anv: drop unused code
[mesa.git] / src / intel / vulkan / genX_query.c
index b1ed4d3854a5cacac0e338e6e9fcf8a06422c482..aa0cf8b947118c3dbee6ad7ba0faf27959a19ae6 100644 (file)
 #include "genxml/gen_macros.h"
 #include "genxml/genX_pack.h"
 
+/* We reserve GPR 14 and 15 for conditional rendering */
+#define GEN_MI_BUILDER_NUM_ALLOC_GPRS 14
+#define __gen_get_batch_dwords anv_batch_emit_dwords
+#define __gen_address_offset anv_address_add
+#include "common/gen_mi_builder.h"
+
 VkResult genX(CreateQueryPool)(
     VkDevice                                    _device,
     const VkQueryPoolCreateInfo*                pCreateInfo,
@@ -39,6 +45,7 @@ VkResult genX(CreateQueryPool)(
     VkQueryPool*                                pQueryPool)
 {
    ANV_FROM_HANDLE(anv_device, device, _device);
+   const struct anv_physical_device *pdevice = &device->instance->physicalDevice;
    struct anv_query_pool *pool;
    VkResult result;
 
@@ -69,7 +76,13 @@ VkResult genX(CreateQueryPool)(
       pipeline_statistics &= ANV_PIPELINE_STATISTICS_MASK;
 
       /* Statistics queries have a min and max for every statistic */
-      uint64s_per_slot += 2 * _mesa_bitcount(pipeline_statistics);
+      uint64s_per_slot += 2 * util_bitcount(pipeline_statistics);
+      break;
+   case VK_QUERY_TYPE_TRANSFORM_FEEDBACK_STREAM_EXT:
+      /* Transform feedback queries are 4 values, begin/end for
+       * written/available.
+       */
+      uint64s_per_slot += 4;
       break;
    default:
       assert(!"Invalid query type");
@@ -90,6 +103,25 @@ VkResult genX(CreateQueryPool)(
    if (result != VK_SUCCESS)
       goto fail;
 
+   if (pdevice->supports_48bit_addresses)
+      pool->bo.flags |= EXEC_OBJECT_SUPPORTS_48B_ADDRESS;
+
+   if (pdevice->use_softpin)
+      pool->bo.flags |= EXEC_OBJECT_PINNED;
+
+   if (pdevice->has_exec_async)
+      pool->bo.flags |= EXEC_OBJECT_ASYNC;
+
+   anv_vma_alloc(device, &pool->bo);
+
+   /* For query pools, we set the caching mode to I915_CACHING_CACHED.  On LLC
+    * platforms, this does nothing.  On non-LLC platforms, this means snooping
+    * which comes at a slight cost.  However, the buffers aren't big, won't be
+    * written frequently, and trying to handle the flushing manually without
+    * doing too much flushing is extremely painful.
+    */
+   anv_gem_set_caching(device, pool->bo.gem_handle, I915_CACHING_CACHED);
+
    pool->bo.map = anv_gem_mmap(device, pool->bo.gem_handle, 0, size, 0);
 
    *pQueryPool = anv_query_pool_to_handle(pool);
@@ -114,10 +146,20 @@ void genX(DestroyQueryPool)(
       return;
 
    anv_gem_munmap(pool->bo.map, pool->bo.size);
+   anv_vma_free(device, &pool->bo);
    anv_gem_close(device, pool->bo.gem_handle);
    vk_free2(&device->alloc, pAllocator, pool);
 }
 
+static struct anv_address
+anv_query_address(struct anv_query_pool *pool, uint32_t query)
+{
+   return (struct anv_address) {
+      .bo = &pool->bo,
+      .offset = query * pool->stride,
+   };
+}
+
 static void
 cpu_write_query_result(void *dst_slot, VkQueryResultFlags flags,
                        uint32_t value_index, uint64_t result)
@@ -131,6 +173,50 @@ cpu_write_query_result(void *dst_slot, VkQueryResultFlags flags,
    }
 }
 
+static bool
+query_is_available(uint64_t *slot)
+{
+   return *(volatile uint64_t *)slot;
+}
+
+static VkResult
+wait_for_available(struct anv_device *device,
+                   struct anv_query_pool *pool, uint64_t *slot)
+{
+   while (true) {
+      if (query_is_available(slot))
+         return VK_SUCCESS;
+
+      int ret = anv_gem_busy(device, pool->bo.gem_handle);
+      if (ret == 1) {
+         /* The BO is still busy, keep waiting. */
+         continue;
+      } else if (ret == -1) {
+         /* We don't know the real error. */
+         return anv_device_set_lost(device, "gem wait failed: %m");
+      } else {
+         assert(ret == 0);
+         /* The BO is no longer busy. */
+         if (query_is_available(slot)) {
+            return VK_SUCCESS;
+         } else {
+            VkResult status = anv_device_query_status(device);
+            if (status != VK_SUCCESS)
+               return status;
+
+            /* If we haven't seen availability yet, then we never will.  This
+             * can only happen if we have a client error where they call
+             * GetQueryPoolResults on a query that they haven't submitted to
+             * the GPU yet.  The spec allows us to do anything in this case,
+             * but returning VK_SUCCESS doesn't seem right and we shouldn't
+             * just keep spinning.
+             */
+            return VK_NOT_READY;
+         }
+      }
+   }
+}
+
 VkResult genX(GetQueryPoolResults)(
     VkDevice                                    _device,
     VkQueryPool                                 queryPool,
@@ -143,37 +229,20 @@ VkResult genX(GetQueryPoolResults)(
 {
    ANV_FROM_HANDLE(anv_device, device, _device);
    ANV_FROM_HANDLE(anv_query_pool, pool, queryPool);
-   int64_t timeout = INT64_MAX;
-   int ret;
 
    assert(pool->type == VK_QUERY_TYPE_OCCLUSION ||
           pool->type == VK_QUERY_TYPE_PIPELINE_STATISTICS ||
-          pool->type == VK_QUERY_TYPE_TIMESTAMP);
+          pool->type == VK_QUERY_TYPE_TIMESTAMP ||
+          pool->type == VK_QUERY_TYPE_TRANSFORM_FEEDBACK_STREAM_EXT);
 
-   if (unlikely(device->lost))
+   if (anv_device_is_lost(device))
       return VK_ERROR_DEVICE_LOST;
 
    if (pData == NULL)
       return VK_SUCCESS;
 
-   if (flags & VK_QUERY_RESULT_WAIT_BIT) {
-      ret = anv_gem_wait(device, pool->bo.gem_handle, &timeout);
-      if (ret == -1) {
-         /* We don't know the real error. */
-         return vk_errorf(VK_ERROR_OUT_OF_DEVICE_MEMORY,
-                          "gem_wait failed %m");
-      }
-   }
-
    void *data_end = pData + dataSize;
 
-   if (!device->info.has_llc) {
-      uint64_t offset = firstQuery * pool->stride;
-      uint64_t size = queryCount * pool->stride;
-      anv_invalidate_range(pool->bo.map + offset,
-                           MIN2(size, pool->bo.size - offset));
-   }
-
    VkResult status = VK_SUCCESS;
    for (uint32_t i = 0; i < queryCount; i++) {
       uint64_t *slot = pool->bo.map + (firstQuery + i) * pool->stride;
@@ -181,6 +250,14 @@ VkResult genX(GetQueryPoolResults)(
       /* Availability is always at the start of the slot */
       bool available = slot[0];
 
+      if (!available && (flags & VK_QUERY_RESULT_WAIT_BIT)) {
+         status = wait_for_available(device, pool, slot);
+         if (status != VK_SUCCESS)
+            return status;
+
+         available = true;
+      }
+
       /* From the Vulkan 1.0.42 spec:
        *
        *    "If VK_QUERY_RESULT_WAIT_BIT and VK_QUERY_RESULT_PARTIAL_BIT are
@@ -192,18 +269,19 @@ VkResult genX(GetQueryPoolResults)(
        */
       bool write_results = available || (flags & VK_QUERY_RESULT_PARTIAL_BIT);
 
-      if (write_results) {
-         switch (pool->type) {
-         case VK_QUERY_TYPE_OCCLUSION: {
-            cpu_write_query_result(pData, flags, 0, slot[2] - slot[1]);
-            break;
-         }
+      uint32_t idx = 0;
+      switch (pool->type) {
+      case VK_QUERY_TYPE_OCCLUSION:
+         if (write_results)
+            cpu_write_query_result(pData, flags, idx, slot[2] - slot[1]);
+         idx++;
+         break;
 
-         case VK_QUERY_TYPE_PIPELINE_STATISTICS: {
-            uint32_t statistics = pool->pipeline_statistics;
-            uint32_t idx = 0;
-            while (statistics) {
-               uint32_t stat = u_bit_scan(&statistics);
+      case VK_QUERY_TYPE_PIPELINE_STATISTICS: {
+         uint32_t statistics = pool->pipeline_statistics;
+         while (statistics) {
+            uint32_t stat = u_bit_scan(&statistics);
+            if (write_results) {
                uint64_t result = slot[idx * 2 + 2] - slot[idx * 2 + 1];
 
                /* WaDividePSInvocationCountBy4:HSW,BDW */
@@ -212,29 +290,37 @@ VkResult genX(GetQueryPoolResults)(
                   result >>= 2;
 
                cpu_write_query_result(pData, flags, idx, result);
-
-               idx++;
             }
-            assert(idx == _mesa_bitcount(pool->pipeline_statistics));
-            break;
+            idx++;
          }
+         assert(idx == util_bitcount(pool->pipeline_statistics));
+         break;
+      }
 
-         case VK_QUERY_TYPE_TIMESTAMP: {
-            cpu_write_query_result(pData, flags, 0, slot[1]);
-            break;
-         }
-         default:
-            unreachable("invalid pool type");
-         }
-      } else {
-         status = VK_NOT_READY;
+      case VK_QUERY_TYPE_TRANSFORM_FEEDBACK_STREAM_EXT:
+         if (write_results)
+            cpu_write_query_result(pData, flags, idx, slot[2] - slot[1]);
+         idx++;
+         if (write_results)
+            cpu_write_query_result(pData, flags, idx, slot[4] - slot[3]);
+         idx++;
+         break;
+
+      case VK_QUERY_TYPE_TIMESTAMP:
+         if (write_results)
+            cpu_write_query_result(pData, flags, idx, slot[1]);
+         idx++;
+         break;
+
+      default:
+         unreachable("invalid pool type");
       }
 
-      if (flags & VK_QUERY_RESULT_WITH_AVAILABILITY_BIT) {
-         uint32_t idx = (pool->type == VK_QUERY_TYPE_PIPELINE_STATISTICS) ?
-                        _mesa_bitcount(pool->pipeline_statistics) : 1;
+      if (!write_results)
+         status = VK_NOT_READY;
+
+      if (flags & VK_QUERY_RESULT_WITH_AVAILABILITY_BIT)
          cpu_write_query_result(pData, flags, idx, available);
-      }
 
       pData += stride;
       if (pData >= data_end)
@@ -246,13 +332,13 @@ VkResult genX(GetQueryPoolResults)(
 
 static void
 emit_ps_depth_count(struct anv_cmd_buffer *cmd_buffer,
-                    struct anv_bo *bo, uint32_t offset)
+                    struct anv_address addr)
 {
    anv_batch_emit(&cmd_buffer->batch, GENX(PIPE_CONTROL), pc) {
       pc.DestinationAddressType  = DAT_PPGTT;
       pc.PostSyncOperation       = WritePSDepthCount;
       pc.DepthStallEnable        = true;
-      pc.Address                 = (struct anv_address) { bo, offset };
+      pc.Address                 = addr;
 
       if (GEN_GEN == 9 && cmd_buffer->device->info.gt == 4)
          pc.CommandStreamerStallEnable = true;
@@ -260,14 +346,68 @@ emit_ps_depth_count(struct anv_cmd_buffer *cmd_buffer,
 }
 
 static void
-emit_query_availability(struct anv_cmd_buffer *cmd_buffer,
-                        struct anv_bo *bo, uint32_t offset)
+emit_query_mi_availability(struct gen_mi_builder *b,
+                           struct anv_address addr,
+                           bool available)
+{
+   gen_mi_store(b, gen_mi_mem64(addr), gen_mi_imm(available));
+}
+
+static void
+emit_query_pc_availability(struct anv_cmd_buffer *cmd_buffer,
+                           struct anv_address addr,
+                           bool available)
 {
    anv_batch_emit(&cmd_buffer->batch, GENX(PIPE_CONTROL), pc) {
       pc.DestinationAddressType  = DAT_PPGTT;
       pc.PostSyncOperation       = WriteImmediateData;
-      pc.Address                 = (struct anv_address) { bo, offset };
-      pc.ImmediateData           = 1;
+      pc.Address                 = addr;
+      pc.ImmediateData           = available;
+   }
+}
+
+/**
+ * Goes through a series of consecutive query indices in the given pool
+ * setting all element values to 0 and emitting them as available.
+ */
+static void
+emit_zero_queries(struct anv_cmd_buffer *cmd_buffer,
+                  struct gen_mi_builder *b, struct anv_query_pool *pool,
+                  uint32_t first_index, uint32_t num_queries)
+{
+   switch (pool->type) {
+   case VK_QUERY_TYPE_OCCLUSION:
+   case VK_QUERY_TYPE_TIMESTAMP:
+      /* These queries are written with a PIPE_CONTROL so clear them using the
+       * PIPE_CONTROL as well so we don't have to synchronize between 2 types
+       * of operations.
+       */
+      assert((pool->stride % 8) == 0);
+      for (uint32_t i = 0; i < num_queries; i++) {
+         struct anv_address slot_addr =
+            anv_query_address(pool, first_index + i);
+
+         for (uint32_t qword = 1; qword < (pool->stride / 8); qword++) {
+            emit_query_pc_availability(cmd_buffer,
+                                       anv_address_add(slot_addr, qword * 8),
+                                       false);
+         }
+         emit_query_pc_availability(cmd_buffer, slot_addr, true);
+      }
+      break;
+
+   case VK_QUERY_TYPE_PIPELINE_STATISTICS:
+   case VK_QUERY_TYPE_TRANSFORM_FEEDBACK_STREAM_EXT:
+      for (uint32_t i = 0; i < num_queries; i++) {
+         struct anv_address slot_addr =
+            anv_query_address(pool, first_index + i);
+         gen_mi_memset(b, anv_address_add(slot_addr, 8), 0, pool->stride - 8);
+         emit_query_mi_availability(b, slot_addr, true);
+      }
+      break;
+
+   default:
+      unreachable("Unsupported query type");
    }
 }
 
@@ -280,14 +420,42 @@ void genX(CmdResetQueryPool)(
    ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
    ANV_FROM_HANDLE(anv_query_pool, pool, queryPool);
 
-   for (uint32_t i = 0; i < queryCount; i++) {
-      anv_batch_emit(&cmd_buffer->batch, GENX(MI_STORE_DATA_IMM), sdm) {
-         sdm.Address = (struct anv_address) {
-            .bo = &pool->bo,
-            .offset = (firstQuery + i) * pool->stride,
-         };
-         sdm.ImmediateData = 0;
+   switch (pool->type) {
+   case VK_QUERY_TYPE_OCCLUSION:
+   case VK_QUERY_TYPE_TIMESTAMP:
+      for (uint32_t i = 0; i < queryCount; i++) {
+         emit_query_pc_availability(cmd_buffer,
+                                    anv_query_address(pool, firstQuery + i),
+                                    false);
       }
+      break;
+
+   case VK_QUERY_TYPE_PIPELINE_STATISTICS:
+   case VK_QUERY_TYPE_TRANSFORM_FEEDBACK_STREAM_EXT: {
+      struct gen_mi_builder b;
+      gen_mi_builder_init(&b, &cmd_buffer->batch);
+
+      for (uint32_t i = 0; i < queryCount; i++)
+         emit_query_mi_availability(&b, anv_query_address(pool, firstQuery + i), false);
+      break;
+   }
+
+   default:
+      unreachable("Unsupported query type");
+   }
+}
+
+void genX(ResetQueryPoolEXT)(
+    VkDevice                                    _device,
+    VkQueryPool                                 queryPool,
+    uint32_t                                    firstQuery,
+    uint32_t                                    queryCount)
+{
+   ANV_FROM_HANDLE(anv_query_pool, pool, queryPool);
+
+   for (uint32_t i = 0; i < queryCount; i++) {
+      uint64_t *slot = pool->bo.map + (firstQuery + i) * pool->stride;
+      *slot = 0;
    }
 }
 
@@ -306,23 +474,27 @@ static const uint32_t vk_pipeline_stat_to_reg[] = {
 };
 
 static void
-emit_pipeline_stat(struct anv_cmd_buffer *cmd_buffer, uint32_t stat,
-                   struct anv_bo *bo, uint32_t offset)
+emit_pipeline_stat(struct gen_mi_builder *b, uint32_t stat,
+                   struct anv_address addr)
 {
    STATIC_ASSERT(ANV_PIPELINE_STATISTICS_MASK ==
                  (1 << ARRAY_SIZE(vk_pipeline_stat_to_reg)) - 1);
 
    assert(stat < ARRAY_SIZE(vk_pipeline_stat_to_reg));
-   uint32_t reg = vk_pipeline_stat_to_reg[stat];
+   gen_mi_store(b, gen_mi_mem64(addr),
+                gen_mi_reg64(vk_pipeline_stat_to_reg[stat]));
+}
 
-   anv_batch_emit(&cmd_buffer->batch, GENX(MI_STORE_REGISTER_MEM), lrm) {
-      lrm.RegisterAddress  = reg,
-      lrm.MemoryAddress    = (struct anv_address) { bo, offset };
-   }
-   anv_batch_emit(&cmd_buffer->batch, GENX(MI_STORE_REGISTER_MEM), lrm) {
-      lrm.RegisterAddress  = reg + 4,
-      lrm.MemoryAddress    = (struct anv_address) { bo, offset + 4 };
-   }
+static void
+emit_xfb_query(struct gen_mi_builder *b, uint32_t stream,
+               struct anv_address addr)
+{
+   assert(stream < MAX_XFB_STREAMS);
+
+   gen_mi_store(b, gen_mi_mem64(anv_address_add(addr, 0)),
+                gen_mi_reg64(GENX(SO_NUM_PRIMS_WRITTEN0_num) + stream * 8));
+   gen_mi_store(b, gen_mi_mem64(anv_address_add(addr, 16)),
+                gen_mi_reg64(GENX(SO_PRIM_STORAGE_NEEDED0_num) + stream * 8));
 }
 
 void genX(CmdBeginQuery)(
@@ -330,27 +502,27 @@ void genX(CmdBeginQuery)(
     VkQueryPool                                 queryPool,
     uint32_t                                    query,
     VkQueryControlFlags                         flags)
+{
+   genX(CmdBeginQueryIndexedEXT)(commandBuffer, queryPool, query, flags, 0);
+}
+
+void genX(CmdBeginQueryIndexedEXT)(
+    VkCommandBuffer                             commandBuffer,
+    VkQueryPool                                 queryPool,
+    uint32_t                                    query,
+    VkQueryControlFlags                         flags,
+    uint32_t                                    index)
 {
    ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
    ANV_FROM_HANDLE(anv_query_pool, pool, queryPool);
+   struct anv_address query_addr = anv_query_address(pool, query);
 
-   /* Workaround: When meta uses the pipeline with the VS disabled, it seems
-    * that the pipelining of the depth write breaks. What we see is that
-    * samples from the render pass clear leaks into the first query
-    * immediately after the clear. Doing a pipecontrol with a post-sync
-    * operation and DepthStallEnable seems to work around the issue.
-    */
-   if (cmd_buffer->state.need_query_wa) {
-      cmd_buffer->state.need_query_wa = false;
-      anv_batch_emit(&cmd_buffer->batch, GENX(PIPE_CONTROL), pc) {
-         pc.DepthCacheFlushEnable   = true;
-         pc.DepthStallEnable        = true;
-      }
-   }
+   struct gen_mi_builder b;
+   gen_mi_builder_init(&b, &cmd_buffer->batch);
 
    switch (pool->type) {
    case VK_QUERY_TYPE_OCCLUSION:
-      emit_ps_depth_count(cmd_buffer, &pool->bo, query * pool->stride + 8);
+      emit_ps_depth_count(cmd_buffer, anv_address_add(query_addr, 8));
       break;
 
    case VK_QUERY_TYPE_PIPELINE_STATISTICS: {
@@ -361,15 +533,23 @@ void genX(CmdBeginQuery)(
       }
 
       uint32_t statistics = pool->pipeline_statistics;
-      uint32_t offset = query * pool->stride + 8;
+      uint32_t offset = 8;
       while (statistics) {
          uint32_t stat = u_bit_scan(&statistics);
-         emit_pipeline_stat(cmd_buffer, stat, &pool->bo, offset);
+         emit_pipeline_stat(&b, stat, anv_address_add(query_addr, offset));
          offset += 16;
       }
       break;
    }
 
+   case VK_QUERY_TYPE_TRANSFORM_FEEDBACK_STREAM_EXT:
+      anv_batch_emit(&cmd_buffer->batch, GENX(PIPE_CONTROL), pc) {
+         pc.CommandStreamerStallEnable = true;
+         pc.StallAtPixelScoreboard = true;
+      }
+      emit_xfb_query(&b, index, anv_address_add(query_addr, 8));
+      break;
+
    default:
       unreachable("");
    }
@@ -379,14 +559,27 @@ void genX(CmdEndQuery)(
     VkCommandBuffer                             commandBuffer,
     VkQueryPool                                 queryPool,
     uint32_t                                    query)
+{
+   genX(CmdEndQueryIndexedEXT)(commandBuffer, queryPool, query, 0);
+}
+
+void genX(CmdEndQueryIndexedEXT)(
+    VkCommandBuffer                             commandBuffer,
+    VkQueryPool                                 queryPool,
+    uint32_t                                    query,
+    uint32_t                                    index)
 {
    ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
    ANV_FROM_HANDLE(anv_query_pool, pool, queryPool);
+   struct anv_address query_addr = anv_query_address(pool, query);
+
+   struct gen_mi_builder b;
+   gen_mi_builder_init(&b, &cmd_buffer->batch);
 
    switch (pool->type) {
    case VK_QUERY_TYPE_OCCLUSION:
-      emit_ps_depth_count(cmd_buffer, &pool->bo, query * pool->stride + 16);
-      emit_query_availability(cmd_buffer, &pool->bo, query * pool->stride);
+      emit_ps_depth_count(cmd_buffer, anv_address_add(query_addr, 16));
+      emit_query_pc_availability(cmd_buffer, query_addr, true);
       break;
 
    case VK_QUERY_TYPE_PIPELINE_STATISTICS: {
@@ -397,20 +590,45 @@ void genX(CmdEndQuery)(
       }
 
       uint32_t statistics = pool->pipeline_statistics;
-      uint32_t offset = query * pool->stride + 16;
+      uint32_t offset = 16;
       while (statistics) {
          uint32_t stat = u_bit_scan(&statistics);
-         emit_pipeline_stat(cmd_buffer, stat, &pool->bo, offset);
+         emit_pipeline_stat(&b, stat, anv_address_add(query_addr, offset));
          offset += 16;
       }
 
-      emit_query_availability(cmd_buffer, &pool->bo, query * pool->stride);
+      emit_query_mi_availability(&b, query_addr, true);
       break;
    }
 
+   case VK_QUERY_TYPE_TRANSFORM_FEEDBACK_STREAM_EXT:
+      anv_batch_emit(&cmd_buffer->batch, GENX(PIPE_CONTROL), pc) {
+         pc.CommandStreamerStallEnable = true;
+         pc.StallAtPixelScoreboard = true;
+      }
+
+      emit_xfb_query(&b, index, anv_address_add(query_addr, 16));
+      emit_query_mi_availability(&b, query_addr, true);
+      break;
+
    default:
       unreachable("");
    }
+
+   /* When multiview is active the spec requires that N consecutive query
+    * indices are used, where N is the number of active views in the subpass.
+    * The spec allows that we only write the results to one of the queries
+    * but we still need to manage result availability for all the query indices.
+    * Since we only emit a single query for all active views in the
+    * first index, mark the other query indices as being already available
+    * with result 0.
+    */
+   if (cmd_buffer->state.subpass && cmd_buffer->state.subpass->view_mask) {
+      const uint32_t num_queries =
+         util_bitcount(cmd_buffer->state.subpass->view_mask);
+      if (num_queries > 1)
+         emit_zero_queries(cmd_buffer, &b, pool, query + 1, num_queries - 1);
+   }
 }
 
 #define TIMESTAMP 0x2358
@@ -423,20 +641,17 @@ void genX(CmdWriteTimestamp)(
 {
    ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
    ANV_FROM_HANDLE(anv_query_pool, pool, queryPool);
-   uint32_t offset = query * pool->stride;
+   struct anv_address query_addr = anv_query_address(pool, query);
 
    assert(pool->type == VK_QUERY_TYPE_TIMESTAMP);
 
+   struct gen_mi_builder b;
+   gen_mi_builder_init(&b, &cmd_buffer->batch);
+
    switch (pipelineStage) {
    case VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT:
-      anv_batch_emit(&cmd_buffer->batch, GENX(MI_STORE_REGISTER_MEM), srm) {
-         srm.RegisterAddress  = TIMESTAMP;
-         srm.MemoryAddress    = (struct anv_address) { &pool->bo, offset + 8 };
-      }
-      anv_batch_emit(&cmd_buffer->batch, GENX(MI_STORE_REGISTER_MEM), srm) {
-         srm.RegisterAddress  = TIMESTAMP + 4;
-         srm.MemoryAddress    = (struct anv_address) { &pool->bo, offset + 12 };
-      }
+      gen_mi_store(&b, gen_mi_mem64(anv_address_add(query_addr, 8)),
+                       gen_mi_reg64(TIMESTAMP));
       break;
 
    default:
@@ -444,7 +659,7 @@ void genX(CmdWriteTimestamp)(
       anv_batch_emit(&cmd_buffer->batch, GENX(PIPE_CONTROL), pc) {
          pc.DestinationAddressType  = DAT_PPGTT;
          pc.PostSyncOperation       = WriteTimestamp;
-         pc.Address = (struct anv_address) { &pool->bo, offset + 8 };
+         pc.Address                 = anv_address_add(query_addr, 8);
 
          if (GEN_GEN == 9 && cmd_buffer->device->info.gt == 4)
             pc.CommandStreamerStallEnable = true;
@@ -452,185 +667,47 @@ void genX(CmdWriteTimestamp)(
       break;
    }
 
-   emit_query_availability(cmd_buffer, &pool->bo, offset);
-}
+   emit_query_pc_availability(cmd_buffer, query_addr, true);
 
-#if GEN_GEN > 7 || GEN_IS_HASWELL
-
-#define alu_opcode(v)   __gen_uint((v),  20, 31)
-#define alu_operand1(v) __gen_uint((v),  10, 19)
-#define alu_operand2(v) __gen_uint((v),   0,  9)
-#define alu(opcode, operand1, operand2) \
-   alu_opcode(opcode) | alu_operand1(operand1) | alu_operand2(operand2)
-
-#define OPCODE_NOOP      0x000
-#define OPCODE_LOAD      0x080
-#define OPCODE_LOADINV   0x480
-#define OPCODE_LOAD0     0x081
-#define OPCODE_LOAD1     0x481
-#define OPCODE_ADD       0x100
-#define OPCODE_SUB       0x101
-#define OPCODE_AND       0x102
-#define OPCODE_OR        0x103
-#define OPCODE_XOR       0x104
-#define OPCODE_STORE     0x180
-#define OPCODE_STOREINV  0x580
-
-#define OPERAND_R0   0x00
-#define OPERAND_R1   0x01
-#define OPERAND_R2   0x02
-#define OPERAND_R3   0x03
-#define OPERAND_R4   0x04
-#define OPERAND_SRCA 0x20
-#define OPERAND_SRCB 0x21
-#define OPERAND_ACCU 0x31
-#define OPERAND_ZF   0x32
-#define OPERAND_CF   0x33
-
-#define CS_GPR(n) (0x2600 + (n) * 8)
-
-static void
-emit_load_alu_reg_u64(struct anv_batch *batch, uint32_t reg,
-                      struct anv_bo *bo, uint32_t offset)
-{
-   anv_batch_emit(batch, GENX(MI_LOAD_REGISTER_MEM), lrm) {
-      lrm.RegisterAddress  = reg,
-      lrm.MemoryAddress    = (struct anv_address) { bo, offset };
-   }
-   anv_batch_emit(batch, GENX(MI_LOAD_REGISTER_MEM), lrm) {
-      lrm.RegisterAddress  = reg + 4;
-      lrm.MemoryAddress    = (struct anv_address) { bo, offset + 4 };
-   }
-}
-
-static void
-emit_load_alu_reg_imm32(struct anv_batch *batch, uint32_t reg, uint32_t imm)
-{
-   anv_batch_emit(batch, GENX(MI_LOAD_REGISTER_IMM), lri) {
-      lri.RegisterOffset   = reg;
-      lri.DataDWord        = imm;
-   }
-}
-
-static void
-emit_load_alu_reg_imm64(struct anv_batch *batch, uint32_t reg, uint64_t imm)
-{
-   emit_load_alu_reg_imm32(batch, reg, (uint32_t)imm);
-   emit_load_alu_reg_imm32(batch, reg + 4, (uint32_t)(imm >> 32));
-}
-
-static void
-emit_load_alu_reg_reg32(struct anv_batch *batch, uint32_t src, uint32_t dst)
-{
-   anv_batch_emit(batch, GENX(MI_LOAD_REGISTER_REG), lrr) {
-      lrr.SourceRegisterAddress      = src;
-      lrr.DestinationRegisterAddress = dst;
-   }
-}
-
-/*
- * GPR0 = GPR0 & ((1ull << n) - 1);
- */
-static void
-keep_gpr0_lower_n_bits(struct anv_batch *batch, uint32_t n)
-{
-   assert(n < 64);
-   emit_load_alu_reg_imm64(batch, CS_GPR(1), (1ull << n) - 1);
-
-   uint32_t *dw = anv_batch_emitn(batch, 5, GENX(MI_MATH));
-   dw[1] = alu(OPCODE_LOAD, OPERAND_SRCA, OPERAND_R0);
-   dw[2] = alu(OPCODE_LOAD, OPERAND_SRCB, OPERAND_R1);
-   dw[3] = alu(OPCODE_AND, 0, 0);
-   dw[4] = alu(OPCODE_STORE, OPERAND_R0, OPERAND_ACCU);
-}
-
-/*
- * GPR0 = GPR0 << 30;
- */
-static void
-shl_gpr0_by_30_bits(struct anv_batch *batch)
-{
-   /* First we mask 34 bits of GPR0 to prevent overflow */
-   keep_gpr0_lower_n_bits(batch, 34);
-
-   const uint32_t outer_count = 5;
-   const uint32_t inner_count = 6;
-   STATIC_ASSERT(outer_count * inner_count == 30);
-   const uint32_t cmd_len = 1 + inner_count * 4;
-
-   /* We'll emit 5 commands, each shifting GPR0 left by 6 bits, for a total of
-    * 30 left shifts.
+   /* When multiview is active the spec requires that N consecutive query
+    * indices are used, where N is the number of active views in the subpass.
+    * The spec allows that we only write the results to one of the queries
+    * but we still need to manage result availability for all the query indices.
+    * Since we only emit a single query for all active views in the
+    * first index, mark the other query indices as being already available
+    * with result 0.
     */
-   for (int o = 0; o < outer_count; o++) {
-      /* Submit one MI_MATH to shift left by 6 bits */
-      uint32_t *dw = anv_batch_emitn(batch, cmd_len, GENX(MI_MATH));
-      dw++;
-      for (int i = 0; i < inner_count; i++, dw += 4) {
-         dw[0] = alu(OPCODE_LOAD, OPERAND_SRCA, OPERAND_R0);
-         dw[1] = alu(OPCODE_LOAD, OPERAND_SRCB, OPERAND_R0);
-         dw[2] = alu(OPCODE_ADD, 0, 0);
-         dw[3] = alu(OPCODE_STORE, OPERAND_R0, OPERAND_ACCU);
-      }
+   if (cmd_buffer->state.subpass && cmd_buffer->state.subpass->view_mask) {
+      const uint32_t num_queries =
+         util_bitcount(cmd_buffer->state.subpass->view_mask);
+      if (num_queries > 1)
+         emit_zero_queries(cmd_buffer, &b, pool, query + 1, num_queries - 1);
    }
 }
 
-/*
- * GPR0 = GPR0 >> 2;
- *
- * Note that the upper 30 bits of GPR are lost!
- */
-static void
-shr_gpr0_by_2_bits(struct anv_batch *batch)
-{
-   shl_gpr0_by_30_bits(batch);
-   emit_load_alu_reg_reg32(batch, CS_GPR(0) + 4, CS_GPR(0));
-   emit_load_alu_reg_imm32(batch, CS_GPR(0) + 4, 0);
-}
+#if GEN_GEN > 7 || GEN_IS_HASWELL
 
 static void
-gpu_write_query_result(struct anv_batch *batch,
-                       struct anv_buffer *dst_buffer, uint32_t dst_offset,
+gpu_write_query_result(struct gen_mi_builder *b,
+                       struct anv_address dst_addr,
                        VkQueryResultFlags flags,
-                       uint32_t value_index, uint32_t reg)
+                       uint32_t value_index,
+                       struct gen_mi_value query_result)
 {
-   if (flags & VK_QUERY_RESULT_64_BIT)
-      dst_offset += value_index * 8;
-   else
-      dst_offset += value_index * 4;
-
-   anv_batch_emit(batch, GENX(MI_STORE_REGISTER_MEM), srm) {
-      srm.RegisterAddress  = reg;
-      srm.MemoryAddress    = (struct anv_address) {
-         .bo = dst_buffer->bo,
-         .offset = dst_buffer->offset + dst_offset,
-      };
-   }
-
    if (flags & VK_QUERY_RESULT_64_BIT) {
-      anv_batch_emit(batch, GENX(MI_STORE_REGISTER_MEM), srm) {
-         srm.RegisterAddress  = reg + 4;
-         srm.MemoryAddress    = (struct anv_address) {
-            .bo = dst_buffer->bo,
-            .offset = dst_buffer->offset + dst_offset + 4,
-         };
-      }
+      struct anv_address res_addr = anv_address_add(dst_addr, value_index * 8);
+      gen_mi_store(b, gen_mi_mem64(res_addr), query_result);
+   } else {
+      struct anv_address res_addr = anv_address_add(dst_addr, value_index * 4);
+      gen_mi_store(b, gen_mi_mem32(res_addr), query_result);
    }
 }
 
-static void
-compute_query_result(struct anv_batch *batch, uint32_t dst_reg,
-                     struct anv_bo *bo, uint32_t offset)
+static struct gen_mi_value
+compute_query_result(struct gen_mi_builder *b, struct anv_address addr)
 {
-   emit_load_alu_reg_u64(batch, CS_GPR(0), bo, offset);
-   emit_load_alu_reg_u64(batch, CS_GPR(1), bo, offset + 8);
-
-   /* FIXME: We need to clamp the result for 32 bit. */
-
-   uint32_t *dw = anv_batch_emitn(batch, 5, GENX(MI_MATH));
-   dw[1] = alu(OPCODE_LOAD, OPERAND_SRCA, OPERAND_R1);
-   dw[2] = alu(OPCODE_LOAD, OPERAND_SRCB, OPERAND_R0);
-   dw[3] = alu(OPCODE_SUB, 0, 0);
-   dw[4] = alu(OPCODE_STORE, dst_reg, OPERAND_ACCU);
+   return gen_mi_isub(b, gen_mi_mem64(anv_address_add(addr, 8)),
+                         gen_mi_mem64(anv_address_add(addr, 0)));
 }
 
 void genX(CmdCopyQueryPoolResults)(
@@ -646,55 +723,80 @@ void genX(CmdCopyQueryPoolResults)(
    ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, commandBuffer);
    ANV_FROM_HANDLE(anv_query_pool, pool, queryPool);
    ANV_FROM_HANDLE(anv_buffer, buffer, destBuffer);
-   uint32_t slot_offset;
 
-   if (flags & VK_QUERY_RESULT_WAIT_BIT) {
-      anv_batch_emit(&cmd_buffer->batch, GENX(PIPE_CONTROL), pc) {
-         pc.CommandStreamerStallEnable = true;
-         pc.StallAtPixelScoreboard     = true;
-      }
+   struct gen_mi_builder b;
+   gen_mi_builder_init(&b, &cmd_buffer->batch);
+   struct gen_mi_value result;
+
+   /* If render target writes are ongoing, request a render target cache flush
+    * to ensure proper ordering of the commands from the 3d pipe and the
+    * command streamer.
+    */
+   if (cmd_buffer->state.pending_pipe_bits & ANV_PIPE_RENDER_TARGET_BUFFER_WRITES) {
+      cmd_buffer->state.pending_pipe_bits |=
+         ANV_PIPE_RENDER_TARGET_CACHE_FLUSH_BIT;
    }
 
+   if ((flags & VK_QUERY_RESULT_WAIT_BIT) ||
+       (cmd_buffer->state.pending_pipe_bits & ANV_PIPE_FLUSH_BITS) ||
+       /* Occlusion & timestamp queries are written using a PIPE_CONTROL and
+        * because we're about to copy values from MI commands, we need to
+        * stall the command streamer to make sure the PIPE_CONTROL values have
+        * landed, otherwise we could see inconsistent values & availability.
+        *
+        *  From the vulkan spec:
+        *
+        *     "vkCmdCopyQueryPoolResults is guaranteed to see the effect of
+        *     previous uses of vkCmdResetQueryPool in the same queue, without
+        *     any additional synchronization."
+        */
+       pool->type == VK_QUERY_TYPE_OCCLUSION ||
+       pool->type == VK_QUERY_TYPE_TIMESTAMP) {
+      cmd_buffer->state.pending_pipe_bits |= ANV_PIPE_CS_STALL_BIT;
+      genX(cmd_buffer_apply_pipe_flushes)(cmd_buffer);
+   }
+
+   struct anv_address dest_addr = anv_address_add(buffer->address, destOffset);
    for (uint32_t i = 0; i < queryCount; i++) {
-      slot_offset = (firstQuery + i) * pool->stride;
+      struct anv_address query_addr = anv_query_address(pool, firstQuery + i);
+      uint32_t idx = 0;
       switch (pool->type) {
       case VK_QUERY_TYPE_OCCLUSION:
-         compute_query_result(&cmd_buffer->batch, OPERAND_R2,
-                              &pool->bo, slot_offset + 8);
-         gpu_write_query_result(&cmd_buffer->batch, buffer, destOffset,
-                                flags, 0, CS_GPR(2));
+         result = compute_query_result(&b, anv_address_add(query_addr, 8));
+         gpu_write_query_result(&b, dest_addr, flags, idx++, result);
          break;
 
       case VK_QUERY_TYPE_PIPELINE_STATISTICS: {
          uint32_t statistics = pool->pipeline_statistics;
-         uint32_t idx = 0;
          while (statistics) {
             uint32_t stat = u_bit_scan(&statistics);
 
-            compute_query_result(&cmd_buffer->batch, OPERAND_R0,
-                                 &pool->bo, slot_offset + idx * 16 + 8);
+            result = compute_query_result(&b, anv_address_add(query_addr,
+                                                              idx * 16 + 8));
 
             /* WaDividePSInvocationCountBy4:HSW,BDW */
             if ((cmd_buffer->device->info.gen == 8 ||
                  cmd_buffer->device->info.is_haswell) &&
                 (1 << stat) == VK_QUERY_PIPELINE_STATISTIC_FRAGMENT_SHADER_INVOCATIONS_BIT) {
-               shr_gpr0_by_2_bits(&cmd_buffer->batch);
+               result = gen_mi_ushr32_imm(&b, result, 2);
             }
 
-            gpu_write_query_result(&cmd_buffer->batch, buffer, destOffset,
-                                   flags, idx, CS_GPR(0));
-
-            idx++;
+            gpu_write_query_result(&b, dest_addr, flags, idx++, result);
          }
-         assert(idx == _mesa_bitcount(pool->pipeline_statistics));
+         assert(idx == util_bitcount(pool->pipeline_statistics));
          break;
       }
 
+      case VK_QUERY_TYPE_TRANSFORM_FEEDBACK_STREAM_EXT:
+         result = compute_query_result(&b, anv_address_add(query_addr, 8));
+         gpu_write_query_result(&b, dest_addr, flags, idx++, result);
+         result = compute_query_result(&b, anv_address_add(query_addr, 24));
+         gpu_write_query_result(&b, dest_addr, flags, idx++, result);
+         break;
+
       case VK_QUERY_TYPE_TIMESTAMP:
-         emit_load_alu_reg_u64(&cmd_buffer->batch,
-                               CS_GPR(2), &pool->bo, slot_offset + 8);
-         gpu_write_query_result(&cmd_buffer->batch, buffer, destOffset,
-                                flags, 0, CS_GPR(2));
+         result = gen_mi_mem64(anv_address_add(query_addr, 8));
+         gpu_write_query_result(&b, dest_addr, flags, 0, result);
          break;
 
       default:
@@ -702,16 +804,11 @@ void genX(CmdCopyQueryPoolResults)(
       }
 
       if (flags & VK_QUERY_RESULT_WITH_AVAILABILITY_BIT) {
-         uint32_t idx = (pool->type == VK_QUERY_TYPE_PIPELINE_STATISTICS) ?
-                        _mesa_bitcount(pool->pipeline_statistics) : 1;
-
-         emit_load_alu_reg_u64(&cmd_buffer->batch, CS_GPR(0),
-                               &pool->bo, slot_offset);
-         gpu_write_query_result(&cmd_buffer->batch, buffer, destOffset,
-                                flags, idx, CS_GPR(0));
+         gpu_write_query_result(&b, dest_addr, flags, idx,
+                                gen_mi_mem64(query_addr));
       }
 
-      destOffset += destStride;
+      dest_addr = anv_address_add(dest_addr, destStride);
    }
 }