+ anv_device_release_bo(device, pool->bo);
+ vk_object_base_finish(&pool->base);
+ vk_free2(&device->vk.alloc, pAllocator, pool);
+}
+
+/**
+ * VK_KHR_performance_query layout (576 bytes * number of passes) :
+ *
+ * -----------------------------------------
+ * | availability (8b) | | |
+ * |----------------------------| | |
+ * | Small batch loading | | |
+ * | ANV_PERF_QUERY_OFFSET_REG | | |
+ * | (56b) | | Pass 0 |
+ * |----------------------------| | |
+ * | begin MI_RPC (256b) | | |
+ * |----------------------------| | |
+ * | end MI_RPC (256b) | | |
+ * |----------------------------|-- | Query 0
+ * | availability (8b) | | |
+ * |----------------------------| | |
+ * | Small batch loading | | |
+ * | ANV_PERF_QUERY_OFFSET_REG | | |
+ * | (56b) | | Pass 1 |
+ * |----------------------------| | |
+ * | begin MI_RPC (256b) | | |
+ * |----------------------------| | |
+ * | end MI_RPC (256b) | | |
+ * |----------------------------|-----------
+ * | availability (8b) | | |
+ * |----------------------------| | |
+ * | Unused (48b) | | |
+ * |----------------------------| | Pass 0 |
+ * | begin MI_RPC (256b) | | |
+ * |----------------------------| | | Query 1
+ * | end MI_RPC (256b) | | |
+ * |----------------------------|-- |
+ * | ... | | |
+ * -----------------------------------------
+ */
+UNUSED static uint64_t
+khr_perf_query_availability_offset(struct anv_query_pool *pool, uint32_t query, uint32_t pass)
+{
+ return query * (pool->n_passes * ANV_KHR_PERF_QUERY_SIZE) +
+ pass * ANV_KHR_PERF_QUERY_SIZE;
+}
+
+UNUSED static uint64_t
+khr_perf_query_oa_offset(struct anv_query_pool *pool, uint32_t query, uint32_t pass, bool end)
+{
+ return query * (pool->n_passes * ANV_KHR_PERF_QUERY_SIZE) +
+ pass * ANV_KHR_PERF_QUERY_SIZE +
+ 64 + (end ? OA_SNAPSHOT_SIZE : 0);
+}
+
+UNUSED static struct anv_address
+khr_perf_query_availability_address(struct anv_query_pool *pool, uint32_t query, uint32_t pass)
+{
+ return anv_address_add(
+ (struct anv_address) { .bo = pool->bo, },
+ khr_perf_query_availability_offset(pool, query, pass));
+}
+
+UNUSED static struct anv_address
+khr_perf_query_oa_address(struct anv_query_pool *pool, uint32_t query, uint32_t pass, bool end)
+{
+ return anv_address_add(
+ (struct anv_address) { .bo = pool->bo, },
+ khr_perf_query_oa_offset(pool, query, pass, end));
+}
+
+
+/**
+ * VK_INTEL_performance_query layout (576 bytes) :
+ *
+ * ------------------------------
+ * | availability (8b) |
+ * |----------------------------|
+ * | marker (8b) |
+ * |----------------------------|
+ * | begin RPSTAT register (4b) |
+ * |----------------------------|
+ * | end RPSTAT register (4b) |
+ * |----------------------------|
+ * | begin perfcntr 1 & 2 (16b) |
+ * |----------------------------|
+ * | end perfcntr 1 & 2 (16b) |
+ * |----------------------------|
+ * | Unused (8b) |
+ * |----------------------------|
+ * | begin MI_RPC (256b) |
+ * |----------------------------|
+ * | end MI_RPC (256b) |
+ * ------------------------------
+ */
+
+static uint32_t
+intel_perf_marker_offset(void)
+{
+ return 8;
+}
+
+static uint32_t
+intel_perf_rpstart_offset(bool end)
+{
+ return 16 + (end ? sizeof(uint32_t) : 0);
+}
+
+#if GEN_GEN >= 8 && GEN_GEN <= 11
+static uint32_t
+intel_perf_counter(bool end)
+{
+ return 24 + (end ? (2 * sizeof(uint64_t)) : 0);
+}
+#endif
+
+static uint32_t
+intel_perf_mi_rpc_offset(bool end)
+{
+ return 64 + (end ? 256 : 0);
+}
+
+static void
+cpu_write_query_result(void *dst_slot, VkQueryResultFlags flags,
+ uint32_t value_index, uint64_t result)
+{
+ if (flags & VK_QUERY_RESULT_64_BIT) {
+ uint64_t *dst64 = dst_slot;
+ dst64[value_index] = result;
+ } else {
+ uint32_t *dst32 = dst_slot;
+ dst32[value_index] = result;
+ }
+}
+
+static void *
+query_slot(struct anv_query_pool *pool, uint32_t query)
+{
+ return pool->bo->map + query * pool->stride;
+}
+
+static bool
+query_is_available(struct anv_query_pool *pool, uint32_t query)
+{
+ if (pool->type == VK_QUERY_TYPE_PERFORMANCE_QUERY_KHR) {
+ for (uint32_t p = 0; p < pool->n_passes; p++) {
+ volatile uint64_t *slot =
+ pool->bo->map + khr_perf_query_availability_offset(pool, query, p);
+ if (!slot[0])
+ return false;
+ }
+ return true;
+ } else {
+ return *(volatile uint64_t *)query_slot(pool, query);
+ }
+}
+
+static VkResult
+wait_for_available(struct anv_device *device,
+ struct anv_query_pool *pool, uint32_t query)
+{
+ uint64_t abs_timeout = anv_get_absolute_timeout(5 * NSEC_PER_SEC);
+
+ while (anv_gettime_ns() < abs_timeout) {
+ if (query_is_available(pool, query))
+ return VK_SUCCESS;
+ VkResult status = anv_device_query_status(device);
+ if (status != VK_SUCCESS)
+ return status;
+ }
+
+ return anv_device_set_lost(device, "query timeout");