+static void si_pc_query_destroy(struct si_context *sctx,
+ struct si_query *squery)
+{
+ struct si_query_pc *query = (struct si_query_pc *)squery;
+
+ while (query->groups) {
+ struct si_query_group *group = query->groups;
+ query->groups = group->next;
+ FREE(group);
+ }
+
+ FREE(query->counters);
+
+ si_query_buffer_destroy(sctx->screen, &query->buffer);
+ FREE(query);
+}
+
+static void si_pc_query_resume(struct si_context *sctx, struct si_query *squery)
+/*
+ struct si_query_hw *hwquery,
+ struct si_resource *buffer, uint64_t va)*/
+{
+ struct si_query_pc *query = (struct si_query_pc *)squery;
+ int current_se = -1;
+ int current_instance = -1;
+
+ if (!si_query_buffer_alloc(sctx, &query->buffer, NULL, query->result_size))
+ return;
+ si_need_gfx_cs_space(sctx);
+
+ if (query->shaders)
+ si_pc_emit_shaders(sctx, query->shaders);
+
+ for (struct si_query_group *group = query->groups; group; group = group->next) {
+ struct si_pc_block *block = group->block;
+
+ if (group->se != current_se || group->instance != current_instance) {
+ current_se = group->se;
+ current_instance = group->instance;
+ si_pc_emit_instance(sctx, group->se, group->instance);
+ }
+
+ si_pc_emit_select(sctx, block, group->num_counters, group->selectors);
+ }
+
+ if (current_se != -1 || current_instance != -1)
+ si_pc_emit_instance(sctx, -1, -1);
+
+ uint64_t va = query->buffer.buf->gpu_address + query->buffer.results_end;
+ si_pc_emit_start(sctx, query->buffer.buf, va);
+}
+
+static void si_pc_query_suspend(struct si_context *sctx, struct si_query *squery)
+{
+ struct si_query_pc *query = (struct si_query_pc *)squery;
+
+ if (!query->buffer.buf)
+ return;
+
+ uint64_t va = query->buffer.buf->gpu_address + query->buffer.results_end;
+ query->buffer.results_end += query->result_size;
+
+ si_pc_emit_stop(sctx, query->buffer.buf, va);
+
+ for (struct si_query_group *group = query->groups; group; group = group->next) {
+ struct si_pc_block *block = group->block;
+ unsigned se = group->se >= 0 ? group->se : 0;
+ unsigned se_end = se + 1;
+
+ if ((block->b->b->flags & SI_PC_BLOCK_SE) && (group->se < 0))
+ se_end = sctx->screen->info.max_se;
+
+ do {
+ unsigned instance = group->instance >= 0 ? group->instance : 0;
+
+ do {
+ si_pc_emit_instance(sctx, se, instance);
+ si_pc_emit_read(sctx, block, group->num_counters, va);
+ va += sizeof(uint64_t) * group->num_counters;
+ } while (group->instance < 0 && ++instance < block->num_instances);
+ } while (++se < se_end);
+ }
+
+ si_pc_emit_instance(sctx, -1, -1);
+}
+
+static bool si_pc_query_begin(struct si_context *ctx, struct si_query *squery)
+{
+ struct si_query_pc *query = (struct si_query_pc *)squery;
+
+ si_query_buffer_reset(ctx, &query->buffer);
+
+ LIST_ADDTAIL(&query->b.active_list, &ctx->active_queries);
+ ctx->num_cs_dw_queries_suspend += query->b.num_cs_dw_suspend;
+
+ si_pc_query_resume(ctx, squery);
+
+ return true;
+}
+
+static bool si_pc_query_end(struct si_context *ctx, struct si_query *squery)
+{
+ struct si_query_pc *query = (struct si_query_pc *)squery;
+
+ si_pc_query_suspend(ctx, squery);
+
+ LIST_DEL(&squery->active_list);
+ ctx->num_cs_dw_queries_suspend -= squery->num_cs_dw_suspend;
+
+ return query->buffer.buf != NULL;
+}
+
+static void si_pc_query_add_result(struct si_query_pc *query,
+ void *buffer,
+ union pipe_query_result *result)
+{
+ uint64_t *results = buffer;
+ unsigned i, j;
+
+ for (i = 0; i < query->num_counters; ++i) {
+ struct si_query_counter *counter = &query->counters[i];
+
+ for (j = 0; j < counter->qwords; ++j) {
+ uint32_t value = results[counter->base + j * counter->stride];
+ result->batch[i].u64 += value;
+ }
+ }
+}
+
+static bool si_pc_query_get_result(struct si_context *sctx, struct si_query *squery,
+ bool wait, union pipe_query_result *result)
+{
+ struct si_query_pc *query = (struct si_query_pc *)squery;
+
+ memset(result, 0, sizeof(result->batch[0]) * query->num_counters);
+
+ for (struct si_query_buffer *qbuf = &query->buffer; qbuf; qbuf = qbuf->previous) {
+ unsigned usage = PIPE_TRANSFER_READ |
+ (wait ? 0 : PIPE_TRANSFER_DONTBLOCK);
+ unsigned results_base = 0;
+ void *map;
+
+ if (squery->b.flushed)
+ map = sctx->ws->buffer_map(qbuf->buf->buf, NULL, usage);
+ else
+ map = si_buffer_map_sync_with_rings(sctx, qbuf->buf, usage);
+
+ if (!map)
+ return false;
+
+ while (results_base != qbuf->results_end) {
+ si_pc_query_add_result(query, map + results_base, result);
+ results_base += query->result_size;
+ }
+ }
+
+ return true;
+}
+
+static const struct si_query_ops batch_query_ops = {
+ .destroy = si_pc_query_destroy,
+ .begin = si_pc_query_begin,
+ .end = si_pc_query_end,
+ .get_result = si_pc_query_get_result,
+
+ .suspend = si_pc_query_suspend,
+ .resume = si_pc_query_resume,
+};
+
+static struct si_query_group *get_group_state(struct si_screen *screen,
+ struct si_query_pc *query,
+ struct si_pc_block *block,
+ unsigned sub_gid)
+{
+ struct si_query_group *group = query->groups;
+
+ while (group) {
+ if (group->block == block && group->sub_gid == sub_gid)
+ return group;
+ group = group->next;
+ }
+
+ group = CALLOC_STRUCT(si_query_group);
+ if (!group)
+ return NULL;
+
+ group->block = block;
+ group->sub_gid = sub_gid;
+
+ if (block->b->b->flags & SI_PC_BLOCK_SHADER) {
+ unsigned sub_gids = block->num_instances;
+ unsigned shader_id;
+ unsigned shaders;
+ unsigned query_shaders;
+
+ if (si_pc_block_has_per_se_groups(screen->perfcounters, block))
+ sub_gids = sub_gids * screen->info.max_se;
+ shader_id = sub_gid / sub_gids;
+ sub_gid = sub_gid % sub_gids;
+
+ shaders = si_pc_shader_type_bits[shader_id];
+
+ query_shaders = query->shaders & ~SI_PC_SHADERS_WINDOWING;
+ if (query_shaders && query_shaders != shaders) {
+ fprintf(stderr, "si_perfcounter: incompatible shader groups\n");
+ FREE(group);
+ return NULL;
+ }
+ query->shaders = shaders;
+ }
+
+ if (block->b->b->flags & SI_PC_BLOCK_SHADER_WINDOWED && !query->shaders) {
+ // A non-zero value in query->shaders ensures that the shader
+ // masking is reset unless the user explicitly requests one.
+ query->shaders = SI_PC_SHADERS_WINDOWING;
+ }
+
+ if (si_pc_block_has_per_se_groups(screen->perfcounters, block)) {
+ group->se = sub_gid / block->num_instances;
+ sub_gid = sub_gid % block->num_instances;
+ } else {
+ group->se = -1;
+ }
+
+ if (si_pc_block_has_per_instance_groups(screen->perfcounters, block)) {
+ group->instance = sub_gid;
+ } else {
+ group->instance = -1;
+ }
+
+ group->next = query->groups;
+ query->groups = group;
+
+ return group;
+}
+
+struct pipe_query *si_create_batch_query(struct pipe_context *ctx,
+ unsigned num_queries,
+ unsigned *query_types)
+{
+ struct si_screen *screen =
+ (struct si_screen *)ctx->screen;
+ struct si_perfcounters *pc = screen->perfcounters;
+ struct si_pc_block *block;
+ struct si_query_group *group;
+ struct si_query_pc *query;
+ unsigned base_gid, sub_gid, sub_index;
+ unsigned i, j;
+
+ if (!pc)
+ return NULL;
+
+ query = CALLOC_STRUCT(si_query_pc);
+ if (!query)
+ return NULL;
+
+ query->b.ops = &batch_query_ops;
+
+ query->num_counters = num_queries;
+
+ /* Collect selectors per group */
+ for (i = 0; i < num_queries; ++i) {
+ unsigned sub_gid;
+
+ if (query_types[i] < SI_QUERY_FIRST_PERFCOUNTER)
+ goto error;
+
+ block = lookup_counter(pc, query_types[i] - SI_QUERY_FIRST_PERFCOUNTER,
+ &base_gid, &sub_index);
+ if (!block)
+ goto error;
+
+ sub_gid = sub_index / block->b->selectors;
+ sub_index = sub_index % block->b->selectors;
+
+ group = get_group_state(screen, query, block, sub_gid);
+ if (!group)
+ goto error;
+
+ if (group->num_counters >= block->b->b->num_counters) {
+ fprintf(stderr,
+ "perfcounter group %s: too many selected\n",
+ block->b->b->name);
+ goto error;
+ }
+ group->selectors[group->num_counters] = sub_index;
+ ++group->num_counters;
+ }
+
+ /* Compute result bases and CS size per group */
+ query->b.num_cs_dw_suspend = pc->num_stop_cs_dwords;
+ query->b.num_cs_dw_suspend += pc->num_instance_cs_dwords;
+
+ i = 0;
+ for (group = query->groups; group; group = group->next) {
+ struct si_pc_block *block = group->block;
+ unsigned read_dw;
+ unsigned instances = 1;
+
+ if ((block->b->b->flags & SI_PC_BLOCK_SE) && group->se < 0)
+ instances = screen->info.max_se;
+ if (group->instance < 0)
+ instances *= block->num_instances;
+
+ group->result_base = i;
+ query->result_size += sizeof(uint64_t) * instances * group->num_counters;
+ i += instances * group->num_counters;
+
+ read_dw = 6 * group->num_counters;
+ query->b.num_cs_dw_suspend += instances * read_dw;
+ query->b.num_cs_dw_suspend += instances * pc->num_instance_cs_dwords;
+ }
+
+ if (query->shaders) {
+ if (query->shaders == SI_PC_SHADERS_WINDOWING)
+ query->shaders = 0xffffffff;
+ }
+
+ /* Map user-supplied query array to result indices */
+ query->counters = CALLOC(num_queries, sizeof(*query->counters));
+ for (i = 0; i < num_queries; ++i) {
+ struct si_query_counter *counter = &query->counters[i];
+ struct si_pc_block *block;
+
+ block = lookup_counter(pc, query_types[i] - SI_QUERY_FIRST_PERFCOUNTER,
+ &base_gid, &sub_index);
+
+ sub_gid = sub_index / block->b->selectors;
+ sub_index = sub_index % block->b->selectors;
+
+ group = get_group_state(screen, query, block, sub_gid);
+ assert(group != NULL);
+
+ for (j = 0; j < group->num_counters; ++j) {
+ if (group->selectors[j] == sub_index)
+ break;
+ }
+
+ counter->base = group->result_base + j;
+ counter->stride = group->num_counters;
+
+ counter->qwords = 1;
+ if ((block->b->b->flags & SI_PC_BLOCK_SE) && group->se < 0)
+ counter->qwords = screen->info.max_se;
+ if (group->instance < 0)
+ counter->qwords *= block->num_instances;
+ }
+
+ return (struct pipe_query *)query;
+
+error:
+ si_pc_query_destroy((struct si_context *)ctx, &query->b);
+ return NULL;
+}
+
+static bool si_init_block_names(struct si_screen *screen,
+ struct si_pc_block *block)