+static void si_pc_emit_stop(struct si_context *sctx, struct si_resource *buffer, uint64_t va)
+{
+ struct radeon_cmdbuf *cs = sctx->gfx_cs;
+
+ si_cp_release_mem(sctx, cs, V_028A90_BOTTOM_OF_PIPE_TS, 0, EOP_DST_SEL_MEM, EOP_INT_SEL_NONE,
+ EOP_DATA_SEL_VALUE_32BIT, buffer, va, 0, SI_NOT_QUERY);
+ si_cp_wait_mem(sctx, cs, va, 0, 0xffffffff, WAIT_REG_MEM_EQUAL);
+
+ radeon_emit(cs, PKT3(PKT3_EVENT_WRITE, 0, 0));
+ radeon_emit(cs, EVENT_TYPE(V_028A90_PERFCOUNTER_SAMPLE) | EVENT_INDEX(0));
+ radeon_emit(cs, PKT3(PKT3_EVENT_WRITE, 0, 0));
+ radeon_emit(cs, EVENT_TYPE(V_028A90_PERFCOUNTER_STOP) | EVENT_INDEX(0));
+ radeon_set_uconfig_reg(
+ cs, R_036020_CP_PERFMON_CNTL,
+ S_036020_PERFMON_STATE(V_036020_STOP_COUNTING) | S_036020_PERFMON_SAMPLE_ENABLE(1));
+}
+
+static void si_pc_emit_read(struct si_context *sctx, struct si_pc_block *block, unsigned count,
+ uint64_t va)
+{
+ struct si_pc_block_base *regs = block->b->b;
+ struct radeon_cmdbuf *cs = sctx->gfx_cs;
+ unsigned idx;
+ unsigned reg = regs->counter0_lo;
+ unsigned reg_delta = 8;
+
+ if (!(regs->layout & SI_PC_FAKE)) {
+ if (regs->layout & SI_PC_REG_REVERSE)
+ reg_delta = -reg_delta;
+
+ for (idx = 0; idx < count; ++idx) {
+ if (regs->counters)
+ reg = regs->counters[idx];
+
+ radeon_emit(cs, PKT3(PKT3_COPY_DATA, 4, 0));
+ radeon_emit(cs, COPY_DATA_SRC_SEL(COPY_DATA_PERF) | COPY_DATA_DST_SEL(COPY_DATA_DST_MEM) |
+ COPY_DATA_COUNT_SEL); /* 64 bits */
+ radeon_emit(cs, reg >> 2);
+ radeon_emit(cs, 0); /* unused */
+ radeon_emit(cs, va);
+ radeon_emit(cs, va >> 32);
+ va += sizeof(uint64_t);
+ reg += reg_delta;
+ }
+ } else {
+ for (idx = 0; idx < count; ++idx) {
+ radeon_emit(cs, PKT3(PKT3_COPY_DATA, 4, 0));
+ radeon_emit(cs, COPY_DATA_SRC_SEL(COPY_DATA_IMM) | COPY_DATA_DST_SEL(COPY_DATA_DST_MEM) |
+ COPY_DATA_COUNT_SEL);
+ radeon_emit(cs, 0); /* immediate */
+ radeon_emit(cs, 0);
+ radeon_emit(cs, va);
+ radeon_emit(cs, va >> 32);
+ va += sizeof(uint64_t);
+ }
+ }
+}
+
+static void si_pc_query_destroy(struct si_context *sctx, struct si_query *squery)
+{
+ struct si_query_pc *query = (struct si_query_pc *)squery;
+
+ while (query->groups) {
+ struct si_query_group *group = query->groups;
+ query->groups = group->next;
+ FREE(group);
+ }
+
+ FREE(query->counters);
+
+ si_query_buffer_destroy(sctx->screen, &query->buffer);
+ FREE(query);
+}
+
+static void si_inhibit_clockgating(struct si_context *sctx, bool inhibit)
+{
+ if (sctx->chip_class >= GFX10) {
+ radeon_set_uconfig_reg(sctx->gfx_cs, R_037390_RLC_PERFMON_CLK_CNTL,
+ S_037390_PERFMON_CLOCK_STATE(inhibit));
+ } else if (sctx->chip_class >= GFX8) {
+ radeon_set_uconfig_reg(sctx->gfx_cs, R_0372FC_RLC_PERFMON_CLK_CNTL,
+ S_0372FC_PERFMON_CLOCK_STATE(inhibit));
+ }
+}
+
+static void si_pc_query_resume(struct si_context *sctx, struct si_query *squery)
+/*
+ struct si_query_hw *hwquery,
+ struct si_resource *buffer, uint64_t va)*/
+{
+ struct si_query_pc *query = (struct si_query_pc *)squery;
+ int current_se = -1;
+ int current_instance = -1;
+
+ if (!si_query_buffer_alloc(sctx, &query->buffer, NULL, query->result_size))
+ return;
+ si_need_gfx_cs_space(sctx);
+
+ if (query->shaders)
+ si_pc_emit_shaders(sctx, query->shaders);
+
+ si_inhibit_clockgating(sctx, true);
+
+ for (struct si_query_group *group = query->groups; group; group = group->next) {
+ struct si_pc_block *block = group->block;
+
+ if (group->se != current_se || group->instance != current_instance) {
+ current_se = group->se;
+ current_instance = group->instance;
+ si_pc_emit_instance(sctx, group->se, group->instance);
+ }
+
+ si_pc_emit_select(sctx, block, group->num_counters, group->selectors);
+ }
+
+ if (current_se != -1 || current_instance != -1)
+ si_pc_emit_instance(sctx, -1, -1);
+
+ uint64_t va = query->buffer.buf->gpu_address + query->buffer.results_end;
+ si_pc_emit_start(sctx, query->buffer.buf, va);
+}
+
+static void si_pc_query_suspend(struct si_context *sctx, struct si_query *squery)
+{
+ struct si_query_pc *query = (struct si_query_pc *)squery;
+
+ if (!query->buffer.buf)
+ return;
+
+ uint64_t va = query->buffer.buf->gpu_address + query->buffer.results_end;
+ query->buffer.results_end += query->result_size;
+
+ si_pc_emit_stop(sctx, query->buffer.buf, va);
+
+ for (struct si_query_group *group = query->groups; group; group = group->next) {
+ struct si_pc_block *block = group->block;
+ unsigned se = group->se >= 0 ? group->se : 0;
+ unsigned se_end = se + 1;
+
+ if ((block->b->b->flags & SI_PC_BLOCK_SE) && (group->se < 0))
+ se_end = sctx->screen->info.max_se;
+
+ do {
+ unsigned instance = group->instance >= 0 ? group->instance : 0;
+
+ do {
+ si_pc_emit_instance(sctx, se, instance);
+ si_pc_emit_read(sctx, block, group->num_counters, va);
+ va += sizeof(uint64_t) * group->num_counters;
+ } while (group->instance < 0 && ++instance < block->num_instances);
+ } while (++se < se_end);
+ }
+
+ si_pc_emit_instance(sctx, -1, -1);
+
+ si_inhibit_clockgating(sctx, false);
+}
+
+static bool si_pc_query_begin(struct si_context *ctx, struct si_query *squery)
+{
+ struct si_query_pc *query = (struct si_query_pc *)squery;
+
+ si_query_buffer_reset(ctx, &query->buffer);
+
+ list_addtail(&query->b.active_list, &ctx->active_queries);
+ ctx->num_cs_dw_queries_suspend += query->b.num_cs_dw_suspend;
+
+ si_pc_query_resume(ctx, squery);
+
+ return true;
+}
+
+static bool si_pc_query_end(struct si_context *ctx, struct si_query *squery)
+{
+ struct si_query_pc *query = (struct si_query_pc *)squery;
+
+ si_pc_query_suspend(ctx, squery);
+
+ list_del(&squery->active_list);
+ ctx->num_cs_dw_queries_suspend -= squery->num_cs_dw_suspend;
+
+ return query->buffer.buf != NULL;
+}
+
+static void si_pc_query_add_result(struct si_query_pc *query, void *buffer,
+ union pipe_query_result *result)
+{
+ uint64_t *results = buffer;
+ unsigned i, j;
+
+ for (i = 0; i < query->num_counters; ++i) {
+ struct si_query_counter *counter = &query->counters[i];
+
+ for (j = 0; j < counter->qwords; ++j) {
+ uint32_t value = results[counter->base + j * counter->stride];
+ result->batch[i].u64 += value;
+ }
+ }
+}
+
+static bool si_pc_query_get_result(struct si_context *sctx, struct si_query *squery, bool wait,
+ union pipe_query_result *result)
+{
+ struct si_query_pc *query = (struct si_query_pc *)squery;
+
+ memset(result, 0, sizeof(result->batch[0]) * query->num_counters);
+
+ for (struct si_query_buffer *qbuf = &query->buffer; qbuf; qbuf = qbuf->previous) {
+ unsigned usage = PIPE_TRANSFER_READ | (wait ? 0 : PIPE_TRANSFER_DONTBLOCK);
+ unsigned results_base = 0;
+ void *map;
+
+ if (squery->b.flushed)
+ map = sctx->ws->buffer_map(qbuf->buf->buf, NULL, usage);
+ else
+ map = si_buffer_map_sync_with_rings(sctx, qbuf->buf, usage);
+
+ if (!map)
+ return false;
+
+ while (results_base != qbuf->results_end) {
+ si_pc_query_add_result(query, map + results_base, result);
+ results_base += query->result_size;
+ }
+ }
+
+ return true;
+}
+
+static const struct si_query_ops batch_query_ops = {
+ .destroy = si_pc_query_destroy,
+ .begin = si_pc_query_begin,
+ .end = si_pc_query_end,
+ .get_result = si_pc_query_get_result,
+
+ .suspend = si_pc_query_suspend,
+ .resume = si_pc_query_resume,
+};
+
+static struct si_query_group *get_group_state(struct si_screen *screen, struct si_query_pc *query,
+ struct si_pc_block *block, unsigned sub_gid)
+{
+ struct si_query_group *group = query->groups;
+
+ while (group) {
+ if (group->block == block && group->sub_gid == sub_gid)
+ return group;
+ group = group->next;
+ }
+
+ group = CALLOC_STRUCT(si_query_group);
+ if (!group)
+ return NULL;
+
+ group->block = block;
+ group->sub_gid = sub_gid;
+
+ if (block->b->b->flags & SI_PC_BLOCK_SHADER) {
+ unsigned sub_gids = block->num_instances;
+ unsigned shader_id;
+ unsigned shaders;
+ unsigned query_shaders;
+
+ if (si_pc_block_has_per_se_groups(screen->perfcounters, block))
+ sub_gids = sub_gids * screen->info.max_se;
+ shader_id = sub_gid / sub_gids;
+ sub_gid = sub_gid % sub_gids;
+
+ shaders = si_pc_shader_type_bits[shader_id];
+
+ query_shaders = query->shaders & ~SI_PC_SHADERS_WINDOWING;
+ if (query_shaders && query_shaders != shaders) {
+ fprintf(stderr, "si_perfcounter: incompatible shader groups\n");
+ FREE(group);
+ return NULL;
+ }
+ query->shaders = shaders;
+ }
+
+ if (block->b->b->flags & SI_PC_BLOCK_SHADER_WINDOWED && !query->shaders) {
+ // A non-zero value in query->shaders ensures that the shader
+ // masking is reset unless the user explicitly requests one.
+ query->shaders = SI_PC_SHADERS_WINDOWING;
+ }
+
+ if (si_pc_block_has_per_se_groups(screen->perfcounters, block)) {
+ group->se = sub_gid / block->num_instances;
+ sub_gid = sub_gid % block->num_instances;
+ } else {
+ group->se = -1;
+ }
+
+ if (si_pc_block_has_per_instance_groups(screen->perfcounters, block)) {
+ group->instance = sub_gid;
+ } else {
+ group->instance = -1;
+ }
+
+ group->next = query->groups;
+ query->groups = group;
+
+ return group;
+}
+
+struct pipe_query *si_create_batch_query(struct pipe_context *ctx, unsigned num_queries,
+ unsigned *query_types)
+{
+ struct si_screen *screen = (struct si_screen *)ctx->screen;
+ struct si_perfcounters *pc = screen->perfcounters;
+ struct si_pc_block *block;
+ struct si_query_group *group;
+ struct si_query_pc *query;
+ unsigned base_gid, sub_gid, sub_index;
+ unsigned i, j;
+
+ if (!pc)
+ return NULL;
+
+ query = CALLOC_STRUCT(si_query_pc);
+ if (!query)
+ return NULL;
+
+ query->b.ops = &batch_query_ops;
+
+ query->num_counters = num_queries;
+
+ /* Collect selectors per group */
+ for (i = 0; i < num_queries; ++i) {
+ unsigned sub_gid;
+
+ if (query_types[i] < SI_QUERY_FIRST_PERFCOUNTER)
+ goto error;
+
+ block =
+ lookup_counter(pc, query_types[i] - SI_QUERY_FIRST_PERFCOUNTER, &base_gid, &sub_index);
+ if (!block)
+ goto error;
+
+ sub_gid = sub_index / block->b->selectors;
+ sub_index = sub_index % block->b->selectors;
+
+ group = get_group_state(screen, query, block, sub_gid);
+ if (!group)
+ goto error;
+
+ if (group->num_counters >= block->b->b->num_counters) {
+ fprintf(stderr, "perfcounter group %s: too many selected\n", block->b->b->name);
+ goto error;
+ }
+ group->selectors[group->num_counters] = sub_index;
+ ++group->num_counters;
+ }
+
+ /* Compute result bases and CS size per group */
+ query->b.num_cs_dw_suspend = pc->num_stop_cs_dwords;
+ query->b.num_cs_dw_suspend += pc->num_instance_cs_dwords;
+
+ i = 0;
+ for (group = query->groups; group; group = group->next) {
+ struct si_pc_block *block = group->block;
+ unsigned read_dw;
+ unsigned instances = 1;
+
+ if ((block->b->b->flags & SI_PC_BLOCK_SE) && group->se < 0)
+ instances = screen->info.max_se;
+ if (group->instance < 0)
+ instances *= block->num_instances;
+
+ group->result_base = i;
+ query->result_size += sizeof(uint64_t) * instances * group->num_counters;
+ i += instances * group->num_counters;
+
+ read_dw = 6 * group->num_counters;
+ query->b.num_cs_dw_suspend += instances * read_dw;
+ query->b.num_cs_dw_suspend += instances * pc->num_instance_cs_dwords;
+ }
+
+ if (query->shaders) {
+ if (query->shaders == SI_PC_SHADERS_WINDOWING)
+ query->shaders = 0xffffffff;
+ }
+
+ /* Map user-supplied query array to result indices */
+ query->counters = CALLOC(num_queries, sizeof(*query->counters));
+ for (i = 0; i < num_queries; ++i) {
+ struct si_query_counter *counter = &query->counters[i];
+ struct si_pc_block *block;
+
+ block =
+ lookup_counter(pc, query_types[i] - SI_QUERY_FIRST_PERFCOUNTER, &base_gid, &sub_index);
+
+ sub_gid = sub_index / block->b->selectors;
+ sub_index = sub_index % block->b->selectors;
+
+ group = get_group_state(screen, query, block, sub_gid);
+ assert(group != NULL);
+
+ for (j = 0; j < group->num_counters; ++j) {
+ if (group->selectors[j] == sub_index)
+ break;
+ }
+
+ counter->base = group->result_base + j;
+ counter->stride = group->num_counters;
+
+ counter->qwords = 1;
+ if ((block->b->b->flags & SI_PC_BLOCK_SE) && group->se < 0)
+ counter->qwords = screen->info.max_se;
+ if (group->instance < 0)
+ counter->qwords *= block->num_instances;
+ }
+
+ return (struct pipe_query *)query;
+
+error:
+ si_pc_query_destroy((struct si_context *)ctx, &query->b);
+ return NULL;
+}
+
+static bool si_init_block_names(struct si_screen *screen, struct si_pc_block *block)
+{
+ bool per_instance_groups = si_pc_block_has_per_instance_groups(screen->perfcounters, block);
+ bool per_se_groups = si_pc_block_has_per_se_groups(screen->perfcounters, block);
+ unsigned i, j, k;
+ unsigned groups_shader = 1, groups_se = 1, groups_instance = 1;
+ unsigned namelen;
+ char *groupname;
+ char *p;
+
+ if (per_instance_groups)
+ groups_instance = block->num_instances;
+ if (per_se_groups)
+ groups_se = screen->info.max_se;
+ if (block->b->b->flags & SI_PC_BLOCK_SHADER)
+ groups_shader = ARRAY_SIZE(si_pc_shader_type_bits);
+
+ namelen = strlen(block->b->b->name);
+ block->group_name_stride = namelen + 1;
+ if (block->b->b->flags & SI_PC_BLOCK_SHADER)
+ block->group_name_stride += 3;
+ if (per_se_groups) {
+ assert(groups_se <= 10);
+ block->group_name_stride += 1;
+
+ if (per_instance_groups)
+ block->group_name_stride += 1;
+ }
+ if (per_instance_groups) {
+ assert(groups_instance <= 100);
+ block->group_name_stride += 2;
+ }
+
+ block->group_names = MALLOC(block->num_groups * block->group_name_stride);
+ if (!block->group_names)
+ return false;
+
+ groupname = block->group_names;
+ for (i = 0; i < groups_shader; ++i) {
+ const char *shader_suffix = si_pc_shader_type_suffixes[i];
+ unsigned shaderlen = strlen(shader_suffix);
+ for (j = 0; j < groups_se; ++j) {
+ for (k = 0; k < groups_instance; ++k) {
+ strcpy(groupname, block->b->b->name);
+ p = groupname + namelen;
+
+ if (block->b->b->flags & SI_PC_BLOCK_SHADER) {
+ strcpy(p, shader_suffix);
+ p += shaderlen;
+ }
+
+ if (per_se_groups) {
+ p += sprintf(p, "%d", j);
+ if (per_instance_groups)
+ *p++ = '_';
+ }
+
+ if (per_instance_groups)
+ p += sprintf(p, "%d", k);
+
+ groupname += block->group_name_stride;
+ }
+ }
+ }
+
+ assert(block->b->selectors <= 1000);
+ block->selector_name_stride = block->group_name_stride + 4;
+ block->selector_names =
+ MALLOC(block->num_groups * block->b->selectors * block->selector_name_stride);
+ if (!block->selector_names)
+ return false;
+
+ groupname = block->group_names;
+ p = block->selector_names;
+ for (i = 0; i < block->num_groups; ++i) {
+ for (j = 0; j < block->b->selectors; ++j) {
+ sprintf(p, "%s_%03d", groupname, j);
+ p += block->selector_name_stride;
+ }
+ groupname += block->group_name_stride;
+ }
+
+ return true;
+}
+
+int si_get_perfcounter_info(struct si_screen *screen, unsigned index,
+ struct pipe_driver_query_info *info)