src/gallium/drivers/freedreno/freedreno_query_hw.c

   1 /* -*- mode: C; c-file-style: "k&r"; tab-width 4; indent-tabs-mode: t; -*- */
   2
   3 /*
   4  * Copyright (C) 2014 Rob Clark <robclark@freedesktop.org>
   5  *
   6  * Permission is hereby granted, free of charge, to any person obtaining a
   7  * copy of this software and associated documentation files (the "Software"),
   8  * to deal in the Software without restriction, including without limitation
   9  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  10  * and/or sell copies of the Software, and to permit persons to whom the
  11  * Software is furnished to do so, subject to the following conditions:
  12  *
  13  * The above copyright notice and this permission notice (including the next
  14  * paragraph) shall be included in all copies or substantial portions of the
  15  * Software.
  16  *
  17  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  18  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  19  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  20  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  21  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  22  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  23  * SOFTWARE.
  24  *
  25  * Authors:
  26  *    Rob Clark <robclark@freedesktop.org>
  27  */
  28
  29 #include "pipe/p_state.h"
  30 #include "util/u_memory.h"
  31 #include "util/u_inlines.h"
  32
  33 #include "freedreno_query_hw.h"
  34 #include "freedreno_context.h"
  35 #include "freedreno_resource.h"
  36 #include "freedreno_util.h"
  37
  38 struct fd_hw_sample_period {
  39         struct fd_hw_sample *start, *end;
  40         struct list_head list;
  41 };
  42
  43 /* maps query_type to sample provider idx: */
  44 static int pidx(unsigned query_type)
  45 {
  46         switch (query_type) {
  47         case PIPE_QUERY_OCCLUSION_COUNTER:
  48                 return 0;
  49         case PIPE_QUERY_OCCLUSION_PREDICATE:
  50                 return 1;
  51         /* TODO currently queries only emitted in main pass (not in binning pass)..
  52          * which is fine for occlusion query, but pretty much not anything else.
  53          */
  54         case PIPE_QUERY_TIME_ELAPSED:
  55                 return 2;
  56         case PIPE_QUERY_TIMESTAMP:
  57                 return 3;
  58         default:
  59                 return -1;
  60         }
  61 }
  62
  63 static struct fd_hw_sample *
  64 get_sample(struct fd_batch *batch, struct fd_ringbuffer *ring,
  65                 unsigned query_type)
  66 {
  67         struct fd_context *ctx = batch->ctx;
  68         struct fd_hw_sample *samp = NULL;
  69         int idx = pidx(query_type);
  70
  71         assume(idx >= 0);   /* query never would have been created otherwise */
  72
  73         if (!batch->sample_cache[idx]) {
  74                 struct fd_hw_sample *new_samp =
  75                         ctx->sample_providers[idx]->get_sample(batch, ring);
  76                 fd_hw_sample_reference(ctx, &batch->sample_cache[idx], new_samp);
  77                 util_dynarray_append(&batch->samples, struct fd_hw_sample *, new_samp);
  78                 batch->needs_flush = true;
  79         }
  80
  81         fd_hw_sample_reference(ctx, &samp, batch->sample_cache[idx]);
  82
  83         return samp;
  84 }
  85
  86 static void
  87 clear_sample_cache(struct fd_batch *batch)
  88 {
  89         int i;
  90
  91         for (i = 0; i < ARRAY_SIZE(batch->sample_cache); i++)
  92                 fd_hw_sample_reference(batch->ctx, &batch->sample_cache[i], NULL);
  93 }
  94
  95 static bool
  96 is_active(struct fd_hw_query *hq, enum fd_render_stage stage)
  97 {
  98         return !!(hq->provider->active & stage);
  99 }
 100
 101
 102 static void
 103 resume_query(struct fd_batch *batch, struct fd_hw_query *hq,
 104                 struct fd_ringbuffer *ring)
 105 {
 106         int idx = pidx(hq->provider->query_type);
 107         DBG("%p", hq);
 108         assert(idx >= 0);   /* query never would have been created otherwise */
 109         assert(!hq->period);
 110         batch->active_providers |= (1 << idx);
 111         hq->period = slab_alloc_st(&batch->ctx->sample_period_pool);
 112         list_inithead(&hq->period->list);
 113         hq->period->start = get_sample(batch, ring, hq->base.type);
 114         /* NOTE: slab_alloc_st() does not zero out the buffer: */
 115         hq->period->end = NULL;
 116 }
 117
 118 static void
 119 pause_query(struct fd_batch *batch, struct fd_hw_query *hq,
 120                 struct fd_ringbuffer *ring)
 121 {
 122         int idx = pidx(hq->provider->query_type);
 123         DBG("%p", hq);
 124         assert(idx >= 0);   /* query never would have been created otherwise */
 125         assert(hq->period && !hq->period->end);
 126         assert(batch->active_providers & (1 << idx));
 127         hq->period->end = get_sample(batch, ring, hq->base.type);
 128         list_addtail(&hq->period->list, &hq->periods);
 129         hq->period = NULL;
 130 }
 131
 132 static void
 133 destroy_periods(struct fd_context *ctx, struct fd_hw_query *hq)
 134 {
 135         struct fd_hw_sample_period *period, *s;
 136         LIST_FOR_EACH_ENTRY_SAFE(period, s, &hq->periods, list) {
 137                 fd_hw_sample_reference(ctx, &period->start, NULL);
 138                 fd_hw_sample_reference(ctx, &period->end, NULL);
 139                 list_del(&period->list);
 140                 slab_free_st(&ctx->sample_period_pool, period);
 141         }
 142 }
 143
 144 static void
 145 fd_hw_destroy_query(struct fd_context *ctx, struct fd_query *q)
 146 {
 147         struct fd_hw_query *hq = fd_hw_query(q);
 148
 149         DBG("%p: active=%d", q, q->active);
 150
 151         destroy_periods(ctx, hq);
 152         list_del(&hq->list);
 153
 154         free(hq);
 155 }
 156
 157 static boolean
 158 fd_hw_begin_query(struct fd_context *ctx, struct fd_query *q)
 159 {
 160         struct fd_batch *batch = ctx->batch;
 161         struct fd_hw_query *hq = fd_hw_query(q);
 162
 163         DBG("%p: active=%d", q, q->active);
 164
 165         /* begin_query() should clear previous results: */
 166         destroy_periods(ctx, hq);
 167
 168         if (batch && is_active(hq, batch->stage))
 169                 resume_query(batch, hq, batch->draw);
 170
 171         /* add to active list: */
 172         assert(list_empty(&hq->list));
 173         list_addtail(&hq->list, &ctx->active_queries);
 174
 175         return true;
 176 }
 177
 178 static void
 179 fd_hw_end_query(struct fd_context *ctx, struct fd_query *q)
 180 {
 181         struct fd_batch *batch = ctx->batch;
 182         struct fd_hw_query *hq = fd_hw_query(q);
 183
 184         DBG("%p: active=%d", q, q->active);
 185
 186         if (batch && is_active(hq, batch->stage))
 187                 pause_query(batch, hq, batch->draw);
 188
 189         /* remove from active list: */
 190         list_delinit(&hq->list);
 191 }
 192
 193 /* helper to get ptr to specified sample: */
 194 static void * sampptr(struct fd_hw_sample *samp, uint32_t n, void *ptr)
 195 {
 196         return ((char *)ptr) + (samp->tile_stride * n) + samp->offset;
 197 }
 198
 199 static boolean
 200 fd_hw_get_query_result(struct fd_context *ctx, struct fd_query *q,
 201                 boolean wait, union pipe_query_result *result)
 202 {
 203         struct fd_hw_query *hq = fd_hw_query(q);
 204         const struct fd_hw_sample_provider *p = hq->provider;
 205         struct fd_hw_sample_period *period;
 206
 207         DBG("%p: wait=%d, active=%d", q, wait, q->active);
 208
 209         if (LIST_IS_EMPTY(&hq->periods))
 210                 return true;
 211
 212         assert(LIST_IS_EMPTY(&hq->list));
 213         assert(!hq->period);
 214
 215         /* if !wait, then check the last sample (the one most likely to
 216          * not be ready yet) and bail if it is not ready:
 217          */
 218         if (!wait) {
 219                 int ret;
 220
 221                 period = LIST_ENTRY(struct fd_hw_sample_period,
 222                                 hq->periods.prev, list);
 223
 224                 struct fd_resource *rsc = fd_resource(period->end->prsc);
 225
 226                 if (pending(rsc, false)) {
 227                         /* piglit spec@arb_occlusion_query@occlusion_query_conform
 228                          * test, and silly apps perhaps, get stuck in a loop trying
 229                          * to get  query result forever with wait==false..  we don't
 230                          * wait to flush unnecessarily but we also don't want to
 231                          * spin forever:
 232                          */
 233                         if (hq->no_wait_cnt++ > 5)
 234                                 fd_batch_flush(rsc->write_batch, false);
 235                         return false;
 236                 }
 237
 238                 if (!rsc->bo)
 239                         return false;
 240
 241                 ret = fd_bo_cpu_prep(rsc->bo, ctx->screen->pipe,
 242                                 DRM_FREEDRENO_PREP_READ | DRM_FREEDRENO_PREP_NOSYNC);
 243                 if (ret)
 244                         return false;
 245
 246                 fd_bo_cpu_fini(rsc->bo);
 247         }
 248
 249         /* sum the result across all sample periods: */
 250         LIST_FOR_EACH_ENTRY(period, &hq->periods, list) {
 251                 struct fd_hw_sample *start = period->start;
 252                 struct fd_hw_sample *end = period->end;
 253                 unsigned i;
 254
 255                 /* start and end samples should be from same batch: */
 256                 assert(start->prsc == end->prsc);
 257                 assert(start->num_tiles == end->num_tiles);
 258
 259                 struct fd_resource *rsc = fd_resource(start->prsc);
 260
 261                 if (rsc->write_batch)
 262                         fd_batch_flush(rsc->write_batch, true);
 263
 264                 /* some piglit tests at least do query with no draws, I guess: */
 265                 if (!rsc->bo)
 266                         continue;
 267
 268                 fd_bo_cpu_prep(rsc->bo, ctx->screen->pipe, DRM_FREEDRENO_PREP_READ);
 269
 270                 void *ptr = fd_bo_map(rsc->bo);
 271
 272                 for (i = 0; i < start->num_tiles; i++) {
 273                         p->accumulate_result(ctx, sampptr(period->start, i, ptr),
 274                                         sampptr(period->end, i, ptr), result);
 275                 }
 276
 277                 fd_bo_cpu_fini(rsc->bo);
 278         }
 279
 280         return true;
 281 }
 282
 283 static const struct fd_query_funcs hw_query_funcs = {
 284                 .destroy_query    = fd_hw_destroy_query,
 285                 .begin_query      = fd_hw_begin_query,
 286                 .end_query        = fd_hw_end_query,
 287                 .get_query_result = fd_hw_get_query_result,
 288 };
 289
 290 struct fd_query *
 291 fd_hw_create_query(struct fd_context *ctx, unsigned query_type)
 292 {
 293         struct fd_hw_query *hq;
 294         struct fd_query *q;
 295         int idx = pidx(query_type);
 296
 297         if ((idx < 0) || !ctx->sample_providers[idx])
 298                 return NULL;
 299
 300         hq = CALLOC_STRUCT(fd_hw_query);
 301         if (!hq)
 302                 return NULL;
 303
 304         DBG("%p: query_type=%u", hq, query_type);
 305
 306         hq->provider = ctx->sample_providers[idx];
 307
 308         list_inithead(&hq->periods);
 309         list_inithead(&hq->list);
 310
 311         q = &hq->base;
 312         q->funcs = &hw_query_funcs;
 313         q->type = query_type;
 314
 315         return q;
 316 }
 317
 318 struct fd_hw_sample *
 319 fd_hw_sample_init(struct fd_batch *batch, uint32_t size)
 320 {
 321         struct fd_hw_sample *samp = slab_alloc_st(&batch->ctx->sample_pool);
 322         pipe_reference_init(&samp->reference, 1);
 323         samp->size = size;
 324         debug_assert(util_is_power_of_two(size));
 325         batch->next_sample_offset = align(batch->next_sample_offset, size);
 326         samp->offset = batch->next_sample_offset;
 327         /* NOTE: slab_alloc_st() does not zero out the buffer: */
 328         samp->prsc = NULL;
 329         samp->num_tiles = 0;
 330         samp->tile_stride = 0;
 331         batch->next_sample_offset += size;
 332
 333         if (!batch->query_buf) {
 334                 struct pipe_screen *pscreen = &batch->ctx->screen->base;
 335                 struct pipe_resource templ = {
 336                         .target  = PIPE_BUFFER,
 337                         .format  = PIPE_FORMAT_R8_UNORM,
 338                         .bind    = PIPE_BIND_QUERY_BUFFER,
 339                         .width0  = 0,    /* create initially zero size buffer */
 340                         .height0 = 1,
 341                         .depth0  = 1,
 342                         .array_size = 1,
 343                         .last_level = 0,
 344                         .nr_samples = 1,
 345                 };
 346                 batch->query_buf = pscreen->resource_create(pscreen, &templ);
 347         }
 348
 349         pipe_resource_reference(&samp->prsc, batch->query_buf);
 350
 351         return samp;
 352 }
 353
 354 void
 355 __fd_hw_sample_destroy(struct fd_context *ctx, struct fd_hw_sample *samp)
 356 {
 357         pipe_resource_reference(&samp->prsc, NULL);
 358         slab_free_st(&ctx->sample_pool, samp);
 359 }
 360
 361 /* called from gmem code once total storage requirements are known (ie.
 362  * number of samples times number of tiles)
 363  */
 364 void
 365 fd_hw_query_prepare(struct fd_batch *batch, uint32_t num_tiles)
 366 {
 367         uint32_t tile_stride = batch->next_sample_offset;
 368
 369         if (tile_stride > 0)
 370                 fd_resource_resize(batch->query_buf, tile_stride * num_tiles);
 371
 372         batch->query_tile_stride = tile_stride;
 373
 374         while (batch->samples.size > 0) {
 375                 struct fd_hw_sample *samp =
 376                         util_dynarray_pop(&batch->samples, struct fd_hw_sample *);
 377                 samp->num_tiles = num_tiles;
 378                 samp->tile_stride = tile_stride;
 379                 fd_hw_sample_reference(batch->ctx, &samp, NULL);
 380         }
 381
 382         /* reset things for next batch: */
 383         batch->next_sample_offset = 0;
 384 }
 385
 386 void
 387 fd_hw_query_prepare_tile(struct fd_batch *batch, uint32_t n,
 388                 struct fd_ringbuffer *ring)
 389 {
 390         uint32_t tile_stride = batch->query_tile_stride;
 391         uint32_t offset = tile_stride * n;
 392
 393         /* bail if no queries: */
 394         if (tile_stride == 0)
 395                 return;
 396
 397         fd_wfi(batch, ring);
 398         OUT_PKT0 (ring, HW_QUERY_BASE_REG, 1);
 399         OUT_RELOCW(ring, fd_resource(batch->query_buf)->bo, offset, 0, 0);
 400 }
 401
 402 void
 403 fd_hw_query_set_stage(struct fd_batch *batch, struct fd_ringbuffer *ring,
 404                 enum fd_render_stage stage)
 405 {
 406         if (stage != batch->stage) {
 407                 struct fd_hw_query *hq;
 408                 LIST_FOR_EACH_ENTRY(hq, &batch->ctx->active_queries, list) {
 409                         bool was_active = is_active(hq, batch->stage);
 410                         bool now_active = is_active(hq, stage);
 411
 412                         if (now_active && !was_active)
 413                                 resume_query(batch, hq, ring);
 414                         else if (was_active && !now_active)
 415                                 pause_query(batch, hq, ring);
 416                 }
 417         }
 418         clear_sample_cache(batch);
 419 }
 420
 421 /* call the provider->enable() for all the hw queries that were active
 422  * in the current batch.  This sets up perfctr selector regs statically
 423  * for the duration of the batch.
 424  */
 425 void
 426 fd_hw_query_enable(struct fd_batch *batch, struct fd_ringbuffer *ring)
 427 {
 428         struct fd_context *ctx = batch->ctx;
 429         for (int idx = 0; idx < MAX_HW_SAMPLE_PROVIDERS; idx++) {
 430                 if (batch->active_providers & (1 << idx)) {
 431                         assert(ctx->sample_providers[idx]);
 432                         if (ctx->sample_providers[idx]->enable)
 433                                 ctx->sample_providers[idx]->enable(ctx, ring);
 434                 }
 435         }
 436         batch->active_providers = 0;  /* clear it for next frame */
 437 }
 438
 439 void
 440 fd_hw_query_register_provider(struct pipe_context *pctx,
 441                 const struct fd_hw_sample_provider *provider)
 442 {
 443         struct fd_context *ctx = fd_context(pctx);
 444         int idx = pidx(provider->query_type);
 445
 446         assert((0 <= idx) && (idx < MAX_HW_SAMPLE_PROVIDERS));
 447         assert(!ctx->sample_providers[idx]);
 448
 449         ctx->sample_providers[idx] = provider;
 450 }
 451
 452 void
 453 fd_hw_query_init(struct pipe_context *pctx)
 454 {
 455         struct fd_context *ctx = fd_context(pctx);
 456
 457         slab_create(&ctx->sample_pool, sizeof(struct fd_hw_sample),
 458                         16);
 459         slab_create(&ctx->sample_period_pool, sizeof(struct fd_hw_sample_period),
 460                         16);
 461         list_inithead(&ctx->active_queries);
 462 }
 463
 464 void
 465 fd_hw_query_fini(struct pipe_context *pctx)
 466 {
 467         struct fd_context *ctx = fd_context(pctx);
 468
 469         slab_destroy(&ctx->sample_pool);
 470         slab_destroy(&ctx->sample_period_pool);
 471 }