src/gallium/drivers/freedreno/freedreno_query_hw.c

   1 /* -*- mode: C; c-file-style: "k&r"; tab-width 4; indent-tabs-mode: t; -*- */
   2
   3 /*
   4  * Copyright (C) 2014 Rob Clark <robclark@freedesktop.org>
   5  *
   6  * Permission is hereby granted, free of charge, to any person obtaining a
   7  * copy of this software and associated documentation files (the "Software"),
   8  * to deal in the Software without restriction, including without limitation
   9  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  10  * and/or sell copies of the Software, and to permit persons to whom the
  11  * Software is furnished to do so, subject to the following conditions:
  12  *
  13  * The above copyright notice and this permission notice (including the next
  14  * paragraph) shall be included in all copies or substantial portions of the
  15  * Software.
  16  *
  17  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  18  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  19  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  20  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  21  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  22  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  23  * SOFTWARE.
  24  *
  25  * Authors:
  26  *    Rob Clark <robclark@freedesktop.org>
  27  */
  28
  29 #include "pipe/p_state.h"
  30 #include "util/u_memory.h"
  31 #include "util/u_inlines.h"
  32
  33 #include "freedreno_query_hw.h"
  34 #include "freedreno_context.h"
  35 #include "freedreno_util.h"
  36
  37 struct fd_hw_sample_period {
  38         struct fd_hw_sample *start, *end;
  39         struct list_head list;
  40 };
  41
  42 /* maps query_type to sample provider idx: */
  43 static int pidx(unsigned query_type)
  44 {
  45         switch (query_type) {
  46         case PIPE_QUERY_OCCLUSION_COUNTER:
  47                 return 0;
  48         case PIPE_QUERY_OCCLUSION_PREDICATE:
  49                 return 1;
  50         case PIPE_QUERY_TIME_ELAPSED:
  51                 return 2;
  52         default:
  53                 return -1;
  54         }
  55 }
  56
  57 static struct fd_hw_sample *
  58 get_sample(struct fd_context *ctx, struct fd_ringbuffer *ring,
  59                 unsigned query_type)
  60 {
  61         struct fd_hw_sample *samp = NULL;
  62         int idx = pidx(query_type);
  63
  64         assume(idx >= 0);   /* query never would have been created otherwise */
  65
  66         if (!ctx->sample_cache[idx]) {
  67                 ctx->sample_cache[idx] =
  68                         ctx->sample_providers[idx]->get_sample(ctx, ring);
  69         }
  70
  71         fd_hw_sample_reference(ctx, &samp, ctx->sample_cache[idx]);
  72
  73         return samp;
  74 }
  75
  76 static void
  77 clear_sample_cache(struct fd_context *ctx)
  78 {
  79         int i;
  80
  81         for (i = 0; i < ARRAY_SIZE(ctx->sample_cache); i++)
  82                 fd_hw_sample_reference(ctx, &ctx->sample_cache[i], NULL);
  83 }
  84
  85 static bool
  86 is_active(struct fd_hw_query *hq, enum fd_render_stage stage)
  87 {
  88         return !!(hq->provider->active & stage);
  89 }
  90
  91
  92 static void
  93 resume_query(struct fd_context *ctx, struct fd_hw_query *hq,
  94                 struct fd_ringbuffer *ring)
  95 {
  96         int idx = pidx(hq->provider->query_type);
  97         assert(idx >= 0);   /* query never would have been created otherwise */
  98         assert(!hq->period);
  99         ctx->active_providers |= (1 << idx);
 100         hq->period = util_slab_alloc(&ctx->sample_period_pool);
 101         list_inithead(&hq->period->list);
 102         hq->period->start = get_sample(ctx, ring, hq->base.type);
 103         /* NOTE: util_slab_alloc() does not zero out the buffer: */
 104         hq->period->end = NULL;
 105 }
 106
 107 static void
 108 pause_query(struct fd_context *ctx, struct fd_hw_query *hq,
 109                 struct fd_ringbuffer *ring)
 110 {
 111         int idx = pidx(hq->provider->query_type);
 112         assert(idx >= 0);   /* query never would have been created otherwise */
 113         assert(hq->period && !hq->period->end);
 114         assert(ctx->active_providers & (1 << idx));
 115         hq->period->end = get_sample(ctx, ring, hq->base.type);
 116         list_addtail(&hq->period->list, &hq->current_periods);
 117         hq->period = NULL;
 118 }
 119
 120 static void
 121 destroy_periods(struct fd_context *ctx, struct list_head *list)
 122 {
 123         struct fd_hw_sample_period *period, *s;
 124         LIST_FOR_EACH_ENTRY_SAFE(period, s, list, list) {
 125                 fd_hw_sample_reference(ctx, &period->start, NULL);
 126                 fd_hw_sample_reference(ctx, &period->end, NULL);
 127                 list_del(&period->list);
 128                 util_slab_free(&ctx->sample_period_pool, period);
 129         }
 130 }
 131
 132 static void
 133 fd_hw_destroy_query(struct fd_context *ctx, struct fd_query *q)
 134 {
 135         struct fd_hw_query *hq = fd_hw_query(q);
 136
 137         destroy_periods(ctx, &hq->periods);
 138         destroy_periods(ctx, &hq->current_periods);
 139         list_del(&hq->list);
 140
 141         free(hq);
 142 }
 143
 144 static boolean
 145 fd_hw_begin_query(struct fd_context *ctx, struct fd_query *q)
 146 {
 147         struct fd_hw_query *hq = fd_hw_query(q);
 148         if (q->active)
 149                 return false;
 150
 151         /* begin_query() should clear previous results: */
 152         destroy_periods(ctx, &hq->periods);
 153
 154         if (is_active(hq, ctx->stage))
 155                 resume_query(ctx, hq, ctx->ring);
 156
 157         q->active = true;
 158
 159         /* add to active list: */
 160         list_del(&hq->list);
 161         list_addtail(&hq->list, &ctx->active_queries);
 162    return true;
 163 }
 164
 165 static void
 166 fd_hw_end_query(struct fd_context *ctx, struct fd_query *q)
 167 {
 168         struct fd_hw_query *hq = fd_hw_query(q);
 169         /* there are a couple special cases, which don't have
 170          * a matching ->begin_query():
 171          */
 172         if (skip_begin_query(q->type) && !q->active) {
 173                 fd_hw_begin_query(ctx, q);
 174         }
 175         if (!q->active)
 176                 return;
 177         if (is_active(hq, ctx->stage))
 178                 pause_query(ctx, hq, ctx->ring);
 179         q->active = false;
 180         /* move to current list: */
 181         list_del(&hq->list);
 182         list_addtail(&hq->list, &ctx->current_queries);
 183 }
 184
 185 /* helper to get ptr to specified sample: */
 186 static void * sampptr(struct fd_hw_sample *samp, uint32_t n, void *ptr)
 187 {
 188         return ((char *)ptr) + (samp->tile_stride * n) + samp->offset;
 189 }
 190
 191 static boolean
 192 fd_hw_get_query_result(struct fd_context *ctx, struct fd_query *q,
 193                 boolean wait, union pipe_query_result *result)
 194 {
 195         struct fd_hw_query *hq = fd_hw_query(q);
 196         const struct fd_hw_sample_provider *p = hq->provider;
 197         struct fd_hw_sample_period *period;
 198
 199         if (q->active)
 200                 return false;
 201
 202         /* if the app tries to read back the query result before the
 203          * batch is submitted, that forces us to flush so that there
 204          * are actually results to wait for:
 205          */
 206         if (!LIST_IS_EMPTY(&hq->list)) {
 207                 /* if app didn't actually trigger any cmdstream, then
 208                  * we have nothing to do:
 209                  */
 210                 if (!ctx->needs_flush)
 211                         return true;
 212                 DBG("reading query result forces flush!");
 213                 fd_context_render(&ctx->base);
 214         }
 215
 216         util_query_clear_result(result, q->type);
 217
 218         if (LIST_IS_EMPTY(&hq->periods))
 219                 return true;
 220
 221         assert(LIST_IS_EMPTY(&hq->list));
 222         assert(LIST_IS_EMPTY(&hq->current_periods));
 223         assert(!hq->period);
 224
 225         /* if !wait, then check the last sample (the one most likely to
 226          * not be ready yet) and bail if it is not ready:
 227          */
 228         if (!wait) {
 229                 int ret;
 230
 231                 period = LIST_ENTRY(struct fd_hw_sample_period,
 232                                 hq->periods.prev, list);
 233
 234                 ret = fd_bo_cpu_prep(period->end->bo, ctx->screen->pipe,
 235                                 DRM_FREEDRENO_PREP_READ | DRM_FREEDRENO_PREP_NOSYNC);
 236                 if (ret)
 237                         return false;
 238
 239                 fd_bo_cpu_fini(period->end->bo);
 240         }
 241
 242         /* sum the result across all sample periods: */
 243         LIST_FOR_EACH_ENTRY(period, &hq->periods, list) {
 244                 struct fd_hw_sample *start = period->start;
 245                 struct fd_hw_sample *end = period->end;
 246                 unsigned i;
 247
 248                 /* start and end samples should be from same batch: */
 249                 assert(start->bo == end->bo);
 250                 assert(start->num_tiles == end->num_tiles);
 251
 252                 for (i = 0; i < start->num_tiles; i++) {
 253                         void *ptr;
 254
 255                         fd_bo_cpu_prep(start->bo, ctx->screen->pipe,
 256                                         DRM_FREEDRENO_PREP_READ);
 257
 258                         ptr = fd_bo_map(start->bo);
 259
 260                         p->accumulate_result(ctx, sampptr(period->start, i, ptr),
 261                                         sampptr(period->end, i, ptr), result);
 262
 263                         fd_bo_cpu_fini(start->bo);
 264                 }
 265         }
 266
 267         return true;
 268 }
 269
 270 static const struct fd_query_funcs hw_query_funcs = {
 271                 .destroy_query    = fd_hw_destroy_query,
 272                 .begin_query      = fd_hw_begin_query,
 273                 .end_query        = fd_hw_end_query,
 274                 .get_query_result = fd_hw_get_query_result,
 275 };
 276
 277 struct fd_query *
 278 fd_hw_create_query(struct fd_context *ctx, unsigned query_type)
 279 {
 280         struct fd_hw_query *hq;
 281         struct fd_query *q;
 282         int idx = pidx(query_type);
 283
 284         if ((idx < 0) || !ctx->sample_providers[idx])
 285                 return NULL;
 286
 287         hq = CALLOC_STRUCT(fd_hw_query);
 288         if (!hq)
 289                 return NULL;
 290
 291         hq->provider = ctx->sample_providers[idx];
 292
 293         list_inithead(&hq->periods);
 294         list_inithead(&hq->current_periods);
 295         list_inithead(&hq->list);
 296
 297         q = &hq->base;
 298         q->funcs = &hw_query_funcs;
 299         q->type = query_type;
 300
 301         return q;
 302 }
 303
 304 struct fd_hw_sample *
 305 fd_hw_sample_init(struct fd_context *ctx, uint32_t size)
 306 {
 307         struct fd_hw_sample *samp = util_slab_alloc(&ctx->sample_pool);
 308         pipe_reference_init(&samp->reference, 1);
 309         samp->size = size;
 310         debug_assert(util_is_power_of_two(size));
 311         ctx->next_sample_offset = align(ctx->next_sample_offset, size);
 312         samp->offset = ctx->next_sample_offset;
 313         /* NOTE: util_slab_alloc() does not zero out the buffer: */
 314         samp->bo = NULL;
 315         samp->num_tiles = 0;
 316         samp->tile_stride = 0;
 317         ctx->next_sample_offset += size;
 318         return samp;
 319 }
 320
 321 void
 322 __fd_hw_sample_destroy(struct fd_context *ctx, struct fd_hw_sample *samp)
 323 {
 324         if (samp->bo)
 325                 fd_bo_del(samp->bo);
 326         util_slab_free(&ctx->sample_pool, samp);
 327 }
 328
 329 static void
 330 prepare_sample(struct fd_hw_sample *samp, struct fd_bo *bo,
 331                 uint32_t num_tiles, uint32_t tile_stride)
 332 {
 333         if (samp->bo) {
 334                 assert(samp->bo == bo);
 335                 assert(samp->num_tiles == num_tiles);
 336                 assert(samp->tile_stride == tile_stride);
 337                 return;
 338         }
 339         samp->bo = fd_bo_ref(bo);
 340         samp->num_tiles = num_tiles;
 341         samp->tile_stride = tile_stride;
 342 }
 343
 344 static void
 345 prepare_query(struct fd_hw_query *hq, struct fd_bo *bo,
 346                 uint32_t num_tiles, uint32_t tile_stride)
 347 {
 348         struct fd_hw_sample_period *period, *s;
 349
 350         /* prepare all the samples in the query: */
 351         LIST_FOR_EACH_ENTRY_SAFE(period, s, &hq->current_periods, list) {
 352                 prepare_sample(period->start, bo, num_tiles, tile_stride);
 353                 prepare_sample(period->end, bo, num_tiles, tile_stride);
 354
 355                 /* move from current_periods list to periods list: */
 356                 list_del(&period->list);
 357                 list_addtail(&period->list, &hq->periods);
 358         }
 359 }
 360
 361 static void
 362 prepare_queries(struct fd_context *ctx, struct fd_bo *bo,
 363                 uint32_t num_tiles, uint32_t tile_stride,
 364                 struct list_head *list, bool remove)
 365 {
 366         struct fd_hw_query *hq, *s;
 367         LIST_FOR_EACH_ENTRY_SAFE(hq, s, list, list) {
 368                 prepare_query(hq, bo, num_tiles, tile_stride);
 369                 if (remove)
 370                         list_delinit(&hq->list);
 371         }
 372 }
 373
 374 /* called from gmem code once total storage requirements are known (ie.
 375  * number of samples times number of tiles)
 376  */
 377 void
 378 fd_hw_query_prepare(struct fd_context *ctx, uint32_t num_tiles)
 379 {
 380         uint32_t tile_stride = ctx->next_sample_offset;
 381         struct fd_bo *bo;
 382
 383         if (ctx->query_bo)
 384                 fd_bo_del(ctx->query_bo);
 385
 386         if (tile_stride > 0) {
 387                 bo = fd_bo_new(ctx->dev, tile_stride * num_tiles,
 388                                 DRM_FREEDRENO_GEM_CACHE_WCOMBINE |
 389                                 DRM_FREEDRENO_GEM_TYPE_KMEM);
 390         } else {
 391                 bo = NULL;
 392         }
 393
 394         ctx->query_bo = bo;
 395         ctx->query_tile_stride = tile_stride;
 396
 397         prepare_queries(ctx, bo, num_tiles, tile_stride,
 398                         &ctx->active_queries, false);
 399         prepare_queries(ctx, bo, num_tiles, tile_stride,
 400                         &ctx->current_queries, true);
 401
 402         /* reset things for next batch: */
 403         ctx->next_sample_offset = 0;
 404 }
 405
 406 void
 407 fd_hw_query_prepare_tile(struct fd_context *ctx, uint32_t n,
 408                 struct fd_ringbuffer *ring)
 409 {
 410         uint32_t tile_stride = ctx->query_tile_stride;
 411         uint32_t offset = tile_stride * n;
 412
 413         /* bail if no queries: */
 414         if (tile_stride == 0)
 415                 return;
 416
 417         fd_wfi(ctx, ring);
 418         OUT_PKT0 (ring, HW_QUERY_BASE_REG, 1);
 419         OUT_RELOCW(ring, ctx->query_bo, offset, 0, 0);
 420 }
 421
 422 void
 423 fd_hw_query_set_stage(struct fd_context *ctx, struct fd_ringbuffer *ring,
 424                 enum fd_render_stage stage)
 425 {
 426         /* special case: internal blits (like mipmap level generation)
 427          * go through normal draw path (via util_blitter_blit()).. but
 428          * we need to ignore the FD_STAGE_DRAW which will be set, so we
 429          * don't enable queries which should be paused during internal
 430          * blits:
 431          */
 432         if ((ctx->stage == FD_STAGE_BLIT) &&
 433                         (stage != FD_STAGE_NULL))
 434                 return;
 435
 436         if (stage != ctx->stage) {
 437                 struct fd_hw_query *hq;
 438                 LIST_FOR_EACH_ENTRY(hq, &ctx->active_queries, list) {
 439                         bool was_active = is_active(hq, ctx->stage);
 440                         bool now_active = is_active(hq, stage);
 441
 442                         if (now_active && !was_active)
 443                                 resume_query(ctx, hq, ring);
 444                         else if (was_active && !now_active)
 445                                 pause_query(ctx, hq, ring);
 446                 }
 447         }
 448         clear_sample_cache(ctx);
 449         ctx->stage = stage;
 450 }
 451
 452 /* call the provider->enable() for all the hw queries that were active
 453  * in the current batch.  This sets up perfctr selector regs statically
 454  * for the duration of the batch.
 455  */
 456 void
 457 fd_hw_query_enable(struct fd_context *ctx, struct fd_ringbuffer *ring)
 458 {
 459         for (int idx = 0; idx < MAX_HW_SAMPLE_PROVIDERS; idx++) {
 460                 if (ctx->active_providers & (1 << idx)) {
 461                         assert(ctx->sample_providers[idx]);
 462                         if (ctx->sample_providers[idx]->enable)
 463                                 ctx->sample_providers[idx]->enable(ctx, ring);
 464                 }
 465         }
 466         ctx->active_providers = 0;  /* clear it for next frame */
 467 }
 468
 469 void
 470 fd_hw_query_register_provider(struct pipe_context *pctx,
 471                 const struct fd_hw_sample_provider *provider)
 472 {
 473         struct fd_context *ctx = fd_context(pctx);
 474         int idx = pidx(provider->query_type);
 475
 476         assert((0 <= idx) && (idx < MAX_HW_SAMPLE_PROVIDERS));
 477         assert(!ctx->sample_providers[idx]);
 478
 479         ctx->sample_providers[idx] = provider;
 480 }
 481
 482 void
 483 fd_hw_query_init(struct pipe_context *pctx)
 484 {
 485         struct fd_context *ctx = fd_context(pctx);
 486
 487         util_slab_create(&ctx->sample_pool, sizeof(struct fd_hw_sample),
 488                         16, UTIL_SLAB_SINGLETHREADED);
 489         util_slab_create(&ctx->sample_period_pool, sizeof(struct fd_hw_sample_period),
 490                         16, UTIL_SLAB_SINGLETHREADED);
 491         list_inithead(&ctx->active_queries);
 492         list_inithead(&ctx->current_queries);
 493 }
 494
 495 void
 496 fd_hw_query_fini(struct pipe_context *pctx)
 497 {
 498         struct fd_context *ctx = fd_context(pctx);
 499
 500         util_slab_destroy(&ctx->sample_pool);
 501         util_slab_destroy(&ctx->sample_period_pool);
 502 }