src/gallium/drivers/freedreno/freedreno_query_hw.c

   1 /* -*- mode: C; c-file-style: "k&r"; tab-width 4; indent-tabs-mode: t; -*- */
   2
   3 /*
   4  * Copyright (C) 2014 Rob Clark <robclark@freedesktop.org>
   5  *
   6  * Permission is hereby granted, free of charge, to any person obtaining a
   7  * copy of this software and associated documentation files (the "Software"),
   8  * to deal in the Software without restriction, including without limitation
   9  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  10  * and/or sell copies of the Software, and to permit persons to whom the
  11  * Software is furnished to do so, subject to the following conditions:
  12  *
  13  * The above copyright notice and this permission notice (including the next
  14  * paragraph) shall be included in all copies or substantial portions of the
  15  * Software.
  16  *
  17  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  18  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  19  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  20  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  21  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  22  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  23  * SOFTWARE.
  24  *
  25  * Authors:
  26  *    Rob Clark <robclark@freedesktop.org>
  27  */
  28
  29 #include "pipe/p_state.h"
  30 #include "util/u_memory.h"
  31 #include "util/u_inlines.h"
  32
  33 #include "freedreno_query_hw.h"
  34 #include "freedreno_context.h"
  35 #include "freedreno_util.h"
  36
  37 struct fd_hw_sample_period {
  38         struct fd_hw_sample *start, *end;
  39         struct list_head list;
  40 };
  41
  42 /* maps query_type to sample provider idx: */
  43 static int pidx(unsigned query_type)
  44 {
  45         switch (query_type) {
  46         case PIPE_QUERY_OCCLUSION_COUNTER:
  47                 return 0;
  48         case PIPE_QUERY_OCCLUSION_PREDICATE:
  49                 return 1;
  50         case PIPE_QUERY_TIME_ELAPSED:
  51                 return 2;
  52         default:
  53                 return -1;
  54         }
  55 }
  56
  57 static struct fd_hw_sample *
  58 get_sample(struct fd_context *ctx, struct fd_ringbuffer *ring,
  59                 unsigned query_type)
  60 {
  61         struct fd_hw_sample *samp = NULL;
  62         int idx = pidx(query_type);
  63
  64         if (!ctx->sample_cache[idx]) {
  65                 ctx->sample_cache[idx] =
  66                         ctx->sample_providers[idx]->get_sample(ctx, ring);
  67         }
  68
  69         fd_hw_sample_reference(ctx, &samp, ctx->sample_cache[idx]);
  70
  71         return samp;
  72 }
  73
  74 static void
  75 clear_sample_cache(struct fd_context *ctx)
  76 {
  77         int i;
  78
  79         for (i = 0; i < ARRAY_SIZE(ctx->sample_cache); i++)
  80                 fd_hw_sample_reference(ctx, &ctx->sample_cache[i], NULL);
  81 }
  82
  83 static bool
  84 is_active(struct fd_hw_query *hq, enum fd_render_stage stage)
  85 {
  86         return !!(hq->provider->active & stage);
  87 }
  88
  89
  90 static void
  91 resume_query(struct fd_context *ctx, struct fd_hw_query *hq,
  92                 struct fd_ringbuffer *ring)
  93 {
  94         int idx = pidx(hq->provider->query_type);
  95         assert(!hq->period);
  96         ctx->active_providers |= (1 << idx);
  97         hq->period = util_slab_alloc(&ctx->sample_period_pool);
  98         list_inithead(&hq->period->list);
  99         hq->period->start = get_sample(ctx, ring, hq->base.type);
 100         /* NOTE: util_slab_alloc() does not zero out the buffer: */
 101         hq->period->end = NULL;
 102 }
 103
 104 static void
 105 pause_query(struct fd_context *ctx, struct fd_hw_query *hq,
 106                 struct fd_ringbuffer *ring)
 107 {
 108         int idx = pidx(hq->provider->query_type);
 109         assert(hq->period && !hq->period->end);
 110         assert(ctx->active_providers & (1 << idx));
 111         hq->period->end = get_sample(ctx, ring, hq->base.type);
 112         list_addtail(&hq->period->list, &hq->current_periods);
 113         hq->period = NULL;
 114 }
 115
 116 static void
 117 destroy_periods(struct fd_context *ctx, struct list_head *list)
 118 {
 119         struct fd_hw_sample_period *period, *s;
 120         LIST_FOR_EACH_ENTRY_SAFE(period, s, list, list) {
 121                 fd_hw_sample_reference(ctx, &period->start, NULL);
 122                 fd_hw_sample_reference(ctx, &period->end, NULL);
 123                 list_del(&period->list);
 124                 util_slab_free(&ctx->sample_period_pool, period);
 125         }
 126 }
 127
 128 static void
 129 fd_hw_destroy_query(struct fd_context *ctx, struct fd_query *q)
 130 {
 131         struct fd_hw_query *hq = fd_hw_query(q);
 132
 133         destroy_periods(ctx, &hq->periods);
 134         destroy_periods(ctx, &hq->current_periods);
 135         list_del(&hq->list);
 136
 137         free(hq);
 138 }
 139
 140 static boolean
 141 fd_hw_begin_query(struct fd_context *ctx, struct fd_query *q)
 142 {
 143         struct fd_hw_query *hq = fd_hw_query(q);
 144         if (q->active)
 145                 return false;
 146
 147         /* begin_query() should clear previous results: */
 148         destroy_periods(ctx, &hq->periods);
 149
 150         if (is_active(hq, ctx->stage))
 151                 resume_query(ctx, hq, ctx->ring);
 152
 153         q->active = true;
 154
 155         /* add to active list: */
 156         list_del(&hq->list);
 157         list_addtail(&hq->list, &ctx->active_queries);
 158    return true;
 159 }
 160
 161 static void
 162 fd_hw_end_query(struct fd_context *ctx, struct fd_query *q)
 163 {
 164         struct fd_hw_query *hq = fd_hw_query(q);
 165         /* there are a couple special cases, which don't have
 166          * a matching ->begin_query():
 167          */
 168         if (skip_begin_query(q->type) && !q->active) {
 169                 fd_hw_begin_query(ctx, q);
 170         }
 171         if (!q->active)
 172                 return;
 173         if (is_active(hq, ctx->stage))
 174                 pause_query(ctx, hq, ctx->ring);
 175         q->active = false;
 176         /* move to current list: */
 177         list_del(&hq->list);
 178         list_addtail(&hq->list, &ctx->current_queries);
 179 }
 180
 181 /* helper to get ptr to specified sample: */
 182 static void * sampptr(struct fd_hw_sample *samp, uint32_t n, void *ptr)
 183 {
 184         return ((char *)ptr) + (samp->tile_stride * n) + samp->offset;
 185 }
 186
 187 static boolean
 188 fd_hw_get_query_result(struct fd_context *ctx, struct fd_query *q,
 189                 boolean wait, union pipe_query_result *result)
 190 {
 191         struct fd_hw_query *hq = fd_hw_query(q);
 192         const struct fd_hw_sample_provider *p = hq->provider;
 193         struct fd_hw_sample_period *period;
 194
 195         if (q->active)
 196                 return false;
 197
 198         /* if the app tries to read back the query result before the
 199          * batch is submitted, that forces us to flush so that there
 200          * are actually results to wait for:
 201          */
 202         if (!LIST_IS_EMPTY(&hq->list)) {
 203                 /* if app didn't actually trigger any cmdstream, then
 204                  * we have nothing to do:
 205                  */
 206                 if (!ctx->needs_flush)
 207                         return true;
 208                 DBG("reading query result forces flush!");
 209                 fd_context_render(&ctx->base);
 210         }
 211
 212         util_query_clear_result(result, q->type);
 213
 214         if (LIST_IS_EMPTY(&hq->periods))
 215                 return true;
 216
 217         assert(LIST_IS_EMPTY(&hq->list));
 218         assert(LIST_IS_EMPTY(&hq->current_periods));
 219         assert(!hq->period);
 220
 221         /* if !wait, then check the last sample (the one most likely to
 222          * not be ready yet) and bail if it is not ready:
 223          */
 224         if (!wait) {
 225                 int ret;
 226
 227                 period = LIST_ENTRY(struct fd_hw_sample_period,
 228                                 hq->periods.prev, list);
 229
 230                 ret = fd_bo_cpu_prep(period->end->bo, ctx->screen->pipe,
 231                                 DRM_FREEDRENO_PREP_READ | DRM_FREEDRENO_PREP_NOSYNC);
 232                 if (ret)
 233                         return false;
 234
 235                 fd_bo_cpu_fini(period->end->bo);
 236         }
 237
 238         /* sum the result across all sample periods: */
 239         LIST_FOR_EACH_ENTRY(period, &hq->periods, list) {
 240                 struct fd_hw_sample *start = period->start;
 241                 struct fd_hw_sample *end = period->end;
 242                 unsigned i;
 243
 244                 /* start and end samples should be from same batch: */
 245                 assert(start->bo == end->bo);
 246                 assert(start->num_tiles == end->num_tiles);
 247
 248                 for (i = 0; i < start->num_tiles; i++) {
 249                         void *ptr;
 250
 251                         fd_bo_cpu_prep(start->bo, ctx->screen->pipe,
 252                                         DRM_FREEDRENO_PREP_READ);
 253
 254                         ptr = fd_bo_map(start->bo);
 255
 256                         p->accumulate_result(ctx, sampptr(period->start, i, ptr),
 257                                         sampptr(period->end, i, ptr), result);
 258
 259                         fd_bo_cpu_fini(start->bo);
 260                 }
 261         }
 262
 263         return true;
 264 }
 265
 266 static const struct fd_query_funcs hw_query_funcs = {
 267                 .destroy_query    = fd_hw_destroy_query,
 268                 .begin_query      = fd_hw_begin_query,
 269                 .end_query        = fd_hw_end_query,
 270                 .get_query_result = fd_hw_get_query_result,
 271 };
 272
 273 struct fd_query *
 274 fd_hw_create_query(struct fd_context *ctx, unsigned query_type)
 275 {
 276         struct fd_hw_query *hq;
 277         struct fd_query *q;
 278         int idx = pidx(query_type);
 279
 280         if ((idx < 0) || !ctx->sample_providers[idx])
 281                 return NULL;
 282
 283         hq = CALLOC_STRUCT(fd_hw_query);
 284         if (!hq)
 285                 return NULL;
 286
 287         hq->provider = ctx->sample_providers[idx];
 288
 289         list_inithead(&hq->periods);
 290         list_inithead(&hq->current_periods);
 291         list_inithead(&hq->list);
 292
 293         q = &hq->base;
 294         q->funcs = &hw_query_funcs;
 295         q->type = query_type;
 296
 297         return q;
 298 }
 299
 300 struct fd_hw_sample *
 301 fd_hw_sample_init(struct fd_context *ctx, uint32_t size)
 302 {
 303         struct fd_hw_sample *samp = util_slab_alloc(&ctx->sample_pool);
 304         pipe_reference_init(&samp->reference, 1);
 305         samp->size = size;
 306         debug_assert(util_is_power_of_two(size));
 307         ctx->next_sample_offset = align(ctx->next_sample_offset, size);
 308         samp->offset = ctx->next_sample_offset;
 309         /* NOTE: util_slab_alloc() does not zero out the buffer: */
 310         samp->bo = NULL;
 311         samp->num_tiles = 0;
 312         samp->tile_stride = 0;
 313         ctx->next_sample_offset += size;
 314         return samp;
 315 }
 316
 317 void
 318 __fd_hw_sample_destroy(struct fd_context *ctx, struct fd_hw_sample *samp)
 319 {
 320         if (samp->bo)
 321                 fd_bo_del(samp->bo);
 322         util_slab_free(&ctx->sample_pool, samp);
 323 }
 324
 325 static void
 326 prepare_sample(struct fd_hw_sample *samp, struct fd_bo *bo,
 327                 uint32_t num_tiles, uint32_t tile_stride)
 328 {
 329         if (samp->bo) {
 330                 assert(samp->bo == bo);
 331                 assert(samp->num_tiles == num_tiles);
 332                 assert(samp->tile_stride == tile_stride);
 333                 return;
 334         }
 335         samp->bo = fd_bo_ref(bo);
 336         samp->num_tiles = num_tiles;
 337         samp->tile_stride = tile_stride;
 338 }
 339
 340 static void
 341 prepare_query(struct fd_hw_query *hq, struct fd_bo *bo,
 342                 uint32_t num_tiles, uint32_t tile_stride)
 343 {
 344         struct fd_hw_sample_period *period, *s;
 345
 346         /* prepare all the samples in the query: */
 347         LIST_FOR_EACH_ENTRY_SAFE(period, s, &hq->current_periods, list) {
 348                 prepare_sample(period->start, bo, num_tiles, tile_stride);
 349                 prepare_sample(period->end, bo, num_tiles, tile_stride);
 350
 351                 /* move from current_periods list to periods list: */
 352                 list_del(&period->list);
 353                 list_addtail(&period->list, &hq->periods);
 354         }
 355 }
 356
 357 static void
 358 prepare_queries(struct fd_context *ctx, struct fd_bo *bo,
 359                 uint32_t num_tiles, uint32_t tile_stride,
 360                 struct list_head *list, bool remove)
 361 {
 362         struct fd_hw_query *hq, *s;
 363         LIST_FOR_EACH_ENTRY_SAFE(hq, s, list, list) {
 364                 prepare_query(hq, bo, num_tiles, tile_stride);
 365                 if (remove)
 366                         list_delinit(&hq->list);
 367         }
 368 }
 369
 370 /* called from gmem code once total storage requirements are known (ie.
 371  * number of samples times number of tiles)
 372  */
 373 void
 374 fd_hw_query_prepare(struct fd_context *ctx, uint32_t num_tiles)
 375 {
 376         uint32_t tile_stride = ctx->next_sample_offset;
 377         struct fd_bo *bo;
 378
 379         if (ctx->query_bo)
 380                 fd_bo_del(ctx->query_bo);
 381
 382         if (tile_stride > 0) {
 383                 bo = fd_bo_new(ctx->dev, tile_stride * num_tiles,
 384                                 DRM_FREEDRENO_GEM_CACHE_WCOMBINE |
 385                                 DRM_FREEDRENO_GEM_TYPE_KMEM);
 386         } else {
 387                 bo = NULL;
 388         }
 389
 390         ctx->query_bo = bo;
 391         ctx->query_tile_stride = tile_stride;
 392
 393         prepare_queries(ctx, bo, num_tiles, tile_stride,
 394                         &ctx->active_queries, false);
 395         prepare_queries(ctx, bo, num_tiles, tile_stride,
 396                         &ctx->current_queries, true);
 397
 398         /* reset things for next batch: */
 399         ctx->next_sample_offset = 0;
 400 }
 401
 402 void
 403 fd_hw_query_prepare_tile(struct fd_context *ctx, uint32_t n,
 404                 struct fd_ringbuffer *ring)
 405 {
 406         uint32_t tile_stride = ctx->query_tile_stride;
 407         uint32_t offset = tile_stride * n;
 408
 409         /* bail if no queries: */
 410         if (tile_stride == 0)
 411                 return;
 412
 413         fd_wfi(ctx, ring);
 414         OUT_PKT0 (ring, HW_QUERY_BASE_REG, 1);
 415         OUT_RELOCW(ring, ctx->query_bo, offset, 0, 0);
 416 }
 417
 418 void
 419 fd_hw_query_set_stage(struct fd_context *ctx, struct fd_ringbuffer *ring,
 420                 enum fd_render_stage stage)
 421 {
 422         /* special case: internal blits (like mipmap level generation)
 423          * go through normal draw path (via util_blitter_blit()).. but
 424          * we need to ignore the FD_STAGE_DRAW which will be set, so we
 425          * don't enable queries which should be paused during internal
 426          * blits:
 427          */
 428         if ((ctx->stage == FD_STAGE_BLIT) &&
 429                         (stage != FD_STAGE_NULL))
 430                 return;
 431
 432         if (stage != ctx->stage) {
 433                 struct fd_hw_query *hq;
 434                 LIST_FOR_EACH_ENTRY(hq, &ctx->active_queries, list) {
 435                         bool was_active = is_active(hq, ctx->stage);
 436                         bool now_active = is_active(hq, stage);
 437
 438                         if (now_active && !was_active)
 439                                 resume_query(ctx, hq, ring);
 440                         else if (was_active && !now_active)
 441                                 pause_query(ctx, hq, ring);
 442                 }
 443         }
 444         clear_sample_cache(ctx);
 445         ctx->stage = stage;
 446 }
 447
 448 /* call the provider->enable() for all the hw queries that were active
 449  * in the current batch.  This sets up perfctr selector regs statically
 450  * for the duration of the batch.
 451  */
 452 void
 453 fd_hw_query_enable(struct fd_context *ctx, struct fd_ringbuffer *ring)
 454 {
 455         for (int idx = 0; idx < MAX_HW_SAMPLE_PROVIDERS; idx++) {
 456                 if (ctx->active_providers & (1 << idx)) {
 457                         assert(ctx->sample_providers[idx]);
 458                         if (ctx->sample_providers[idx]->enable)
 459                                 ctx->sample_providers[idx]->enable(ctx, ring);
 460                 }
 461         }
 462         ctx->active_providers = 0;  /* clear it for next frame */
 463 }
 464
 465 void
 466 fd_hw_query_register_provider(struct pipe_context *pctx,
 467                 const struct fd_hw_sample_provider *provider)
 468 {
 469         struct fd_context *ctx = fd_context(pctx);
 470         int idx = pidx(provider->query_type);
 471
 472         assert((0 <= idx) && (idx < MAX_HW_SAMPLE_PROVIDERS));
 473         assert(!ctx->sample_providers[idx]);
 474
 475         ctx->sample_providers[idx] = provider;
 476 }
 477
 478 void
 479 fd_hw_query_init(struct pipe_context *pctx)
 480 {
 481         struct fd_context *ctx = fd_context(pctx);
 482
 483         util_slab_create(&ctx->sample_pool, sizeof(struct fd_hw_sample),
 484                         16, UTIL_SLAB_SINGLETHREADED);
 485         util_slab_create(&ctx->sample_period_pool, sizeof(struct fd_hw_sample_period),
 486                         16, UTIL_SLAB_SINGLETHREADED);
 487         list_inithead(&ctx->active_queries);
 488         list_inithead(&ctx->current_queries);
 489 }
 490
 491 void
 492 fd_hw_query_fini(struct pipe_context *pctx)
 493 {
 494         struct fd_context *ctx = fd_context(pctx);
 495
 496         util_slab_destroy(&ctx->sample_pool);
 497         util_slab_destroy(&ctx->sample_period_pool);
 498 }