src/gallium/drivers/zink/zink_query.c

   1 #include "zink_query.h"
   2
   3 #include "zink_context.h"
   4 #include "zink_fence.h"
   5 #include "zink_resource.h"
   6 #include "zink_screen.h"
   7
   8 #include "util/hash_table.h"
   9 #include "util/set.h"
  10 #include "util/u_dump.h"
  11 #include "util/u_inlines.h"
  12 #include "util/u_memory.h"
  13
  14 #define NUM_QUERIES 50
  15
  16 struct zink_query {
  17    enum pipe_query_type type;
  18
  19    VkQueryPool query_pool;
  20    unsigned last_checked_query, curr_query, num_queries;
  21
  22    VkQueryType vkqtype;
  23    unsigned index;
  24    bool use_64bit;
  25    bool precise;
  26    bool xfb_running;
  27
  28    bool active; /* query is considered active by vk */
  29    bool needs_reset; /* query is considered active by vk and cannot be destroyed */
  30    bool dead; /* query should be destroyed when its fence finishes */
  31
  32    unsigned fences;
  33    struct list_head active_list;
  34
  35    union pipe_query_result accumulated_result;
  36 };
  37
  38 static VkQueryType
  39 convert_query_type(unsigned query_type, bool *use_64bit, bool *precise)
  40 {
  41    *use_64bit = false;
  42    *precise = false;
  43    switch (query_type) {
  44    case PIPE_QUERY_OCCLUSION_COUNTER:
  45       *precise = true;
  46       *use_64bit = true;
  47       /* fallthrough */
  48    case PIPE_QUERY_OCCLUSION_PREDICATE:
  49    case PIPE_QUERY_OCCLUSION_PREDICATE_CONSERVATIVE:
  50       return VK_QUERY_TYPE_OCCLUSION;
  51    case PIPE_QUERY_TIMESTAMP:
  52       *use_64bit = true;
  53       return VK_QUERY_TYPE_TIMESTAMP;
  54    case PIPE_QUERY_PIPELINE_STATISTICS:
  55    case PIPE_QUERY_PRIMITIVES_GENERATED:
  56       return VK_QUERY_TYPE_PIPELINE_STATISTICS;
  57    case PIPE_QUERY_PRIMITIVES_EMITTED:
  58       *use_64bit = true;
  59       return VK_QUERY_TYPE_TRANSFORM_FEEDBACK_STREAM_EXT;
  60    default:
  61       debug_printf("unknown query: %s\n",
  62                    util_str_query_type(query_type, true));
  63       unreachable("zink: unknown query type");
  64    }
  65 }
  66
  67 static struct pipe_query *
  68 zink_create_query(struct pipe_context *pctx,
  69                   unsigned query_type, unsigned index)
  70 {
  71    struct zink_screen *screen = zink_screen(pctx->screen);
  72    struct zink_query *query = CALLOC_STRUCT(zink_query);
  73    VkQueryPoolCreateInfo pool_create = {};
  74
  75    if (!query)
  76       return NULL;
  77
  78    query->index = index;
  79    query->type = query_type;
  80    query->vkqtype = convert_query_type(query_type, &query->use_64bit, &query->precise);
  81    if (query->vkqtype == -1)
  82       return NULL;
  83
  84    query->num_queries = query_type == PIPE_QUERY_TIMESTAMP ? 1 : NUM_QUERIES;
  85    query->curr_query = 0;
  86
  87    pool_create.sType = VK_STRUCTURE_TYPE_QUERY_POOL_CREATE_INFO;
  88    pool_create.queryType = query->vkqtype;
  89    pool_create.queryCount = query->num_queries;
  90    if (query_type == PIPE_QUERY_PRIMITIVES_GENERATED)
  91      pool_create.pipelineStatistics = VK_QUERY_PIPELINE_STATISTIC_CLIPPING_INVOCATIONS_BIT;
  92
  93    VkResult status = vkCreateQueryPool(screen->dev, &pool_create, NULL, &query->query_pool);
  94    if (status != VK_SUCCESS) {
  95       FREE(query);
  96       return NULL;
  97    }
  98    struct zink_batch *batch = zink_batch_no_rp(zink_context(pctx));
  99    vkCmdResetQueryPool(batch->cmdbuf, query->query_pool, 0, query->num_queries);
 100    return (struct pipe_query *)query;
 101 }
 102
 103 static void
 104 wait_query(struct pipe_context *pctx, struct zink_query *query)
 105 {
 106    struct pipe_fence_handle *fence = NULL;
 107
 108    pctx->flush(pctx, &fence, PIPE_FLUSH_HINT_FINISH);
 109    if (fence) {
 110       pctx->screen->fence_finish(pctx->screen, NULL, fence,
 111                                  PIPE_TIMEOUT_INFINITE);
 112       pctx->screen->fence_reference(pctx->screen, &fence, NULL);
 113    }
 114 }
 115
 116 static void
 117 destroy_query(struct zink_screen *screen, struct zink_query *query)
 118 {
 119    assert(!p_atomic_read(&query->fences));
 120    vkDestroyQueryPool(screen->dev, query->query_pool, NULL);
 121    FREE(query);
 122 }
 123
 124 static void
 125 zink_destroy_query(struct pipe_context *pctx,
 126                    struct pipe_query *q)
 127 {
 128    struct zink_screen *screen = zink_screen(pctx->screen);
 129    struct zink_query *query = (struct zink_query *)q;
 130
 131    p_atomic_set(&query->dead, true);
 132    if (p_atomic_read(&query->fences)) {
 133       if (query->xfb_running)
 134         wait_query(pctx, query);
 135       return;
 136    }
 137
 138    destroy_query(screen, query);
 139 }
 140
 141 void
 142 zink_prune_queries(struct zink_screen *screen, struct zink_fence *fence)
 143 {
 144    set_foreach(fence->active_queries, entry) {
 145       struct zink_query *query = (void*)entry->key;
 146       if (!p_atomic_dec_return(&query->fences)) {
 147          if (p_atomic_read(&query->dead))
 148             destroy_query(screen, query);
 149       }
 150    }
 151    _mesa_set_destroy(fence->active_queries, NULL);
 152    fence->active_queries = NULL;
 153 }
 154
 155 static bool
 156 get_query_result(struct pipe_context *pctx,
 157                       struct pipe_query *q,
 158                       bool wait,
 159                       union pipe_query_result *result)
 160 {
 161    struct zink_screen *screen = zink_screen(pctx->screen);
 162    struct zink_query *query = (struct zink_query *)q;
 163    VkQueryResultFlagBits flags = 0;
 164
 165    if (wait)
 166       flags |= VK_QUERY_RESULT_WAIT_BIT;
 167
 168    if (query->use_64bit)
 169       flags |= VK_QUERY_RESULT_64_BIT;
 170
 171    if (result != &query->accumulated_result) {
 172       memcpy(result, &query->accumulated_result, sizeof(query->accumulated_result));
 173       util_query_clear_result(&query->accumulated_result, query->type);
 174    } else {
 175       assert(query->vkqtype != VK_QUERY_TYPE_TIMESTAMP);
 176       flags |= VK_QUERY_RESULT_PARTIAL_BIT;
 177    }
 178
 179    // union pipe_query_result results[NUM_QUERIES * 2];
 180    /* xfb queries return 2 results */
 181    uint64_t results[NUM_QUERIES * 2];
 182    memset(results, 0, sizeof(results));
 183    int num_results = query->curr_query - query->last_checked_query;
 184    if (query->vkqtype == VK_QUERY_TYPE_TRANSFORM_FEEDBACK_STREAM_EXT) {
 185       /* this query emits 2 values */
 186       assert(query->curr_query <= ARRAY_SIZE(results) / 2);
 187       VkResult status = vkGetQueryPoolResults(screen->dev, query->query_pool,
 188                                               query->last_checked_query, num_results,
 189                                               sizeof(results),
 190                                               results,
 191                                               sizeof(uint64_t),
 192                                               flags);
 193       if (status != VK_SUCCESS)
 194          return false;
 195       /* multiply for correct looping behavior below */
 196       num_results *= 2;
 197    } else {
 198       assert(query->curr_query <= ARRAY_SIZE(results));
 199       VkResult status = vkGetQueryPoolResults(screen->dev, query->query_pool,
 200                                               query->last_checked_query, num_results,
 201                                               sizeof(results),
 202                                               results,
 203                                               sizeof(uint64_t),
 204                                               flags);
 205       if (status != VK_SUCCESS)
 206          return false;
 207    }
 208
 209    for (int i = 0; i < num_results; ++i) {
 210       switch (query->type) {
 211       case PIPE_QUERY_OCCLUSION_PREDICATE:
 212       case PIPE_QUERY_OCCLUSION_PREDICATE_CONSERVATIVE:
 213       case PIPE_QUERY_SO_OVERFLOW_PREDICATE:
 214       case PIPE_QUERY_SO_OVERFLOW_ANY_PREDICATE:
 215       case PIPE_QUERY_GPU_FINISHED:
 216          result->b |= results[i] != 0;
 217          break;
 218
 219       case PIPE_QUERY_OCCLUSION_COUNTER:
 220          result->u64 += results[i];
 221          break;
 222       case PIPE_QUERY_PRIMITIVES_GENERATED:
 223          result->u32 += results[i];
 224          break;
 225       case PIPE_QUERY_PRIMITIVES_EMITTED:
 226          /* A query pool created with this type will capture 2 integers -
 227           * numPrimitivesWritten and numPrimitivesNeeded -
 228           * for the specified vertex stream output from the last vertex processing stage.
 229           * - from VK_EXT_transform_feedback spec
 230           */
 231          result->u64 += results[i];
 232          i++;
 233          break;
 234
 235       default:
 236          debug_printf("unhangled query type: %s\n",
 237                       util_str_query_type(query->type, true));
 238          unreachable("unexpected query type");
 239       }
 240    }
 241    query->last_checked_query = query->curr_query;
 242
 243    return TRUE;
 244 }
 245
 246 static void
 247 reset_pool(struct zink_context *ctx, struct zink_batch *batch, struct zink_query *q)
 248 {
 249    /* This command must only be called outside of a render pass instance
 250     *
 251     * - vkCmdResetQueryPool spec
 252     */
 253    batch = zink_batch_no_rp(ctx);
 254
 255    get_query_result(&ctx->base, (struct pipe_query*)q, false, &q->accumulated_result);
 256    vkCmdResetQueryPool(batch->cmdbuf, q->query_pool, 0, q->num_queries);
 257    q->last_checked_query = q->curr_query = 0;
 258    q->needs_reset = false;
 259 }
 260
 261 static void
 262 begin_query(struct zink_context *ctx, struct zink_batch *batch, struct zink_query *q)
 263 {
 264    VkQueryControlFlags flags = 0;
 265
 266    if (q->needs_reset)
 267       reset_pool(ctx, batch, q);
 268    assert(q->curr_query < q->num_queries);
 269    if (q->precise)
 270       flags |= VK_QUERY_CONTROL_PRECISE_BIT;
 271    if (q->vkqtype == VK_QUERY_TYPE_TRANSFORM_FEEDBACK_STREAM_EXT) {
 272       zink_screen(ctx->base.screen)->vk_CmdBeginQueryIndexedEXT(batch->cmdbuf,
 273                                                                 q->query_pool,
 274                                                                 q->curr_query,
 275                                                                 flags,
 276                                                                 q->index);
 277       q->xfb_running = true;
 278    } else
 279       vkCmdBeginQuery(batch->cmdbuf, q->query_pool, q->curr_query, flags);
 280    q->active = true;
 281    if (!batch->active_queries)
 282       batch->active_queries = _mesa_set_create(NULL, _mesa_hash_pointer, _mesa_key_pointer_equal);
 283    assert(batch->active_queries);
 284    p_atomic_inc(&q->fences);
 285    _mesa_set_add(batch->active_queries, q);
 286 }
 287
 288 static bool
 289 zink_begin_query(struct pipe_context *pctx,
 290                  struct pipe_query *q)
 291 {
 292    struct zink_query *query = (struct zink_query *)q;
 293    struct zink_batch *batch = zink_curr_batch(zink_context(pctx));
 294
 295    /* ignore begin_query for timestamps */
 296    if (query->type == PIPE_QUERY_TIMESTAMP)
 297       return true;
 298    util_query_clear_result(&query->accumulated_result, query->type);
 299
 300    begin_query(zink_context(pctx), batch, query);
 301
 302    return true;
 303 }
 304
 305 static void
 306 end_query(struct zink_context *ctx, struct zink_batch *batch, struct zink_query *q)
 307 {
 308    struct zink_screen *screen = zink_screen(ctx->base.screen);
 309    assert(q->type != PIPE_QUERY_TIMESTAMP);
 310    q->active = false;
 311    if (q->vkqtype == VK_QUERY_TYPE_TRANSFORM_FEEDBACK_STREAM_EXT)
 312       screen->vk_CmdEndQueryIndexedEXT(batch->cmdbuf, q->query_pool, q->curr_query, q->index);
 313    else
 314       vkCmdEndQuery(batch->cmdbuf, q->query_pool, q->curr_query);
 315    if (++q->curr_query == q->num_queries) {
 316       /* can't do zink_batch_no_rp here because we might already be inside a zink_batch_no_rp */
 317       if (batch->rp)
 318          q->needs_reset = true;
 319       else
 320         reset_pool(ctx, batch, q);
 321    }
 322 }
 323
 324 static bool
 325 zink_end_query(struct pipe_context *pctx,
 326                struct pipe_query *q)
 327 {
 328    struct zink_context *ctx = zink_context(pctx);
 329    struct zink_query *query = (struct zink_query *)q;
 330    struct zink_batch *batch = zink_curr_batch(ctx);
 331
 332    if (query->type == PIPE_QUERY_TIMESTAMP) {
 333       assert(query->curr_query == 0);
 334       vkCmdWriteTimestamp(batch->cmdbuf, VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT,
 335                           query->query_pool, 0);
 336    } else if (query->active)
 337       end_query(ctx, batch, query);
 338
 339    return true;
 340 }
 341
 342 static bool
 343 zink_get_query_result(struct pipe_context *pctx,
 344                       struct pipe_query *q,
 345                       bool wait,
 346                       union pipe_query_result *result)
 347 {
 348    struct zink_query *query = (struct zink_query *)q;
 349
 350    if (wait) {
 351       wait_query(pctx, query);
 352    } else
 353       pctx->flush(pctx, NULL, 0);
 354    return get_query_result(pctx, q, wait, result);
 355 }
 356
 357 void
 358 zink_suspend_queries(struct zink_context *ctx, struct zink_batch *batch)
 359 {
 360    if (!batch->active_queries)
 361       return;
 362    set_foreach(batch->active_queries, entry) {
 363       struct zink_query *query = (void*)entry->key;
 364       /* if a query isn't active here then we don't need to reactivate it on the next batch */
 365       if (query->active) {
 366          end_query(ctx, batch, query);
 367          /* the fence is going to steal the set off the batch, so we have to copy
 368           * the active queries onto a list
 369           */
 370          list_addtail(&query->active_list, &ctx->suspended_queries);
 371       }
 372    }
 373 }
 374
 375 void
 376 zink_resume_queries(struct zink_context *ctx, struct zink_batch *batch)
 377 {
 378    struct zink_query *query, *next;
 379    LIST_FOR_EACH_ENTRY_SAFE(query, next, &ctx->suspended_queries, active_list) {
 380       begin_query(ctx, batch, query);
 381       list_delinit(&query->active_list);
 382    }
 383 }
 384
 385 static void
 386 zink_set_active_query_state(struct pipe_context *pctx, bool enable)
 387 {
 388    struct zink_context *ctx = zink_context(pctx);
 389    ctx->queries_disabled = !enable;
 390
 391    struct zink_batch *batch = zink_curr_batch(ctx);
 392    if (ctx->queries_disabled)
 393       zink_suspend_queries(ctx, batch);
 394    else
 395       zink_resume_queries(ctx, batch);
 396 }
 397
 398 static void
 399 zink_render_condition(struct pipe_context *pctx,
 400                       struct pipe_query *pquery,
 401                       bool condition,
 402                       enum pipe_render_cond_flag mode)
 403 {
 404    struct zink_context *ctx = zink_context(pctx);
 405    struct zink_screen *screen = zink_screen(pctx->screen);
 406    struct zink_query *query = (struct zink_query *)pquery;
 407    struct zink_batch *batch = zink_batch_no_rp(ctx);
 408    VkQueryResultFlagBits flags = 0;
 409
 410    if (query == NULL) {
 411       screen->vk_CmdEndConditionalRenderingEXT(batch->cmdbuf);
 412       return;
 413    }
 414
 415    struct pipe_resource *pres;
 416    struct zink_resource *res;
 417    struct pipe_resource templ = {};
 418    templ.width0 = 8;
 419    templ.height0 = 1;
 420    templ.depth0 = 1;
 421    templ.format = PIPE_FORMAT_R8_UINT;
 422    templ.target = PIPE_BUFFER;
 423
 424    /* need to create a vulkan buffer to copy the data into */
 425    pres = pctx->screen->resource_create(pctx->screen, &templ);
 426    if (!pres)
 427       return;
 428
 429    res = (struct zink_resource *)pres;
 430
 431    if (mode == PIPE_RENDER_COND_WAIT || mode == PIPE_RENDER_COND_BY_REGION_WAIT)
 432       flags |= VK_QUERY_RESULT_WAIT_BIT;
 433
 434    if (query->use_64bit)
 435       flags |= VK_QUERY_RESULT_64_BIT;
 436    int num_results = query->curr_query - query->last_checked_query;
 437    vkCmdCopyQueryPoolResults(batch->cmdbuf, query->query_pool, query->last_checked_query, num_results,
 438                              res->buffer, 0, 0, flags);
 439
 440    query->last_checked_query = query->curr_query;
 441    VkConditionalRenderingFlagsEXT begin_flags = 0;
 442    if (condition)
 443       begin_flags = VK_CONDITIONAL_RENDERING_INVERTED_BIT_EXT;
 444    VkConditionalRenderingBeginInfoEXT begin_info = {};
 445    begin_info.sType = VK_STRUCTURE_TYPE_CONDITIONAL_RENDERING_BEGIN_INFO_EXT;
 446    begin_info.buffer = res->buffer;
 447    begin_info.flags = begin_flags;
 448    screen->vk_CmdBeginConditionalRenderingEXT(batch->cmdbuf, &begin_info);
 449
 450    zink_batch_reference_resoure(batch, res);
 451
 452    pipe_resource_reference(&pres, NULL);
 453 }
 454
 455 void
 456 zink_context_query_init(struct pipe_context *pctx)
 457 {
 458    struct zink_context *ctx = zink_context(pctx);
 459    list_inithead(&ctx->suspended_queries);
 460
 461    pctx->create_query = zink_create_query;
 462    pctx->destroy_query = zink_destroy_query;
 463    pctx->begin_query = zink_begin_query;
 464    pctx->end_query = zink_end_query;
 465    pctx->get_query_result = zink_get_query_result;
 466    pctx->set_active_query_state = zink_set_active_query_state;
 467    pctx->render_condition = zink_render_condition;
 468 }