src/gallium/drivers/zink/zink_query.c

   1 #include "zink_query.h"
   2
   3 #include "zink_context.h"
   4 #include "zink_fence.h"
   5 #include "zink_resource.h"
   6 #include "zink_screen.h"
   7
   8 #include "util/hash_table.h"
   9 #include "util/set.h"
  10 #include "util/u_dump.h"
  11 #include "util/u_inlines.h"
  12 #include "util/u_memory.h"
  13
  14 #define NUM_QUERIES 50
  15
  16 struct zink_query {
  17    enum pipe_query_type type;
  18
  19    VkQueryPool query_pool;
  20    unsigned last_checked_query, curr_query, num_queries;
  21
  22    VkQueryType vkqtype;
  23    unsigned index;
  24    bool use_64bit;
  25    bool precise;
  26    bool xfb_running;
  27
  28    bool active; /* query is considered active by vk */
  29    bool dead; /* query should be destroyed when its fence finishes */
  30
  31    unsigned fences;
  32    struct list_head active_list;
  33 };
  34
  35 static VkQueryType
  36 convert_query_type(unsigned query_type, bool *use_64bit, bool *precise)
  37 {
  38    *use_64bit = false;
  39    *precise = false;
  40    switch (query_type) {
  41    case PIPE_QUERY_OCCLUSION_COUNTER:
  42       *precise = true;
  43       *use_64bit = true;
  44       /* fallthrough */
  45    case PIPE_QUERY_OCCLUSION_PREDICATE:
  46    case PIPE_QUERY_OCCLUSION_PREDICATE_CONSERVATIVE:
  47       return VK_QUERY_TYPE_OCCLUSION;
  48    case PIPE_QUERY_TIMESTAMP:
  49       *use_64bit = true;
  50       return VK_QUERY_TYPE_TIMESTAMP;
  51    case PIPE_QUERY_PIPELINE_STATISTICS:
  52    case PIPE_QUERY_PRIMITIVES_GENERATED:
  53       return VK_QUERY_TYPE_PIPELINE_STATISTICS;
  54    case PIPE_QUERY_PRIMITIVES_EMITTED:
  55       *use_64bit = true;
  56       return VK_QUERY_TYPE_TRANSFORM_FEEDBACK_STREAM_EXT;
  57    default:
  58       debug_printf("unknown query: %s\n",
  59                    util_str_query_type(query_type, true));
  60       unreachable("zink: unknown query type");
  61    }
  62 }
  63
  64 static struct pipe_query *
  65 zink_create_query(struct pipe_context *pctx,
  66                   unsigned query_type, unsigned index)
  67 {
  68    struct zink_screen *screen = zink_screen(pctx->screen);
  69    struct zink_query *query = CALLOC_STRUCT(zink_query);
  70    VkQueryPoolCreateInfo pool_create = {};
  71
  72    if (!query)
  73       return NULL;
  74
  75    query->index = index;
  76    query->type = query_type;
  77    query->vkqtype = convert_query_type(query_type, &query->use_64bit, &query->precise);
  78    if (query->vkqtype == -1)
  79       return NULL;
  80
  81    query->num_queries = query_type == PIPE_QUERY_TIMESTAMP ? 1 : NUM_QUERIES;
  82    query->curr_query = 0;
  83
  84    pool_create.sType = VK_STRUCTURE_TYPE_QUERY_POOL_CREATE_INFO;
  85    pool_create.queryType = query->vkqtype;
  86    pool_create.queryCount = query->num_queries;
  87    if (query_type == PIPE_QUERY_PRIMITIVES_GENERATED)
  88      pool_create.pipelineStatistics = VK_QUERY_PIPELINE_STATISTIC_INPUT_ASSEMBLY_PRIMITIVES_BIT;
  89
  90    VkResult status = vkCreateQueryPool(screen->dev, &pool_create, NULL, &query->query_pool);
  91    if (status != VK_SUCCESS) {
  92       FREE(query);
  93       return NULL;
  94    }
  95    struct zink_batch *batch = zink_batch_no_rp(zink_context(pctx));
  96    vkCmdResetQueryPool(batch->cmdbuf, query->query_pool, 0, query->num_queries);
  97    return (struct pipe_query *)query;
  98 }
  99
 100 static void
 101 wait_query(struct pipe_context *pctx, struct zink_query *query)
 102 {
 103    struct pipe_fence_handle *fence = NULL;
 104
 105    pctx->flush(pctx, &fence, PIPE_FLUSH_HINT_FINISH);
 106    if (fence) {
 107       pctx->screen->fence_finish(pctx->screen, NULL, fence,
 108                                  PIPE_TIMEOUT_INFINITE);
 109       pctx->screen->fence_reference(pctx->screen, &fence, NULL);
 110    }
 111 }
 112
 113 static void
 114 destroy_query(struct zink_screen *screen, struct zink_query *query)
 115 {
 116    assert(!p_atomic_read(&query->fences));
 117    vkDestroyQueryPool(screen->dev, query->query_pool, NULL);
 118    FREE(query);
 119 }
 120
 121 static void
 122 zink_destroy_query(struct pipe_context *pctx,
 123                    struct pipe_query *q)
 124 {
 125    struct zink_screen *screen = zink_screen(pctx->screen);
 126    struct zink_query *query = (struct zink_query *)q;
 127
 128    p_atomic_set(&query->dead, true);
 129    if (p_atomic_read(&query->fences)) {
 130       if (query->xfb_running)
 131         wait_query(pctx, query);
 132       return;
 133    }
 134
 135    destroy_query(screen, query);
 136 }
 137
 138 void
 139 zink_prune_queries(struct zink_screen *screen, struct zink_fence *fence)
 140 {
 141    set_foreach(fence->active_queries, entry) {
 142       struct zink_query *query = (void*)entry->key;
 143       if (!p_atomic_dec_return(&query->fences)) {
 144          if (p_atomic_read(&query->dead))
 145             destroy_query(screen, query);
 146       }
 147    }
 148    _mesa_set_destroy(fence->active_queries, NULL);
 149    fence->active_queries = NULL;
 150 }
 151
 152 static void
 153 begin_query(struct zink_context *ctx, struct zink_batch *batch, struct zink_query *q)
 154 {
 155    VkQueryControlFlags flags = 0;
 156
 157    if (q->precise)
 158       flags |= VK_QUERY_CONTROL_PRECISE_BIT;
 159    if (q->vkqtype == VK_QUERY_TYPE_TRANSFORM_FEEDBACK_STREAM_EXT) {
 160       zink_screen(ctx->base.screen)->vk_CmdBeginQueryIndexedEXT(batch->cmdbuf,
 161                                                                 q->query_pool,
 162                                                                 q->curr_query,
 163                                                                 flags,
 164                                                                 q->index);
 165       q->xfb_running = true;
 166    } else
 167       vkCmdBeginQuery(batch->cmdbuf, q->query_pool, q->curr_query, flags);
 168    q->active = true;
 169    if (!batch->active_queries)
 170       batch->active_queries = _mesa_set_create(NULL, _mesa_hash_pointer, _mesa_key_pointer_equal);
 171    assert(batch->active_queries);
 172    p_atomic_inc(&q->fences);
 173    _mesa_set_add(batch->active_queries, q);
 174 }
 175
 176 static bool
 177 zink_begin_query(struct pipe_context *pctx,
 178                  struct pipe_query *q)
 179 {
 180    struct zink_query *query = (struct zink_query *)q;
 181    struct zink_batch *batch = zink_curr_batch(zink_context(pctx));
 182
 183    /* ignore begin_query for timestamps */
 184    if (query->type == PIPE_QUERY_TIMESTAMP)
 185       return true;
 186
 187    begin_query(zink_context(pctx), batch, query);
 188
 189    return true;
 190 }
 191
 192 static bool
 193 get_query_result(struct pipe_context *pctx,
 194                       struct pipe_query *q,
 195                       bool wait,
 196                       union pipe_query_result *result)
 197 {
 198    struct zink_screen *screen = zink_screen(pctx->screen);
 199    struct zink_query *query = (struct zink_query *)q;
 200    VkQueryResultFlagBits flags = 0;
 201
 202    if (wait)
 203       flags |= VK_QUERY_RESULT_WAIT_BIT;
 204
 205    if (query->use_64bit)
 206       flags |= VK_QUERY_RESULT_64_BIT;
 207
 208    // union pipe_query_result results[NUM_QUERIES * 2];
 209    /* xfb queries return 2 results */
 210    uint64_t results[NUM_QUERIES * 2];
 211    memset(results, 0, sizeof(results));
 212    int num_results = query->curr_query - query->last_checked_query;
 213    if (query->vkqtype == VK_QUERY_TYPE_TRANSFORM_FEEDBACK_STREAM_EXT) {
 214       char tf_result[16] = {};
 215       /* this query emits 2 values */
 216       assert(query->curr_query <= ARRAY_SIZE(results) / 2);
 217       VkResult status = vkGetQueryPoolResults(screen->dev, query->query_pool,
 218                                               query->last_checked_query, num_results,
 219                                               sizeof(results),
 220                                               results,
 221                                               sizeof(uint64_t),
 222                                               flags);
 223       if (status != VK_SUCCESS)
 224          return false;
 225       memcpy(result, tf_result + (query->type == PIPE_QUERY_PRIMITIVES_GENERATED ? 8 : 0), 8);
 226       /* multiply for correct looping behavior below */
 227       num_results *= 2;
 228    } else {
 229       assert(query->curr_query <= ARRAY_SIZE(results));
 230       VkResult status = vkGetQueryPoolResults(screen->dev, query->query_pool,
 231                                               query->last_checked_query, num_results,
 232                                               sizeof(results),
 233                                               results,
 234                                               sizeof(uint64_t),
 235                                               flags);
 236       if (status != VK_SUCCESS)
 237          return false;
 238    }
 239
 240    util_query_clear_result(result, query->type);
 241    for (int i = 0; i < num_results; ++i) {
 242       switch (query->type) {
 243       case PIPE_QUERY_OCCLUSION_PREDICATE:
 244       case PIPE_QUERY_OCCLUSION_PREDICATE_CONSERVATIVE:
 245       case PIPE_QUERY_SO_OVERFLOW_PREDICATE:
 246       case PIPE_QUERY_SO_OVERFLOW_ANY_PREDICATE:
 247       case PIPE_QUERY_GPU_FINISHED:
 248          result->b |= results[i] != 0;
 249          break;
 250
 251       case PIPE_QUERY_OCCLUSION_COUNTER:
 252          result->u64 += results[i];
 253          break;
 254       case PIPE_QUERY_PRIMITIVES_GENERATED:
 255          result->u32 += results[i];
 256          break;
 257       case PIPE_QUERY_PRIMITIVES_EMITTED:
 258          /* A query pool created with this type will capture 2 integers -
 259           * numPrimitivesWritten and numPrimitivesNeeded -
 260           * for the specified vertex stream output from the last vertex processing stage.
 261           * - from VK_EXT_transform_feedback spec
 262           */
 263          result->u64 += results[i];
 264          i++;
 265          break;
 266
 267       default:
 268          debug_printf("unhangled query type: %s\n",
 269                       util_str_query_type(query->type, true));
 270          unreachable("unexpected query type");
 271       }
 272    }
 273    query->last_checked_query = query->curr_query;
 274
 275    return TRUE;
 276 }
 277
 278 static void
 279 end_query(struct zink_context *ctx, struct zink_batch *batch, struct zink_query *q)
 280 {
 281    struct zink_screen *screen = zink_screen(ctx->base.screen);
 282    assert(q->type != PIPE_QUERY_TIMESTAMP);
 283    q->active = false;
 284    if (q->vkqtype == VK_QUERY_TYPE_TRANSFORM_FEEDBACK_STREAM_EXT)
 285       screen->vk_CmdEndQueryIndexedEXT(batch->cmdbuf, q->query_pool, q->curr_query, q->index);
 286    else
 287       vkCmdEndQuery(batch->cmdbuf, q->query_pool, q->curr_query);
 288    if (++q->curr_query == q->num_queries) {
 289       vkCmdResetQueryPool(batch->cmdbuf, q->query_pool, 0, q->num_queries);
 290       q->last_checked_query = q->curr_query = 0;
 291    }
 292 }
 293
 294 static bool
 295 zink_end_query(struct pipe_context *pctx,
 296                struct pipe_query *q)
 297 {
 298    struct zink_context *ctx = zink_context(pctx);
 299    struct zink_query *query = (struct zink_query *)q;
 300    struct zink_batch *batch = zink_curr_batch(ctx);
 301
 302    if (query->type == PIPE_QUERY_TIMESTAMP) {
 303       assert(query->curr_query == 0);
 304       vkCmdWriteTimestamp(batch->cmdbuf, VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT,
 305                           query->query_pool, 0);
 306    } else if (query->active)
 307       end_query(ctx, batch, query);
 308
 309    return true;
 310 }
 311
 312 static bool
 313 zink_get_query_result(struct pipe_context *pctx,
 314                       struct pipe_query *q,
 315                       bool wait,
 316                       union pipe_query_result *result)
 317 {
 318    struct zink_query *query = (struct zink_query *)q;
 319
 320    if (wait) {
 321       wait_query(pctx, query);
 322    } else
 323       pctx->flush(pctx, NULL, 0);
 324    return get_query_result(pctx, q, wait, result);
 325 }
 326
 327 void
 328 zink_suspend_queries(struct zink_context *ctx, struct zink_batch *batch)
 329 {
 330    if (!batch->active_queries)
 331       return;
 332    set_foreach(batch->active_queries, entry) {
 333       struct zink_query *query = (void*)entry->key;
 334       /* if a query isn't active here then we don't need to reactivate it on the next batch */
 335       if (query->active) {
 336          end_query(ctx, batch, query);
 337          /* the fence is going to steal the set off the batch, so we have to copy
 338           * the active queries onto a list
 339           */
 340          list_addtail(&query->active_list, &ctx->suspended_queries);
 341       }
 342    }
 343 }
 344
 345 void
 346 zink_resume_queries(struct zink_context *ctx, struct zink_batch *batch)
 347 {
 348    struct zink_query *query, *next;
 349    LIST_FOR_EACH_ENTRY_SAFE(query, next, &ctx->suspended_queries, active_list) {
 350       begin_query(ctx, batch, query);
 351       list_delinit(&query->active_list);
 352    }
 353 }
 354
 355 static void
 356 zink_set_active_query_state(struct pipe_context *pctx, bool enable)
 357 {
 358    struct zink_context *ctx = zink_context(pctx);
 359    ctx->queries_disabled = !enable;
 360
 361    struct zink_batch *batch = zink_curr_batch(ctx);
 362    if (ctx->queries_disabled)
 363       zink_suspend_queries(ctx, batch);
 364    else
 365       zink_resume_queries(ctx, batch);
 366 }
 367
 368 static void
 369 zink_render_condition(struct pipe_context *pctx,
 370                       struct pipe_query *pquery,
 371                       bool condition,
 372                       enum pipe_render_cond_flag mode)
 373 {
 374    struct zink_context *ctx = zink_context(pctx);
 375    struct zink_screen *screen = zink_screen(pctx->screen);
 376    struct zink_query *query = (struct zink_query *)pquery;
 377    struct zink_batch *batch = zink_batch_no_rp(ctx);
 378    VkQueryResultFlagBits flags = 0;
 379
 380    if (query == NULL) {
 381       screen->vk_CmdEndConditionalRenderingEXT(batch->cmdbuf);
 382       return;
 383    }
 384
 385    struct pipe_resource *pres;
 386    struct zink_resource *res;
 387    struct pipe_resource templ = {};
 388    templ.width0 = 8;
 389    templ.height0 = 1;
 390    templ.depth0 = 1;
 391    templ.format = PIPE_FORMAT_R8_UINT;
 392    templ.target = PIPE_BUFFER;
 393
 394    /* need to create a vulkan buffer to copy the data into */
 395    pres = pctx->screen->resource_create(pctx->screen, &templ);
 396    if (!pres)
 397       return;
 398
 399    res = (struct zink_resource *)pres;
 400
 401    if (mode == PIPE_RENDER_COND_WAIT || mode == PIPE_RENDER_COND_BY_REGION_WAIT)
 402       flags |= VK_QUERY_RESULT_WAIT_BIT;
 403
 404    if (query->use_64bit)
 405       flags |= VK_QUERY_RESULT_64_BIT;
 406    int num_results = query->curr_query - query->last_checked_query;
 407    vkCmdCopyQueryPoolResults(batch->cmdbuf, query->query_pool, query->last_checked_query, num_results,
 408                              res->buffer, 0, 0, flags);
 409
 410    query->last_checked_query = query->curr_query;
 411    VkConditionalRenderingFlagsEXT begin_flags = 0;
 412    if (condition)
 413       begin_flags = VK_CONDITIONAL_RENDERING_INVERTED_BIT_EXT;
 414    VkConditionalRenderingBeginInfoEXT begin_info = {};
 415    begin_info.sType = VK_STRUCTURE_TYPE_CONDITIONAL_RENDERING_BEGIN_INFO_EXT;
 416    begin_info.buffer = res->buffer;
 417    begin_info.flags = begin_flags;
 418    screen->vk_CmdBeginConditionalRenderingEXT(batch->cmdbuf, &begin_info);
 419
 420    zink_batch_reference_resoure(batch, res);
 421
 422    pipe_resource_reference(&pres, NULL);
 423 }
 424
 425 void
 426 zink_context_query_init(struct pipe_context *pctx)
 427 {
 428    struct zink_context *ctx = zink_context(pctx);
 429    list_inithead(&ctx->suspended_queries);
 430
 431    pctx->create_query = zink_create_query;
 432    pctx->destroy_query = zink_destroy_query;
 433    pctx->begin_query = zink_begin_query;
 434    pctx->end_query = zink_end_query;
 435    pctx->get_query_result = zink_get_query_result;
 436    pctx->set_active_query_state = zink_set_active_query_state;
 437    pctx->render_condition = zink_render_condition;
 438 }