src/gallium/drivers/nouveau/nvc0/nvc0_query_hw.c

   1 /*
   2  * Copyright 2011 Christoph Bumiller
   3  * Copyright 2015 Samuel Pitoiset
   4  *
   5  * Permission is hereby granted, free of charge, to any person obtaining a
   6  * copy of this software and associated documentation files (the "Software"),
   7  * to deal in the Software without restriction, including without limitation
   8  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   9  * and/or sell copies of the Software, and to permit persons to whom the
  10  * Software is furnished to do so, subject to the following conditions:
  11  *
  12  * The above copyright notice and this permission notice shall be included in
  13  * all copies or substantial portions of the Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
  19  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  20  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  21  * OTHER DEALINGS IN THE SOFTWARE.
  22  */
  23
  24 #define NVC0_PUSH_EXPLICIT_SPACE_CHECKING
  25
  26 #include "nvc0/nvc0_context.h"
  27 #include "nvc0/nvc0_query_hw.h"
  28 #include "nvc0/nvc0_query_hw_metric.h"
  29 #include "nvc0/nvc0_query_hw_sm.h"
  30
  31 #define NVC0_HW_QUERY_STATE_READY   0
  32 #define NVC0_HW_QUERY_STATE_ACTIVE  1
  33 #define NVC0_HW_QUERY_STATE_ENDED   2
  34 #define NVC0_HW_QUERY_STATE_FLUSHED 3
  35
  36 #define NVC0_HW_QUERY_ALLOC_SPACE 256
  37
  38 bool
  39 nvc0_hw_query_allocate(struct nvc0_context *nvc0, struct nvc0_query *q,
  40                        int size)
  41 {
  42    struct nvc0_hw_query *hq = nvc0_hw_query(q);
  43    struct nvc0_screen *screen = nvc0->screen;
  44    int ret;
  45
  46    if (hq->bo) {
  47       nouveau_bo_ref(NULL, &hq->bo);
  48       if (hq->mm) {
  49          if (hq->state == NVC0_HW_QUERY_STATE_READY)
  50             nouveau_mm_free(hq->mm);
  51          else
  52             nouveau_fence_work(screen->base.fence.current,
  53                                nouveau_mm_free_work, hq->mm);
  54       }
  55    }
  56    if (size) {
  57       hq->mm = nouveau_mm_allocate(screen->base.mm_GART, size, &hq->bo,
  58                                    &hq->base_offset);
  59       if (!hq->bo)
  60          return false;
  61       hq->offset = hq->base_offset;
  62
  63       ret = nouveau_bo_map(hq->bo, 0, screen->base.client);
  64       if (ret) {
  65          nvc0_hw_query_allocate(nvc0, q, 0);
  66          return false;
  67       }
  68       hq->data = (uint32_t *)((uint8_t *)hq->bo->map + hq->base_offset);
  69    }
  70    return true;
  71 }
  72
  73 static void
  74 nvc0_hw_query_get(struct nouveau_pushbuf *push, struct nvc0_query *q,
  75                   unsigned offset, uint32_t get)
  76 {
  77    struct nvc0_hw_query *hq = nvc0_hw_query(q);
  78
  79    offset += hq->offset;
  80
  81    PUSH_SPACE(push, 5);
  82    PUSH_REFN (push, hq->bo, NOUVEAU_BO_GART | NOUVEAU_BO_WR);
  83    BEGIN_NVC0(push, NVC0_3D(QUERY_ADDRESS_HIGH), 4);
  84    PUSH_DATAh(push, hq->bo->offset + offset);
  85    PUSH_DATA (push, hq->bo->offset + offset);
  86    PUSH_DATA (push, hq->sequence);
  87    PUSH_DATA (push, get);
  88 }
  89
  90 static void
  91 nvc0_hw_query_rotate(struct nvc0_context *nvc0, struct nvc0_query *q)
  92 {
  93    struct nvc0_hw_query *hq = nvc0_hw_query(q);
  94
  95    hq->offset += hq->rotate;
  96    hq->data += hq->rotate / sizeof(*hq->data);
  97    if (hq->offset - hq->base_offset == NVC0_HW_QUERY_ALLOC_SPACE)
  98       nvc0_hw_query_allocate(nvc0, q, NVC0_HW_QUERY_ALLOC_SPACE);
  99 }
 100
 101 static inline void
 102 nvc0_hw_query_update(struct nouveau_client *cli, struct nvc0_query *q)
 103 {
 104    struct nvc0_hw_query *hq = nvc0_hw_query(q);
 105
 106    if (hq->is64bit) {
 107       if (nouveau_fence_signalled(hq->fence))
 108          hq->state = NVC0_HW_QUERY_STATE_READY;
 109    } else {
 110       if (hq->data[0] == hq->sequence)
 111          hq->state = NVC0_HW_QUERY_STATE_READY;
 112    }
 113 }
 114
 115 static void
 116 nvc0_hw_destroy_query(struct nvc0_context *nvc0, struct nvc0_query *q)
 117 {
 118    struct nvc0_hw_query *hq = nvc0_hw_query(q);
 119
 120    if (hq->funcs && hq->funcs->destroy_query) {
 121       hq->funcs->destroy_query(nvc0, hq);
 122       return;
 123    }
 124
 125    nvc0_hw_query_allocate(nvc0, q, 0);
 126    nouveau_fence_ref(NULL, &hq->fence);
 127    FREE(hq);
 128 }
 129
 130 static boolean
 131 nvc0_hw_begin_query(struct nvc0_context *nvc0, struct nvc0_query *q)
 132 {
 133    struct nouveau_pushbuf *push = nvc0->base.pushbuf;
 134    struct nvc0_hw_query *hq = nvc0_hw_query(q);
 135    bool ret = true;
 136
 137    if (hq->funcs && hq->funcs->begin_query)
 138       return hq->funcs->begin_query(nvc0, hq);
 139
 140    /* For occlusion queries we have to change the storage, because a previous
 141     * query might set the initial render conition to false even *after* we re-
 142     * initialized it to true.
 143     */
 144    if (hq->rotate) {
 145       nvc0_hw_query_rotate(nvc0, q);
 146
 147       /* XXX: can we do this with the GPU, and sync with respect to a previous
 148        *  query ?
 149        */
 150       hq->data[0] = hq->sequence; /* initialize sequence */
 151       hq->data[1] = 1; /* initial render condition = true */
 152       hq->data[4] = hq->sequence + 1; /* for comparison COND_MODE */
 153       hq->data[5] = 0;
 154    }
 155    hq->sequence++;
 156
 157    switch (q->type) {
 158    case PIPE_QUERY_OCCLUSION_COUNTER:
 159    case PIPE_QUERY_OCCLUSION_PREDICATE:
 160       hq->nesting = nvc0->screen->num_occlusion_queries_active++;
 161       if (hq->nesting) {
 162          nvc0_hw_query_get(push, q, 0x10, 0x0100f002);
 163       } else {
 164          PUSH_SPACE(push, 3);
 165          BEGIN_NVC0(push, NVC0_3D(COUNTER_RESET), 1);
 166          PUSH_DATA (push, NVC0_3D_COUNTER_RESET_SAMPLECNT);
 167          IMMED_NVC0(push, NVC0_3D(SAMPLECNT_ENABLE), 1);
 168       }
 169       break;
 170    case PIPE_QUERY_PRIMITIVES_GENERATED:
 171       nvc0_hw_query_get(push, q, 0x10, 0x09005002 | (q->index << 5));
 172       break;
 173    case PIPE_QUERY_PRIMITIVES_EMITTED:
 174       nvc0_hw_query_get(push, q, 0x10, 0x05805002 | (q->index << 5));
 175       break;
 176    case PIPE_QUERY_SO_STATISTICS:
 177       nvc0_hw_query_get(push, q, 0x20, 0x05805002 | (q->index << 5));
 178       nvc0_hw_query_get(push, q, 0x30, 0x06805002 | (q->index << 5));
 179       break;
 180    case PIPE_QUERY_SO_OVERFLOW_PREDICATE:
 181       nvc0_hw_query_get(push, q, 0x10, 0x03005002 | (q->index << 5));
 182       break;
 183    case PIPE_QUERY_TIME_ELAPSED:
 184       nvc0_hw_query_get(push, q, 0x10, 0x00005002);
 185       break;
 186    case PIPE_QUERY_PIPELINE_STATISTICS:
 187       nvc0_hw_query_get(push, q, 0xc0 + 0x00, 0x00801002); /* VFETCH, VERTICES */
 188       nvc0_hw_query_get(push, q, 0xc0 + 0x10, 0x01801002); /* VFETCH, PRIMS */
 189       nvc0_hw_query_get(push, q, 0xc0 + 0x20, 0x02802002); /* VP, LAUNCHES */
 190       nvc0_hw_query_get(push, q, 0xc0 + 0x30, 0x03806002); /* GP, LAUNCHES */
 191       nvc0_hw_query_get(push, q, 0xc0 + 0x40, 0x04806002); /* GP, PRIMS_OUT */
 192       nvc0_hw_query_get(push, q, 0xc0 + 0x50, 0x07804002); /* RAST, PRIMS_IN */
 193       nvc0_hw_query_get(push, q, 0xc0 + 0x60, 0x08804002); /* RAST, PRIMS_OUT */
 194       nvc0_hw_query_get(push, q, 0xc0 + 0x70, 0x0980a002); /* ROP, PIXELS */
 195       nvc0_hw_query_get(push, q, 0xc0 + 0x80, 0x0d808002); /* TCP, LAUNCHES */
 196       nvc0_hw_query_get(push, q, 0xc0 + 0x90, 0x0e809002); /* TEP, LAUNCHES */
 197       break;
 198    default:
 199       break;
 200    }
 201    hq->state = NVC0_HW_QUERY_STATE_ACTIVE;
 202    return ret;
 203 }
 204
 205 static void
 206 nvc0_hw_end_query(struct nvc0_context *nvc0, struct nvc0_query *q)
 207 {
 208    struct nouveau_pushbuf *push = nvc0->base.pushbuf;
 209    struct nvc0_hw_query *hq = nvc0_hw_query(q);
 210
 211    if (hq->funcs && hq->funcs->end_query) {
 212       hq->funcs->end_query(nvc0, hq);
 213       return;
 214    }
 215
 216    if (hq->state != NVC0_HW_QUERY_STATE_ACTIVE) {
 217       /* some queries don't require 'begin' to be called (e.g. GPU_FINISHED) */
 218       if (hq->rotate)
 219          nvc0_hw_query_rotate(nvc0, q);
 220       hq->sequence++;
 221    }
 222    hq->state = NVC0_HW_QUERY_STATE_ENDED;
 223
 224    switch (q->type) {
 225    case PIPE_QUERY_OCCLUSION_COUNTER:
 226    case PIPE_QUERY_OCCLUSION_PREDICATE:
 227       nvc0_hw_query_get(push, q, 0, 0x0100f002);
 228       if (--nvc0->screen->num_occlusion_queries_active == 0) {
 229          PUSH_SPACE(push, 1);
 230          IMMED_NVC0(push, NVC0_3D(SAMPLECNT_ENABLE), 0);
 231       }
 232       break;
 233    case PIPE_QUERY_PRIMITIVES_GENERATED:
 234       nvc0_hw_query_get(push, q, 0, 0x09005002 | (q->index << 5));
 235       break;
 236    case PIPE_QUERY_PRIMITIVES_EMITTED:
 237       nvc0_hw_query_get(push, q, 0, 0x05805002 | (q->index << 5));
 238       break;
 239    case PIPE_QUERY_SO_STATISTICS:
 240       nvc0_hw_query_get(push, q, 0x00, 0x05805002 | (q->index << 5));
 241       nvc0_hw_query_get(push, q, 0x10, 0x06805002 | (q->index << 5));
 242       break;
 243    case PIPE_QUERY_SO_OVERFLOW_PREDICATE:
 244       /* TODO: How do we sum over all streams for render condition ? */
 245       /* PRIMS_DROPPED doesn't write sequence, use a ZERO query to sync on */
 246       nvc0_hw_query_get(push, q, 0x00, 0x03005002 | (q->index << 5));
 247       nvc0_hw_query_get(push, q, 0x20, 0x00005002);
 248       break;
 249    case PIPE_QUERY_TIMESTAMP:
 250    case PIPE_QUERY_TIME_ELAPSED:
 251       nvc0_hw_query_get(push, q, 0, 0x00005002);
 252       break;
 253    case PIPE_QUERY_GPU_FINISHED:
 254       nvc0_hw_query_get(push, q, 0, 0x1000f010);
 255       break;
 256    case PIPE_QUERY_PIPELINE_STATISTICS:
 257       nvc0_hw_query_get(push, q, 0x00, 0x00801002); /* VFETCH, VERTICES */
 258       nvc0_hw_query_get(push, q, 0x10, 0x01801002); /* VFETCH, PRIMS */
 259       nvc0_hw_query_get(push, q, 0x20, 0x02802002); /* VP, LAUNCHES */
 260       nvc0_hw_query_get(push, q, 0x30, 0x03806002); /* GP, LAUNCHES */
 261       nvc0_hw_query_get(push, q, 0x40, 0x04806002); /* GP, PRIMS_OUT */
 262       nvc0_hw_query_get(push, q, 0x50, 0x07804002); /* RAST, PRIMS_IN */
 263       nvc0_hw_query_get(push, q, 0x60, 0x08804002); /* RAST, PRIMS_OUT */
 264       nvc0_hw_query_get(push, q, 0x70, 0x0980a002); /* ROP, PIXELS */
 265       nvc0_hw_query_get(push, q, 0x80, 0x0d808002); /* TCP, LAUNCHES */
 266       nvc0_hw_query_get(push, q, 0x90, 0x0e809002); /* TEP, LAUNCHES */
 267       break;
 268    case PIPE_QUERY_TIMESTAMP_DISJOINT:
 269       /* This query is not issued on GPU because disjoint is forced to false */
 270       hq->state = NVC0_HW_QUERY_STATE_READY;
 271       break;
 272    case NVC0_HW_QUERY_TFB_BUFFER_OFFSET:
 273       /* indexed by TFB buffer instead of by vertex stream */
 274       nvc0_hw_query_get(push, q, 0x00, 0x0d005002 | (q->index << 5));
 275       break;
 276    default:
 277       break;
 278    }
 279    if (hq->is64bit)
 280       nouveau_fence_ref(nvc0->screen->base.fence.current, &hq->fence);
 281 }
 282
 283 static boolean
 284 nvc0_hw_get_query_result(struct nvc0_context *nvc0, struct nvc0_query *q,
 285                          boolean wait, union pipe_query_result *result)
 286 {
 287    struct nvc0_hw_query *hq = nvc0_hw_query(q);
 288    uint64_t *res64 = (uint64_t*)result;
 289    uint32_t *res32 = (uint32_t*)result;
 290    uint8_t *res8 = (uint8_t*)result;
 291    uint64_t *data64 = (uint64_t *)hq->data;
 292    unsigned i;
 293
 294    if (hq->funcs && hq->funcs->get_query_result)
 295       return hq->funcs->get_query_result(nvc0, hq, wait, result);
 296
 297    if (hq->state != NVC0_HW_QUERY_STATE_READY)
 298       nvc0_hw_query_update(nvc0->screen->base.client, q);
 299
 300    if (hq->state != NVC0_HW_QUERY_STATE_READY) {
 301       if (!wait) {
 302          if (hq->state != NVC0_HW_QUERY_STATE_FLUSHED) {
 303             hq->state = NVC0_HW_QUERY_STATE_FLUSHED;
 304             /* flush for silly apps that spin on GL_QUERY_RESULT_AVAILABLE */
 305             PUSH_KICK(nvc0->base.pushbuf);
 306          }
 307          return false;
 308       }
 309       if (nouveau_bo_wait(hq->bo, NOUVEAU_BO_RD, nvc0->screen->base.client))
 310          return false;
 311       NOUVEAU_DRV_STAT(&nvc0->screen->base, query_sync_count, 1);
 312    }
 313    hq->state = NVC0_HW_QUERY_STATE_READY;
 314
 315    switch (q->type) {
 316    case PIPE_QUERY_GPU_FINISHED:
 317       res8[0] = true;
 318       break;
 319    case PIPE_QUERY_OCCLUSION_COUNTER: /* u32 sequence, u32 count, u64 time */
 320       res64[0] = hq->data[1] - hq->data[5];
 321       break;
 322    case PIPE_QUERY_OCCLUSION_PREDICATE:
 323       res8[0] = hq->data[1] != hq->data[5];
 324       break;
 325    case PIPE_QUERY_PRIMITIVES_GENERATED: /* u64 count, u64 time */
 326    case PIPE_QUERY_PRIMITIVES_EMITTED: /* u64 count, u64 time */
 327       res64[0] = data64[0] - data64[2];
 328       break;
 329    case PIPE_QUERY_SO_STATISTICS:
 330       res64[0] = data64[0] - data64[4];
 331       res64[1] = data64[2] - data64[6];
 332       break;
 333    case PIPE_QUERY_SO_OVERFLOW_PREDICATE:
 334       res8[0] = data64[0] != data64[2];
 335       break;
 336    case PIPE_QUERY_TIMESTAMP:
 337       res64[0] = data64[1];
 338       break;
 339    case PIPE_QUERY_TIMESTAMP_DISJOINT:
 340       res64[0] = 1000000000;
 341       res8[8] = false;
 342       break;
 343    case PIPE_QUERY_TIME_ELAPSED:
 344       res64[0] = data64[1] - data64[3];
 345       break;
 346    case PIPE_QUERY_PIPELINE_STATISTICS:
 347       for (i = 0; i < 10; ++i)
 348          res64[i] = data64[i * 2] - data64[24 + i * 2];
 349       break;
 350    case NVC0_HW_QUERY_TFB_BUFFER_OFFSET:
 351       res32[0] = hq->data[1];
 352       break;
 353    default:
 354       assert(0); /* can't happen, we don't create queries with invalid type */
 355       return false;
 356    }
 357
 358    return true;
 359 }
 360
 361 static const struct nvc0_query_funcs hw_query_funcs = {
 362    .destroy_query = nvc0_hw_destroy_query,
 363    .begin_query = nvc0_hw_begin_query,
 364    .end_query = nvc0_hw_end_query,
 365    .get_query_result = nvc0_hw_get_query_result,
 366 };
 367
 368 struct nvc0_query *
 369 nvc0_hw_create_query(struct nvc0_context *nvc0, unsigned type, unsigned index)
 370 {
 371    struct nvc0_hw_query *hq;
 372    struct nvc0_query *q;
 373    unsigned space = NVC0_HW_QUERY_ALLOC_SPACE;
 374
 375    hq = nvc0_hw_sm_create_query(nvc0, type);
 376    if (hq) {
 377       hq->base.funcs = &hw_query_funcs;
 378       return (struct nvc0_query *)hq;
 379    }
 380
 381    hq = nvc0_hw_metric_create_query(nvc0, type);
 382    if (hq) {
 383       hq->base.funcs = &hw_query_funcs;
 384       return (struct nvc0_query *)hq;
 385    }
 386
 387    hq = CALLOC_STRUCT(nvc0_hw_query);
 388    if (!hq)
 389       return NULL;
 390
 391    q = &hq->base;
 392    q->funcs = &hw_query_funcs;
 393    q->type = type;
 394
 395    switch (q->type) {
 396    case PIPE_QUERY_OCCLUSION_COUNTER:
 397    case PIPE_QUERY_OCCLUSION_PREDICATE:
 398       hq->rotate = 32;
 399       space = NVC0_HW_QUERY_ALLOC_SPACE;
 400       break;
 401    case PIPE_QUERY_PIPELINE_STATISTICS:
 402       hq->is64bit = true;
 403       space = 512;
 404       break;
 405    case PIPE_QUERY_SO_STATISTICS:
 406    case PIPE_QUERY_SO_OVERFLOW_PREDICATE:
 407       hq->is64bit = true;
 408       space = 64;
 409       break;
 410    case PIPE_QUERY_PRIMITIVES_GENERATED:
 411    case PIPE_QUERY_PRIMITIVES_EMITTED:
 412       hq->is64bit = true;
 413       q->index = index;
 414       space = 32;
 415       break;
 416    case PIPE_QUERY_TIME_ELAPSED:
 417    case PIPE_QUERY_TIMESTAMP:
 418    case PIPE_QUERY_TIMESTAMP_DISJOINT:
 419    case PIPE_QUERY_GPU_FINISHED:
 420       space = 32;
 421       break;
 422    case NVC0_HW_QUERY_TFB_BUFFER_OFFSET:
 423       space = 16;
 424       break;
 425    default:
 426       debug_printf("invalid query type: %u\n", type);
 427       FREE(q);
 428       return NULL;
 429    }
 430
 431    if (!nvc0_hw_query_allocate(nvc0, q, space)) {
 432       FREE(hq);
 433       return NULL;
 434    }
 435
 436    if (hq->rotate) {
 437       /* we advance before query_begin ! */
 438       hq->offset -= hq->rotate;
 439       hq->data -= hq->rotate / sizeof(*hq->data);
 440    } else
 441    if (!hq->is64bit)
 442       hq->data[0] = 0; /* initialize sequence */
 443
 444    return q;
 445 }
 446
 447 int
 448 nvc0_hw_get_driver_query_info(struct nvc0_screen *screen, unsigned id,
 449                               struct pipe_driver_query_info *info)
 450 {
 451    int num_hw_sm_queries = 0, num_hw_metric_queries = 0;
 452
 453    num_hw_sm_queries = nvc0_hw_sm_get_driver_query_info(screen, 0, NULL);
 454    num_hw_metric_queries =
 455       nvc0_hw_metric_get_driver_query_info(screen, 0, NULL);
 456
 457    if (!info)
 458       return num_hw_sm_queries + num_hw_metric_queries;
 459
 460    if (id < num_hw_sm_queries)
 461       return nvc0_hw_sm_get_driver_query_info(screen, id, info);
 462
 463    return nvc0_hw_metric_get_driver_query_info(screen,
 464                                                id - num_hw_sm_queries, info);
 465 }
 466
 467 void
 468 nvc0_hw_query_pushbuf_submit(struct nouveau_pushbuf *push,
 469                              struct nvc0_query *q, unsigned result_offset)
 470 {
 471    struct nvc0_hw_query *hq = nvc0_hw_query(q);
 472
 473    PUSH_REFN(push, hq->bo, NOUVEAU_BO_RD | NOUVEAU_BO_GART);
 474    nouveau_pushbuf_data(push, hq->bo, hq->offset + result_offset, 4 |
 475                         NVC0_IB_ENTRY_1_NO_PREFETCH);
 476 }
 477
 478 void
 479 nvc0_hw_query_fifo_wait(struct nouveau_pushbuf *push, struct nvc0_query *q)
 480 {
 481    struct nvc0_hw_query *hq = nvc0_hw_query(q);
 482    unsigned offset = hq->offset;
 483
 484    if (q->type == PIPE_QUERY_SO_OVERFLOW_PREDICATE) offset += 0x20;
 485
 486    PUSH_SPACE(push, 5);
 487    PUSH_REFN (push, hq->bo, NOUVEAU_BO_GART | NOUVEAU_BO_RD);
 488    BEGIN_NVC0(push, SUBC_3D(NV84_SUBCHAN_SEMAPHORE_ADDRESS_HIGH), 4);
 489    PUSH_DATAh(push, hq->bo->offset + offset);
 490    PUSH_DATA (push, hq->bo->offset + offset);
 491    PUSH_DATA (push, hq->sequence);
 492    PUSH_DATA (push, (1 << 12) |
 493               NV84_SUBCHAN_SEMAPHORE_TRIGGER_ACQUIRE_EQUAL);
 494 }