src/gallium/drivers/nouveau/nvc0/nvc0_query_hw.c

   1 /*
   2  * Copyright 2011 Christoph Bumiller
   3  * Copyright 2015 Samuel Pitoiset
   4  *
   5  * Permission is hereby granted, free of charge, to any person obtaining a
   6  * copy of this software and associated documentation files (the "Software"),
   7  * to deal in the Software without restriction, including without limitation
   8  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   9  * and/or sell copies of the Software, and to permit persons to whom the
  10  * Software is furnished to do so, subject to the following conditions:
  11  *
  12  * The above copyright notice and this permission notice shall be included in
  13  * all copies or substantial portions of the Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
  19  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  20  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  21  * OTHER DEALINGS IN THE SOFTWARE.
  22  */
  23
  24 #define NVC0_PUSH_EXPLICIT_SPACE_CHECKING
  25
  26 #include "nvc0/nvc0_context.h"
  27 #include "nvc0/nvc0_query_hw.h"
  28 #include "nvc0/nvc0_query_hw_metric.h"
  29 #include "nvc0/nvc0_query_hw_sm.h"
  30
  31 #define NVC0_HW_QUERY_STATE_READY   0
  32 #define NVC0_HW_QUERY_STATE_ACTIVE  1
  33 #define NVC0_HW_QUERY_STATE_ENDED   2
  34 #define NVC0_HW_QUERY_STATE_FLUSHED 3
  35
  36 #define NVC0_HW_QUERY_ALLOC_SPACE 256
  37
  38 bool
  39 nvc0_hw_query_allocate(struct nvc0_context *nvc0, struct nvc0_query *q,
  40                        int size)
  41 {
  42    struct nvc0_hw_query *hq = nvc0_hw_query(q);
  43    struct nvc0_screen *screen = nvc0->screen;
  44    int ret;
  45
  46    if (hq->bo) {
  47       nouveau_bo_ref(NULL, &hq->bo);
  48       if (hq->mm) {
  49          if (hq->state == NVC0_HW_QUERY_STATE_READY)
  50             nouveau_mm_free(hq->mm);
  51          else
  52             nouveau_fence_work(screen->base.fence.current,
  53                                nouveau_mm_free_work, hq->mm);
  54       }
  55    }
  56    if (size) {
  57       hq->mm = nouveau_mm_allocate(screen->base.mm_GART, size, &hq->bo,
  58                                    &hq->base_offset);
  59       if (!hq->bo)
  60          return false;
  61       hq->offset = hq->base_offset;
  62
  63       ret = nouveau_bo_map(hq->bo, 0, screen->base.client);
  64       if (ret) {
  65          nvc0_hw_query_allocate(nvc0, q, 0);
  66          return false;
  67       }
  68       hq->data = (uint32_t *)((uint8_t *)hq->bo->map + hq->base_offset);
  69    }
  70    return true;
  71 }
  72
  73 static void
  74 nvc0_hw_query_get(struct nouveau_pushbuf *push, struct nvc0_query *q,
  75                   unsigned offset, uint32_t get)
  76 {
  77    struct nvc0_hw_query *hq = nvc0_hw_query(q);
  78
  79    offset += hq->offset;
  80
  81    PUSH_SPACE(push, 5);
  82    PUSH_REFN (push, hq->bo, NOUVEAU_BO_GART | NOUVEAU_BO_WR);
  83    BEGIN_NVC0(push, NVC0_3D(QUERY_ADDRESS_HIGH), 4);
  84    PUSH_DATAh(push, hq->bo->offset + offset);
  85    PUSH_DATA (push, hq->bo->offset + offset);
  86    PUSH_DATA (push, hq->sequence);
  87    PUSH_DATA (push, get);
  88 }
  89
  90 static void
  91 nvc0_hw_query_rotate(struct nvc0_context *nvc0, struct nvc0_query *q)
  92 {
  93    struct nvc0_hw_query *hq = nvc0_hw_query(q);
  94
  95    hq->offset += hq->rotate;
  96    hq->data += hq->rotate / sizeof(*hq->data);
  97    if (hq->offset - hq->base_offset == NVC0_HW_QUERY_ALLOC_SPACE)
  98       nvc0_hw_query_allocate(nvc0, q, NVC0_HW_QUERY_ALLOC_SPACE);
  99 }
 100
 101 static inline void
 102 nvc0_hw_query_update(struct nouveau_client *cli, struct nvc0_query *q)
 103 {
 104    struct nvc0_hw_query *hq = nvc0_hw_query(q);
 105
 106    if (hq->is64bit) {
 107       if (nouveau_fence_signalled(hq->fence))
 108          hq->state = NVC0_HW_QUERY_STATE_READY;
 109    } else {
 110       if (hq->data[0] == hq->sequence)
 111          hq->state = NVC0_HW_QUERY_STATE_READY;
 112    }
 113 }
 114
 115 static void
 116 nvc0_hw_destroy_query(struct nvc0_context *nvc0, struct nvc0_query *q)
 117 {
 118    struct nvc0_hw_query *hq = nvc0_hw_query(q);
 119
 120    if (hq->funcs && hq->funcs->destroy_query) {
 121       hq->funcs->destroy_query(nvc0, hq);
 122       return;
 123    }
 124
 125    nvc0_hw_query_allocate(nvc0, q, 0);
 126    nouveau_fence_ref(NULL, &hq->fence);
 127    FREE(hq);
 128 }
 129
 130 static boolean
 131 nvc0_hw_begin_query(struct nvc0_context *nvc0, struct nvc0_query *q)
 132 {
 133    struct nouveau_pushbuf *push = nvc0->base.pushbuf;
 134    struct nvc0_hw_query *hq = nvc0_hw_query(q);
 135    bool ret = true;
 136
 137    if (hq->funcs && hq->funcs->begin_query)
 138       return hq->funcs->begin_query(nvc0, hq);
 139
 140    /* For occlusion queries we have to change the storage, because a previous
 141     * query might set the initial render conition to false even *after* we re-
 142     * initialized it to true.
 143     */
 144    if (hq->rotate) {
 145       nvc0_hw_query_rotate(nvc0, q);
 146
 147       /* XXX: can we do this with the GPU, and sync with respect to a previous
 148        *  query ?
 149        */
 150       hq->data[0] = hq->sequence; /* initialize sequence */
 151       hq->data[1] = 1; /* initial render condition = true */
 152       hq->data[4] = hq->sequence + 1; /* for comparison COND_MODE */
 153       hq->data[5] = 0;
 154    }
 155    hq->sequence++;
 156
 157    switch (q->type) {
 158    case PIPE_QUERY_OCCLUSION_COUNTER:
 159    case PIPE_QUERY_OCCLUSION_PREDICATE:
 160       hq->nesting = nvc0->screen->num_occlusion_queries_active++;
 161       if (hq->nesting) {
 162          nvc0_hw_query_get(push, q, 0x10, 0x0100f002);
 163       } else {
 164          PUSH_SPACE(push, 3);
 165          BEGIN_NVC0(push, NVC0_3D(COUNTER_RESET), 1);
 166          PUSH_DATA (push, NVC0_3D_COUNTER_RESET_SAMPLECNT);
 167          IMMED_NVC0(push, NVC0_3D(SAMPLECNT_ENABLE), 1);
 168       }
 169       break;
 170    case PIPE_QUERY_PRIMITIVES_GENERATED:
 171       nvc0_hw_query_get(push, q, 0x10, 0x09005002 | (q->index << 5));
 172       break;
 173    case PIPE_QUERY_PRIMITIVES_EMITTED:
 174       nvc0_hw_query_get(push, q, 0x10, 0x05805002 | (q->index << 5));
 175       break;
 176    case PIPE_QUERY_SO_STATISTICS:
 177       nvc0_hw_query_get(push, q, 0x20, 0x05805002 | (q->index << 5));
 178       nvc0_hw_query_get(push, q, 0x30, 0x06805002 | (q->index << 5));
 179       break;
 180    case PIPE_QUERY_SO_OVERFLOW_PREDICATE:
 181       nvc0_hw_query_get(push, q, 0x10, 0x03005002 | (q->index << 5));
 182       break;
 183    case PIPE_QUERY_TIME_ELAPSED:
 184       nvc0_hw_query_get(push, q, 0x10, 0x00005002);
 185       break;
 186    case PIPE_QUERY_PIPELINE_STATISTICS:
 187       nvc0_hw_query_get(push, q, 0xc0 + 0x00, 0x00801002); /* VFETCH, VERTICES */
 188       nvc0_hw_query_get(push, q, 0xc0 + 0x10, 0x01801002); /* VFETCH, PRIMS */
 189       nvc0_hw_query_get(push, q, 0xc0 + 0x20, 0x02802002); /* VP, LAUNCHES */
 190       nvc0_hw_query_get(push, q, 0xc0 + 0x30, 0x03806002); /* GP, LAUNCHES */
 191       nvc0_hw_query_get(push, q, 0xc0 + 0x40, 0x04806002); /* GP, PRIMS_OUT */
 192       nvc0_hw_query_get(push, q, 0xc0 + 0x50, 0x07804002); /* RAST, PRIMS_IN */
 193       nvc0_hw_query_get(push, q, 0xc0 + 0x60, 0x08804002); /* RAST, PRIMS_OUT */
 194       nvc0_hw_query_get(push, q, 0xc0 + 0x70, 0x0980a002); /* ROP, PIXELS */
 195       nvc0_hw_query_get(push, q, 0xc0 + 0x80, 0x0d808002); /* TCP, LAUNCHES */
 196       nvc0_hw_query_get(push, q, 0xc0 + 0x90, 0x0e809002); /* TEP, LAUNCHES */
 197       break;
 198    default:
 199       break;
 200    }
 201    hq->state = NVC0_HW_QUERY_STATE_ACTIVE;
 202    return ret;
 203 }
 204
 205 static void
 206 nvc0_hw_end_query(struct nvc0_context *nvc0, struct nvc0_query *q)
 207 {
 208    struct nouveau_pushbuf *push = nvc0->base.pushbuf;
 209    struct nvc0_hw_query *hq = nvc0_hw_query(q);
 210
 211    if (hq->funcs && hq->funcs->end_query) {
 212       hq->funcs->end_query(nvc0, hq);
 213       return;
 214    }
 215
 216    if (hq->state != NVC0_HW_QUERY_STATE_ACTIVE) {
 217       /* some queries don't require 'begin' to be called (e.g. GPU_FINISHED) */
 218       if (hq->rotate)
 219          nvc0_hw_query_rotate(nvc0, q);
 220       hq->sequence++;
 221    }
 222    hq->state = NVC0_HW_QUERY_STATE_ENDED;
 223
 224    switch (q->type) {
 225    case PIPE_QUERY_OCCLUSION_COUNTER:
 226    case PIPE_QUERY_OCCLUSION_PREDICATE:
 227       nvc0_hw_query_get(push, q, 0, 0x0100f002);
 228       if (--nvc0->screen->num_occlusion_queries_active == 0) {
 229          PUSH_SPACE(push, 1);
 230          IMMED_NVC0(push, NVC0_3D(SAMPLECNT_ENABLE), 0);
 231       }
 232       break;
 233    case PIPE_QUERY_PRIMITIVES_GENERATED:
 234       nvc0_hw_query_get(push, q, 0, 0x09005002 | (q->index << 5));
 235       break;
 236    case PIPE_QUERY_PRIMITIVES_EMITTED:
 237       nvc0_hw_query_get(push, q, 0, 0x05805002 | (q->index << 5));
 238       break;
 239    case PIPE_QUERY_SO_STATISTICS:
 240       nvc0_hw_query_get(push, q, 0x00, 0x05805002 | (q->index << 5));
 241       nvc0_hw_query_get(push, q, 0x10, 0x06805002 | (q->index << 5));
 242       break;
 243    case PIPE_QUERY_SO_OVERFLOW_PREDICATE:
 244       /* TODO: How do we sum over all streams for render condition ? */
 245       /* PRIMS_DROPPED doesn't write sequence, use a ZERO query to sync on */
 246       nvc0_hw_query_get(push, q, 0x00, 0x03005002 | (q->index << 5));
 247       nvc0_hw_query_get(push, q, 0x20, 0x00005002);
 248       break;
 249    case PIPE_QUERY_TIMESTAMP:
 250    case PIPE_QUERY_TIME_ELAPSED:
 251       nvc0_hw_query_get(push, q, 0, 0x00005002);
 252       break;
 253    case PIPE_QUERY_GPU_FINISHED:
 254       nvc0_hw_query_get(push, q, 0, 0x1000f010);
 255       break;
 256    case PIPE_QUERY_PIPELINE_STATISTICS:
 257       nvc0_hw_query_get(push, q, 0x00, 0x00801002); /* VFETCH, VERTICES */
 258       nvc0_hw_query_get(push, q, 0x10, 0x01801002); /* VFETCH, PRIMS */
 259       nvc0_hw_query_get(push, q, 0x20, 0x02802002); /* VP, LAUNCHES */
 260       nvc0_hw_query_get(push, q, 0x30, 0x03806002); /* GP, LAUNCHES */
 261       nvc0_hw_query_get(push, q, 0x40, 0x04806002); /* GP, PRIMS_OUT */
 262       nvc0_hw_query_get(push, q, 0x50, 0x07804002); /* RAST, PRIMS_IN */
 263       nvc0_hw_query_get(push, q, 0x60, 0x08804002); /* RAST, PRIMS_OUT */
 264       nvc0_hw_query_get(push, q, 0x70, 0x0980a002); /* ROP, PIXELS */
 265       nvc0_hw_query_get(push, q, 0x80, 0x0d808002); /* TCP, LAUNCHES */
 266       nvc0_hw_query_get(push, q, 0x90, 0x0e809002); /* TEP, LAUNCHES */
 267       break;
 268    case PIPE_QUERY_TIMESTAMP_DISJOINT:
 269       /* This query is not issued on GPU because disjoint is forced to false */
 270       hq->state = NVC0_HW_QUERY_STATE_READY;
 271       break;
 272    case NVC0_HW_QUERY_TFB_BUFFER_OFFSET:
 273       /* indexed by TFB buffer instead of by vertex stream */
 274       nvc0_hw_query_get(push, q, 0x00, 0x0d005002 | (q->index << 5));
 275       break;
 276    default:
 277       break;
 278    }
 279    if (hq->is64bit)
 280       nouveau_fence_ref(nvc0->screen->base.fence.current, &hq->fence);
 281 }
 282
 283 static boolean
 284 nvc0_hw_get_query_result(struct nvc0_context *nvc0, struct nvc0_query *q,
 285                          boolean wait, union pipe_query_result *result)
 286 {
 287    struct nvc0_hw_query *hq = nvc0_hw_query(q);
 288    uint64_t *res64 = (uint64_t*)result;
 289    uint32_t *res32 = (uint32_t*)result;
 290    uint8_t *res8 = (uint8_t*)result;
 291    uint64_t *data64 = (uint64_t *)hq->data;
 292    unsigned i;
 293
 294    if (hq->funcs && hq->funcs->get_query_result)
 295       return hq->funcs->get_query_result(nvc0, hq, wait, result);
 296
 297    if (hq->state != NVC0_HW_QUERY_STATE_READY)
 298       nvc0_hw_query_update(nvc0->screen->base.client, q);
 299
 300    if (hq->state != NVC0_HW_QUERY_STATE_READY) {
 301       if (!wait) {
 302          if (hq->state != NVC0_HW_QUERY_STATE_FLUSHED) {
 303             hq->state = NVC0_HW_QUERY_STATE_FLUSHED;
 304             /* flush for silly apps that spin on GL_QUERY_RESULT_AVAILABLE */
 305             PUSH_KICK(nvc0->base.pushbuf);
 306          }
 307          return false;
 308       }
 309       if (nouveau_bo_wait(hq->bo, NOUVEAU_BO_RD, nvc0->screen->base.client))
 310          return false;
 311       NOUVEAU_DRV_STAT(&nvc0->screen->base, query_sync_count, 1);
 312    }
 313    hq->state = NVC0_HW_QUERY_STATE_READY;
 314
 315    switch (q->type) {
 316    case PIPE_QUERY_GPU_FINISHED:
 317       res8[0] = true;
 318       break;
 319    case PIPE_QUERY_OCCLUSION_COUNTER: /* u32 sequence, u32 count, u64 time */
 320       res64[0] = hq->data[1] - hq->data[5];
 321       break;
 322    case PIPE_QUERY_OCCLUSION_PREDICATE:
 323       res8[0] = hq->data[1] != hq->data[5];
 324       break;
 325    case PIPE_QUERY_PRIMITIVES_GENERATED: /* u64 count, u64 time */
 326    case PIPE_QUERY_PRIMITIVES_EMITTED: /* u64 count, u64 time */
 327       res64[0] = data64[0] - data64[2];
 328       break;
 329    case PIPE_QUERY_SO_STATISTICS:
 330       res64[0] = data64[0] - data64[4];
 331       res64[1] = data64[2] - data64[6];
 332       break;
 333    case PIPE_QUERY_SO_OVERFLOW_PREDICATE:
 334       res8[0] = data64[0] != data64[2];
 335       break;
 336    case PIPE_QUERY_TIMESTAMP:
 337       res64[0] = data64[1];
 338       break;
 339    case PIPE_QUERY_TIMESTAMP_DISJOINT:
 340       res64[0] = 1000000000;
 341       res8[8] = false;
 342       break;
 343    case PIPE_QUERY_TIME_ELAPSED:
 344       res64[0] = data64[1] - data64[3];
 345       break;
 346    case PIPE_QUERY_PIPELINE_STATISTICS:
 347       for (i = 0; i < 10; ++i)
 348          res64[i] = data64[i * 2] - data64[24 + i * 2];
 349       break;
 350    case NVC0_HW_QUERY_TFB_BUFFER_OFFSET:
 351       res32[0] = hq->data[1];
 352       break;
 353    default:
 354       assert(0); /* can't happen, we don't create queries with invalid type */
 355       return false;
 356    }
 357
 358    return true;
 359 }
 360
 361 static void
 362 nvc0_hw_get_query_result_resource(struct nvc0_context *nvc0,
 363                                   struct nvc0_query *q,
 364                                   boolean wait,
 365                                   enum pipe_query_value_type result_type,
 366                                   int index,
 367                                   struct pipe_resource *resource,
 368                                   unsigned offset)
 369 {
 370    struct nouveau_pushbuf *push = nvc0->base.pushbuf;
 371    struct nvc0_hw_query *hq = nvc0_hw_query(q);
 372    struct nv04_resource *buf = nv04_resource(resource);
 373    unsigned qoffset = 0, stride;
 374
 375    assert(!hq->funcs || !hq->funcs->get_query_result);
 376
 377    if (index == -1) {
 378       /* TODO: Use a macro to write the availability of the query */
 379       if (hq->state != NVC0_HW_QUERY_STATE_READY)
 380          nvc0_hw_query_update(nvc0->screen->base.client, q);
 381       uint32_t ready[2] = {hq->state == NVC0_HW_QUERY_STATE_READY};
 382       nvc0->base.push_cb(&nvc0->base, buf, offset,
 383                          result_type >= PIPE_QUERY_TYPE_I64 ? 2 : 1,
 384                          ready);
 385       return;
 386    }
 387
 388    /* If the fence guarding this query has not been emitted, that makes a lot
 389     * of the following logic more complicated.
 390     */
 391    if (hq->is64bit && hq->fence->state < NOUVEAU_FENCE_STATE_EMITTED)
 392       nouveau_fence_emit(hq->fence);
 393
 394    /* We either need to compute a 32- or 64-bit difference between 2 values,
 395     * and then store the result as either a 32- or 64-bit value. As such let's
 396     * treat all inputs as 64-bit (and just push an extra 0 for the 32-bit
 397     * ones), and have one macro that clamps result to i32, u32, or just
 398     * outputs the difference (no need to worry about 64-bit clamping).
 399     */
 400    if (hq->state != NVC0_HW_QUERY_STATE_READY)
 401       nvc0_hw_query_update(nvc0->screen->base.client, q);
 402
 403    if (wait && hq->state != NVC0_HW_QUERY_STATE_READY)
 404       nvc0_hw_query_fifo_wait(nvc0, q);
 405
 406    nouveau_pushbuf_space(push, 32, 2, 0);
 407    PUSH_REFN (push, hq->bo, NOUVEAU_BO_GART | NOUVEAU_BO_RD);
 408    PUSH_REFN (push, buf->bo, buf->domain | NOUVEAU_BO_WR);
 409    BEGIN_1IC0(push, NVC0_3D(MACRO_QUERY_BUFFER_WRITE), 9);
 410    if (q->type == PIPE_QUERY_OCCLUSION_PREDICATE) /* XXX what if 64-bit? */
 411       PUSH_DATA(push, 0x00000001);
 412    else if (result_type == PIPE_QUERY_TYPE_I32)
 413       PUSH_DATA(push, 0x7fffffff);
 414    else if (result_type == PIPE_QUERY_TYPE_U32)
 415       PUSH_DATA(push, 0xffffffff);
 416    else
 417       PUSH_DATA(push, 0x00000000);
 418
 419    switch (q->type) {
 420    case PIPE_QUERY_SO_STATISTICS:
 421       stride = 2;
 422       break;
 423    case PIPE_QUERY_PIPELINE_STATISTICS:
 424       stride = 12;
 425       break;
 426    case PIPE_QUERY_TIME_ELAPSED:
 427    case PIPE_QUERY_TIMESTAMP:
 428       qoffset = 8;
 429       /* fallthrough */
 430    default:
 431       assert(index == 0);
 432       stride = 1;
 433       break;
 434    }
 435
 436    if (hq->is64bit || qoffset) {
 437       nouveau_pushbuf_data(push, hq->bo, hq->offset + qoffset + 16 * index,
 438                            8 | NVC0_IB_ENTRY_1_NO_PREFETCH);
 439       if (q->type == PIPE_QUERY_TIMESTAMP) {
 440          PUSH_DATA(push, 0);
 441          PUSH_DATA(push, 0);
 442       } else {
 443          nouveau_pushbuf_data(push, hq->bo, hq->offset + qoffset +
 444                               16 * (index + stride),
 445                               8 | NVC0_IB_ENTRY_1_NO_PREFETCH);
 446       }
 447    } else {
 448       nouveau_pushbuf_data(push, hq->bo, hq->offset + 4,
 449                            4 | NVC0_IB_ENTRY_1_NO_PREFETCH);
 450       PUSH_DATA(push, 0);
 451       nouveau_pushbuf_data(push, hq->bo, hq->offset + 16 + 4,
 452                            4 | NVC0_IB_ENTRY_1_NO_PREFETCH);
 453       PUSH_DATA(push, 0);
 454    }
 455
 456    if (wait || hq->state == NVC0_HW_QUERY_STATE_READY) {
 457       PUSH_DATA(push, 0);
 458       PUSH_DATA(push, 0);
 459    } else if (hq->is64bit) {
 460       PUSH_DATA(push, hq->fence->sequence);
 461       nouveau_pushbuf_data(push, nvc0->screen->fence.bo, 0,
 462                            4 | NVC0_IB_ENTRY_1_NO_PREFETCH);
 463    } else {
 464       PUSH_DATA(push, hq->sequence);
 465       nouveau_pushbuf_data(push, hq->bo, hq->offset,
 466                            4 | NVC0_IB_ENTRY_1_NO_PREFETCH);
 467    }
 468    PUSH_DATAh(push, buf->address + offset);
 469    PUSH_DATA (push, buf->address + offset);
 470
 471    if (buf->mm) {
 472       nouveau_fence_ref(nvc0->screen->base.fence.current, &buf->fence);
 473       nouveau_fence_ref(nvc0->screen->base.fence.current, &buf->fence_wr);
 474    }
 475 }
 476
 477 static const struct nvc0_query_funcs hw_query_funcs = {
 478    .destroy_query = nvc0_hw_destroy_query,
 479    .begin_query = nvc0_hw_begin_query,
 480    .end_query = nvc0_hw_end_query,
 481    .get_query_result = nvc0_hw_get_query_result,
 482    .get_query_result_resource = nvc0_hw_get_query_result_resource,
 483 };
 484
 485 struct nvc0_query *
 486 nvc0_hw_create_query(struct nvc0_context *nvc0, unsigned type, unsigned index)
 487 {
 488    struct nvc0_hw_query *hq;
 489    struct nvc0_query *q;
 490    unsigned space = NVC0_HW_QUERY_ALLOC_SPACE;
 491
 492    hq = nvc0_hw_sm_create_query(nvc0, type);
 493    if (hq) {
 494       hq->base.funcs = &hw_query_funcs;
 495       return (struct nvc0_query *)hq;
 496    }
 497
 498    hq = nvc0_hw_metric_create_query(nvc0, type);
 499    if (hq) {
 500       hq->base.funcs = &hw_query_funcs;
 501       return (struct nvc0_query *)hq;
 502    }
 503
 504    hq = CALLOC_STRUCT(nvc0_hw_query);
 505    if (!hq)
 506       return NULL;
 507
 508    q = &hq->base;
 509    q->funcs = &hw_query_funcs;
 510    q->type = type;
 511
 512    switch (q->type) {
 513    case PIPE_QUERY_OCCLUSION_COUNTER:
 514    case PIPE_QUERY_OCCLUSION_PREDICATE:
 515       hq->rotate = 32;
 516       space = NVC0_HW_QUERY_ALLOC_SPACE;
 517       break;
 518    case PIPE_QUERY_PIPELINE_STATISTICS:
 519       hq->is64bit = true;
 520       space = 512;
 521       break;
 522    case PIPE_QUERY_SO_STATISTICS:
 523    case PIPE_QUERY_SO_OVERFLOW_PREDICATE:
 524       hq->is64bit = true;
 525       space = 64;
 526       break;
 527    case PIPE_QUERY_PRIMITIVES_GENERATED:
 528    case PIPE_QUERY_PRIMITIVES_EMITTED:
 529       hq->is64bit = true;
 530       q->index = index;
 531       space = 32;
 532       break;
 533    case PIPE_QUERY_TIME_ELAPSED:
 534    case PIPE_QUERY_TIMESTAMP:
 535    case PIPE_QUERY_TIMESTAMP_DISJOINT:
 536    case PIPE_QUERY_GPU_FINISHED:
 537       space = 32;
 538       break;
 539    case NVC0_HW_QUERY_TFB_BUFFER_OFFSET:
 540       space = 16;
 541       break;
 542    default:
 543       debug_printf("invalid query type: %u\n", type);
 544       FREE(q);
 545       return NULL;
 546    }
 547
 548    if (!nvc0_hw_query_allocate(nvc0, q, space)) {
 549       FREE(hq);
 550       return NULL;
 551    }
 552
 553    if (hq->rotate) {
 554       /* we advance before query_begin ! */
 555       hq->offset -= hq->rotate;
 556       hq->data -= hq->rotate / sizeof(*hq->data);
 557    } else
 558    if (!hq->is64bit)
 559       hq->data[0] = 0; /* initialize sequence */
 560
 561    return q;
 562 }
 563
 564 int
 565 nvc0_hw_get_driver_query_info(struct nvc0_screen *screen, unsigned id,
 566                               struct pipe_driver_query_info *info)
 567 {
 568    int num_hw_sm_queries = 0, num_hw_metric_queries = 0;
 569
 570    num_hw_sm_queries = nvc0_hw_sm_get_driver_query_info(screen, 0, NULL);
 571    num_hw_metric_queries =
 572       nvc0_hw_metric_get_driver_query_info(screen, 0, NULL);
 573
 574    if (!info)
 575       return num_hw_sm_queries + num_hw_metric_queries;
 576
 577    if (id < num_hw_sm_queries)
 578       return nvc0_hw_sm_get_driver_query_info(screen, id, info);
 579
 580    return nvc0_hw_metric_get_driver_query_info(screen,
 581                                                id - num_hw_sm_queries, info);
 582 }
 583
 584 void
 585 nvc0_hw_query_pushbuf_submit(struct nouveau_pushbuf *push,
 586                              struct nvc0_query *q, unsigned result_offset)
 587 {
 588    struct nvc0_hw_query *hq = nvc0_hw_query(q);
 589
 590    PUSH_REFN(push, hq->bo, NOUVEAU_BO_RD | NOUVEAU_BO_GART);
 591    nouveau_pushbuf_data(push, hq->bo, hq->offset + result_offset, 4 |
 592                         NVC0_IB_ENTRY_1_NO_PREFETCH);
 593 }
 594
 595 void
 596 nvc0_hw_query_fifo_wait(struct nvc0_context *nvc0, struct nvc0_query *q)
 597 {
 598    struct nouveau_pushbuf *push = nvc0->base.pushbuf;
 599    struct nvc0_hw_query *hq = nvc0_hw_query(q);
 600    unsigned offset = hq->offset;
 601
 602    if (q->type == PIPE_QUERY_SO_OVERFLOW_PREDICATE) offset += 0x20;
 603
 604    PUSH_SPACE(push, 5);
 605    PUSH_REFN (push, hq->bo, NOUVEAU_BO_GART | NOUVEAU_BO_RD);
 606    BEGIN_NVC0(push, SUBC_3D(NV84_SUBCHAN_SEMAPHORE_ADDRESS_HIGH), 4);
 607    if (hq->is64bit) {
 608       PUSH_DATAh(push, nvc0->screen->fence.bo->offset);
 609       PUSH_DATA (push, nvc0->screen->fence.bo->offset);
 610       PUSH_DATA (push, hq->fence->sequence);
 611    } else {
 612       PUSH_DATAh(push, hq->bo->offset + offset);
 613       PUSH_DATA (push, hq->bo->offset + offset);
 614       PUSH_DATA (push, hq->sequence);
 615    }
 616    PUSH_DATA (push, (1 << 12) |
 617               NV84_SUBCHAN_SEMAPHORE_TRIGGER_ACQUIRE_EQUAL);
 618 }