src/gallium/drivers/nouveau/nvc0/nvc0_query_hw.c

   1 /*
   2  * Copyright 2011 Christoph Bumiller
   3  * Copyright 2015 Samuel Pitoiset
   4  *
   5  * Permission is hereby granted, free of charge, to any person obtaining a
   6  * copy of this software and associated documentation files (the "Software"),
   7  * to deal in the Software without restriction, including without limitation
   8  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   9  * and/or sell copies of the Software, and to permit persons to whom the
  10  * Software is furnished to do so, subject to the following conditions:
  11  *
  12  * The above copyright notice and this permission notice shall be included in
  13  * all copies or substantial portions of the Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
  19  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  20  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  21  * OTHER DEALINGS IN THE SOFTWARE.
  22  */
  23
  24 #define NVC0_PUSH_EXPLICIT_SPACE_CHECKING
  25
  26 #include "nvc0/nvc0_context.h"
  27 #include "nvc0/nvc0_query_hw.h"
  28 #include "nvc0/nvc0_query_hw_metric.h"
  29 #include "nvc0/nvc0_query_hw_sm.h"
  30
  31 #define NVC0_HW_QUERY_STATE_READY   0
  32 #define NVC0_HW_QUERY_STATE_ACTIVE  1
  33 #define NVC0_HW_QUERY_STATE_ENDED   2
  34 #define NVC0_HW_QUERY_STATE_FLUSHED 3
  35
  36 #define NVC0_HW_QUERY_ALLOC_SPACE 256
  37
  38 bool
  39 nvc0_hw_query_allocate(struct nvc0_context *nvc0, struct nvc0_query *q,
  40                        int size)
  41 {
  42    struct nvc0_hw_query *hq = nvc0_hw_query(q);
  43    struct nvc0_screen *screen = nvc0->screen;
  44    int ret;
  45
  46    if (hq->bo) {
  47       nouveau_bo_ref(NULL, &hq->bo);
  48       if (hq->mm) {
  49          if (hq->state == NVC0_HW_QUERY_STATE_READY)
  50             nouveau_mm_free(hq->mm);
  51          else
  52             nouveau_fence_work(screen->base.fence.current,
  53                                nouveau_mm_free_work, hq->mm);
  54       }
  55    }
  56    if (size) {
  57       hq->mm = nouveau_mm_allocate(screen->base.mm_GART, size, &hq->bo,
  58                                    &hq->base_offset);
  59       if (!hq->bo)
  60          return false;
  61       hq->offset = hq->base_offset;
  62
  63       ret = nouveau_bo_map(hq->bo, 0, screen->base.client);
  64       if (ret) {
  65          nvc0_hw_query_allocate(nvc0, q, 0);
  66          return false;
  67       }
  68       hq->data = (uint32_t *)((uint8_t *)hq->bo->map + hq->base_offset);
  69    }
  70    return true;
  71 }
  72
  73 static void
  74 nvc0_hw_query_get(struct nouveau_pushbuf *push, struct nvc0_query *q,
  75                   unsigned offset, uint32_t get)
  76 {
  77    struct nvc0_hw_query *hq = nvc0_hw_query(q);
  78
  79    offset += hq->offset;
  80
  81    PUSH_SPACE(push, 5);
  82    PUSH_REFN (push, hq->bo, NOUVEAU_BO_GART | NOUVEAU_BO_WR);
  83    BEGIN_NVC0(push, NVC0_3D(QUERY_ADDRESS_HIGH), 4);
  84    PUSH_DATAh(push, hq->bo->offset + offset);
  85    PUSH_DATA (push, hq->bo->offset + offset);
  86    PUSH_DATA (push, hq->sequence);
  87    PUSH_DATA (push, get);
  88 }
  89
  90 static void
  91 nvc0_hw_query_rotate(struct nvc0_context *nvc0, struct nvc0_query *q)
  92 {
  93    struct nvc0_hw_query *hq = nvc0_hw_query(q);
  94
  95    hq->offset += hq->rotate;
  96    hq->data += hq->rotate / sizeof(*hq->data);
  97    if (hq->offset - hq->base_offset == NVC0_HW_QUERY_ALLOC_SPACE)
  98       nvc0_hw_query_allocate(nvc0, q, NVC0_HW_QUERY_ALLOC_SPACE);
  99 }
 100
 101 static inline void
 102 nvc0_hw_query_update(struct nouveau_client *cli, struct nvc0_query *q)
 103 {
 104    struct nvc0_hw_query *hq = nvc0_hw_query(q);
 105
 106    if (hq->is64bit) {
 107       if (nouveau_fence_signalled(hq->fence))
 108          hq->state = NVC0_HW_QUERY_STATE_READY;
 109    } else {
 110       if (hq->data[0] == hq->sequence)
 111          hq->state = NVC0_HW_QUERY_STATE_READY;
 112    }
 113 }
 114
 115 static void
 116 nvc0_hw_destroy_query(struct nvc0_context *nvc0, struct nvc0_query *q)
 117 {
 118    struct nvc0_hw_query *hq = nvc0_hw_query(q);
 119
 120    if (hq->funcs && hq->funcs->destroy_query) {
 121       hq->funcs->destroy_query(nvc0, hq);
 122       return;
 123    }
 124
 125    nvc0_hw_query_allocate(nvc0, q, 0);
 126    nouveau_fence_ref(NULL, &hq->fence);
 127    FREE(hq);
 128 }
 129
 130 static boolean
 131 nvc0_hw_begin_query(struct nvc0_context *nvc0, struct nvc0_query *q)
 132 {
 133    struct nouveau_pushbuf *push = nvc0->base.pushbuf;
 134    struct nvc0_hw_query *hq = nvc0_hw_query(q);
 135    bool ret = true;
 136
 137    if (hq->funcs && hq->funcs->begin_query)
 138       return hq->funcs->begin_query(nvc0, hq);
 139
 140    /* For occlusion queries we have to change the storage, because a previous
 141     * query might set the initial render conition to false even *after* we re-
 142     * initialized it to true.
 143     */
 144    if (hq->rotate) {
 145       nvc0_hw_query_rotate(nvc0, q);
 146
 147       /* XXX: can we do this with the GPU, and sync with respect to a previous
 148        *  query ?
 149        */
 150       hq->data[0] = hq->sequence; /* initialize sequence */
 151       hq->data[1] = 1; /* initial render condition = true */
 152       hq->data[4] = hq->sequence + 1; /* for comparison COND_MODE */
 153       hq->data[5] = 0;
 154    }
 155    hq->sequence++;
 156
 157    switch (q->type) {
 158    case PIPE_QUERY_OCCLUSION_COUNTER:
 159    case PIPE_QUERY_OCCLUSION_PREDICATE:
 160    case PIPE_QUERY_OCCLUSION_PREDICATE_CONSERVATIVE:
 161       hq->nesting = nvc0->screen->num_occlusion_queries_active++;
 162       if (hq->nesting) {
 163          nvc0_hw_query_get(push, q, 0x10, 0x0100f002);
 164       } else {
 165          PUSH_SPACE(push, 3);
 166          BEGIN_NVC0(push, NVC0_3D(COUNTER_RESET), 1);
 167          PUSH_DATA (push, NVC0_3D_COUNTER_RESET_SAMPLECNT);
 168          IMMED_NVC0(push, NVC0_3D(SAMPLECNT_ENABLE), 1);
 169       }
 170       break;
 171    case PIPE_QUERY_PRIMITIVES_GENERATED:
 172       nvc0_hw_query_get(push, q, 0x10, 0x09005002 | (q->index << 5));
 173       break;
 174    case PIPE_QUERY_PRIMITIVES_EMITTED:
 175       nvc0_hw_query_get(push, q, 0x10, 0x05805002 | (q->index << 5));
 176       break;
 177    case PIPE_QUERY_SO_STATISTICS:
 178       nvc0_hw_query_get(push, q, 0x20, 0x05805002 | (q->index << 5));
 179       nvc0_hw_query_get(push, q, 0x30, 0x06805002 | (q->index << 5));
 180       break;
 181    case PIPE_QUERY_SO_OVERFLOW_PREDICATE:
 182       nvc0_hw_query_get(push, q, 0x10, 0x03005002 | (q->index << 5));
 183       break;
 184    case PIPE_QUERY_TIME_ELAPSED:
 185       nvc0_hw_query_get(push, q, 0x10, 0x00005002);
 186       break;
 187    case PIPE_QUERY_PIPELINE_STATISTICS:
 188       nvc0_hw_query_get(push, q, 0xc0 + 0x00, 0x00801002); /* VFETCH, VERTICES */
 189       nvc0_hw_query_get(push, q, 0xc0 + 0x10, 0x01801002); /* VFETCH, PRIMS */
 190       nvc0_hw_query_get(push, q, 0xc0 + 0x20, 0x02802002); /* VP, LAUNCHES */
 191       nvc0_hw_query_get(push, q, 0xc0 + 0x30, 0x03806002); /* GP, LAUNCHES */
 192       nvc0_hw_query_get(push, q, 0xc0 + 0x40, 0x04806002); /* GP, PRIMS_OUT */
 193       nvc0_hw_query_get(push, q, 0xc0 + 0x50, 0x07804002); /* RAST, PRIMS_IN */
 194       nvc0_hw_query_get(push, q, 0xc0 + 0x60, 0x08804002); /* RAST, PRIMS_OUT */
 195       nvc0_hw_query_get(push, q, 0xc0 + 0x70, 0x0980a002); /* ROP, PIXELS */
 196       nvc0_hw_query_get(push, q, 0xc0 + 0x80, 0x0d808002); /* TCP, LAUNCHES */
 197       nvc0_hw_query_get(push, q, 0xc0 + 0x90, 0x0e809002); /* TEP, LAUNCHES */
 198       break;
 199    default:
 200       break;
 201    }
 202    hq->state = NVC0_HW_QUERY_STATE_ACTIVE;
 203    return ret;
 204 }
 205
 206 static void
 207 nvc0_hw_end_query(struct nvc0_context *nvc0, struct nvc0_query *q)
 208 {
 209    struct nouveau_pushbuf *push = nvc0->base.pushbuf;
 210    struct nvc0_hw_query *hq = nvc0_hw_query(q);
 211
 212    if (hq->funcs && hq->funcs->end_query) {
 213       hq->funcs->end_query(nvc0, hq);
 214       return;
 215    }
 216
 217    if (hq->state != NVC0_HW_QUERY_STATE_ACTIVE) {
 218       /* some queries don't require 'begin' to be called (e.g. GPU_FINISHED) */
 219       if (hq->rotate)
 220          nvc0_hw_query_rotate(nvc0, q);
 221       hq->sequence++;
 222    }
 223    hq->state = NVC0_HW_QUERY_STATE_ENDED;
 224
 225    switch (q->type) {
 226    case PIPE_QUERY_OCCLUSION_COUNTER:
 227    case PIPE_QUERY_OCCLUSION_PREDICATE:
 228    case PIPE_QUERY_OCCLUSION_PREDICATE_CONSERVATIVE:
 229       nvc0_hw_query_get(push, q, 0, 0x0100f002);
 230       if (--nvc0->screen->num_occlusion_queries_active == 0) {
 231          PUSH_SPACE(push, 1);
 232          IMMED_NVC0(push, NVC0_3D(SAMPLECNT_ENABLE), 0);
 233       }
 234       break;
 235    case PIPE_QUERY_PRIMITIVES_GENERATED:
 236       nvc0_hw_query_get(push, q, 0, 0x09005002 | (q->index << 5));
 237       break;
 238    case PIPE_QUERY_PRIMITIVES_EMITTED:
 239       nvc0_hw_query_get(push, q, 0, 0x05805002 | (q->index << 5));
 240       break;
 241    case PIPE_QUERY_SO_STATISTICS:
 242       nvc0_hw_query_get(push, q, 0x00, 0x05805002 | (q->index << 5));
 243       nvc0_hw_query_get(push, q, 0x10, 0x06805002 | (q->index << 5));
 244       break;
 245    case PIPE_QUERY_SO_OVERFLOW_PREDICATE:
 246       /* TODO: How do we sum over all streams for render condition ? */
 247       /* PRIMS_DROPPED doesn't write sequence, use a ZERO query to sync on */
 248       nvc0_hw_query_get(push, q, 0x00, 0x03005002 | (q->index << 5));
 249       nvc0_hw_query_get(push, q, 0x20, 0x00005002);
 250       break;
 251    case PIPE_QUERY_TIMESTAMP:
 252    case PIPE_QUERY_TIME_ELAPSED:
 253       nvc0_hw_query_get(push, q, 0, 0x00005002);
 254       break;
 255    case PIPE_QUERY_GPU_FINISHED:
 256       nvc0_hw_query_get(push, q, 0, 0x1000f010);
 257       break;
 258    case PIPE_QUERY_PIPELINE_STATISTICS:
 259       nvc0_hw_query_get(push, q, 0x00, 0x00801002); /* VFETCH, VERTICES */
 260       nvc0_hw_query_get(push, q, 0x10, 0x01801002); /* VFETCH, PRIMS */
 261       nvc0_hw_query_get(push, q, 0x20, 0x02802002); /* VP, LAUNCHES */
 262       nvc0_hw_query_get(push, q, 0x30, 0x03806002); /* GP, LAUNCHES */
 263       nvc0_hw_query_get(push, q, 0x40, 0x04806002); /* GP, PRIMS_OUT */
 264       nvc0_hw_query_get(push, q, 0x50, 0x07804002); /* RAST, PRIMS_IN */
 265       nvc0_hw_query_get(push, q, 0x60, 0x08804002); /* RAST, PRIMS_OUT */
 266       nvc0_hw_query_get(push, q, 0x70, 0x0980a002); /* ROP, PIXELS */
 267       nvc0_hw_query_get(push, q, 0x80, 0x0d808002); /* TCP, LAUNCHES */
 268       nvc0_hw_query_get(push, q, 0x90, 0x0e809002); /* TEP, LAUNCHES */
 269       break;
 270    case PIPE_QUERY_TIMESTAMP_DISJOINT:
 271       /* This query is not issued on GPU because disjoint is forced to false */
 272       hq->state = NVC0_HW_QUERY_STATE_READY;
 273       break;
 274    case NVC0_HW_QUERY_TFB_BUFFER_OFFSET:
 275       /* indexed by TFB buffer instead of by vertex stream */
 276       nvc0_hw_query_get(push, q, 0x00, 0x0d005002 | (q->index << 5));
 277       break;
 278    default:
 279       break;
 280    }
 281    if (hq->is64bit)
 282       nouveau_fence_ref(nvc0->screen->base.fence.current, &hq->fence);
 283 }
 284
 285 static boolean
 286 nvc0_hw_get_query_result(struct nvc0_context *nvc0, struct nvc0_query *q,
 287                          boolean wait, union pipe_query_result *result)
 288 {
 289    struct nvc0_hw_query *hq = nvc0_hw_query(q);
 290    uint64_t *res64 = (uint64_t*)result;
 291    uint32_t *res32 = (uint32_t*)result;
 292    uint8_t *res8 = (uint8_t*)result;
 293    uint64_t *data64 = (uint64_t *)hq->data;
 294    unsigned i;
 295
 296    if (hq->funcs && hq->funcs->get_query_result)
 297       return hq->funcs->get_query_result(nvc0, hq, wait, result);
 298
 299    if (hq->state != NVC0_HW_QUERY_STATE_READY)
 300       nvc0_hw_query_update(nvc0->screen->base.client, q);
 301
 302    if (hq->state != NVC0_HW_QUERY_STATE_READY) {
 303       if (!wait) {
 304          if (hq->state != NVC0_HW_QUERY_STATE_FLUSHED) {
 305             hq->state = NVC0_HW_QUERY_STATE_FLUSHED;
 306             /* flush for silly apps that spin on GL_QUERY_RESULT_AVAILABLE */
 307             PUSH_KICK(nvc0->base.pushbuf);
 308          }
 309          return false;
 310       }
 311       if (nouveau_bo_wait(hq->bo, NOUVEAU_BO_RD, nvc0->screen->base.client))
 312          return false;
 313       NOUVEAU_DRV_STAT(&nvc0->screen->base, query_sync_count, 1);
 314    }
 315    hq->state = NVC0_HW_QUERY_STATE_READY;
 316
 317    switch (q->type) {
 318    case PIPE_QUERY_GPU_FINISHED:
 319       res8[0] = true;
 320       break;
 321    case PIPE_QUERY_OCCLUSION_COUNTER: /* u32 sequence, u32 count, u64 time */
 322       res64[0] = hq->data[1] - hq->data[5];
 323       break;
 324    case PIPE_QUERY_OCCLUSION_PREDICATE:
 325    case PIPE_QUERY_OCCLUSION_PREDICATE_CONSERVATIVE:
 326       res8[0] = hq->data[1] != hq->data[5];
 327       break;
 328    case PIPE_QUERY_PRIMITIVES_GENERATED: /* u64 count, u64 time */
 329    case PIPE_QUERY_PRIMITIVES_EMITTED: /* u64 count, u64 time */
 330       res64[0] = data64[0] - data64[2];
 331       break;
 332    case PIPE_QUERY_SO_STATISTICS:
 333       res64[0] = data64[0] - data64[4];
 334       res64[1] = data64[2] - data64[6];
 335       break;
 336    case PIPE_QUERY_SO_OVERFLOW_PREDICATE:
 337       res8[0] = data64[0] != data64[2];
 338       break;
 339    case PIPE_QUERY_TIMESTAMP:
 340       res64[0] = data64[1];
 341       break;
 342    case PIPE_QUERY_TIMESTAMP_DISJOINT:
 343       res64[0] = 1000000000;
 344       res8[8] = false;
 345       break;
 346    case PIPE_QUERY_TIME_ELAPSED:
 347       res64[0] = data64[1] - data64[3];
 348       break;
 349    case PIPE_QUERY_PIPELINE_STATISTICS:
 350       for (i = 0; i < 10; ++i)
 351          res64[i] = data64[i * 2] - data64[24 + i * 2];
 352       result->pipeline_statistics.cs_invocations = 0;
 353       break;
 354    case NVC0_HW_QUERY_TFB_BUFFER_OFFSET:
 355       res32[0] = hq->data[1];
 356       break;
 357    default:
 358       assert(0); /* can't happen, we don't create queries with invalid type */
 359       return false;
 360    }
 361
 362    return true;
 363 }
 364
 365 static void
 366 nvc0_hw_get_query_result_resource(struct nvc0_context *nvc0,
 367                                   struct nvc0_query *q,
 368                                   boolean wait,
 369                                   enum pipe_query_value_type result_type,
 370                                   int index,
 371                                   struct pipe_resource *resource,
 372                                   unsigned offset)
 373 {
 374    struct nouveau_pushbuf *push = nvc0->base.pushbuf;
 375    struct nvc0_hw_query *hq = nvc0_hw_query(q);
 376    struct nv04_resource *buf = nv04_resource(resource);
 377    unsigned qoffset = 0, stride;
 378
 379    assert(!hq->funcs || !hq->funcs->get_query_result);
 380
 381    if (index == -1) {
 382       /* TODO: Use a macro to write the availability of the query */
 383       if (hq->state != NVC0_HW_QUERY_STATE_READY)
 384          nvc0_hw_query_update(nvc0->screen->base.client, q);
 385       uint32_t ready[2] = {hq->state == NVC0_HW_QUERY_STATE_READY};
 386       nvc0->base.push_cb(&nvc0->base, buf, offset,
 387                          result_type >= PIPE_QUERY_TYPE_I64 ? 2 : 1,
 388                          ready);
 389
 390       util_range_add(&buf->valid_buffer_range, offset,
 391                      offset + (result_type >= PIPE_QUERY_TYPE_I64 ? 8 : 4));
 392
 393       nvc0_resource_validate(buf, NOUVEAU_BO_WR);
 394
 395       return;
 396    }
 397
 398    /* If the fence guarding this query has not been emitted, that makes a lot
 399     * of the following logic more complicated.
 400     */
 401    if (hq->is64bit && hq->fence->state < NOUVEAU_FENCE_STATE_EMITTED)
 402       nouveau_fence_emit(hq->fence);
 403
 404    /* We either need to compute a 32- or 64-bit difference between 2 values,
 405     * and then store the result as either a 32- or 64-bit value. As such let's
 406     * treat all inputs as 64-bit (and just push an extra 0 for the 32-bit
 407     * ones), and have one macro that clamps result to i32, u32, or just
 408     * outputs the difference (no need to worry about 64-bit clamping).
 409     */
 410    if (hq->state != NVC0_HW_QUERY_STATE_READY)
 411       nvc0_hw_query_update(nvc0->screen->base.client, q);
 412
 413    if (wait && hq->state != NVC0_HW_QUERY_STATE_READY)
 414       nvc0_hw_query_fifo_wait(nvc0, q);
 415
 416    nouveau_pushbuf_space(push, 32, 2, 0);
 417    PUSH_REFN (push, hq->bo, NOUVEAU_BO_GART | NOUVEAU_BO_RD);
 418    PUSH_REFN (push, buf->bo, buf->domain | NOUVEAU_BO_WR);
 419    BEGIN_1IC0(push, NVC0_3D(MACRO_QUERY_BUFFER_WRITE), 9);
 420    if (q->type == PIPE_QUERY_OCCLUSION_PREDICATE ||
 421        q->type == PIPE_QUERY_OCCLUSION_PREDICATE_CONSERVATIVE) /* XXX what if 64-bit? */
 422       PUSH_DATA(push, 0x00000001);
 423    else if (result_type == PIPE_QUERY_TYPE_I32)
 424       PUSH_DATA(push, 0x7fffffff);
 425    else if (result_type == PIPE_QUERY_TYPE_U32)
 426       PUSH_DATA(push, 0xffffffff);
 427    else
 428       PUSH_DATA(push, 0x00000000);
 429
 430    switch (q->type) {
 431    case PIPE_QUERY_SO_STATISTICS:
 432       stride = 2;
 433       break;
 434    case PIPE_QUERY_PIPELINE_STATISTICS:
 435       stride = 12;
 436       break;
 437    case PIPE_QUERY_TIME_ELAPSED:
 438    case PIPE_QUERY_TIMESTAMP:
 439       qoffset = 8;
 440       /* fallthrough */
 441    default:
 442       assert(index == 0);
 443       stride = 1;
 444       break;
 445    }
 446
 447    if (hq->is64bit || qoffset) {
 448       nouveau_pushbuf_data(push, hq->bo, hq->offset + qoffset + 16 * index,
 449                            8 | NVC0_IB_ENTRY_1_NO_PREFETCH);
 450       if (q->type == PIPE_QUERY_TIMESTAMP) {
 451          PUSH_DATA(push, 0);
 452          PUSH_DATA(push, 0);
 453       } else {
 454          nouveau_pushbuf_data(push, hq->bo, hq->offset + qoffset +
 455                               16 * (index + stride),
 456                               8 | NVC0_IB_ENTRY_1_NO_PREFETCH);
 457       }
 458    } else {
 459       nouveau_pushbuf_data(push, hq->bo, hq->offset + 4,
 460                            4 | NVC0_IB_ENTRY_1_NO_PREFETCH);
 461       PUSH_DATA(push, 0);
 462       nouveau_pushbuf_data(push, hq->bo, hq->offset + 16 + 4,
 463                            4 | NVC0_IB_ENTRY_1_NO_PREFETCH);
 464       PUSH_DATA(push, 0);
 465    }
 466
 467    if (wait || hq->state == NVC0_HW_QUERY_STATE_READY) {
 468       PUSH_DATA(push, 0);
 469       PUSH_DATA(push, 0);
 470    } else if (hq->is64bit) {
 471       PUSH_DATA(push, hq->fence->sequence);
 472       nouveau_pushbuf_data(push, nvc0->screen->fence.bo, 0,
 473                            4 | NVC0_IB_ENTRY_1_NO_PREFETCH);
 474    } else {
 475       PUSH_DATA(push, hq->sequence);
 476       nouveau_pushbuf_data(push, hq->bo, hq->offset,
 477                            4 | NVC0_IB_ENTRY_1_NO_PREFETCH);
 478    }
 479    PUSH_DATAh(push, buf->address + offset);
 480    PUSH_DATA (push, buf->address + offset);
 481
 482    util_range_add(&buf->valid_buffer_range, offset,
 483                   offset + (result_type >= PIPE_QUERY_TYPE_I64 ? 8 : 4));
 484
 485    nvc0_resource_validate(buf, NOUVEAU_BO_WR);
 486 }
 487
 488 static const struct nvc0_query_funcs hw_query_funcs = {
 489    .destroy_query = nvc0_hw_destroy_query,
 490    .begin_query = nvc0_hw_begin_query,
 491    .end_query = nvc0_hw_end_query,
 492    .get_query_result = nvc0_hw_get_query_result,
 493    .get_query_result_resource = nvc0_hw_get_query_result_resource,
 494 };
 495
 496 struct nvc0_query *
 497 nvc0_hw_create_query(struct nvc0_context *nvc0, unsigned type, unsigned index)
 498 {
 499    struct nvc0_hw_query *hq;
 500    struct nvc0_query *q;
 501    unsigned space = NVC0_HW_QUERY_ALLOC_SPACE;
 502
 503    hq = nvc0_hw_sm_create_query(nvc0, type);
 504    if (hq) {
 505       hq->base.funcs = &hw_query_funcs;
 506       return (struct nvc0_query *)hq;
 507    }
 508
 509    hq = nvc0_hw_metric_create_query(nvc0, type);
 510    if (hq) {
 511       hq->base.funcs = &hw_query_funcs;
 512       return (struct nvc0_query *)hq;
 513    }
 514
 515    hq = CALLOC_STRUCT(nvc0_hw_query);
 516    if (!hq)
 517       return NULL;
 518
 519    q = &hq->base;
 520    q->funcs = &hw_query_funcs;
 521    q->type = type;
 522
 523    switch (q->type) {
 524    case PIPE_QUERY_OCCLUSION_COUNTER:
 525    case PIPE_QUERY_OCCLUSION_PREDICATE:
 526    case PIPE_QUERY_OCCLUSION_PREDICATE_CONSERVATIVE:
 527       hq->rotate = 32;
 528       space = NVC0_HW_QUERY_ALLOC_SPACE;
 529       break;
 530    case PIPE_QUERY_PIPELINE_STATISTICS:
 531       hq->is64bit = true;
 532       space = 512;
 533       break;
 534    case PIPE_QUERY_SO_STATISTICS:
 535    case PIPE_QUERY_SO_OVERFLOW_PREDICATE:
 536       hq->is64bit = true;
 537       space = 64;
 538       break;
 539    case PIPE_QUERY_PRIMITIVES_GENERATED:
 540    case PIPE_QUERY_PRIMITIVES_EMITTED:
 541       hq->is64bit = true;
 542       q->index = index;
 543       space = 32;
 544       break;
 545    case PIPE_QUERY_TIME_ELAPSED:
 546    case PIPE_QUERY_TIMESTAMP:
 547    case PIPE_QUERY_TIMESTAMP_DISJOINT:
 548    case PIPE_QUERY_GPU_FINISHED:
 549       space = 32;
 550       break;
 551    case NVC0_HW_QUERY_TFB_BUFFER_OFFSET:
 552       space = 16;
 553       break;
 554    default:
 555       debug_printf("invalid query type: %u\n", type);
 556       FREE(q);
 557       return NULL;
 558    }
 559
 560    if (!nvc0_hw_query_allocate(nvc0, q, space)) {
 561       FREE(hq);
 562       return NULL;
 563    }
 564
 565    if (hq->rotate) {
 566       /* we advance before query_begin ! */
 567       hq->offset -= hq->rotate;
 568       hq->data -= hq->rotate / sizeof(*hq->data);
 569    } else
 570    if (!hq->is64bit)
 571       hq->data[0] = 0; /* initialize sequence */
 572
 573    return q;
 574 }
 575
 576 int
 577 nvc0_hw_get_driver_query_info(struct nvc0_screen *screen, unsigned id,
 578                               struct pipe_driver_query_info *info)
 579 {
 580    int num_hw_sm_queries = 0, num_hw_metric_queries = 0;
 581
 582    num_hw_sm_queries = nvc0_hw_sm_get_driver_query_info(screen, 0, NULL);
 583    num_hw_metric_queries =
 584       nvc0_hw_metric_get_driver_query_info(screen, 0, NULL);
 585
 586    if (!info)
 587       return num_hw_sm_queries + num_hw_metric_queries;
 588
 589    if (id < num_hw_sm_queries)
 590       return nvc0_hw_sm_get_driver_query_info(screen, id, info);
 591
 592    return nvc0_hw_metric_get_driver_query_info(screen,
 593                                                id - num_hw_sm_queries, info);
 594 }
 595
 596 void
 597 nvc0_hw_query_pushbuf_submit(struct nouveau_pushbuf *push,
 598                              struct nvc0_query *q, unsigned result_offset)
 599 {
 600    struct nvc0_hw_query *hq = nvc0_hw_query(q);
 601
 602    PUSH_REFN(push, hq->bo, NOUVEAU_BO_RD | NOUVEAU_BO_GART);
 603    nouveau_pushbuf_data(push, hq->bo, hq->offset + result_offset, 4 |
 604                         NVC0_IB_ENTRY_1_NO_PREFETCH);
 605 }
 606
 607 void
 608 nvc0_hw_query_fifo_wait(struct nvc0_context *nvc0, struct nvc0_query *q)
 609 {
 610    struct nouveau_pushbuf *push = nvc0->base.pushbuf;
 611    struct nvc0_hw_query *hq = nvc0_hw_query(q);
 612    unsigned offset = hq->offset;
 613
 614    /* ensure the query's fence has been emitted */
 615    if (hq->is64bit && hq->fence->state < NOUVEAU_FENCE_STATE_EMITTED)
 616       nouveau_fence_emit(hq->fence);
 617
 618    if (q->type == PIPE_QUERY_SO_OVERFLOW_PREDICATE) offset += 0x20;
 619
 620    PUSH_SPACE(push, 5);
 621    PUSH_REFN (push, hq->bo, NOUVEAU_BO_GART | NOUVEAU_BO_RD);
 622    BEGIN_NVC0(push, SUBC_3D(NV84_SUBCHAN_SEMAPHORE_ADDRESS_HIGH), 4);
 623    if (hq->is64bit) {
 624       PUSH_DATAh(push, nvc0->screen->fence.bo->offset);
 625       PUSH_DATA (push, nvc0->screen->fence.bo->offset);
 626       PUSH_DATA (push, hq->fence->sequence);
 627    } else {
 628       PUSH_DATAh(push, hq->bo->offset + offset);
 629       PUSH_DATA (push, hq->bo->offset + offset);
 630       PUSH_DATA (push, hq->sequence);
 631    }
 632    PUSH_DATA (push, (1 << 12) |
 633               NV84_SUBCHAN_SEMAPHORE_TRIGGER_ACQUIRE_EQUAL);
 634 }