src/gallium/drivers/nouveau/nv50/nv50_query_hw.c

   1 /*
   2  * Copyright 2011 Christoph Bumiller
   3  * Copyright 2015 Samuel Pitoiset
   4  *
   5  * Permission is hereby granted, free of charge, to any person obtaining a
   6  * copy of this software and associated documentation files (the "Software"),
   7  * to deal in the Software without restriction, including without limitation
   8  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   9  * and/or sell copies of the Software, and to permit persons to whom the
  10  * Software is furnished to do so, subject to the following conditions:
  11  *
  12  * The above copyright notice and this permission notice shall be included in
  13  * all copies or substantial portions of the Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
  19  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  20  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  21  * OTHER DEALINGS IN THE SOFTWARE.
  22  */
  23
  24 #define NV50_PUSH_EXPLICIT_SPACE_CHECKING
  25
  26 #include "nv50/nv50_context.h"
  27 #include "nv50/nv50_query_hw.h"
  28 #include "nv50/nv50_query_hw_metric.h"
  29 #include "nv50/nv50_query_hw_sm.h"
  30 #include "nv_object.xml.h"
  31
  32 #define NV50_HW_QUERY_STATE_READY   0
  33 #define NV50_HW_QUERY_STATE_ACTIVE  1
  34 #define NV50_HW_QUERY_STATE_ENDED   2
  35 #define NV50_HW_QUERY_STATE_FLUSHED 3
  36
  37 /* XXX: Nested queries, and simultaneous queries on multiple gallium contexts
  38  * (since we use only a single GPU channel per screen) will not work properly.
  39  *
  40  * The first is not that big of an issue because OpenGL does not allow nested
  41  * queries anyway.
  42  */
  43
  44 #define NV50_HW_QUERY_ALLOC_SPACE 256
  45
  46 bool
  47 nv50_hw_query_allocate(struct nv50_context *nv50, struct nv50_query *q,
  48                        int size)
  49 {
  50    struct nv50_screen *screen = nv50->screen;
  51    struct nv50_hw_query *hq = nv50_hw_query(q);
  52    int ret;
  53
  54    if (hq->bo) {
  55       nouveau_bo_ref(NULL, &hq->bo);
  56       if (hq->mm) {
  57          if (hq->state == NV50_HW_QUERY_STATE_READY)
  58             nouveau_mm_free(hq->mm);
  59          else
  60             nouveau_fence_work(screen->base.fence.current,
  61                                nouveau_mm_free_work, hq->mm);
  62       }
  63    }
  64    if (size) {
  65       hq->mm = nouveau_mm_allocate(screen->base.mm_GART, size,
  66                                    &hq->bo, &hq->base_offset);
  67       if (!hq->bo)
  68          return false;
  69       hq->offset = hq->base_offset;
  70
  71       ret = nouveau_bo_map(hq->bo, 0, screen->base.client);
  72       if (ret) {
  73          nv50_hw_query_allocate(nv50, q, 0);
  74          return false;
  75       }
  76       hq->data = (uint32_t *)((uint8_t *)hq->bo->map + hq->base_offset);
  77    }
  78    return true;
  79 }
  80
  81 static void
  82 nv50_hw_query_get(struct nouveau_pushbuf *push, struct nv50_query *q,
  83                unsigned offset, uint32_t get)
  84 {
  85    struct nv50_hw_query *hq = nv50_hw_query(q);
  86
  87    offset += hq->offset;
  88
  89    PUSH_SPACE(push, 5);
  90    PUSH_REFN (push, hq->bo, NOUVEAU_BO_GART | NOUVEAU_BO_WR);
  91    BEGIN_NV04(push, NV50_3D(QUERY_ADDRESS_HIGH), 4);
  92    PUSH_DATAh(push, hq->bo->offset + offset);
  93    PUSH_DATA (push, hq->bo->offset + offset);
  94    PUSH_DATA (push, hq->sequence);
  95    PUSH_DATA (push, get);
  96 }
  97
  98 static inline void
  99 nv50_hw_query_update(struct nv50_query *q)
 100 {
 101    struct nv50_hw_query *hq = nv50_hw_query(q);
 102
 103    if (hq->is64bit) {
 104       if (nouveau_fence_signalled(hq->fence))
 105          hq->state = NV50_HW_QUERY_STATE_READY;
 106    } else {
 107       if (hq->data[0] == hq->sequence)
 108          hq->state = NV50_HW_QUERY_STATE_READY;
 109    }
 110 }
 111
 112 static void
 113 nv50_hw_destroy_query(struct nv50_context *nv50, struct nv50_query *q)
 114 {
 115    struct nv50_hw_query *hq = nv50_hw_query(q);
 116
 117    if (hq->funcs && hq->funcs->destroy_query) {
 118       hq->funcs->destroy_query(nv50, hq);
 119       return;
 120    }
 121
 122    nv50_hw_query_allocate(nv50, q, 0);
 123    nouveau_fence_ref(NULL, &hq->fence);
 124    FREE(hq);
 125 }
 126
 127 static boolean
 128 nv50_hw_begin_query(struct nv50_context *nv50, struct nv50_query *q)
 129 {
 130    struct nouveau_pushbuf *push = nv50->base.pushbuf;
 131    struct nv50_hw_query *hq = nv50_hw_query(q);
 132
 133    if (hq->funcs && hq->funcs->begin_query)
 134       return hq->funcs->begin_query(nv50, hq);
 135
 136    /* For occlusion queries we have to change the storage, because a previous
 137     * query might set the initial render condition to false even *after* we re-
 138     * initialized it to true.
 139     */
 140    if (hq->rotate) {
 141       hq->offset += hq->rotate;
 142       hq->data += hq->rotate / sizeof(*hq->data);
 143       if (hq->offset - hq->base_offset == NV50_HW_QUERY_ALLOC_SPACE)
 144          nv50_hw_query_allocate(nv50, q, NV50_HW_QUERY_ALLOC_SPACE);
 145
 146       /* XXX: can we do this with the GPU, and sync with respect to a previous
 147        *  query ?
 148        */
 149       hq->data[0] = hq->sequence; /* initialize sequence */
 150       hq->data[1] = 1; /* initial render condition = true */
 151       hq->data[4] = hq->sequence + 1; /* for comparison COND_MODE */
 152       hq->data[5] = 0;
 153    }
 154    if (!hq->is64bit)
 155       hq->data[0] = hq->sequence++; /* the previously used one */
 156
 157    switch (q->type) {
 158    case PIPE_QUERY_OCCLUSION_COUNTER:
 159       hq->nesting = nv50->screen->num_occlusion_queries_active++;
 160       if (hq->nesting) {
 161          nv50_hw_query_get(push, q, 0x10, 0x0100f002);
 162       } else {
 163          PUSH_SPACE(push, 4);
 164          BEGIN_NV04(push, NV50_3D(COUNTER_RESET), 1);
 165          PUSH_DATA (push, NV50_3D_COUNTER_RESET_SAMPLECNT);
 166          BEGIN_NV04(push, NV50_3D(SAMPLECNT_ENABLE), 1);
 167          PUSH_DATA (push, 1);
 168       }
 169       break;
 170    case PIPE_QUERY_PRIMITIVES_GENERATED:
 171       nv50_hw_query_get(push, q, 0x10, 0x06805002);
 172       break;
 173    case PIPE_QUERY_PRIMITIVES_EMITTED:
 174       nv50_hw_query_get(push, q, 0x10, 0x05805002);
 175       break;
 176    case PIPE_QUERY_SO_STATISTICS:
 177       nv50_hw_query_get(push, q, 0x20, 0x05805002);
 178       nv50_hw_query_get(push, q, 0x30, 0x06805002);
 179       break;
 180    case PIPE_QUERY_PIPELINE_STATISTICS:
 181       nv50_hw_query_get(push, q, 0x80, 0x00801002); /* VFETCH, VERTICES */
 182       nv50_hw_query_get(push, q, 0x90, 0x01801002); /* VFETCH, PRIMS */
 183       nv50_hw_query_get(push, q, 0xa0, 0x02802002); /* VP, LAUNCHES */
 184       nv50_hw_query_get(push, q, 0xb0, 0x03806002); /* GP, LAUNCHES */
 185       nv50_hw_query_get(push, q, 0xc0, 0x04806002); /* GP, PRIMS_OUT */
 186       nv50_hw_query_get(push, q, 0xd0, 0x07804002); /* RAST, PRIMS_IN */
 187       nv50_hw_query_get(push, q, 0xe0, 0x08804002); /* RAST, PRIMS_OUT */
 188       nv50_hw_query_get(push, q, 0xf0, 0x0980a002); /* ROP, PIXELS */
 189       break;
 190    case PIPE_QUERY_TIME_ELAPSED:
 191       nv50_hw_query_get(push, q, 0x10, 0x00005002);
 192       break;
 193    default:
 194       assert(0);
 195       return false;
 196    }
 197    hq->state = NV50_HW_QUERY_STATE_ACTIVE;
 198    return true;
 199 }
 200
 201 static void
 202 nv50_hw_end_query(struct nv50_context *nv50, struct nv50_query *q)
 203 {
 204    struct nouveau_pushbuf *push = nv50->base.pushbuf;
 205    struct nv50_hw_query *hq = nv50_hw_query(q);
 206
 207    if (hq->funcs && hq->funcs->end_query) {
 208       hq->funcs->end_query(nv50, hq);
 209       return;
 210    }
 211
 212    hq->state = NV50_HW_QUERY_STATE_ENDED;
 213
 214    switch (q->type) {
 215    case PIPE_QUERY_OCCLUSION_COUNTER:
 216       nv50_hw_query_get(push, q, 0, 0x0100f002);
 217       if (--nv50->screen->num_occlusion_queries_active == 0) {
 218          PUSH_SPACE(push, 2);
 219          BEGIN_NV04(push, NV50_3D(SAMPLECNT_ENABLE), 1);
 220          PUSH_DATA (push, 0);
 221       }
 222       break;
 223    case PIPE_QUERY_PRIMITIVES_GENERATED:
 224       nv50_hw_query_get(push, q, 0, 0x06805002);
 225       break;
 226    case PIPE_QUERY_PRIMITIVES_EMITTED:
 227       nv50_hw_query_get(push, q, 0, 0x05805002);
 228       break;
 229    case PIPE_QUERY_SO_STATISTICS:
 230       nv50_hw_query_get(push, q, 0x00, 0x05805002);
 231       nv50_hw_query_get(push, q, 0x10, 0x06805002);
 232       break;
 233    case PIPE_QUERY_PIPELINE_STATISTICS:
 234       nv50_hw_query_get(push, q, 0x00, 0x00801002); /* VFETCH, VERTICES */
 235       nv50_hw_query_get(push, q, 0x10, 0x01801002); /* VFETCH, PRIMS */
 236       nv50_hw_query_get(push, q, 0x20, 0x02802002); /* VP, LAUNCHES */
 237       nv50_hw_query_get(push, q, 0x30, 0x03806002); /* GP, LAUNCHES */
 238       nv50_hw_query_get(push, q, 0x40, 0x04806002); /* GP, PRIMS_OUT */
 239       nv50_hw_query_get(push, q, 0x50, 0x07804002); /* RAST, PRIMS_IN */
 240       nv50_hw_query_get(push, q, 0x60, 0x08804002); /* RAST, PRIMS_OUT */
 241       nv50_hw_query_get(push, q, 0x70, 0x0980a002); /* ROP, PIXELS */
 242       break;
 243    case PIPE_QUERY_TIMESTAMP:
 244       hq->sequence++;
 245       /* fall through */
 246    case PIPE_QUERY_TIME_ELAPSED:
 247       nv50_hw_query_get(push, q, 0, 0x00005002);
 248       break;
 249    case PIPE_QUERY_GPU_FINISHED:
 250       hq->sequence++;
 251       nv50_hw_query_get(push, q, 0, 0x1000f010);
 252       break;
 253    case NVA0_HW_QUERY_STREAM_OUTPUT_BUFFER_OFFSET:
 254       hq->sequence++;
 255       nv50_hw_query_get(push, q, 0, 0x0d005002 | (q->index << 5));
 256       break;
 257    case PIPE_QUERY_TIMESTAMP_DISJOINT:
 258       /* This query is not issued on GPU because disjoint is forced to false */
 259       hq->state = NV50_HW_QUERY_STATE_READY;
 260       break;
 261    default:
 262       assert(0);
 263       break;
 264    }
 265    if (hq->is64bit)
 266       nouveau_fence_ref(nv50->screen->base.fence.current, &hq->fence);
 267 }
 268
 269 static boolean
 270 nv50_hw_get_query_result(struct nv50_context *nv50, struct nv50_query *q,
 271                          boolean wait, union pipe_query_result *result)
 272 {
 273    struct nv50_hw_query *hq = nv50_hw_query(q);
 274    uint64_t *res64 = (uint64_t *)result;
 275    uint32_t *res32 = (uint32_t *)result;
 276    uint8_t *res8 = (uint8_t *)result;
 277    uint64_t *data64 = (uint64_t *)hq->data;
 278    int i;
 279
 280    if (hq->funcs && hq->funcs->get_query_result)
 281       return hq->funcs->get_query_result(nv50, hq, wait, result);
 282
 283    if (hq->state != NV50_HW_QUERY_STATE_READY)
 284       nv50_hw_query_update(q);
 285
 286    if (hq->state != NV50_HW_QUERY_STATE_READY) {
 287       if (!wait) {
 288          /* for broken apps that spin on GL_QUERY_RESULT_AVAILABLE */
 289          if (hq->state != NV50_HW_QUERY_STATE_FLUSHED) {
 290             hq->state = NV50_HW_QUERY_STATE_FLUSHED;
 291             PUSH_KICK(nv50->base.pushbuf);
 292          }
 293          return false;
 294       }
 295       if (nouveau_bo_wait(hq->bo, NOUVEAU_BO_RD, nv50->screen->base.client))
 296          return false;
 297    }
 298    hq->state = NV50_HW_QUERY_STATE_READY;
 299
 300    switch (q->type) {
 301    case PIPE_QUERY_GPU_FINISHED:
 302       res8[0] = true;
 303       break;
 304    case PIPE_QUERY_OCCLUSION_COUNTER: /* u32 sequence, u32 count, u64 time */
 305       res64[0] = hq->data[1] - hq->data[5];
 306       break;
 307    case PIPE_QUERY_PRIMITIVES_GENERATED: /* u64 count, u64 time */
 308    case PIPE_QUERY_PRIMITIVES_EMITTED: /* u64 count, u64 time */
 309       res64[0] = data64[0] - data64[2];
 310       break;
 311    case PIPE_QUERY_SO_STATISTICS:
 312       res64[0] = data64[0] - data64[4];
 313       res64[1] = data64[2] - data64[6];
 314       break;
 315    case PIPE_QUERY_PIPELINE_STATISTICS:
 316       for (i = 0; i < 8; ++i)
 317          res64[i] = data64[i * 2] - data64[16 + i * 2];
 318       break;
 319    case PIPE_QUERY_TIMESTAMP:
 320       res64[0] = data64[1];
 321       break;
 322    case PIPE_QUERY_TIMESTAMP_DISJOINT:
 323       res64[0] = 1000000000;
 324       res8[8] = false;
 325       break;
 326    case PIPE_QUERY_TIME_ELAPSED:
 327       res64[0] = data64[1] - data64[3];
 328       break;
 329    case NVA0_HW_QUERY_STREAM_OUTPUT_BUFFER_OFFSET:
 330       res32[0] = hq->data[1];
 331       break;
 332    default:
 333       assert(0);
 334       return false;
 335    }
 336
 337    return true;
 338 }
 339
 340 static const struct nv50_query_funcs hw_query_funcs = {
 341    .destroy_query = nv50_hw_destroy_query,
 342    .begin_query = nv50_hw_begin_query,
 343    .end_query = nv50_hw_end_query,
 344    .get_query_result = nv50_hw_get_query_result,
 345 };
 346
 347 struct nv50_query *
 348 nv50_hw_create_query(struct nv50_context *nv50, unsigned type, unsigned index)
 349 {
 350    struct nv50_hw_query *hq;
 351    struct nv50_query *q;
 352
 353    hq = nv50_hw_sm_create_query(nv50, type);
 354    if (hq) {
 355       hq->base.funcs = &hw_query_funcs;
 356       return (struct nv50_query *)hq;
 357    }
 358
 359    hq = nv50_hw_metric_create_query(nv50, type);
 360    if (hq) {
 361       hq->base.funcs = &hw_query_funcs;
 362       return (struct nv50_query *)hq;
 363    }
 364
 365    hq = CALLOC_STRUCT(nv50_hw_query);
 366    if (!hq)
 367       return NULL;
 368
 369    q = &hq->base;
 370    q->funcs = &hw_query_funcs;
 371    q->type = type;
 372
 373    switch (q->type) {
 374    case PIPE_QUERY_OCCLUSION_COUNTER:
 375       hq->rotate = 32;
 376       break;
 377    case PIPE_QUERY_PRIMITIVES_GENERATED:
 378    case PIPE_QUERY_PRIMITIVES_EMITTED:
 379    case PIPE_QUERY_SO_STATISTICS:
 380    case PIPE_QUERY_PIPELINE_STATISTICS:
 381       hq->is64bit = true;
 382       break;
 383    case PIPE_QUERY_TIME_ELAPSED:
 384    case PIPE_QUERY_TIMESTAMP:
 385    case PIPE_QUERY_TIMESTAMP_DISJOINT:
 386    case PIPE_QUERY_GPU_FINISHED:
 387    case NVA0_HW_QUERY_STREAM_OUTPUT_BUFFER_OFFSET:
 388       break;
 389    default:
 390       debug_printf("invalid query type: %u\n", type);
 391       FREE(q);
 392       return NULL;
 393    }
 394
 395    if (!nv50_hw_query_allocate(nv50, q, NV50_HW_QUERY_ALLOC_SPACE)) {
 396       FREE(hq);
 397       return NULL;
 398    }
 399
 400    if (hq->rotate) {
 401       /* we advance before query_begin ! */
 402       hq->offset -= hq->rotate;
 403       hq->data -= hq->rotate / sizeof(*hq->data);
 404    }
 405
 406    return q;
 407 }
 408
 409 int
 410 nv50_hw_get_driver_query_info(struct nv50_screen *screen, unsigned id,
 411                               struct pipe_driver_query_info *info)
 412 {
 413    int num_hw_sm_queries = 0, num_hw_metric_queries = 0;
 414
 415    num_hw_sm_queries = nv50_hw_sm_get_driver_query_info(screen, 0, NULL);
 416    num_hw_metric_queries =
 417       nv50_hw_metric_get_driver_query_info(screen, 0, NULL);
 418
 419    if (!info)
 420       return num_hw_sm_queries + num_hw_metric_queries;
 421
 422    if (id < num_hw_sm_queries)
 423       return nv50_hw_sm_get_driver_query_info(screen, id, info);
 424
 425    return nv50_hw_metric_get_driver_query_info(screen,
 426                                                id - num_hw_sm_queries, info);
 427 }
 428
 429 void
 430 nv50_hw_query_pushbuf_submit(struct nouveau_pushbuf *push, uint16_t method,
 431                              struct nv50_query *q, unsigned result_offset)
 432 {
 433    struct nv50_hw_query *hq = nv50_hw_query(q);
 434
 435    nv50_hw_query_update(q);
 436    if (hq->state != NV50_HW_QUERY_STATE_READY)
 437       nouveau_bo_wait(hq->bo, NOUVEAU_BO_RD, push->client);
 438    hq->state = NV50_HW_QUERY_STATE_READY;
 439
 440    BEGIN_NV04(push, SUBC_3D(method), 1);
 441    PUSH_DATA (push, hq->data[result_offset / 4]);
 442 }
 443
 444 void
 445 nv84_hw_query_fifo_wait(struct nouveau_pushbuf *push, struct nv50_query *q)
 446 {
 447    struct nv50_hw_query *hq = nv50_hw_query(q);
 448    unsigned offset = hq->offset;
 449
 450    PUSH_SPACE(push, 5);
 451    PUSH_REFN (push, hq->bo, NOUVEAU_BO_GART | NOUVEAU_BO_RD);
 452    BEGIN_NV04(push, SUBC_3D(NV84_SUBCHAN_SEMAPHORE_ADDRESS_HIGH), 4);
 453    PUSH_DATAh(push, hq->bo->offset + offset);
 454    PUSH_DATA (push, hq->bo->offset + offset);
 455    PUSH_DATA (push, hq->sequence);
 456    PUSH_DATA (push, NV84_SUBCHAN_SEMAPHORE_TRIGGER_ACQUIRE_EQUAL);
 457 }