src/gallium/drivers/nouveau/nv50/nv50_query_hw.c

   1 /*
   2  * Copyright 2011 Christoph Bumiller
   3  * Copyright 2015 Samuel Pitoiset
   4  *
   5  * Permission is hereby granted, free of charge, to any person obtaining a
   6  * copy of this software and associated documentation files (the "Software"),
   7  * to deal in the Software without restriction, including without limitation
   8  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   9  * and/or sell copies of the Software, and to permit persons to whom the
  10  * Software is furnished to do so, subject to the following conditions:
  11  *
  12  * The above copyright notice and this permission notice shall be included in
  13  * all copies or substantial portions of the Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
  19  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  20  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  21  * OTHER DEALINGS IN THE SOFTWARE.
  22  */
  23
  24 #define NV50_PUSH_EXPLICIT_SPACE_CHECKING
  25
  26 #include "nv50/nv50_context.h"
  27 #include "nv50/nv50_query_hw.h"
  28 #include "nv50/nv50_query_hw_metric.h"
  29 #include "nv50/nv50_query_hw_sm.h"
  30 #include "nv_object.xml.h"
  31
  32 #define NV50_HW_QUERY_STATE_READY   0
  33 #define NV50_HW_QUERY_STATE_ACTIVE  1
  34 #define NV50_HW_QUERY_STATE_ENDED   2
  35 #define NV50_HW_QUERY_STATE_FLUSHED 3
  36
  37 /* XXX: Nested queries, and simultaneous queries on multiple gallium contexts
  38  * (since we use only a single GPU channel per screen) will not work properly.
  39  *
  40  * The first is not that big of an issue because OpenGL does not allow nested
  41  * queries anyway.
  42  */
  43
  44 #define NV50_HW_QUERY_ALLOC_SPACE 256
  45
  46 bool
  47 nv50_hw_query_allocate(struct nv50_context *nv50, struct nv50_query *q,
  48                        int size)
  49 {
  50    struct nv50_screen *screen = nv50->screen;
  51    struct nv50_hw_query *hq = nv50_hw_query(q);
  52    int ret;
  53
  54    if (hq->bo) {
  55       nouveau_bo_ref(NULL, &hq->bo);
  56       if (hq->mm) {
  57          if (hq->state == NV50_HW_QUERY_STATE_READY)
  58             nouveau_mm_free(hq->mm);
  59          else
  60             nouveau_fence_work(screen->base.fence.current,
  61                                nouveau_mm_free_work, hq->mm);
  62       }
  63    }
  64    if (size) {
  65       hq->mm = nouveau_mm_allocate(screen->base.mm_GART, size,
  66                                    &hq->bo, &hq->base_offset);
  67       if (!hq->bo)
  68          return false;
  69       hq->offset = hq->base_offset;
  70
  71       ret = nouveau_bo_map(hq->bo, 0, screen->base.client);
  72       if (ret) {
  73          nv50_hw_query_allocate(nv50, q, 0);
  74          return false;
  75       }
  76       hq->data = (uint32_t *)((uint8_t *)hq->bo->map + hq->base_offset);
  77    }
  78    return true;
  79 }
  80
  81 static void
  82 nv50_hw_query_get(struct nouveau_pushbuf *push, struct nv50_query *q,
  83                unsigned offset, uint32_t get)
  84 {
  85    struct nv50_hw_query *hq = nv50_hw_query(q);
  86
  87    offset += hq->offset;
  88
  89    PUSH_SPACE(push, 5);
  90    PUSH_REFN (push, hq->bo, NOUVEAU_BO_GART | NOUVEAU_BO_WR);
  91    BEGIN_NV04(push, NV50_3D(QUERY_ADDRESS_HIGH), 4);
  92    PUSH_DATAh(push, hq->bo->offset + offset);
  93    PUSH_DATA (push, hq->bo->offset + offset);
  94    PUSH_DATA (push, hq->sequence);
  95    PUSH_DATA (push, get);
  96 }
  97
  98 static inline void
  99 nv50_hw_query_update(struct nv50_query *q)
 100 {
 101    struct nv50_hw_query *hq = nv50_hw_query(q);
 102
 103    if (hq->is64bit) {
 104       if (nouveau_fence_signalled(hq->fence))
 105          hq->state = NV50_HW_QUERY_STATE_READY;
 106    } else {
 107       if (hq->data[0] == hq->sequence)
 108          hq->state = NV50_HW_QUERY_STATE_READY;
 109    }
 110 }
 111
 112 static void
 113 nv50_hw_destroy_query(struct nv50_context *nv50, struct nv50_query *q)
 114 {
 115    struct nv50_hw_query *hq = nv50_hw_query(q);
 116
 117    if (hq->funcs && hq->funcs->destroy_query) {
 118       hq->funcs->destroy_query(nv50, hq);
 119       return;
 120    }
 121
 122    nv50_hw_query_allocate(nv50, q, 0);
 123    nouveau_fence_ref(NULL, &hq->fence);
 124    FREE(hq);
 125 }
 126
 127 static boolean
 128 nv50_hw_begin_query(struct nv50_context *nv50, struct nv50_query *q)
 129 {
 130    struct nouveau_pushbuf *push = nv50->base.pushbuf;
 131    struct nv50_hw_query *hq = nv50_hw_query(q);
 132
 133    if (hq->funcs && hq->funcs->begin_query)
 134       return hq->funcs->begin_query(nv50, hq);
 135
 136    /* For occlusion queries we have to change the storage, because a previous
 137     * query might set the initial render condition to false even *after* we re-
 138     * initialized it to true.
 139     */
 140    if (hq->rotate) {
 141       hq->offset += hq->rotate;
 142       hq->data += hq->rotate / sizeof(*hq->data);
 143       if (hq->offset - hq->base_offset == NV50_HW_QUERY_ALLOC_SPACE)
 144          nv50_hw_query_allocate(nv50, q, NV50_HW_QUERY_ALLOC_SPACE);
 145
 146       /* XXX: can we do this with the GPU, and sync with respect to a previous
 147        *  query ?
 148        */
 149       hq->data[0] = hq->sequence; /* initialize sequence */
 150       hq->data[1] = 1; /* initial render condition = true */
 151       hq->data[4] = hq->sequence + 1; /* for comparison COND_MODE */
 152       hq->data[5] = 0;
 153    }
 154    if (!hq->is64bit)
 155       hq->data[0] = hq->sequence++; /* the previously used one */
 156
 157    switch (q->type) {
 158    case PIPE_QUERY_OCCLUSION_COUNTER:
 159    case PIPE_QUERY_OCCLUSION_PREDICATE:
 160       hq->nesting = nv50->screen->num_occlusion_queries_active++;
 161       if (hq->nesting) {
 162          nv50_hw_query_get(push, q, 0x10, 0x0100f002);
 163       } else {
 164          PUSH_SPACE(push, 4);
 165          BEGIN_NV04(push, NV50_3D(COUNTER_RESET), 1);
 166          PUSH_DATA (push, NV50_3D_COUNTER_RESET_SAMPLECNT);
 167          BEGIN_NV04(push, NV50_3D(SAMPLECNT_ENABLE), 1);
 168          PUSH_DATA (push, 1);
 169       }
 170       break;
 171    case PIPE_QUERY_PRIMITIVES_GENERATED:
 172       nv50_hw_query_get(push, q, 0x10, 0x06805002);
 173       break;
 174    case PIPE_QUERY_PRIMITIVES_EMITTED:
 175       nv50_hw_query_get(push, q, 0x10, 0x05805002);
 176       break;
 177    case PIPE_QUERY_SO_STATISTICS:
 178       nv50_hw_query_get(push, q, 0x20, 0x05805002);
 179       nv50_hw_query_get(push, q, 0x30, 0x06805002);
 180       break;
 181    case PIPE_QUERY_PIPELINE_STATISTICS:
 182       nv50_hw_query_get(push, q, 0x80, 0x00801002); /* VFETCH, VERTICES */
 183       nv50_hw_query_get(push, q, 0x90, 0x01801002); /* VFETCH, PRIMS */
 184       nv50_hw_query_get(push, q, 0xa0, 0x02802002); /* VP, LAUNCHES */
 185       nv50_hw_query_get(push, q, 0xb0, 0x03806002); /* GP, LAUNCHES */
 186       nv50_hw_query_get(push, q, 0xc0, 0x04806002); /* GP, PRIMS_OUT */
 187       nv50_hw_query_get(push, q, 0xd0, 0x07804002); /* RAST, PRIMS_IN */
 188       nv50_hw_query_get(push, q, 0xe0, 0x08804002); /* RAST, PRIMS_OUT */
 189       nv50_hw_query_get(push, q, 0xf0, 0x0980a002); /* ROP, PIXELS */
 190       break;
 191    case PIPE_QUERY_TIME_ELAPSED:
 192       nv50_hw_query_get(push, q, 0x10, 0x00005002);
 193       break;
 194    default:
 195       assert(0);
 196       return false;
 197    }
 198    hq->state = NV50_HW_QUERY_STATE_ACTIVE;
 199    return true;
 200 }
 201
 202 static void
 203 nv50_hw_end_query(struct nv50_context *nv50, struct nv50_query *q)
 204 {
 205    struct nouveau_pushbuf *push = nv50->base.pushbuf;
 206    struct nv50_hw_query *hq = nv50_hw_query(q);
 207
 208    if (hq->funcs && hq->funcs->end_query) {
 209       hq->funcs->end_query(nv50, hq);
 210       return;
 211    }
 212
 213    hq->state = NV50_HW_QUERY_STATE_ENDED;
 214
 215    switch (q->type) {
 216    case PIPE_QUERY_OCCLUSION_COUNTER:
 217    case PIPE_QUERY_OCCLUSION_PREDICATE:
 218       nv50_hw_query_get(push, q, 0, 0x0100f002);
 219       if (--nv50->screen->num_occlusion_queries_active == 0) {
 220          PUSH_SPACE(push, 2);
 221          BEGIN_NV04(push, NV50_3D(SAMPLECNT_ENABLE), 1);
 222          PUSH_DATA (push, 0);
 223       }
 224       break;
 225    case PIPE_QUERY_PRIMITIVES_GENERATED:
 226       nv50_hw_query_get(push, q, 0, 0x06805002);
 227       break;
 228    case PIPE_QUERY_PRIMITIVES_EMITTED:
 229       nv50_hw_query_get(push, q, 0, 0x05805002);
 230       break;
 231    case PIPE_QUERY_SO_STATISTICS:
 232       nv50_hw_query_get(push, q, 0x00, 0x05805002);
 233       nv50_hw_query_get(push, q, 0x10, 0x06805002);
 234       break;
 235    case PIPE_QUERY_PIPELINE_STATISTICS:
 236       nv50_hw_query_get(push, q, 0x00, 0x00801002); /* VFETCH, VERTICES */
 237       nv50_hw_query_get(push, q, 0x10, 0x01801002); /* VFETCH, PRIMS */
 238       nv50_hw_query_get(push, q, 0x20, 0x02802002); /* VP, LAUNCHES */
 239       nv50_hw_query_get(push, q, 0x30, 0x03806002); /* GP, LAUNCHES */
 240       nv50_hw_query_get(push, q, 0x40, 0x04806002); /* GP, PRIMS_OUT */
 241       nv50_hw_query_get(push, q, 0x50, 0x07804002); /* RAST, PRIMS_IN */
 242       nv50_hw_query_get(push, q, 0x60, 0x08804002); /* RAST, PRIMS_OUT */
 243       nv50_hw_query_get(push, q, 0x70, 0x0980a002); /* ROP, PIXELS */
 244       break;
 245    case PIPE_QUERY_TIMESTAMP:
 246       hq->sequence++;
 247       /* fall through */
 248    case PIPE_QUERY_TIME_ELAPSED:
 249       nv50_hw_query_get(push, q, 0, 0x00005002);
 250       break;
 251    case PIPE_QUERY_GPU_FINISHED:
 252       hq->sequence++;
 253       nv50_hw_query_get(push, q, 0, 0x1000f010);
 254       break;
 255    case NVA0_HW_QUERY_STREAM_OUTPUT_BUFFER_OFFSET:
 256       hq->sequence++;
 257       nv50_hw_query_get(push, q, 0, 0x0d005002 | (q->index << 5));
 258       break;
 259    case PIPE_QUERY_TIMESTAMP_DISJOINT:
 260       /* This query is not issued on GPU because disjoint is forced to false */
 261       hq->state = NV50_HW_QUERY_STATE_READY;
 262       break;
 263    default:
 264       assert(0);
 265       break;
 266    }
 267    if (hq->is64bit)
 268       nouveau_fence_ref(nv50->screen->base.fence.current, &hq->fence);
 269 }
 270
 271 static boolean
 272 nv50_hw_get_query_result(struct nv50_context *nv50, struct nv50_query *q,
 273                          boolean wait, union pipe_query_result *result)
 274 {
 275    struct nv50_hw_query *hq = nv50_hw_query(q);
 276    uint64_t *res64 = (uint64_t *)result;
 277    uint32_t *res32 = (uint32_t *)result;
 278    uint8_t *res8 = (uint8_t *)result;
 279    uint64_t *data64 = (uint64_t *)hq->data;
 280    int i;
 281
 282    if (hq->funcs && hq->funcs->get_query_result)
 283       return hq->funcs->get_query_result(nv50, hq, wait, result);
 284
 285    if (hq->state != NV50_HW_QUERY_STATE_READY)
 286       nv50_hw_query_update(q);
 287
 288    if (hq->state != NV50_HW_QUERY_STATE_READY) {
 289       if (!wait) {
 290          /* for broken apps that spin on GL_QUERY_RESULT_AVAILABLE */
 291          if (hq->state != NV50_HW_QUERY_STATE_FLUSHED) {
 292             hq->state = NV50_HW_QUERY_STATE_FLUSHED;
 293             PUSH_KICK(nv50->base.pushbuf);
 294          }
 295          return false;
 296       }
 297       if (nouveau_bo_wait(hq->bo, NOUVEAU_BO_RD, nv50->screen->base.client))
 298          return false;
 299    }
 300    hq->state = NV50_HW_QUERY_STATE_READY;
 301
 302    switch (q->type) {
 303    case PIPE_QUERY_GPU_FINISHED:
 304       res8[0] = true;
 305       break;
 306    case PIPE_QUERY_OCCLUSION_COUNTER: /* u32 sequence, u32 count, u64 time */
 307       res64[0] = hq->data[1] - hq->data[5];
 308       break;
 309    case PIPE_QUERY_OCCLUSION_PREDICATE:
 310       res8[0] = hq->data[1] != hq->data[5];
 311       break;
 312    case PIPE_QUERY_PRIMITIVES_GENERATED: /* u64 count, u64 time */
 313    case PIPE_QUERY_PRIMITIVES_EMITTED: /* u64 count, u64 time */
 314       res64[0] = data64[0] - data64[2];
 315       break;
 316    case PIPE_QUERY_SO_STATISTICS:
 317       res64[0] = data64[0] - data64[4];
 318       res64[1] = data64[2] - data64[6];
 319       break;
 320    case PIPE_QUERY_PIPELINE_STATISTICS:
 321       for (i = 0; i < 8; ++i)
 322          res64[i] = data64[i * 2] - data64[16 + i * 2];
 323       break;
 324    case PIPE_QUERY_TIMESTAMP:
 325       res64[0] = data64[1];
 326       break;
 327    case PIPE_QUERY_TIMESTAMP_DISJOINT:
 328       res64[0] = 1000000000;
 329       res8[8] = false;
 330       break;
 331    case PIPE_QUERY_TIME_ELAPSED:
 332       res64[0] = data64[1] - data64[3];
 333       break;
 334    case NVA0_HW_QUERY_STREAM_OUTPUT_BUFFER_OFFSET:
 335       res32[0] = hq->data[1];
 336       break;
 337    default:
 338       assert(0);
 339       return false;
 340    }
 341
 342    return true;
 343 }
 344
 345 static const struct nv50_query_funcs hw_query_funcs = {
 346    .destroy_query = nv50_hw_destroy_query,
 347    .begin_query = nv50_hw_begin_query,
 348    .end_query = nv50_hw_end_query,
 349    .get_query_result = nv50_hw_get_query_result,
 350 };
 351
 352 struct nv50_query *
 353 nv50_hw_create_query(struct nv50_context *nv50, unsigned type, unsigned index)
 354 {
 355    struct nv50_hw_query *hq;
 356    struct nv50_query *q;
 357
 358    hq = nv50_hw_sm_create_query(nv50, type);
 359    if (hq) {
 360       hq->base.funcs = &hw_query_funcs;
 361       return (struct nv50_query *)hq;
 362    }
 363
 364    hq = nv50_hw_metric_create_query(nv50, type);
 365    if (hq) {
 366       hq->base.funcs = &hw_query_funcs;
 367       return (struct nv50_query *)hq;
 368    }
 369
 370    hq = CALLOC_STRUCT(nv50_hw_query);
 371    if (!hq)
 372       return NULL;
 373
 374    q = &hq->base;
 375    q->funcs = &hw_query_funcs;
 376    q->type = type;
 377
 378    switch (q->type) {
 379    case PIPE_QUERY_OCCLUSION_COUNTER:
 380    case PIPE_QUERY_OCCLUSION_PREDICATE:
 381       hq->rotate = 32;
 382       break;
 383    case PIPE_QUERY_PRIMITIVES_GENERATED:
 384    case PIPE_QUERY_PRIMITIVES_EMITTED:
 385    case PIPE_QUERY_SO_STATISTICS:
 386    case PIPE_QUERY_PIPELINE_STATISTICS:
 387       hq->is64bit = true;
 388       break;
 389    case PIPE_QUERY_TIME_ELAPSED:
 390    case PIPE_QUERY_TIMESTAMP:
 391    case PIPE_QUERY_TIMESTAMP_DISJOINT:
 392    case PIPE_QUERY_GPU_FINISHED:
 393    case NVA0_HW_QUERY_STREAM_OUTPUT_BUFFER_OFFSET:
 394       break;
 395    default:
 396       debug_printf("invalid query type: %u\n", type);
 397       FREE(q);
 398       return NULL;
 399    }
 400
 401    if (!nv50_hw_query_allocate(nv50, q, NV50_HW_QUERY_ALLOC_SPACE)) {
 402       FREE(hq);
 403       return NULL;
 404    }
 405
 406    if (hq->rotate) {
 407       /* we advance before query_begin ! */
 408       hq->offset -= hq->rotate;
 409       hq->data -= hq->rotate / sizeof(*hq->data);
 410    }
 411
 412    return q;
 413 }
 414
 415 int
 416 nv50_hw_get_driver_query_info(struct nv50_screen *screen, unsigned id,
 417                               struct pipe_driver_query_info *info)
 418 {
 419    int num_hw_sm_queries = 0, num_hw_metric_queries = 0;
 420
 421    num_hw_sm_queries = nv50_hw_sm_get_driver_query_info(screen, 0, NULL);
 422    num_hw_metric_queries =
 423       nv50_hw_metric_get_driver_query_info(screen, 0, NULL);
 424
 425    if (!info)
 426       return num_hw_sm_queries + num_hw_metric_queries;
 427
 428    if (id < num_hw_sm_queries)
 429       return nv50_hw_sm_get_driver_query_info(screen, id, info);
 430
 431    return nv50_hw_metric_get_driver_query_info(screen,
 432                                                id - num_hw_sm_queries, info);
 433 }
 434
 435 void
 436 nv50_hw_query_pushbuf_submit(struct nouveau_pushbuf *push, uint16_t method,
 437                              struct nv50_query *q, unsigned result_offset)
 438 {
 439    struct nv50_hw_query *hq = nv50_hw_query(q);
 440
 441    nv50_hw_query_update(q);
 442    if (hq->state != NV50_HW_QUERY_STATE_READY)
 443       nouveau_bo_wait(hq->bo, NOUVEAU_BO_RD, push->client);
 444    hq->state = NV50_HW_QUERY_STATE_READY;
 445
 446    BEGIN_NV04(push, SUBC_3D(method), 1);
 447    PUSH_DATA (push, hq->data[result_offset / 4]);
 448 }
 449
 450 void
 451 nv84_hw_query_fifo_wait(struct nouveau_pushbuf *push, struct nv50_query *q)
 452 {
 453    struct nv50_hw_query *hq = nv50_hw_query(q);
 454    unsigned offset = hq->offset;
 455
 456    PUSH_SPACE(push, 5);
 457    PUSH_REFN (push, hq->bo, NOUVEAU_BO_GART | NOUVEAU_BO_RD);
 458    BEGIN_NV04(push, SUBC_3D(NV84_SUBCHAN_SEMAPHORE_ADDRESS_HIGH), 4);
 459    PUSH_DATAh(push, hq->bo->offset + offset);
 460    PUSH_DATA (push, hq->bo->offset + offset);
 461    PUSH_DATA (push, hq->sequence);
 462    PUSH_DATA (push, NV84_SUBCHAN_SEMAPHORE_TRIGGER_ACQUIRE_EQUAL);
 463 }