src/mesa/drivers/dri/i965/hsw_queryobj.c

   1 /*
   2  * Copyright (c) 2016 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  *
  23  */
  24
  25 /** @file hsw_queryobj.c
  26  *
  27  * Support for query buffer objects (GL_ARB_query_buffer_object) on Haswell+.
  28  */
  29 #include "main/imports.h"
  30
  31 #include "brw_context.h"
  32 #include "brw_defines.h"
  33 #include "intel_batchbuffer.h"
  34 #include "intel_buffer_objects.h"
  35
  36 /*
  37  * GPR0 = 80 * GPR0;
  38  */
  39 static void
  40 mult_gpr0_by_80(struct brw_context *brw)
  41 {
  42    static const uint32_t maths[] = {
  43       MI_MATH_ALU2(LOAD, SRCA, R0),
  44       MI_MATH_ALU2(LOAD, SRCB, R0),
  45       MI_MATH_ALU0(ADD),
  46       MI_MATH_ALU2(STORE, R1, ACCU),
  47       MI_MATH_ALU2(LOAD, SRCA, R1),
  48       MI_MATH_ALU2(LOAD, SRCB, R1),
  49       MI_MATH_ALU0(ADD),
  50       MI_MATH_ALU2(STORE, R1, ACCU),
  51       MI_MATH_ALU2(LOAD, SRCA, R1),
  52       MI_MATH_ALU2(LOAD, SRCB, R1),
  53       MI_MATH_ALU0(ADD),
  54       MI_MATH_ALU2(STORE, R1, ACCU),
  55       MI_MATH_ALU2(LOAD, SRCA, R1),
  56       MI_MATH_ALU2(LOAD, SRCB, R1),
  57       MI_MATH_ALU0(ADD),
  58       /* GPR1 = 16 * GPR0 */
  59       MI_MATH_ALU2(STORE, R1, ACCU),
  60       MI_MATH_ALU2(LOAD, SRCA, R1),
  61       MI_MATH_ALU2(LOAD, SRCB, R1),
  62       MI_MATH_ALU0(ADD),
  63       MI_MATH_ALU2(STORE, R2, ACCU),
  64       MI_MATH_ALU2(LOAD, SRCA, R2),
  65       MI_MATH_ALU2(LOAD, SRCB, R2),
  66       MI_MATH_ALU0(ADD),
  67       /* GPR2 = 64 * GPR0 */
  68       MI_MATH_ALU2(STORE, R2, ACCU),
  69       MI_MATH_ALU2(LOAD, SRCA, R1),
  70       MI_MATH_ALU2(LOAD, SRCB, R2),
  71       MI_MATH_ALU0(ADD),
  72       /* GPR0 = 80 * GPR0 */
  73       MI_MATH_ALU2(STORE, R0, ACCU),
  74    };
  75
  76    BEGIN_BATCH(1 + ARRAY_SIZE(maths));
  77    OUT_BATCH(HSW_MI_MATH | (1 + ARRAY_SIZE(maths) - 2));
  78
  79    for (int m = 0; m < ARRAY_SIZE(maths); m++)
  80       OUT_BATCH(maths[m]);
  81
  82    ADVANCE_BATCH();
  83 }
  84
  85 /*
  86  * GPR0 = GPR0 & ((1ull << n) - 1);
  87  */
  88 static void
  89 keep_gpr0_lower_n_bits(struct brw_context *brw, uint32_t n)
  90 {
  91    static const uint32_t maths[] = {
  92       MI_MATH_ALU2(LOAD, SRCA, R0),
  93       MI_MATH_ALU2(LOAD, SRCB, R1),
  94       MI_MATH_ALU0(AND),
  95       MI_MATH_ALU2(STORE, R0, ACCU),
  96    };
  97
  98    assert(n < 64);
  99    brw_load_register_imm64(brw, HSW_CS_GPR(1), (1ull << n) - 1);
 100
 101    BEGIN_BATCH(1 + ARRAY_SIZE(maths));
 102    OUT_BATCH(HSW_MI_MATH | (1 + ARRAY_SIZE(maths) - 2));
 103
 104    for (int m = 0; m < ARRAY_SIZE(maths); m++)
 105       OUT_BATCH(maths[m]);
 106
 107    ADVANCE_BATCH();
 108 }
 109
 110 /*
 111  * GPR0 = GPR0 << 30;
 112  */
 113 static void
 114 shl_gpr0_by_30_bits(struct brw_context *brw)
 115 {
 116    /* First we mask 34 bits of GPR0 to prevent overflow */
 117    keep_gpr0_lower_n_bits(brw, 34);
 118
 119    static const uint32_t shl_maths[] = {
 120       MI_MATH_ALU2(LOAD, SRCA, R0),
 121       MI_MATH_ALU2(LOAD, SRCB, R0),
 122       MI_MATH_ALU0(ADD),
 123       MI_MATH_ALU2(STORE, R0, ACCU),
 124    };
 125
 126    const uint32_t outer_count = 5;
 127    const uint32_t inner_count = 6;
 128    STATIC_ASSERT(outer_count * inner_count == 30);
 129    const uint32_t cmd_len = 1 + inner_count * ARRAY_SIZE(shl_maths);
 130    const uint32_t batch_len = cmd_len * outer_count;
 131
 132    BEGIN_BATCH(batch_len);
 133
 134    /* We'll emit 5 commands, each shifting GPR0 left by 6 bits, for a total of
 135     * 30 left shifts.
 136     */
 137    for (int o = 0; o < outer_count; o++) {
 138       /* Submit one MI_MATH to shift left by 6 bits */
 139       OUT_BATCH(HSW_MI_MATH | (cmd_len - 2));
 140       for (int i = 0; i < inner_count; i++)
 141          for (int m = 0; m < ARRAY_SIZE(shl_maths); m++)
 142             OUT_BATCH(shl_maths[m]);
 143    }
 144
 145    ADVANCE_BATCH();
 146 }
 147
 148 /*
 149  * GPR0 = GPR0 >> 2;
 150  *
 151  * Note that the upper 30 bits of GPR0 are lost!
 152  */
 153 static void
 154 shr_gpr0_by_2_bits(struct brw_context *brw)
 155 {
 156    shl_gpr0_by_30_bits(brw);
 157    brw_load_register_reg(brw, HSW_CS_GPR(0) + 4, HSW_CS_GPR(0));
 158    brw_load_register_imm32(brw, HSW_CS_GPR(0) + 4, 0);
 159 }
 160
 161 /*
 162  * GPR0 = (GPR0 == 0) ? 0 : 1;
 163  */
 164 static void
 165 gpr0_to_bool(struct brw_context *brw)
 166 {
 167    static const uint32_t maths[] = {
 168       MI_MATH_ALU2(LOAD, SRCA, R0),
 169       MI_MATH_ALU1(LOAD0, SRCB),
 170       MI_MATH_ALU0(ADD),
 171       MI_MATH_ALU2(STOREINV, R0, ZF),
 172       MI_MATH_ALU2(LOAD, SRCA, R0),
 173       MI_MATH_ALU2(LOAD, SRCB, R1),
 174       MI_MATH_ALU0(AND),
 175       MI_MATH_ALU2(STORE, R0, ACCU),
 176    };
 177
 178    brw_load_register_imm64(brw, HSW_CS_GPR(1), 1ull);
 179
 180    BEGIN_BATCH(1 + ARRAY_SIZE(maths));
 181    OUT_BATCH(HSW_MI_MATH | (1 + ARRAY_SIZE(maths) - 2));
 182
 183    for (int m = 0; m < ARRAY_SIZE(maths); m++)
 184       OUT_BATCH(maths[m]);
 185
 186    ADVANCE_BATCH();
 187 }
 188
 189 static void
 190 hsw_result_to_gpr0(struct gl_context *ctx, struct brw_query_object *query,
 191                    struct gl_buffer_object *buf, intptr_t offset,
 192                    GLenum pname, GLenum ptype)
 193 {
 194    struct brw_context *brw = brw_context(ctx);
 195
 196    assert(query->bo);
 197    assert(pname != GL_QUERY_TARGET);
 198
 199    if (pname == GL_QUERY_RESULT_AVAILABLE) {
 200       /* The query result availability is stored at offset 0 of the buffer. */
 201       brw_load_register_mem64(brw,
 202                               HSW_CS_GPR(0),
 203                               query->bo,
 204                               I915_GEM_DOMAIN_INSTRUCTION,
 205                               I915_GEM_DOMAIN_INSTRUCTION,
 206                               2 * sizeof(uint64_t));
 207       return;
 208    }
 209
 210    if (pname == GL_QUERY_RESULT) {
 211       /* Since GL_QUERY_RESULT_NO_WAIT wasn't used, they want us to stall to
 212        * make sure the query is available.
 213        */
 214       brw_emit_pipe_control_flush(brw,
 215                                   PIPE_CONTROL_CS_STALL |
 216                                   PIPE_CONTROL_STALL_AT_SCOREBOARD);
 217    }
 218
 219    if (query->Base.Target == GL_TIMESTAMP) {
 220       brw_load_register_mem64(brw,
 221                               HSW_CS_GPR(0),
 222                               query->bo,
 223                               I915_GEM_DOMAIN_INSTRUCTION,
 224                               I915_GEM_DOMAIN_INSTRUCTION,
 225                               0 * sizeof(uint64_t));
 226    } else {
 227       brw_load_register_mem64(brw,
 228                               HSW_CS_GPR(1),
 229                               query->bo,
 230                               I915_GEM_DOMAIN_INSTRUCTION,
 231                               I915_GEM_DOMAIN_INSTRUCTION,
 232                               0 * sizeof(uint64_t));
 233       brw_load_register_mem64(brw,
 234                               HSW_CS_GPR(2),
 235                               query->bo,
 236                               I915_GEM_DOMAIN_INSTRUCTION,
 237                               I915_GEM_DOMAIN_INSTRUCTION,
 238                               1 * sizeof(uint64_t));
 239
 240       BEGIN_BATCH(5);
 241       OUT_BATCH(HSW_MI_MATH | (5 - 2));
 242
 243       OUT_BATCH(MI_MATH_ALU2(LOAD, SRCA, R2));
 244       OUT_BATCH(MI_MATH_ALU2(LOAD, SRCB, R1));
 245       OUT_BATCH(MI_MATH_ALU0(SUB));
 246       OUT_BATCH(MI_MATH_ALU2(STORE, R0, ACCU));
 247
 248       ADVANCE_BATCH();
 249    }
 250
 251    switch (query->Base.Target) {
 252    case GL_FRAGMENT_SHADER_INVOCATIONS_ARB:
 253       /* Implement the "WaDividePSInvocationCountBy4:HSW,BDW" workaround:
 254        * "Invocation counter is 4 times actual.  WA: SW to divide HW reported
 255        *  PS Invocations value by 4."
 256        *
 257        * Prior to Haswell, invocation count was counted by the WM, and it
 258        * buggily counted invocations in units of subspans (2x2 unit). To get the
 259        * correct value, the CS multiplied this by 4. With HSW the logic moved,
 260        * and correctly emitted the number of pixel shader invocations, but,
 261        * whomever forgot to undo the multiply by 4.
 262        */
 263       if (brw->gen == 8 || brw->is_haswell)
 264          shr_gpr0_by_2_bits(brw);
 265       break;
 266    case GL_TIME_ELAPSED:
 267    case GL_TIMESTAMP:
 268       mult_gpr0_by_80(brw);
 269       if (query->Base.Target == GL_TIMESTAMP) {
 270          keep_gpr0_lower_n_bits(brw, 36);
 271       }
 272       break;
 273    case GL_ANY_SAMPLES_PASSED:
 274    case GL_ANY_SAMPLES_PASSED_CONSERVATIVE:
 275       gpr0_to_bool(brw);
 276       break;
 277    }
 278 }
 279
 280 /*
 281  * Store immediate data into the user buffer using the requested size.
 282  */
 283 static void
 284 store_query_result_imm(struct brw_context *brw, drm_intel_bo *bo,
 285                        uint32_t offset, GLenum ptype, uint64_t imm)
 286 {
 287    switch (ptype) {
 288    case GL_INT:
 289    case GL_UNSIGNED_INT:
 290       brw_store_data_imm32(brw, bo, offset, imm);
 291       break;
 292    case GL_INT64_ARB:
 293    case GL_UNSIGNED_INT64_ARB:
 294       brw_store_data_imm64(brw, bo, offset, imm);
 295       break;
 296    default:
 297       unreachable("Unexpected result type");
 298    }
 299 }
 300
 301 static void
 302 set_predicate(struct brw_context *brw, drm_intel_bo *query_bo)
 303 {
 304    brw_load_register_imm64(brw, MI_PREDICATE_SRC1, 0ull);
 305
 306    /* Load query availability into SRC0 */
 307    brw_load_register_mem64(brw, MI_PREDICATE_SRC0, query_bo,
 308                            I915_GEM_DOMAIN_INSTRUCTION, 0,
 309                            2 * sizeof(uint64_t));
 310
 311    /* predicate = !(query_availability == 0); */
 312    BEGIN_BATCH(1);
 313    OUT_BATCH(GEN7_MI_PREDICATE |
 314              MI_PREDICATE_LOADOP_LOADINV |
 315              MI_PREDICATE_COMBINEOP_SET |
 316              MI_PREDICATE_COMPAREOP_SRCS_EQUAL);
 317    ADVANCE_BATCH();
 318 }
 319
 320 /*
 321  * Store data from the register into the user buffer using the requested size.
 322  * The write also enables the predication to prevent writing the result if the
 323  * query has not finished yet.
 324  */
 325 static void
 326 store_query_result_reg(struct brw_context *brw, drm_intel_bo *bo,
 327                        uint32_t offset, GLenum ptype, uint32_t reg,
 328                        const bool pipelined)
 329 {
 330    uint32_t cmd_size = brw->gen >= 8 ? 4 : 3;
 331    uint32_t dwords = (ptype == GL_INT || ptype == GL_UNSIGNED_INT) ? 1 : 2;
 332    assert(brw->gen >= 6);
 333
 334    BEGIN_BATCH(dwords * cmd_size);
 335    for (int i = 0; i < dwords; i++) {
 336       OUT_BATCH(MI_STORE_REGISTER_MEM |
 337                 (pipelined ? MI_STORE_REGISTER_MEM_PREDICATE : 0) |
 338                 (cmd_size - 2));
 339       OUT_BATCH(reg + 4 * i);
 340       if (brw->gen >= 8) {
 341          OUT_RELOC64(bo, I915_GEM_DOMAIN_INSTRUCTION,
 342                      I915_GEM_DOMAIN_INSTRUCTION, offset + 4 * i);
 343       } else {
 344          OUT_RELOC(bo, I915_GEM_DOMAIN_INSTRUCTION,
 345                    I915_GEM_DOMAIN_INSTRUCTION, offset + 4 * i);
 346       }
 347    }
 348    ADVANCE_BATCH();
 349 }
 350
 351 static void
 352 hsw_store_query_result(struct gl_context *ctx, struct gl_query_object *q,
 353                        struct gl_buffer_object *buf, intptr_t offset,
 354                        GLenum pname, GLenum ptype)
 355 {
 356    struct brw_context *brw = brw_context(ctx);
 357    struct brw_query_object *query = (struct brw_query_object *)q;
 358    struct intel_buffer_object *bo = intel_buffer_object(buf);
 359    const bool pipelined = brw_is_query_pipelined(query);
 360
 361    if (pname == GL_QUERY_TARGET) {
 362       store_query_result_imm(brw, bo->buffer, offset, ptype,
 363                              query->Base.Target);
 364       return;
 365    } else if (pname == GL_QUERY_RESULT_AVAILABLE && !pipelined) {
 366       store_query_result_imm(brw, bo->buffer, offset, ptype, 1ull);
 367    } else if (query->bo) {
 368       /* The query bo still around. Therefore, we:
 369        *
 370        *  1. Compute the current result in GPR0
 371        *  2. Set the command streamer predicate based on query availability
 372        *  3. (With predication) Write GPR0 to the requested buffer
 373        */
 374       hsw_result_to_gpr0(ctx, query, buf, offset, pname, ptype);
 375       if (pipelined)
 376          set_predicate(brw, query->bo);
 377       store_query_result_reg(brw, bo->buffer, offset, ptype, HSW_CS_GPR(0),
 378                              pipelined);
 379    } else {
 380       /* The query bo is gone, so the query must have been processed into
 381        * client memory. In this case we can fill the buffer location with the
 382        * requested data using MI_STORE_DATA_IMM.
 383        */
 384       switch (pname) {
 385       case GL_QUERY_RESULT_AVAILABLE:
 386          store_query_result_imm(brw, bo->buffer, offset, ptype, 1ull);
 387          break;
 388       case GL_QUERY_RESULT_NO_WAIT:
 389       case GL_QUERY_RESULT:
 390          store_query_result_imm(brw, bo->buffer, offset, ptype,
 391                                 q->Result);
 392          break;
 393       default:
 394          unreachable("Unexpected result type");
 395       }
 396    }
 397
 398 }
 399
 400 /* Initialize hsw+-specific query object functions. */
 401 void hsw_init_queryobj_functions(struct dd_function_table *functions)
 402 {
 403    gen6_init_queryobj_functions(functions);
 404    functions->StoreQueryResult = hsw_store_query_result;
 405 }