src/mesa/drivers/dri/i965/hsw_queryobj.c

   1 /*
   2  * Copyright (c) 2016 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  *
  23  */
  24
  25 /** @file hsw_queryobj.c
  26  *
  27  * Support for query buffer objects (GL_ARB_query_buffer_object) on Haswell+.
  28  */
  29 #include "main/imports.h"
  30
  31 #include "brw_context.h"
  32 #include "brw_defines.h"
  33 #include "intel_batchbuffer.h"
  34 #include "intel_buffer_objects.h"
  35 #include "intel_reg.h"
  36
  37 /*
  38  * GPR0 = 80 * GPR0;
  39  */
  40 static void
  41 mult_gpr0_by_80(struct brw_context *brw)
  42 {
  43    static const uint32_t maths[] = {
  44       MI_MATH_ALU2(LOAD, SRCA, R0),
  45       MI_MATH_ALU2(LOAD, SRCB, R0),
  46       MI_MATH_ALU0(ADD),
  47       MI_MATH_ALU2(STORE, R1, ACCU),
  48       MI_MATH_ALU2(LOAD, SRCA, R1),
  49       MI_MATH_ALU2(LOAD, SRCB, R1),
  50       MI_MATH_ALU0(ADD),
  51       MI_MATH_ALU2(STORE, R1, ACCU),
  52       MI_MATH_ALU2(LOAD, SRCA, R1),
  53       MI_MATH_ALU2(LOAD, SRCB, R1),
  54       MI_MATH_ALU0(ADD),
  55       MI_MATH_ALU2(STORE, R1, ACCU),
  56       MI_MATH_ALU2(LOAD, SRCA, R1),
  57       MI_MATH_ALU2(LOAD, SRCB, R1),
  58       MI_MATH_ALU0(ADD),
  59       /* GPR1 = 16 * GPR0 */
  60       MI_MATH_ALU2(STORE, R1, ACCU),
  61       MI_MATH_ALU2(LOAD, SRCA, R1),
  62       MI_MATH_ALU2(LOAD, SRCB, R1),
  63       MI_MATH_ALU0(ADD),
  64       MI_MATH_ALU2(STORE, R2, ACCU),
  65       MI_MATH_ALU2(LOAD, SRCA, R2),
  66       MI_MATH_ALU2(LOAD, SRCB, R2),
  67       MI_MATH_ALU0(ADD),
  68       /* GPR2 = 64 * GPR0 */
  69       MI_MATH_ALU2(STORE, R2, ACCU),
  70       MI_MATH_ALU2(LOAD, SRCA, R1),
  71       MI_MATH_ALU2(LOAD, SRCB, R2),
  72       MI_MATH_ALU0(ADD),
  73       /* GPR0 = 80 * GPR0 */
  74       MI_MATH_ALU2(STORE, R0, ACCU),
  75    };
  76
  77    BEGIN_BATCH(1 + ARRAY_SIZE(maths));
  78    OUT_BATCH(HSW_MI_MATH | (1 + ARRAY_SIZE(maths) - 2));
  79
  80    for (int m = 0; m < ARRAY_SIZE(maths); m++)
  81       OUT_BATCH(maths[m]);
  82
  83    ADVANCE_BATCH();
  84 }
  85
  86 /*
  87  * GPR0 = GPR0 & ((1ull << n) - 1);
  88  */
  89 static void
  90 keep_gpr0_lower_n_bits(struct brw_context *brw, uint32_t n)
  91 {
  92    static const uint32_t maths[] = {
  93       MI_MATH_ALU2(LOAD, SRCA, R0),
  94       MI_MATH_ALU2(LOAD, SRCB, R1),
  95       MI_MATH_ALU0(AND),
  96       MI_MATH_ALU2(STORE, R0, ACCU),
  97    };
  98
  99    assert(n < 64);
 100    brw_load_register_imm64(brw, HSW_CS_GPR(1), (1ull << n) - 1);
 101
 102    BEGIN_BATCH(1 + ARRAY_SIZE(maths));
 103    OUT_BATCH(HSW_MI_MATH | (1 + ARRAY_SIZE(maths) - 2));
 104
 105    for (int m = 0; m < ARRAY_SIZE(maths); m++)
 106       OUT_BATCH(maths[m]);
 107
 108    ADVANCE_BATCH();
 109 }
 110
 111 /*
 112  * GPR0 = GPR0 << 30;
 113  */
 114 static void
 115 shl_gpr0_by_30_bits(struct brw_context *brw)
 116 {
 117    /* First we mask 34 bits of GPR0 to prevent overflow */
 118    keep_gpr0_lower_n_bits(brw, 34);
 119
 120    static const uint32_t shl_maths[] = {
 121       MI_MATH_ALU2(LOAD, SRCA, R0),
 122       MI_MATH_ALU2(LOAD, SRCB, R0),
 123       MI_MATH_ALU0(ADD),
 124       MI_MATH_ALU2(STORE, R0, ACCU),
 125    };
 126
 127    const uint32_t outer_count = 5;
 128    const uint32_t inner_count = 6;
 129    STATIC_ASSERT(outer_count * inner_count == 30);
 130    const uint32_t cmd_len = 1 + inner_count * ARRAY_SIZE(shl_maths);
 131    const uint32_t batch_len = cmd_len * outer_count;
 132
 133    BEGIN_BATCH(batch_len);
 134
 135    /* We'll emit 5 commands, each shifting GPR0 left by 6 bits, for a total of
 136     * 30 left shifts.
 137     */
 138    for (int o = 0; o < outer_count; o++) {
 139       /* Submit one MI_MATH to shift left by 6 bits */
 140       OUT_BATCH(HSW_MI_MATH | (cmd_len - 2));
 141       for (int i = 0; i < inner_count; i++)
 142          for (int m = 0; m < ARRAY_SIZE(shl_maths); m++)
 143             OUT_BATCH(shl_maths[m]);
 144    }
 145
 146    ADVANCE_BATCH();
 147 }
 148
 149 /*
 150  * GPR0 = GPR0 >> 2;
 151  *
 152  * Note that the upper 30 bits of GPR0 are lost!
 153  */
 154 static void
 155 shr_gpr0_by_2_bits(struct brw_context *brw)
 156 {
 157    shl_gpr0_by_30_bits(brw);
 158    brw_load_register_reg(brw, HSW_CS_GPR(0) + 4, HSW_CS_GPR(0));
 159    brw_load_register_imm32(brw, HSW_CS_GPR(0) + 4, 0);
 160 }
 161
 162 /*
 163  * GPR0 = (GPR0 == 0) ? 0 : 1;
 164  */
 165 static void
 166 gpr0_to_bool(struct brw_context *brw)
 167 {
 168    static const uint32_t maths[] = {
 169       MI_MATH_ALU2(LOAD, SRCA, R0),
 170       MI_MATH_ALU1(LOAD0, SRCB),
 171       MI_MATH_ALU0(ADD),
 172       MI_MATH_ALU2(STOREINV, R0, ZF),
 173       MI_MATH_ALU2(LOAD, SRCA, R0),
 174       MI_MATH_ALU2(LOAD, SRCB, R1),
 175       MI_MATH_ALU0(AND),
 176       MI_MATH_ALU2(STORE, R0, ACCU),
 177    };
 178
 179    brw_load_register_imm64(brw, HSW_CS_GPR(1), 1ull);
 180
 181    BEGIN_BATCH(1 + ARRAY_SIZE(maths));
 182    OUT_BATCH(HSW_MI_MATH | (1 + ARRAY_SIZE(maths) - 2));
 183
 184    for (int m = 0; m < ARRAY_SIZE(maths); m++)
 185       OUT_BATCH(maths[m]);
 186
 187    ADVANCE_BATCH();
 188 }
 189
 190 static void
 191 hsw_result_to_gpr0(struct gl_context *ctx, struct brw_query_object *query,
 192                    struct gl_buffer_object *buf, intptr_t offset,
 193                    GLenum pname, GLenum ptype)
 194 {
 195    struct brw_context *brw = brw_context(ctx);
 196
 197    assert(query->bo);
 198    assert(pname != GL_QUERY_TARGET);
 199
 200    if (pname == GL_QUERY_RESULT_AVAILABLE) {
 201       /* The query result availability is stored at offset 0 of the buffer. */
 202       brw_load_register_mem64(brw,
 203                               HSW_CS_GPR(0),
 204                               query->bo,
 205                               I915_GEM_DOMAIN_INSTRUCTION,
 206                               I915_GEM_DOMAIN_INSTRUCTION,
 207                               2 * sizeof(uint64_t));
 208       return;
 209    }
 210
 211    if (pname == GL_QUERY_RESULT) {
 212       /* Since GL_QUERY_RESULT_NO_WAIT wasn't used, they want us to stall to
 213        * make sure the query is available.
 214        */
 215       brw_emit_pipe_control_flush(brw,
 216                                   PIPE_CONTROL_CS_STALL |
 217                                   PIPE_CONTROL_STALL_AT_SCOREBOARD);
 218    }
 219
 220    if (query->Base.Target == GL_TIMESTAMP) {
 221       brw_load_register_mem64(brw,
 222                               HSW_CS_GPR(0),
 223                               query->bo,
 224                               I915_GEM_DOMAIN_INSTRUCTION,
 225                               I915_GEM_DOMAIN_INSTRUCTION,
 226                               0 * sizeof(uint64_t));
 227    } else {
 228       brw_load_register_mem64(brw,
 229                               HSW_CS_GPR(1),
 230                               query->bo,
 231                               I915_GEM_DOMAIN_INSTRUCTION,
 232                               I915_GEM_DOMAIN_INSTRUCTION,
 233                               0 * sizeof(uint64_t));
 234       brw_load_register_mem64(brw,
 235                               HSW_CS_GPR(2),
 236                               query->bo,
 237                               I915_GEM_DOMAIN_INSTRUCTION,
 238                               I915_GEM_DOMAIN_INSTRUCTION,
 239                               1 * sizeof(uint64_t));
 240
 241       BEGIN_BATCH(5);
 242       OUT_BATCH(HSW_MI_MATH | (5 - 2));
 243
 244       OUT_BATCH(MI_MATH_ALU2(LOAD, SRCA, R2));
 245       OUT_BATCH(MI_MATH_ALU2(LOAD, SRCB, R1));
 246       OUT_BATCH(MI_MATH_ALU0(SUB));
 247       OUT_BATCH(MI_MATH_ALU2(STORE, R0, ACCU));
 248
 249       ADVANCE_BATCH();
 250    }
 251
 252    switch (query->Base.Target) {
 253    case GL_FRAGMENT_SHADER_INVOCATIONS_ARB:
 254       /* Implement the "WaDividePSInvocationCountBy4:HSW,BDW" workaround:
 255        * "Invocation counter is 4 times actual.  WA: SW to divide HW reported
 256        *  PS Invocations value by 4."
 257        *
 258        * Prior to Haswell, invocation count was counted by the WM, and it
 259        * buggily counted invocations in units of subspans (2x2 unit). To get the
 260        * correct value, the CS multiplied this by 4. With HSW the logic moved,
 261        * and correctly emitted the number of pixel shader invocations, but,
 262        * whomever forgot to undo the multiply by 4.
 263        */
 264       if (brw->gen == 8 || brw->is_haswell)
 265          shr_gpr0_by_2_bits(brw);
 266       break;
 267    case GL_TIME_ELAPSED:
 268    case GL_TIMESTAMP:
 269       mult_gpr0_by_80(brw);
 270       if (query->Base.Target == GL_TIMESTAMP) {
 271          keep_gpr0_lower_n_bits(brw, 36);
 272       }
 273       break;
 274    case GL_ANY_SAMPLES_PASSED:
 275    case GL_ANY_SAMPLES_PASSED_CONSERVATIVE:
 276       gpr0_to_bool(brw);
 277       break;
 278    }
 279 }
 280
 281 /*
 282  * Store immediate data into the user buffer using the requested size.
 283  */
 284 static void
 285 store_query_result_imm(struct brw_context *brw, drm_intel_bo *bo,
 286                        uint32_t offset, GLenum ptype, uint64_t imm)
 287 {
 288    switch (ptype) {
 289    case GL_INT:
 290    case GL_UNSIGNED_INT:
 291       brw_store_data_imm32(brw, bo, offset, imm);
 292       break;
 293    case GL_INT64_ARB:
 294    case GL_UNSIGNED_INT64_ARB:
 295       brw_store_data_imm64(brw, bo, offset, imm);
 296       break;
 297    default:
 298       unreachable("Unexpected result type");
 299    }
 300 }
 301
 302 static void
 303 set_predicate(struct brw_context *brw, drm_intel_bo *query_bo)
 304 {
 305    brw_load_register_imm64(brw, MI_PREDICATE_SRC1, 0ull);
 306
 307    /* Load query availability into SRC0 */
 308    brw_load_register_mem64(brw, MI_PREDICATE_SRC0, query_bo,
 309                            I915_GEM_DOMAIN_INSTRUCTION, 0,
 310                            2 * sizeof(uint64_t));
 311
 312    /* predicate = !(query_availability == 0); */
 313    BEGIN_BATCH(1);
 314    OUT_BATCH(GEN7_MI_PREDICATE |
 315              MI_PREDICATE_LOADOP_LOADINV |
 316              MI_PREDICATE_COMBINEOP_SET |
 317              MI_PREDICATE_COMPAREOP_SRCS_EQUAL);
 318    ADVANCE_BATCH();
 319 }
 320
 321 /*
 322  * Store data from the register into the user buffer using the requested size.
 323  * The write also enables the predication to prevent writing the result if the
 324  * query has not finished yet.
 325  */
 326 static void
 327 store_query_result_reg(struct brw_context *brw, drm_intel_bo *bo,
 328                        uint32_t offset, GLenum ptype, uint32_t reg,
 329                        const bool pipelined)
 330 {
 331    uint32_t cmd_size = brw->gen >= 8 ? 4 : 3;
 332    uint32_t dwords = (ptype == GL_INT || ptype == GL_UNSIGNED_INT) ? 1 : 2;
 333    assert(brw->gen >= 6);
 334
 335    BEGIN_BATCH(dwords * cmd_size);
 336    for (int i = 0; i < dwords; i++) {
 337       OUT_BATCH(MI_STORE_REGISTER_MEM |
 338                 (pipelined ? MI_STORE_REGISTER_MEM_PREDICATE : 0) |
 339                 (cmd_size - 2));
 340       OUT_BATCH(reg + 4 * i);
 341       if (brw->gen >= 8) {
 342          OUT_RELOC64(bo, I915_GEM_DOMAIN_INSTRUCTION,
 343                      I915_GEM_DOMAIN_INSTRUCTION, offset + 4 * i);
 344       } else {
 345          OUT_RELOC(bo, I915_GEM_DOMAIN_INSTRUCTION,
 346                    I915_GEM_DOMAIN_INSTRUCTION, offset + 4 * i);
 347       }
 348    }
 349    ADVANCE_BATCH();
 350 }
 351
 352 static void
 353 hsw_store_query_result(struct gl_context *ctx, struct gl_query_object *q,
 354                        struct gl_buffer_object *buf, intptr_t offset,
 355                        GLenum pname, GLenum ptype)
 356 {
 357    struct brw_context *brw = brw_context(ctx);
 358    struct brw_query_object *query = (struct brw_query_object *)q;
 359    struct intel_buffer_object *bo = intel_buffer_object(buf);
 360    const bool pipelined = brw_is_query_pipelined(query);
 361
 362    if (pname == GL_QUERY_TARGET) {
 363       store_query_result_imm(brw, bo->buffer, offset, ptype,
 364                              query->Base.Target);
 365       return;
 366    } else if (pname == GL_QUERY_RESULT_AVAILABLE && !pipelined) {
 367       store_query_result_imm(brw, bo->buffer, offset, ptype, 1ull);
 368    } else if (query->bo) {
 369       /* The query bo still around. Therefore, we:
 370        *
 371        *  1. Compute the current result in GPR0
 372        *  2. Set the command streamer predicate based on query availability
 373        *  3. (With predication) Write GPR0 to the requested buffer
 374        */
 375       hsw_result_to_gpr0(ctx, query, buf, offset, pname, ptype);
 376       if (pipelined)
 377          set_predicate(brw, query->bo);
 378       store_query_result_reg(brw, bo->buffer, offset, ptype, HSW_CS_GPR(0),
 379                              pipelined);
 380    } else {
 381       /* The query bo is gone, so the query must have been processed into
 382        * client memory. In this case we can fill the buffer location with the
 383        * requested data using MI_STORE_DATA_IMM.
 384        */
 385       switch (pname) {
 386       case GL_QUERY_RESULT_AVAILABLE:
 387          store_query_result_imm(brw, bo->buffer, offset, ptype, 1ull);
 388          break;
 389       case GL_QUERY_RESULT_NO_WAIT:
 390       case GL_QUERY_RESULT:
 391          store_query_result_imm(brw, bo->buffer, offset, ptype,
 392                                 q->Result);
 393          break;
 394       default:
 395          unreachable("Unexpected result type");
 396       }
 397    }
 398
 399 }
 400
 401 /* Initialize hsw+-specific query object functions. */
 402 void hsw_init_queryobj_functions(struct dd_function_table *functions)
 403 {
 404    gen6_init_queryobj_functions(functions);
 405    functions->StoreQueryResult = hsw_store_query_result;
 406 }