src/mesa/drivers/dri/i965/gen7_sol_state.c

   1 /*
   2  * Copyright © 2011 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  */
  23
  24 /**
  25  * @file gen7_sol_state.c
  26  *
  27  * Controls the stream output logic (SOL) stage of the gen7 hardware, which is
  28  * used to implement GL_EXT_transform_feedback.
  29  */
  30
  31 #include "brw_context.h"
  32 #include "brw_state.h"
  33 #include "brw_defines.h"
  34 #include "intel_batchbuffer.h"
  35 #include "intel_buffer_objects.h"
  36 #include "main/transformfeedback.h"
  37
  38 static void
  39 upload_3dstate_so_buffers(struct brw_context *brw)
  40 {
  41    struct gl_context *ctx = &brw->ctx;
  42    /* BRW_NEW_TRANSFORM_FEEDBACK */
  43    struct gl_transform_feedback_object *xfb_obj =
  44       ctx->TransformFeedback.CurrentObject;
  45    const struct gl_transform_feedback_info *linked_xfb_info =
  46       &xfb_obj->shader_program->LinkedTransformFeedback;
  47    int i;
  48
  49    /* Set up the up to 4 output buffers.  These are the ranges defined in the
  50     * gl_transform_feedback_object.
  51     */
  52    for (i = 0; i < 4; i++) {
  53       struct intel_buffer_object *bufferobj =
  54          intel_buffer_object(xfb_obj->Buffers[i]);
  55       drm_intel_bo *bo;
  56       uint32_t start, end;
  57       uint32_t stride;
  58
  59       if (!xfb_obj->Buffers[i]) {
  60          /* The pitch of 0 in this command indicates that the buffer is
  61           * unbound and won't be written to.
  62           */
  63          BEGIN_BATCH(4);
  64          OUT_BATCH(_3DSTATE_SO_BUFFER << 16 | (4 - 2));
  65          OUT_BATCH((i << SO_BUFFER_INDEX_SHIFT));
  66          OUT_BATCH(0);
  67          OUT_BATCH(0);
  68          ADVANCE_BATCH();
  69
  70          continue;
  71       }
  72
  73       stride = linked_xfb_info->BufferStride[i] * 4;
  74
  75       start = xfb_obj->Offset[i];
  76       assert(start % 4 == 0);
  77       end = ALIGN(start + xfb_obj->Size[i], 4);
  78       bo = intel_bufferobj_buffer(brw, bufferobj, start, end - start);
  79       assert(end <= bo->size);
  80
  81       BEGIN_BATCH(4);
  82       OUT_BATCH(_3DSTATE_SO_BUFFER << 16 | (4 - 2));
  83       OUT_BATCH((i << SO_BUFFER_INDEX_SHIFT) | stride);
  84       OUT_RELOC(bo, I915_GEM_DOMAIN_RENDER, I915_GEM_DOMAIN_RENDER, start);
  85       OUT_RELOC(bo, I915_GEM_DOMAIN_RENDER, I915_GEM_DOMAIN_RENDER, end);
  86       ADVANCE_BATCH();
  87    }
  88 }
  89
  90 /**
  91  * Outputs the 3DSTATE_SO_DECL_LIST command.
  92  *
  93  * The data output is a series of 64-bit entries containing a SO_DECL per
  94  * stream.  We only have one stream of rendering coming out of the GS unit, so
  95  * we only emit stream 0 (low 16 bits) SO_DECLs.
  96  */
  97 void
  98 gen7_upload_3dstate_so_decl_list(struct brw_context *brw,
  99                                  const struct brw_vue_map *vue_map)
 100 {
 101    struct gl_context *ctx = &brw->ctx;
 102    /* BRW_NEW_TRANSFORM_FEEDBACK */
 103    struct gl_transform_feedback_object *xfb_obj =
 104       ctx->TransformFeedback.CurrentObject;
 105    const struct gl_transform_feedback_info *linked_xfb_info =
 106       &xfb_obj->shader_program->LinkedTransformFeedback;
 107    uint16_t so_decl[128];
 108    int buffer_mask = 0;
 109    int next_offset[4] = {0, 0, 0, 0};
 110    int decls = 0;
 111
 112    STATIC_ASSERT(ARRAY_SIZE(so_decl) >= MAX_PROGRAM_OUTPUTS);
 113
 114    /* Construct the list of SO_DECLs to be emitted.  The formatting of the
 115     * command is feels strange -- each dword pair contains a SO_DECL per stream.
 116     */
 117    for (int i = 0; i < linked_xfb_info->NumOutputs; i++) {
 118       int buffer = linked_xfb_info->Outputs[i].OutputBuffer;
 119       uint16_t decl = 0;
 120       int varying = linked_xfb_info->Outputs[i].OutputRegister;
 121       const unsigned components = linked_xfb_info->Outputs[i].NumComponents;
 122       unsigned component_mask = (1 << components) - 1;
 123
 124       /* gl_PointSize is stored in VARYING_SLOT_PSIZ.w
 125        * gl_Layer is stored in VARYING_SLOT_PSIZ.y
 126        */
 127       if (varying == VARYING_SLOT_PSIZ) {
 128          assert(components == 1);
 129          component_mask <<= 3;
 130       } else if (varying == VARYING_SLOT_LAYER) {
 131          assert(components == 1);
 132          component_mask <<= 1;
 133       } else {
 134          component_mask <<= linked_xfb_info->Outputs[i].ComponentOffset;
 135       }
 136
 137       buffer_mask |= 1 << buffer;
 138
 139       decl |= buffer << SO_DECL_OUTPUT_BUFFER_SLOT_SHIFT;
 140       if (varying == VARYING_SLOT_LAYER) {
 141          decl |= vue_map->varying_to_slot[VARYING_SLOT_PSIZ] <<
 142             SO_DECL_REGISTER_INDEX_SHIFT;
 143       } else {
 144          assert(vue_map->varying_to_slot[varying] >= 0);
 145          decl |= vue_map->varying_to_slot[varying] <<
 146             SO_DECL_REGISTER_INDEX_SHIFT;
 147       }
 148       decl |= component_mask << SO_DECL_COMPONENT_MASK_SHIFT;
 149
 150       /* Mesa doesn't store entries for gl_SkipComponents in the Outputs[]
 151        * array.  Instead, it simply increments DstOffset for the following
 152        * input by the number of components that should be skipped.
 153        *
 154        * Our hardware is unusual in that it requires us to program SO_DECLs
 155        * for fake "hole" components, rather than simply taking the offset
 156        * for each real varying.  Each hole can have size 1, 2, 3, or 4; we
 157        * program as many size = 4 holes as we can, then a final hole to
 158        * accomodate the final 1, 2, or 3 remaining.
 159        */
 160       int skip_components =
 161          linked_xfb_info->Outputs[i].DstOffset - next_offset[buffer];
 162
 163       next_offset[buffer] += skip_components;
 164
 165       while (skip_components >= 4) {
 166          so_decl[decls++] = SO_DECL_HOLE_FLAG | 0xf;
 167          skip_components -= 4;
 168       }
 169       if (skip_components > 0)
 170          so_decl[decls++] = SO_DECL_HOLE_FLAG | ((1 << skip_components) - 1);
 171
 172       assert(linked_xfb_info->Outputs[i].DstOffset == next_offset[buffer]);
 173
 174       next_offset[buffer] += components;
 175
 176       so_decl[decls++] = decl;
 177    }
 178
 179    BEGIN_BATCH(decls * 2 + 3);
 180    OUT_BATCH(_3DSTATE_SO_DECL_LIST << 16 | (decls * 2 + 1));
 181
 182    OUT_BATCH((buffer_mask << SO_STREAM_TO_BUFFER_SELECTS_0_SHIFT) |
 183              (0 << SO_STREAM_TO_BUFFER_SELECTS_1_SHIFT) |
 184              (0 << SO_STREAM_TO_BUFFER_SELECTS_2_SHIFT) |
 185              (0 << SO_STREAM_TO_BUFFER_SELECTS_3_SHIFT));
 186
 187    OUT_BATCH((decls << SO_NUM_ENTRIES_0_SHIFT) |
 188              (0 << SO_NUM_ENTRIES_1_SHIFT) |
 189              (0 << SO_NUM_ENTRIES_2_SHIFT) |
 190              (0 << SO_NUM_ENTRIES_3_SHIFT));
 191
 192    for (int i = 0; i < decls; i++) {
 193       OUT_BATCH(so_decl[i]);
 194       OUT_BATCH(0);
 195    }
 196
 197    ADVANCE_BATCH();
 198 }
 199
 200 static void
 201 upload_3dstate_streamout(struct brw_context *brw, bool active,
 202                          const struct brw_vue_map *vue_map)
 203 {
 204    struct gl_context *ctx = &brw->ctx;
 205    /* BRW_NEW_TRANSFORM_FEEDBACK */
 206    struct gl_transform_feedback_object *xfb_obj =
 207       ctx->TransformFeedback.CurrentObject;
 208    uint32_t dw1 = 0, dw2 = 0;
 209    int i;
 210
 211    if (active) {
 212       int urb_entry_read_offset = 0;
 213       int urb_entry_read_length = (vue_map->num_slots + 1) / 2 -
 214          urb_entry_read_offset;
 215
 216       dw1 |= SO_FUNCTION_ENABLE;
 217       dw1 |= SO_STATISTICS_ENABLE;
 218
 219       /* _NEW_LIGHT */
 220       if (ctx->Light.ProvokingVertex != GL_FIRST_VERTEX_CONVENTION)
 221          dw1 |= SO_REORDER_TRAILING;
 222
 223       for (i = 0; i < 4; i++) {
 224          if (xfb_obj->Buffers[i]) {
 225             dw1 |= SO_BUFFER_ENABLE(i);
 226          }
 227       }
 228
 229       /* We always read the whole vertex.  This could be reduced at some
 230        * point by reading less and offsetting the register index in the
 231        * SO_DECLs.
 232        */
 233       dw2 |= urb_entry_read_offset << SO_STREAM_0_VERTEX_READ_OFFSET_SHIFT;
 234       dw2 |= (urb_entry_read_length - 1) <<
 235          SO_STREAM_0_VERTEX_READ_LENGTH_SHIFT;
 236    }
 237
 238    BEGIN_BATCH(3);
 239    OUT_BATCH(_3DSTATE_STREAMOUT << 16 | (3 - 2));
 240    OUT_BATCH(dw1);
 241    OUT_BATCH(dw2);
 242    ADVANCE_BATCH();
 243 }
 244
 245 static void
 246 upload_sol_state(struct brw_context *brw)
 247 {
 248    struct gl_context *ctx = &brw->ctx;
 249    /* BRW_NEW_TRANSFORM_FEEDBACK */
 250    bool active = _mesa_is_xfb_active_and_unpaused(ctx);
 251
 252    if (active) {
 253       upload_3dstate_so_buffers(brw);
 254       /* BRW_NEW_VUE_MAP_GEOM_OUT */
 255       gen7_upload_3dstate_so_decl_list(brw, &brw->vue_map_geom_out);
 256    }
 257
 258    /* Finally, set up the SOL stage.  This command must always follow updates to
 259     * the nonpipelined SOL state (3DSTATE_SO_BUFFER, 3DSTATE_SO_DECL_LIST) or
 260     * MMIO register updates (current performed by the kernel at each batch
 261     * emit).
 262     */
 263    upload_3dstate_streamout(brw, active, &brw->vue_map_geom_out);
 264 }
 265
 266 const struct brw_tracked_state gen7_sol_state = {
 267    .dirty = {
 268       .mesa  = (_NEW_LIGHT),
 269       .brw   = (BRW_NEW_BATCH |
 270                 BRW_NEW_VUE_MAP_GEOM_OUT |
 271                 BRW_NEW_TRANSFORM_FEEDBACK)
 272    },
 273    .emit = upload_sol_state,
 274 };
 275
 276 /**
 277  * Tally the number of primitives generated so far.
 278  *
 279  * The buffer contains a series of pairs:
 280  * (<start0, start1, start2, start3>, <end0, end1, end2, end3>) ;
 281  * (<start0, start1, start2, start3>, <end0, end1, end2, end3>) ;
 282  *
 283  * For each stream, we subtract the pair of values (end - start) to get the
 284  * number of primitives generated during one section.  We accumulate these
 285  * values, adding them up to get the total number of primitives generated.
 286  */
 287 static void
 288 gen7_tally_prims_generated(struct brw_context *brw,
 289                            struct brw_transform_feedback_object *obj)
 290 {
 291    /* If the current batch is still contributing to the number of primitives
 292     * generated, flush it now so the results will be present when mapped.
 293     */
 294    if (drm_intel_bo_references(brw->batch.bo, obj->prim_count_bo))
 295       intel_batchbuffer_flush(brw);
 296
 297    if (unlikely(brw->perf_debug && drm_intel_bo_busy(obj->prim_count_bo)))
 298       perf_debug("Stalling for # of transform feedback primitives written.\n");
 299
 300    drm_intel_bo_map(obj->prim_count_bo, false);
 301    uint64_t *prim_counts = obj->prim_count_bo->virtual;
 302
 303    assert(obj->prim_count_buffer_index % (2 * BRW_MAX_XFB_STREAMS) == 0);
 304    int pairs = obj->prim_count_buffer_index / (2 * BRW_MAX_XFB_STREAMS);
 305
 306    for (int i = 0; i < pairs; i++) {
 307       for (int s = 0; s < BRW_MAX_XFB_STREAMS; s++) {
 308          obj->prims_generated[s] +=
 309             prim_counts[BRW_MAX_XFB_STREAMS + s] - prim_counts[s];
 310       }
 311       prim_counts += 2 * BRW_MAX_XFB_STREAMS; /* move to the next pair */
 312    }
 313
 314    drm_intel_bo_unmap(obj->prim_count_bo);
 315
 316    /* We've already gathered up the old data; we can safely overwrite it now. */
 317    obj->prim_count_buffer_index = 0;
 318 }
 319
 320 /**
 321  * Store the SO_NUM_PRIMS_WRITTEN counters for each stream (4 uint64_t values)
 322  * to prim_count_bo.
 323  *
 324  * If prim_count_bo is out of space, gather up the results so far into
 325  * prims_generated[] and allocate a new buffer with enough space.
 326  *
 327  * The number of primitives written is used to compute the number of vertices
 328  * written to a transform feedback stream, which is required to implement
 329  * DrawTransformFeedback().
 330  */
 331 static void
 332 gen7_save_primitives_written_counters(struct brw_context *brw,
 333                                 struct brw_transform_feedback_object *obj)
 334 {
 335    const int streams = BRW_MAX_XFB_STREAMS;
 336
 337    /* Check if there's enough space for a new pair of four values. */
 338    if (obj->prim_count_bo != NULL &&
 339        obj->prim_count_buffer_index + 2 * streams >= 4096 / sizeof(uint64_t)) {
 340       /* Gather up the results so far and release the BO. */
 341       gen7_tally_prims_generated(brw, obj);
 342    }
 343
 344    /* Flush any drawing so that the counters have the right values. */
 345    intel_batchbuffer_emit_mi_flush(brw);
 346
 347    /* Emit MI_STORE_REGISTER_MEM commands to write the values. */
 348    for (int i = 0; i < streams; i++) {
 349       brw_store_register_mem64(brw, obj->prim_count_bo,
 350                                GEN7_SO_NUM_PRIMS_WRITTEN(i),
 351                                obj->prim_count_buffer_index + i);
 352    }
 353
 354    /* Update where to write data to. */
 355    obj->prim_count_buffer_index += streams;
 356 }
 357
 358 /**
 359  * Compute the number of vertices written by this transform feedback operation.
 360  */
 361 static void
 362 brw_compute_xfb_vertices_written(struct brw_context *brw,
 363                                  struct brw_transform_feedback_object *obj)
 364 {
 365    if (obj->vertices_written_valid || !obj->base.EndedAnytime)
 366       return;
 367
 368    unsigned vertices_per_prim = 0;
 369
 370    switch (obj->primitive_mode) {
 371    case GL_POINTS:
 372       vertices_per_prim = 1;
 373       break;
 374    case GL_LINES:
 375       vertices_per_prim = 2;
 376       break;
 377    case GL_TRIANGLES:
 378       vertices_per_prim = 3;
 379       break;
 380    default:
 381       assert(!"Invalid transform feedback primitive mode.");
 382    }
 383
 384    /* Get the number of primitives generated. */
 385    gen7_tally_prims_generated(brw, obj);
 386
 387    for (int i = 0; i < BRW_MAX_XFB_STREAMS; i++) {
 388       obj->vertices_written[i] = vertices_per_prim * obj->prims_generated[i];
 389    }
 390    obj->vertices_written_valid = true;
 391 }
 392
 393 /**
 394  * GetTransformFeedbackVertexCount() driver hook.
 395  *
 396  * Returns the number of vertices written to a particular stream by the last
 397  * Begin/EndTransformFeedback block.  Used to implement DrawTransformFeedback().
 398  */
 399 GLsizei
 400 brw_get_transform_feedback_vertex_count(struct gl_context *ctx,
 401                                         struct gl_transform_feedback_object *obj,
 402                                         GLuint stream)
 403 {
 404    struct brw_context *brw = brw_context(ctx);
 405    struct brw_transform_feedback_object *brw_obj =
 406       (struct brw_transform_feedback_object *) obj;
 407
 408    assert(obj->EndedAnytime);
 409    assert(stream < BRW_MAX_XFB_STREAMS);
 410
 411    brw_compute_xfb_vertices_written(brw, brw_obj);
 412    return brw_obj->vertices_written[stream];
 413 }
 414
 415 void
 416 gen7_begin_transform_feedback(struct gl_context *ctx, GLenum mode,
 417                               struct gl_transform_feedback_object *obj)
 418 {
 419    struct brw_context *brw = brw_context(ctx);
 420    struct brw_transform_feedback_object *brw_obj =
 421       (struct brw_transform_feedback_object *) obj;
 422
 423    /* Reset the SO buffer offsets to 0. */
 424    if (brw->gen >= 8) {
 425       brw_obj->zero_offsets = true;
 426    } else {
 427       intel_batchbuffer_flush(brw);
 428       brw->batch.needs_sol_reset = true;
 429    }
 430
 431    /* We're about to lose the information needed to compute the number of
 432     * vertices written during the last Begin/EndTransformFeedback section,
 433     * so we can't delay it any further.
 434     */
 435    brw_compute_xfb_vertices_written(brw, brw_obj);
 436
 437    /* No primitives have been generated yet. */
 438    for (int i = 0; i < BRW_MAX_XFB_STREAMS; i++) {
 439       brw_obj->prims_generated[i] = 0;
 440    }
 441
 442    /* Store the starting value of the SO_NUM_PRIMS_WRITTEN counters. */
 443    gen7_save_primitives_written_counters(brw, brw_obj);
 444
 445    brw_obj->primitive_mode = mode;
 446 }
 447
 448 void
 449 gen7_end_transform_feedback(struct gl_context *ctx,
 450                             struct gl_transform_feedback_object *obj)
 451 {
 452    /* After EndTransformFeedback, it's likely that the client program will try
 453     * to draw using the contents of the transform feedback buffer as vertex
 454     * input.  In order for this to work, we need to flush the data through at
 455     * least the GS stage of the pipeline, and flush out the render cache.  For
 456     * simplicity, just do a full flush.
 457     */
 458    struct brw_context *brw = brw_context(ctx);
 459    struct brw_transform_feedback_object *brw_obj =
 460       (struct brw_transform_feedback_object *) obj;
 461
 462    /* Store the ending value of the SO_NUM_PRIMS_WRITTEN counters. */
 463    gen7_save_primitives_written_counters(brw, brw_obj);
 464
 465    /* EndTransformFeedback() means that we need to update the number of
 466     * vertices written.  Since it's only necessary if DrawTransformFeedback()
 467     * is called and it means mapping a buffer object, we delay computing it
 468     * until it's absolutely necessary to try and avoid stalls.
 469     */
 470    brw_obj->vertices_written_valid = false;
 471 }
 472
 473 void
 474 gen7_pause_transform_feedback(struct gl_context *ctx,
 475                               struct gl_transform_feedback_object *obj)
 476 {
 477    struct brw_context *brw = brw_context(ctx);
 478    struct brw_transform_feedback_object *brw_obj =
 479       (struct brw_transform_feedback_object *) obj;
 480
 481    /* Flush any drawing so that the counters have the right values. */
 482    intel_batchbuffer_emit_mi_flush(brw);
 483
 484    /* Save the SOL buffer offset register values. */
 485    if (brw->gen < 8) {
 486       for (int i = 0; i < 4; i++) {
 487          BEGIN_BATCH(3);
 488          OUT_BATCH(MI_STORE_REGISTER_MEM | (3 - 2));
 489          OUT_BATCH(GEN7_SO_WRITE_OFFSET(i));
 490          OUT_RELOC(brw_obj->offset_bo,
 491                    I915_GEM_DOMAIN_INSTRUCTION, I915_GEM_DOMAIN_INSTRUCTION,
 492                    i * sizeof(uint32_t));
 493          ADVANCE_BATCH();
 494       }
 495    }
 496
 497    /* Store the temporary ending value of the SO_NUM_PRIMS_WRITTEN counters.
 498     * While this operation is paused, other transform feedback actions may
 499     * occur, which will contribute to the counters.  We need to exclude that
 500     * from our counts.
 501     */
 502    gen7_save_primitives_written_counters(brw, brw_obj);
 503 }
 504
 505 void
 506 gen7_resume_transform_feedback(struct gl_context *ctx,
 507                                struct gl_transform_feedback_object *obj)
 508 {
 509    struct brw_context *brw = brw_context(ctx);
 510    struct brw_transform_feedback_object *brw_obj =
 511       (struct brw_transform_feedback_object *) obj;
 512
 513    /* Reload the SOL buffer offset registers. */
 514    if (brw->gen < 8) {
 515       for (int i = 0; i < 4; i++) {
 516          BEGIN_BATCH(3);
 517          OUT_BATCH(GEN7_MI_LOAD_REGISTER_MEM | (3 - 2));
 518          OUT_BATCH(GEN7_SO_WRITE_OFFSET(i));
 519          OUT_RELOC(brw_obj->offset_bo,
 520                    I915_GEM_DOMAIN_INSTRUCTION, I915_GEM_DOMAIN_INSTRUCTION,
 521                    i * sizeof(uint32_t));
 522          ADVANCE_BATCH();
 523       }
 524    }
 525
 526    /* Store the new starting value of the SO_NUM_PRIMS_WRITTEN counters. */
 527    gen7_save_primitives_written_counters(brw, brw_obj);
 528 }