src/mesa/drivers/dri/i965/gen7_sol_state.c

   1 /*
   2  * Copyright © 2011 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  */
  23
  24 /**
  25  * @file gen7_sol_state.c
  26  *
  27  * Controls the stream output logic (SOL) stage of the gen7 hardware, which is
  28  * used to implement GL_EXT_transform_feedback.
  29  */
  30
  31 #include "brw_context.h"
  32 #include "brw_state.h"
  33 #include "brw_defines.h"
  34 #include "intel_batchbuffer.h"
  35 #include "intel_buffer_objects.h"
  36 #include "main/transformfeedback.h"
  37
  38 static void
  39 upload_3dstate_so_buffers(struct brw_context *brw)
  40 {
  41    struct gl_context *ctx = &brw->ctx;
  42    /* BRW_NEW_TRANSFORM_FEEDBACK */
  43    struct gl_transform_feedback_object *xfb_obj =
  44       ctx->TransformFeedback.CurrentObject;
  45    const struct gl_transform_feedback_info *linked_xfb_info =
  46       &xfb_obj->shader_program->LinkedTransformFeedback;
  47    int i;
  48
  49    /* Set up the up to 4 output buffers.  These are the ranges defined in the
  50     * gl_transform_feedback_object.
  51     */
  52    for (i = 0; i < 4; i++) {
  53       struct intel_buffer_object *bufferobj =
  54          intel_buffer_object(xfb_obj->Buffers[i]);
  55       drm_intel_bo *bo;
  56       uint32_t start, end;
  57       uint32_t stride;
  58
  59       if (!xfb_obj->Buffers[i]) {
  60          /* The pitch of 0 in this command indicates that the buffer is
  61           * unbound and won't be written to.
  62           */
  63          BEGIN_BATCH(4);
  64          OUT_BATCH(_3DSTATE_SO_BUFFER << 16 | (4 - 2));
  65          OUT_BATCH((i << SO_BUFFER_INDEX_SHIFT));
  66          OUT_BATCH(0);
  67          OUT_BATCH(0);
  68          ADVANCE_BATCH();
  69
  70          continue;
  71       }
  72
  73       stride = linked_xfb_info->BufferStride[i] * 4;
  74
  75       start = xfb_obj->Offset[i];
  76       assert(start % 4 == 0);
  77       end = ALIGN(start + xfb_obj->Size[i], 4);
  78       bo = intel_bufferobj_buffer(brw, bufferobj, start, end - start);
  79       assert(end <= bo->size);
  80
  81       BEGIN_BATCH(4);
  82       OUT_BATCH(_3DSTATE_SO_BUFFER << 16 | (4 - 2));
  83       OUT_BATCH((i << SO_BUFFER_INDEX_SHIFT) | stride);
  84       OUT_RELOC(bo, I915_GEM_DOMAIN_RENDER, I915_GEM_DOMAIN_RENDER, start);
  85       OUT_RELOC(bo, I915_GEM_DOMAIN_RENDER, I915_GEM_DOMAIN_RENDER, end);
  86       ADVANCE_BATCH();
  87    }
  88 }
  89
  90 /**
  91  * Outputs the 3DSTATE_SO_DECL_LIST command.
  92  *
  93  * The data output is a series of 64-bit entries containing a SO_DECL per
  94  * stream.  We only have one stream of rendering coming out of the GS unit, so
  95  * we only emit stream 0 (low 16 bits) SO_DECLs.
  96  */
  97 void
  98 gen7_upload_3dstate_so_decl_list(struct brw_context *brw,
  99                                  const struct brw_vue_map *vue_map)
 100 {
 101    struct gl_context *ctx = &brw->ctx;
 102    /* BRW_NEW_TRANSFORM_FEEDBACK */
 103    struct gl_transform_feedback_object *xfb_obj =
 104       ctx->TransformFeedback.CurrentObject;
 105    const struct gl_transform_feedback_info *linked_xfb_info =
 106       &xfb_obj->shader_program->LinkedTransformFeedback;
 107    uint16_t so_decl[128];
 108    int buffer_mask = 0;
 109    int next_offset[4] = {0, 0, 0, 0};
 110    int decls = 0;
 111
 112    STATIC_ASSERT(ARRAY_SIZE(so_decl) >= MAX_PROGRAM_OUTPUTS);
 113
 114    /* Construct the list of SO_DECLs to be emitted.  The formatting of the
 115     * command is feels strange -- each dword pair contains a SO_DECL per stream.
 116     */
 117    for (int i = 0; i < linked_xfb_info->NumOutputs; i++) {
 118       int buffer = linked_xfb_info->Outputs[i].OutputBuffer;
 119       uint16_t decl = 0;
 120       int varying = linked_xfb_info->Outputs[i].OutputRegister;
 121       const unsigned components = linked_xfb_info->Outputs[i].NumComponents;
 122       unsigned component_mask = (1 << components) - 1;
 123
 124       /* gl_PointSize is stored in VARYING_SLOT_PSIZ.w. */
 125       if (varying == VARYING_SLOT_PSIZ) {
 126          assert(components == 1);
 127          component_mask <<= 3;
 128       } else {
 129          component_mask <<= linked_xfb_info->Outputs[i].ComponentOffset;
 130       }
 131
 132       buffer_mask |= 1 << buffer;
 133
 134       decl |= buffer << SO_DECL_OUTPUT_BUFFER_SLOT_SHIFT;
 135       decl |= vue_map->varying_to_slot[varying] <<
 136          SO_DECL_REGISTER_INDEX_SHIFT;
 137       decl |= component_mask << SO_DECL_COMPONENT_MASK_SHIFT;
 138
 139       /* Mesa doesn't store entries for gl_SkipComponents in the Outputs[]
 140        * array.  Instead, it simply increments DstOffset for the following
 141        * input by the number of components that should be skipped.
 142        *
 143        * Our hardware is unusual in that it requires us to program SO_DECLs
 144        * for fake "hole" components, rather than simply taking the offset
 145        * for each real varying.  Each hole can have size 1, 2, 3, or 4; we
 146        * program as many size = 4 holes as we can, then a final hole to
 147        * accomodate the final 1, 2, or 3 remaining.
 148        */
 149       int skip_components =
 150          linked_xfb_info->Outputs[i].DstOffset - next_offset[buffer];
 151
 152       next_offset[buffer] += skip_components;
 153
 154       while (skip_components >= 4) {
 155          so_decl[decls++] = SO_DECL_HOLE_FLAG | 0xf;
 156          skip_components -= 4;
 157       }
 158       if (skip_components > 0)
 159          so_decl[decls++] = SO_DECL_HOLE_FLAG | ((1 << skip_components) - 1);
 160
 161       assert(linked_xfb_info->Outputs[i].DstOffset == next_offset[buffer]);
 162
 163       next_offset[buffer] += components;
 164
 165       so_decl[decls++] = decl;
 166    }
 167
 168    BEGIN_BATCH(decls * 2 + 3);
 169    OUT_BATCH(_3DSTATE_SO_DECL_LIST << 16 | (decls * 2 + 1));
 170
 171    OUT_BATCH((buffer_mask << SO_STREAM_TO_BUFFER_SELECTS_0_SHIFT) |
 172              (0 << SO_STREAM_TO_BUFFER_SELECTS_1_SHIFT) |
 173              (0 << SO_STREAM_TO_BUFFER_SELECTS_2_SHIFT) |
 174              (0 << SO_STREAM_TO_BUFFER_SELECTS_3_SHIFT));
 175
 176    OUT_BATCH((decls << SO_NUM_ENTRIES_0_SHIFT) |
 177              (0 << SO_NUM_ENTRIES_1_SHIFT) |
 178              (0 << SO_NUM_ENTRIES_2_SHIFT) |
 179              (0 << SO_NUM_ENTRIES_3_SHIFT));
 180
 181    for (int i = 0; i < decls; i++) {
 182       OUT_BATCH(so_decl[i]);
 183       OUT_BATCH(0);
 184    }
 185
 186    ADVANCE_BATCH();
 187 }
 188
 189 static void
 190 upload_3dstate_streamout(struct brw_context *brw, bool active,
 191                          const struct brw_vue_map *vue_map)
 192 {
 193    struct gl_context *ctx = &brw->ctx;
 194    /* BRW_NEW_TRANSFORM_FEEDBACK */
 195    struct gl_transform_feedback_object *xfb_obj =
 196       ctx->TransformFeedback.CurrentObject;
 197    uint32_t dw1 = 0, dw2 = 0;
 198    int i;
 199
 200    if (active) {
 201       int urb_entry_read_offset = 0;
 202       int urb_entry_read_length = (vue_map->num_slots + 1) / 2 -
 203          urb_entry_read_offset;
 204
 205       dw1 |= SO_FUNCTION_ENABLE;
 206       dw1 |= SO_STATISTICS_ENABLE;
 207
 208       /* _NEW_LIGHT */
 209       if (ctx->Light.ProvokingVertex != GL_FIRST_VERTEX_CONVENTION)
 210          dw1 |= SO_REORDER_TRAILING;
 211
 212       for (i = 0; i < 4; i++) {
 213          if (xfb_obj->Buffers[i]) {
 214             dw1 |= SO_BUFFER_ENABLE(i);
 215          }
 216       }
 217
 218       /* We always read the whole vertex.  This could be reduced at some
 219        * point by reading less and offsetting the register index in the
 220        * SO_DECLs.
 221        */
 222       dw2 |= urb_entry_read_offset << SO_STREAM_0_VERTEX_READ_OFFSET_SHIFT;
 223       dw2 |= (urb_entry_read_length - 1) <<
 224          SO_STREAM_0_VERTEX_READ_LENGTH_SHIFT;
 225    }
 226
 227    BEGIN_BATCH(3);
 228    OUT_BATCH(_3DSTATE_STREAMOUT << 16 | (3 - 2));
 229    OUT_BATCH(dw1);
 230    OUT_BATCH(dw2);
 231    ADVANCE_BATCH();
 232 }
 233
 234 static void
 235 upload_sol_state(struct brw_context *brw)
 236 {
 237    struct gl_context *ctx = &brw->ctx;
 238    /* BRW_NEW_TRANSFORM_FEEDBACK */
 239    bool active = _mesa_is_xfb_active_and_unpaused(ctx);
 240
 241    if (active) {
 242       upload_3dstate_so_buffers(brw);
 243       /* BRW_NEW_VUE_MAP_GEOM_OUT */
 244       gen7_upload_3dstate_so_decl_list(brw, &brw->vue_map_geom_out);
 245    }
 246
 247    /* Finally, set up the SOL stage.  This command must always follow updates to
 248     * the nonpipelined SOL state (3DSTATE_SO_BUFFER, 3DSTATE_SO_DECL_LIST) or
 249     * MMIO register updates (current performed by the kernel at each batch
 250     * emit).
 251     */
 252    upload_3dstate_streamout(brw, active, &brw->vue_map_geom_out);
 253 }
 254
 255 const struct brw_tracked_state gen7_sol_state = {
 256    .dirty = {
 257       .mesa  = (_NEW_LIGHT),
 258       .brw   = (BRW_NEW_BATCH |
 259                 BRW_NEW_VUE_MAP_GEOM_OUT |
 260                 BRW_NEW_TRANSFORM_FEEDBACK)
 261    },
 262    .emit = upload_sol_state,
 263 };
 264
 265 /**
 266  * Tally the number of primitives generated so far.
 267  *
 268  * The buffer contains a series of pairs:
 269  * (<start0, start1, start2, start3>, <end0, end1, end2, end3>) ;
 270  * (<start0, start1, start2, start3>, <end0, end1, end2, end3>) ;
 271  *
 272  * For each stream, we subtract the pair of values (end - start) to get the
 273  * number of primitives generated during one section.  We accumulate these
 274  * values, adding them up to get the total number of primitives generated.
 275  */
 276 static void
 277 gen7_tally_prims_generated(struct brw_context *brw,
 278                            struct brw_transform_feedback_object *obj)
 279 {
 280    /* If the current batch is still contributing to the number of primitives
 281     * generated, flush it now so the results will be present when mapped.
 282     */
 283    if (drm_intel_bo_references(brw->batch.bo, obj->prim_count_bo))
 284       intel_batchbuffer_flush(brw);
 285
 286    if (unlikely(brw->perf_debug && drm_intel_bo_busy(obj->prim_count_bo)))
 287       perf_debug("Stalling for # of transform feedback primitives written.\n");
 288
 289    drm_intel_bo_map(obj->prim_count_bo, false);
 290    uint64_t *prim_counts = obj->prim_count_bo->virtual;
 291
 292    assert(obj->prim_count_buffer_index % (2 * BRW_MAX_XFB_STREAMS) == 0);
 293    int pairs = obj->prim_count_buffer_index / (2 * BRW_MAX_XFB_STREAMS);
 294
 295    for (int i = 0; i < pairs; i++) {
 296       for (int s = 0; s < BRW_MAX_XFB_STREAMS; s++) {
 297          obj->prims_generated[s] +=
 298             prim_counts[BRW_MAX_XFB_STREAMS + s] - prim_counts[s];
 299       }
 300       prim_counts += 2 * BRW_MAX_XFB_STREAMS; /* move to the next pair */
 301    }
 302
 303    drm_intel_bo_unmap(obj->prim_count_bo);
 304
 305    /* We've already gathered up the old data; we can safely overwrite it now. */
 306    obj->prim_count_buffer_index = 0;
 307 }
 308
 309 /**
 310  * Store the SO_NUM_PRIMS_WRITTEN counters for each stream (4 uint64_t values)
 311  * to prim_count_bo.
 312  *
 313  * If prim_count_bo is out of space, gather up the results so far into
 314  * prims_generated[] and allocate a new buffer with enough space.
 315  *
 316  * The number of primitives written is used to compute the number of vertices
 317  * written to a transform feedback stream, which is required to implement
 318  * DrawTransformFeedback().
 319  */
 320 static void
 321 gen7_save_primitives_written_counters(struct brw_context *brw,
 322                                 struct brw_transform_feedback_object *obj)
 323 {
 324    const int streams = BRW_MAX_XFB_STREAMS;
 325
 326    /* Check if there's enough space for a new pair of four values. */
 327    if (obj->prim_count_bo != NULL &&
 328        obj->prim_count_buffer_index + 2 * streams >= 4096 / sizeof(uint64_t)) {
 329       /* Gather up the results so far and release the BO. */
 330       gen7_tally_prims_generated(brw, obj);
 331    }
 332
 333    /* Flush any drawing so that the counters have the right values. */
 334    intel_batchbuffer_emit_mi_flush(brw);
 335
 336    /* Emit MI_STORE_REGISTER_MEM commands to write the values. */
 337    for (int i = 0; i < streams; i++) {
 338       brw_store_register_mem64(brw, obj->prim_count_bo,
 339                                GEN7_SO_NUM_PRIMS_WRITTEN(i),
 340                                obj->prim_count_buffer_index + i);
 341    }
 342
 343    /* Update where to write data to. */
 344    obj->prim_count_buffer_index += streams;
 345 }
 346
 347 /**
 348  * Compute the number of vertices written by this transform feedback operation.
 349  */
 350 static void
 351 brw_compute_xfb_vertices_written(struct brw_context *brw,
 352                                  struct brw_transform_feedback_object *obj)
 353 {
 354    if (obj->vertices_written_valid || !obj->base.EndedAnytime)
 355       return;
 356
 357    unsigned vertices_per_prim = 0;
 358
 359    switch (obj->primitive_mode) {
 360    case GL_POINTS:
 361       vertices_per_prim = 1;
 362       break;
 363    case GL_LINES:
 364       vertices_per_prim = 2;
 365       break;
 366    case GL_TRIANGLES:
 367       vertices_per_prim = 3;
 368       break;
 369    default:
 370       assert(!"Invalid transform feedback primitive mode.");
 371    }
 372
 373    /* Get the number of primitives generated. */
 374    gen7_tally_prims_generated(brw, obj);
 375
 376    for (int i = 0; i < BRW_MAX_XFB_STREAMS; i++) {
 377       obj->vertices_written[i] = vertices_per_prim * obj->prims_generated[i];
 378    }
 379    obj->vertices_written_valid = true;
 380 }
 381
 382 /**
 383  * GetTransformFeedbackVertexCount() driver hook.
 384  *
 385  * Returns the number of vertices written to a particular stream by the last
 386  * Begin/EndTransformFeedback block.  Used to implement DrawTransformFeedback().
 387  */
 388 GLsizei
 389 brw_get_transform_feedback_vertex_count(struct gl_context *ctx,
 390                                         struct gl_transform_feedback_object *obj,
 391                                         GLuint stream)
 392 {
 393    struct brw_context *brw = brw_context(ctx);
 394    struct brw_transform_feedback_object *brw_obj =
 395       (struct brw_transform_feedback_object *) obj;
 396
 397    assert(obj->EndedAnytime);
 398    assert(stream < BRW_MAX_XFB_STREAMS);
 399
 400    brw_compute_xfb_vertices_written(brw, brw_obj);
 401    return brw_obj->vertices_written[stream];
 402 }
 403
 404 void
 405 gen7_begin_transform_feedback(struct gl_context *ctx, GLenum mode,
 406                               struct gl_transform_feedback_object *obj)
 407 {
 408    struct brw_context *brw = brw_context(ctx);
 409    struct brw_transform_feedback_object *brw_obj =
 410       (struct brw_transform_feedback_object *) obj;
 411
 412    intel_batchbuffer_flush(brw);
 413    brw->batch.needs_sol_reset = true;
 414
 415    /* We're about to lose the information needed to compute the number of
 416     * vertices written during the last Begin/EndTransformFeedback section,
 417     * so we can't delay it any further.
 418     */
 419    brw_compute_xfb_vertices_written(brw, brw_obj);
 420
 421    /* No primitives have been generated yet. */
 422    for (int i = 0; i < BRW_MAX_XFB_STREAMS; i++) {
 423       brw_obj->prims_generated[i] = 0;
 424    }
 425
 426    /* Store the starting value of the SO_NUM_PRIMS_WRITTEN counters. */
 427    gen7_save_primitives_written_counters(brw, brw_obj);
 428
 429    brw_obj->primitive_mode = mode;
 430 }
 431
 432 void
 433 gen7_end_transform_feedback(struct gl_context *ctx,
 434                             struct gl_transform_feedback_object *obj)
 435 {
 436    /* After EndTransformFeedback, it's likely that the client program will try
 437     * to draw using the contents of the transform feedback buffer as vertex
 438     * input.  In order for this to work, we need to flush the data through at
 439     * least the GS stage of the pipeline, and flush out the render cache.  For
 440     * simplicity, just do a full flush.
 441     */
 442    struct brw_context *brw = brw_context(ctx);
 443    struct brw_transform_feedback_object *brw_obj =
 444       (struct brw_transform_feedback_object *) obj;
 445
 446    /* Store the ending value of the SO_NUM_PRIMS_WRITTEN counters. */
 447    gen7_save_primitives_written_counters(brw, brw_obj);
 448
 449    /* EndTransformFeedback() means that we need to update the number of
 450     * vertices written.  Since it's only necessary if DrawTransformFeedback()
 451     * is called and it means mapping a buffer object, we delay computing it
 452     * until it's absolutely necessary to try and avoid stalls.
 453     */
 454    brw_obj->vertices_written_valid = false;
 455 }
 456
 457 void
 458 gen7_pause_transform_feedback(struct gl_context *ctx,
 459                               struct gl_transform_feedback_object *obj)
 460 {
 461    struct brw_context *brw = brw_context(ctx);
 462    struct brw_transform_feedback_object *brw_obj =
 463       (struct brw_transform_feedback_object *) obj;
 464
 465    /* Flush any drawing so that the counters have the right values. */
 466    intel_batchbuffer_emit_mi_flush(brw);
 467
 468    /* Save the SOL buffer offset register values. */
 469    for (int i = 0; i < 4; i++) {
 470       BEGIN_BATCH(3);
 471       OUT_BATCH(MI_STORE_REGISTER_MEM | (3 - 2));
 472       OUT_BATCH(GEN7_SO_WRITE_OFFSET(i));
 473       OUT_RELOC(brw_obj->offset_bo,
 474                 I915_GEM_DOMAIN_INSTRUCTION, I915_GEM_DOMAIN_INSTRUCTION,
 475                 i * sizeof(uint32_t));
 476       ADVANCE_BATCH();
 477    }
 478
 479    /* Store the temporary ending value of the SO_NUM_PRIMS_WRITTEN counters.
 480     * While this operation is paused, other transform feedback actions may
 481     * occur, which will contribute to the counters.  We need to exclude that
 482     * from our counts.
 483     */
 484    gen7_save_primitives_written_counters(brw, brw_obj);
 485 }
 486
 487 void
 488 gen7_resume_transform_feedback(struct gl_context *ctx,
 489                                struct gl_transform_feedback_object *obj)
 490 {
 491    struct brw_context *brw = brw_context(ctx);
 492    struct brw_transform_feedback_object *brw_obj =
 493       (struct brw_transform_feedback_object *) obj;
 494
 495    /* Reload the SOL buffer offset registers. */
 496    for (int i = 0; i < 4; i++) {
 497       BEGIN_BATCH(3);
 498       OUT_BATCH(GEN7_MI_LOAD_REGISTER_MEM | (3 - 2));
 499       OUT_BATCH(GEN7_SO_WRITE_OFFSET(i));
 500       OUT_RELOC(brw_obj->offset_bo,
 501                 I915_GEM_DOMAIN_INSTRUCTION, I915_GEM_DOMAIN_INSTRUCTION,
 502                 i * sizeof(uint32_t));
 503       ADVANCE_BATCH();
 504    }
 505
 506    /* Store the new starting value of the SO_NUM_PRIMS_WRITTEN counters. */
 507    gen7_save_primitives_written_counters(brw, brw_obj);
 508 }