src/mesa/drivers/dri/i965/gen7_sol_state.c

   1 /*
   2  * Copyright © 2011 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  */
  23
  24 /**
  25  * @file gen7_sol_state.c
  26  *
  27  * Controls the stream output logic (SOL) stage of the gen7 hardware, which is
  28  * used to implement GL_EXT_transform_feedback.
  29  */
  30
  31 #include "brw_context.h"
  32 #include "brw_state.h"
  33 #include "brw_defines.h"
  34 #include "intel_batchbuffer.h"
  35 #include "intel_buffer_objects.h"
  36 #include "main/transformfeedback.h"
  37
  38 static void
  39 upload_3dstate_so_buffers(struct brw_context *brw)
  40 {
  41    struct gl_context *ctx = &brw->ctx;
  42    /* BRW_NEW_VERTEX_PROGRAM */
  43    const struct gl_shader_program *vs_prog =
  44       ctx->Shader.CurrentProgram[MESA_SHADER_VERTEX];
  45    const struct gl_transform_feedback_info *linked_xfb_info =
  46       &vs_prog->LinkedTransformFeedback;
  47    /* BRW_NEW_TRANSFORM_FEEDBACK */
  48    struct gl_transform_feedback_object *xfb_obj =
  49       ctx->TransformFeedback.CurrentObject;
  50    int i;
  51
  52    /* Set up the up to 4 output buffers.  These are the ranges defined in the
  53     * gl_transform_feedback_object.
  54     */
  55    for (i = 0; i < 4; i++) {
  56       struct intel_buffer_object *bufferobj =
  57          intel_buffer_object(xfb_obj->Buffers[i]);
  58       drm_intel_bo *bo;
  59       uint32_t start, end;
  60       uint32_t stride;
  61
  62       if (!xfb_obj->Buffers[i]) {
  63          /* The pitch of 0 in this command indicates that the buffer is
  64           * unbound and won't be written to.
  65           */
  66          BEGIN_BATCH(4);
  67          OUT_BATCH(_3DSTATE_SO_BUFFER << 16 | (4 - 2));
  68          OUT_BATCH((i << SO_BUFFER_INDEX_SHIFT));
  69          OUT_BATCH(0);
  70          OUT_BATCH(0);
  71          ADVANCE_BATCH();
  72
  73          continue;
  74       }
  75
  76       stride = linked_xfb_info->BufferStride[i] * 4;
  77
  78       start = xfb_obj->Offset[i];
  79       assert(start % 4 == 0);
  80       end = ALIGN(start + xfb_obj->Size[i], 4);
  81       bo = intel_bufferobj_buffer(brw, bufferobj, start, end - start);
  82       assert(end <= bo->size);
  83
  84       BEGIN_BATCH(4);
  85       OUT_BATCH(_3DSTATE_SO_BUFFER << 16 | (4 - 2));
  86       OUT_BATCH((i << SO_BUFFER_INDEX_SHIFT) | stride);
  87       OUT_RELOC(bo, I915_GEM_DOMAIN_RENDER, I915_GEM_DOMAIN_RENDER, start);
  88       OUT_RELOC(bo, I915_GEM_DOMAIN_RENDER, I915_GEM_DOMAIN_RENDER, end);
  89       ADVANCE_BATCH();
  90    }
  91 }
  92
  93 /**
  94  * Outputs the 3DSTATE_SO_DECL_LIST command.
  95  *
  96  * The data output is a series of 64-bit entries containing a SO_DECL per
  97  * stream.  We only have one stream of rendering coming out of the GS unit, so
  98  * we only emit stream 0 (low 16 bits) SO_DECLs.
  99  */
 100 void
 101 gen7_upload_3dstate_so_decl_list(struct brw_context *brw,
 102                                  const struct brw_vue_map *vue_map)
 103 {
 104    struct gl_context *ctx = &brw->ctx;
 105    /* BRW_NEW_VERTEX_PROGRAM */
 106    const struct gl_shader_program *vs_prog =
 107       ctx->Shader.CurrentProgram[MESA_SHADER_VERTEX];
 108    /* BRW_NEW_TRANSFORM_FEEDBACK */
 109    const struct gl_transform_feedback_info *linked_xfb_info =
 110       &vs_prog->LinkedTransformFeedback;
 111    uint16_t so_decl[128];
 112    int buffer_mask = 0;
 113    int next_offset[4] = {0, 0, 0, 0};
 114    int decls = 0;
 115
 116    STATIC_ASSERT(ARRAY_SIZE(so_decl) >= MAX_PROGRAM_OUTPUTS);
 117
 118    /* Construct the list of SO_DECLs to be emitted.  The formatting of the
 119     * command is feels strange -- each dword pair contains a SO_DECL per stream.
 120     */
 121    for (int i = 0; i < linked_xfb_info->NumOutputs; i++) {
 122       int buffer = linked_xfb_info->Outputs[i].OutputBuffer;
 123       uint16_t decl = 0;
 124       int varying = linked_xfb_info->Outputs[i].OutputRegister;
 125       const unsigned components = linked_xfb_info->Outputs[i].NumComponents;
 126       unsigned component_mask = (1 << components) - 1;
 127
 128       /* gl_PointSize is stored in VARYING_SLOT_PSIZ.w. */
 129       if (varying == VARYING_SLOT_PSIZ) {
 130          assert(components == 1);
 131          component_mask <<= 3;
 132       } else {
 133          component_mask <<= linked_xfb_info->Outputs[i].ComponentOffset;
 134       }
 135
 136       buffer_mask |= 1 << buffer;
 137
 138       decl |= buffer << SO_DECL_OUTPUT_BUFFER_SLOT_SHIFT;
 139       decl |= vue_map->varying_to_slot[varying] <<
 140          SO_DECL_REGISTER_INDEX_SHIFT;
 141       decl |= component_mask << SO_DECL_COMPONENT_MASK_SHIFT;
 142
 143       /* Mesa doesn't store entries for gl_SkipComponents in the Outputs[]
 144        * array.  Instead, it simply increments DstOffset for the following
 145        * input by the number of components that should be skipped.
 146        *
 147        * Our hardware is unusual in that it requires us to program SO_DECLs
 148        * for fake "hole" components, rather than simply taking the offset
 149        * for each real varying.  Each hole can have size 1, 2, 3, or 4; we
 150        * program as many size = 4 holes as we can, then a final hole to
 151        * accomodate the final 1, 2, or 3 remaining.
 152        */
 153       int skip_components =
 154          linked_xfb_info->Outputs[i].DstOffset - next_offset[buffer];
 155
 156       next_offset[buffer] += skip_components;
 157
 158       while (skip_components >= 4) {
 159          so_decl[decls++] = SO_DECL_HOLE_FLAG | 0xf;
 160          skip_components -= 4;
 161       }
 162       if (skip_components > 0)
 163          so_decl[decls++] = SO_DECL_HOLE_FLAG | ((1 << skip_components) - 1);
 164
 165       assert(linked_xfb_info->Outputs[i].DstOffset == next_offset[buffer]);
 166
 167       next_offset[buffer] += components;
 168
 169       so_decl[decls++] = decl;
 170    }
 171
 172    BEGIN_BATCH(decls * 2 + 3);
 173    OUT_BATCH(_3DSTATE_SO_DECL_LIST << 16 | (decls * 2 + 1));
 174
 175    OUT_BATCH((buffer_mask << SO_STREAM_TO_BUFFER_SELECTS_0_SHIFT) |
 176              (0 << SO_STREAM_TO_BUFFER_SELECTS_1_SHIFT) |
 177              (0 << SO_STREAM_TO_BUFFER_SELECTS_2_SHIFT) |
 178              (0 << SO_STREAM_TO_BUFFER_SELECTS_3_SHIFT));
 179
 180    OUT_BATCH((decls << SO_NUM_ENTRIES_0_SHIFT) |
 181              (0 << SO_NUM_ENTRIES_1_SHIFT) |
 182              (0 << SO_NUM_ENTRIES_2_SHIFT) |
 183              (0 << SO_NUM_ENTRIES_3_SHIFT));
 184
 185    for (int i = 0; i < decls; i++) {
 186       OUT_BATCH(so_decl[i]);
 187       OUT_BATCH(0);
 188    }
 189
 190    ADVANCE_BATCH();
 191 }
 192
 193 static void
 194 upload_3dstate_streamout(struct brw_context *brw, bool active,
 195                          const struct brw_vue_map *vue_map)
 196 {
 197    struct gl_context *ctx = &brw->ctx;
 198    /* BRW_NEW_TRANSFORM_FEEDBACK */
 199    struct gl_transform_feedback_object *xfb_obj =
 200       ctx->TransformFeedback.CurrentObject;
 201    uint32_t dw1 = 0, dw2 = 0;
 202    int i;
 203
 204    if (active) {
 205       int urb_entry_read_offset = 0;
 206       int urb_entry_read_length = (vue_map->num_slots + 1) / 2 -
 207          urb_entry_read_offset;
 208
 209       dw1 |= SO_FUNCTION_ENABLE;
 210       dw1 |= SO_STATISTICS_ENABLE;
 211
 212       /* _NEW_LIGHT */
 213       if (ctx->Light.ProvokingVertex != GL_FIRST_VERTEX_CONVENTION)
 214          dw1 |= SO_REORDER_TRAILING;
 215
 216       for (i = 0; i < 4; i++) {
 217          if (xfb_obj->Buffers[i]) {
 218             dw1 |= SO_BUFFER_ENABLE(i);
 219          }
 220       }
 221
 222       /* We always read the whole vertex.  This could be reduced at some
 223        * point by reading less and offsetting the register index in the
 224        * SO_DECLs.
 225        */
 226       dw2 |= urb_entry_read_offset << SO_STREAM_0_VERTEX_READ_OFFSET_SHIFT;
 227       dw2 |= (urb_entry_read_length - 1) <<
 228          SO_STREAM_0_VERTEX_READ_LENGTH_SHIFT;
 229    }
 230
 231    BEGIN_BATCH(3);
 232    OUT_BATCH(_3DSTATE_STREAMOUT << 16 | (3 - 2));
 233    OUT_BATCH(dw1);
 234    OUT_BATCH(dw2);
 235    ADVANCE_BATCH();
 236 }
 237
 238 static void
 239 upload_sol_state(struct brw_context *brw)
 240 {
 241    struct gl_context *ctx = &brw->ctx;
 242    /* BRW_NEW_TRANSFORM_FEEDBACK */
 243    bool active = _mesa_is_xfb_active_and_unpaused(ctx);
 244
 245    if (active) {
 246       upload_3dstate_so_buffers(brw);
 247       /* BRW_NEW_VUE_MAP_GEOM_OUT */
 248       gen7_upload_3dstate_so_decl_list(brw, &brw->vue_map_geom_out);
 249    }
 250
 251    /* Finally, set up the SOL stage.  This command must always follow updates to
 252     * the nonpipelined SOL state (3DSTATE_SO_BUFFER, 3DSTATE_SO_DECL_LIST) or
 253     * MMIO register updates (current performed by the kernel at each batch
 254     * emit).
 255     */
 256    upload_3dstate_streamout(brw, active, &brw->vue_map_geom_out);
 257 }
 258
 259 const struct brw_tracked_state gen7_sol_state = {
 260    .dirty = {
 261       .mesa  = (_NEW_LIGHT),
 262       .brw   = (BRW_NEW_BATCH |
 263                 BRW_NEW_VERTEX_PROGRAM |
 264                 BRW_NEW_VUE_MAP_GEOM_OUT |
 265                 BRW_NEW_TRANSFORM_FEEDBACK)
 266    },
 267    .emit = upload_sol_state,
 268 };
 269
 270 /**
 271  * Tally the number of primitives generated so far.
 272  *
 273  * The buffer contains a series of pairs:
 274  * (<start0, start1, start2, start3>, <end0, end1, end2, end3>) ;
 275  * (<start0, start1, start2, start3>, <end0, end1, end2, end3>) ;
 276  *
 277  * For each stream, we subtract the pair of values (end - start) to get the
 278  * number of primitives generated during one section.  We accumulate these
 279  * values, adding them up to get the total number of primitives generated.
 280  */
 281 static void
 282 gen7_tally_prims_generated(struct brw_context *brw,
 283                            struct brw_transform_feedback_object *obj)
 284 {
 285    /* If the current batch is still contributing to the number of primitives
 286     * generated, flush it now so the results will be present when mapped.
 287     */
 288    if (drm_intel_bo_references(brw->batch.bo, obj->prim_count_bo))
 289       intel_batchbuffer_flush(brw);
 290
 291    if (unlikely(brw->perf_debug && drm_intel_bo_busy(obj->prim_count_bo)))
 292       perf_debug("Stalling for # of transform feedback primitives written.\n");
 293
 294    drm_intel_bo_map(obj->prim_count_bo, false);
 295    uint64_t *prim_counts = obj->prim_count_bo->virtual;
 296
 297    assert(obj->prim_count_buffer_index % (2 * BRW_MAX_XFB_STREAMS) == 0);
 298    int pairs = obj->prim_count_buffer_index / (2 * BRW_MAX_XFB_STREAMS);
 299
 300    for (int i = 0; i < pairs; i++) {
 301       for (int s = 0; s < BRW_MAX_XFB_STREAMS; s++) {
 302          obj->prims_generated[s] +=
 303             prim_counts[BRW_MAX_XFB_STREAMS + s] - prim_counts[s];
 304       }
 305       prim_counts += 2 * BRW_MAX_XFB_STREAMS; /* move to the next pair */
 306    }
 307
 308    drm_intel_bo_unmap(obj->prim_count_bo);
 309
 310    /* We've already gathered up the old data; we can safely overwrite it now. */
 311    obj->prim_count_buffer_index = 0;
 312 }
 313
 314 /**
 315  * Store the SO_NUM_PRIMS_WRITTEN counters for each stream (4 uint64_t values)
 316  * to prim_count_bo.
 317  *
 318  * If prim_count_bo is out of space, gather up the results so far into
 319  * prims_generated[] and allocate a new buffer with enough space.
 320  *
 321  * The number of primitives written is used to compute the number of vertices
 322  * written to a transform feedback stream, which is required to implement
 323  * DrawTransformFeedback().
 324  */
 325 static void
 326 gen7_save_primitives_written_counters(struct brw_context *brw,
 327                                 struct brw_transform_feedback_object *obj)
 328 {
 329    const int streams = BRW_MAX_XFB_STREAMS;
 330
 331    /* Check if there's enough space for a new pair of four values. */
 332    if (obj->prim_count_bo != NULL &&
 333        obj->prim_count_buffer_index + 2 * streams >= 4096 / sizeof(uint64_t)) {
 334       /* Gather up the results so far and release the BO. */
 335       gen7_tally_prims_generated(brw, obj);
 336    }
 337
 338    /* Flush any drawing so that the counters have the right values. */
 339    intel_batchbuffer_emit_mi_flush(brw);
 340
 341    /* Emit MI_STORE_REGISTER_MEM commands to write the values. */
 342    for (int i = 0; i < streams; i++) {
 343       brw_store_register_mem64(brw, obj->prim_count_bo,
 344                                GEN7_SO_NUM_PRIMS_WRITTEN(i),
 345                                obj->prim_count_buffer_index + i);
 346    }
 347
 348    /* Update where to write data to. */
 349    obj->prim_count_buffer_index += streams;
 350 }
 351
 352 /**
 353  * Compute the number of vertices written by this transform feedback operation.
 354  */
 355 static void
 356 brw_compute_xfb_vertices_written(struct brw_context *brw,
 357                                  struct brw_transform_feedback_object *obj)
 358 {
 359    if (obj->vertices_written_valid || !obj->base.EndedAnytime)
 360       return;
 361
 362    unsigned vertices_per_prim = 0;
 363
 364    switch (obj->primitive_mode) {
 365    case GL_POINTS:
 366       vertices_per_prim = 1;
 367       break;
 368    case GL_LINES:
 369       vertices_per_prim = 2;
 370       break;
 371    case GL_TRIANGLES:
 372       vertices_per_prim = 3;
 373       break;
 374    default:
 375       assert(!"Invalid transform feedback primitive mode.");
 376    }
 377
 378    /* Get the number of primitives generated. */
 379    gen7_tally_prims_generated(brw, obj);
 380
 381    for (int i = 0; i < BRW_MAX_XFB_STREAMS; i++) {
 382       obj->vertices_written[i] = vertices_per_prim * obj->prims_generated[i];
 383    }
 384    obj->vertices_written_valid = true;
 385 }
 386
 387 /**
 388  * GetTransformFeedbackVertexCount() driver hook.
 389  *
 390  * Returns the number of vertices written to a particular stream by the last
 391  * Begin/EndTransformFeedback block.  Used to implement DrawTransformFeedback().
 392  */
 393 GLsizei
 394 brw_get_transform_feedback_vertex_count(struct gl_context *ctx,
 395                                         struct gl_transform_feedback_object *obj,
 396                                         GLuint stream)
 397 {
 398    struct brw_context *brw = brw_context(ctx);
 399    struct brw_transform_feedback_object *brw_obj =
 400       (struct brw_transform_feedback_object *) obj;
 401
 402    assert(obj->EndedAnytime);
 403    assert(stream < BRW_MAX_XFB_STREAMS);
 404
 405    brw_compute_xfb_vertices_written(brw, brw_obj);
 406    return brw_obj->vertices_written[stream];
 407 }
 408
 409 void
 410 gen7_begin_transform_feedback(struct gl_context *ctx, GLenum mode,
 411                               struct gl_transform_feedback_object *obj)
 412 {
 413    struct brw_context *brw = brw_context(ctx);
 414    struct brw_transform_feedback_object *brw_obj =
 415       (struct brw_transform_feedback_object *) obj;
 416
 417    intel_batchbuffer_flush(brw);
 418    brw->batch.needs_sol_reset = true;
 419
 420    /* We're about to lose the information needed to compute the number of
 421     * vertices written during the last Begin/EndTransformFeedback section,
 422     * so we can't delay it any further.
 423     */
 424    brw_compute_xfb_vertices_written(brw, brw_obj);
 425
 426    /* No primitives have been generated yet. */
 427    for (int i = 0; i < BRW_MAX_XFB_STREAMS; i++) {
 428       brw_obj->prims_generated[i] = 0;
 429    }
 430
 431    /* Store the starting value of the SO_NUM_PRIMS_WRITTEN counters. */
 432    gen7_save_primitives_written_counters(brw, brw_obj);
 433
 434    brw_obj->primitive_mode = mode;
 435 }
 436
 437 void
 438 gen7_end_transform_feedback(struct gl_context *ctx,
 439                             struct gl_transform_feedback_object *obj)
 440 {
 441    /* After EndTransformFeedback, it's likely that the client program will try
 442     * to draw using the contents of the transform feedback buffer as vertex
 443     * input.  In order for this to work, we need to flush the data through at
 444     * least the GS stage of the pipeline, and flush out the render cache.  For
 445     * simplicity, just do a full flush.
 446     */
 447    struct brw_context *brw = brw_context(ctx);
 448    struct brw_transform_feedback_object *brw_obj =
 449       (struct brw_transform_feedback_object *) obj;
 450
 451    /* Store the ending value of the SO_NUM_PRIMS_WRITTEN counters. */
 452    gen7_save_primitives_written_counters(brw, brw_obj);
 453
 454    /* EndTransformFeedback() means that we need to update the number of
 455     * vertices written.  Since it's only necessary if DrawTransformFeedback()
 456     * is called and it means mapping a buffer object, we delay computing it
 457     * until it's absolutely necessary to try and avoid stalls.
 458     */
 459    brw_obj->vertices_written_valid = false;
 460 }
 461
 462 void
 463 gen7_pause_transform_feedback(struct gl_context *ctx,
 464                               struct gl_transform_feedback_object *obj)
 465 {
 466    struct brw_context *brw = brw_context(ctx);
 467    struct brw_transform_feedback_object *brw_obj =
 468       (struct brw_transform_feedback_object *) obj;
 469
 470    /* Flush any drawing so that the counters have the right values. */
 471    intel_batchbuffer_emit_mi_flush(brw);
 472
 473    /* Save the SOL buffer offset register values. */
 474    for (int i = 0; i < 4; i++) {
 475       BEGIN_BATCH(3);
 476       OUT_BATCH(MI_STORE_REGISTER_MEM | (3 - 2));
 477       OUT_BATCH(GEN7_SO_WRITE_OFFSET(i));
 478       OUT_RELOC(brw_obj->offset_bo,
 479                 I915_GEM_DOMAIN_INSTRUCTION, I915_GEM_DOMAIN_INSTRUCTION,
 480                 i * sizeof(uint32_t));
 481       ADVANCE_BATCH();
 482    }
 483
 484    /* Store the temporary ending value of the SO_NUM_PRIMS_WRITTEN counters.
 485     * While this operation is paused, other transform feedback actions may
 486     * occur, which will contribute to the counters.  We need to exclude that
 487     * from our counts.
 488     */
 489    gen7_save_primitives_written_counters(brw, brw_obj);
 490 }
 491
 492 void
 493 gen7_resume_transform_feedback(struct gl_context *ctx,
 494                                struct gl_transform_feedback_object *obj)
 495 {
 496    struct brw_context *brw = brw_context(ctx);
 497    struct brw_transform_feedback_object *brw_obj =
 498       (struct brw_transform_feedback_object *) obj;
 499
 500    /* Reload the SOL buffer offset registers. */
 501    for (int i = 0; i < 4; i++) {
 502       BEGIN_BATCH(3);
 503       OUT_BATCH(GEN7_MI_LOAD_REGISTER_MEM | (3 - 2));
 504       OUT_BATCH(GEN7_SO_WRITE_OFFSET(i));
 505       OUT_RELOC(brw_obj->offset_bo,
 506                 I915_GEM_DOMAIN_INSTRUCTION, I915_GEM_DOMAIN_INSTRUCTION,
 507                 i * sizeof(uint32_t));
 508       ADVANCE_BATCH();
 509    }
 510
 511    /* Store the new starting value of the SO_NUM_PRIMS_WRITTEN counters. */
 512    gen7_save_primitives_written_counters(brw, brw_obj);
 513 }