2 * Copyright © 2011 Intel Corporation
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
25 * @file gen7_sol_state.c
27 * Controls the stream output logic (SOL) stage of the gen7 hardware, which is
28 * used to implement GL_EXT_transform_feedback.
31 #include "brw_context.h"
32 #include "brw_state.h"
33 #include "brw_defines.h"
34 #include "intel_batchbuffer.h"
35 #include "intel_buffer_objects.h"
36 #include "main/transformfeedback.h"
39 upload_3dstate_so_buffers(struct brw_context
*brw
)
41 struct gl_context
*ctx
= &brw
->ctx
;
42 /* BRW_NEW_TRANSFORM_FEEDBACK */
43 struct gl_transform_feedback_object
*xfb_obj
=
44 ctx
->TransformFeedback
.CurrentObject
;
45 const struct gl_transform_feedback_info
*linked_xfb_info
=
46 &xfb_obj
->shader_program
->LinkedTransformFeedback
;
49 /* Set up the up to 4 output buffers. These are the ranges defined in the
50 * gl_transform_feedback_object.
52 for (i
= 0; i
< 4; i
++) {
53 struct intel_buffer_object
*bufferobj
=
54 intel_buffer_object(xfb_obj
->Buffers
[i
]);
59 if (!xfb_obj
->Buffers
[i
]) {
60 /* The pitch of 0 in this command indicates that the buffer is
61 * unbound and won't be written to.
64 OUT_BATCH(_3DSTATE_SO_BUFFER
<< 16 | (4 - 2));
65 OUT_BATCH((i
<< SO_BUFFER_INDEX_SHIFT
));
73 stride
= linked_xfb_info
->BufferStride
[i
] * 4;
75 start
= xfb_obj
->Offset
[i
];
76 assert(start
% 4 == 0);
77 end
= ALIGN(start
+ xfb_obj
->Size
[i
], 4);
78 bo
= intel_bufferobj_buffer(brw
, bufferobj
, start
, end
- start
);
79 assert(end
<= bo
->size
);
82 OUT_BATCH(_3DSTATE_SO_BUFFER
<< 16 | (4 - 2));
83 OUT_BATCH((i
<< SO_BUFFER_INDEX_SHIFT
) | stride
);
84 OUT_RELOC(bo
, I915_GEM_DOMAIN_RENDER
, I915_GEM_DOMAIN_RENDER
, start
);
85 OUT_RELOC(bo
, I915_GEM_DOMAIN_RENDER
, I915_GEM_DOMAIN_RENDER
, end
);
91 * Outputs the 3DSTATE_SO_DECL_LIST command.
93 * The data output is a series of 64-bit entries containing a SO_DECL per
94 * stream. We only have one stream of rendering coming out of the GS unit, so
95 * we only emit stream 0 (low 16 bits) SO_DECLs.
98 gen7_upload_3dstate_so_decl_list(struct brw_context
*brw
,
99 const struct brw_vue_map
*vue_map
)
101 struct gl_context
*ctx
= &brw
->ctx
;
102 /* BRW_NEW_TRANSFORM_FEEDBACK */
103 struct gl_transform_feedback_object
*xfb_obj
=
104 ctx
->TransformFeedback
.CurrentObject
;
105 const struct gl_transform_feedback_info
*linked_xfb_info
=
106 &xfb_obj
->shader_program
->LinkedTransformFeedback
;
107 uint16_t so_decl
[128];
109 int next_offset
[4] = {0, 0, 0, 0};
112 STATIC_ASSERT(ARRAY_SIZE(so_decl
) >= MAX_PROGRAM_OUTPUTS
);
114 /* Construct the list of SO_DECLs to be emitted. The formatting of the
115 * command is feels strange -- each dword pair contains a SO_DECL per stream.
117 for (int i
= 0; i
< linked_xfb_info
->NumOutputs
; i
++) {
118 int buffer
= linked_xfb_info
->Outputs
[i
].OutputBuffer
;
120 int varying
= linked_xfb_info
->Outputs
[i
].OutputRegister
;
121 const unsigned components
= linked_xfb_info
->Outputs
[i
].NumComponents
;
122 unsigned component_mask
= (1 << components
) - 1;
124 /* gl_PointSize is stored in VARYING_SLOT_PSIZ.w
125 * gl_Layer is stored in VARYING_SLOT_PSIZ.y
126 * gl_ViewportIndex is stored in VARYING_SLOT_PSIZ.z
128 if (varying
== VARYING_SLOT_PSIZ
) {
129 assert(components
== 1);
130 component_mask
<<= 3;
131 } else if (varying
== VARYING_SLOT_LAYER
) {
132 assert(components
== 1);
133 component_mask
<<= 1;
134 } else if (varying
== VARYING_SLOT_VIEWPORT
) {
135 assert(components
== 1);
136 component_mask
<<= 2;
138 component_mask
<<= linked_xfb_info
->Outputs
[i
].ComponentOffset
;
141 buffer_mask
|= 1 << buffer
;
143 decl
|= buffer
<< SO_DECL_OUTPUT_BUFFER_SLOT_SHIFT
;
144 if (varying
== VARYING_SLOT_LAYER
|| varying
== VARYING_SLOT_VIEWPORT
) {
145 decl
|= vue_map
->varying_to_slot
[VARYING_SLOT_PSIZ
] <<
146 SO_DECL_REGISTER_INDEX_SHIFT
;
148 assert(vue_map
->varying_to_slot
[varying
] >= 0);
149 decl
|= vue_map
->varying_to_slot
[varying
] <<
150 SO_DECL_REGISTER_INDEX_SHIFT
;
152 decl
|= component_mask
<< SO_DECL_COMPONENT_MASK_SHIFT
;
154 /* Mesa doesn't store entries for gl_SkipComponents in the Outputs[]
155 * array. Instead, it simply increments DstOffset for the following
156 * input by the number of components that should be skipped.
158 * Our hardware is unusual in that it requires us to program SO_DECLs
159 * for fake "hole" components, rather than simply taking the offset
160 * for each real varying. Each hole can have size 1, 2, 3, or 4; we
161 * program as many size = 4 holes as we can, then a final hole to
162 * accomodate the final 1, 2, or 3 remaining.
164 int skip_components
=
165 linked_xfb_info
->Outputs
[i
].DstOffset
- next_offset
[buffer
];
167 next_offset
[buffer
] += skip_components
;
169 while (skip_components
>= 4) {
170 so_decl
[decls
++] = SO_DECL_HOLE_FLAG
| 0xf;
171 skip_components
-= 4;
173 if (skip_components
> 0)
174 so_decl
[decls
++] = SO_DECL_HOLE_FLAG
| ((1 << skip_components
) - 1);
176 assert(linked_xfb_info
->Outputs
[i
].DstOffset
== next_offset
[buffer
]);
178 next_offset
[buffer
] += components
;
180 so_decl
[decls
++] = decl
;
183 BEGIN_BATCH(decls
* 2 + 3);
184 OUT_BATCH(_3DSTATE_SO_DECL_LIST
<< 16 | (decls
* 2 + 1));
186 OUT_BATCH((buffer_mask
<< SO_STREAM_TO_BUFFER_SELECTS_0_SHIFT
) |
187 (0 << SO_STREAM_TO_BUFFER_SELECTS_1_SHIFT
) |
188 (0 << SO_STREAM_TO_BUFFER_SELECTS_2_SHIFT
) |
189 (0 << SO_STREAM_TO_BUFFER_SELECTS_3_SHIFT
));
191 OUT_BATCH((decls
<< SO_NUM_ENTRIES_0_SHIFT
) |
192 (0 << SO_NUM_ENTRIES_1_SHIFT
) |
193 (0 << SO_NUM_ENTRIES_2_SHIFT
) |
194 (0 << SO_NUM_ENTRIES_3_SHIFT
));
196 for (int i
= 0; i
< decls
; i
++) {
197 OUT_BATCH(so_decl
[i
]);
205 upload_3dstate_streamout(struct brw_context
*brw
, bool active
,
206 const struct brw_vue_map
*vue_map
)
208 struct gl_context
*ctx
= &brw
->ctx
;
209 /* BRW_NEW_TRANSFORM_FEEDBACK */
210 struct gl_transform_feedback_object
*xfb_obj
=
211 ctx
->TransformFeedback
.CurrentObject
;
212 uint32_t dw1
= 0, dw2
= 0;
216 int urb_entry_read_offset
= 0;
217 int urb_entry_read_length
= (vue_map
->num_slots
+ 1) / 2 -
218 urb_entry_read_offset
;
220 dw1
|= SO_FUNCTION_ENABLE
;
221 dw1
|= SO_STATISTICS_ENABLE
;
224 if (ctx
->Light
.ProvokingVertex
!= GL_FIRST_VERTEX_CONVENTION
)
225 dw1
|= SO_REORDER_TRAILING
;
227 for (i
= 0; i
< 4; i
++) {
228 if (xfb_obj
->Buffers
[i
]) {
229 dw1
|= SO_BUFFER_ENABLE(i
);
233 /* We always read the whole vertex. This could be reduced at some
234 * point by reading less and offsetting the register index in the
237 dw2
|= urb_entry_read_offset
<< SO_STREAM_0_VERTEX_READ_OFFSET_SHIFT
;
238 dw2
|= (urb_entry_read_length
- 1) <<
239 SO_STREAM_0_VERTEX_READ_LENGTH_SHIFT
;
243 OUT_BATCH(_3DSTATE_STREAMOUT
<< 16 | (3 - 2));
250 upload_sol_state(struct brw_context
*brw
)
252 struct gl_context
*ctx
= &brw
->ctx
;
253 /* BRW_NEW_TRANSFORM_FEEDBACK */
254 bool active
= _mesa_is_xfb_active_and_unpaused(ctx
);
257 upload_3dstate_so_buffers(brw
);
258 /* BRW_NEW_VUE_MAP_GEOM_OUT */
259 gen7_upload_3dstate_so_decl_list(brw
, &brw
->vue_map_geom_out
);
262 /* Finally, set up the SOL stage. This command must always follow updates to
263 * the nonpipelined SOL state (3DSTATE_SO_BUFFER, 3DSTATE_SO_DECL_LIST) or
264 * MMIO register updates (current performed by the kernel at each batch
267 upload_3dstate_streamout(brw
, active
, &brw
->vue_map_geom_out
);
270 const struct brw_tracked_state gen7_sol_state
= {
272 .mesa
= (_NEW_LIGHT
),
273 .brw
= (BRW_NEW_BATCH
|
274 BRW_NEW_VUE_MAP_GEOM_OUT
|
275 BRW_NEW_TRANSFORM_FEEDBACK
)
277 .emit
= upload_sol_state
,
281 * Tally the number of primitives generated so far.
283 * The buffer contains a series of pairs:
284 * (<start0, start1, start2, start3>, <end0, end1, end2, end3>) ;
285 * (<start0, start1, start2, start3>, <end0, end1, end2, end3>) ;
287 * For each stream, we subtract the pair of values (end - start) to get the
288 * number of primitives generated during one section. We accumulate these
289 * values, adding them up to get the total number of primitives generated.
292 gen7_tally_prims_generated(struct brw_context
*brw
,
293 struct brw_transform_feedback_object
*obj
)
295 /* If the current batch is still contributing to the number of primitives
296 * generated, flush it now so the results will be present when mapped.
298 if (drm_intel_bo_references(brw
->batch
.bo
, obj
->prim_count_bo
))
299 intel_batchbuffer_flush(brw
);
301 if (unlikely(brw
->perf_debug
&& drm_intel_bo_busy(obj
->prim_count_bo
)))
302 perf_debug("Stalling for # of transform feedback primitives written.\n");
304 drm_intel_bo_map(obj
->prim_count_bo
, false);
305 uint64_t *prim_counts
= obj
->prim_count_bo
->virtual;
307 assert(obj
->prim_count_buffer_index
% (2 * BRW_MAX_XFB_STREAMS
) == 0);
308 int pairs
= obj
->prim_count_buffer_index
/ (2 * BRW_MAX_XFB_STREAMS
);
310 for (int i
= 0; i
< pairs
; i
++) {
311 for (int s
= 0; s
< BRW_MAX_XFB_STREAMS
; s
++) {
312 obj
->prims_generated
[s
] +=
313 prim_counts
[BRW_MAX_XFB_STREAMS
+ s
] - prim_counts
[s
];
315 prim_counts
+= 2 * BRW_MAX_XFB_STREAMS
; /* move to the next pair */
318 drm_intel_bo_unmap(obj
->prim_count_bo
);
320 /* We've already gathered up the old data; we can safely overwrite it now. */
321 obj
->prim_count_buffer_index
= 0;
325 * Store the SO_NUM_PRIMS_WRITTEN counters for each stream (4 uint64_t values)
328 * If prim_count_bo is out of space, gather up the results so far into
329 * prims_generated[] and allocate a new buffer with enough space.
331 * The number of primitives written is used to compute the number of vertices
332 * written to a transform feedback stream, which is required to implement
333 * DrawTransformFeedback().
336 gen7_save_primitives_written_counters(struct brw_context
*brw
,
337 struct brw_transform_feedback_object
*obj
)
339 const int streams
= BRW_MAX_XFB_STREAMS
;
341 /* Check if there's enough space for a new pair of four values. */
342 if (obj
->prim_count_bo
!= NULL
&&
343 obj
->prim_count_buffer_index
+ 2 * streams
>= 4096 / sizeof(uint64_t)) {
344 /* Gather up the results so far and release the BO. */
345 gen7_tally_prims_generated(brw
, obj
);
348 /* Flush any drawing so that the counters have the right values. */
349 intel_batchbuffer_emit_mi_flush(brw
);
351 /* Emit MI_STORE_REGISTER_MEM commands to write the values. */
352 for (int i
= 0; i
< streams
; i
++) {
353 brw_store_register_mem64(brw
, obj
->prim_count_bo
,
354 GEN7_SO_NUM_PRIMS_WRITTEN(i
),
355 obj
->prim_count_buffer_index
+ i
);
358 /* Update where to write data to. */
359 obj
->prim_count_buffer_index
+= streams
;
363 * Compute the number of vertices written by this transform feedback operation.
366 brw_compute_xfb_vertices_written(struct brw_context
*brw
,
367 struct brw_transform_feedback_object
*obj
)
369 if (obj
->vertices_written_valid
|| !obj
->base
.EndedAnytime
)
372 unsigned vertices_per_prim
= 0;
374 switch (obj
->primitive_mode
) {
376 vertices_per_prim
= 1;
379 vertices_per_prim
= 2;
382 vertices_per_prim
= 3;
385 assert(!"Invalid transform feedback primitive mode.");
388 /* Get the number of primitives generated. */
389 gen7_tally_prims_generated(brw
, obj
);
391 for (int i
= 0; i
< BRW_MAX_XFB_STREAMS
; i
++) {
392 obj
->vertices_written
[i
] = vertices_per_prim
* obj
->prims_generated
[i
];
394 obj
->vertices_written_valid
= true;
398 * GetTransformFeedbackVertexCount() driver hook.
400 * Returns the number of vertices written to a particular stream by the last
401 * Begin/EndTransformFeedback block. Used to implement DrawTransformFeedback().
404 brw_get_transform_feedback_vertex_count(struct gl_context
*ctx
,
405 struct gl_transform_feedback_object
*obj
,
408 struct brw_context
*brw
= brw_context(ctx
);
409 struct brw_transform_feedback_object
*brw_obj
=
410 (struct brw_transform_feedback_object
*) obj
;
412 assert(obj
->EndedAnytime
);
413 assert(stream
< BRW_MAX_XFB_STREAMS
);
415 brw_compute_xfb_vertices_written(brw
, brw_obj
);
416 return brw_obj
->vertices_written
[stream
];
420 gen7_begin_transform_feedback(struct gl_context
*ctx
, GLenum mode
,
421 struct gl_transform_feedback_object
*obj
)
423 struct brw_context
*brw
= brw_context(ctx
);
424 struct brw_transform_feedback_object
*brw_obj
=
425 (struct brw_transform_feedback_object
*) obj
;
427 /* Reset the SO buffer offsets to 0. */
429 brw_obj
->zero_offsets
= true;
431 intel_batchbuffer_flush(brw
);
432 brw
->batch
.needs_sol_reset
= true;
435 /* We're about to lose the information needed to compute the number of
436 * vertices written during the last Begin/EndTransformFeedback section,
437 * so we can't delay it any further.
439 brw_compute_xfb_vertices_written(brw
, brw_obj
);
441 /* No primitives have been generated yet. */
442 for (int i
= 0; i
< BRW_MAX_XFB_STREAMS
; i
++) {
443 brw_obj
->prims_generated
[i
] = 0;
446 /* Store the starting value of the SO_NUM_PRIMS_WRITTEN counters. */
447 gen7_save_primitives_written_counters(brw
, brw_obj
);
449 brw_obj
->primitive_mode
= mode
;
453 gen7_end_transform_feedback(struct gl_context
*ctx
,
454 struct gl_transform_feedback_object
*obj
)
456 /* After EndTransformFeedback, it's likely that the client program will try
457 * to draw using the contents of the transform feedback buffer as vertex
458 * input. In order for this to work, we need to flush the data through at
459 * least the GS stage of the pipeline, and flush out the render cache. For
460 * simplicity, just do a full flush.
462 struct brw_context
*brw
= brw_context(ctx
);
463 struct brw_transform_feedback_object
*brw_obj
=
464 (struct brw_transform_feedback_object
*) obj
;
466 /* Store the ending value of the SO_NUM_PRIMS_WRITTEN counters. */
467 gen7_save_primitives_written_counters(brw
, brw_obj
);
469 /* EndTransformFeedback() means that we need to update the number of
470 * vertices written. Since it's only necessary if DrawTransformFeedback()
471 * is called and it means mapping a buffer object, we delay computing it
472 * until it's absolutely necessary to try and avoid stalls.
474 brw_obj
->vertices_written_valid
= false;
478 gen7_pause_transform_feedback(struct gl_context
*ctx
,
479 struct gl_transform_feedback_object
*obj
)
481 struct brw_context
*brw
= brw_context(ctx
);
482 struct brw_transform_feedback_object
*brw_obj
=
483 (struct brw_transform_feedback_object
*) obj
;
485 /* Flush any drawing so that the counters have the right values. */
486 intel_batchbuffer_emit_mi_flush(brw
);
488 /* Save the SOL buffer offset register values. */
490 for (int i
= 0; i
< 4; i
++) {
492 OUT_BATCH(MI_STORE_REGISTER_MEM
| (3 - 2));
493 OUT_BATCH(GEN7_SO_WRITE_OFFSET(i
));
494 OUT_RELOC(brw_obj
->offset_bo
,
495 I915_GEM_DOMAIN_INSTRUCTION
, I915_GEM_DOMAIN_INSTRUCTION
,
496 i
* sizeof(uint32_t));
501 /* Store the temporary ending value of the SO_NUM_PRIMS_WRITTEN counters.
502 * While this operation is paused, other transform feedback actions may
503 * occur, which will contribute to the counters. We need to exclude that
506 gen7_save_primitives_written_counters(brw
, brw_obj
);
510 gen7_resume_transform_feedback(struct gl_context
*ctx
,
511 struct gl_transform_feedback_object
*obj
)
513 struct brw_context
*brw
= brw_context(ctx
);
514 struct brw_transform_feedback_object
*brw_obj
=
515 (struct brw_transform_feedback_object
*) obj
;
517 /* Reload the SOL buffer offset registers. */
519 for (int i
= 0; i
< 4; i
++) {
521 OUT_BATCH(GEN7_MI_LOAD_REGISTER_MEM
| (3 - 2));
522 OUT_BATCH(GEN7_SO_WRITE_OFFSET(i
));
523 OUT_RELOC(brw_obj
->offset_bo
,
524 I915_GEM_DOMAIN_INSTRUCTION
, I915_GEM_DOMAIN_INSTRUCTION
,
525 i
* sizeof(uint32_t));
530 /* Store the new starting value of the SO_NUM_PRIMS_WRITTEN counters. */
531 gen7_save_primitives_written_counters(brw
, brw_obj
);