2 Copyright (C) Intel Corp. 2006. All Rights Reserved.
3 Intel funded Tungsten Graphics to
4 develop this 3D driver.
6 Permission is hereby granted, free of charge, to any person obtaining
7 a copy of this software and associated documentation files (the
8 "Software"), to deal in the Software without restriction, including
9 without limitation the rights to use, copy, modify, merge, publish,
10 distribute, sublicense, and/or sell copies of the Software, and to
11 permit persons to whom the Software is furnished to do so, subject to
12 the following conditions:
14 The above copyright notice and this permission notice (including the
15 next paragraph) shall be included in all copies or substantial
16 portions of the Software.
18 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
19 EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
20 MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
21 IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE
22 LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
23 OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
24 WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
26 **********************************************************************/
29 * Keith Whitwell <keithw@vmware.com>
34 #include "intel_batchbuffer.h"
35 #include "intel_fbo.h"
36 #include "intel_mipmap_tree.h"
38 #include "brw_context.h"
39 #include "brw_state.h"
40 #include "brw_defines.h"
41 #include "compiler/brw_eu_defines.h"
43 #include "main/framebuffer.h"
44 #include "main/fbobject.h"
45 #include "main/format_utils.h"
46 #include "main/glformats.h"
49 * Upload pointers to the per-stage state.
51 * The state pointers in this packet are all relative to the general state
52 * base address set by CMD_STATE_BASE_ADDRESS, which is 0.
55 upload_pipelined_state_pointers(struct brw_context
*brw
)
57 const struct gen_device_info
*devinfo
= &brw
->screen
->devinfo
;
59 if (devinfo
->gen
== 5) {
60 /* Need to flush before changing clip max threads for errata. */
67 OUT_BATCH(_3DSTATE_PIPELINED_POINTERS
<< 16 | (7 - 2));
68 OUT_RELOC(brw
->batch
.state
.bo
, 0, brw
->vs
.base
.state_offset
);
69 if (brw
->ff_gs
.prog_active
)
70 OUT_RELOC(brw
->batch
.state
.bo
, 0, brw
->ff_gs
.state_offset
| 1);
73 OUT_RELOC(brw
->batch
.state
.bo
, 0, brw
->clip
.state_offset
| 1);
74 OUT_RELOC(brw
->batch
.state
.bo
, 0, brw
->sf
.state_offset
);
75 OUT_RELOC(brw
->batch
.state
.bo
, 0, brw
->wm
.base
.state_offset
);
76 OUT_RELOC(brw
->batch
.state
.bo
, 0, brw
->cc
.state_offset
);
79 brw
->ctx
.NewDriverState
|= BRW_NEW_PSP
;
83 upload_psp_urb_cbs(struct brw_context
*brw
)
85 upload_pipelined_state_pointers(brw
);
86 brw_upload_urb_fence(brw
);
87 brw_upload_cs_urb_state(brw
);
90 const struct brw_tracked_state brw_psp_urb_cbs
= {
93 .brw
= BRW_NEW_BATCH
|
95 BRW_NEW_FF_GS_PROG_DATA
|
96 BRW_NEW_GEN4_UNIT_STATE
|
97 BRW_NEW_STATE_BASE_ADDRESS
|
100 .emit
= upload_psp_urb_cbs
,
104 brw_depthbuffer_format(struct brw_context
*brw
)
106 struct gl_context
*ctx
= &brw
->ctx
;
107 struct gl_framebuffer
*fb
= ctx
->DrawBuffer
;
108 struct intel_renderbuffer
*drb
= intel_get_renderbuffer(fb
, BUFFER_DEPTH
);
109 struct intel_renderbuffer
*srb
;
112 (srb
= intel_get_renderbuffer(fb
, BUFFER_STENCIL
)) &&
113 !srb
->mt
->stencil_mt
&&
114 (intel_rb_format(srb
) == MESA_FORMAT_Z24_UNORM_S8_UINT
||
115 intel_rb_format(srb
) == MESA_FORMAT_Z32_FLOAT_S8X24_UINT
)) {
120 return BRW_DEPTHFORMAT_D32_FLOAT
;
122 return brw_depth_format(brw
, drb
->mt
->format
);
125 static struct intel_mipmap_tree
*
126 get_stencil_miptree(struct intel_renderbuffer
*irb
)
130 if (irb
->mt
->stencil_mt
)
131 return irb
->mt
->stencil_mt
;
132 return intel_renderbuffer_get_mt(irb
);
136 rebase_depth_stencil(struct brw_context
*brw
, struct intel_renderbuffer
*irb
,
139 const struct gen_device_info
*devinfo
= &brw
->screen
->devinfo
;
140 struct gl_context
*ctx
= &brw
->ctx
;
141 uint32_t tile_mask_x
= 0, tile_mask_y
= 0;
143 intel_get_tile_masks(irb
->mt
->surf
.tiling
, irb
->mt
->cpp
,
144 &tile_mask_x
, &tile_mask_y
);
145 assert(!intel_miptree_level_has_hiz(irb
->mt
, irb
->mt_level
));
147 uint32_t tile_x
= irb
->draw_x
& tile_mask_x
;
148 uint32_t tile_y
= irb
->draw_y
& tile_mask_y
;
150 /* According to the Sandy Bridge PRM, volume 2 part 1, pp326-327
151 * (3DSTATE_DEPTH_BUFFER dw5), in the documentation for "Depth
152 * Coordinate Offset X/Y":
154 * "The 3 LSBs of both offsets must be zero to ensure correct
157 bool rebase
= tile_x
& 7 || tile_y
& 7;
159 /* We didn't even have intra-tile offsets before g45. */
160 rebase
|= (!devinfo
->has_surface_tile_offset
&& (tile_x
|| tile_y
));
163 perf_debug("HW workaround: blitting depth level %d to a temporary "
164 "to fix alignment (depth tile offset %d,%d)\n",
165 irb
->mt_level
, tile_x
, tile_y
);
166 intel_renderbuffer_move_to_temp(brw
, irb
, invalidate
);
168 /* There is now only single slice miptree. */
169 brw
->depthstencil
.tile_x
= 0;
170 brw
->depthstencil
.tile_y
= 0;
171 brw
->depthstencil
.depth_offset
= 0;
175 /* While we just tried to get everything aligned, we may have failed to do
176 * so in the case of rendering to array or 3D textures, where nonzero faces
177 * will still have an offset post-rebase. At least give an informative
180 WARN_ONCE((tile_x
& 7) || (tile_y
& 7),
181 "Depth/stencil buffer needs alignment to 8-pixel boundaries.\n"
182 "Truncating offset (%u:%u), bad rendering may occur.\n",
187 brw
->depthstencil
.tile_x
= tile_x
;
188 brw
->depthstencil
.tile_y
= tile_y
;
189 brw
->depthstencil
.depth_offset
= intel_miptree_get_aligned_offset(
191 irb
->draw_x
& ~tile_mask_x
,
192 irb
->draw_y
& ~tile_mask_y
);
198 brw_workaround_depthstencil_alignment(struct brw_context
*brw
,
199 GLbitfield clear_mask
)
201 const struct gen_device_info
*devinfo
= &brw
->screen
->devinfo
;
202 struct gl_context
*ctx
= &brw
->ctx
;
203 struct gl_framebuffer
*fb
= ctx
->DrawBuffer
;
204 struct intel_renderbuffer
*depth_irb
= intel_get_renderbuffer(fb
, BUFFER_DEPTH
);
205 struct intel_renderbuffer
*stencil_irb
= intel_get_renderbuffer(fb
, BUFFER_STENCIL
);
206 struct intel_mipmap_tree
*depth_mt
= NULL
;
207 bool invalidate_depth
= clear_mask
& BUFFER_BIT_DEPTH
;
208 bool invalidate_stencil
= clear_mask
& BUFFER_BIT_STENCIL
;
211 depth_mt
= depth_irb
->mt
;
213 /* Initialize brw->depthstencil to 'nop' workaround state.
215 brw
->depthstencil
.tile_x
= 0;
216 brw
->depthstencil
.tile_y
= 0;
217 brw
->depthstencil
.depth_offset
= 0;
219 /* Gen6+ doesn't require the workarounds, since we always program the
220 * surface state at the start of the whole surface.
222 if (devinfo
->gen
>= 6)
225 /* Check if depth buffer is in depth/stencil format. If so, then it's only
226 * safe to invalidate it if we're also clearing stencil.
228 if (depth_irb
&& invalidate_depth
&&
229 _mesa_get_format_base_format(depth_mt
->format
) == GL_DEPTH_STENCIL
)
230 invalidate_depth
= invalidate_stencil
&& stencil_irb
;
233 if (rebase_depth_stencil(brw
, depth_irb
, invalidate_depth
)) {
234 /* In the case of stencil_irb being the same packed depth/stencil
235 * texture but not the same rb, make it point at our rebased mt, too.
238 stencil_irb
!= depth_irb
&&
239 stencil_irb
->mt
== depth_mt
) {
240 intel_miptree_reference(&stencil_irb
->mt
, depth_irb
->mt
);
241 intel_renderbuffer_set_draw_offset(stencil_irb
);
246 assert(stencil_irb
->mt
== depth_irb
->mt
);
247 assert(stencil_irb
->mt_level
== depth_irb
->mt_level
);
248 assert(stencil_irb
->mt_layer
== depth_irb
->mt_layer
);
252 /* If there is no depth attachment, consider if stencil needs rebase. */
253 if (!depth_irb
&& stencil_irb
)
254 rebase_depth_stencil(brw
, stencil_irb
, invalidate_stencil
);
258 brw_emit_depth_stencil_hiz(struct brw_context
*brw
,
259 struct intel_renderbuffer
*depth_irb
,
260 struct intel_mipmap_tree
*depth_mt
,
261 struct intel_renderbuffer
*stencil_irb
,
262 struct intel_mipmap_tree
*stencil_mt
)
264 uint32_t tile_x
= brw
->depthstencil
.tile_x
;
265 uint32_t tile_y
= brw
->depthstencil
.tile_y
;
266 uint32_t depth_surface_type
= BRW_SURFACE_NULL
;
267 uint32_t depthbuffer_format
= BRW_DEPTHFORMAT_D32_FLOAT
;
268 uint32_t depth_offset
= 0;
269 uint32_t width
= 1, height
= 1;
270 bool tiled_surface
= true;
272 /* If there's a packed depth/stencil bound to stencil only, we need to
273 * emit the packed depth/stencil buffer packet.
275 if (!depth_irb
&& stencil_irb
) {
276 depth_irb
= stencil_irb
;
277 depth_mt
= stencil_mt
;
280 if (depth_irb
&& depth_mt
) {
281 depthbuffer_format
= brw_depthbuffer_format(brw
);
282 depth_surface_type
= BRW_SURFACE_2D
;
283 depth_offset
= brw
->depthstencil
.depth_offset
;
284 width
= depth_irb
->Base
.Base
.Width
;
285 height
= depth_irb
->Base
.Base
.Height
;
286 tiled_surface
= depth_mt
->surf
.tiling
!= ISL_TILING_LINEAR
;
289 const struct gen_device_info
*devinfo
= &brw
->screen
->devinfo
;
290 const unsigned len
= (devinfo
->is_g4x
|| devinfo
->gen
== 5) ? 6 : 5;
293 OUT_BATCH(_3DSTATE_DEPTH_BUFFER
<< 16 | (len
- 2));
294 OUT_BATCH((depth_mt
? depth_mt
->surf
.row_pitch_B
- 1 : 0) |
295 (depthbuffer_format
<< 18) |
296 (BRW_TILEWALK_YMAJOR
<< 26) |
297 (tiled_surface
<< 27) |
298 (depth_surface_type
<< 29));
301 OUT_RELOC(depth_mt
->bo
, RELOC_WRITE
, depth_offset
);
306 OUT_BATCH(((width
+ tile_x
- 1) << 6) |
307 ((height
+ tile_y
- 1) << 19));
310 if (devinfo
->is_g4x
|| devinfo
->gen
>= 5)
311 OUT_BATCH(tile_x
| (tile_y
<< 16));
313 assert(tile_x
== 0 && tile_y
== 0);
315 if (devinfo
->gen
>= 6)
322 brw_emit_depthbuffer(struct brw_context
*brw
)
324 const struct gen_device_info
*devinfo
= &brw
->screen
->devinfo
;
325 struct gl_context
*ctx
= &brw
->ctx
;
326 struct gl_framebuffer
*fb
= ctx
->DrawBuffer
;
328 struct intel_renderbuffer
*depth_irb
= intel_get_renderbuffer(fb
, BUFFER_DEPTH
);
329 struct intel_renderbuffer
*stencil_irb
= intel_get_renderbuffer(fb
, BUFFER_STENCIL
);
330 struct intel_mipmap_tree
*depth_mt
= intel_renderbuffer_get_mt(depth_irb
);
331 struct intel_mipmap_tree
*stencil_mt
= get_stencil_miptree(stencil_irb
);
334 brw_cache_flush_for_depth(brw
, depth_mt
->bo
);
336 brw_cache_flush_for_depth(brw
, stencil_mt
->bo
);
338 if (devinfo
->gen
< 6) {
339 brw_emit_depth_stencil_hiz(brw
, depth_irb
, depth_mt
,
340 stencil_irb
, stencil_mt
);
344 /* Skip repeated NULL depth/stencil emits (think 2D rendering). */
345 if (!depth_mt
&& !stencil_mt
&& brw
->no_depth_or_stencil
) {
350 brw_emit_depth_stall_flushes(brw
);
352 const unsigned ds_dwords
= brw
->isl_dev
.ds
.size
/ 4;
353 intel_batchbuffer_begin(brw
, ds_dwords
);
354 uint32_t *ds_map
= brw
->batch
.map_next
;
355 const uint32_t ds_offset
= (char *)ds_map
- (char *)brw
->batch
.batch
.map
;
357 struct isl_view view
= {
358 /* Some nice defaults */
361 .base_array_layer
= 0,
363 .swizzle
= ISL_SWIZZLE_IDENTITY
,
366 struct isl_depth_stencil_hiz_emit_info info
= {
371 view
.usage
|= ISL_SURF_USAGE_DEPTH_BIT
;
372 info
.depth_surf
= &depth_mt
->surf
;
375 brw_batch_reloc(&brw
->batch
,
376 ds_offset
+ brw
->isl_dev
.ds
.depth_offset
,
377 depth_mt
->bo
, depth_mt
->offset
, RELOC_WRITE
);
379 info
.mocs
= brw_get_bo_mocs(devinfo
, depth_mt
->bo
);
380 view
.base_level
= depth_irb
->mt_level
- depth_irb
->mt
->first_level
;
381 view
.base_array_layer
= depth_irb
->mt_layer
;
382 view
.array_len
= MAX2(depth_irb
->layer_count
, 1);
383 view
.format
= depth_mt
->surf
.format
;
385 info
.hiz_usage
= depth_mt
->aux_usage
;
386 if (!intel_renderbuffer_has_hiz(depth_irb
)) {
387 /* Just because a miptree has ISL_AUX_USAGE_HIZ does not mean that
388 * all miplevels of that miptree are guaranteed to support HiZ. See
389 * intel_miptree_level_enable_hiz for details.
391 info
.hiz_usage
= ISL_AUX_USAGE_NONE
;
394 if (info
.hiz_usage
== ISL_AUX_USAGE_HIZ
) {
395 info
.hiz_surf
= &depth_mt
->aux_buf
->surf
;
397 uint32_t hiz_offset
= 0;
398 if (devinfo
->gen
== 6) {
399 /* HiZ surfaces on Sandy Bridge technically don't support
400 * mip-mapping. However, we can fake it by offsetting to the
401 * first slice of LOD0 in the HiZ surface.
403 isl_surf_get_image_offset_B_tile_sa(&depth_mt
->aux_buf
->surf
,
404 view
.base_level
, 0, 0,
405 &hiz_offset
, NULL
, NULL
);
409 brw_batch_reloc(&brw
->batch
,
410 ds_offset
+ brw
->isl_dev
.ds
.hiz_offset
,
411 depth_mt
->aux_buf
->bo
,
412 depth_mt
->aux_buf
->offset
+ hiz_offset
,
416 info
.depth_clear_value
= depth_mt
->fast_clear_color
.f32
[0];
420 view
.usage
|= ISL_SURF_USAGE_STENCIL_BIT
;
421 info
.stencil_surf
= &stencil_mt
->surf
;
424 info
.mocs
= brw_get_bo_mocs(devinfo
, stencil_mt
->bo
);
425 view
.base_level
= stencil_irb
->mt_level
- stencil_irb
->mt
->first_level
;
426 view
.base_array_layer
= stencil_irb
->mt_layer
;
427 view
.array_len
= MAX2(stencil_irb
->layer_count
, 1);
428 view
.format
= stencil_mt
->surf
.format
;
431 uint32_t stencil_offset
= 0;
432 if (devinfo
->gen
== 6) {
433 /* Stencil surfaces on Sandy Bridge technically don't support
434 * mip-mapping. However, we can fake it by offsetting to the
435 * first slice of LOD0 in the stencil surface.
437 isl_surf_get_image_offset_B_tile_sa(&stencil_mt
->surf
,
438 view
.base_level
, 0, 0,
439 &stencil_offset
, NULL
, NULL
);
442 info
.stencil_address
=
443 brw_batch_reloc(&brw
->batch
,
444 ds_offset
+ brw
->isl_dev
.ds
.stencil_offset
,
446 stencil_mt
->offset
+ stencil_offset
,
450 isl_emit_depth_stencil_hiz_s(&brw
->isl_dev
, ds_map
, &info
);
452 brw
->batch
.map_next
+= ds_dwords
;
453 intel_batchbuffer_advance(brw
);
455 brw
->no_depth_or_stencil
= !depth_mt
&& !stencil_mt
;
458 const struct brw_tracked_state brw_depthbuffer
= {
460 .mesa
= _NEW_BUFFERS
,
461 .brw
= BRW_NEW_AUX_STATE
|
465 .emit
= brw_emit_depthbuffer
,
469 brw_emit_select_pipeline(struct brw_context
*brw
, enum brw_pipeline pipeline
)
471 const struct gen_device_info
*devinfo
= &brw
->screen
->devinfo
;
472 const bool is_965
= devinfo
->gen
== 4 && !devinfo
->is_g4x
;
473 const uint32_t _3DSTATE_PIPELINE_SELECT
=
474 is_965
? CMD_PIPELINE_SELECT_965
: CMD_PIPELINE_SELECT_GM45
;
476 if (devinfo
->gen
>= 8 && devinfo
->gen
< 10) {
477 /* From the Broadwell PRM, Volume 2a: Instructions, PIPELINE_SELECT:
479 * Software must clear the COLOR_CALC_STATE Valid field in
480 * 3DSTATE_CC_STATE_POINTERS command prior to send a PIPELINE_SELECT
481 * with Pipeline Select set to GPGPU.
483 * The internal hardware docs recommend the same workaround for Gen9
486 if (pipeline
== BRW_COMPUTE_PIPELINE
) {
488 OUT_BATCH(_3DSTATE_CC_STATE_POINTERS
<< 16 | (2 - 2));
492 brw
->ctx
.NewDriverState
|= BRW_NEW_CC_STATE
;
496 if (devinfo
->gen
== 9 && pipeline
== BRW_RENDER_PIPELINE
) {
497 /* We seem to have issues with geometry flickering when 3D and compute
498 * are combined in the same batch and this appears to fix it.
500 const uint32_t subslices
= MAX2(brw
->screen
->subslice_total
, 1);
501 const uint32_t maxNumberofThreads
=
502 devinfo
->max_cs_threads
* subslices
- 1;
505 OUT_BATCH(MEDIA_VFE_STATE
<< 16 | (9 - 2));
508 OUT_BATCH(2 << 8 | maxNumberofThreads
<< 16);
517 if (devinfo
->gen
>= 6) {
518 /* From "BXML » GT » MI » vol1a GPU Overview » [Instruction]
519 * PIPELINE_SELECT [DevBWR+]":
523 * Software must ensure all the write caches are flushed through a
524 * stalling PIPE_CONTROL command followed by another PIPE_CONTROL
525 * command to invalidate read only caches prior to programming
526 * MI_PIPELINE_SELECT command to change the Pipeline Select Mode.
528 const unsigned dc_flush
=
529 devinfo
->gen
>= 7 ? PIPE_CONTROL_DATA_CACHE_FLUSH
: 0;
531 brw_emit_pipe_control_flush(brw
,
532 PIPE_CONTROL_RENDER_TARGET_FLUSH
|
533 PIPE_CONTROL_DEPTH_CACHE_FLUSH
|
535 PIPE_CONTROL_CS_STALL
);
537 brw_emit_pipe_control_flush(brw
,
538 PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE
|
539 PIPE_CONTROL_CONST_CACHE_INVALIDATE
|
540 PIPE_CONTROL_STATE_CACHE_INVALIDATE
|
541 PIPE_CONTROL_INSTRUCTION_INVALIDATE
);
544 /* From "BXML » GT » MI » vol1a GPU Overview » [Instruction]
545 * PIPELINE_SELECT [DevBWR+]":
547 * Project: PRE-DEVSNB
549 * Software must ensure the current pipeline is flushed via an
550 * MI_FLUSH or PIPE_CONTROL prior to the execution of PIPELINE_SELECT.
557 /* Select the pipeline */
559 OUT_BATCH(_3DSTATE_PIPELINE_SELECT
<< 16 |
560 (devinfo
->gen
>= 9 ? (3 << 8) : 0) |
561 (pipeline
== BRW_COMPUTE_PIPELINE
? 2 : 0));
564 if (devinfo
->gen
== 7 && !devinfo
->is_haswell
&&
565 pipeline
== BRW_RENDER_PIPELINE
) {
566 /* From "BXML » GT » MI » vol1a GPU Overview » [Instruction]
567 * PIPELINE_SELECT [DevBWR+]":
569 * Project: DEVIVB, DEVHSW:GT3:A0
571 * Software must send a pipe_control with a CS stall and a post sync
572 * operation and then a dummy DRAW after every MI_SET_CONTEXT and
573 * after any PIPELINE_SELECT that is enabling 3D mode.
575 gen7_emit_cs_stall_flush(brw
);
578 OUT_BATCH(CMD_3D_PRIM
<< 16 | (7 - 2));
579 OUT_BATCH(_3DPRIM_POINTLIST
);
588 if (devinfo
->is_geminilake
) {
591 * "This chicken bit works around a hardware issue with barrier logic
592 * encountered when switching between GPGPU and 3D pipelines. To
593 * workaround the issue, this mode bit should be set after a pipeline
596 const unsigned barrier_mode
=
597 pipeline
== BRW_RENDER_PIPELINE
? GLK_SCEC_BARRIER_MODE_3D_HULL
598 : GLK_SCEC_BARRIER_MODE_GPGPU
;
599 brw_load_register_imm32(brw
, SLICE_COMMON_ECO_CHICKEN1
,
600 barrier_mode
| GLK_SCEC_BARRIER_MODE_MASK
);
605 * Update the pixel hashing modes that determine the balancing of PS threads
606 * across subslices and slices.
608 * \param width Width bound of the rendering area (already scaled down if \p
609 * scale is greater than 1).
610 * \param height Height bound of the rendering area (already scaled down if \p
611 * scale is greater than 1).
612 * \param scale The number of framebuffer samples that could potentially be
613 * affected by an individual channel of the PS thread. This is
614 * typically one for single-sampled rendering, but for operations
615 * like CCS resolves and fast clears a single PS invocation may
616 * update a huge number of pixels, in which case a finer
617 * balancing is desirable in order to maximally utilize the
618 * bandwidth available. UINT_MAX can be used as shorthand for
619 * "finest hashing mode available".
622 brw_emit_hashing_mode(struct brw_context
*brw
, unsigned width
,
623 unsigned height
, unsigned scale
)
625 const struct gen_device_info
*devinfo
= &brw
->screen
->devinfo
;
627 if (devinfo
->gen
== 9) {
628 const uint32_t slice_hashing
[] = {
629 /* Because all Gen9 platforms with more than one slice require
630 * three-way subslice hashing, a single "normal" 16x16 slice hashing
631 * block is guaranteed to suffer from substantial imbalance, with one
632 * subslice receiving twice as much work as the other two in the
635 * The performance impact of that would be particularly severe when
636 * three-way hashing is also in use for slice balancing (which is the
637 * case for all Gen9 GT4 platforms), because one of the slices
638 * receives one every three 16x16 blocks in either direction, which
639 * is roughly the periodicity of the underlying subslice imbalance
640 * pattern ("roughly" because in reality the hardware's
641 * implementation of three-way hashing doesn't do exact modulo 3
642 * arithmetic, which somewhat decreases the magnitude of this effect
643 * in practice). This leads to a systematic subslice imbalance
644 * within that slice regardless of the size of the primitive. The
645 * 32x32 hashing mode guarantees that the subslice imbalance within a
646 * single slice hashing block is minimal, largely eliminating this
649 GEN9_SLICE_HASHING_32x32
,
650 /* Finest slice hashing mode available. */
651 GEN9_SLICE_HASHING_NORMAL
653 const uint32_t subslice_hashing
[] = {
654 /* The 16x16 subslice hashing mode is used on non-LLC platforms to
655 * match the performance of previous Mesa versions. 16x16 has a
656 * slight cache locality benefit especially visible in the sampler L1
657 * cache efficiency of low-bandwidth platforms, but it comes at the
658 * cost of greater subslice imbalance for primitives of dimensions
659 * approximately intermediate between 16x4 and 16x16.
661 (devinfo
->has_llc
? GEN9_SUBSLICE_HASHING_16x4
:
662 GEN9_SUBSLICE_HASHING_16x16
),
663 /* Finest subslice hashing mode available. */
664 GEN9_SUBSLICE_HASHING_8x4
666 /* Dimensions of the smallest hashing block of a given hashing mode. If
667 * the rendering area is smaller than this there can't possibly be any
668 * benefit from switching to this mode, so we optimize out the
671 const unsigned min_size
[][2] = {
675 const unsigned idx
= scale
> 1;
677 if (width
> min_size
[idx
][0] || height
> min_size
[idx
][1]) {
678 const uint32_t gt_mode
=
679 (devinfo
->num_slices
== 1 ? 0 :
680 GEN9_SLICE_HASHING_MASK_BITS
| slice_hashing
[idx
]) |
681 GEN9_SUBSLICE_HASHING_MASK_BITS
| subslice_hashing
[idx
];
683 brw_emit_pipe_control_flush(brw
,
684 PIPE_CONTROL_STALL_AT_SCOREBOARD
|
685 PIPE_CONTROL_CS_STALL
);
687 brw_load_register_imm32(brw
, GEN7_GT_MODE
, gt_mode
);
689 brw
->current_hash_scale
= scale
;
695 * Misc invariant state packets
698 brw_upload_invariant_state(struct brw_context
*brw
)
700 const struct gen_device_info
*devinfo
= &brw
->screen
->devinfo
;
701 const bool is_965
= devinfo
->gen
== 4 && !devinfo
->is_g4x
;
703 brw_emit_select_pipeline(brw
, BRW_RENDER_PIPELINE
);
704 brw
->last_pipeline
= BRW_RENDER_PIPELINE
;
706 if (devinfo
->gen
>= 8) {
708 OUT_BATCH(CMD_STATE_SIP
<< 16 | (3 - 2));
714 OUT_BATCH(CMD_STATE_SIP
<< 16 | (2 - 2));
719 /* Original Gen4 doesn't have 3DSTATE_AA_LINE_PARAMETERS. */
722 OUT_BATCH(_3DSTATE_AA_LINE_PARAMETERS
<< 16 | (3 - 2));
723 /* use legacy aa line coverage computation */
731 * Define the base addresses which some state is referenced from.
733 * This allows us to avoid having to emit relocations for the objects,
734 * and is actually required for binding table pointers on gen6.
736 * Surface state base address covers binding table pointers and
737 * surface state objects, but not the surfaces that the surface state
741 brw_upload_state_base_address(struct brw_context
*brw
)
743 const struct gen_device_info
*devinfo
= &brw
->screen
->devinfo
;
745 if (brw
->batch
.state_base_address_emitted
)
748 /* FINISHME: According to section 3.6.1 "STATE_BASE_ADDRESS" of
749 * vol1a of the G45 PRM, MI_FLUSH with the ISC invalidate should be
750 * programmed prior to STATE_BASE_ADDRESS.
752 * However, given that the instruction SBA (general state base
753 * address) on this chipset is always set to 0 across X and GL,
754 * maybe this isn't required for us in particular.
757 if (devinfo
->gen
>= 6) {
758 const unsigned dc_flush
=
759 devinfo
->gen
>= 7 ? PIPE_CONTROL_DATA_CACHE_FLUSH
: 0;
761 /* Emit a render target cache flush.
763 * This isn't documented anywhere in the PRM. However, it seems to be
764 * necessary prior to changing the surface state base adress. We've
765 * seen issues in Vulkan where we get GPU hangs when using multi-level
766 * command buffers which clear depth, reset state base address, and then
769 * Normally, in GL, we would trust the kernel to do sufficient stalls
770 * and flushes prior to executing our batch. However, it doesn't seem
771 * as if the kernel's flushing is always sufficient and we don't want to
774 * We make this an end-of-pipe sync instead of a normal flush because we
775 * do not know the current status of the GPU. On Haswell at least,
776 * having a fast-clear operation in flight at the same time as a normal
777 * rendering operation can cause hangs. Since the kernel's flushing is
778 * insufficient, we need to ensure that any rendering operations from
779 * other processes are definitely complete before we try to do our own
780 * rendering. It's a bit of a big hammer but it appears to work.
782 brw_emit_end_of_pipe_sync(brw
,
783 PIPE_CONTROL_RENDER_TARGET_FLUSH
|
784 PIPE_CONTROL_DEPTH_CACHE_FLUSH
|
788 if (devinfo
->gen
>= 8) {
789 /* STATE_BASE_ADDRESS has issues with 48-bit address spaces. If the
790 * address + size as seen by STATE_BASE_ADDRESS overflows 48 bits,
791 * the GPU appears to treat all accesses to the buffer as being out
792 * of bounds and returns zero. To work around this, we pin all SBAs
795 uint32_t mocs_wb
= devinfo
->gen
>= 9 ? SKL_MOCS_WB
: BDW_MOCS_WB
;
796 int pkt_len
= devinfo
->gen
>= 10 ? 22 : (devinfo
->gen
>= 9 ? 19 : 16);
798 BEGIN_BATCH(pkt_len
);
799 OUT_BATCH(CMD_STATE_BASE_ADDRESS
<< 16 | (pkt_len
- 2));
800 /* General state base address: stateless DP read/write requests */
801 OUT_BATCH(mocs_wb
<< 4 | 1);
803 OUT_BATCH(mocs_wb
<< 16);
804 /* Surface state base address: */
805 OUT_RELOC64(brw
->batch
.state
.bo
, RELOC_32BIT
, mocs_wb
<< 4 | 1);
806 /* Dynamic state base address: */
807 OUT_RELOC64(brw
->batch
.state
.bo
, RELOC_32BIT
, mocs_wb
<< 4 | 1);
808 /* Indirect object base address: MEDIA_OBJECT data */
809 OUT_BATCH(mocs_wb
<< 4 | 1);
811 /* Instruction base address: shader kernels (incl. SIP) */
812 OUT_RELOC64(brw
->cache
.bo
, RELOC_32BIT
, mocs_wb
<< 4 | 1);
813 /* General state buffer size */
814 OUT_BATCH(0xfffff001);
815 /* Dynamic state buffer size */
816 OUT_BATCH(ALIGN(MAX_STATE_SIZE
, 4096) | 1);
817 /* Indirect object upper bound */
818 OUT_BATCH(0xfffff001);
819 /* Instruction access upper bound */
820 OUT_BATCH(ALIGN(brw
->cache
.bo
->size
, 4096) | 1);
821 if (devinfo
->gen
>= 9) {
826 if (devinfo
->gen
>= 10) {
832 } else if (devinfo
->gen
>= 6) {
833 uint8_t mocs
= devinfo
->gen
== 7 ? GEN7_MOCS_L3
: 0;
836 OUT_BATCH(CMD_STATE_BASE_ADDRESS
<< 16 | (10 - 2));
837 OUT_BATCH(mocs
<< 8 | /* General State Memory Object Control State */
838 mocs
<< 4 | /* Stateless Data Port Access Memory Object Control State */
839 1); /* General State Base Address Modify Enable */
840 /* Surface state base address:
841 * BINDING_TABLE_STATE
844 OUT_RELOC(brw
->batch
.state
.bo
, 0, 1);
845 /* Dynamic state base address:
847 * SAMPLER_BORDER_COLOR_STATE
848 * CLIP, SF, WM/CC viewport state
850 * DEPTH_STENCIL_STATE
852 * Push constants (when INSTPM: CONSTANT_BUFFER Address Offset
853 * Disable is clear, which we rely on)
855 OUT_RELOC(brw
->batch
.state
.bo
, 0, 1);
857 OUT_BATCH(1); /* Indirect object base address: MEDIA_OBJECT data */
859 /* Instruction base address: shader kernels (incl. SIP) */
860 OUT_RELOC(brw
->cache
.bo
, 0, 1);
862 OUT_BATCH(1); /* General state upper bound */
863 /* Dynamic state upper bound. Although the documentation says that
864 * programming it to zero will cause it to be ignored, that is a lie.
865 * If this isn't programmed to a real bound, the sampler border color
866 * pointer is rejected, causing border color to mysteriously fail.
868 OUT_BATCH(0xfffff001);
869 OUT_BATCH(1); /* Indirect object upper bound */
870 OUT_BATCH(1); /* Instruction access upper bound */
872 } else if (devinfo
->gen
== 5) {
874 OUT_BATCH(CMD_STATE_BASE_ADDRESS
<< 16 | (8 - 2));
875 OUT_BATCH(1); /* General state base address */
876 OUT_RELOC(brw
->batch
.state
.bo
, 0, 1); /* Surface state base address */
877 OUT_BATCH(1); /* Indirect object base address */
878 OUT_RELOC(brw
->cache
.bo
, 0, 1); /* Instruction base address */
879 OUT_BATCH(0xfffff001); /* General state upper bound */
880 OUT_BATCH(1); /* Indirect object upper bound */
881 OUT_BATCH(1); /* Instruction access upper bound */
885 OUT_BATCH(CMD_STATE_BASE_ADDRESS
<< 16 | (6 - 2));
886 OUT_BATCH(1); /* General state base address */
887 OUT_RELOC(brw
->batch
.state
.bo
, 0, 1); /* Surface state base address */
888 OUT_BATCH(1); /* Indirect object base address */
889 OUT_BATCH(1); /* General state upper bound */
890 OUT_BATCH(1); /* Indirect object upper bound */
894 if (devinfo
->gen
>= 6) {
895 brw_emit_pipe_control_flush(brw
,
896 PIPE_CONTROL_INSTRUCTION_INVALIDATE
|
897 PIPE_CONTROL_STATE_CACHE_INVALIDATE
|
898 PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE
);
901 /* According to section 3.6.1 of VOL1 of the 965 PRM,
902 * STATE_BASE_ADDRESS updates require a reissue of:
904 * 3DSTATE_PIPELINE_POINTERS
905 * 3DSTATE_BINDING_TABLE_POINTERS
906 * MEDIA_STATE_POINTERS
908 * and this continues through Ironlake. The Sandy Bridge PRM, vol
909 * 1 part 1 says that the folowing packets must be reissued:
911 * 3DSTATE_CC_POINTERS
912 * 3DSTATE_BINDING_TABLE_POINTERS
913 * 3DSTATE_SAMPLER_STATE_POINTERS
914 * 3DSTATE_VIEWPORT_STATE_POINTERS
915 * MEDIA_STATE_POINTERS
917 * Those are always reissued following SBA updates anyway (new
918 * batch time), except in the case of the program cache BO
919 * changing. Having a separate state flag makes the sequence more
923 brw
->ctx
.NewDriverState
|= BRW_NEW_STATE_BASE_ADDRESS
;
924 brw
->batch
.state_base_address_emitted
= true;