iris: add support for gl_ClipVertex in geometry shaders
[mesa.git] / src / gallium / drivers / iris / iris_state.c
1 /*
2 * Copyright © 2017 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included
12 * in all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
15 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
19 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
20 * DEALINGS IN THE SOFTWARE.
21 */
22
23 /**
24 * @file iris_state.c
25 *
26 * ============================= GENXML CODE =============================
27 * [This file is compiled once per generation.]
28 * =======================================================================
29 *
30 * This is the main state upload code.
31 *
32 * Gallium uses Constant State Objects, or CSOs, for most state. Large,
33 * complex, or highly reusable state can be created once, and bound and
34 * rebound multiple times. This is modeled with the pipe->create_*_state()
35 * and pipe->bind_*_state() hooks. Highly dynamic or inexpensive state is
36 * streamed out on the fly, via pipe->set_*_state() hooks.
37 *
38 * OpenGL involves frequently mutating context state, which is mirrored in
39 * core Mesa by highly mutable data structures. However, most applications
40 * typically draw the same things over and over - from frame to frame, most
41 * of the same objects are still visible and need to be redrawn. So, rather
42 * than inventing new state all the time, applications usually mutate to swap
43 * between known states that we've seen before.
44 *
45 * Gallium isolates us from this mutation by tracking API state, and
46 * distilling it into a set of Constant State Objects, or CSOs. Large,
47 * complex, or typically reusable state can be created once, then reused
48 * multiple times. Drivers can create and store their own associated data.
49 * This create/bind model corresponds to the pipe->create_*_state() and
50 * pipe->bind_*_state() driver hooks.
51 *
52 * Some state is cheap to create, or expected to be highly dynamic. Rather
53 * than creating and caching piles of CSOs for these, Gallium simply streams
54 * them out, via the pipe->set_*_state() driver hooks.
55 *
56 * To reduce draw time overhead, we try to compute as much state at create
57 * time as possible. Wherever possible, we translate the Gallium pipe state
58 * to 3DSTATE commands, and store those commands in the CSO. At draw time,
59 * we can simply memcpy them into a batch buffer.
60 *
61 * No hardware matches the abstraction perfectly, so some commands require
62 * information from multiple CSOs. In this case, we can store two copies
63 * of the packet (one in each CSO), and simply | together their DWords at
64 * draw time. Sometimes the second set is trivial (one or two fields), so
65 * we simply pack it at draw time.
66 *
67 * There are two main components in the file below. First, the CSO hooks
68 * create/bind/track state. The second are the draw-time upload functions,
69 * iris_upload_render_state() and iris_upload_compute_state(), which read
70 * the context state and emit the commands into the actual batch.
71 */
72
73 #include <stdio.h>
74 #include <errno.h>
75
76 #if HAVE_VALGRIND
77 #include <valgrind.h>
78 #include <memcheck.h>
79 #define VG(x) x
80 #ifdef DEBUG
81 #define __gen_validate_value(x) VALGRIND_CHECK_MEM_IS_DEFINED(&(x), sizeof(x))
82 #endif
83 #else
84 #define VG(x)
85 #endif
86
87 #include "pipe/p_defines.h"
88 #include "pipe/p_state.h"
89 #include "pipe/p_context.h"
90 #include "pipe/p_screen.h"
91 #include "util/u_dual_blend.h"
92 #include "util/u_inlines.h"
93 #include "util/u_format.h"
94 #include "util/u_framebuffer.h"
95 #include "util/u_transfer.h"
96 #include "util/u_upload_mgr.h"
97 #include "util/u_viewport.h"
98 #include "drm-uapi/i915_drm.h"
99 #include "nir.h"
100 #include "intel/compiler/brw_compiler.h"
101 #include "intel/common/gen_l3_config.h"
102 #include "intel/common/gen_sample_positions.h"
103 #include "iris_batch.h"
104 #include "iris_context.h"
105 #include "iris_defines.h"
106 #include "iris_pipe.h"
107 #include "iris_resource.h"
108
109 #include "iris_genx_macros.h"
110 #include "intel/common/gen_guardband.h"
111
112 #if GEN_GEN == 8
113 #define MOCS_PTE 0x18
114 #define MOCS_WB 0x78
115 #else
116 #define MOCS_PTE (1 << 1)
117 #define MOCS_WB (2 << 1)
118 #endif
119
120 static uint32_t
121 mocs(const struct iris_bo *bo)
122 {
123 return bo && bo->external ? MOCS_PTE : MOCS_WB;
124 }
125
126 /**
127 * Statically assert that PIPE_* enums match the hardware packets.
128 * (As long as they match, we don't need to translate them.)
129 */
130 UNUSED static void pipe_asserts()
131 {
132 #define PIPE_ASSERT(x) STATIC_ASSERT((int)x)
133
134 /* pipe_logicop happens to match the hardware. */
135 PIPE_ASSERT(PIPE_LOGICOP_CLEAR == LOGICOP_CLEAR);
136 PIPE_ASSERT(PIPE_LOGICOP_NOR == LOGICOP_NOR);
137 PIPE_ASSERT(PIPE_LOGICOP_AND_INVERTED == LOGICOP_AND_INVERTED);
138 PIPE_ASSERT(PIPE_LOGICOP_COPY_INVERTED == LOGICOP_COPY_INVERTED);
139 PIPE_ASSERT(PIPE_LOGICOP_AND_REVERSE == LOGICOP_AND_REVERSE);
140 PIPE_ASSERT(PIPE_LOGICOP_INVERT == LOGICOP_INVERT);
141 PIPE_ASSERT(PIPE_LOGICOP_XOR == LOGICOP_XOR);
142 PIPE_ASSERT(PIPE_LOGICOP_NAND == LOGICOP_NAND);
143 PIPE_ASSERT(PIPE_LOGICOP_AND == LOGICOP_AND);
144 PIPE_ASSERT(PIPE_LOGICOP_EQUIV == LOGICOP_EQUIV);
145 PIPE_ASSERT(PIPE_LOGICOP_NOOP == LOGICOP_NOOP);
146 PIPE_ASSERT(PIPE_LOGICOP_OR_INVERTED == LOGICOP_OR_INVERTED);
147 PIPE_ASSERT(PIPE_LOGICOP_COPY == LOGICOP_COPY);
148 PIPE_ASSERT(PIPE_LOGICOP_OR_REVERSE == LOGICOP_OR_REVERSE);
149 PIPE_ASSERT(PIPE_LOGICOP_OR == LOGICOP_OR);
150 PIPE_ASSERT(PIPE_LOGICOP_SET == LOGICOP_SET);
151
152 /* pipe_blend_func happens to match the hardware. */
153 PIPE_ASSERT(PIPE_BLENDFACTOR_ONE == BLENDFACTOR_ONE);
154 PIPE_ASSERT(PIPE_BLENDFACTOR_SRC_COLOR == BLENDFACTOR_SRC_COLOR);
155 PIPE_ASSERT(PIPE_BLENDFACTOR_SRC_ALPHA == BLENDFACTOR_SRC_ALPHA);
156 PIPE_ASSERT(PIPE_BLENDFACTOR_DST_ALPHA == BLENDFACTOR_DST_ALPHA);
157 PIPE_ASSERT(PIPE_BLENDFACTOR_DST_COLOR == BLENDFACTOR_DST_COLOR);
158 PIPE_ASSERT(PIPE_BLENDFACTOR_SRC_ALPHA_SATURATE == BLENDFACTOR_SRC_ALPHA_SATURATE);
159 PIPE_ASSERT(PIPE_BLENDFACTOR_CONST_COLOR == BLENDFACTOR_CONST_COLOR);
160 PIPE_ASSERT(PIPE_BLENDFACTOR_CONST_ALPHA == BLENDFACTOR_CONST_ALPHA);
161 PIPE_ASSERT(PIPE_BLENDFACTOR_SRC1_COLOR == BLENDFACTOR_SRC1_COLOR);
162 PIPE_ASSERT(PIPE_BLENDFACTOR_SRC1_ALPHA == BLENDFACTOR_SRC1_ALPHA);
163 PIPE_ASSERT(PIPE_BLENDFACTOR_ZERO == BLENDFACTOR_ZERO);
164 PIPE_ASSERT(PIPE_BLENDFACTOR_INV_SRC_COLOR == BLENDFACTOR_INV_SRC_COLOR);
165 PIPE_ASSERT(PIPE_BLENDFACTOR_INV_SRC_ALPHA == BLENDFACTOR_INV_SRC_ALPHA);
166 PIPE_ASSERT(PIPE_BLENDFACTOR_INV_DST_ALPHA == BLENDFACTOR_INV_DST_ALPHA);
167 PIPE_ASSERT(PIPE_BLENDFACTOR_INV_DST_COLOR == BLENDFACTOR_INV_DST_COLOR);
168 PIPE_ASSERT(PIPE_BLENDFACTOR_INV_CONST_COLOR == BLENDFACTOR_INV_CONST_COLOR);
169 PIPE_ASSERT(PIPE_BLENDFACTOR_INV_CONST_ALPHA == BLENDFACTOR_INV_CONST_ALPHA);
170 PIPE_ASSERT(PIPE_BLENDFACTOR_INV_SRC1_COLOR == BLENDFACTOR_INV_SRC1_COLOR);
171 PIPE_ASSERT(PIPE_BLENDFACTOR_INV_SRC1_ALPHA == BLENDFACTOR_INV_SRC1_ALPHA);
172
173 /* pipe_blend_func happens to match the hardware. */
174 PIPE_ASSERT(PIPE_BLEND_ADD == BLENDFUNCTION_ADD);
175 PIPE_ASSERT(PIPE_BLEND_SUBTRACT == BLENDFUNCTION_SUBTRACT);
176 PIPE_ASSERT(PIPE_BLEND_REVERSE_SUBTRACT == BLENDFUNCTION_REVERSE_SUBTRACT);
177 PIPE_ASSERT(PIPE_BLEND_MIN == BLENDFUNCTION_MIN);
178 PIPE_ASSERT(PIPE_BLEND_MAX == BLENDFUNCTION_MAX);
179
180 /* pipe_stencil_op happens to match the hardware. */
181 PIPE_ASSERT(PIPE_STENCIL_OP_KEEP == STENCILOP_KEEP);
182 PIPE_ASSERT(PIPE_STENCIL_OP_ZERO == STENCILOP_ZERO);
183 PIPE_ASSERT(PIPE_STENCIL_OP_REPLACE == STENCILOP_REPLACE);
184 PIPE_ASSERT(PIPE_STENCIL_OP_INCR == STENCILOP_INCRSAT);
185 PIPE_ASSERT(PIPE_STENCIL_OP_DECR == STENCILOP_DECRSAT);
186 PIPE_ASSERT(PIPE_STENCIL_OP_INCR_WRAP == STENCILOP_INCR);
187 PIPE_ASSERT(PIPE_STENCIL_OP_DECR_WRAP == STENCILOP_DECR);
188 PIPE_ASSERT(PIPE_STENCIL_OP_INVERT == STENCILOP_INVERT);
189
190 /* pipe_sprite_coord_mode happens to match 3DSTATE_SBE */
191 PIPE_ASSERT(PIPE_SPRITE_COORD_UPPER_LEFT == UPPERLEFT);
192 PIPE_ASSERT(PIPE_SPRITE_COORD_LOWER_LEFT == LOWERLEFT);
193 #undef PIPE_ASSERT
194 }
195
196 static unsigned
197 translate_prim_type(enum pipe_prim_type prim, uint8_t verts_per_patch)
198 {
199 static const unsigned map[] = {
200 [PIPE_PRIM_POINTS] = _3DPRIM_POINTLIST,
201 [PIPE_PRIM_LINES] = _3DPRIM_LINELIST,
202 [PIPE_PRIM_LINE_LOOP] = _3DPRIM_LINELOOP,
203 [PIPE_PRIM_LINE_STRIP] = _3DPRIM_LINESTRIP,
204 [PIPE_PRIM_TRIANGLES] = _3DPRIM_TRILIST,
205 [PIPE_PRIM_TRIANGLE_STRIP] = _3DPRIM_TRISTRIP,
206 [PIPE_PRIM_TRIANGLE_FAN] = _3DPRIM_TRIFAN,
207 [PIPE_PRIM_QUADS] = _3DPRIM_QUADLIST,
208 [PIPE_PRIM_QUAD_STRIP] = _3DPRIM_QUADSTRIP,
209 [PIPE_PRIM_POLYGON] = _3DPRIM_POLYGON,
210 [PIPE_PRIM_LINES_ADJACENCY] = _3DPRIM_LINELIST_ADJ,
211 [PIPE_PRIM_LINE_STRIP_ADJACENCY] = _3DPRIM_LINESTRIP_ADJ,
212 [PIPE_PRIM_TRIANGLES_ADJACENCY] = _3DPRIM_TRILIST_ADJ,
213 [PIPE_PRIM_TRIANGLE_STRIP_ADJACENCY] = _3DPRIM_TRISTRIP_ADJ,
214 [PIPE_PRIM_PATCHES] = _3DPRIM_PATCHLIST_1 - 1,
215 };
216
217 return map[prim] + (prim == PIPE_PRIM_PATCHES ? verts_per_patch : 0);
218 }
219
220 static unsigned
221 translate_compare_func(enum pipe_compare_func pipe_func)
222 {
223 static const unsigned map[] = {
224 [PIPE_FUNC_NEVER] = COMPAREFUNCTION_NEVER,
225 [PIPE_FUNC_LESS] = COMPAREFUNCTION_LESS,
226 [PIPE_FUNC_EQUAL] = COMPAREFUNCTION_EQUAL,
227 [PIPE_FUNC_LEQUAL] = COMPAREFUNCTION_LEQUAL,
228 [PIPE_FUNC_GREATER] = COMPAREFUNCTION_GREATER,
229 [PIPE_FUNC_NOTEQUAL] = COMPAREFUNCTION_NOTEQUAL,
230 [PIPE_FUNC_GEQUAL] = COMPAREFUNCTION_GEQUAL,
231 [PIPE_FUNC_ALWAYS] = COMPAREFUNCTION_ALWAYS,
232 };
233 return map[pipe_func];
234 }
235
236 static unsigned
237 translate_shadow_func(enum pipe_compare_func pipe_func)
238 {
239 /* Gallium specifies the result of shadow comparisons as:
240 *
241 * 1 if ref <op> texel,
242 * 0 otherwise.
243 *
244 * The hardware does:
245 *
246 * 0 if texel <op> ref,
247 * 1 otherwise.
248 *
249 * So we need to flip the operator and also negate.
250 */
251 static const unsigned map[] = {
252 [PIPE_FUNC_NEVER] = PREFILTEROPALWAYS,
253 [PIPE_FUNC_LESS] = PREFILTEROPLEQUAL,
254 [PIPE_FUNC_EQUAL] = PREFILTEROPNOTEQUAL,
255 [PIPE_FUNC_LEQUAL] = PREFILTEROPLESS,
256 [PIPE_FUNC_GREATER] = PREFILTEROPGEQUAL,
257 [PIPE_FUNC_NOTEQUAL] = PREFILTEROPEQUAL,
258 [PIPE_FUNC_GEQUAL] = PREFILTEROPGREATER,
259 [PIPE_FUNC_ALWAYS] = PREFILTEROPNEVER,
260 };
261 return map[pipe_func];
262 }
263
264 static unsigned
265 translate_cull_mode(unsigned pipe_face)
266 {
267 static const unsigned map[4] = {
268 [PIPE_FACE_NONE] = CULLMODE_NONE,
269 [PIPE_FACE_FRONT] = CULLMODE_FRONT,
270 [PIPE_FACE_BACK] = CULLMODE_BACK,
271 [PIPE_FACE_FRONT_AND_BACK] = CULLMODE_BOTH,
272 };
273 return map[pipe_face];
274 }
275
276 static unsigned
277 translate_fill_mode(unsigned pipe_polymode)
278 {
279 static const unsigned map[4] = {
280 [PIPE_POLYGON_MODE_FILL] = FILL_MODE_SOLID,
281 [PIPE_POLYGON_MODE_LINE] = FILL_MODE_WIREFRAME,
282 [PIPE_POLYGON_MODE_POINT] = FILL_MODE_POINT,
283 [PIPE_POLYGON_MODE_FILL_RECTANGLE] = FILL_MODE_SOLID,
284 };
285 return map[pipe_polymode];
286 }
287
288 static unsigned
289 translate_mip_filter(enum pipe_tex_mipfilter pipe_mip)
290 {
291 static const unsigned map[] = {
292 [PIPE_TEX_MIPFILTER_NEAREST] = MIPFILTER_NEAREST,
293 [PIPE_TEX_MIPFILTER_LINEAR] = MIPFILTER_LINEAR,
294 [PIPE_TEX_MIPFILTER_NONE] = MIPFILTER_NONE,
295 };
296 return map[pipe_mip];
297 }
298
299 static uint32_t
300 translate_wrap(unsigned pipe_wrap)
301 {
302 static const unsigned map[] = {
303 [PIPE_TEX_WRAP_REPEAT] = TCM_WRAP,
304 [PIPE_TEX_WRAP_CLAMP] = TCM_HALF_BORDER,
305 [PIPE_TEX_WRAP_CLAMP_TO_EDGE] = TCM_CLAMP,
306 [PIPE_TEX_WRAP_CLAMP_TO_BORDER] = TCM_CLAMP_BORDER,
307 [PIPE_TEX_WRAP_MIRROR_REPEAT] = TCM_MIRROR,
308 [PIPE_TEX_WRAP_MIRROR_CLAMP_TO_EDGE] = TCM_MIRROR_ONCE,
309
310 /* These are unsupported. */
311 [PIPE_TEX_WRAP_MIRROR_CLAMP] = -1,
312 [PIPE_TEX_WRAP_MIRROR_CLAMP_TO_BORDER] = -1,
313 };
314 return map[pipe_wrap];
315 }
316
317 /**
318 * Allocate space for some indirect state.
319 *
320 * Return a pointer to the map (to fill it out) and a state ref (for
321 * referring to the state in GPU commands).
322 */
323 static void *
324 upload_state(struct u_upload_mgr *uploader,
325 struct iris_state_ref *ref,
326 unsigned size,
327 unsigned alignment)
328 {
329 void *p = NULL;
330 u_upload_alloc(uploader, 0, size, alignment, &ref->offset, &ref->res, &p);
331 return p;
332 }
333
334 /**
335 * Stream out temporary/short-lived state.
336 *
337 * This allocates space, pins the BO, and includes the BO address in the
338 * returned offset (which works because all state lives in 32-bit memory
339 * zones).
340 */
341 static uint32_t *
342 stream_state(struct iris_batch *batch,
343 struct u_upload_mgr *uploader,
344 struct pipe_resource **out_res,
345 unsigned size,
346 unsigned alignment,
347 uint32_t *out_offset)
348 {
349 void *ptr = NULL;
350
351 u_upload_alloc(uploader, 0, size, alignment, out_offset, out_res, &ptr);
352
353 struct iris_bo *bo = iris_resource_bo(*out_res);
354 iris_use_pinned_bo(batch, bo, false);
355
356 *out_offset += iris_bo_offset_from_base_address(bo);
357
358 iris_record_state_size(batch->state_sizes, *out_offset, size);
359
360 return ptr;
361 }
362
363 /**
364 * stream_state() + memcpy.
365 */
366 static uint32_t
367 emit_state(struct iris_batch *batch,
368 struct u_upload_mgr *uploader,
369 struct pipe_resource **out_res,
370 const void *data,
371 unsigned size,
372 unsigned alignment)
373 {
374 unsigned offset = 0;
375 uint32_t *map =
376 stream_state(batch, uploader, out_res, size, alignment, &offset);
377
378 if (map)
379 memcpy(map, data, size);
380
381 return offset;
382 }
383
384 /**
385 * Did field 'x' change between 'old_cso' and 'new_cso'?
386 *
387 * (If so, we may want to set some dirty flags.)
388 */
389 #define cso_changed(x) (!old_cso || (old_cso->x != new_cso->x))
390 #define cso_changed_memcmp(x) \
391 (!old_cso || memcmp(old_cso->x, new_cso->x, sizeof(old_cso->x)) != 0)
392
393 static void
394 flush_for_state_base_change(struct iris_batch *batch)
395 {
396 /* Flush before emitting STATE_BASE_ADDRESS.
397 *
398 * This isn't documented anywhere in the PRM. However, it seems to be
399 * necessary prior to changing the surface state base adress. We've
400 * seen issues in Vulkan where we get GPU hangs when using multi-level
401 * command buffers which clear depth, reset state base address, and then
402 * go render stuff.
403 *
404 * Normally, in GL, we would trust the kernel to do sufficient stalls
405 * and flushes prior to executing our batch. However, it doesn't seem
406 * as if the kernel's flushing is always sufficient and we don't want to
407 * rely on it.
408 *
409 * We make this an end-of-pipe sync instead of a normal flush because we
410 * do not know the current status of the GPU. On Haswell at least,
411 * having a fast-clear operation in flight at the same time as a normal
412 * rendering operation can cause hangs. Since the kernel's flushing is
413 * insufficient, we need to ensure that any rendering operations from
414 * other processes are definitely complete before we try to do our own
415 * rendering. It's a bit of a big hammer but it appears to work.
416 */
417 iris_emit_end_of_pipe_sync(batch,
418 "change STATE_BASE_ADDRESS",
419 PIPE_CONTROL_RENDER_TARGET_FLUSH |
420 PIPE_CONTROL_DEPTH_CACHE_FLUSH |
421 PIPE_CONTROL_DATA_CACHE_FLUSH);
422 }
423
424 static void
425 _iris_emit_lri(struct iris_batch *batch, uint32_t reg, uint32_t val)
426 {
427 iris_emit_cmd(batch, GENX(MI_LOAD_REGISTER_IMM), lri) {
428 lri.RegisterOffset = reg;
429 lri.DataDWord = val;
430 }
431 }
432 #define iris_emit_lri(b, r, v) _iris_emit_lri(b, GENX(r##_num), v)
433
434 static void
435 _iris_emit_lrr(struct iris_batch *batch, uint32_t dst, uint32_t src)
436 {
437 iris_emit_cmd(batch, GENX(MI_LOAD_REGISTER_REG), lrr) {
438 lrr.SourceRegisterAddress = src;
439 lrr.DestinationRegisterAddress = dst;
440 }
441 }
442
443 static void
444 emit_pipeline_select(struct iris_batch *batch, uint32_t pipeline)
445 {
446 #if GEN_GEN >= 8 && GEN_GEN < 10
447 /* From the Broadwell PRM, Volume 2a: Instructions, PIPELINE_SELECT:
448 *
449 * Software must clear the COLOR_CALC_STATE Valid field in
450 * 3DSTATE_CC_STATE_POINTERS command prior to send a PIPELINE_SELECT
451 * with Pipeline Select set to GPGPU.
452 *
453 * The internal hardware docs recommend the same workaround for Gen9
454 * hardware too.
455 */
456 if (pipeline == GPGPU)
457 iris_emit_cmd(batch, GENX(3DSTATE_CC_STATE_POINTERS), t);
458 #endif
459
460
461 /* From "BXML » GT » MI » vol1a GPU Overview » [Instruction]
462 * PIPELINE_SELECT [DevBWR+]":
463 *
464 * "Project: DEVSNB+
465 *
466 * Software must ensure all the write caches are flushed through a
467 * stalling PIPE_CONTROL command followed by another PIPE_CONTROL
468 * command to invalidate read only caches prior to programming
469 * MI_PIPELINE_SELECT command to change the Pipeline Select Mode."
470 */
471 iris_emit_pipe_control_flush(batch,
472 "workaround: PIPELINE_SELECT flushes (1/2)",
473 PIPE_CONTROL_RENDER_TARGET_FLUSH |
474 PIPE_CONTROL_DEPTH_CACHE_FLUSH |
475 PIPE_CONTROL_DATA_CACHE_FLUSH |
476 PIPE_CONTROL_CS_STALL);
477
478 iris_emit_pipe_control_flush(batch,
479 "workaround: PIPELINE_SELECT flushes (2/2)",
480 PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE |
481 PIPE_CONTROL_CONST_CACHE_INVALIDATE |
482 PIPE_CONTROL_STATE_CACHE_INVALIDATE |
483 PIPE_CONTROL_INSTRUCTION_INVALIDATE);
484
485 iris_emit_cmd(batch, GENX(PIPELINE_SELECT), sel) {
486 #if GEN_GEN >= 9
487 sel.MaskBits = 3;
488 #endif
489 sel.PipelineSelection = pipeline;
490 }
491 }
492
493 UNUSED static void
494 init_glk_barrier_mode(struct iris_batch *batch, uint32_t value)
495 {
496 #if GEN_GEN == 9
497 /* Project: DevGLK
498 *
499 * "This chicken bit works around a hardware issue with barrier
500 * logic encountered when switching between GPGPU and 3D pipelines.
501 * To workaround the issue, this mode bit should be set after a
502 * pipeline is selected."
503 */
504 uint32_t reg_val;
505 iris_pack_state(GENX(SLICE_COMMON_ECO_CHICKEN1), &reg_val, reg) {
506 reg.GLKBarrierMode = value;
507 reg.GLKBarrierModeMask = 1;
508 }
509 iris_emit_lri(batch, SLICE_COMMON_ECO_CHICKEN1, reg_val);
510 #endif
511 }
512
513 static void
514 init_state_base_address(struct iris_batch *batch)
515 {
516 flush_for_state_base_change(batch);
517
518 /* We program most base addresses once at context initialization time.
519 * Each base address points at a 4GB memory zone, and never needs to
520 * change. See iris_bufmgr.h for a description of the memory zones.
521 *
522 * The one exception is Surface State Base Address, which needs to be
523 * updated occasionally. See iris_binder.c for the details there.
524 */
525 iris_emit_cmd(batch, GENX(STATE_BASE_ADDRESS), sba) {
526 sba.GeneralStateMOCS = MOCS_WB;
527 sba.StatelessDataPortAccessMOCS = MOCS_WB;
528 sba.DynamicStateMOCS = MOCS_WB;
529 sba.IndirectObjectMOCS = MOCS_WB;
530 sba.InstructionMOCS = MOCS_WB;
531
532 sba.GeneralStateBaseAddressModifyEnable = true;
533 sba.DynamicStateBaseAddressModifyEnable = true;
534 sba.IndirectObjectBaseAddressModifyEnable = true;
535 sba.InstructionBaseAddressModifyEnable = true;
536 sba.GeneralStateBufferSizeModifyEnable = true;
537 sba.DynamicStateBufferSizeModifyEnable = true;
538 #if (GEN_GEN >= 9)
539 sba.BindlessSurfaceStateBaseAddressModifyEnable = true;
540 sba.BindlessSurfaceStateMOCS = MOCS_WB;
541 #endif
542 sba.IndirectObjectBufferSizeModifyEnable = true;
543 sba.InstructionBuffersizeModifyEnable = true;
544
545 sba.InstructionBaseAddress = ro_bo(NULL, IRIS_MEMZONE_SHADER_START);
546 sba.DynamicStateBaseAddress = ro_bo(NULL, IRIS_MEMZONE_DYNAMIC_START);
547
548 sba.GeneralStateBufferSize = 0xfffff;
549 sba.IndirectObjectBufferSize = 0xfffff;
550 sba.InstructionBufferSize = 0xfffff;
551 sba.DynamicStateBufferSize = 0xfffff;
552 }
553 }
554
555 static void
556 iris_emit_l3_config(struct iris_batch *batch, const struct gen_l3_config *cfg,
557 bool has_slm, bool wants_dc_cache)
558 {
559 uint32_t reg_val;
560 iris_pack_state(GENX(L3CNTLREG), &reg_val, reg) {
561 reg.SLMEnable = has_slm;
562 #if GEN_GEN == 11
563 /* WA_1406697149: Bit 9 "Error Detection Behavior Control" must be set
564 * in L3CNTLREG register. The default setting of the bit is not the
565 * desirable behavior.
566 */
567 reg.ErrorDetectionBehaviorControl = true;
568 reg.UseFullWays = true;
569 #endif
570 reg.URBAllocation = cfg->n[GEN_L3P_URB];
571 reg.ROAllocation = cfg->n[GEN_L3P_RO];
572 reg.DCAllocation = cfg->n[GEN_L3P_DC];
573 reg.AllAllocation = cfg->n[GEN_L3P_ALL];
574 }
575 iris_emit_lri(batch, L3CNTLREG, reg_val);
576 }
577
578 static void
579 iris_emit_default_l3_config(struct iris_batch *batch,
580 const struct gen_device_info *devinfo,
581 bool compute)
582 {
583 bool wants_dc_cache = true;
584 bool has_slm = compute;
585 const struct gen_l3_weights w =
586 gen_get_default_l3_weights(devinfo, wants_dc_cache, has_slm);
587 const struct gen_l3_config *cfg = gen_get_l3_config(devinfo, w);
588 iris_emit_l3_config(batch, cfg, has_slm, wants_dc_cache);
589 }
590
591 #if GEN_GEN == 9 || GEN_GEN == 10
592 static void
593 iris_enable_obj_preemption(struct iris_batch *batch, bool enable)
594 {
595 uint32_t reg_val;
596
597 /* A fixed function pipe flush is required before modifying this field */
598 iris_emit_end_of_pipe_sync(batch, enable ? "enable preemption"
599 : "disable preemption",
600 PIPE_CONTROL_RENDER_TARGET_FLUSH);
601
602 /* enable object level preemption */
603 iris_pack_state(GENX(CS_CHICKEN1), &reg_val, reg) {
604 reg.ReplayMode = enable;
605 reg.ReplayModeMask = true;
606 }
607 iris_emit_lri(batch, CS_CHICKEN1, reg_val);
608 }
609 #endif
610
611 /**
612 * Upload the initial GPU state for a render context.
613 *
614 * This sets some invariant state that needs to be programmed a particular
615 * way, but we never actually change.
616 */
617 static void
618 iris_init_render_context(struct iris_screen *screen,
619 struct iris_batch *batch,
620 struct iris_vtable *vtbl,
621 struct pipe_debug_callback *dbg)
622 {
623 UNUSED const struct gen_device_info *devinfo = &screen->devinfo;
624 uint32_t reg_val;
625
626 emit_pipeline_select(batch, _3D);
627
628 iris_emit_default_l3_config(batch, devinfo, false);
629
630 init_state_base_address(batch);
631
632 #if GEN_GEN >= 9
633 iris_pack_state(GENX(CS_DEBUG_MODE2), &reg_val, reg) {
634 reg.CONSTANT_BUFFERAddressOffsetDisable = true;
635 reg.CONSTANT_BUFFERAddressOffsetDisableMask = true;
636 }
637 iris_emit_lri(batch, CS_DEBUG_MODE2, reg_val);
638 #else
639 iris_pack_state(GENX(INSTPM), &reg_val, reg) {
640 reg.CONSTANT_BUFFERAddressOffsetDisable = true;
641 reg.CONSTANT_BUFFERAddressOffsetDisableMask = true;
642 }
643 iris_emit_lri(batch, INSTPM, reg_val);
644 #endif
645
646 #if GEN_GEN == 9
647 iris_pack_state(GENX(CACHE_MODE_1), &reg_val, reg) {
648 reg.FloatBlendOptimizationEnable = true;
649 reg.FloatBlendOptimizationEnableMask = true;
650 reg.PartialResolveDisableInVC = true;
651 reg.PartialResolveDisableInVCMask = true;
652 }
653 iris_emit_lri(batch, CACHE_MODE_1, reg_val);
654
655 if (devinfo->is_geminilake)
656 init_glk_barrier_mode(batch, GLK_BARRIER_MODE_3D_HULL);
657 #endif
658
659 #if GEN_GEN == 11
660 iris_pack_state(GENX(SAMPLER_MODE), &reg_val, reg) {
661 reg.HeaderlessMessageforPreemptableContexts = 1;
662 reg.HeaderlessMessageforPreemptableContextsMask = 1;
663 }
664 iris_emit_lri(batch, SAMPLER_MODE, reg_val);
665
666 /* Bit 1 must be set in HALF_SLICE_CHICKEN7. */
667 iris_pack_state(GENX(HALF_SLICE_CHICKEN7), &reg_val, reg) {
668 reg.EnabledTexelOffsetPrecisionFix = 1;
669 reg.EnabledTexelOffsetPrecisionFixMask = 1;
670 }
671 iris_emit_lri(batch, HALF_SLICE_CHICKEN7, reg_val);
672
673 iris_pack_state(GENX(SLICE_COMMON_ECO_CHICKEN1), &reg_val, reg) {
674 reg.StateCacheRedirectToCSSectionEnable = true;
675 reg.StateCacheRedirectToCSSectionEnableMask = true;
676 }
677 iris_emit_lri(batch, SLICE_COMMON_ECO_CHICKEN1, reg_val);
678
679 /* Hardware specification recommends disabling repacking for the
680 * compatibility with decompression mechanism in display controller.
681 */
682 if (devinfo->disable_ccs_repack) {
683 iris_pack_state(GENX(CACHE_MODE_0), &reg_val, reg) {
684 reg.DisableRepackingforCompression = true;
685 reg.DisableRepackingforCompressionMask = true;
686 }
687 iris_emit_lri(batch, CACHE_MODE_0, reg_val);
688 }
689
690 // XXX: 3D_MODE?
691 #endif
692
693 /* 3DSTATE_DRAWING_RECTANGLE is non-pipelined, so we want to avoid
694 * changing it dynamically. We set it to the maximum size here, and
695 * instead include the render target dimensions in the viewport, so
696 * viewport extents clipping takes care of pruning stray geometry.
697 */
698 iris_emit_cmd(batch, GENX(3DSTATE_DRAWING_RECTANGLE), rect) {
699 rect.ClippedDrawingRectangleXMax = UINT16_MAX;
700 rect.ClippedDrawingRectangleYMax = UINT16_MAX;
701 }
702
703 /* Set the initial MSAA sample positions. */
704 iris_emit_cmd(batch, GENX(3DSTATE_SAMPLE_PATTERN), pat) {
705 GEN_SAMPLE_POS_1X(pat._1xSample);
706 GEN_SAMPLE_POS_2X(pat._2xSample);
707 GEN_SAMPLE_POS_4X(pat._4xSample);
708 GEN_SAMPLE_POS_8X(pat._8xSample);
709 #if GEN_GEN >= 9
710 GEN_SAMPLE_POS_16X(pat._16xSample);
711 #endif
712 }
713
714 /* Use the legacy AA line coverage computation. */
715 iris_emit_cmd(batch, GENX(3DSTATE_AA_LINE_PARAMETERS), foo);
716
717 /* Disable chromakeying (it's for media) */
718 iris_emit_cmd(batch, GENX(3DSTATE_WM_CHROMAKEY), foo);
719
720 /* We want regular rendering, not special HiZ operations. */
721 iris_emit_cmd(batch, GENX(3DSTATE_WM_HZ_OP), foo);
722
723 /* No polygon stippling offsets are necessary. */
724 /* TODO: may need to set an offset for origin-UL framebuffers */
725 iris_emit_cmd(batch, GENX(3DSTATE_POLY_STIPPLE_OFFSET), foo);
726
727 /* Set a static partitioning of the push constant area. */
728 /* TODO: this may be a bad idea...could starve the push ringbuffers... */
729 for (int i = 0; i <= MESA_SHADER_FRAGMENT; i++) {
730 iris_emit_cmd(batch, GENX(3DSTATE_PUSH_CONSTANT_ALLOC_VS), alloc) {
731 alloc._3DCommandSubOpcode = 18 + i;
732 alloc.ConstantBufferOffset = 6 * i;
733 alloc.ConstantBufferSize = i == MESA_SHADER_FRAGMENT ? 8 : 6;
734 }
735 }
736
737 #if GEN_GEN == 10
738 /* Gen11+ is enabled for us by the kernel. */
739 iris_enable_obj_preemption(batch, true);
740 #endif
741 }
742
743 static void
744 iris_init_compute_context(struct iris_screen *screen,
745 struct iris_batch *batch,
746 struct iris_vtable *vtbl,
747 struct pipe_debug_callback *dbg)
748 {
749 UNUSED const struct gen_device_info *devinfo = &screen->devinfo;
750
751 emit_pipeline_select(batch, GPGPU);
752
753 iris_emit_default_l3_config(batch, devinfo, true);
754
755 init_state_base_address(batch);
756
757 #if GEN_GEN == 9
758 if (devinfo->is_geminilake)
759 init_glk_barrier_mode(batch, GLK_BARRIER_MODE_GPGPU);
760 #endif
761 }
762
763 struct iris_vertex_buffer_state {
764 /** The VERTEX_BUFFER_STATE hardware structure. */
765 uint32_t state[GENX(VERTEX_BUFFER_STATE_length)];
766
767 /** The resource to source vertex data from. */
768 struct pipe_resource *resource;
769 };
770
771 struct iris_depth_buffer_state {
772 /* Depth/HiZ/Stencil related hardware packets. */
773 uint32_t packets[GENX(3DSTATE_DEPTH_BUFFER_length) +
774 GENX(3DSTATE_STENCIL_BUFFER_length) +
775 GENX(3DSTATE_HIER_DEPTH_BUFFER_length) +
776 GENX(3DSTATE_CLEAR_PARAMS_length)];
777 };
778
779 /**
780 * Generation-specific context state (ice->state.genx->...).
781 *
782 * Most state can go in iris_context directly, but these encode hardware
783 * packets which vary by generation.
784 */
785 struct iris_genx_state {
786 struct iris_vertex_buffer_state vertex_buffers[33];
787 uint32_t last_index_buffer[GENX(3DSTATE_INDEX_BUFFER_length)];
788
789 struct iris_depth_buffer_state depth_buffer;
790
791 uint32_t so_buffers[4 * GENX(3DSTATE_SO_BUFFER_length)];
792
793 #if GEN_GEN == 9
794 /* Is object level preemption enabled? */
795 bool object_preemption;
796 #endif
797
798 struct {
799 #if GEN_GEN == 8
800 struct brw_image_param image_param[PIPE_MAX_SHADER_IMAGES];
801 #endif
802 } shaders[MESA_SHADER_STAGES];
803 };
804
805 /**
806 * The pipe->set_blend_color() driver hook.
807 *
808 * This corresponds to our COLOR_CALC_STATE.
809 */
810 static void
811 iris_set_blend_color(struct pipe_context *ctx,
812 const struct pipe_blend_color *state)
813 {
814 struct iris_context *ice = (struct iris_context *) ctx;
815
816 /* Our COLOR_CALC_STATE is exactly pipe_blend_color, so just memcpy */
817 memcpy(&ice->state.blend_color, state, sizeof(struct pipe_blend_color));
818 ice->state.dirty |= IRIS_DIRTY_COLOR_CALC_STATE;
819 }
820
821 /**
822 * Gallium CSO for blend state (see pipe_blend_state).
823 */
824 struct iris_blend_state {
825 /** Partial 3DSTATE_PS_BLEND */
826 uint32_t ps_blend[GENX(3DSTATE_PS_BLEND_length)];
827
828 /** Partial BLEND_STATE */
829 uint32_t blend_state[GENX(BLEND_STATE_length) +
830 BRW_MAX_DRAW_BUFFERS * GENX(BLEND_STATE_ENTRY_length)];
831
832 bool alpha_to_coverage; /* for shader key */
833
834 /** Bitfield of whether blending is enabled for RT[i] - for aux resolves */
835 uint8_t blend_enables;
836
837 /** Bitfield of whether color writes are enabled for RT[i] */
838 uint8_t color_write_enables;
839
840 /** Does RT[0] use dual color blending? */
841 bool dual_color_blending;
842 };
843
844 static enum pipe_blendfactor
845 fix_blendfactor(enum pipe_blendfactor f, bool alpha_to_one)
846 {
847 if (alpha_to_one) {
848 if (f == PIPE_BLENDFACTOR_SRC1_ALPHA)
849 return PIPE_BLENDFACTOR_ONE;
850
851 if (f == PIPE_BLENDFACTOR_INV_SRC1_ALPHA)
852 return PIPE_BLENDFACTOR_ZERO;
853 }
854
855 return f;
856 }
857
858 /**
859 * The pipe->create_blend_state() driver hook.
860 *
861 * Translates a pipe_blend_state into iris_blend_state.
862 */
863 static void *
864 iris_create_blend_state(struct pipe_context *ctx,
865 const struct pipe_blend_state *state)
866 {
867 struct iris_blend_state *cso = malloc(sizeof(struct iris_blend_state));
868 uint32_t *blend_entry = cso->blend_state + GENX(BLEND_STATE_length);
869
870 cso->blend_enables = 0;
871 cso->color_write_enables = 0;
872 STATIC_ASSERT(BRW_MAX_DRAW_BUFFERS <= 8);
873
874 cso->alpha_to_coverage = state->alpha_to_coverage;
875
876 bool indep_alpha_blend = false;
877
878 for (int i = 0; i < BRW_MAX_DRAW_BUFFERS; i++) {
879 const struct pipe_rt_blend_state *rt =
880 &state->rt[state->independent_blend_enable ? i : 0];
881
882 enum pipe_blendfactor src_rgb =
883 fix_blendfactor(rt->rgb_src_factor, state->alpha_to_one);
884 enum pipe_blendfactor src_alpha =
885 fix_blendfactor(rt->alpha_src_factor, state->alpha_to_one);
886 enum pipe_blendfactor dst_rgb =
887 fix_blendfactor(rt->rgb_dst_factor, state->alpha_to_one);
888 enum pipe_blendfactor dst_alpha =
889 fix_blendfactor(rt->alpha_dst_factor, state->alpha_to_one);
890
891 if (rt->rgb_func != rt->alpha_func ||
892 src_rgb != src_alpha || dst_rgb != dst_alpha)
893 indep_alpha_blend = true;
894
895 if (rt->blend_enable)
896 cso->blend_enables |= 1u << i;
897
898 if (rt->colormask)
899 cso->color_write_enables |= 1u << i;
900
901 iris_pack_state(GENX(BLEND_STATE_ENTRY), blend_entry, be) {
902 be.LogicOpEnable = state->logicop_enable;
903 be.LogicOpFunction = state->logicop_func;
904
905 be.PreBlendSourceOnlyClampEnable = false;
906 be.ColorClampRange = COLORCLAMP_RTFORMAT;
907 be.PreBlendColorClampEnable = true;
908 be.PostBlendColorClampEnable = true;
909
910 be.ColorBufferBlendEnable = rt->blend_enable;
911
912 be.ColorBlendFunction = rt->rgb_func;
913 be.AlphaBlendFunction = rt->alpha_func;
914 be.SourceBlendFactor = src_rgb;
915 be.SourceAlphaBlendFactor = src_alpha;
916 be.DestinationBlendFactor = dst_rgb;
917 be.DestinationAlphaBlendFactor = dst_alpha;
918
919 be.WriteDisableRed = !(rt->colormask & PIPE_MASK_R);
920 be.WriteDisableGreen = !(rt->colormask & PIPE_MASK_G);
921 be.WriteDisableBlue = !(rt->colormask & PIPE_MASK_B);
922 be.WriteDisableAlpha = !(rt->colormask & PIPE_MASK_A);
923 }
924 blend_entry += GENX(BLEND_STATE_ENTRY_length);
925 }
926
927 iris_pack_command(GENX(3DSTATE_PS_BLEND), cso->ps_blend, pb) {
928 /* pb.HasWriteableRT is filled in at draw time.
929 * pb.AlphaTestEnable is filled in at draw time.
930 *
931 * pb.ColorBufferBlendEnable is filled in at draw time so we can avoid
932 * setting it when dual color blending without an appropriate shader.
933 */
934
935 pb.AlphaToCoverageEnable = state->alpha_to_coverage;
936 pb.IndependentAlphaBlendEnable = indep_alpha_blend;
937
938 pb.SourceBlendFactor =
939 fix_blendfactor(state->rt[0].rgb_src_factor, state->alpha_to_one);
940 pb.SourceAlphaBlendFactor =
941 fix_blendfactor(state->rt[0].alpha_src_factor, state->alpha_to_one);
942 pb.DestinationBlendFactor =
943 fix_blendfactor(state->rt[0].rgb_dst_factor, state->alpha_to_one);
944 pb.DestinationAlphaBlendFactor =
945 fix_blendfactor(state->rt[0].alpha_dst_factor, state->alpha_to_one);
946 }
947
948 iris_pack_state(GENX(BLEND_STATE), cso->blend_state, bs) {
949 bs.AlphaToCoverageEnable = state->alpha_to_coverage;
950 bs.IndependentAlphaBlendEnable = indep_alpha_blend;
951 bs.AlphaToOneEnable = state->alpha_to_one;
952 bs.AlphaToCoverageDitherEnable = state->alpha_to_coverage;
953 bs.ColorDitherEnable = state->dither;
954 /* bl.AlphaTestEnable and bs.AlphaTestFunction are filled in later. */
955 }
956
957 cso->dual_color_blending = util_blend_state_is_dual(state, 0);
958
959 return cso;
960 }
961
962 /**
963 * The pipe->bind_blend_state() driver hook.
964 *
965 * Bind a blending CSO and flag related dirty bits.
966 */
967 static void
968 iris_bind_blend_state(struct pipe_context *ctx, void *state)
969 {
970 struct iris_context *ice = (struct iris_context *) ctx;
971 struct iris_blend_state *cso = state;
972
973 ice->state.cso_blend = cso;
974 ice->state.blend_enables = cso ? cso->blend_enables : 0;
975
976 ice->state.dirty |= IRIS_DIRTY_PS_BLEND;
977 ice->state.dirty |= IRIS_DIRTY_BLEND_STATE;
978 ice->state.dirty |= IRIS_DIRTY_RENDER_RESOLVES_AND_FLUSHES;
979 ice->state.dirty |= ice->state.dirty_for_nos[IRIS_NOS_BLEND];
980 }
981
982 /**
983 * Return true if the FS writes to any color outputs which are not disabled
984 * via color masking.
985 */
986 static bool
987 has_writeable_rt(const struct iris_blend_state *cso_blend,
988 const struct shader_info *fs_info)
989 {
990 if (!fs_info)
991 return false;
992
993 unsigned rt_outputs = fs_info->outputs_written >> FRAG_RESULT_DATA0;
994
995 if (fs_info->outputs_written & BITFIELD64_BIT(FRAG_RESULT_COLOR))
996 rt_outputs = (1 << BRW_MAX_DRAW_BUFFERS) - 1;
997
998 return cso_blend->color_write_enables & rt_outputs;
999 }
1000
1001 /**
1002 * Gallium CSO for depth, stencil, and alpha testing state.
1003 */
1004 struct iris_depth_stencil_alpha_state {
1005 /** Partial 3DSTATE_WM_DEPTH_STENCIL. */
1006 uint32_t wmds[GENX(3DSTATE_WM_DEPTH_STENCIL_length)];
1007
1008 /** Outbound to BLEND_STATE, 3DSTATE_PS_BLEND, COLOR_CALC_STATE. */
1009 struct pipe_alpha_state alpha;
1010
1011 /** Outbound to resolve and cache set tracking. */
1012 bool depth_writes_enabled;
1013 bool stencil_writes_enabled;
1014 };
1015
1016 /**
1017 * The pipe->create_depth_stencil_alpha_state() driver hook.
1018 *
1019 * We encode most of 3DSTATE_WM_DEPTH_STENCIL, and just save off the alpha
1020 * testing state since we need pieces of it in a variety of places.
1021 */
1022 static void *
1023 iris_create_zsa_state(struct pipe_context *ctx,
1024 const struct pipe_depth_stencil_alpha_state *state)
1025 {
1026 struct iris_depth_stencil_alpha_state *cso =
1027 malloc(sizeof(struct iris_depth_stencil_alpha_state));
1028
1029 bool two_sided_stencil = state->stencil[1].enabled;
1030
1031 cso->alpha = state->alpha;
1032 cso->depth_writes_enabled = state->depth.writemask;
1033 cso->stencil_writes_enabled =
1034 state->stencil[0].writemask != 0 ||
1035 (two_sided_stencil && state->stencil[1].writemask != 0);
1036
1037 /* The state tracker needs to optimize away EQUAL writes for us. */
1038 assert(!(state->depth.func == PIPE_FUNC_EQUAL && state->depth.writemask));
1039
1040 iris_pack_command(GENX(3DSTATE_WM_DEPTH_STENCIL), cso->wmds, wmds) {
1041 wmds.StencilFailOp = state->stencil[0].fail_op;
1042 wmds.StencilPassDepthFailOp = state->stencil[0].zfail_op;
1043 wmds.StencilPassDepthPassOp = state->stencil[0].zpass_op;
1044 wmds.StencilTestFunction =
1045 translate_compare_func(state->stencil[0].func);
1046 wmds.BackfaceStencilFailOp = state->stencil[1].fail_op;
1047 wmds.BackfaceStencilPassDepthFailOp = state->stencil[1].zfail_op;
1048 wmds.BackfaceStencilPassDepthPassOp = state->stencil[1].zpass_op;
1049 wmds.BackfaceStencilTestFunction =
1050 translate_compare_func(state->stencil[1].func);
1051 wmds.DepthTestFunction = translate_compare_func(state->depth.func);
1052 wmds.DoubleSidedStencilEnable = two_sided_stencil;
1053 wmds.StencilTestEnable = state->stencil[0].enabled;
1054 wmds.StencilBufferWriteEnable =
1055 state->stencil[0].writemask != 0 ||
1056 (two_sided_stencil && state->stencil[1].writemask != 0);
1057 wmds.DepthTestEnable = state->depth.enabled;
1058 wmds.DepthBufferWriteEnable = state->depth.writemask;
1059 wmds.StencilTestMask = state->stencil[0].valuemask;
1060 wmds.StencilWriteMask = state->stencil[0].writemask;
1061 wmds.BackfaceStencilTestMask = state->stencil[1].valuemask;
1062 wmds.BackfaceStencilWriteMask = state->stencil[1].writemask;
1063 /* wmds.[Backface]StencilReferenceValue are merged later */
1064 }
1065
1066 return cso;
1067 }
1068
1069 /**
1070 * The pipe->bind_depth_stencil_alpha_state() driver hook.
1071 *
1072 * Bind a depth/stencil/alpha CSO and flag related dirty bits.
1073 */
1074 static void
1075 iris_bind_zsa_state(struct pipe_context *ctx, void *state)
1076 {
1077 struct iris_context *ice = (struct iris_context *) ctx;
1078 struct iris_depth_stencil_alpha_state *old_cso = ice->state.cso_zsa;
1079 struct iris_depth_stencil_alpha_state *new_cso = state;
1080
1081 if (new_cso) {
1082 if (cso_changed(alpha.ref_value))
1083 ice->state.dirty |= IRIS_DIRTY_COLOR_CALC_STATE;
1084
1085 if (cso_changed(alpha.enabled))
1086 ice->state.dirty |= IRIS_DIRTY_PS_BLEND | IRIS_DIRTY_BLEND_STATE;
1087
1088 if (cso_changed(alpha.func))
1089 ice->state.dirty |= IRIS_DIRTY_BLEND_STATE;
1090
1091 if (cso_changed(depth_writes_enabled))
1092 ice->state.dirty |= IRIS_DIRTY_RENDER_RESOLVES_AND_FLUSHES;
1093
1094 ice->state.depth_writes_enabled = new_cso->depth_writes_enabled;
1095 ice->state.stencil_writes_enabled = new_cso->stencil_writes_enabled;
1096 }
1097
1098 ice->state.cso_zsa = new_cso;
1099 ice->state.dirty |= IRIS_DIRTY_CC_VIEWPORT;
1100 ice->state.dirty |= IRIS_DIRTY_WM_DEPTH_STENCIL;
1101 ice->state.dirty |= ice->state.dirty_for_nos[IRIS_NOS_DEPTH_STENCIL_ALPHA];
1102 }
1103
1104 /**
1105 * Gallium CSO for rasterizer state.
1106 */
1107 struct iris_rasterizer_state {
1108 uint32_t sf[GENX(3DSTATE_SF_length)];
1109 uint32_t clip[GENX(3DSTATE_CLIP_length)];
1110 uint32_t raster[GENX(3DSTATE_RASTER_length)];
1111 uint32_t wm[GENX(3DSTATE_WM_length)];
1112 uint32_t line_stipple[GENX(3DSTATE_LINE_STIPPLE_length)];
1113
1114 uint8_t num_clip_plane_consts;
1115 bool clip_halfz; /* for CC_VIEWPORT */
1116 bool depth_clip_near; /* for CC_VIEWPORT */
1117 bool depth_clip_far; /* for CC_VIEWPORT */
1118 bool flatshade; /* for shader state */
1119 bool flatshade_first; /* for stream output */
1120 bool clamp_fragment_color; /* for shader state */
1121 bool light_twoside; /* for shader state */
1122 bool rasterizer_discard; /* for 3DSTATE_STREAMOUT and 3DSTATE_CLIP */
1123 bool half_pixel_center; /* for 3DSTATE_MULTISAMPLE */
1124 bool line_stipple_enable;
1125 bool poly_stipple_enable;
1126 bool multisample;
1127 bool force_persample_interp;
1128 bool conservative_rasterization;
1129 bool fill_mode_point_or_line;
1130 enum pipe_sprite_coord_mode sprite_coord_mode; /* PIPE_SPRITE_* */
1131 uint16_t sprite_coord_enable;
1132 };
1133
1134 static float
1135 get_line_width(const struct pipe_rasterizer_state *state)
1136 {
1137 float line_width = state->line_width;
1138
1139 /* From the OpenGL 4.4 spec:
1140 *
1141 * "The actual width of non-antialiased lines is determined by rounding
1142 * the supplied width to the nearest integer, then clamping it to the
1143 * implementation-dependent maximum non-antialiased line width."
1144 */
1145 if (!state->multisample && !state->line_smooth)
1146 line_width = roundf(state->line_width);
1147
1148 if (!state->multisample && state->line_smooth && line_width < 1.5f) {
1149 /* For 1 pixel line thickness or less, the general anti-aliasing
1150 * algorithm gives up, and a garbage line is generated. Setting a
1151 * Line Width of 0.0 specifies the rasterization of the "thinnest"
1152 * (one-pixel-wide), non-antialiased lines.
1153 *
1154 * Lines rendered with zero Line Width are rasterized using the
1155 * "Grid Intersection Quantization" rules as specified by the
1156 * "Zero-Width (Cosmetic) Line Rasterization" section of the docs.
1157 */
1158 line_width = 0.0f;
1159 }
1160
1161 return line_width;
1162 }
1163
1164 /**
1165 * The pipe->create_rasterizer_state() driver hook.
1166 */
1167 static void *
1168 iris_create_rasterizer_state(struct pipe_context *ctx,
1169 const struct pipe_rasterizer_state *state)
1170 {
1171 struct iris_rasterizer_state *cso =
1172 malloc(sizeof(struct iris_rasterizer_state));
1173
1174 cso->multisample = state->multisample;
1175 cso->force_persample_interp = state->force_persample_interp;
1176 cso->clip_halfz = state->clip_halfz;
1177 cso->depth_clip_near = state->depth_clip_near;
1178 cso->depth_clip_far = state->depth_clip_far;
1179 cso->flatshade = state->flatshade;
1180 cso->flatshade_first = state->flatshade_first;
1181 cso->clamp_fragment_color = state->clamp_fragment_color;
1182 cso->light_twoside = state->light_twoside;
1183 cso->rasterizer_discard = state->rasterizer_discard;
1184 cso->half_pixel_center = state->half_pixel_center;
1185 cso->sprite_coord_mode = state->sprite_coord_mode;
1186 cso->sprite_coord_enable = state->sprite_coord_enable;
1187 cso->line_stipple_enable = state->line_stipple_enable;
1188 cso->poly_stipple_enable = state->poly_stipple_enable;
1189 cso->conservative_rasterization =
1190 state->conservative_raster_mode == PIPE_CONSERVATIVE_RASTER_POST_SNAP;
1191
1192 cso->fill_mode_point_or_line =
1193 state->fill_front == PIPE_POLYGON_MODE_LINE ||
1194 state->fill_front == PIPE_POLYGON_MODE_POINT ||
1195 state->fill_back == PIPE_POLYGON_MODE_LINE ||
1196 state->fill_back == PIPE_POLYGON_MODE_POINT;
1197
1198 if (state->clip_plane_enable != 0)
1199 cso->num_clip_plane_consts = util_logbase2(state->clip_plane_enable) + 1;
1200 else
1201 cso->num_clip_plane_consts = 0;
1202
1203 float line_width = get_line_width(state);
1204
1205 iris_pack_command(GENX(3DSTATE_SF), cso->sf, sf) {
1206 sf.StatisticsEnable = true;
1207 sf.ViewportTransformEnable = true;
1208 sf.AALineDistanceMode = AALINEDISTANCE_TRUE;
1209 sf.LineEndCapAntialiasingRegionWidth =
1210 state->line_smooth ? _10pixels : _05pixels;
1211 sf.LastPixelEnable = state->line_last_pixel;
1212 sf.LineWidth = line_width;
1213 sf.SmoothPointEnable = (state->point_smooth || state->multisample) &&
1214 !state->point_quad_rasterization;
1215 sf.PointWidthSource = state->point_size_per_vertex ? Vertex : State;
1216 sf.PointWidth = state->point_size;
1217
1218 if (state->flatshade_first) {
1219 sf.TriangleFanProvokingVertexSelect = 1;
1220 } else {
1221 sf.TriangleStripListProvokingVertexSelect = 2;
1222 sf.TriangleFanProvokingVertexSelect = 2;
1223 sf.LineStripListProvokingVertexSelect = 1;
1224 }
1225 }
1226
1227 iris_pack_command(GENX(3DSTATE_RASTER), cso->raster, rr) {
1228 rr.FrontWinding = state->front_ccw ? CounterClockwise : Clockwise;
1229 rr.CullMode = translate_cull_mode(state->cull_face);
1230 rr.FrontFaceFillMode = translate_fill_mode(state->fill_front);
1231 rr.BackFaceFillMode = translate_fill_mode(state->fill_back);
1232 rr.DXMultisampleRasterizationEnable = state->multisample;
1233 rr.GlobalDepthOffsetEnableSolid = state->offset_tri;
1234 rr.GlobalDepthOffsetEnableWireframe = state->offset_line;
1235 rr.GlobalDepthOffsetEnablePoint = state->offset_point;
1236 rr.GlobalDepthOffsetConstant = state->offset_units * 2;
1237 rr.GlobalDepthOffsetScale = state->offset_scale;
1238 rr.GlobalDepthOffsetClamp = state->offset_clamp;
1239 rr.SmoothPointEnable = state->point_smooth;
1240 rr.AntialiasingEnable = state->line_smooth;
1241 rr.ScissorRectangleEnable = state->scissor;
1242 #if GEN_GEN >= 9
1243 rr.ViewportZNearClipTestEnable = state->depth_clip_near;
1244 rr.ViewportZFarClipTestEnable = state->depth_clip_far;
1245 rr.ConservativeRasterizationEnable =
1246 cso->conservative_rasterization;
1247 #else
1248 rr.ViewportZClipTestEnable = (state->depth_clip_near || state->depth_clip_far);
1249 #endif
1250 }
1251
1252 iris_pack_command(GENX(3DSTATE_CLIP), cso->clip, cl) {
1253 /* cl.NonPerspectiveBarycentricEnable is filled in at draw time from
1254 * the FS program; cl.ForceZeroRTAIndexEnable is filled in from the FB.
1255 */
1256 cl.EarlyCullEnable = true;
1257 cl.UserClipDistanceClipTestEnableBitmask = state->clip_plane_enable;
1258 cl.ForceUserClipDistanceClipTestEnableBitmask = true;
1259 cl.APIMode = state->clip_halfz ? APIMODE_D3D : APIMODE_OGL;
1260 cl.GuardbandClipTestEnable = true;
1261 cl.ClipEnable = true;
1262 cl.MinimumPointWidth = 0.125;
1263 cl.MaximumPointWidth = 255.875;
1264
1265 if (state->flatshade_first) {
1266 cl.TriangleFanProvokingVertexSelect = 1;
1267 } else {
1268 cl.TriangleStripListProvokingVertexSelect = 2;
1269 cl.TriangleFanProvokingVertexSelect = 2;
1270 cl.LineStripListProvokingVertexSelect = 1;
1271 }
1272 }
1273
1274 iris_pack_command(GENX(3DSTATE_WM), cso->wm, wm) {
1275 /* wm.BarycentricInterpolationMode and wm.EarlyDepthStencilControl are
1276 * filled in at draw time from the FS program.
1277 */
1278 wm.LineAntialiasingRegionWidth = _10pixels;
1279 wm.LineEndCapAntialiasingRegionWidth = _05pixels;
1280 wm.PointRasterizationRule = RASTRULE_UPPER_RIGHT;
1281 wm.LineStippleEnable = state->line_stipple_enable;
1282 wm.PolygonStippleEnable = state->poly_stipple_enable;
1283 }
1284
1285 /* Remap from 0..255 back to 1..256 */
1286 const unsigned line_stipple_factor = state->line_stipple_factor + 1;
1287
1288 iris_pack_command(GENX(3DSTATE_LINE_STIPPLE), cso->line_stipple, line) {
1289 line.LineStipplePattern = state->line_stipple_pattern;
1290 line.LineStippleInverseRepeatCount = 1.0f / line_stipple_factor;
1291 line.LineStippleRepeatCount = line_stipple_factor;
1292 }
1293
1294 return cso;
1295 }
1296
1297 /**
1298 * The pipe->bind_rasterizer_state() driver hook.
1299 *
1300 * Bind a rasterizer CSO and flag related dirty bits.
1301 */
1302 static void
1303 iris_bind_rasterizer_state(struct pipe_context *ctx, void *state)
1304 {
1305 struct iris_context *ice = (struct iris_context *) ctx;
1306 struct iris_rasterizer_state *old_cso = ice->state.cso_rast;
1307 struct iris_rasterizer_state *new_cso = state;
1308
1309 if (new_cso) {
1310 /* Try to avoid re-emitting 3DSTATE_LINE_STIPPLE, it's non-pipelined */
1311 if (cso_changed_memcmp(line_stipple))
1312 ice->state.dirty |= IRIS_DIRTY_LINE_STIPPLE;
1313
1314 if (cso_changed(half_pixel_center))
1315 ice->state.dirty |= IRIS_DIRTY_MULTISAMPLE;
1316
1317 if (cso_changed(line_stipple_enable) || cso_changed(poly_stipple_enable))
1318 ice->state.dirty |= IRIS_DIRTY_WM;
1319
1320 if (cso_changed(rasterizer_discard))
1321 ice->state.dirty |= IRIS_DIRTY_STREAMOUT | IRIS_DIRTY_CLIP;
1322
1323 if (cso_changed(flatshade_first))
1324 ice->state.dirty |= IRIS_DIRTY_STREAMOUT;
1325
1326 if (cso_changed(depth_clip_near) || cso_changed(depth_clip_far) ||
1327 cso_changed(clip_halfz))
1328 ice->state.dirty |= IRIS_DIRTY_CC_VIEWPORT;
1329
1330 if (cso_changed(sprite_coord_enable) ||
1331 cso_changed(sprite_coord_mode) ||
1332 cso_changed(light_twoside))
1333 ice->state.dirty |= IRIS_DIRTY_SBE;
1334
1335 if (cso_changed(conservative_rasterization))
1336 ice->state.dirty |= IRIS_DIRTY_FS;
1337 }
1338
1339 ice->state.cso_rast = new_cso;
1340 ice->state.dirty |= IRIS_DIRTY_RASTER;
1341 ice->state.dirty |= IRIS_DIRTY_CLIP;
1342 ice->state.dirty |= ice->state.dirty_for_nos[IRIS_NOS_RASTERIZER];
1343 }
1344
1345 /**
1346 * Return true if the given wrap mode requires the border color to exist.
1347 *
1348 * (We can skip uploading it if the sampler isn't going to use it.)
1349 */
1350 static bool
1351 wrap_mode_needs_border_color(unsigned wrap_mode)
1352 {
1353 return wrap_mode == TCM_CLAMP_BORDER || wrap_mode == TCM_HALF_BORDER;
1354 }
1355
1356 /**
1357 * Gallium CSO for sampler state.
1358 */
1359 struct iris_sampler_state {
1360 union pipe_color_union border_color;
1361 bool needs_border_color;
1362
1363 uint32_t sampler_state[GENX(SAMPLER_STATE_length)];
1364 };
1365
1366 /**
1367 * The pipe->create_sampler_state() driver hook.
1368 *
1369 * We fill out SAMPLER_STATE (except for the border color pointer), and
1370 * store that on the CPU. It doesn't make sense to upload it to a GPU
1371 * buffer object yet, because 3DSTATE_SAMPLER_STATE_POINTERS requires
1372 * all bound sampler states to be in contiguous memor.
1373 */
1374 static void *
1375 iris_create_sampler_state(struct pipe_context *ctx,
1376 const struct pipe_sampler_state *state)
1377 {
1378 struct iris_sampler_state *cso = CALLOC_STRUCT(iris_sampler_state);
1379
1380 if (!cso)
1381 return NULL;
1382
1383 STATIC_ASSERT(PIPE_TEX_FILTER_NEAREST == MAPFILTER_NEAREST);
1384 STATIC_ASSERT(PIPE_TEX_FILTER_LINEAR == MAPFILTER_LINEAR);
1385
1386 unsigned wrap_s = translate_wrap(state->wrap_s);
1387 unsigned wrap_t = translate_wrap(state->wrap_t);
1388 unsigned wrap_r = translate_wrap(state->wrap_r);
1389
1390 memcpy(&cso->border_color, &state->border_color, sizeof(cso->border_color));
1391
1392 cso->needs_border_color = wrap_mode_needs_border_color(wrap_s) ||
1393 wrap_mode_needs_border_color(wrap_t) ||
1394 wrap_mode_needs_border_color(wrap_r);
1395
1396 float min_lod = state->min_lod;
1397 unsigned mag_img_filter = state->mag_img_filter;
1398
1399 // XXX: explain this code ported from ilo...I don't get it at all...
1400 if (state->min_mip_filter == PIPE_TEX_MIPFILTER_NONE &&
1401 state->min_lod > 0.0f) {
1402 min_lod = 0.0f;
1403 mag_img_filter = state->min_img_filter;
1404 }
1405
1406 iris_pack_state(GENX(SAMPLER_STATE), cso->sampler_state, samp) {
1407 samp.TCXAddressControlMode = wrap_s;
1408 samp.TCYAddressControlMode = wrap_t;
1409 samp.TCZAddressControlMode = wrap_r;
1410 samp.CubeSurfaceControlMode = state->seamless_cube_map;
1411 samp.NonnormalizedCoordinateEnable = !state->normalized_coords;
1412 samp.MinModeFilter = state->min_img_filter;
1413 samp.MagModeFilter = mag_img_filter;
1414 samp.MipModeFilter = translate_mip_filter(state->min_mip_filter);
1415 samp.MaximumAnisotropy = RATIO21;
1416
1417 if (state->max_anisotropy >= 2) {
1418 if (state->min_img_filter == PIPE_TEX_FILTER_LINEAR) {
1419 samp.MinModeFilter = MAPFILTER_ANISOTROPIC;
1420 samp.AnisotropicAlgorithm = EWAApproximation;
1421 }
1422
1423 if (state->mag_img_filter == PIPE_TEX_FILTER_LINEAR)
1424 samp.MagModeFilter = MAPFILTER_ANISOTROPIC;
1425
1426 samp.MaximumAnisotropy =
1427 MIN2((state->max_anisotropy - 2) / 2, RATIO161);
1428 }
1429
1430 /* Set address rounding bits if not using nearest filtering. */
1431 if (state->min_img_filter != PIPE_TEX_FILTER_NEAREST) {
1432 samp.UAddressMinFilterRoundingEnable = true;
1433 samp.VAddressMinFilterRoundingEnable = true;
1434 samp.RAddressMinFilterRoundingEnable = true;
1435 }
1436
1437 if (state->mag_img_filter != PIPE_TEX_FILTER_NEAREST) {
1438 samp.UAddressMagFilterRoundingEnable = true;
1439 samp.VAddressMagFilterRoundingEnable = true;
1440 samp.RAddressMagFilterRoundingEnable = true;
1441 }
1442
1443 if (state->compare_mode == PIPE_TEX_COMPARE_R_TO_TEXTURE)
1444 samp.ShadowFunction = translate_shadow_func(state->compare_func);
1445
1446 const float hw_max_lod = GEN_GEN >= 7 ? 14 : 13;
1447
1448 samp.LODPreClampMode = CLAMP_MODE_OGL;
1449 samp.MinLOD = CLAMP(min_lod, 0, hw_max_lod);
1450 samp.MaxLOD = CLAMP(state->max_lod, 0, hw_max_lod);
1451 samp.TextureLODBias = CLAMP(state->lod_bias, -16, 15);
1452
1453 /* .BorderColorPointer is filled in by iris_bind_sampler_states. */
1454 }
1455
1456 return cso;
1457 }
1458
1459 /**
1460 * The pipe->bind_sampler_states() driver hook.
1461 */
1462 static void
1463 iris_bind_sampler_states(struct pipe_context *ctx,
1464 enum pipe_shader_type p_stage,
1465 unsigned start, unsigned count,
1466 void **states)
1467 {
1468 struct iris_context *ice = (struct iris_context *) ctx;
1469 gl_shader_stage stage = stage_from_pipe(p_stage);
1470 struct iris_shader_state *shs = &ice->state.shaders[stage];
1471
1472 assert(start + count <= IRIS_MAX_TEXTURE_SAMPLERS);
1473
1474 for (int i = 0; i < count; i++) {
1475 shs->samplers[start + i] = states[i];
1476 }
1477
1478 ice->state.dirty |= IRIS_DIRTY_SAMPLER_STATES_VS << stage;
1479 }
1480
1481 /**
1482 * Upload the sampler states into a contiguous area of GPU memory, for
1483 * for 3DSTATE_SAMPLER_STATE_POINTERS_*.
1484 *
1485 * Also fill out the border color state pointers.
1486 */
1487 static void
1488 iris_upload_sampler_states(struct iris_context *ice, gl_shader_stage stage)
1489 {
1490 struct iris_shader_state *shs = &ice->state.shaders[stage];
1491 const struct shader_info *info = iris_get_shader_info(ice, stage);
1492
1493 /* We assume the state tracker will call pipe->bind_sampler_states()
1494 * if the program's number of textures changes.
1495 */
1496 unsigned count = info ? util_last_bit(info->textures_used) : 0;
1497
1498 if (!count)
1499 return;
1500
1501 /* Assemble the SAMPLER_STATEs into a contiguous table that lives
1502 * in the dynamic state memory zone, so we can point to it via the
1503 * 3DSTATE_SAMPLER_STATE_POINTERS_* commands.
1504 */
1505 unsigned size = count * 4 * GENX(SAMPLER_STATE_length);
1506 uint32_t *map =
1507 upload_state(ice->state.dynamic_uploader, &shs->sampler_table, size, 32);
1508 if (unlikely(!map))
1509 return;
1510
1511 struct pipe_resource *res = shs->sampler_table.res;
1512 shs->sampler_table.offset +=
1513 iris_bo_offset_from_base_address(iris_resource_bo(res));
1514
1515 iris_record_state_size(ice->state.sizes, shs->sampler_table.offset, size);
1516
1517 /* Make sure all land in the same BO */
1518 iris_border_color_pool_reserve(ice, IRIS_MAX_TEXTURE_SAMPLERS);
1519
1520 ice->state.need_border_colors &= ~(1 << stage);
1521
1522 for (int i = 0; i < count; i++) {
1523 struct iris_sampler_state *state = shs->samplers[i];
1524 struct iris_sampler_view *tex = shs->textures[i];
1525
1526 if (!state) {
1527 memset(map, 0, 4 * GENX(SAMPLER_STATE_length));
1528 } else if (!state->needs_border_color) {
1529 memcpy(map, state->sampler_state, 4 * GENX(SAMPLER_STATE_length));
1530 } else {
1531 ice->state.need_border_colors |= 1 << stage;
1532
1533 /* We may need to swizzle the border color for format faking.
1534 * A/LA formats are faked as R/RG with 000R or R00G swizzles.
1535 * This means we need to move the border color's A channel into
1536 * the R or G channels so that those read swizzles will move it
1537 * back into A.
1538 */
1539 union pipe_color_union *color = &state->border_color;
1540 union pipe_color_union tmp;
1541 if (tex) {
1542 enum pipe_format internal_format = tex->res->internal_format;
1543
1544 if (util_format_is_alpha(internal_format)) {
1545 unsigned char swz[4] = {
1546 PIPE_SWIZZLE_W, PIPE_SWIZZLE_0,
1547 PIPE_SWIZZLE_0, PIPE_SWIZZLE_0
1548 };
1549 util_format_apply_color_swizzle(&tmp, color, swz, true);
1550 color = &tmp;
1551 } else if (util_format_is_luminance_alpha(internal_format) &&
1552 internal_format != PIPE_FORMAT_L8A8_SRGB) {
1553 unsigned char swz[4] = {
1554 PIPE_SWIZZLE_X, PIPE_SWIZZLE_W,
1555 PIPE_SWIZZLE_0, PIPE_SWIZZLE_0
1556 };
1557 util_format_apply_color_swizzle(&tmp, color, swz, true);
1558 color = &tmp;
1559 }
1560 }
1561
1562 /* Stream out the border color and merge the pointer. */
1563 uint32_t offset = iris_upload_border_color(ice, color);
1564
1565 uint32_t dynamic[GENX(SAMPLER_STATE_length)];
1566 iris_pack_state(GENX(SAMPLER_STATE), dynamic, dyns) {
1567 dyns.BorderColorPointer = offset;
1568 }
1569
1570 for (uint32_t j = 0; j < GENX(SAMPLER_STATE_length); j++)
1571 map[j] = state->sampler_state[j] | dynamic[j];
1572 }
1573
1574 map += GENX(SAMPLER_STATE_length);
1575 }
1576 }
1577
1578 static enum isl_channel_select
1579 fmt_swizzle(const struct iris_format_info *fmt, enum pipe_swizzle swz)
1580 {
1581 switch (swz) {
1582 case PIPE_SWIZZLE_X: return fmt->swizzle.r;
1583 case PIPE_SWIZZLE_Y: return fmt->swizzle.g;
1584 case PIPE_SWIZZLE_Z: return fmt->swizzle.b;
1585 case PIPE_SWIZZLE_W: return fmt->swizzle.a;
1586 case PIPE_SWIZZLE_1: return SCS_ONE;
1587 case PIPE_SWIZZLE_0: return SCS_ZERO;
1588 default: unreachable("invalid swizzle");
1589 }
1590 }
1591
1592 static void
1593 fill_buffer_surface_state(struct isl_device *isl_dev,
1594 struct iris_resource *res,
1595 void *map,
1596 enum isl_format format,
1597 struct isl_swizzle swizzle,
1598 unsigned offset,
1599 unsigned size)
1600 {
1601 const struct isl_format_layout *fmtl = isl_format_get_layout(format);
1602 const unsigned cpp = format == ISL_FORMAT_RAW ? 1 : fmtl->bpb / 8;
1603
1604 /* The ARB_texture_buffer_specification says:
1605 *
1606 * "The number of texels in the buffer texture's texel array is given by
1607 *
1608 * floor(<buffer_size> / (<components> * sizeof(<base_type>)),
1609 *
1610 * where <buffer_size> is the size of the buffer object, in basic
1611 * machine units and <components> and <base_type> are the element count
1612 * and base data type for elements, as specified in Table X.1. The
1613 * number of texels in the texel array is then clamped to the
1614 * implementation-dependent limit MAX_TEXTURE_BUFFER_SIZE_ARB."
1615 *
1616 * We need to clamp the size in bytes to MAX_TEXTURE_BUFFER_SIZE * stride,
1617 * so that when ISL divides by stride to obtain the number of texels, that
1618 * texel count is clamped to MAX_TEXTURE_BUFFER_SIZE.
1619 */
1620 unsigned final_size =
1621 MIN3(size, res->bo->size - res->offset - offset,
1622 IRIS_MAX_TEXTURE_BUFFER_SIZE * cpp);
1623
1624 isl_buffer_fill_state(isl_dev, map,
1625 .address = res->bo->gtt_offset + res->offset + offset,
1626 .size_B = final_size,
1627 .format = format,
1628 .swizzle = swizzle,
1629 .stride_B = cpp,
1630 .mocs = mocs(res->bo));
1631 }
1632
1633 #define SURFACE_STATE_ALIGNMENT 64
1634
1635 /**
1636 * Allocate several contiguous SURFACE_STATE structures, one for each
1637 * supported auxiliary surface mode.
1638 */
1639 static void *
1640 alloc_surface_states(struct u_upload_mgr *mgr,
1641 struct iris_state_ref *ref,
1642 unsigned aux_usages)
1643 {
1644 const unsigned surf_size = 4 * GENX(RENDER_SURFACE_STATE_length);
1645
1646 /* If this changes, update this to explicitly align pointers */
1647 STATIC_ASSERT(surf_size == SURFACE_STATE_ALIGNMENT);
1648
1649 assert(aux_usages != 0);
1650
1651 void *map =
1652 upload_state(mgr, ref, util_bitcount(aux_usages) * surf_size,
1653 SURFACE_STATE_ALIGNMENT);
1654
1655 ref->offset += iris_bo_offset_from_base_address(iris_resource_bo(ref->res));
1656
1657 return map;
1658 }
1659
1660 static void
1661 fill_surface_state(struct isl_device *isl_dev,
1662 void *map,
1663 struct iris_resource *res,
1664 struct isl_view *view,
1665 unsigned aux_usage)
1666 {
1667 struct isl_surf_fill_state_info f = {
1668 .surf = &res->surf,
1669 .view = view,
1670 .mocs = mocs(res->bo),
1671 .address = res->bo->gtt_offset + res->offset,
1672 };
1673
1674 if (aux_usage != ISL_AUX_USAGE_NONE) {
1675 f.aux_surf = &res->aux.surf;
1676 f.aux_usage = aux_usage;
1677 f.aux_address = res->aux.bo->gtt_offset + res->aux.offset;
1678
1679 struct iris_bo *clear_bo = NULL;
1680 uint64_t clear_offset = 0;
1681 f.clear_color =
1682 iris_resource_get_clear_color(res, &clear_bo, &clear_offset);
1683 if (clear_bo) {
1684 f.clear_address = clear_bo->gtt_offset + clear_offset;
1685 f.use_clear_address = isl_dev->info->gen > 9;
1686 }
1687 }
1688
1689 isl_surf_fill_state_s(isl_dev, map, &f);
1690 }
1691
1692 /**
1693 * The pipe->create_sampler_view() driver hook.
1694 */
1695 static struct pipe_sampler_view *
1696 iris_create_sampler_view(struct pipe_context *ctx,
1697 struct pipe_resource *tex,
1698 const struct pipe_sampler_view *tmpl)
1699 {
1700 struct iris_context *ice = (struct iris_context *) ctx;
1701 struct iris_screen *screen = (struct iris_screen *)ctx->screen;
1702 const struct gen_device_info *devinfo = &screen->devinfo;
1703 struct iris_sampler_view *isv = calloc(1, sizeof(struct iris_sampler_view));
1704
1705 if (!isv)
1706 return NULL;
1707
1708 /* initialize base object */
1709 isv->base = *tmpl;
1710 isv->base.context = ctx;
1711 isv->base.texture = NULL;
1712 pipe_reference_init(&isv->base.reference, 1);
1713 pipe_resource_reference(&isv->base.texture, tex);
1714
1715 if (util_format_is_depth_or_stencil(tmpl->format)) {
1716 struct iris_resource *zres, *sres;
1717 const struct util_format_description *desc =
1718 util_format_description(tmpl->format);
1719
1720 iris_get_depth_stencil_resources(tex, &zres, &sres);
1721
1722 tex = util_format_has_depth(desc) ? &zres->base : &sres->base;
1723 }
1724
1725 isv->res = (struct iris_resource *) tex;
1726
1727 void *map = alloc_surface_states(ice->state.surface_uploader,
1728 &isv->surface_state,
1729 isv->res->aux.sampler_usages);
1730 if (!unlikely(map))
1731 return NULL;
1732
1733 isl_surf_usage_flags_t usage = ISL_SURF_USAGE_TEXTURE_BIT;
1734
1735 if (isv->base.target == PIPE_TEXTURE_CUBE ||
1736 isv->base.target == PIPE_TEXTURE_CUBE_ARRAY)
1737 usage |= ISL_SURF_USAGE_CUBE_BIT;
1738
1739 const struct iris_format_info fmt =
1740 iris_format_for_usage(devinfo, tmpl->format, usage);
1741
1742 isv->clear_color = isv->res->aux.clear_color;
1743
1744 isv->view = (struct isl_view) {
1745 .format = fmt.fmt,
1746 .swizzle = (struct isl_swizzle) {
1747 .r = fmt_swizzle(&fmt, tmpl->swizzle_r),
1748 .g = fmt_swizzle(&fmt, tmpl->swizzle_g),
1749 .b = fmt_swizzle(&fmt, tmpl->swizzle_b),
1750 .a = fmt_swizzle(&fmt, tmpl->swizzle_a),
1751 },
1752 .usage = usage,
1753 };
1754
1755 /* Fill out SURFACE_STATE for this view. */
1756 if (tmpl->target != PIPE_BUFFER) {
1757 isv->view.base_level = tmpl->u.tex.first_level;
1758 isv->view.levels = tmpl->u.tex.last_level - tmpl->u.tex.first_level + 1;
1759 // XXX: do I need to port f9fd0cf4790cb2a530e75d1a2206dbb9d8af7cb2?
1760 isv->view.base_array_layer = tmpl->u.tex.first_layer;
1761 isv->view.array_len =
1762 tmpl->u.tex.last_layer - tmpl->u.tex.first_layer + 1;
1763
1764 unsigned aux_modes = isv->res->aux.sampler_usages;
1765 while (aux_modes) {
1766 enum isl_aux_usage aux_usage = u_bit_scan(&aux_modes);
1767
1768 /* If we have a multisampled depth buffer, do not create a sampler
1769 * surface state with HiZ.
1770 */
1771 fill_surface_state(&screen->isl_dev, map, isv->res, &isv->view,
1772 aux_usage);
1773
1774 map += SURFACE_STATE_ALIGNMENT;
1775 }
1776 } else {
1777 fill_buffer_surface_state(&screen->isl_dev, isv->res, map,
1778 isv->view.format, isv->view.swizzle,
1779 tmpl->u.buf.offset, tmpl->u.buf.size);
1780 }
1781
1782 return &isv->base;
1783 }
1784
1785 static void
1786 iris_sampler_view_destroy(struct pipe_context *ctx,
1787 struct pipe_sampler_view *state)
1788 {
1789 struct iris_sampler_view *isv = (void *) state;
1790 pipe_resource_reference(&state->texture, NULL);
1791 pipe_resource_reference(&isv->surface_state.res, NULL);
1792 free(isv);
1793 }
1794
1795 /**
1796 * The pipe->create_surface() driver hook.
1797 *
1798 * In Gallium nomenclature, "surfaces" are a view of a resource that
1799 * can be bound as a render target or depth/stencil buffer.
1800 */
1801 static struct pipe_surface *
1802 iris_create_surface(struct pipe_context *ctx,
1803 struct pipe_resource *tex,
1804 const struct pipe_surface *tmpl)
1805 {
1806 struct iris_context *ice = (struct iris_context *) ctx;
1807 struct iris_screen *screen = (struct iris_screen *)ctx->screen;
1808 const struct gen_device_info *devinfo = &screen->devinfo;
1809
1810 isl_surf_usage_flags_t usage = 0;
1811 if (tmpl->writable)
1812 usage = ISL_SURF_USAGE_STORAGE_BIT;
1813 else if (util_format_is_depth_or_stencil(tmpl->format))
1814 usage = ISL_SURF_USAGE_DEPTH_BIT;
1815 else
1816 usage = ISL_SURF_USAGE_RENDER_TARGET_BIT;
1817
1818 const struct iris_format_info fmt =
1819 iris_format_for_usage(devinfo, tmpl->format, usage);
1820
1821 if ((usage & ISL_SURF_USAGE_RENDER_TARGET_BIT) &&
1822 !isl_format_supports_rendering(devinfo, fmt.fmt)) {
1823 /* Framebuffer validation will reject this invalid case, but it
1824 * hasn't had the opportunity yet. In the meantime, we need to
1825 * avoid hitting ISL asserts about unsupported formats below.
1826 */
1827 return NULL;
1828 }
1829
1830 struct iris_surface *surf = calloc(1, sizeof(struct iris_surface));
1831 struct pipe_surface *psurf = &surf->base;
1832 struct iris_resource *res = (struct iris_resource *) tex;
1833
1834 if (!surf)
1835 return NULL;
1836
1837 pipe_reference_init(&psurf->reference, 1);
1838 pipe_resource_reference(&psurf->texture, tex);
1839 psurf->context = ctx;
1840 psurf->format = tmpl->format;
1841 psurf->width = tex->width0;
1842 psurf->height = tex->height0;
1843 psurf->texture = tex;
1844 psurf->u.tex.first_layer = tmpl->u.tex.first_layer;
1845 psurf->u.tex.last_layer = tmpl->u.tex.last_layer;
1846 psurf->u.tex.level = tmpl->u.tex.level;
1847
1848 struct isl_view *view = &surf->view;
1849 *view = (struct isl_view) {
1850 .format = fmt.fmt,
1851 .base_level = tmpl->u.tex.level,
1852 .levels = 1,
1853 .base_array_layer = tmpl->u.tex.first_layer,
1854 .array_len = tmpl->u.tex.last_layer - tmpl->u.tex.first_layer + 1,
1855 .swizzle = ISL_SWIZZLE_IDENTITY,
1856 .usage = usage,
1857 };
1858
1859 surf->clear_color = res->aux.clear_color;
1860
1861 /* Bail early for depth/stencil - we don't want SURFACE_STATE for them. */
1862 if (res->surf.usage & (ISL_SURF_USAGE_DEPTH_BIT |
1863 ISL_SURF_USAGE_STENCIL_BIT))
1864 return psurf;
1865
1866
1867 void *map = alloc_surface_states(ice->state.surface_uploader,
1868 &surf->surface_state,
1869 res->aux.possible_usages);
1870 if (!unlikely(map))
1871 return NULL;
1872
1873 if (!isl_format_is_compressed(res->surf.format)) {
1874 /* This is a normal surface. Fill out a SURFACE_STATE for each possible
1875 * auxiliary surface mode and return the pipe_surface.
1876 */
1877 unsigned aux_modes = res->aux.possible_usages;
1878 while (aux_modes) {
1879 enum isl_aux_usage aux_usage = u_bit_scan(&aux_modes);
1880
1881 fill_surface_state(&screen->isl_dev, map, res, view, aux_usage);
1882
1883 map += SURFACE_STATE_ALIGNMENT;
1884 }
1885
1886 return psurf;
1887 }
1888
1889 /* The resource has a compressed format, which is not renderable, but we
1890 * have a renderable view format. We must be attempting to upload blocks
1891 * of compressed data via an uncompressed view.
1892 *
1893 * In this case, we can assume there are no auxiliary buffers, a single
1894 * miplevel, and that the resource is single-sampled. Gallium may try
1895 * and create an uncompressed view with multiple layers, however.
1896 */
1897 assert(!isl_format_is_compressed(fmt.fmt));
1898 assert(res->aux.possible_usages == 1 << ISL_AUX_USAGE_NONE);
1899 assert(res->surf.samples == 1);
1900 assert(view->levels == 1);
1901
1902 struct isl_surf isl_surf;
1903 uint32_t offset_B = 0, tile_x_sa = 0, tile_y_sa = 0;
1904
1905 if (view->base_level > 0) {
1906 /* We can't rely on the hardware's miplevel selection with such
1907 * a substantial lie about the format, so we select a single image
1908 * using the Tile X/Y Offset fields. In this case, we can't handle
1909 * multiple array slices.
1910 *
1911 * On Broadwell, HALIGN and VALIGN are specified in pixels and are
1912 * hard-coded to align to exactly the block size of the compressed
1913 * texture. This means that, when reinterpreted as a non-compressed
1914 * texture, the tile offsets may be anything and we can't rely on
1915 * X/Y Offset.
1916 *
1917 * Return NULL to force the state tracker to take fallback paths.
1918 */
1919 if (view->array_len > 1 || GEN_GEN == 8)
1920 return NULL;
1921
1922 const bool is_3d = res->surf.dim == ISL_SURF_DIM_3D;
1923 isl_surf_get_image_surf(&screen->isl_dev, &res->surf,
1924 view->base_level,
1925 is_3d ? 0 : view->base_array_layer,
1926 is_3d ? view->base_array_layer : 0,
1927 &isl_surf,
1928 &offset_B, &tile_x_sa, &tile_y_sa);
1929
1930 /* We use address and tile offsets to access a single level/layer
1931 * as a subimage, so reset level/layer so it doesn't offset again.
1932 */
1933 view->base_array_layer = 0;
1934 view->base_level = 0;
1935 } else {
1936 /* Level 0 doesn't require tile offsets, and the hardware can find
1937 * array slices using QPitch even with the format override, so we
1938 * can allow layers in this case. Copy the original ISL surface.
1939 */
1940 memcpy(&isl_surf, &res->surf, sizeof(isl_surf));
1941 }
1942
1943 /* Scale down the image dimensions by the block size. */
1944 const struct isl_format_layout *fmtl =
1945 isl_format_get_layout(res->surf.format);
1946 isl_surf.format = fmt.fmt;
1947 isl_surf.logical_level0_px = isl_surf_get_logical_level0_el(&isl_surf);
1948 isl_surf.phys_level0_sa = isl_surf_get_phys_level0_el(&isl_surf);
1949 tile_x_sa /= fmtl->bw;
1950 tile_y_sa /= fmtl->bh;
1951
1952 psurf->width = isl_surf.logical_level0_px.width;
1953 psurf->height = isl_surf.logical_level0_px.height;
1954
1955 struct isl_surf_fill_state_info f = {
1956 .surf = &isl_surf,
1957 .view = view,
1958 .mocs = mocs(res->bo),
1959 .address = res->bo->gtt_offset + offset_B,
1960 .x_offset_sa = tile_x_sa,
1961 .y_offset_sa = tile_y_sa,
1962 };
1963
1964 isl_surf_fill_state_s(&screen->isl_dev, map, &f);
1965 return psurf;
1966 }
1967
1968 #if GEN_GEN < 9
1969 static void
1970 fill_default_image_param(struct brw_image_param *param)
1971 {
1972 memset(param, 0, sizeof(*param));
1973 /* Set the swizzling shifts to all-ones to effectively disable swizzling --
1974 * See emit_address_calculation() in brw_fs_surface_builder.cpp for a more
1975 * detailed explanation of these parameters.
1976 */
1977 param->swizzling[0] = 0xff;
1978 param->swizzling[1] = 0xff;
1979 }
1980
1981 static void
1982 fill_buffer_image_param(struct brw_image_param *param,
1983 enum pipe_format pfmt,
1984 unsigned size)
1985 {
1986 const unsigned cpp = util_format_get_blocksize(pfmt);
1987
1988 fill_default_image_param(param);
1989 param->size[0] = size / cpp;
1990 param->stride[0] = cpp;
1991 }
1992 #else
1993 #define isl_surf_fill_image_param(x, ...)
1994 #define fill_default_image_param(x, ...)
1995 #define fill_buffer_image_param(x, ...)
1996 #endif
1997
1998 /**
1999 * The pipe->set_shader_images() driver hook.
2000 */
2001 static void
2002 iris_set_shader_images(struct pipe_context *ctx,
2003 enum pipe_shader_type p_stage,
2004 unsigned start_slot, unsigned count,
2005 const struct pipe_image_view *p_images)
2006 {
2007 struct iris_context *ice = (struct iris_context *) ctx;
2008 struct iris_screen *screen = (struct iris_screen *)ctx->screen;
2009 const struct gen_device_info *devinfo = &screen->devinfo;
2010 gl_shader_stage stage = stage_from_pipe(p_stage);
2011 struct iris_shader_state *shs = &ice->state.shaders[stage];
2012 #if GEN_GEN == 8
2013 struct iris_genx_state *genx = ice->state.genx;
2014 struct brw_image_param *image_params = genx->shaders[stage].image_param;
2015 #endif
2016
2017 shs->bound_image_views &= ~u_bit_consecutive(start_slot, count);
2018
2019 for (unsigned i = 0; i < count; i++) {
2020 struct iris_image_view *iv = &shs->image[start_slot + i];
2021
2022 if (p_images && p_images[i].resource) {
2023 const struct pipe_image_view *img = &p_images[i];
2024 struct iris_resource *res = (void *) img->resource;
2025
2026 void *map =
2027 alloc_surface_states(ice->state.surface_uploader,
2028 &iv->surface_state, 1 << ISL_AUX_USAGE_NONE);
2029 if (!unlikely(map))
2030 return;
2031
2032 util_copy_image_view(&iv->base, img);
2033
2034 shs->bound_image_views |= 1 << (start_slot + i);
2035
2036 res->bind_history |= PIPE_BIND_SHADER_IMAGE;
2037
2038 isl_surf_usage_flags_t usage = ISL_SURF_USAGE_STORAGE_BIT;
2039 enum isl_format isl_fmt =
2040 iris_format_for_usage(devinfo, img->format, usage).fmt;
2041
2042 bool untyped_fallback = false;
2043
2044 if (img->shader_access & PIPE_IMAGE_ACCESS_READ) {
2045 /* On Gen8, try to use typed surfaces reads (which support a
2046 * limited number of formats), and if not possible, fall back
2047 * to untyped reads.
2048 */
2049 untyped_fallback = GEN_GEN == 8 &&
2050 !isl_has_matching_typed_storage_image_format(devinfo, isl_fmt);
2051
2052 if (untyped_fallback)
2053 isl_fmt = ISL_FORMAT_RAW;
2054 else
2055 isl_fmt = isl_lower_storage_image_format(devinfo, isl_fmt);
2056 }
2057
2058 if (res->base.target != PIPE_BUFFER) {
2059 struct isl_view view = {
2060 .format = isl_fmt,
2061 .base_level = img->u.tex.level,
2062 .levels = 1,
2063 .base_array_layer = img->u.tex.first_layer,
2064 .array_len = img->u.tex.last_layer - img->u.tex.first_layer + 1,
2065 .swizzle = ISL_SWIZZLE_IDENTITY,
2066 .usage = usage,
2067 };
2068
2069 if (untyped_fallback) {
2070 fill_buffer_surface_state(&screen->isl_dev, res, map,
2071 isl_fmt, ISL_SWIZZLE_IDENTITY,
2072 0, res->bo->size);
2073 } else {
2074 /* Images don't support compression */
2075 unsigned aux_modes = 1 << ISL_AUX_USAGE_NONE;
2076 while (aux_modes) {
2077 enum isl_aux_usage usage = u_bit_scan(&aux_modes);
2078
2079 fill_surface_state(&screen->isl_dev, map, res, &view, usage);
2080
2081 map += SURFACE_STATE_ALIGNMENT;
2082 }
2083 }
2084
2085 isl_surf_fill_image_param(&screen->isl_dev,
2086 &image_params[start_slot + i],
2087 &res->surf, &view);
2088 } else {
2089 util_range_add(&res->valid_buffer_range, img->u.buf.offset,
2090 img->u.buf.offset + img->u.buf.size);
2091
2092 fill_buffer_surface_state(&screen->isl_dev, res, map,
2093 isl_fmt, ISL_SWIZZLE_IDENTITY,
2094 img->u.buf.offset, img->u.buf.size);
2095 fill_buffer_image_param(&image_params[start_slot + i],
2096 img->format, img->u.buf.size);
2097 }
2098 } else {
2099 pipe_resource_reference(&iv->base.resource, NULL);
2100 pipe_resource_reference(&iv->surface_state.res, NULL);
2101 fill_default_image_param(&image_params[start_slot + i]);
2102 }
2103 }
2104
2105 ice->state.dirty |= IRIS_DIRTY_BINDINGS_VS << stage;
2106 ice->state.dirty |=
2107 stage == MESA_SHADER_COMPUTE ? IRIS_DIRTY_COMPUTE_RESOLVES_AND_FLUSHES
2108 : IRIS_DIRTY_RENDER_RESOLVES_AND_FLUSHES;
2109
2110 /* Broadwell also needs brw_image_params re-uploaded */
2111 if (GEN_GEN < 9) {
2112 ice->state.dirty |= IRIS_DIRTY_CONSTANTS_VS << stage;
2113 shs->sysvals_need_upload = true;
2114 }
2115 }
2116
2117
2118 /**
2119 * The pipe->set_sampler_views() driver hook.
2120 */
2121 static void
2122 iris_set_sampler_views(struct pipe_context *ctx,
2123 enum pipe_shader_type p_stage,
2124 unsigned start, unsigned count,
2125 struct pipe_sampler_view **views)
2126 {
2127 struct iris_context *ice = (struct iris_context *) ctx;
2128 gl_shader_stage stage = stage_from_pipe(p_stage);
2129 struct iris_shader_state *shs = &ice->state.shaders[stage];
2130
2131 shs->bound_sampler_views &= ~u_bit_consecutive(start, count);
2132
2133 for (unsigned i = 0; i < count; i++) {
2134 struct pipe_sampler_view *pview = views ? views[i] : NULL;
2135 pipe_sampler_view_reference((struct pipe_sampler_view **)
2136 &shs->textures[start + i], pview);
2137 struct iris_sampler_view *view = (void *) pview;
2138 if (view) {
2139 view->res->bind_history |= PIPE_BIND_SAMPLER_VIEW;
2140 shs->bound_sampler_views |= 1 << (start + i);
2141 }
2142 }
2143
2144 ice->state.dirty |= (IRIS_DIRTY_BINDINGS_VS << stage);
2145 ice->state.dirty |=
2146 stage == MESA_SHADER_COMPUTE ? IRIS_DIRTY_COMPUTE_RESOLVES_AND_FLUSHES
2147 : IRIS_DIRTY_RENDER_RESOLVES_AND_FLUSHES;
2148 }
2149
2150 /**
2151 * The pipe->set_tess_state() driver hook.
2152 */
2153 static void
2154 iris_set_tess_state(struct pipe_context *ctx,
2155 const float default_outer_level[4],
2156 const float default_inner_level[2])
2157 {
2158 struct iris_context *ice = (struct iris_context *) ctx;
2159 struct iris_shader_state *shs = &ice->state.shaders[MESA_SHADER_TESS_CTRL];
2160
2161 memcpy(&ice->state.default_outer_level[0], &default_outer_level[0], 4 * sizeof(float));
2162 memcpy(&ice->state.default_inner_level[0], &default_inner_level[0], 2 * sizeof(float));
2163
2164 ice->state.dirty |= IRIS_DIRTY_CONSTANTS_TCS;
2165 shs->sysvals_need_upload = true;
2166 }
2167
2168 static void
2169 iris_surface_destroy(struct pipe_context *ctx, struct pipe_surface *p_surf)
2170 {
2171 struct iris_surface *surf = (void *) p_surf;
2172 pipe_resource_reference(&p_surf->texture, NULL);
2173 pipe_resource_reference(&surf->surface_state.res, NULL);
2174 free(surf);
2175 }
2176
2177 static void
2178 iris_set_clip_state(struct pipe_context *ctx,
2179 const struct pipe_clip_state *state)
2180 {
2181 struct iris_context *ice = (struct iris_context *) ctx;
2182 struct iris_shader_state *shs = &ice->state.shaders[MESA_SHADER_VERTEX];
2183 struct iris_shader_state *gshs = &ice->state.shaders[MESA_SHADER_GEOMETRY];
2184
2185 memcpy(&ice->state.clip_planes, state, sizeof(*state));
2186
2187 ice->state.dirty |= IRIS_DIRTY_CONSTANTS_VS | IRIS_DIRTY_CONSTANTS_GS;
2188 shs->sysvals_need_upload = true;
2189 gshs->sysvals_need_upload = true;
2190 }
2191
2192 /**
2193 * The pipe->set_polygon_stipple() driver hook.
2194 */
2195 static void
2196 iris_set_polygon_stipple(struct pipe_context *ctx,
2197 const struct pipe_poly_stipple *state)
2198 {
2199 struct iris_context *ice = (struct iris_context *) ctx;
2200 memcpy(&ice->state.poly_stipple, state, sizeof(*state));
2201 ice->state.dirty |= IRIS_DIRTY_POLYGON_STIPPLE;
2202 }
2203
2204 /**
2205 * The pipe->set_sample_mask() driver hook.
2206 */
2207 static void
2208 iris_set_sample_mask(struct pipe_context *ctx, unsigned sample_mask)
2209 {
2210 struct iris_context *ice = (struct iris_context *) ctx;
2211
2212 /* We only support 16x MSAA, so we have 16 bits of sample maks.
2213 * st/mesa may pass us 0xffffffff though, meaning "enable all samples".
2214 */
2215 ice->state.sample_mask = sample_mask & 0xffff;
2216 ice->state.dirty |= IRIS_DIRTY_SAMPLE_MASK;
2217 }
2218
2219 /**
2220 * The pipe->set_scissor_states() driver hook.
2221 *
2222 * This corresponds to our SCISSOR_RECT state structures. It's an
2223 * exact match, so we just store them, and memcpy them out later.
2224 */
2225 static void
2226 iris_set_scissor_states(struct pipe_context *ctx,
2227 unsigned start_slot,
2228 unsigned num_scissors,
2229 const struct pipe_scissor_state *rects)
2230 {
2231 struct iris_context *ice = (struct iris_context *) ctx;
2232
2233 for (unsigned i = 0; i < num_scissors; i++) {
2234 if (rects[i].minx == rects[i].maxx || rects[i].miny == rects[i].maxy) {
2235 /* If the scissor was out of bounds and got clamped to 0 width/height
2236 * at the bounds, the subtraction of 1 from maximums could produce a
2237 * negative number and thus not clip anything. Instead, just provide
2238 * a min > max scissor inside the bounds, which produces the expected
2239 * no rendering.
2240 */
2241 ice->state.scissors[start_slot + i] = (struct pipe_scissor_state) {
2242 .minx = 1, .maxx = 0, .miny = 1, .maxy = 0,
2243 };
2244 } else {
2245 ice->state.scissors[start_slot + i] = (struct pipe_scissor_state) {
2246 .minx = rects[i].minx, .miny = rects[i].miny,
2247 .maxx = rects[i].maxx - 1, .maxy = rects[i].maxy - 1,
2248 };
2249 }
2250 }
2251
2252 ice->state.dirty |= IRIS_DIRTY_SCISSOR_RECT;
2253 }
2254
2255 /**
2256 * The pipe->set_stencil_ref() driver hook.
2257 *
2258 * This is added to 3DSTATE_WM_DEPTH_STENCIL dynamically at draw time.
2259 */
2260 static void
2261 iris_set_stencil_ref(struct pipe_context *ctx,
2262 const struct pipe_stencil_ref *state)
2263 {
2264 struct iris_context *ice = (struct iris_context *) ctx;
2265 memcpy(&ice->state.stencil_ref, state, sizeof(*state));
2266 if (GEN_GEN == 8)
2267 ice->state.dirty |= IRIS_DIRTY_COLOR_CALC_STATE;
2268 else
2269 ice->state.dirty |= IRIS_DIRTY_WM_DEPTH_STENCIL;
2270 }
2271
2272 static float
2273 viewport_extent(const struct pipe_viewport_state *state, int axis, float sign)
2274 {
2275 return copysignf(state->scale[axis], sign) + state->translate[axis];
2276 }
2277
2278 /**
2279 * The pipe->set_viewport_states() driver hook.
2280 *
2281 * This corresponds to our SF_CLIP_VIEWPORT states. We can't calculate
2282 * the guardband yet, as we need the framebuffer dimensions, but we can
2283 * at least fill out the rest.
2284 */
2285 static void
2286 iris_set_viewport_states(struct pipe_context *ctx,
2287 unsigned start_slot,
2288 unsigned count,
2289 const struct pipe_viewport_state *states)
2290 {
2291 struct iris_context *ice = (struct iris_context *) ctx;
2292
2293 memcpy(&ice->state.viewports[start_slot], states, sizeof(*states) * count);
2294
2295 ice->state.dirty |= IRIS_DIRTY_SF_CL_VIEWPORT;
2296
2297 if (ice->state.cso_rast && (!ice->state.cso_rast->depth_clip_near ||
2298 !ice->state.cso_rast->depth_clip_far))
2299 ice->state.dirty |= IRIS_DIRTY_CC_VIEWPORT;
2300 }
2301
2302 /**
2303 * The pipe->set_framebuffer_state() driver hook.
2304 *
2305 * Sets the current draw FBO, including color render targets, depth,
2306 * and stencil buffers.
2307 */
2308 static void
2309 iris_set_framebuffer_state(struct pipe_context *ctx,
2310 const struct pipe_framebuffer_state *state)
2311 {
2312 struct iris_context *ice = (struct iris_context *) ctx;
2313 struct iris_screen *screen = (struct iris_screen *)ctx->screen;
2314 struct isl_device *isl_dev = &screen->isl_dev;
2315 struct pipe_framebuffer_state *cso = &ice->state.framebuffer;
2316 struct iris_resource *zres;
2317 struct iris_resource *stencil_res;
2318
2319 unsigned samples = util_framebuffer_get_num_samples(state);
2320 unsigned layers = util_framebuffer_get_num_layers(state);
2321
2322 if (cso->samples != samples) {
2323 ice->state.dirty |= IRIS_DIRTY_MULTISAMPLE;
2324
2325 /* We need to toggle 3DSTATE_PS::32 Pixel Dispatch Enable */
2326 if (GEN_GEN >= 9 && (cso->samples == 16 || samples == 16))
2327 ice->state.dirty |= IRIS_DIRTY_FS;
2328 }
2329
2330 if (cso->nr_cbufs != state->nr_cbufs) {
2331 ice->state.dirty |= IRIS_DIRTY_BLEND_STATE;
2332 }
2333
2334 if ((cso->layers == 0) != (layers == 0)) {
2335 ice->state.dirty |= IRIS_DIRTY_CLIP;
2336 }
2337
2338 if (cso->width != state->width || cso->height != state->height) {
2339 ice->state.dirty |= IRIS_DIRTY_SF_CL_VIEWPORT;
2340 }
2341
2342 if (cso->zsbuf || state->zsbuf) {
2343 ice->state.dirty |= IRIS_DIRTY_DEPTH_BUFFER;
2344 }
2345
2346 util_copy_framebuffer_state(cso, state);
2347 cso->samples = samples;
2348 cso->layers = layers;
2349
2350 struct iris_depth_buffer_state *cso_z = &ice->state.genx->depth_buffer;
2351
2352 struct isl_view view = {
2353 .base_level = 0,
2354 .levels = 1,
2355 .base_array_layer = 0,
2356 .array_len = 1,
2357 .swizzle = ISL_SWIZZLE_IDENTITY,
2358 };
2359
2360 struct isl_depth_stencil_hiz_emit_info info = { .view = &view };
2361
2362 if (cso->zsbuf) {
2363 iris_get_depth_stencil_resources(cso->zsbuf->texture, &zres,
2364 &stencil_res);
2365
2366 view.base_level = cso->zsbuf->u.tex.level;
2367 view.base_array_layer = cso->zsbuf->u.tex.first_layer;
2368 view.array_len =
2369 cso->zsbuf->u.tex.last_layer - cso->zsbuf->u.tex.first_layer + 1;
2370
2371 if (zres) {
2372 view.usage |= ISL_SURF_USAGE_DEPTH_BIT;
2373
2374 info.depth_surf = &zres->surf;
2375 info.depth_address = zres->bo->gtt_offset + zres->offset;
2376 info.mocs = mocs(zres->bo);
2377
2378 view.format = zres->surf.format;
2379
2380 if (iris_resource_level_has_hiz(zres, view.base_level)) {
2381 info.hiz_usage = ISL_AUX_USAGE_HIZ;
2382 info.hiz_surf = &zres->aux.surf;
2383 info.hiz_address = zres->aux.bo->gtt_offset;
2384 }
2385 }
2386
2387 if (stencil_res) {
2388 view.usage |= ISL_SURF_USAGE_STENCIL_BIT;
2389 info.stencil_surf = &stencil_res->surf;
2390 info.stencil_address = stencil_res->bo->gtt_offset + stencil_res->offset;
2391 if (!zres) {
2392 view.format = stencil_res->surf.format;
2393 info.mocs = mocs(stencil_res->bo);
2394 }
2395 }
2396 }
2397
2398 isl_emit_depth_stencil_hiz_s(isl_dev, cso_z->packets, &info);
2399
2400 /* Make a null surface for unbound buffers */
2401 void *null_surf_map =
2402 upload_state(ice->state.surface_uploader, &ice->state.null_fb,
2403 4 * GENX(RENDER_SURFACE_STATE_length), 64);
2404 isl_null_fill_state(&screen->isl_dev, null_surf_map,
2405 isl_extent3d(MAX2(cso->width, 1),
2406 MAX2(cso->height, 1),
2407 cso->layers ? cso->layers : 1));
2408 ice->state.null_fb.offset +=
2409 iris_bo_offset_from_base_address(iris_resource_bo(ice->state.null_fb.res));
2410
2411 /* Render target change */
2412 ice->state.dirty |= IRIS_DIRTY_BINDINGS_FS;
2413
2414 ice->state.dirty |= IRIS_DIRTY_RENDER_RESOLVES_AND_FLUSHES;
2415
2416 ice->state.dirty |= ice->state.dirty_for_nos[IRIS_NOS_FRAMEBUFFER];
2417
2418 #if GEN_GEN == 11
2419 // XXX: we may want to flag IRIS_DIRTY_MULTISAMPLE (or SAMPLE_MASK?)
2420 // XXX: see commit 979fc1bc9bcc64027ff2cfafd285676f31b930a6
2421
2422 /* The PIPE_CONTROL command description says:
2423 *
2424 * "Whenever a Binding Table Index (BTI) used by a Render Target Message
2425 * points to a different RENDER_SURFACE_STATE, SW must issue a Render
2426 * Target Cache Flush by enabling this bit. When render target flush
2427 * is set due to new association of BTI, PS Scoreboard Stall bit must
2428 * be set in this packet."
2429 */
2430 // XXX: does this need to happen at 3DSTATE_BTP_PS time?
2431 iris_emit_pipe_control_flush(&ice->batches[IRIS_BATCH_RENDER],
2432 "workaround: RT BTI change [draw]",
2433 PIPE_CONTROL_RENDER_TARGET_FLUSH |
2434 PIPE_CONTROL_STALL_AT_SCOREBOARD);
2435 #endif
2436 }
2437
2438 /**
2439 * The pipe->set_constant_buffer() driver hook.
2440 *
2441 * This uploads any constant data in user buffers, and references
2442 * any UBO resources containing constant data.
2443 */
2444 static void
2445 iris_set_constant_buffer(struct pipe_context *ctx,
2446 enum pipe_shader_type p_stage, unsigned index,
2447 const struct pipe_constant_buffer *input)
2448 {
2449 struct iris_context *ice = (struct iris_context *) ctx;
2450 gl_shader_stage stage = stage_from_pipe(p_stage);
2451 struct iris_shader_state *shs = &ice->state.shaders[stage];
2452 struct pipe_shader_buffer *cbuf = &shs->constbuf[index];
2453
2454 if (input && input->buffer_size && (input->buffer || input->user_buffer)) {
2455 shs->bound_cbufs |= 1u << index;
2456
2457 if (input->user_buffer) {
2458 void *map = NULL;
2459 pipe_resource_reference(&cbuf->buffer, NULL);
2460 u_upload_alloc(ice->ctx.const_uploader, 0, input->buffer_size, 64,
2461 &cbuf->buffer_offset, &cbuf->buffer, (void **) &map);
2462
2463 if (!cbuf->buffer) {
2464 /* Allocation was unsuccessful - just unbind */
2465 iris_set_constant_buffer(ctx, p_stage, index, NULL);
2466 return;
2467 }
2468
2469 assert(map);
2470 memcpy(map, input->user_buffer, input->buffer_size);
2471 } else if (input->buffer) {
2472 pipe_resource_reference(&cbuf->buffer, input->buffer);
2473
2474 cbuf->buffer_offset = input->buffer_offset;
2475 cbuf->buffer_size =
2476 MIN2(input->buffer_size,
2477 iris_resource_bo(cbuf->buffer)->size - cbuf->buffer_offset);
2478 }
2479
2480 struct iris_resource *res = (void *) cbuf->buffer;
2481 res->bind_history |= PIPE_BIND_CONSTANT_BUFFER;
2482
2483 iris_upload_ubo_ssbo_surf_state(ice, cbuf,
2484 &shs->constbuf_surf_state[index],
2485 false);
2486 } else {
2487 shs->bound_cbufs &= ~(1u << index);
2488 pipe_resource_reference(&cbuf->buffer, NULL);
2489 pipe_resource_reference(&shs->constbuf_surf_state[index].res, NULL);
2490 }
2491
2492 ice->state.dirty |= IRIS_DIRTY_CONSTANTS_VS << stage;
2493 // XXX: maybe not necessary all the time...?
2494 // XXX: we need 3DS_BTP to commit these changes, and if we fell back to
2495 // XXX: pull model we may need actual new bindings...
2496 ice->state.dirty |= IRIS_DIRTY_BINDINGS_VS << stage;
2497 }
2498
2499 static void
2500 upload_sysvals(struct iris_context *ice,
2501 gl_shader_stage stage)
2502 {
2503 UNUSED struct iris_genx_state *genx = ice->state.genx;
2504 struct iris_shader_state *shs = &ice->state.shaders[stage];
2505
2506 struct iris_compiled_shader *shader = ice->shaders.prog[stage];
2507 if (!shader || shader->num_system_values == 0)
2508 return;
2509
2510 assert(shader->num_cbufs > 0);
2511
2512 unsigned sysval_cbuf_index = shader->num_cbufs - 1;
2513 struct pipe_shader_buffer *cbuf = &shs->constbuf[sysval_cbuf_index];
2514 unsigned upload_size = shader->num_system_values * sizeof(uint32_t);
2515 uint32_t *map = NULL;
2516
2517 assert(sysval_cbuf_index < PIPE_MAX_CONSTANT_BUFFERS);
2518 u_upload_alloc(ice->ctx.const_uploader, 0, upload_size, 64,
2519 &cbuf->buffer_offset, &cbuf->buffer, (void **) &map);
2520
2521 for (int i = 0; i < shader->num_system_values; i++) {
2522 uint32_t sysval = shader->system_values[i];
2523 uint32_t value = 0;
2524
2525 if (BRW_PARAM_DOMAIN(sysval) == BRW_PARAM_DOMAIN_IMAGE) {
2526 #if GEN_GEN == 8
2527 unsigned img = BRW_PARAM_IMAGE_IDX(sysval);
2528 unsigned offset = BRW_PARAM_IMAGE_OFFSET(sysval);
2529 struct brw_image_param *param =
2530 &genx->shaders[stage].image_param[img];
2531
2532 assert(offset < sizeof(struct brw_image_param));
2533 value = ((uint32_t *) param)[offset];
2534 #endif
2535 } else if (sysval == BRW_PARAM_BUILTIN_ZERO) {
2536 value = 0;
2537 } else if (BRW_PARAM_BUILTIN_IS_CLIP_PLANE(sysval)) {
2538 int plane = BRW_PARAM_BUILTIN_CLIP_PLANE_IDX(sysval);
2539 int comp = BRW_PARAM_BUILTIN_CLIP_PLANE_COMP(sysval);
2540 value = fui(ice->state.clip_planes.ucp[plane][comp]);
2541 } else if (sysval == BRW_PARAM_BUILTIN_PATCH_VERTICES_IN) {
2542 if (stage == MESA_SHADER_TESS_CTRL) {
2543 value = ice->state.vertices_per_patch;
2544 } else {
2545 assert(stage == MESA_SHADER_TESS_EVAL);
2546 const struct shader_info *tcs_info =
2547 iris_get_shader_info(ice, MESA_SHADER_TESS_CTRL);
2548 if (tcs_info)
2549 value = tcs_info->tess.tcs_vertices_out;
2550 else
2551 value = ice->state.vertices_per_patch;
2552 }
2553 } else if (sysval >= BRW_PARAM_BUILTIN_TESS_LEVEL_OUTER_X &&
2554 sysval <= BRW_PARAM_BUILTIN_TESS_LEVEL_OUTER_W) {
2555 unsigned i = sysval - BRW_PARAM_BUILTIN_TESS_LEVEL_OUTER_X;
2556 value = fui(ice->state.default_outer_level[i]);
2557 } else if (sysval == BRW_PARAM_BUILTIN_TESS_LEVEL_INNER_X) {
2558 value = fui(ice->state.default_inner_level[0]);
2559 } else if (sysval == BRW_PARAM_BUILTIN_TESS_LEVEL_INNER_Y) {
2560 value = fui(ice->state.default_inner_level[1]);
2561 } else {
2562 assert(!"unhandled system value");
2563 }
2564
2565 *map++ = value;
2566 }
2567
2568 cbuf->buffer_size = upload_size;
2569 iris_upload_ubo_ssbo_surf_state(ice, cbuf,
2570 &shs->constbuf_surf_state[sysval_cbuf_index], false);
2571
2572 shs->sysvals_need_upload = false;
2573 }
2574
2575 /**
2576 * The pipe->set_shader_buffers() driver hook.
2577 *
2578 * This binds SSBOs and ABOs. Unfortunately, we need to stream out
2579 * SURFACE_STATE here, as the buffer offset may change each time.
2580 */
2581 static void
2582 iris_set_shader_buffers(struct pipe_context *ctx,
2583 enum pipe_shader_type p_stage,
2584 unsigned start_slot, unsigned count,
2585 const struct pipe_shader_buffer *buffers,
2586 unsigned writable_bitmask)
2587 {
2588 struct iris_context *ice = (struct iris_context *) ctx;
2589 gl_shader_stage stage = stage_from_pipe(p_stage);
2590 struct iris_shader_state *shs = &ice->state.shaders[stage];
2591
2592 unsigned modified_bits = u_bit_consecutive(start_slot, count);
2593
2594 shs->bound_ssbos &= ~modified_bits;
2595 shs->writable_ssbos &= ~modified_bits;
2596 shs->writable_ssbos |= writable_bitmask << start_slot;
2597
2598 for (unsigned i = 0; i < count; i++) {
2599 if (buffers && buffers[i].buffer) {
2600 struct iris_resource *res = (void *) buffers[i].buffer;
2601 struct pipe_shader_buffer *ssbo = &shs->ssbo[start_slot + i];
2602 struct iris_state_ref *surf_state =
2603 &shs->ssbo_surf_state[start_slot + i];
2604 pipe_resource_reference(&ssbo->buffer, &res->base);
2605 ssbo->buffer_offset = buffers[i].buffer_offset;
2606 ssbo->buffer_size =
2607 MIN2(buffers[i].buffer_size, res->bo->size - ssbo->buffer_offset);
2608
2609 shs->bound_ssbos |= 1 << (start_slot + i);
2610
2611 iris_upload_ubo_ssbo_surf_state(ice, ssbo, surf_state, true);
2612
2613 res->bind_history |= PIPE_BIND_SHADER_BUFFER;
2614
2615 util_range_add(&res->valid_buffer_range, ssbo->buffer_offset,
2616 ssbo->buffer_offset + ssbo->buffer_size);
2617 } else {
2618 pipe_resource_reference(&shs->ssbo[start_slot + i].buffer, NULL);
2619 pipe_resource_reference(&shs->ssbo_surf_state[start_slot + i].res,
2620 NULL);
2621 }
2622 }
2623
2624 ice->state.dirty |= IRIS_DIRTY_BINDINGS_VS << stage;
2625 }
2626
2627 static void
2628 iris_delete_state(struct pipe_context *ctx, void *state)
2629 {
2630 free(state);
2631 }
2632
2633 /**
2634 * The pipe->set_vertex_buffers() driver hook.
2635 *
2636 * This translates pipe_vertex_buffer to our 3DSTATE_VERTEX_BUFFERS packet.
2637 */
2638 static void
2639 iris_set_vertex_buffers(struct pipe_context *ctx,
2640 unsigned start_slot, unsigned count,
2641 const struct pipe_vertex_buffer *buffers)
2642 {
2643 struct iris_context *ice = (struct iris_context *) ctx;
2644 struct iris_genx_state *genx = ice->state.genx;
2645
2646 ice->state.bound_vertex_buffers &= ~u_bit_consecutive64(start_slot, count);
2647
2648 for (unsigned i = 0; i < count; i++) {
2649 const struct pipe_vertex_buffer *buffer = buffers ? &buffers[i] : NULL;
2650 struct iris_vertex_buffer_state *state =
2651 &genx->vertex_buffers[start_slot + i];
2652
2653 if (!buffer) {
2654 pipe_resource_reference(&state->resource, NULL);
2655 continue;
2656 }
2657
2658 /* We may see user buffers that are NULL bindings. */
2659 assert(!(buffer->is_user_buffer && buffer->buffer.user != NULL));
2660
2661 pipe_resource_reference(&state->resource, buffer->buffer.resource);
2662 struct iris_resource *res = (void *) state->resource;
2663
2664 if (res) {
2665 ice->state.bound_vertex_buffers |= 1ull << (start_slot + i);
2666 res->bind_history |= PIPE_BIND_VERTEX_BUFFER;
2667 }
2668
2669 iris_pack_state(GENX(VERTEX_BUFFER_STATE), state->state, vb) {
2670 vb.VertexBufferIndex = start_slot + i;
2671 vb.AddressModifyEnable = true;
2672 vb.BufferPitch = buffer->stride;
2673 if (res) {
2674 vb.BufferSize = res->bo->size - (int) buffer->buffer_offset;
2675 vb.BufferStartingAddress =
2676 ro_bo(NULL, res->bo->gtt_offset + (int) buffer->buffer_offset);
2677 vb.MOCS = mocs(res->bo);
2678 } else {
2679 vb.NullVertexBuffer = true;
2680 }
2681 }
2682 }
2683
2684 ice->state.dirty |= IRIS_DIRTY_VERTEX_BUFFERS;
2685 }
2686
2687 /**
2688 * Gallium CSO for vertex elements.
2689 */
2690 struct iris_vertex_element_state {
2691 uint32_t vertex_elements[1 + 33 * GENX(VERTEX_ELEMENT_STATE_length)];
2692 uint32_t vf_instancing[33 * GENX(3DSTATE_VF_INSTANCING_length)];
2693 uint32_t edgeflag_ve[GENX(VERTEX_ELEMENT_STATE_length)];
2694 uint32_t edgeflag_vfi[GENX(3DSTATE_VF_INSTANCING_length)];
2695 unsigned count;
2696 };
2697
2698 /**
2699 * The pipe->create_vertex_elements() driver hook.
2700 *
2701 * This translates pipe_vertex_element to our 3DSTATE_VERTEX_ELEMENTS
2702 * and 3DSTATE_VF_INSTANCING commands. The vertex_elements and vf_instancing
2703 * arrays are ready to be emitted at draw time if no EdgeFlag or SGVs are
2704 * needed. In these cases we will need information available at draw time.
2705 * We setup edgeflag_ve and edgeflag_vfi as alternatives last
2706 * 3DSTATE_VERTEX_ELEMENT and 3DSTATE_VF_INSTANCING that can be used at
2707 * draw time if we detect that EdgeFlag is needed by the Vertex Shader.
2708 */
2709 static void *
2710 iris_create_vertex_elements(struct pipe_context *ctx,
2711 unsigned count,
2712 const struct pipe_vertex_element *state)
2713 {
2714 struct iris_screen *screen = (struct iris_screen *)ctx->screen;
2715 const struct gen_device_info *devinfo = &screen->devinfo;
2716 struct iris_vertex_element_state *cso =
2717 malloc(sizeof(struct iris_vertex_element_state));
2718
2719 cso->count = count;
2720
2721 iris_pack_command(GENX(3DSTATE_VERTEX_ELEMENTS), cso->vertex_elements, ve) {
2722 ve.DWordLength =
2723 1 + GENX(VERTEX_ELEMENT_STATE_length) * MAX2(count, 1) - 2;
2724 }
2725
2726 uint32_t *ve_pack_dest = &cso->vertex_elements[1];
2727 uint32_t *vfi_pack_dest = cso->vf_instancing;
2728
2729 if (count == 0) {
2730 iris_pack_state(GENX(VERTEX_ELEMENT_STATE), ve_pack_dest, ve) {
2731 ve.Valid = true;
2732 ve.SourceElementFormat = ISL_FORMAT_R32G32B32A32_FLOAT;
2733 ve.Component0Control = VFCOMP_STORE_0;
2734 ve.Component1Control = VFCOMP_STORE_0;
2735 ve.Component2Control = VFCOMP_STORE_0;
2736 ve.Component3Control = VFCOMP_STORE_1_FP;
2737 }
2738
2739 iris_pack_command(GENX(3DSTATE_VF_INSTANCING), vfi_pack_dest, vi) {
2740 }
2741 }
2742
2743 for (int i = 0; i < count; i++) {
2744 const struct iris_format_info fmt =
2745 iris_format_for_usage(devinfo, state[i].src_format, 0);
2746 unsigned comp[4] = { VFCOMP_STORE_SRC, VFCOMP_STORE_SRC,
2747 VFCOMP_STORE_SRC, VFCOMP_STORE_SRC };
2748
2749 switch (isl_format_get_num_channels(fmt.fmt)) {
2750 case 0: comp[0] = VFCOMP_STORE_0; /* fallthrough */
2751 case 1: comp[1] = VFCOMP_STORE_0; /* fallthrough */
2752 case 2: comp[2] = VFCOMP_STORE_0; /* fallthrough */
2753 case 3:
2754 comp[3] = isl_format_has_int_channel(fmt.fmt) ? VFCOMP_STORE_1_INT
2755 : VFCOMP_STORE_1_FP;
2756 break;
2757 }
2758 iris_pack_state(GENX(VERTEX_ELEMENT_STATE), ve_pack_dest, ve) {
2759 ve.EdgeFlagEnable = false;
2760 ve.VertexBufferIndex = state[i].vertex_buffer_index;
2761 ve.Valid = true;
2762 ve.SourceElementOffset = state[i].src_offset;
2763 ve.SourceElementFormat = fmt.fmt;
2764 ve.Component0Control = comp[0];
2765 ve.Component1Control = comp[1];
2766 ve.Component2Control = comp[2];
2767 ve.Component3Control = comp[3];
2768 }
2769
2770 iris_pack_command(GENX(3DSTATE_VF_INSTANCING), vfi_pack_dest, vi) {
2771 vi.VertexElementIndex = i;
2772 vi.InstancingEnable = state[i].instance_divisor > 0;
2773 vi.InstanceDataStepRate = state[i].instance_divisor;
2774 }
2775
2776 ve_pack_dest += GENX(VERTEX_ELEMENT_STATE_length);
2777 vfi_pack_dest += GENX(3DSTATE_VF_INSTANCING_length);
2778 }
2779
2780 /* An alternative version of the last VE and VFI is stored so it
2781 * can be used at draw time in case Vertex Shader uses EdgeFlag
2782 */
2783 if (count) {
2784 const unsigned edgeflag_index = count - 1;
2785 const struct iris_format_info fmt =
2786 iris_format_for_usage(devinfo, state[edgeflag_index].src_format, 0);
2787 iris_pack_state(GENX(VERTEX_ELEMENT_STATE), cso->edgeflag_ve, ve) {
2788 ve.EdgeFlagEnable = true ;
2789 ve.VertexBufferIndex = state[edgeflag_index].vertex_buffer_index;
2790 ve.Valid = true;
2791 ve.SourceElementOffset = state[edgeflag_index].src_offset;
2792 ve.SourceElementFormat = fmt.fmt;
2793 ve.Component0Control = VFCOMP_STORE_SRC;
2794 ve.Component1Control = VFCOMP_STORE_0;
2795 ve.Component2Control = VFCOMP_STORE_0;
2796 ve.Component3Control = VFCOMP_STORE_0;
2797 }
2798 iris_pack_command(GENX(3DSTATE_VF_INSTANCING), cso->edgeflag_vfi, vi) {
2799 /* The vi.VertexElementIndex of the EdgeFlag Vertex Element is filled
2800 * at draw time, as it should change if SGVs are emitted.
2801 */
2802 vi.InstancingEnable = state[edgeflag_index].instance_divisor > 0;
2803 vi.InstanceDataStepRate = state[edgeflag_index].instance_divisor;
2804 }
2805 }
2806
2807 return cso;
2808 }
2809
2810 /**
2811 * The pipe->bind_vertex_elements_state() driver hook.
2812 */
2813 static void
2814 iris_bind_vertex_elements_state(struct pipe_context *ctx, void *state)
2815 {
2816 struct iris_context *ice = (struct iris_context *) ctx;
2817 struct iris_vertex_element_state *old_cso = ice->state.cso_vertex_elements;
2818 struct iris_vertex_element_state *new_cso = state;
2819
2820 /* 3DSTATE_VF_SGVs overrides the last VE, so if the count is changing,
2821 * we need to re-emit it to ensure we're overriding the right one.
2822 */
2823 if (new_cso && cso_changed(count))
2824 ice->state.dirty |= IRIS_DIRTY_VF_SGVS;
2825
2826 ice->state.cso_vertex_elements = state;
2827 ice->state.dirty |= IRIS_DIRTY_VERTEX_ELEMENTS;
2828 }
2829
2830 /**
2831 * The pipe->create_stream_output_target() driver hook.
2832 *
2833 * "Target" here refers to a destination buffer. We translate this into
2834 * a 3DSTATE_SO_BUFFER packet. We can handle most fields, but don't yet
2835 * know which buffer this represents, or whether we ought to zero the
2836 * write-offsets, or append. Those are handled in the set() hook.
2837 */
2838 static struct pipe_stream_output_target *
2839 iris_create_stream_output_target(struct pipe_context *ctx,
2840 struct pipe_resource *p_res,
2841 unsigned buffer_offset,
2842 unsigned buffer_size)
2843 {
2844 struct iris_resource *res = (void *) p_res;
2845 struct iris_stream_output_target *cso = calloc(1, sizeof(*cso));
2846 if (!cso)
2847 return NULL;
2848
2849 res->bind_history |= PIPE_BIND_STREAM_OUTPUT;
2850
2851 pipe_reference_init(&cso->base.reference, 1);
2852 pipe_resource_reference(&cso->base.buffer, p_res);
2853 cso->base.buffer_offset = buffer_offset;
2854 cso->base.buffer_size = buffer_size;
2855 cso->base.context = ctx;
2856
2857 util_range_add(&res->valid_buffer_range, buffer_offset,
2858 buffer_offset + buffer_size);
2859
2860 upload_state(ctx->stream_uploader, &cso->offset, sizeof(uint32_t), 4);
2861
2862 return &cso->base;
2863 }
2864
2865 static void
2866 iris_stream_output_target_destroy(struct pipe_context *ctx,
2867 struct pipe_stream_output_target *state)
2868 {
2869 struct iris_stream_output_target *cso = (void *) state;
2870
2871 pipe_resource_reference(&cso->base.buffer, NULL);
2872 pipe_resource_reference(&cso->offset.res, NULL);
2873
2874 free(cso);
2875 }
2876
2877 /**
2878 * The pipe->set_stream_output_targets() driver hook.
2879 *
2880 * At this point, we know which targets are bound to a particular index,
2881 * and also whether we want to append or start over. We can finish the
2882 * 3DSTATE_SO_BUFFER packets we started earlier.
2883 */
2884 static void
2885 iris_set_stream_output_targets(struct pipe_context *ctx,
2886 unsigned num_targets,
2887 struct pipe_stream_output_target **targets,
2888 const unsigned *offsets)
2889 {
2890 struct iris_context *ice = (struct iris_context *) ctx;
2891 struct iris_genx_state *genx = ice->state.genx;
2892 uint32_t *so_buffers = genx->so_buffers;
2893
2894 const bool active = num_targets > 0;
2895 if (ice->state.streamout_active != active) {
2896 ice->state.streamout_active = active;
2897 ice->state.dirty |= IRIS_DIRTY_STREAMOUT;
2898
2899 /* We only emit 3DSTATE_SO_DECL_LIST when streamout is active, because
2900 * it's a non-pipelined command. If we're switching streamout on, we
2901 * may have missed emitting it earlier, so do so now. (We're already
2902 * taking a stall to update 3DSTATE_SO_BUFFERS anyway...)
2903 */
2904 if (active) {
2905 ice->state.dirty |= IRIS_DIRTY_SO_DECL_LIST;
2906 } else {
2907 uint32_t flush = 0;
2908 for (int i = 0; i < PIPE_MAX_SO_BUFFERS; i++) {
2909 struct iris_stream_output_target *tgt =
2910 (void *) ice->state.so_target[i];
2911 if (tgt) {
2912 struct iris_resource *res = (void *) tgt->base.buffer;
2913
2914 flush |= iris_flush_bits_for_history(res);
2915 iris_dirty_for_history(ice, res);
2916 }
2917 }
2918 iris_emit_pipe_control_flush(&ice->batches[IRIS_BATCH_RENDER],
2919 "make streamout results visible", flush);
2920 }
2921 }
2922
2923 for (int i = 0; i < 4; i++) {
2924 pipe_so_target_reference(&ice->state.so_target[i],
2925 i < num_targets ? targets[i] : NULL);
2926 }
2927
2928 /* No need to update 3DSTATE_SO_BUFFER unless SOL is active. */
2929 if (!active)
2930 return;
2931
2932 for (unsigned i = 0; i < 4; i++,
2933 so_buffers += GENX(3DSTATE_SO_BUFFER_length)) {
2934
2935 struct iris_stream_output_target *tgt = (void *) ice->state.so_target[i];
2936 unsigned offset = offsets[i];
2937
2938 if (!tgt) {
2939 iris_pack_command(GENX(3DSTATE_SO_BUFFER), so_buffers, sob)
2940 sob.SOBufferIndex = i;
2941 continue;
2942 }
2943
2944 struct iris_resource *res = (void *) tgt->base.buffer;
2945
2946 /* Note that offsets[i] will either be 0, causing us to zero
2947 * the value in the buffer, or 0xFFFFFFFF, which happens to mean
2948 * "continue appending at the existing offset."
2949 */
2950 assert(offset == 0 || offset == 0xFFFFFFFF);
2951
2952 /* We might be called by Begin (offset = 0), Pause, then Resume
2953 * (offset = 0xFFFFFFFF) before ever drawing (where these commands
2954 * will actually be sent to the GPU). In this case, we don't want
2955 * to append - we still want to do our initial zeroing.
2956 */
2957 if (!tgt->zeroed)
2958 offset = 0;
2959
2960 iris_pack_command(GENX(3DSTATE_SO_BUFFER), so_buffers, sob) {
2961 sob.SurfaceBaseAddress =
2962 rw_bo(NULL, res->bo->gtt_offset + tgt->base.buffer_offset);
2963 sob.SOBufferEnable = true;
2964 sob.StreamOffsetWriteEnable = true;
2965 sob.StreamOutputBufferOffsetAddressEnable = true;
2966 sob.MOCS = mocs(res->bo);
2967
2968 sob.SurfaceSize = MAX2(tgt->base.buffer_size / 4, 1) - 1;
2969
2970 sob.SOBufferIndex = i;
2971 sob.StreamOffset = offset;
2972 sob.StreamOutputBufferOffsetAddress =
2973 rw_bo(NULL, iris_resource_bo(tgt->offset.res)->gtt_offset +
2974 tgt->offset.offset);
2975 }
2976 }
2977
2978 ice->state.dirty |= IRIS_DIRTY_SO_BUFFERS;
2979 }
2980
2981 /**
2982 * An iris-vtable helper for encoding the 3DSTATE_SO_DECL_LIST and
2983 * 3DSTATE_STREAMOUT packets.
2984 *
2985 * 3DSTATE_SO_DECL_LIST is a list of shader outputs we want the streamout
2986 * hardware to record. We can create it entirely based on the shader, with
2987 * no dynamic state dependencies.
2988 *
2989 * 3DSTATE_STREAMOUT is an annoying mix of shader-based information and
2990 * state-based settings. We capture the shader-related ones here, and merge
2991 * the rest in at draw time.
2992 */
2993 static uint32_t *
2994 iris_create_so_decl_list(const struct pipe_stream_output_info *info,
2995 const struct brw_vue_map *vue_map)
2996 {
2997 struct GENX(SO_DECL) so_decl[MAX_VERTEX_STREAMS][128];
2998 int buffer_mask[MAX_VERTEX_STREAMS] = {0, 0, 0, 0};
2999 int next_offset[MAX_VERTEX_STREAMS] = {0, 0, 0, 0};
3000 int decls[MAX_VERTEX_STREAMS] = {0, 0, 0, 0};
3001 int max_decls = 0;
3002 STATIC_ASSERT(ARRAY_SIZE(so_decl[0]) >= MAX_PROGRAM_OUTPUTS);
3003
3004 memset(so_decl, 0, sizeof(so_decl));
3005
3006 /* Construct the list of SO_DECLs to be emitted. The formatting of the
3007 * command feels strange -- each dword pair contains a SO_DECL per stream.
3008 */
3009 for (unsigned i = 0; i < info->num_outputs; i++) {
3010 const struct pipe_stream_output *output = &info->output[i];
3011 const int buffer = output->output_buffer;
3012 const int varying = output->register_index;
3013 const unsigned stream_id = output->stream;
3014 assert(stream_id < MAX_VERTEX_STREAMS);
3015
3016 buffer_mask[stream_id] |= 1 << buffer;
3017
3018 assert(vue_map->varying_to_slot[varying] >= 0);
3019
3020 /* Mesa doesn't store entries for gl_SkipComponents in the Outputs[]
3021 * array. Instead, it simply increments DstOffset for the following
3022 * input by the number of components that should be skipped.
3023 *
3024 * Our hardware is unusual in that it requires us to program SO_DECLs
3025 * for fake "hole" components, rather than simply taking the offset
3026 * for each real varying. Each hole can have size 1, 2, 3, or 4; we
3027 * program as many size = 4 holes as we can, then a final hole to
3028 * accommodate the final 1, 2, or 3 remaining.
3029 */
3030 int skip_components = output->dst_offset - next_offset[buffer];
3031
3032 while (skip_components > 0) {
3033 so_decl[stream_id][decls[stream_id]++] = (struct GENX(SO_DECL)) {
3034 .HoleFlag = 1,
3035 .OutputBufferSlot = output->output_buffer,
3036 .ComponentMask = (1 << MIN2(skip_components, 4)) - 1,
3037 };
3038 skip_components -= 4;
3039 }
3040
3041 next_offset[buffer] = output->dst_offset + output->num_components;
3042
3043 so_decl[stream_id][decls[stream_id]++] = (struct GENX(SO_DECL)) {
3044 .OutputBufferSlot = output->output_buffer,
3045 .RegisterIndex = vue_map->varying_to_slot[varying],
3046 .ComponentMask =
3047 ((1 << output->num_components) - 1) << output->start_component,
3048 };
3049
3050 if (decls[stream_id] > max_decls)
3051 max_decls = decls[stream_id];
3052 }
3053
3054 unsigned dwords = GENX(3DSTATE_STREAMOUT_length) + (3 + 2 * max_decls);
3055 uint32_t *map = ralloc_size(NULL, sizeof(uint32_t) * dwords);
3056 uint32_t *so_decl_map = map + GENX(3DSTATE_STREAMOUT_length);
3057
3058 iris_pack_command(GENX(3DSTATE_STREAMOUT), map, sol) {
3059 int urb_entry_read_offset = 0;
3060 int urb_entry_read_length = (vue_map->num_slots + 1) / 2 -
3061 urb_entry_read_offset;
3062
3063 /* We always read the whole vertex. This could be reduced at some
3064 * point by reading less and offsetting the register index in the
3065 * SO_DECLs.
3066 */
3067 sol.Stream0VertexReadOffset = urb_entry_read_offset;
3068 sol.Stream0VertexReadLength = urb_entry_read_length - 1;
3069 sol.Stream1VertexReadOffset = urb_entry_read_offset;
3070 sol.Stream1VertexReadLength = urb_entry_read_length - 1;
3071 sol.Stream2VertexReadOffset = urb_entry_read_offset;
3072 sol.Stream2VertexReadLength = urb_entry_read_length - 1;
3073 sol.Stream3VertexReadOffset = urb_entry_read_offset;
3074 sol.Stream3VertexReadLength = urb_entry_read_length - 1;
3075
3076 /* Set buffer pitches; 0 means unbound. */
3077 sol.Buffer0SurfacePitch = 4 * info->stride[0];
3078 sol.Buffer1SurfacePitch = 4 * info->stride[1];
3079 sol.Buffer2SurfacePitch = 4 * info->stride[2];
3080 sol.Buffer3SurfacePitch = 4 * info->stride[3];
3081 }
3082
3083 iris_pack_command(GENX(3DSTATE_SO_DECL_LIST), so_decl_map, list) {
3084 list.DWordLength = 3 + 2 * max_decls - 2;
3085 list.StreamtoBufferSelects0 = buffer_mask[0];
3086 list.StreamtoBufferSelects1 = buffer_mask[1];
3087 list.StreamtoBufferSelects2 = buffer_mask[2];
3088 list.StreamtoBufferSelects3 = buffer_mask[3];
3089 list.NumEntries0 = decls[0];
3090 list.NumEntries1 = decls[1];
3091 list.NumEntries2 = decls[2];
3092 list.NumEntries3 = decls[3];
3093 }
3094
3095 for (int i = 0; i < max_decls; i++) {
3096 iris_pack_state(GENX(SO_DECL_ENTRY), so_decl_map + 3 + i * 2, entry) {
3097 entry.Stream0Decl = so_decl[0][i];
3098 entry.Stream1Decl = so_decl[1][i];
3099 entry.Stream2Decl = so_decl[2][i];
3100 entry.Stream3Decl = so_decl[3][i];
3101 }
3102 }
3103
3104 return map;
3105 }
3106
3107 static void
3108 iris_compute_sbe_urb_read_interval(uint64_t fs_input_slots,
3109 const struct brw_vue_map *last_vue_map,
3110 bool two_sided_color,
3111 unsigned *out_offset,
3112 unsigned *out_length)
3113 {
3114 /* The compiler computes the first URB slot without considering COL/BFC
3115 * swizzling (because it doesn't know whether it's enabled), so we need
3116 * to do that here too. This may result in a smaller offset, which
3117 * should be safe.
3118 */
3119 const unsigned first_slot =
3120 brw_compute_first_urb_slot_required(fs_input_slots, last_vue_map);
3121
3122 /* This becomes the URB read offset (counted in pairs of slots). */
3123 assert(first_slot % 2 == 0);
3124 *out_offset = first_slot / 2;
3125
3126 /* We need to adjust the inputs read to account for front/back color
3127 * swizzling, as it can make the URB length longer.
3128 */
3129 for (int c = 0; c <= 1; c++) {
3130 if (fs_input_slots & (VARYING_BIT_COL0 << c)) {
3131 /* If two sided color is enabled, the fragment shader's gl_Color
3132 * (COL0) input comes from either the gl_FrontColor (COL0) or
3133 * gl_BackColor (BFC0) input varyings. Mark BFC as used, too.
3134 */
3135 if (two_sided_color)
3136 fs_input_slots |= (VARYING_BIT_BFC0 << c);
3137
3138 /* If front color isn't written, we opt to give them back color
3139 * instead of an undefined value. Switch from COL to BFC.
3140 */
3141 if (last_vue_map->varying_to_slot[VARYING_SLOT_COL0 + c] == -1) {
3142 fs_input_slots &= ~(VARYING_BIT_COL0 << c);
3143 fs_input_slots |= (VARYING_BIT_BFC0 << c);
3144 }
3145 }
3146 }
3147
3148 /* Compute the minimum URB Read Length necessary for the FS inputs.
3149 *
3150 * From the Sandy Bridge PRM, Volume 2, Part 1, documentation for
3151 * 3DSTATE_SF DWord 1 bits 15:11, "Vertex URB Entry Read Length":
3152 *
3153 * "This field should be set to the minimum length required to read the
3154 * maximum source attribute. The maximum source attribute is indicated
3155 * by the maximum value of the enabled Attribute # Source Attribute if
3156 * Attribute Swizzle Enable is set, Number of Output Attributes-1 if
3157 * enable is not set.
3158 * read_length = ceiling((max_source_attr + 1) / 2)
3159 *
3160 * [errata] Corruption/Hang possible if length programmed larger than
3161 * recommended"
3162 *
3163 * Similar text exists for Ivy Bridge.
3164 *
3165 * We find the last URB slot that's actually read by the FS.
3166 */
3167 unsigned last_read_slot = last_vue_map->num_slots - 1;
3168 while (last_read_slot > first_slot && !(fs_input_slots &
3169 (1ull << last_vue_map->slot_to_varying[last_read_slot])))
3170 --last_read_slot;
3171
3172 /* The URB read length is the difference of the two, counted in pairs. */
3173 *out_length = DIV_ROUND_UP(last_read_slot - first_slot + 1, 2);
3174 }
3175
3176 static void
3177 iris_emit_sbe_swiz(struct iris_batch *batch,
3178 const struct iris_context *ice,
3179 unsigned urb_read_offset,
3180 unsigned sprite_coord_enables)
3181 {
3182 struct GENX(SF_OUTPUT_ATTRIBUTE_DETAIL) attr_overrides[16] = {};
3183 const struct brw_wm_prog_data *wm_prog_data = (void *)
3184 ice->shaders.prog[MESA_SHADER_FRAGMENT]->prog_data;
3185 const struct brw_vue_map *vue_map = ice->shaders.last_vue_map;
3186 const struct iris_rasterizer_state *cso_rast = ice->state.cso_rast;
3187
3188 /* XXX: this should be generated when putting programs in place */
3189
3190 for (int fs_attr = 0; fs_attr < VARYING_SLOT_MAX; fs_attr++) {
3191 const int input_index = wm_prog_data->urb_setup[fs_attr];
3192 if (input_index < 0 || input_index >= 16)
3193 continue;
3194
3195 struct GENX(SF_OUTPUT_ATTRIBUTE_DETAIL) *attr =
3196 &attr_overrides[input_index];
3197 int slot = vue_map->varying_to_slot[fs_attr];
3198
3199 /* Viewport and Layer are stored in the VUE header. We need to override
3200 * them to zero if earlier stages didn't write them, as GL requires that
3201 * they read back as zero when not explicitly set.
3202 */
3203 switch (fs_attr) {
3204 case VARYING_SLOT_VIEWPORT:
3205 case VARYING_SLOT_LAYER:
3206 attr->ComponentOverrideX = true;
3207 attr->ComponentOverrideW = true;
3208 attr->ConstantSource = CONST_0000;
3209
3210 if (!(vue_map->slots_valid & VARYING_BIT_LAYER))
3211 attr->ComponentOverrideY = true;
3212 if (!(vue_map->slots_valid & VARYING_BIT_VIEWPORT))
3213 attr->ComponentOverrideZ = true;
3214 continue;
3215
3216 case VARYING_SLOT_PRIMITIVE_ID:
3217 /* Override if the previous shader stage didn't write gl_PrimitiveID. */
3218 if (slot == -1) {
3219 attr->ComponentOverrideX = true;
3220 attr->ComponentOverrideY = true;
3221 attr->ComponentOverrideZ = true;
3222 attr->ComponentOverrideW = true;
3223 attr->ConstantSource = PRIM_ID;
3224 continue;
3225 }
3226
3227 default:
3228 break;
3229 }
3230
3231 if (sprite_coord_enables & (1 << input_index))
3232 continue;
3233
3234 /* If there was only a back color written but not front, use back
3235 * as the color instead of undefined.
3236 */
3237 if (slot == -1 && fs_attr == VARYING_SLOT_COL0)
3238 slot = vue_map->varying_to_slot[VARYING_SLOT_BFC0];
3239 if (slot == -1 && fs_attr == VARYING_SLOT_COL1)
3240 slot = vue_map->varying_to_slot[VARYING_SLOT_BFC1];
3241
3242 /* Not written by the previous stage - undefined. */
3243 if (slot == -1) {
3244 attr->ComponentOverrideX = true;
3245 attr->ComponentOverrideY = true;
3246 attr->ComponentOverrideZ = true;
3247 attr->ComponentOverrideW = true;
3248 attr->ConstantSource = CONST_0001_FLOAT;
3249 continue;
3250 }
3251
3252 /* Compute the location of the attribute relative to the read offset,
3253 * which is counted in 256-bit increments (two 128-bit VUE slots).
3254 */
3255 const int source_attr = slot - 2 * urb_read_offset;
3256 assert(source_attr >= 0 && source_attr <= 32);
3257 attr->SourceAttribute = source_attr;
3258
3259 /* If we are doing two-sided color, and the VUE slot following this one
3260 * represents a back-facing color, then we need to instruct the SF unit
3261 * to do back-facing swizzling.
3262 */
3263 if (cso_rast->light_twoside &&
3264 ((vue_map->slot_to_varying[slot] == VARYING_SLOT_COL0 &&
3265 vue_map->slot_to_varying[slot+1] == VARYING_SLOT_BFC0) ||
3266 (vue_map->slot_to_varying[slot] == VARYING_SLOT_COL1 &&
3267 vue_map->slot_to_varying[slot+1] == VARYING_SLOT_BFC1)))
3268 attr->SwizzleSelect = INPUTATTR_FACING;
3269 }
3270
3271 iris_emit_cmd(batch, GENX(3DSTATE_SBE_SWIZ), sbes) {
3272 for (int i = 0; i < 16; i++)
3273 sbes.Attribute[i] = attr_overrides[i];
3274 }
3275 }
3276
3277 static unsigned
3278 iris_calculate_point_sprite_overrides(const struct brw_wm_prog_data *prog_data,
3279 const struct iris_rasterizer_state *cso)
3280 {
3281 unsigned overrides = 0;
3282
3283 if (prog_data->urb_setup[VARYING_SLOT_PNTC] != -1)
3284 overrides |= 1 << prog_data->urb_setup[VARYING_SLOT_PNTC];
3285
3286 for (int i = 0; i < 8; i++) {
3287 if ((cso->sprite_coord_enable & (1 << i)) &&
3288 prog_data->urb_setup[VARYING_SLOT_TEX0 + i] != -1)
3289 overrides |= 1 << prog_data->urb_setup[VARYING_SLOT_TEX0 + i];
3290 }
3291
3292 return overrides;
3293 }
3294
3295 static void
3296 iris_emit_sbe(struct iris_batch *batch, const struct iris_context *ice)
3297 {
3298 const struct iris_rasterizer_state *cso_rast = ice->state.cso_rast;
3299 const struct brw_wm_prog_data *wm_prog_data = (void *)
3300 ice->shaders.prog[MESA_SHADER_FRAGMENT]->prog_data;
3301 const struct shader_info *fs_info =
3302 iris_get_shader_info(ice, MESA_SHADER_FRAGMENT);
3303
3304 unsigned urb_read_offset, urb_read_length;
3305 iris_compute_sbe_urb_read_interval(fs_info->inputs_read,
3306 ice->shaders.last_vue_map,
3307 cso_rast->light_twoside,
3308 &urb_read_offset, &urb_read_length);
3309
3310 unsigned sprite_coord_overrides =
3311 iris_calculate_point_sprite_overrides(wm_prog_data, cso_rast);
3312
3313 iris_emit_cmd(batch, GENX(3DSTATE_SBE), sbe) {
3314 sbe.AttributeSwizzleEnable = true;
3315 sbe.NumberofSFOutputAttributes = wm_prog_data->num_varying_inputs;
3316 sbe.PointSpriteTextureCoordinateOrigin = cso_rast->sprite_coord_mode;
3317 sbe.VertexURBEntryReadOffset = urb_read_offset;
3318 sbe.VertexURBEntryReadLength = urb_read_length;
3319 sbe.ForceVertexURBEntryReadOffset = true;
3320 sbe.ForceVertexURBEntryReadLength = true;
3321 sbe.ConstantInterpolationEnable = wm_prog_data->flat_inputs;
3322 sbe.PointSpriteTextureCoordinateEnable = sprite_coord_overrides;
3323 #if GEN_GEN >= 9
3324 for (int i = 0; i < 32; i++) {
3325 sbe.AttributeActiveComponentFormat[i] = ACTIVE_COMPONENT_XYZW;
3326 }
3327 #endif
3328 }
3329
3330 iris_emit_sbe_swiz(batch, ice, urb_read_offset, sprite_coord_overrides);
3331 }
3332
3333 /* ------------------------------------------------------------------- */
3334
3335 /**
3336 * Populate VS program key fields based on the current state.
3337 */
3338 static void
3339 iris_populate_vs_key(const struct iris_context *ice,
3340 const struct shader_info *info,
3341 gl_shader_stage last_stage,
3342 struct brw_vs_prog_key *key)
3343 {
3344 const struct iris_rasterizer_state *cso_rast = ice->state.cso_rast;
3345
3346 if (info->clip_distance_array_size == 0 &&
3347 (info->outputs_written & (VARYING_BIT_POS | VARYING_BIT_CLIP_VERTEX)) &&
3348 last_stage == MESA_SHADER_VERTEX)
3349 key->nr_userclip_plane_consts = cso_rast->num_clip_plane_consts;
3350 }
3351
3352 /**
3353 * Populate TCS program key fields based on the current state.
3354 */
3355 static void
3356 iris_populate_tcs_key(const struct iris_context *ice,
3357 struct brw_tcs_prog_key *key)
3358 {
3359 }
3360
3361 /**
3362 * Populate TES program key fields based on the current state.
3363 */
3364 static void
3365 iris_populate_tes_key(const struct iris_context *ice,
3366 struct brw_tes_prog_key *key)
3367 {
3368 }
3369
3370 /**
3371 * Populate GS program key fields based on the current state.
3372 */
3373 static void
3374 iris_populate_gs_key(const struct iris_context *ice,
3375 const struct shader_info *info,
3376 gl_shader_stage last_stage,
3377 struct brw_gs_prog_key *key)
3378 {
3379 const struct iris_rasterizer_state *cso_rast = ice->state.cso_rast;
3380
3381 if (info->clip_distance_array_size == 0 &&
3382 (info->outputs_written & (VARYING_BIT_POS | VARYING_BIT_CLIP_VERTEX)) &&
3383 last_stage == MESA_SHADER_GEOMETRY)
3384 key->nr_userclip_plane_consts = cso_rast->num_clip_plane_consts;
3385 }
3386
3387 /**
3388 * Populate FS program key fields based on the current state.
3389 */
3390 static void
3391 iris_populate_fs_key(const struct iris_context *ice,
3392 const struct shader_info *info,
3393 struct brw_wm_prog_key *key)
3394 {
3395 struct iris_screen *screen = (void *) ice->ctx.screen;
3396 const struct pipe_framebuffer_state *fb = &ice->state.framebuffer;
3397 const struct iris_depth_stencil_alpha_state *zsa = ice->state.cso_zsa;
3398 const struct iris_rasterizer_state *rast = ice->state.cso_rast;
3399 const struct iris_blend_state *blend = ice->state.cso_blend;
3400
3401 key->nr_color_regions = fb->nr_cbufs;
3402
3403 key->clamp_fragment_color = rast->clamp_fragment_color;
3404
3405 key->alpha_to_coverage = blend->alpha_to_coverage;
3406
3407 key->alpha_test_replicate_alpha = fb->nr_cbufs > 1 && zsa->alpha.enabled;
3408
3409 key->flat_shade = rast->flatshade &&
3410 (info->inputs_read & (VARYING_BIT_COL0 | VARYING_BIT_COL1));
3411
3412 key->persample_interp = rast->force_persample_interp;
3413 key->multisample_fbo = rast->multisample && fb->samples > 1;
3414
3415 key->coherent_fb_fetch = true;
3416
3417 key->force_dual_color_blend =
3418 screen->driconf.dual_color_blend_by_location &&
3419 (blend->blend_enables & 1) && blend->dual_color_blending;
3420
3421 /* TODO: Respect glHint for key->high_quality_derivatives */
3422 }
3423
3424 static void
3425 iris_populate_cs_key(const struct iris_context *ice,
3426 struct brw_cs_prog_key *key)
3427 {
3428 }
3429
3430 static uint64_t
3431 KSP(const struct iris_compiled_shader *shader)
3432 {
3433 struct iris_resource *res = (void *) shader->assembly.res;
3434 return iris_bo_offset_from_base_address(res->bo) + shader->assembly.offset;
3435 }
3436
3437 /* Gen11 workaround table #2056 WABTPPrefetchDisable suggests to disable
3438 * prefetching of binding tables in A0 and B0 steppings. XXX: Revisit
3439 * this WA on C0 stepping.
3440 *
3441 * TODO: Fill out SamplerCount for prefetching?
3442 */
3443
3444 #define INIT_THREAD_DISPATCH_FIELDS(pkt, prefix, stage) \
3445 pkt.KernelStartPointer = KSP(shader); \
3446 pkt.BindingTableEntryCount = GEN_GEN == 11 ? 0 : \
3447 shader->bt.size_bytes / 4; \
3448 pkt.FloatingPointMode = prog_data->use_alt_mode; \
3449 \
3450 pkt.DispatchGRFStartRegisterForURBData = \
3451 prog_data->dispatch_grf_start_reg; \
3452 pkt.prefix##URBEntryReadLength = vue_prog_data->urb_read_length; \
3453 pkt.prefix##URBEntryReadOffset = 0; \
3454 \
3455 pkt.StatisticsEnable = true; \
3456 pkt.Enable = true; \
3457 \
3458 if (prog_data->total_scratch) { \
3459 struct iris_bo *bo = \
3460 iris_get_scratch_space(ice, prog_data->total_scratch, stage); \
3461 uint32_t scratch_addr = bo->gtt_offset; \
3462 pkt.PerThreadScratchSpace = ffs(prog_data->total_scratch) - 11; \
3463 pkt.ScratchSpaceBasePointer = rw_bo(NULL, scratch_addr); \
3464 }
3465
3466 /**
3467 * Encode most of 3DSTATE_VS based on the compiled shader.
3468 */
3469 static void
3470 iris_store_vs_state(struct iris_context *ice,
3471 const struct gen_device_info *devinfo,
3472 struct iris_compiled_shader *shader)
3473 {
3474 struct brw_stage_prog_data *prog_data = shader->prog_data;
3475 struct brw_vue_prog_data *vue_prog_data = (void *) prog_data;
3476
3477 iris_pack_command(GENX(3DSTATE_VS), shader->derived_data, vs) {
3478 INIT_THREAD_DISPATCH_FIELDS(vs, Vertex, MESA_SHADER_VERTEX);
3479 vs.MaximumNumberofThreads = devinfo->max_vs_threads - 1;
3480 vs.SIMD8DispatchEnable = true;
3481 vs.UserClipDistanceCullTestEnableBitmask =
3482 vue_prog_data->cull_distance_mask;
3483 }
3484 }
3485
3486 /**
3487 * Encode most of 3DSTATE_HS based on the compiled shader.
3488 */
3489 static void
3490 iris_store_tcs_state(struct iris_context *ice,
3491 const struct gen_device_info *devinfo,
3492 struct iris_compiled_shader *shader)
3493 {
3494 struct brw_stage_prog_data *prog_data = shader->prog_data;
3495 struct brw_vue_prog_data *vue_prog_data = (void *) prog_data;
3496 struct brw_tcs_prog_data *tcs_prog_data = (void *) prog_data;
3497
3498 iris_pack_command(GENX(3DSTATE_HS), shader->derived_data, hs) {
3499 INIT_THREAD_DISPATCH_FIELDS(hs, Vertex, MESA_SHADER_TESS_CTRL);
3500
3501 hs.InstanceCount = tcs_prog_data->instances - 1;
3502 hs.MaximumNumberofThreads = devinfo->max_tcs_threads - 1;
3503 hs.IncludeVertexHandles = true;
3504
3505 #if GEN_GEN >= 9
3506 hs.DispatchMode = vue_prog_data->dispatch_mode;
3507 hs.IncludePrimitiveID = tcs_prog_data->include_primitive_id;
3508 #endif
3509 }
3510 }
3511
3512 /**
3513 * Encode 3DSTATE_TE and most of 3DSTATE_DS based on the compiled shader.
3514 */
3515 static void
3516 iris_store_tes_state(struct iris_context *ice,
3517 const struct gen_device_info *devinfo,
3518 struct iris_compiled_shader *shader)
3519 {
3520 struct brw_stage_prog_data *prog_data = shader->prog_data;
3521 struct brw_vue_prog_data *vue_prog_data = (void *) prog_data;
3522 struct brw_tes_prog_data *tes_prog_data = (void *) prog_data;
3523
3524 uint32_t *te_state = (void *) shader->derived_data;
3525 uint32_t *ds_state = te_state + GENX(3DSTATE_TE_length);
3526
3527 iris_pack_command(GENX(3DSTATE_TE), te_state, te) {
3528 te.Partitioning = tes_prog_data->partitioning;
3529 te.OutputTopology = tes_prog_data->output_topology;
3530 te.TEDomain = tes_prog_data->domain;
3531 te.TEEnable = true;
3532 te.MaximumTessellationFactorOdd = 63.0;
3533 te.MaximumTessellationFactorNotOdd = 64.0;
3534 }
3535
3536 iris_pack_command(GENX(3DSTATE_DS), ds_state, ds) {
3537 INIT_THREAD_DISPATCH_FIELDS(ds, Patch, MESA_SHADER_TESS_EVAL);
3538
3539 ds.DispatchMode = DISPATCH_MODE_SIMD8_SINGLE_PATCH;
3540 ds.MaximumNumberofThreads = devinfo->max_tes_threads - 1;
3541 ds.ComputeWCoordinateEnable =
3542 tes_prog_data->domain == BRW_TESS_DOMAIN_TRI;
3543
3544 ds.UserClipDistanceCullTestEnableBitmask =
3545 vue_prog_data->cull_distance_mask;
3546 }
3547
3548 }
3549
3550 /**
3551 * Encode most of 3DSTATE_GS based on the compiled shader.
3552 */
3553 static void
3554 iris_store_gs_state(struct iris_context *ice,
3555 const struct gen_device_info *devinfo,
3556 struct iris_compiled_shader *shader)
3557 {
3558 struct brw_stage_prog_data *prog_data = shader->prog_data;
3559 struct brw_vue_prog_data *vue_prog_data = (void *) prog_data;
3560 struct brw_gs_prog_data *gs_prog_data = (void *) prog_data;
3561
3562 iris_pack_command(GENX(3DSTATE_GS), shader->derived_data, gs) {
3563 INIT_THREAD_DISPATCH_FIELDS(gs, Vertex, MESA_SHADER_GEOMETRY);
3564
3565 gs.OutputVertexSize = gs_prog_data->output_vertex_size_hwords * 2 - 1;
3566 gs.OutputTopology = gs_prog_data->output_topology;
3567 gs.ControlDataHeaderSize =
3568 gs_prog_data->control_data_header_size_hwords;
3569 gs.InstanceControl = gs_prog_data->invocations - 1;
3570 gs.DispatchMode = DISPATCH_MODE_SIMD8;
3571 gs.IncludePrimitiveID = gs_prog_data->include_primitive_id;
3572 gs.ControlDataFormat = gs_prog_data->control_data_format;
3573 gs.ReorderMode = TRAILING;
3574 gs.ExpectedVertexCount = gs_prog_data->vertices_in;
3575 gs.MaximumNumberofThreads =
3576 GEN_GEN == 8 ? (devinfo->max_gs_threads / 2 - 1)
3577 : (devinfo->max_gs_threads - 1);
3578
3579 if (gs_prog_data->static_vertex_count != -1) {
3580 gs.StaticOutput = true;
3581 gs.StaticOutputVertexCount = gs_prog_data->static_vertex_count;
3582 }
3583 gs.IncludeVertexHandles = vue_prog_data->include_vue_handles;
3584
3585 gs.UserClipDistanceCullTestEnableBitmask =
3586 vue_prog_data->cull_distance_mask;
3587
3588 const int urb_entry_write_offset = 1;
3589 const uint32_t urb_entry_output_length =
3590 DIV_ROUND_UP(vue_prog_data->vue_map.num_slots, 2) -
3591 urb_entry_write_offset;
3592
3593 gs.VertexURBEntryOutputReadOffset = urb_entry_write_offset;
3594 gs.VertexURBEntryOutputLength = MAX2(urb_entry_output_length, 1);
3595 }
3596 }
3597
3598 /**
3599 * Encode most of 3DSTATE_PS and 3DSTATE_PS_EXTRA based on the shader.
3600 */
3601 static void
3602 iris_store_fs_state(struct iris_context *ice,
3603 const struct gen_device_info *devinfo,
3604 struct iris_compiled_shader *shader)
3605 {
3606 struct brw_stage_prog_data *prog_data = shader->prog_data;
3607 struct brw_wm_prog_data *wm_prog_data = (void *) shader->prog_data;
3608
3609 uint32_t *ps_state = (void *) shader->derived_data;
3610 uint32_t *psx_state = ps_state + GENX(3DSTATE_PS_length);
3611
3612 iris_pack_command(GENX(3DSTATE_PS), ps_state, ps) {
3613 ps.VectorMaskEnable = true;
3614 // XXX: WABTPPrefetchDisable, see above, drop at C0
3615 ps.BindingTableEntryCount = GEN_GEN == 11 ? 0 :
3616 shader->bt.size_bytes / 4;
3617 ps.FloatingPointMode = prog_data->use_alt_mode;
3618 ps.MaximumNumberofThreadsPerPSD = 64 - (GEN_GEN == 8 ? 2 : 1);
3619
3620 ps.PushConstantEnable = prog_data->ubo_ranges[0].length > 0;
3621
3622 /* From the documentation for this packet:
3623 * "If the PS kernel does not need the Position XY Offsets to
3624 * compute a Position Value, then this field should be programmed
3625 * to POSOFFSET_NONE."
3626 *
3627 * "SW Recommendation: If the PS kernel needs the Position Offsets
3628 * to compute a Position XY value, this field should match Position
3629 * ZW Interpolation Mode to ensure a consistent position.xyzw
3630 * computation."
3631 *
3632 * We only require XY sample offsets. So, this recommendation doesn't
3633 * look useful at the moment. We might need this in future.
3634 */
3635 ps.PositionXYOffsetSelect =
3636 wm_prog_data->uses_pos_offset ? POSOFFSET_SAMPLE : POSOFFSET_NONE;
3637 ps._8PixelDispatchEnable = wm_prog_data->dispatch_8;
3638 ps._16PixelDispatchEnable = wm_prog_data->dispatch_16;
3639 /* ps._32PixelDispatchEnable is filled in at draw time. */
3640
3641 ps.DispatchGRFStartRegisterForConstantSetupData0 =
3642 brw_wm_prog_data_dispatch_grf_start_reg(wm_prog_data, ps, 0);
3643 ps.DispatchGRFStartRegisterForConstantSetupData1 =
3644 brw_wm_prog_data_dispatch_grf_start_reg(wm_prog_data, ps, 1);
3645 ps.DispatchGRFStartRegisterForConstantSetupData2 =
3646 brw_wm_prog_data_dispatch_grf_start_reg(wm_prog_data, ps, 2);
3647
3648 ps.KernelStartPointer0 =
3649 KSP(shader) + brw_wm_prog_data_prog_offset(wm_prog_data, ps, 0);
3650 ps.KernelStartPointer1 =
3651 KSP(shader) + brw_wm_prog_data_prog_offset(wm_prog_data, ps, 1);
3652 ps.KernelStartPointer2 =
3653 KSP(shader) + brw_wm_prog_data_prog_offset(wm_prog_data, ps, 2);
3654
3655 if (prog_data->total_scratch) {
3656 struct iris_bo *bo =
3657 iris_get_scratch_space(ice, prog_data->total_scratch,
3658 MESA_SHADER_FRAGMENT);
3659 uint32_t scratch_addr = bo->gtt_offset;
3660 ps.PerThreadScratchSpace = ffs(prog_data->total_scratch) - 11;
3661 ps.ScratchSpaceBasePointer = rw_bo(NULL, scratch_addr);
3662 }
3663 }
3664
3665 iris_pack_command(GENX(3DSTATE_PS_EXTRA), psx_state, psx) {
3666 psx.PixelShaderValid = true;
3667 psx.PixelShaderComputedDepthMode = wm_prog_data->computed_depth_mode;
3668 psx.PixelShaderKillsPixel = wm_prog_data->uses_kill;
3669 psx.AttributeEnable = wm_prog_data->num_varying_inputs != 0;
3670 psx.PixelShaderUsesSourceDepth = wm_prog_data->uses_src_depth;
3671 psx.PixelShaderUsesSourceW = wm_prog_data->uses_src_w;
3672 psx.PixelShaderIsPerSample = wm_prog_data->persample_dispatch;
3673 psx.oMaskPresenttoRenderTarget = wm_prog_data->uses_omask;
3674
3675 #if GEN_GEN >= 9
3676 psx.PixelShaderPullsBary = wm_prog_data->pulls_bary;
3677 psx.PixelShaderComputesStencil = wm_prog_data->computed_stencil;
3678 #else
3679 psx.PixelShaderUsesInputCoverageMask = wm_prog_data->uses_sample_mask;
3680 #endif
3681 }
3682 }
3683
3684 /**
3685 * Compute the size of the derived data (shader command packets).
3686 *
3687 * This must match the data written by the iris_store_xs_state() functions.
3688 */
3689 static void
3690 iris_store_cs_state(struct iris_context *ice,
3691 const struct gen_device_info *devinfo,
3692 struct iris_compiled_shader *shader)
3693 {
3694 struct brw_stage_prog_data *prog_data = shader->prog_data;
3695 struct brw_cs_prog_data *cs_prog_data = (void *) shader->prog_data;
3696 void *map = shader->derived_data;
3697
3698 iris_pack_state(GENX(INTERFACE_DESCRIPTOR_DATA), map, desc) {
3699 desc.KernelStartPointer = KSP(shader);
3700 desc.ConstantURBEntryReadLength = cs_prog_data->push.per_thread.regs;
3701 desc.NumberofThreadsinGPGPUThreadGroup = cs_prog_data->threads;
3702 desc.SharedLocalMemorySize =
3703 encode_slm_size(GEN_GEN, prog_data->total_shared);
3704 desc.BarrierEnable = cs_prog_data->uses_barrier;
3705 desc.CrossThreadConstantDataReadLength =
3706 cs_prog_data->push.cross_thread.regs;
3707 }
3708 }
3709
3710 static unsigned
3711 iris_derived_program_state_size(enum iris_program_cache_id cache_id)
3712 {
3713 assert(cache_id <= IRIS_CACHE_BLORP);
3714
3715 static const unsigned dwords[] = {
3716 [IRIS_CACHE_VS] = GENX(3DSTATE_VS_length),
3717 [IRIS_CACHE_TCS] = GENX(3DSTATE_HS_length),
3718 [IRIS_CACHE_TES] = GENX(3DSTATE_TE_length) + GENX(3DSTATE_DS_length),
3719 [IRIS_CACHE_GS] = GENX(3DSTATE_GS_length),
3720 [IRIS_CACHE_FS] =
3721 GENX(3DSTATE_PS_length) + GENX(3DSTATE_PS_EXTRA_length),
3722 [IRIS_CACHE_CS] = GENX(INTERFACE_DESCRIPTOR_DATA_length),
3723 [IRIS_CACHE_BLORP] = 0,
3724 };
3725
3726 return sizeof(uint32_t) * dwords[cache_id];
3727 }
3728
3729 /**
3730 * Create any state packets corresponding to the given shader stage
3731 * (i.e. 3DSTATE_VS) and save them as "derived data" in the shader variant.
3732 * This means that we can look up a program in the in-memory cache and
3733 * get most of the state packet without having to reconstruct it.
3734 */
3735 static void
3736 iris_store_derived_program_state(struct iris_context *ice,
3737 enum iris_program_cache_id cache_id,
3738 struct iris_compiled_shader *shader)
3739 {
3740 struct iris_screen *screen = (void *) ice->ctx.screen;
3741 const struct gen_device_info *devinfo = &screen->devinfo;
3742
3743 switch (cache_id) {
3744 case IRIS_CACHE_VS:
3745 iris_store_vs_state(ice, devinfo, shader);
3746 break;
3747 case IRIS_CACHE_TCS:
3748 iris_store_tcs_state(ice, devinfo, shader);
3749 break;
3750 case IRIS_CACHE_TES:
3751 iris_store_tes_state(ice, devinfo, shader);
3752 break;
3753 case IRIS_CACHE_GS:
3754 iris_store_gs_state(ice, devinfo, shader);
3755 break;
3756 case IRIS_CACHE_FS:
3757 iris_store_fs_state(ice, devinfo, shader);
3758 break;
3759 case IRIS_CACHE_CS:
3760 iris_store_cs_state(ice, devinfo, shader);
3761 case IRIS_CACHE_BLORP:
3762 break;
3763 default:
3764 break;
3765 }
3766 }
3767
3768 /* ------------------------------------------------------------------- */
3769
3770 static const uint32_t push_constant_opcodes[] = {
3771 [MESA_SHADER_VERTEX] = 21,
3772 [MESA_SHADER_TESS_CTRL] = 25, /* HS */
3773 [MESA_SHADER_TESS_EVAL] = 26, /* DS */
3774 [MESA_SHADER_GEOMETRY] = 22,
3775 [MESA_SHADER_FRAGMENT] = 23,
3776 [MESA_SHADER_COMPUTE] = 0,
3777 };
3778
3779 static uint32_t
3780 use_null_surface(struct iris_batch *batch, struct iris_context *ice)
3781 {
3782 struct iris_bo *state_bo = iris_resource_bo(ice->state.unbound_tex.res);
3783
3784 iris_use_pinned_bo(batch, state_bo, false);
3785
3786 return ice->state.unbound_tex.offset;
3787 }
3788
3789 static uint32_t
3790 use_null_fb_surface(struct iris_batch *batch, struct iris_context *ice)
3791 {
3792 /* If set_framebuffer_state() was never called, fall back to 1x1x1 */
3793 if (!ice->state.null_fb.res)
3794 return use_null_surface(batch, ice);
3795
3796 struct iris_bo *state_bo = iris_resource_bo(ice->state.null_fb.res);
3797
3798 iris_use_pinned_bo(batch, state_bo, false);
3799
3800 return ice->state.null_fb.offset;
3801 }
3802
3803 static uint32_t
3804 surf_state_offset_for_aux(struct iris_resource *res,
3805 unsigned aux_modes,
3806 enum isl_aux_usage aux_usage)
3807 {
3808 return SURFACE_STATE_ALIGNMENT *
3809 util_bitcount(res->aux.possible_usages & ((1 << aux_usage) - 1));
3810 }
3811
3812 static void
3813 surf_state_update_clear_value(struct iris_batch *batch,
3814 struct iris_resource *res,
3815 struct iris_state_ref *state,
3816 unsigned aux_modes,
3817 enum isl_aux_usage aux_usage)
3818 {
3819 struct isl_device *isl_dev = &batch->screen->isl_dev;
3820 struct iris_bo *state_bo = iris_resource_bo(state->res);
3821 uint64_t real_offset = state->offset +
3822 IRIS_MEMZONE_BINDER_START;
3823 uint32_t offset_into_bo = real_offset - state_bo->gtt_offset;
3824 uint32_t clear_offset = offset_into_bo +
3825 isl_dev->ss.clear_value_offset +
3826 surf_state_offset_for_aux(res, aux_modes, aux_usage);
3827
3828 batch->vtbl->copy_mem_mem(batch, state_bo, clear_offset,
3829 res->aux.clear_color_bo,
3830 res->aux.clear_color_offset,
3831 isl_dev->ss.clear_value_size);
3832 }
3833
3834 static void
3835 update_clear_value(struct iris_context *ice,
3836 struct iris_batch *batch,
3837 struct iris_resource *res,
3838 struct iris_state_ref *state,
3839 unsigned aux_modes,
3840 struct isl_view *view)
3841 {
3842 struct iris_screen *screen = batch->screen;
3843 const struct gen_device_info *devinfo = &screen->devinfo;
3844
3845 /* We only need to update the clear color in the surface state for gen8 and
3846 * gen9. Newer gens can read it directly from the clear color state buffer.
3847 */
3848 if (devinfo->gen > 9)
3849 return;
3850
3851 if (devinfo->gen == 9) {
3852 /* Skip updating the ISL_AUX_USAGE_NONE surface state */
3853 aux_modes &= ~(1 << ISL_AUX_USAGE_NONE);
3854
3855 while (aux_modes) {
3856 enum isl_aux_usage aux_usage = u_bit_scan(&aux_modes);
3857
3858 surf_state_update_clear_value(batch, res, state, aux_modes,
3859 aux_usage);
3860 }
3861 } else if (devinfo->gen == 8) {
3862 pipe_resource_reference(&state->res, NULL);
3863 void *map = alloc_surface_states(ice->state.surface_uploader,
3864 state, res->aux.possible_usages);
3865 while (aux_modes) {
3866 enum isl_aux_usage aux_usage = u_bit_scan(&aux_modes);
3867 fill_surface_state(&screen->isl_dev, map, res, view, aux_usage);
3868 map += SURFACE_STATE_ALIGNMENT;
3869 }
3870 }
3871 }
3872
3873 /**
3874 * Add a surface to the validation list, as well as the buffer containing
3875 * the corresponding SURFACE_STATE.
3876 *
3877 * Returns the binding table entry (offset to SURFACE_STATE).
3878 */
3879 static uint32_t
3880 use_surface(struct iris_context *ice,
3881 struct iris_batch *batch,
3882 struct pipe_surface *p_surf,
3883 bool writeable,
3884 enum isl_aux_usage aux_usage)
3885 {
3886 struct iris_surface *surf = (void *) p_surf;
3887 struct iris_resource *res = (void *) p_surf->texture;
3888
3889 iris_use_pinned_bo(batch, iris_resource_bo(p_surf->texture), writeable);
3890 iris_use_pinned_bo(batch, iris_resource_bo(surf->surface_state.res), false);
3891
3892 if (res->aux.bo) {
3893 iris_use_pinned_bo(batch, res->aux.bo, writeable);
3894 if (res->aux.clear_color_bo)
3895 iris_use_pinned_bo(batch, res->aux.clear_color_bo, false);
3896
3897 if (memcmp(&res->aux.clear_color, &surf->clear_color,
3898 sizeof(surf->clear_color)) != 0) {
3899 update_clear_value(ice, batch, res, &surf->surface_state,
3900 res->aux.possible_usages, &surf->view);
3901 surf->clear_color = res->aux.clear_color;
3902 }
3903 }
3904
3905 return surf->surface_state.offset +
3906 surf_state_offset_for_aux(res, res->aux.possible_usages, aux_usage);
3907 }
3908
3909 static uint32_t
3910 use_sampler_view(struct iris_context *ice,
3911 struct iris_batch *batch,
3912 struct iris_sampler_view *isv)
3913 {
3914 // XXX: ASTC hacks
3915 enum isl_aux_usage aux_usage =
3916 iris_resource_texture_aux_usage(ice, isv->res, isv->view.format, 0);
3917
3918 iris_use_pinned_bo(batch, isv->res->bo, false);
3919 iris_use_pinned_bo(batch, iris_resource_bo(isv->surface_state.res), false);
3920
3921 if (isv->res->aux.bo) {
3922 iris_use_pinned_bo(batch, isv->res->aux.bo, false);
3923 if (isv->res->aux.clear_color_bo)
3924 iris_use_pinned_bo(batch, isv->res->aux.clear_color_bo, false);
3925 if (memcmp(&isv->res->aux.clear_color, &isv->clear_color,
3926 sizeof(isv->clear_color)) != 0) {
3927 update_clear_value(ice, batch, isv->res, &isv->surface_state,
3928 isv->res->aux.sampler_usages, &isv->view);
3929 isv->clear_color = isv->res->aux.clear_color;
3930 }
3931 }
3932
3933 return isv->surface_state.offset +
3934 surf_state_offset_for_aux(isv->res, isv->res->aux.sampler_usages,
3935 aux_usage);
3936 }
3937
3938 static uint32_t
3939 use_ubo_ssbo(struct iris_batch *batch,
3940 struct iris_context *ice,
3941 struct pipe_shader_buffer *buf,
3942 struct iris_state_ref *surf_state,
3943 bool writable)
3944 {
3945 if (!buf->buffer)
3946 return use_null_surface(batch, ice);
3947
3948 iris_use_pinned_bo(batch, iris_resource_bo(buf->buffer), writable);
3949 iris_use_pinned_bo(batch, iris_resource_bo(surf_state->res), false);
3950
3951 return surf_state->offset;
3952 }
3953
3954 static uint32_t
3955 use_image(struct iris_batch *batch, struct iris_context *ice,
3956 struct iris_shader_state *shs, int i)
3957 {
3958 struct iris_image_view *iv = &shs->image[i];
3959 struct iris_resource *res = (void *) iv->base.resource;
3960
3961 if (!res)
3962 return use_null_surface(batch, ice);
3963
3964 bool write = iv->base.shader_access & PIPE_IMAGE_ACCESS_WRITE;
3965
3966 iris_use_pinned_bo(batch, res->bo, write);
3967 iris_use_pinned_bo(batch, iris_resource_bo(iv->surface_state.res), false);
3968
3969 if (res->aux.bo)
3970 iris_use_pinned_bo(batch, res->aux.bo, write);
3971
3972 return iv->surface_state.offset;
3973 }
3974
3975 #define push_bt_entry(addr) \
3976 assert(addr >= binder_addr); \
3977 assert(s < shader->bt.size_bytes / sizeof(uint32_t)); \
3978 if (!pin_only) bt_map[s++] = (addr) - binder_addr;
3979
3980 #define bt_assert(section) \
3981 if (!pin_only && shader->bt.used_mask[section] != 0) \
3982 assert(shader->bt.offsets[section] == s);
3983
3984 /**
3985 * Populate the binding table for a given shader stage.
3986 *
3987 * This fills out the table of pointers to surfaces required by the shader,
3988 * and also adds those buffers to the validation list so the kernel can make
3989 * resident before running our batch.
3990 */
3991 static void
3992 iris_populate_binding_table(struct iris_context *ice,
3993 struct iris_batch *batch,
3994 gl_shader_stage stage,
3995 bool pin_only)
3996 {
3997 const struct iris_binder *binder = &ice->state.binder;
3998 struct iris_uncompiled_shader *ish = ice->shaders.uncompiled[stage];
3999 struct iris_compiled_shader *shader = ice->shaders.prog[stage];
4000 if (!shader)
4001 return;
4002
4003 struct iris_binding_table *bt = &shader->bt;
4004 UNUSED struct brw_stage_prog_data *prog_data = shader->prog_data;
4005 struct iris_shader_state *shs = &ice->state.shaders[stage];
4006 uint32_t binder_addr = binder->bo->gtt_offset;
4007
4008 uint32_t *bt_map = binder->map + binder->bt_offset[stage];
4009 int s = 0;
4010
4011 const struct shader_info *info = iris_get_shader_info(ice, stage);
4012 if (!info) {
4013 /* TCS passthrough doesn't need a binding table. */
4014 assert(stage == MESA_SHADER_TESS_CTRL);
4015 return;
4016 }
4017
4018 if (stage == MESA_SHADER_COMPUTE &&
4019 shader->bt.used_mask[IRIS_SURFACE_GROUP_CS_WORK_GROUPS]) {
4020 /* surface for gl_NumWorkGroups */
4021 struct iris_state_ref *grid_data = &ice->state.grid_size;
4022 struct iris_state_ref *grid_state = &ice->state.grid_surf_state;
4023 iris_use_pinned_bo(batch, iris_resource_bo(grid_data->res), false);
4024 iris_use_pinned_bo(batch, iris_resource_bo(grid_state->res), false);
4025 push_bt_entry(grid_state->offset);
4026 }
4027
4028 if (stage == MESA_SHADER_FRAGMENT) {
4029 struct pipe_framebuffer_state *cso_fb = &ice->state.framebuffer;
4030 /* Note that cso_fb->nr_cbufs == fs_key->nr_color_regions. */
4031 if (cso_fb->nr_cbufs) {
4032 for (unsigned i = 0; i < cso_fb->nr_cbufs; i++) {
4033 uint32_t addr;
4034 if (cso_fb->cbufs[i]) {
4035 addr = use_surface(ice, batch, cso_fb->cbufs[i], true,
4036 ice->state.draw_aux_usage[i]);
4037 } else {
4038 addr = use_null_fb_surface(batch, ice);
4039 }
4040 push_bt_entry(addr);
4041 }
4042 } else {
4043 uint32_t addr = use_null_fb_surface(batch, ice);
4044 push_bt_entry(addr);
4045 }
4046 }
4047
4048 #define foreach_surface_used(index, group) \
4049 bt_assert(group); \
4050 for (int index = 0; index < bt->sizes[group]; index++) \
4051 if (iris_group_index_to_bti(bt, group, index) != \
4052 IRIS_SURFACE_NOT_USED)
4053
4054 foreach_surface_used(i, IRIS_SURFACE_GROUP_TEXTURE) {
4055 struct iris_sampler_view *view = shs->textures[i];
4056 uint32_t addr = view ? use_sampler_view(ice, batch, view)
4057 : use_null_surface(batch, ice);
4058 push_bt_entry(addr);
4059 }
4060
4061 foreach_surface_used(i, IRIS_SURFACE_GROUP_IMAGE) {
4062 uint32_t addr = use_image(batch, ice, shs, i);
4063 push_bt_entry(addr);
4064 }
4065
4066 foreach_surface_used(i, IRIS_SURFACE_GROUP_UBO) {
4067 uint32_t addr;
4068
4069 if (i == bt->sizes[IRIS_SURFACE_GROUP_UBO] - 1) {
4070 if (ish->const_data) {
4071 iris_use_pinned_bo(batch, iris_resource_bo(ish->const_data), false);
4072 iris_use_pinned_bo(batch, iris_resource_bo(ish->const_data_state.res),
4073 false);
4074 addr = ish->const_data_state.offset;
4075 } else {
4076 /* This can only happen with INTEL_DISABLE_COMPACT_BINDING_TABLE=1. */
4077 addr = use_null_surface(batch, ice);
4078 }
4079 } else {
4080 addr = use_ubo_ssbo(batch, ice, &shs->constbuf[i],
4081 &shs->constbuf_surf_state[i], false);
4082 }
4083
4084 push_bt_entry(addr);
4085 }
4086
4087 foreach_surface_used(i, IRIS_SURFACE_GROUP_SSBO) {
4088 uint32_t addr =
4089 use_ubo_ssbo(batch, ice, &shs->ssbo[i], &shs->ssbo_surf_state[i],
4090 shs->writable_ssbos & (1u << i));
4091 push_bt_entry(addr);
4092 }
4093
4094 #if 0
4095 /* XXX: YUV surfaces not implemented yet */
4096 bt_assert(plane_start[1], ...);
4097 bt_assert(plane_start[2], ...);
4098 #endif
4099 }
4100
4101 static void
4102 iris_use_optional_res(struct iris_batch *batch,
4103 struct pipe_resource *res,
4104 bool writeable)
4105 {
4106 if (res) {
4107 struct iris_bo *bo = iris_resource_bo(res);
4108 iris_use_pinned_bo(batch, bo, writeable);
4109 }
4110 }
4111
4112 static void
4113 pin_depth_and_stencil_buffers(struct iris_batch *batch,
4114 struct pipe_surface *zsbuf,
4115 struct iris_depth_stencil_alpha_state *cso_zsa)
4116 {
4117 if (!zsbuf)
4118 return;
4119
4120 struct iris_resource *zres, *sres;
4121 iris_get_depth_stencil_resources(zsbuf->texture, &zres, &sres);
4122
4123 if (zres) {
4124 iris_use_pinned_bo(batch, zres->bo, cso_zsa->depth_writes_enabled);
4125 if (zres->aux.bo) {
4126 iris_use_pinned_bo(batch, zres->aux.bo,
4127 cso_zsa->depth_writes_enabled);
4128 }
4129 }
4130
4131 if (sres) {
4132 iris_use_pinned_bo(batch, sres->bo, cso_zsa->stencil_writes_enabled);
4133 }
4134 }
4135
4136 /* ------------------------------------------------------------------- */
4137
4138 /**
4139 * Pin any BOs which were installed by a previous batch, and restored
4140 * via the hardware logical context mechanism.
4141 *
4142 * We don't need to re-emit all state every batch - the hardware context
4143 * mechanism will save and restore it for us. This includes pointers to
4144 * various BOs...which won't exist unless we ask the kernel to pin them
4145 * by adding them to the validation list.
4146 *
4147 * We can skip buffers if we've re-emitted those packets, as we're
4148 * overwriting those stale pointers with new ones, and don't actually
4149 * refer to the old BOs.
4150 */
4151 static void
4152 iris_restore_render_saved_bos(struct iris_context *ice,
4153 struct iris_batch *batch,
4154 const struct pipe_draw_info *draw)
4155 {
4156 struct iris_genx_state *genx = ice->state.genx;
4157
4158 const uint64_t clean = ~ice->state.dirty;
4159
4160 if (clean & IRIS_DIRTY_CC_VIEWPORT) {
4161 iris_use_optional_res(batch, ice->state.last_res.cc_vp, false);
4162 }
4163
4164 if (clean & IRIS_DIRTY_SF_CL_VIEWPORT) {
4165 iris_use_optional_res(batch, ice->state.last_res.sf_cl_vp, false);
4166 }
4167
4168 if (clean & IRIS_DIRTY_BLEND_STATE) {
4169 iris_use_optional_res(batch, ice->state.last_res.blend, false);
4170 }
4171
4172 if (clean & IRIS_DIRTY_COLOR_CALC_STATE) {
4173 iris_use_optional_res(batch, ice->state.last_res.color_calc, false);
4174 }
4175
4176 if (clean & IRIS_DIRTY_SCISSOR_RECT) {
4177 iris_use_optional_res(batch, ice->state.last_res.scissor, false);
4178 }
4179
4180 if (ice->state.streamout_active && (clean & IRIS_DIRTY_SO_BUFFERS)) {
4181 for (int i = 0; i < 4; i++) {
4182 struct iris_stream_output_target *tgt =
4183 (void *) ice->state.so_target[i];
4184 if (tgt) {
4185 iris_use_pinned_bo(batch, iris_resource_bo(tgt->base.buffer),
4186 true);
4187 iris_use_pinned_bo(batch, iris_resource_bo(tgt->offset.res),
4188 true);
4189 }
4190 }
4191 }
4192
4193 for (int stage = 0; stage <= MESA_SHADER_FRAGMENT; stage++) {
4194 if (!(clean & (IRIS_DIRTY_CONSTANTS_VS << stage)))
4195 continue;
4196
4197 struct iris_shader_state *shs = &ice->state.shaders[stage];
4198 struct iris_compiled_shader *shader = ice->shaders.prog[stage];
4199
4200 if (!shader)
4201 continue;
4202
4203 struct brw_stage_prog_data *prog_data = (void *) shader->prog_data;
4204
4205 for (int i = 0; i < 4; i++) {
4206 const struct brw_ubo_range *range = &prog_data->ubo_ranges[i];
4207
4208 if (range->length == 0)
4209 continue;
4210
4211 /* Range block is a binding table index, map back to UBO index. */
4212 unsigned block_index = iris_bti_to_group_index(
4213 &shader->bt, IRIS_SURFACE_GROUP_UBO, range->block);
4214 assert(block_index != IRIS_SURFACE_NOT_USED);
4215
4216 struct pipe_shader_buffer *cbuf = &shs->constbuf[block_index];
4217 struct iris_resource *res = (void *) cbuf->buffer;
4218
4219 if (res)
4220 iris_use_pinned_bo(batch, res->bo, false);
4221 else
4222 iris_use_pinned_bo(batch, batch->screen->workaround_bo, false);
4223 }
4224 }
4225
4226 for (int stage = 0; stage <= MESA_SHADER_FRAGMENT; stage++) {
4227 if (clean & (IRIS_DIRTY_BINDINGS_VS << stage)) {
4228 /* Re-pin any buffers referred to by the binding table. */
4229 iris_populate_binding_table(ice, batch, stage, true);
4230 }
4231 }
4232
4233 for (int stage = 0; stage <= MESA_SHADER_FRAGMENT; stage++) {
4234 struct iris_shader_state *shs = &ice->state.shaders[stage];
4235 struct pipe_resource *res = shs->sampler_table.res;
4236 if (res)
4237 iris_use_pinned_bo(batch, iris_resource_bo(res), false);
4238 }
4239
4240 for (int stage = 0; stage <= MESA_SHADER_FRAGMENT; stage++) {
4241 if (clean & (IRIS_DIRTY_VS << stage)) {
4242 struct iris_compiled_shader *shader = ice->shaders.prog[stage];
4243
4244 if (shader) {
4245 struct iris_bo *bo = iris_resource_bo(shader->assembly.res);
4246 iris_use_pinned_bo(batch, bo, false);
4247
4248 struct brw_stage_prog_data *prog_data = shader->prog_data;
4249
4250 if (prog_data->total_scratch > 0) {
4251 struct iris_bo *bo =
4252 iris_get_scratch_space(ice, prog_data->total_scratch, stage);
4253 iris_use_pinned_bo(batch, bo, true);
4254 }
4255 }
4256 }
4257 }
4258
4259 if ((clean & IRIS_DIRTY_DEPTH_BUFFER) &&
4260 (clean & IRIS_DIRTY_WM_DEPTH_STENCIL)) {
4261 struct pipe_framebuffer_state *cso_fb = &ice->state.framebuffer;
4262 pin_depth_and_stencil_buffers(batch, cso_fb->zsbuf, ice->state.cso_zsa);
4263 }
4264
4265 iris_use_optional_res(batch, ice->state.last_res.index_buffer, false);
4266
4267 if (clean & IRIS_DIRTY_VERTEX_BUFFERS) {
4268 uint64_t bound = ice->state.bound_vertex_buffers;
4269 while (bound) {
4270 const int i = u_bit_scan64(&bound);
4271 struct pipe_resource *res = genx->vertex_buffers[i].resource;
4272 iris_use_pinned_bo(batch, iris_resource_bo(res), false);
4273 }
4274 }
4275 }
4276
4277 static void
4278 iris_restore_compute_saved_bos(struct iris_context *ice,
4279 struct iris_batch *batch,
4280 const struct pipe_grid_info *grid)
4281 {
4282 const uint64_t clean = ~ice->state.dirty;
4283
4284 const int stage = MESA_SHADER_COMPUTE;
4285 struct iris_shader_state *shs = &ice->state.shaders[stage];
4286
4287 if (clean & IRIS_DIRTY_BINDINGS_CS) {
4288 /* Re-pin any buffers referred to by the binding table. */
4289 iris_populate_binding_table(ice, batch, stage, true);
4290 }
4291
4292 struct pipe_resource *sampler_res = shs->sampler_table.res;
4293 if (sampler_res)
4294 iris_use_pinned_bo(batch, iris_resource_bo(sampler_res), false);
4295
4296 if ((clean & IRIS_DIRTY_SAMPLER_STATES_CS) &&
4297 (clean & IRIS_DIRTY_BINDINGS_CS) &&
4298 (clean & IRIS_DIRTY_CONSTANTS_CS) &&
4299 (clean & IRIS_DIRTY_CS)) {
4300 iris_use_optional_res(batch, ice->state.last_res.cs_desc, false);
4301 }
4302
4303 if (clean & IRIS_DIRTY_CS) {
4304 struct iris_compiled_shader *shader = ice->shaders.prog[stage];
4305
4306 if (shader) {
4307 struct iris_bo *bo = iris_resource_bo(shader->assembly.res);
4308 iris_use_pinned_bo(batch, bo, false);
4309
4310 struct iris_bo *curbe_bo =
4311 iris_resource_bo(ice->state.last_res.cs_thread_ids);
4312 iris_use_pinned_bo(batch, curbe_bo, false);
4313
4314 struct brw_stage_prog_data *prog_data = shader->prog_data;
4315
4316 if (prog_data->total_scratch > 0) {
4317 struct iris_bo *bo =
4318 iris_get_scratch_space(ice, prog_data->total_scratch, stage);
4319 iris_use_pinned_bo(batch, bo, true);
4320 }
4321 }
4322 }
4323 }
4324
4325 /**
4326 * Possibly emit STATE_BASE_ADDRESS to update Surface State Base Address.
4327 */
4328 static void
4329 iris_update_surface_base_address(struct iris_batch *batch,
4330 struct iris_binder *binder)
4331 {
4332 if (batch->last_surface_base_address == binder->bo->gtt_offset)
4333 return;
4334
4335 flush_for_state_base_change(batch);
4336
4337 iris_emit_cmd(batch, GENX(STATE_BASE_ADDRESS), sba) {
4338 sba.SurfaceStateMOCS = MOCS_WB;
4339 sba.SurfaceStateBaseAddressModifyEnable = true;
4340 sba.SurfaceStateBaseAddress = ro_bo(binder->bo, 0);
4341 }
4342
4343 batch->last_surface_base_address = binder->bo->gtt_offset;
4344 }
4345
4346 static void
4347 iris_upload_dirty_render_state(struct iris_context *ice,
4348 struct iris_batch *batch,
4349 const struct pipe_draw_info *draw)
4350 {
4351 const uint64_t dirty = ice->state.dirty;
4352
4353 if (!(dirty & IRIS_ALL_DIRTY_FOR_RENDER))
4354 return;
4355
4356 struct iris_genx_state *genx = ice->state.genx;
4357 struct iris_binder *binder = &ice->state.binder;
4358 struct brw_wm_prog_data *wm_prog_data = (void *)
4359 ice->shaders.prog[MESA_SHADER_FRAGMENT]->prog_data;
4360
4361 if (dirty & IRIS_DIRTY_CC_VIEWPORT) {
4362 const struct iris_rasterizer_state *cso_rast = ice->state.cso_rast;
4363 uint32_t cc_vp_address;
4364
4365 /* XXX: could avoid streaming for depth_clip [0,1] case. */
4366 uint32_t *cc_vp_map =
4367 stream_state(batch, ice->state.dynamic_uploader,
4368 &ice->state.last_res.cc_vp,
4369 4 * ice->state.num_viewports *
4370 GENX(CC_VIEWPORT_length), 32, &cc_vp_address);
4371 for (int i = 0; i < ice->state.num_viewports; i++) {
4372 float zmin, zmax;
4373 util_viewport_zmin_zmax(&ice->state.viewports[i],
4374 cso_rast->clip_halfz, &zmin, &zmax);
4375 if (cso_rast->depth_clip_near)
4376 zmin = 0.0;
4377 if (cso_rast->depth_clip_far)
4378 zmax = 1.0;
4379
4380 iris_pack_state(GENX(CC_VIEWPORT), cc_vp_map, ccv) {
4381 ccv.MinimumDepth = zmin;
4382 ccv.MaximumDepth = zmax;
4383 }
4384
4385 cc_vp_map += GENX(CC_VIEWPORT_length);
4386 }
4387
4388 iris_emit_cmd(batch, GENX(3DSTATE_VIEWPORT_STATE_POINTERS_CC), ptr) {
4389 ptr.CCViewportPointer = cc_vp_address;
4390 }
4391 }
4392
4393 if (dirty & IRIS_DIRTY_SF_CL_VIEWPORT) {
4394 struct pipe_framebuffer_state *cso_fb = &ice->state.framebuffer;
4395 uint32_t sf_cl_vp_address;
4396 uint32_t *vp_map =
4397 stream_state(batch, ice->state.dynamic_uploader,
4398 &ice->state.last_res.sf_cl_vp,
4399 4 * ice->state.num_viewports *
4400 GENX(SF_CLIP_VIEWPORT_length), 64, &sf_cl_vp_address);
4401
4402 for (unsigned i = 0; i < ice->state.num_viewports; i++) {
4403 const struct pipe_viewport_state *state = &ice->state.viewports[i];
4404 float gb_xmin, gb_xmax, gb_ymin, gb_ymax;
4405
4406 float vp_xmin = viewport_extent(state, 0, -1.0f);
4407 float vp_xmax = viewport_extent(state, 0, 1.0f);
4408 float vp_ymin = viewport_extent(state, 1, -1.0f);
4409 float vp_ymax = viewport_extent(state, 1, 1.0f);
4410
4411 gen_calculate_guardband_size(cso_fb->width, cso_fb->height,
4412 state->scale[0], state->scale[1],
4413 state->translate[0], state->translate[1],
4414 &gb_xmin, &gb_xmax, &gb_ymin, &gb_ymax);
4415
4416 iris_pack_state(GENX(SF_CLIP_VIEWPORT), vp_map, vp) {
4417 vp.ViewportMatrixElementm00 = state->scale[0];
4418 vp.ViewportMatrixElementm11 = state->scale[1];
4419 vp.ViewportMatrixElementm22 = state->scale[2];
4420 vp.ViewportMatrixElementm30 = state->translate[0];
4421 vp.ViewportMatrixElementm31 = state->translate[1];
4422 vp.ViewportMatrixElementm32 = state->translate[2];
4423 vp.XMinClipGuardband = gb_xmin;
4424 vp.XMaxClipGuardband = gb_xmax;
4425 vp.YMinClipGuardband = gb_ymin;
4426 vp.YMaxClipGuardband = gb_ymax;
4427 vp.XMinViewPort = MAX2(vp_xmin, 0);
4428 vp.XMaxViewPort = MIN2(vp_xmax, cso_fb->width) - 1;
4429 vp.YMinViewPort = MAX2(vp_ymin, 0);
4430 vp.YMaxViewPort = MIN2(vp_ymax, cso_fb->height) - 1;
4431 }
4432
4433 vp_map += GENX(SF_CLIP_VIEWPORT_length);
4434 }
4435
4436 iris_emit_cmd(batch, GENX(3DSTATE_VIEWPORT_STATE_POINTERS_SF_CLIP), ptr) {
4437 ptr.SFClipViewportPointer = sf_cl_vp_address;
4438 }
4439 }
4440
4441 if (dirty & IRIS_DIRTY_URB) {
4442 unsigned size[4];
4443
4444 for (int i = MESA_SHADER_VERTEX; i <= MESA_SHADER_GEOMETRY; i++) {
4445 if (!ice->shaders.prog[i]) {
4446 size[i] = 1;
4447 } else {
4448 struct brw_vue_prog_data *vue_prog_data =
4449 (void *) ice->shaders.prog[i]->prog_data;
4450 size[i] = vue_prog_data->urb_entry_size;
4451 }
4452 assert(size[i] != 0);
4453 }
4454
4455 genX(emit_urb_setup)(ice, batch, size,
4456 ice->shaders.prog[MESA_SHADER_TESS_EVAL] != NULL,
4457 ice->shaders.prog[MESA_SHADER_GEOMETRY] != NULL);
4458 }
4459
4460 if (dirty & IRIS_DIRTY_BLEND_STATE) {
4461 struct iris_blend_state *cso_blend = ice->state.cso_blend;
4462 struct pipe_framebuffer_state *cso_fb = &ice->state.framebuffer;
4463 struct iris_depth_stencil_alpha_state *cso_zsa = ice->state.cso_zsa;
4464 const int header_dwords = GENX(BLEND_STATE_length);
4465
4466 /* Always write at least one BLEND_STATE - the final RT message will
4467 * reference BLEND_STATE[0] even if there aren't color writes. There
4468 * may still be alpha testing, computed depth, and so on.
4469 */
4470 const int rt_dwords =
4471 MAX2(cso_fb->nr_cbufs, 1) * GENX(BLEND_STATE_ENTRY_length);
4472
4473 uint32_t blend_offset;
4474 uint32_t *blend_map =
4475 stream_state(batch, ice->state.dynamic_uploader,
4476 &ice->state.last_res.blend,
4477 4 * (header_dwords + rt_dwords), 64, &blend_offset);
4478
4479 uint32_t blend_state_header;
4480 iris_pack_state(GENX(BLEND_STATE), &blend_state_header, bs) {
4481 bs.AlphaTestEnable = cso_zsa->alpha.enabled;
4482 bs.AlphaTestFunction = translate_compare_func(cso_zsa->alpha.func);
4483 }
4484
4485 blend_map[0] = blend_state_header | cso_blend->blend_state[0];
4486 memcpy(&blend_map[1], &cso_blend->blend_state[1], 4 * rt_dwords);
4487
4488 iris_emit_cmd(batch, GENX(3DSTATE_BLEND_STATE_POINTERS), ptr) {
4489 ptr.BlendStatePointer = blend_offset;
4490 ptr.BlendStatePointerValid = true;
4491 }
4492 }
4493
4494 if (dirty & IRIS_DIRTY_COLOR_CALC_STATE) {
4495 struct iris_depth_stencil_alpha_state *cso = ice->state.cso_zsa;
4496 #if GEN_GEN == 8
4497 struct pipe_stencil_ref *p_stencil_refs = &ice->state.stencil_ref;
4498 #endif
4499 uint32_t cc_offset;
4500 void *cc_map =
4501 stream_state(batch, ice->state.dynamic_uploader,
4502 &ice->state.last_res.color_calc,
4503 sizeof(uint32_t) * GENX(COLOR_CALC_STATE_length),
4504 64, &cc_offset);
4505 iris_pack_state(GENX(COLOR_CALC_STATE), cc_map, cc) {
4506 cc.AlphaTestFormat = ALPHATEST_FLOAT32;
4507 cc.AlphaReferenceValueAsFLOAT32 = cso->alpha.ref_value;
4508 cc.BlendConstantColorRed = ice->state.blend_color.color[0];
4509 cc.BlendConstantColorGreen = ice->state.blend_color.color[1];
4510 cc.BlendConstantColorBlue = ice->state.blend_color.color[2];
4511 cc.BlendConstantColorAlpha = ice->state.blend_color.color[3];
4512 #if GEN_GEN == 8
4513 cc.StencilReferenceValue = p_stencil_refs->ref_value[0];
4514 cc.BackfaceStencilReferenceValue = p_stencil_refs->ref_value[1];
4515 #endif
4516 }
4517 iris_emit_cmd(batch, GENX(3DSTATE_CC_STATE_POINTERS), ptr) {
4518 ptr.ColorCalcStatePointer = cc_offset;
4519 ptr.ColorCalcStatePointerValid = true;
4520 }
4521 }
4522
4523 for (int stage = 0; stage <= MESA_SHADER_FRAGMENT; stage++) {
4524 if (!(dirty & (IRIS_DIRTY_CONSTANTS_VS << stage)))
4525 continue;
4526
4527 struct iris_shader_state *shs = &ice->state.shaders[stage];
4528 struct iris_compiled_shader *shader = ice->shaders.prog[stage];
4529
4530 if (!shader)
4531 continue;
4532
4533 if (shs->sysvals_need_upload)
4534 upload_sysvals(ice, stage);
4535
4536 struct brw_stage_prog_data *prog_data = (void *) shader->prog_data;
4537
4538 iris_emit_cmd(batch, GENX(3DSTATE_CONSTANT_VS), pkt) {
4539 pkt._3DCommandSubOpcode = push_constant_opcodes[stage];
4540 if (prog_data) {
4541 /* The Skylake PRM contains the following restriction:
4542 *
4543 * "The driver must ensure The following case does not occur
4544 * without a flush to the 3D engine: 3DSTATE_CONSTANT_* with
4545 * buffer 3 read length equal to zero committed followed by a
4546 * 3DSTATE_CONSTANT_* with buffer 0 read length not equal to
4547 * zero committed."
4548 *
4549 * To avoid this, we program the buffers in the highest slots.
4550 * This way, slot 0 is only used if slot 3 is also used.
4551 */
4552 int n = 3;
4553
4554 for (int i = 3; i >= 0; i--) {
4555 const struct brw_ubo_range *range = &prog_data->ubo_ranges[i];
4556
4557 if (range->length == 0)
4558 continue;
4559
4560 /* Range block is a binding table index, map back to UBO index. */
4561 unsigned block_index = iris_bti_to_group_index(
4562 &shader->bt, IRIS_SURFACE_GROUP_UBO, range->block);
4563 assert(block_index != IRIS_SURFACE_NOT_USED);
4564
4565 struct pipe_shader_buffer *cbuf = &shs->constbuf[block_index];
4566 struct iris_resource *res = (void *) cbuf->buffer;
4567
4568 assert(cbuf->buffer_offset % 32 == 0);
4569
4570 pkt.ConstantBody.ReadLength[n] = range->length;
4571 pkt.ConstantBody.Buffer[n] =
4572 res ? ro_bo(res->bo, range->start * 32 + cbuf->buffer_offset)
4573 : ro_bo(batch->screen->workaround_bo, 0);
4574 n--;
4575 }
4576 }
4577 }
4578 }
4579
4580 for (int stage = 0; stage <= MESA_SHADER_FRAGMENT; stage++) {
4581 if (dirty & (IRIS_DIRTY_BINDINGS_VS << stage)) {
4582 iris_emit_cmd(batch, GENX(3DSTATE_BINDING_TABLE_POINTERS_VS), ptr) {
4583 ptr._3DCommandSubOpcode = 38 + stage;
4584 ptr.PointertoVSBindingTable = binder->bt_offset[stage];
4585 }
4586 }
4587 }
4588
4589 for (int stage = 0; stage <= MESA_SHADER_FRAGMENT; stage++) {
4590 if (dirty & (IRIS_DIRTY_BINDINGS_VS << stage)) {
4591 iris_populate_binding_table(ice, batch, stage, false);
4592 }
4593 }
4594
4595 for (int stage = 0; stage <= MESA_SHADER_FRAGMENT; stage++) {
4596 if (!(dirty & (IRIS_DIRTY_SAMPLER_STATES_VS << stage)) ||
4597 !ice->shaders.prog[stage])
4598 continue;
4599
4600 iris_upload_sampler_states(ice, stage);
4601
4602 struct iris_shader_state *shs = &ice->state.shaders[stage];
4603 struct pipe_resource *res = shs->sampler_table.res;
4604 if (res)
4605 iris_use_pinned_bo(batch, iris_resource_bo(res), false);
4606
4607 iris_emit_cmd(batch, GENX(3DSTATE_SAMPLER_STATE_POINTERS_VS), ptr) {
4608 ptr._3DCommandSubOpcode = 43 + stage;
4609 ptr.PointertoVSSamplerState = shs->sampler_table.offset;
4610 }
4611 }
4612
4613 if (ice->state.need_border_colors)
4614 iris_use_pinned_bo(batch, ice->state.border_color_pool.bo, false);
4615
4616 if (dirty & IRIS_DIRTY_MULTISAMPLE) {
4617 iris_emit_cmd(batch, GENX(3DSTATE_MULTISAMPLE), ms) {
4618 ms.PixelLocation =
4619 ice->state.cso_rast->half_pixel_center ? CENTER : UL_CORNER;
4620 if (ice->state.framebuffer.samples > 0)
4621 ms.NumberofMultisamples = ffs(ice->state.framebuffer.samples) - 1;
4622 }
4623 }
4624
4625 if (dirty & IRIS_DIRTY_SAMPLE_MASK) {
4626 iris_emit_cmd(batch, GENX(3DSTATE_SAMPLE_MASK), ms) {
4627 ms.SampleMask = ice->state.sample_mask;
4628 }
4629 }
4630
4631 for (int stage = 0; stage <= MESA_SHADER_FRAGMENT; stage++) {
4632 if (!(dirty & (IRIS_DIRTY_VS << stage)))
4633 continue;
4634
4635 struct iris_compiled_shader *shader = ice->shaders.prog[stage];
4636
4637 if (shader) {
4638 struct brw_stage_prog_data *prog_data = shader->prog_data;
4639 struct iris_resource *cache = (void *) shader->assembly.res;
4640 iris_use_pinned_bo(batch, cache->bo, false);
4641
4642 if (prog_data->total_scratch > 0) {
4643 struct iris_bo *bo =
4644 iris_get_scratch_space(ice, prog_data->total_scratch, stage);
4645 iris_use_pinned_bo(batch, bo, true);
4646 }
4647 #if GEN_GEN >= 9
4648 if (stage == MESA_SHADER_FRAGMENT && wm_prog_data->uses_sample_mask) {
4649 uint32_t *shader_ps = (uint32_t *) shader->derived_data;
4650 uint32_t *shader_psx = shader_ps + GENX(3DSTATE_PS_length);
4651 uint32_t ps_state[GENX(3DSTATE_PS_length)] = {0};
4652 uint32_t psx_state[GENX(3DSTATE_PS_EXTRA_length)] = {0};
4653 struct iris_rasterizer_state *cso = ice->state.cso_rast;
4654 struct pipe_framebuffer_state *cso_fb = &ice->state.framebuffer;
4655
4656 /* The docs for 3DSTATE_PS::32 Pixel Dispatch Enable say:
4657 *
4658 * "When NUM_MULTISAMPLES = 16 or FORCE_SAMPLE_COUNT = 16,
4659 * SIMD32 Dispatch must not be enabled for PER_PIXEL dispatch
4660 * mode."
4661 *
4662 * 16x MSAA only exists on Gen9+, so we can skip this on Gen8.
4663 */
4664 iris_pack_command(GENX(3DSTATE_PS), &ps_state, ps) {
4665 ps._32PixelDispatchEnable = wm_prog_data->dispatch_32 &&
4666 (cso_fb->samples != 16 || wm_prog_data->persample_dispatch);
4667 }
4668
4669 iris_pack_command(GENX(3DSTATE_PS_EXTRA), &psx_state, psx) {
4670 if (wm_prog_data->post_depth_coverage)
4671 psx.InputCoverageMaskState = ICMS_DEPTH_COVERAGE;
4672 else if (wm_prog_data->inner_coverage && cso->conservative_rasterization)
4673 psx.InputCoverageMaskState = ICMS_INNER_CONSERVATIVE;
4674 else
4675 psx.InputCoverageMaskState = ICMS_NORMAL;
4676 }
4677
4678 iris_emit_merge(batch, shader_ps, ps_state,
4679 GENX(3DSTATE_PS_length));
4680 iris_emit_merge(batch,
4681 shader_psx,
4682 psx_state,
4683 GENX(3DSTATE_PS_EXTRA_length));
4684 } else
4685 #endif
4686 iris_batch_emit(batch, shader->derived_data,
4687 iris_derived_program_state_size(stage));
4688 } else {
4689 if (stage == MESA_SHADER_TESS_EVAL) {
4690 iris_emit_cmd(batch, GENX(3DSTATE_HS), hs);
4691 iris_emit_cmd(batch, GENX(3DSTATE_TE), te);
4692 iris_emit_cmd(batch, GENX(3DSTATE_DS), ds);
4693 } else if (stage == MESA_SHADER_GEOMETRY) {
4694 iris_emit_cmd(batch, GENX(3DSTATE_GS), gs);
4695 }
4696 }
4697 }
4698
4699 if (ice->state.streamout_active) {
4700 if (dirty & IRIS_DIRTY_SO_BUFFERS) {
4701 iris_batch_emit(batch, genx->so_buffers,
4702 4 * 4 * GENX(3DSTATE_SO_BUFFER_length));
4703 for (int i = 0; i < 4; i++) {
4704 struct iris_stream_output_target *tgt =
4705 (void *) ice->state.so_target[i];
4706 if (tgt) {
4707 tgt->zeroed = true;
4708 iris_use_pinned_bo(batch, iris_resource_bo(tgt->base.buffer),
4709 true);
4710 iris_use_pinned_bo(batch, iris_resource_bo(tgt->offset.res),
4711 true);
4712 }
4713 }
4714 }
4715
4716 if ((dirty & IRIS_DIRTY_SO_DECL_LIST) && ice->state.streamout) {
4717 uint32_t *decl_list =
4718 ice->state.streamout + GENX(3DSTATE_STREAMOUT_length);
4719 iris_batch_emit(batch, decl_list, 4 * ((decl_list[0] & 0xff) + 2));
4720 }
4721
4722 if (dirty & IRIS_DIRTY_STREAMOUT) {
4723 const struct iris_rasterizer_state *cso_rast = ice->state.cso_rast;
4724
4725 uint32_t dynamic_sol[GENX(3DSTATE_STREAMOUT_length)];
4726 iris_pack_command(GENX(3DSTATE_STREAMOUT), dynamic_sol, sol) {
4727 sol.SOFunctionEnable = true;
4728 sol.SOStatisticsEnable = true;
4729
4730 sol.RenderingDisable = cso_rast->rasterizer_discard &&
4731 !ice->state.prims_generated_query_active;
4732 sol.ReorderMode = cso_rast->flatshade_first ? LEADING : TRAILING;
4733 }
4734
4735 assert(ice->state.streamout);
4736
4737 iris_emit_merge(batch, ice->state.streamout, dynamic_sol,
4738 GENX(3DSTATE_STREAMOUT_length));
4739 }
4740 } else {
4741 if (dirty & IRIS_DIRTY_STREAMOUT) {
4742 iris_emit_cmd(batch, GENX(3DSTATE_STREAMOUT), sol);
4743 }
4744 }
4745
4746 if (dirty & IRIS_DIRTY_CLIP) {
4747 struct iris_rasterizer_state *cso_rast = ice->state.cso_rast;
4748 struct pipe_framebuffer_state *cso_fb = &ice->state.framebuffer;
4749
4750 bool gs_or_tes = ice->shaders.prog[MESA_SHADER_GEOMETRY] ||
4751 ice->shaders.prog[MESA_SHADER_TESS_EVAL];
4752 bool points_or_lines = cso_rast->fill_mode_point_or_line ||
4753 (gs_or_tes ? ice->shaders.output_topology_is_points_or_lines
4754 : ice->state.prim_is_points_or_lines);
4755
4756 uint32_t dynamic_clip[GENX(3DSTATE_CLIP_length)];
4757 iris_pack_command(GENX(3DSTATE_CLIP), &dynamic_clip, cl) {
4758 cl.StatisticsEnable = ice->state.statistics_counters_enabled;
4759 cl.ClipMode = cso_rast->rasterizer_discard ? CLIPMODE_REJECT_ALL
4760 : CLIPMODE_NORMAL;
4761 cl.ViewportXYClipTestEnable = !points_or_lines;
4762
4763 if (wm_prog_data->barycentric_interp_modes &
4764 BRW_BARYCENTRIC_NONPERSPECTIVE_BITS)
4765 cl.NonPerspectiveBarycentricEnable = true;
4766
4767 cl.ForceZeroRTAIndexEnable = cso_fb->layers == 0;
4768 cl.MaximumVPIndex = ice->state.num_viewports - 1;
4769 }
4770 iris_emit_merge(batch, cso_rast->clip, dynamic_clip,
4771 ARRAY_SIZE(cso_rast->clip));
4772 }
4773
4774 if (dirty & IRIS_DIRTY_RASTER) {
4775 struct iris_rasterizer_state *cso = ice->state.cso_rast;
4776 iris_batch_emit(batch, cso->raster, sizeof(cso->raster));
4777 iris_batch_emit(batch, cso->sf, sizeof(cso->sf));
4778
4779 }
4780
4781 if (dirty & IRIS_DIRTY_WM) {
4782 struct iris_rasterizer_state *cso = ice->state.cso_rast;
4783 uint32_t dynamic_wm[GENX(3DSTATE_WM_length)];
4784
4785 iris_pack_command(GENX(3DSTATE_WM), &dynamic_wm, wm) {
4786 wm.StatisticsEnable = ice->state.statistics_counters_enabled;
4787
4788 wm.BarycentricInterpolationMode =
4789 wm_prog_data->barycentric_interp_modes;
4790
4791 if (wm_prog_data->early_fragment_tests)
4792 wm.EarlyDepthStencilControl = EDSC_PREPS;
4793 else if (wm_prog_data->has_side_effects)
4794 wm.EarlyDepthStencilControl = EDSC_PSEXEC;
4795
4796 /* We could skip this bit if color writes are enabled. */
4797 if (wm_prog_data->has_side_effects || wm_prog_data->uses_kill)
4798 wm.ForceThreadDispatchEnable = ForceON;
4799 }
4800 iris_emit_merge(batch, cso->wm, dynamic_wm, ARRAY_SIZE(cso->wm));
4801 }
4802
4803 if (dirty & IRIS_DIRTY_SBE) {
4804 iris_emit_sbe(batch, ice);
4805 }
4806
4807 if (dirty & IRIS_DIRTY_PS_BLEND) {
4808 struct iris_blend_state *cso_blend = ice->state.cso_blend;
4809 struct iris_depth_stencil_alpha_state *cso_zsa = ice->state.cso_zsa;
4810 const struct shader_info *fs_info =
4811 iris_get_shader_info(ice, MESA_SHADER_FRAGMENT);
4812
4813 uint32_t dynamic_pb[GENX(3DSTATE_PS_BLEND_length)];
4814 iris_pack_command(GENX(3DSTATE_PS_BLEND), &dynamic_pb, pb) {
4815 pb.HasWriteableRT = has_writeable_rt(cso_blend, fs_info);
4816 pb.AlphaTestEnable = cso_zsa->alpha.enabled;
4817
4818 /* The dual source blending docs caution against using SRC1 factors
4819 * when the shader doesn't use a dual source render target write.
4820 * Empirically, this can lead to GPU hangs, and the results are
4821 * undefined anyway, so simply disable blending to avoid the hang.
4822 */
4823 pb.ColorBufferBlendEnable = (cso_blend->blend_enables & 1) &&
4824 (!cso_blend->dual_color_blending || wm_prog_data->dual_src_blend);
4825 }
4826
4827 iris_emit_merge(batch, cso_blend->ps_blend, dynamic_pb,
4828 ARRAY_SIZE(cso_blend->ps_blend));
4829 }
4830
4831 if (dirty & IRIS_DIRTY_WM_DEPTH_STENCIL) {
4832 struct iris_depth_stencil_alpha_state *cso = ice->state.cso_zsa;
4833 #if GEN_GEN >= 9
4834 struct pipe_stencil_ref *p_stencil_refs = &ice->state.stencil_ref;
4835 uint32_t stencil_refs[GENX(3DSTATE_WM_DEPTH_STENCIL_length)];
4836 iris_pack_command(GENX(3DSTATE_WM_DEPTH_STENCIL), &stencil_refs, wmds) {
4837 wmds.StencilReferenceValue = p_stencil_refs->ref_value[0];
4838 wmds.BackfaceStencilReferenceValue = p_stencil_refs->ref_value[1];
4839 }
4840 iris_emit_merge(batch, cso->wmds, stencil_refs, ARRAY_SIZE(cso->wmds));
4841 #else
4842 iris_batch_emit(batch, cso->wmds, sizeof(cso->wmds));
4843 #endif
4844 }
4845
4846 if (dirty & IRIS_DIRTY_SCISSOR_RECT) {
4847 uint32_t scissor_offset =
4848 emit_state(batch, ice->state.dynamic_uploader,
4849 &ice->state.last_res.scissor,
4850 ice->state.scissors,
4851 sizeof(struct pipe_scissor_state) *
4852 ice->state.num_viewports, 32);
4853
4854 iris_emit_cmd(batch, GENX(3DSTATE_SCISSOR_STATE_POINTERS), ptr) {
4855 ptr.ScissorRectPointer = scissor_offset;
4856 }
4857 }
4858
4859 if (dirty & IRIS_DIRTY_DEPTH_BUFFER) {
4860 struct iris_depth_buffer_state *cso_z = &ice->state.genx->depth_buffer;
4861
4862 /* Do not emit the clear params yets. We need to update the clear value
4863 * first.
4864 */
4865 uint32_t clear_length = GENX(3DSTATE_CLEAR_PARAMS_length) * 4;
4866 uint32_t cso_z_size = sizeof(cso_z->packets) - clear_length;
4867 iris_batch_emit(batch, cso_z->packets, cso_z_size);
4868
4869 union isl_color_value clear_value = { .f32 = { 0, } };
4870
4871 struct pipe_framebuffer_state *cso_fb = &ice->state.framebuffer;
4872 if (cso_fb->zsbuf) {
4873 struct iris_resource *zres, *sres;
4874 iris_get_depth_stencil_resources(cso_fb->zsbuf->texture,
4875 &zres, &sres);
4876 if (zres && zres->aux.bo)
4877 clear_value = iris_resource_get_clear_color(zres, NULL, NULL);
4878 }
4879
4880 uint32_t clear_params[GENX(3DSTATE_CLEAR_PARAMS_length)];
4881 iris_pack_command(GENX(3DSTATE_CLEAR_PARAMS), clear_params, clear) {
4882 clear.DepthClearValueValid = true;
4883 clear.DepthClearValue = clear_value.f32[0];
4884 }
4885 iris_batch_emit(batch, clear_params, clear_length);
4886 }
4887
4888 if (dirty & (IRIS_DIRTY_DEPTH_BUFFER | IRIS_DIRTY_WM_DEPTH_STENCIL)) {
4889 /* Listen for buffer changes, and also write enable changes. */
4890 struct pipe_framebuffer_state *cso_fb = &ice->state.framebuffer;
4891 pin_depth_and_stencil_buffers(batch, cso_fb->zsbuf, ice->state.cso_zsa);
4892 }
4893
4894 if (dirty & IRIS_DIRTY_POLYGON_STIPPLE) {
4895 iris_emit_cmd(batch, GENX(3DSTATE_POLY_STIPPLE_PATTERN), poly) {
4896 for (int i = 0; i < 32; i++) {
4897 poly.PatternRow[i] = ice->state.poly_stipple.stipple[i];
4898 }
4899 }
4900 }
4901
4902 if (dirty & IRIS_DIRTY_LINE_STIPPLE) {
4903 struct iris_rasterizer_state *cso = ice->state.cso_rast;
4904 iris_batch_emit(batch, cso->line_stipple, sizeof(cso->line_stipple));
4905 }
4906
4907 if (dirty & IRIS_DIRTY_VF_TOPOLOGY) {
4908 iris_emit_cmd(batch, GENX(3DSTATE_VF_TOPOLOGY), topo) {
4909 topo.PrimitiveTopologyType =
4910 translate_prim_type(draw->mode, draw->vertices_per_patch);
4911 }
4912 }
4913
4914 if (dirty & IRIS_DIRTY_VERTEX_BUFFERS) {
4915 int count = util_bitcount64(ice->state.bound_vertex_buffers);
4916 int dynamic_bound = ice->state.bound_vertex_buffers;
4917
4918 if (ice->state.vs_uses_draw_params) {
4919 if (ice->draw.draw_params_offset == 0) {
4920 u_upload_data(ice->ctx.stream_uploader, 0, sizeof(ice->draw.params),
4921 4, &ice->draw.params, &ice->draw.draw_params_offset,
4922 &ice->draw.draw_params_res);
4923 }
4924 assert(ice->draw.draw_params_res);
4925
4926 struct iris_vertex_buffer_state *state =
4927 &(ice->state.genx->vertex_buffers[count]);
4928 pipe_resource_reference(&state->resource, ice->draw.draw_params_res);
4929 struct iris_resource *res = (void *) state->resource;
4930
4931 iris_pack_state(GENX(VERTEX_BUFFER_STATE), state->state, vb) {
4932 vb.VertexBufferIndex = count;
4933 vb.AddressModifyEnable = true;
4934 vb.BufferPitch = 0;
4935 vb.BufferSize = res->bo->size - ice->draw.draw_params_offset;
4936 vb.BufferStartingAddress =
4937 ro_bo(NULL, res->bo->gtt_offset +
4938 (int) ice->draw.draw_params_offset);
4939 vb.MOCS = mocs(res->bo);
4940 }
4941 dynamic_bound |= 1ull << count;
4942 count++;
4943 }
4944
4945 if (ice->state.vs_uses_derived_draw_params) {
4946 u_upload_data(ice->ctx.stream_uploader, 0,
4947 sizeof(ice->draw.derived_params), 4,
4948 &ice->draw.derived_params,
4949 &ice->draw.derived_draw_params_offset,
4950 &ice->draw.derived_draw_params_res);
4951
4952 struct iris_vertex_buffer_state *state =
4953 &(ice->state.genx->vertex_buffers[count]);
4954 pipe_resource_reference(&state->resource,
4955 ice->draw.derived_draw_params_res);
4956 struct iris_resource *res = (void *) ice->draw.derived_draw_params_res;
4957
4958 iris_pack_state(GENX(VERTEX_BUFFER_STATE), state->state, vb) {
4959 vb.VertexBufferIndex = count;
4960 vb.AddressModifyEnable = true;
4961 vb.BufferPitch = 0;
4962 vb.BufferSize =
4963 res->bo->size - ice->draw.derived_draw_params_offset;
4964 vb.BufferStartingAddress =
4965 ro_bo(NULL, res->bo->gtt_offset +
4966 (int) ice->draw.derived_draw_params_offset);
4967 vb.MOCS = mocs(res->bo);
4968 }
4969 dynamic_bound |= 1ull << count;
4970 count++;
4971 }
4972
4973 if (count) {
4974 /* The VF cache designers cut corners, and made the cache key's
4975 * <VertexBufferIndex, Memory Address> tuple only consider the bottom
4976 * 32 bits of the address. If you have two vertex buffers which get
4977 * placed exactly 4 GiB apart and use them in back-to-back draw calls,
4978 * you can get collisions (even within a single batch).
4979 *
4980 * So, we need to do a VF cache invalidate if the buffer for a VB
4981 * slot slot changes [48:32] address bits from the previous time.
4982 */
4983 unsigned flush_flags = 0;
4984
4985 uint64_t bound = dynamic_bound;
4986 while (bound) {
4987 const int i = u_bit_scan64(&bound);
4988 uint16_t high_bits = 0;
4989
4990 struct iris_resource *res =
4991 (void *) genx->vertex_buffers[i].resource;
4992 if (res) {
4993 iris_use_pinned_bo(batch, res->bo, false);
4994
4995 high_bits = res->bo->gtt_offset >> 32ull;
4996 if (high_bits != ice->state.last_vbo_high_bits[i]) {
4997 flush_flags |= PIPE_CONTROL_VF_CACHE_INVALIDATE |
4998 PIPE_CONTROL_CS_STALL;
4999 ice->state.last_vbo_high_bits[i] = high_bits;
5000 }
5001 }
5002 }
5003
5004 if (flush_flags) {
5005 iris_emit_pipe_control_flush(batch,
5006 "workaround: VF cache 32-bit key [VB]",
5007 flush_flags);
5008 }
5009
5010 const unsigned vb_dwords = GENX(VERTEX_BUFFER_STATE_length);
5011
5012 uint32_t *map =
5013 iris_get_command_space(batch, 4 * (1 + vb_dwords * count));
5014 _iris_pack_command(batch, GENX(3DSTATE_VERTEX_BUFFERS), map, vb) {
5015 vb.DWordLength = (vb_dwords * count + 1) - 2;
5016 }
5017 map += 1;
5018
5019 bound = dynamic_bound;
5020 while (bound) {
5021 const int i = u_bit_scan64(&bound);
5022 memcpy(map, genx->vertex_buffers[i].state,
5023 sizeof(uint32_t) * vb_dwords);
5024 map += vb_dwords;
5025 }
5026 }
5027 }
5028
5029 if (dirty & IRIS_DIRTY_VERTEX_ELEMENTS) {
5030 struct iris_vertex_element_state *cso = ice->state.cso_vertex_elements;
5031 const unsigned entries = MAX2(cso->count, 1);
5032 if (!(ice->state.vs_needs_sgvs_element ||
5033 ice->state.vs_uses_derived_draw_params ||
5034 ice->state.vs_needs_edge_flag)) {
5035 iris_batch_emit(batch, cso->vertex_elements, sizeof(uint32_t) *
5036 (1 + entries * GENX(VERTEX_ELEMENT_STATE_length)));
5037 } else {
5038 uint32_t dynamic_ves[1 + 33 * GENX(VERTEX_ELEMENT_STATE_length)];
5039 const unsigned dyn_count = cso->count +
5040 ice->state.vs_needs_sgvs_element +
5041 ice->state.vs_uses_derived_draw_params;
5042
5043 iris_pack_command(GENX(3DSTATE_VERTEX_ELEMENTS),
5044 &dynamic_ves, ve) {
5045 ve.DWordLength =
5046 1 + GENX(VERTEX_ELEMENT_STATE_length) * dyn_count - 2;
5047 }
5048 memcpy(&dynamic_ves[1], &cso->vertex_elements[1],
5049 (cso->count - ice->state.vs_needs_edge_flag) *
5050 GENX(VERTEX_ELEMENT_STATE_length) * sizeof(uint32_t));
5051 uint32_t *ve_pack_dest =
5052 &dynamic_ves[1 + (cso->count - ice->state.vs_needs_edge_flag) *
5053 GENX(VERTEX_ELEMENT_STATE_length)];
5054
5055 if (ice->state.vs_needs_sgvs_element) {
5056 uint32_t base_ctrl = ice->state.vs_uses_draw_params ?
5057 VFCOMP_STORE_SRC : VFCOMP_STORE_0;
5058 iris_pack_state(GENX(VERTEX_ELEMENT_STATE), ve_pack_dest, ve) {
5059 ve.Valid = true;
5060 ve.VertexBufferIndex =
5061 util_bitcount64(ice->state.bound_vertex_buffers);
5062 ve.SourceElementFormat = ISL_FORMAT_R32G32_UINT;
5063 ve.Component0Control = base_ctrl;
5064 ve.Component1Control = base_ctrl;
5065 ve.Component2Control = VFCOMP_STORE_0;
5066 ve.Component3Control = VFCOMP_STORE_0;
5067 }
5068 ve_pack_dest += GENX(VERTEX_ELEMENT_STATE_length);
5069 }
5070 if (ice->state.vs_uses_derived_draw_params) {
5071 iris_pack_state(GENX(VERTEX_ELEMENT_STATE), ve_pack_dest, ve) {
5072 ve.Valid = true;
5073 ve.VertexBufferIndex =
5074 util_bitcount64(ice->state.bound_vertex_buffers) +
5075 ice->state.vs_uses_draw_params;
5076 ve.SourceElementFormat = ISL_FORMAT_R32G32_UINT;
5077 ve.Component0Control = VFCOMP_STORE_SRC;
5078 ve.Component1Control = VFCOMP_STORE_SRC;
5079 ve.Component2Control = VFCOMP_STORE_0;
5080 ve.Component3Control = VFCOMP_STORE_0;
5081 }
5082 ve_pack_dest += GENX(VERTEX_ELEMENT_STATE_length);
5083 }
5084 if (ice->state.vs_needs_edge_flag) {
5085 for (int i = 0; i < GENX(VERTEX_ELEMENT_STATE_length); i++)
5086 ve_pack_dest[i] = cso->edgeflag_ve[i];
5087 }
5088
5089 iris_batch_emit(batch, &dynamic_ves, sizeof(uint32_t) *
5090 (1 + dyn_count * GENX(VERTEX_ELEMENT_STATE_length)));
5091 }
5092
5093 if (!ice->state.vs_needs_edge_flag) {
5094 iris_batch_emit(batch, cso->vf_instancing, sizeof(uint32_t) *
5095 entries * GENX(3DSTATE_VF_INSTANCING_length));
5096 } else {
5097 assert(cso->count > 0);
5098 const unsigned edgeflag_index = cso->count - 1;
5099 uint32_t dynamic_vfi[33 * GENX(3DSTATE_VF_INSTANCING_length)];
5100 memcpy(&dynamic_vfi[0], cso->vf_instancing, edgeflag_index *
5101 GENX(3DSTATE_VF_INSTANCING_length) * sizeof(uint32_t));
5102
5103 uint32_t *vfi_pack_dest = &dynamic_vfi[0] +
5104 edgeflag_index * GENX(3DSTATE_VF_INSTANCING_length);
5105 iris_pack_command(GENX(3DSTATE_VF_INSTANCING), vfi_pack_dest, vi) {
5106 vi.VertexElementIndex = edgeflag_index +
5107 ice->state.vs_needs_sgvs_element +
5108 ice->state.vs_uses_derived_draw_params;
5109 }
5110 for (int i = 0; i < GENX(3DSTATE_VF_INSTANCING_length); i++)
5111 vfi_pack_dest[i] |= cso->edgeflag_vfi[i];
5112
5113 iris_batch_emit(batch, &dynamic_vfi[0], sizeof(uint32_t) *
5114 entries * GENX(3DSTATE_VF_INSTANCING_length));
5115 }
5116 }
5117
5118 if (dirty & IRIS_DIRTY_VF_SGVS) {
5119 const struct brw_vs_prog_data *vs_prog_data = (void *)
5120 ice->shaders.prog[MESA_SHADER_VERTEX]->prog_data;
5121 struct iris_vertex_element_state *cso = ice->state.cso_vertex_elements;
5122
5123 iris_emit_cmd(batch, GENX(3DSTATE_VF_SGVS), sgv) {
5124 if (vs_prog_data->uses_vertexid) {
5125 sgv.VertexIDEnable = true;
5126 sgv.VertexIDComponentNumber = 2;
5127 sgv.VertexIDElementOffset =
5128 cso->count - ice->state.vs_needs_edge_flag;
5129 }
5130
5131 if (vs_prog_data->uses_instanceid) {
5132 sgv.InstanceIDEnable = true;
5133 sgv.InstanceIDComponentNumber = 3;
5134 sgv.InstanceIDElementOffset =
5135 cso->count - ice->state.vs_needs_edge_flag;
5136 }
5137 }
5138 }
5139
5140 if (dirty & IRIS_DIRTY_VF) {
5141 iris_emit_cmd(batch, GENX(3DSTATE_VF), vf) {
5142 if (draw->primitive_restart) {
5143 vf.IndexedDrawCutIndexEnable = true;
5144 vf.CutIndex = draw->restart_index;
5145 }
5146 }
5147 }
5148
5149 if (dirty & IRIS_DIRTY_VF_STATISTICS) {
5150 iris_emit_cmd(batch, GENX(3DSTATE_VF_STATISTICS), vf) {
5151 vf.StatisticsEnable = true;
5152 }
5153 }
5154
5155 /* TODO: Gen8 PMA fix */
5156 }
5157
5158 static void
5159 iris_upload_render_state(struct iris_context *ice,
5160 struct iris_batch *batch,
5161 const struct pipe_draw_info *draw)
5162 {
5163 bool use_predicate = ice->state.predicate == IRIS_PREDICATE_STATE_USE_BIT;
5164
5165 /* Always pin the binder. If we're emitting new binding table pointers,
5166 * we need it. If not, we're probably inheriting old tables via the
5167 * context, and need it anyway. Since true zero-bindings cases are
5168 * practically non-existent, just pin it and avoid last_res tracking.
5169 */
5170 iris_use_pinned_bo(batch, ice->state.binder.bo, false);
5171
5172 if (!batch->contains_draw) {
5173 iris_restore_render_saved_bos(ice, batch, draw);
5174 batch->contains_draw = true;
5175 }
5176
5177 iris_upload_dirty_render_state(ice, batch, draw);
5178
5179 if (draw->index_size > 0) {
5180 unsigned offset;
5181
5182 if (draw->has_user_indices) {
5183 u_upload_data(ice->ctx.stream_uploader, 0,
5184 draw->count * draw->index_size, 4, draw->index.user,
5185 &offset, &ice->state.last_res.index_buffer);
5186 } else {
5187 struct iris_resource *res = (void *) draw->index.resource;
5188 res->bind_history |= PIPE_BIND_INDEX_BUFFER;
5189
5190 pipe_resource_reference(&ice->state.last_res.index_buffer,
5191 draw->index.resource);
5192 offset = 0;
5193 }
5194
5195 struct iris_genx_state *genx = ice->state.genx;
5196 struct iris_bo *bo = iris_resource_bo(ice->state.last_res.index_buffer);
5197
5198 uint32_t ib_packet[GENX(3DSTATE_INDEX_BUFFER_length)];
5199 iris_pack_command(GENX(3DSTATE_INDEX_BUFFER), ib_packet, ib) {
5200 ib.IndexFormat = draw->index_size >> 1;
5201 ib.MOCS = mocs(bo);
5202 ib.BufferSize = bo->size - offset;
5203 ib.BufferStartingAddress = ro_bo(NULL, bo->gtt_offset + offset);
5204 }
5205
5206 if (memcmp(genx->last_index_buffer, ib_packet, sizeof(ib_packet)) != 0) {
5207 memcpy(genx->last_index_buffer, ib_packet, sizeof(ib_packet));
5208 iris_batch_emit(batch, ib_packet, sizeof(ib_packet));
5209 iris_use_pinned_bo(batch, bo, false);
5210 }
5211
5212 /* The VF cache key only uses 32-bits, see vertex buffer comment above */
5213 uint16_t high_bits = bo->gtt_offset >> 32ull;
5214 if (high_bits != ice->state.last_index_bo_high_bits) {
5215 iris_emit_pipe_control_flush(batch,
5216 "workaround: VF cache 32-bit key [IB]",
5217 PIPE_CONTROL_VF_CACHE_INVALIDATE |
5218 PIPE_CONTROL_CS_STALL);
5219 ice->state.last_index_bo_high_bits = high_bits;
5220 }
5221 }
5222
5223 #define _3DPRIM_END_OFFSET 0x2420
5224 #define _3DPRIM_START_VERTEX 0x2430
5225 #define _3DPRIM_VERTEX_COUNT 0x2434
5226 #define _3DPRIM_INSTANCE_COUNT 0x2438
5227 #define _3DPRIM_START_INSTANCE 0x243C
5228 #define _3DPRIM_BASE_VERTEX 0x2440
5229
5230 if (draw->indirect) {
5231 if (draw->indirect->indirect_draw_count) {
5232 use_predicate = true;
5233
5234 struct iris_bo *draw_count_bo =
5235 iris_resource_bo(draw->indirect->indirect_draw_count);
5236 unsigned draw_count_offset =
5237 draw->indirect->indirect_draw_count_offset;
5238
5239 iris_emit_pipe_control_flush(batch,
5240 "ensure indirect draw buffer is flushed",
5241 PIPE_CONTROL_FLUSH_ENABLE);
5242
5243 if (ice->state.predicate == IRIS_PREDICATE_STATE_USE_BIT) {
5244 struct gen_mi_builder b;
5245 gen_mi_builder_init(&b, batch);
5246
5247 /* comparison = draw id < draw count */
5248 struct gen_mi_value comparison =
5249 gen_mi_ult(&b, gen_mi_imm(draw->drawid),
5250 gen_mi_mem32(ro_bo(draw_count_bo,
5251 draw_count_offset)));
5252
5253 /* predicate = comparison & conditional rendering predicate */
5254 gen_mi_store(&b, gen_mi_reg32(MI_PREDICATE_RESULT),
5255 gen_mi_iand(&b, comparison,
5256 gen_mi_reg32(CS_GPR(15))));
5257 } else {
5258 uint32_t mi_predicate;
5259
5260 /* Upload the id of the current primitive to MI_PREDICATE_SRC1. */
5261 ice->vtbl.load_register_imm64(batch, MI_PREDICATE_SRC1,
5262 draw->drawid);
5263 /* Upload the current draw count from the draw parameters buffer
5264 * to MI_PREDICATE_SRC0.
5265 */
5266 ice->vtbl.load_register_mem32(batch, MI_PREDICATE_SRC0,
5267 draw_count_bo, draw_count_offset);
5268 /* Zero the top 32-bits of MI_PREDICATE_SRC0 */
5269 ice->vtbl.load_register_imm32(batch, MI_PREDICATE_SRC0 + 4, 0);
5270
5271 if (draw->drawid == 0) {
5272 mi_predicate = MI_PREDICATE | MI_PREDICATE_LOADOP_LOADINV |
5273 MI_PREDICATE_COMBINEOP_SET |
5274 MI_PREDICATE_COMPAREOP_SRCS_EQUAL;
5275 } else {
5276 /* While draw_index < draw_count the predicate's result will be
5277 * (draw_index == draw_count) ^ TRUE = TRUE
5278 * When draw_index == draw_count the result is
5279 * (TRUE) ^ TRUE = FALSE
5280 * After this all results will be:
5281 * (FALSE) ^ FALSE = FALSE
5282 */
5283 mi_predicate = MI_PREDICATE | MI_PREDICATE_LOADOP_LOAD |
5284 MI_PREDICATE_COMBINEOP_XOR |
5285 MI_PREDICATE_COMPAREOP_SRCS_EQUAL;
5286 }
5287 iris_batch_emit(batch, &mi_predicate, sizeof(uint32_t));
5288 }
5289 }
5290 struct iris_bo *bo = iris_resource_bo(draw->indirect->buffer);
5291 assert(bo);
5292
5293 iris_emit_cmd(batch, GENX(MI_LOAD_REGISTER_MEM), lrm) {
5294 lrm.RegisterAddress = _3DPRIM_VERTEX_COUNT;
5295 lrm.MemoryAddress = ro_bo(bo, draw->indirect->offset + 0);
5296 }
5297 iris_emit_cmd(batch, GENX(MI_LOAD_REGISTER_MEM), lrm) {
5298 lrm.RegisterAddress = _3DPRIM_INSTANCE_COUNT;
5299 lrm.MemoryAddress = ro_bo(bo, draw->indirect->offset + 4);
5300 }
5301 iris_emit_cmd(batch, GENX(MI_LOAD_REGISTER_MEM), lrm) {
5302 lrm.RegisterAddress = _3DPRIM_START_VERTEX;
5303 lrm.MemoryAddress = ro_bo(bo, draw->indirect->offset + 8);
5304 }
5305 if (draw->index_size) {
5306 iris_emit_cmd(batch, GENX(MI_LOAD_REGISTER_MEM), lrm) {
5307 lrm.RegisterAddress = _3DPRIM_BASE_VERTEX;
5308 lrm.MemoryAddress = ro_bo(bo, draw->indirect->offset + 12);
5309 }
5310 iris_emit_cmd(batch, GENX(MI_LOAD_REGISTER_MEM), lrm) {
5311 lrm.RegisterAddress = _3DPRIM_START_INSTANCE;
5312 lrm.MemoryAddress = ro_bo(bo, draw->indirect->offset + 16);
5313 }
5314 } else {
5315 iris_emit_cmd(batch, GENX(MI_LOAD_REGISTER_MEM), lrm) {
5316 lrm.RegisterAddress = _3DPRIM_START_INSTANCE;
5317 lrm.MemoryAddress = ro_bo(bo, draw->indirect->offset + 12);
5318 }
5319 iris_emit_cmd(batch, GENX(MI_LOAD_REGISTER_IMM), lri) {
5320 lri.RegisterOffset = _3DPRIM_BASE_VERTEX;
5321 lri.DataDWord = 0;
5322 }
5323 }
5324 } else if (draw->count_from_stream_output) {
5325 struct iris_stream_output_target *so =
5326 (void *) draw->count_from_stream_output;
5327
5328 /* XXX: Replace with actual cache tracking */
5329 iris_emit_pipe_control_flush(batch,
5330 "draw count from stream output stall",
5331 PIPE_CONTROL_CS_STALL);
5332
5333 struct gen_mi_builder b;
5334 gen_mi_builder_init(&b, batch);
5335
5336 struct iris_address addr =
5337 ro_bo(iris_resource_bo(so->offset.res), so->offset.offset);
5338 struct gen_mi_value offset =
5339 gen_mi_iadd_imm(&b, gen_mi_mem32(addr), -so->base.buffer_offset);
5340
5341 gen_mi_store(&b, gen_mi_reg32(_3DPRIM_VERTEX_COUNT),
5342 gen_mi_udiv32_imm(&b, offset, so->stride));
5343
5344 _iris_emit_lri(batch, _3DPRIM_START_VERTEX, 0);
5345 _iris_emit_lri(batch, _3DPRIM_BASE_VERTEX, 0);
5346 _iris_emit_lri(batch, _3DPRIM_START_INSTANCE, 0);
5347 _iris_emit_lri(batch, _3DPRIM_INSTANCE_COUNT, draw->instance_count);
5348 }
5349
5350 iris_emit_cmd(batch, GENX(3DPRIMITIVE), prim) {
5351 prim.VertexAccessType = draw->index_size > 0 ? RANDOM : SEQUENTIAL;
5352 prim.PredicateEnable = use_predicate;
5353
5354 if (draw->indirect || draw->count_from_stream_output) {
5355 prim.IndirectParameterEnable = true;
5356 } else {
5357 prim.StartInstanceLocation = draw->start_instance;
5358 prim.InstanceCount = draw->instance_count;
5359 prim.VertexCountPerInstance = draw->count;
5360
5361 prim.StartVertexLocation = draw->start;
5362
5363 if (draw->index_size) {
5364 prim.BaseVertexLocation += draw->index_bias;
5365 } else {
5366 prim.StartVertexLocation += draw->index_bias;
5367 }
5368 }
5369 }
5370 }
5371
5372 static void
5373 iris_upload_compute_state(struct iris_context *ice,
5374 struct iris_batch *batch,
5375 const struct pipe_grid_info *grid)
5376 {
5377 const uint64_t dirty = ice->state.dirty;
5378 struct iris_screen *screen = batch->screen;
5379 const struct gen_device_info *devinfo = &screen->devinfo;
5380 struct iris_binder *binder = &ice->state.binder;
5381 struct iris_shader_state *shs = &ice->state.shaders[MESA_SHADER_COMPUTE];
5382 struct iris_compiled_shader *shader =
5383 ice->shaders.prog[MESA_SHADER_COMPUTE];
5384 struct brw_stage_prog_data *prog_data = shader->prog_data;
5385 struct brw_cs_prog_data *cs_prog_data = (void *) prog_data;
5386
5387 /* Always pin the binder. If we're emitting new binding table pointers,
5388 * we need it. If not, we're probably inheriting old tables via the
5389 * context, and need it anyway. Since true zero-bindings cases are
5390 * practically non-existent, just pin it and avoid last_res tracking.
5391 */
5392 iris_use_pinned_bo(batch, ice->state.binder.bo, false);
5393
5394 if ((dirty & IRIS_DIRTY_CONSTANTS_CS) && shs->sysvals_need_upload)
5395 upload_sysvals(ice, MESA_SHADER_COMPUTE);
5396
5397 if (dirty & IRIS_DIRTY_BINDINGS_CS)
5398 iris_populate_binding_table(ice, batch, MESA_SHADER_COMPUTE, false);
5399
5400 if (dirty & IRIS_DIRTY_SAMPLER_STATES_CS)
5401 iris_upload_sampler_states(ice, MESA_SHADER_COMPUTE);
5402
5403 iris_use_optional_res(batch, shs->sampler_table.res, false);
5404 iris_use_pinned_bo(batch, iris_resource_bo(shader->assembly.res), false);
5405
5406 if (ice->state.need_border_colors)
5407 iris_use_pinned_bo(batch, ice->state.border_color_pool.bo, false);
5408
5409 if (dirty & IRIS_DIRTY_CS) {
5410 /* The MEDIA_VFE_STATE documentation for Gen8+ says:
5411 *
5412 * "A stalling PIPE_CONTROL is required before MEDIA_VFE_STATE unless
5413 * the only bits that are changed are scoreboard related: Scoreboard
5414 * Enable, Scoreboard Type, Scoreboard Mask, Scoreboard Delta. For
5415 * these scoreboard related states, a MEDIA_STATE_FLUSH is
5416 * sufficient."
5417 */
5418 iris_emit_pipe_control_flush(batch,
5419 "workaround: stall before MEDIA_VFE_STATE",
5420 PIPE_CONTROL_CS_STALL);
5421
5422 iris_emit_cmd(batch, GENX(MEDIA_VFE_STATE), vfe) {
5423 if (prog_data->total_scratch) {
5424 struct iris_bo *bo =
5425 iris_get_scratch_space(ice, prog_data->total_scratch,
5426 MESA_SHADER_COMPUTE);
5427 vfe.PerThreadScratchSpace = ffs(prog_data->total_scratch) - 11;
5428 vfe.ScratchSpaceBasePointer = rw_bo(bo, 0);
5429 }
5430
5431 vfe.MaximumNumberofThreads =
5432 devinfo->max_cs_threads * screen->subslice_total - 1;
5433 #if GEN_GEN < 11
5434 vfe.ResetGatewayTimer =
5435 Resettingrelativetimerandlatchingtheglobaltimestamp;
5436 #endif
5437 #if GEN_GEN == 8
5438 vfe.BypassGatewayControl = true;
5439 #endif
5440 vfe.NumberofURBEntries = 2;
5441 vfe.URBEntryAllocationSize = 2;
5442
5443 vfe.CURBEAllocationSize =
5444 ALIGN(cs_prog_data->push.per_thread.regs * cs_prog_data->threads +
5445 cs_prog_data->push.cross_thread.regs, 2);
5446 }
5447 }
5448
5449 /* TODO: Combine subgroup-id with cbuf0 so we can push regular uniforms */
5450 if (dirty & IRIS_DIRTY_CS) {
5451 uint32_t curbe_data_offset = 0;
5452 assert(cs_prog_data->push.cross_thread.dwords == 0 &&
5453 cs_prog_data->push.per_thread.dwords == 1 &&
5454 cs_prog_data->base.param[0] == BRW_PARAM_BUILTIN_SUBGROUP_ID);
5455 uint32_t *curbe_data_map =
5456 stream_state(batch, ice->state.dynamic_uploader,
5457 &ice->state.last_res.cs_thread_ids,
5458 ALIGN(cs_prog_data->push.total.size, 64), 64,
5459 &curbe_data_offset);
5460 assert(curbe_data_map);
5461 memset(curbe_data_map, 0x5a, ALIGN(cs_prog_data->push.total.size, 64));
5462 iris_fill_cs_push_const_buffer(cs_prog_data, curbe_data_map);
5463
5464 iris_emit_cmd(batch, GENX(MEDIA_CURBE_LOAD), curbe) {
5465 curbe.CURBETotalDataLength =
5466 ALIGN(cs_prog_data->push.total.size, 64);
5467 curbe.CURBEDataStartAddress = curbe_data_offset;
5468 }
5469 }
5470
5471 if (dirty & (IRIS_DIRTY_SAMPLER_STATES_CS |
5472 IRIS_DIRTY_BINDINGS_CS |
5473 IRIS_DIRTY_CONSTANTS_CS |
5474 IRIS_DIRTY_CS)) {
5475 uint32_t desc[GENX(INTERFACE_DESCRIPTOR_DATA_length)];
5476
5477 iris_pack_state(GENX(INTERFACE_DESCRIPTOR_DATA), desc, idd) {
5478 idd.SamplerStatePointer = shs->sampler_table.offset;
5479 idd.BindingTablePointer = binder->bt_offset[MESA_SHADER_COMPUTE];
5480 }
5481
5482 for (int i = 0; i < GENX(INTERFACE_DESCRIPTOR_DATA_length); i++)
5483 desc[i] |= ((uint32_t *) shader->derived_data)[i];
5484
5485 iris_emit_cmd(batch, GENX(MEDIA_INTERFACE_DESCRIPTOR_LOAD), load) {
5486 load.InterfaceDescriptorTotalLength =
5487 GENX(INTERFACE_DESCRIPTOR_DATA_length) * sizeof(uint32_t);
5488 load.InterfaceDescriptorDataStartAddress =
5489 emit_state(batch, ice->state.dynamic_uploader,
5490 &ice->state.last_res.cs_desc, desc, sizeof(desc), 64);
5491 }
5492 }
5493
5494 uint32_t group_size = grid->block[0] * grid->block[1] * grid->block[2];
5495 uint32_t remainder = group_size & (cs_prog_data->simd_size - 1);
5496 uint32_t right_mask;
5497
5498 if (remainder > 0)
5499 right_mask = ~0u >> (32 - remainder);
5500 else
5501 right_mask = ~0u >> (32 - cs_prog_data->simd_size);
5502
5503 #define GPGPU_DISPATCHDIMX 0x2500
5504 #define GPGPU_DISPATCHDIMY 0x2504
5505 #define GPGPU_DISPATCHDIMZ 0x2508
5506
5507 if (grid->indirect) {
5508 struct iris_state_ref *grid_size = &ice->state.grid_size;
5509 struct iris_bo *bo = iris_resource_bo(grid_size->res);
5510 iris_emit_cmd(batch, GENX(MI_LOAD_REGISTER_MEM), lrm) {
5511 lrm.RegisterAddress = GPGPU_DISPATCHDIMX;
5512 lrm.MemoryAddress = ro_bo(bo, grid_size->offset + 0);
5513 }
5514 iris_emit_cmd(batch, GENX(MI_LOAD_REGISTER_MEM), lrm) {
5515 lrm.RegisterAddress = GPGPU_DISPATCHDIMY;
5516 lrm.MemoryAddress = ro_bo(bo, grid_size->offset + 4);
5517 }
5518 iris_emit_cmd(batch, GENX(MI_LOAD_REGISTER_MEM), lrm) {
5519 lrm.RegisterAddress = GPGPU_DISPATCHDIMZ;
5520 lrm.MemoryAddress = ro_bo(bo, grid_size->offset + 8);
5521 }
5522 }
5523
5524 iris_emit_cmd(batch, GENX(GPGPU_WALKER), ggw) {
5525 ggw.IndirectParameterEnable = grid->indirect != NULL;
5526 ggw.SIMDSize = cs_prog_data->simd_size / 16;
5527 ggw.ThreadDepthCounterMaximum = 0;
5528 ggw.ThreadHeightCounterMaximum = 0;
5529 ggw.ThreadWidthCounterMaximum = cs_prog_data->threads - 1;
5530 ggw.ThreadGroupIDXDimension = grid->grid[0];
5531 ggw.ThreadGroupIDYDimension = grid->grid[1];
5532 ggw.ThreadGroupIDZDimension = grid->grid[2];
5533 ggw.RightExecutionMask = right_mask;
5534 ggw.BottomExecutionMask = 0xffffffff;
5535 }
5536
5537 iris_emit_cmd(batch, GENX(MEDIA_STATE_FLUSH), msf);
5538
5539 if (!batch->contains_draw) {
5540 iris_restore_compute_saved_bos(ice, batch, grid);
5541 batch->contains_draw = true;
5542 }
5543 }
5544
5545 /**
5546 * State module teardown.
5547 */
5548 static void
5549 iris_destroy_state(struct iris_context *ice)
5550 {
5551 struct iris_genx_state *genx = ice->state.genx;
5552
5553 pipe_resource_reference(&ice->draw.draw_params_res, NULL);
5554 pipe_resource_reference(&ice->draw.derived_draw_params_res, NULL);
5555
5556 uint64_t bound_vbs = ice->state.bound_vertex_buffers;
5557 while (bound_vbs) {
5558 const int i = u_bit_scan64(&bound_vbs);
5559 pipe_resource_reference(&genx->vertex_buffers[i].resource, NULL);
5560 }
5561 free(ice->state.genx);
5562
5563 for (int i = 0; i < 4; i++) {
5564 pipe_so_target_reference(&ice->state.so_target[i], NULL);
5565 }
5566
5567 for (unsigned i = 0; i < ice->state.framebuffer.nr_cbufs; i++) {
5568 pipe_surface_reference(&ice->state.framebuffer.cbufs[i], NULL);
5569 }
5570 pipe_surface_reference(&ice->state.framebuffer.zsbuf, NULL);
5571
5572 for (int stage = 0; stage < MESA_SHADER_STAGES; stage++) {
5573 struct iris_shader_state *shs = &ice->state.shaders[stage];
5574 pipe_resource_reference(&shs->sampler_table.res, NULL);
5575 for (int i = 0; i < PIPE_MAX_CONSTANT_BUFFERS; i++) {
5576 pipe_resource_reference(&shs->constbuf[i].buffer, NULL);
5577 pipe_resource_reference(&shs->constbuf_surf_state[i].res, NULL);
5578 }
5579 for (int i = 0; i < PIPE_MAX_SHADER_IMAGES; i++) {
5580 pipe_resource_reference(&shs->image[i].base.resource, NULL);
5581 pipe_resource_reference(&shs->image[i].surface_state.res, NULL);
5582 }
5583 for (int i = 0; i < PIPE_MAX_SHADER_BUFFERS; i++) {
5584 pipe_resource_reference(&shs->ssbo[i].buffer, NULL);
5585 pipe_resource_reference(&shs->ssbo_surf_state[i].res, NULL);
5586 }
5587 for (int i = 0; i < IRIS_MAX_TEXTURE_SAMPLERS; i++) {
5588 pipe_sampler_view_reference((struct pipe_sampler_view **)
5589 &shs->textures[i], NULL);
5590 }
5591 }
5592
5593 pipe_resource_reference(&ice->state.grid_size.res, NULL);
5594 pipe_resource_reference(&ice->state.grid_surf_state.res, NULL);
5595
5596 pipe_resource_reference(&ice->state.null_fb.res, NULL);
5597 pipe_resource_reference(&ice->state.unbound_tex.res, NULL);
5598
5599 pipe_resource_reference(&ice->state.last_res.cc_vp, NULL);
5600 pipe_resource_reference(&ice->state.last_res.sf_cl_vp, NULL);
5601 pipe_resource_reference(&ice->state.last_res.color_calc, NULL);
5602 pipe_resource_reference(&ice->state.last_res.scissor, NULL);
5603 pipe_resource_reference(&ice->state.last_res.blend, NULL);
5604 pipe_resource_reference(&ice->state.last_res.index_buffer, NULL);
5605 pipe_resource_reference(&ice->state.last_res.cs_thread_ids, NULL);
5606 pipe_resource_reference(&ice->state.last_res.cs_desc, NULL);
5607 }
5608
5609 /* ------------------------------------------------------------------- */
5610
5611 static void
5612 iris_rebind_buffer(struct iris_context *ice,
5613 struct iris_resource *res,
5614 uint64_t old_address)
5615 {
5616 struct pipe_context *ctx = &ice->ctx;
5617 struct iris_screen *screen = (void *) ctx->screen;
5618 struct iris_genx_state *genx = ice->state.genx;
5619
5620 assert(res->base.target == PIPE_BUFFER);
5621
5622 /* Buffers can't be framebuffer attachments, nor display related,
5623 * and we don't have upstream Clover support.
5624 */
5625 assert(!(res->bind_history & (PIPE_BIND_DEPTH_STENCIL |
5626 PIPE_BIND_RENDER_TARGET |
5627 PIPE_BIND_BLENDABLE |
5628 PIPE_BIND_DISPLAY_TARGET |
5629 PIPE_BIND_CURSOR |
5630 PIPE_BIND_COMPUTE_RESOURCE |
5631 PIPE_BIND_GLOBAL)));
5632
5633 if (res->bind_history & PIPE_BIND_VERTEX_BUFFER) {
5634 uint64_t bound_vbs = ice->state.bound_vertex_buffers;
5635 while (bound_vbs) {
5636 const int i = u_bit_scan64(&bound_vbs);
5637 struct iris_vertex_buffer_state *state = &genx->vertex_buffers[i];
5638
5639 /* Update the CPU struct */
5640 STATIC_ASSERT(GENX(VERTEX_BUFFER_STATE_BufferStartingAddress_start) == 32);
5641 STATIC_ASSERT(GENX(VERTEX_BUFFER_STATE_BufferStartingAddress_bits) == 64);
5642 uint64_t *addr = (uint64_t *) &state->state[1];
5643
5644 if (*addr == old_address) {
5645 *addr = res->bo->gtt_offset;
5646 ice->state.dirty |= IRIS_DIRTY_VERTEX_BUFFERS;
5647 }
5648 }
5649 }
5650
5651 /* We don't need to handle PIPE_BIND_INDEX_BUFFER here: we re-emit
5652 * the 3DSTATE_INDEX_BUFFER packet whenever the address changes.
5653 *
5654 * There is also no need to handle these:
5655 * - PIPE_BIND_COMMAND_ARGS_BUFFER (emitted for every indirect draw)
5656 * - PIPE_BIND_QUERY_BUFFER (no persistent state references)
5657 */
5658
5659 if (res->bind_history & PIPE_BIND_STREAM_OUTPUT) {
5660 /* XXX: be careful about resetting vs appending... */
5661 assert(false);
5662 }
5663
5664 for (int s = MESA_SHADER_VERTEX; s < MESA_SHADER_STAGES; s++) {
5665 struct iris_shader_state *shs = &ice->state.shaders[s];
5666 enum pipe_shader_type p_stage = stage_to_pipe(s);
5667
5668 if (res->bind_history & PIPE_BIND_CONSTANT_BUFFER) {
5669 /* Skip constant buffer 0, it's for regular uniforms, not UBOs */
5670 uint32_t bound_cbufs = shs->bound_cbufs & ~1u;
5671 while (bound_cbufs) {
5672 const int i = u_bit_scan(&bound_cbufs);
5673 struct pipe_shader_buffer *cbuf = &shs->constbuf[i];
5674 struct iris_state_ref *surf_state = &shs->constbuf_surf_state[i];
5675
5676 if (res->bo == iris_resource_bo(cbuf->buffer)) {
5677 iris_upload_ubo_ssbo_surf_state(ice, cbuf, surf_state, false);
5678 ice->state.dirty |= IRIS_DIRTY_CONSTANTS_VS << s;
5679 }
5680 }
5681 }
5682
5683 if (res->bind_history & PIPE_BIND_SHADER_BUFFER) {
5684 uint32_t bound_ssbos = shs->bound_ssbos;
5685 while (bound_ssbos) {
5686 const int i = u_bit_scan(&bound_ssbos);
5687 struct pipe_shader_buffer *ssbo = &shs->ssbo[i];
5688
5689 if (res->bo == iris_resource_bo(ssbo->buffer)) {
5690 struct pipe_shader_buffer buf = {
5691 .buffer = &res->base,
5692 .buffer_offset = ssbo->buffer_offset,
5693 .buffer_size = ssbo->buffer_size,
5694 };
5695 iris_set_shader_buffers(ctx, p_stage, i, 1, &buf,
5696 (shs->writable_ssbos >> i) & 1);
5697 }
5698 }
5699 }
5700
5701 if (res->bind_history & PIPE_BIND_SAMPLER_VIEW) {
5702 uint32_t bound_sampler_views = shs->bound_sampler_views;
5703 while (bound_sampler_views) {
5704 const int i = u_bit_scan(&bound_sampler_views);
5705 struct iris_sampler_view *isv = shs->textures[i];
5706
5707 if (res->bo == iris_resource_bo(isv->base.texture)) {
5708 void *map = alloc_surface_states(ice->state.surface_uploader,
5709 &isv->surface_state,
5710 isv->res->aux.sampler_usages);
5711 assert(map);
5712 fill_buffer_surface_state(&screen->isl_dev, isv->res, map,
5713 isv->view.format, isv->view.swizzle,
5714 isv->base.u.buf.offset,
5715 isv->base.u.buf.size);
5716 ice->state.dirty |= IRIS_DIRTY_BINDINGS_VS << s;
5717 }
5718 }
5719 }
5720
5721 if (res->bind_history & PIPE_BIND_SHADER_IMAGE) {
5722 uint32_t bound_image_views = shs->bound_image_views;
5723 while (bound_image_views) {
5724 const int i = u_bit_scan(&bound_image_views);
5725 struct iris_image_view *iv = &shs->image[i];
5726
5727 if (res->bo == iris_resource_bo(iv->base.resource)) {
5728 iris_set_shader_images(ctx, p_stage, i, 1, &iv->base);
5729 }
5730 }
5731 }
5732 }
5733 }
5734
5735 /* ------------------------------------------------------------------- */
5736
5737 static void
5738 iris_load_register_reg32(struct iris_batch *batch, uint32_t dst,
5739 uint32_t src)
5740 {
5741 _iris_emit_lrr(batch, dst, src);
5742 }
5743
5744 static void
5745 iris_load_register_reg64(struct iris_batch *batch, uint32_t dst,
5746 uint32_t src)
5747 {
5748 _iris_emit_lrr(batch, dst, src);
5749 _iris_emit_lrr(batch, dst + 4, src + 4);
5750 }
5751
5752 static void
5753 iris_load_register_imm32(struct iris_batch *batch, uint32_t reg,
5754 uint32_t val)
5755 {
5756 _iris_emit_lri(batch, reg, val);
5757 }
5758
5759 static void
5760 iris_load_register_imm64(struct iris_batch *batch, uint32_t reg,
5761 uint64_t val)
5762 {
5763 _iris_emit_lri(batch, reg + 0, val & 0xffffffff);
5764 _iris_emit_lri(batch, reg + 4, val >> 32);
5765 }
5766
5767 /**
5768 * Emit MI_LOAD_REGISTER_MEM to load a 32-bit MMIO register from a buffer.
5769 */
5770 static void
5771 iris_load_register_mem32(struct iris_batch *batch, uint32_t reg,
5772 struct iris_bo *bo, uint32_t offset)
5773 {
5774 iris_emit_cmd(batch, GENX(MI_LOAD_REGISTER_MEM), lrm) {
5775 lrm.RegisterAddress = reg;
5776 lrm.MemoryAddress = ro_bo(bo, offset);
5777 }
5778 }
5779
5780 /**
5781 * Load a 64-bit value from a buffer into a MMIO register via
5782 * two MI_LOAD_REGISTER_MEM commands.
5783 */
5784 static void
5785 iris_load_register_mem64(struct iris_batch *batch, uint32_t reg,
5786 struct iris_bo *bo, uint32_t offset)
5787 {
5788 iris_load_register_mem32(batch, reg + 0, bo, offset + 0);
5789 iris_load_register_mem32(batch, reg + 4, bo, offset + 4);
5790 }
5791
5792 static void
5793 iris_store_register_mem32(struct iris_batch *batch, uint32_t reg,
5794 struct iris_bo *bo, uint32_t offset,
5795 bool predicated)
5796 {
5797 iris_emit_cmd(batch, GENX(MI_STORE_REGISTER_MEM), srm) {
5798 srm.RegisterAddress = reg;
5799 srm.MemoryAddress = rw_bo(bo, offset);
5800 srm.PredicateEnable = predicated;
5801 }
5802 }
5803
5804 static void
5805 iris_store_register_mem64(struct iris_batch *batch, uint32_t reg,
5806 struct iris_bo *bo, uint32_t offset,
5807 bool predicated)
5808 {
5809 iris_store_register_mem32(batch, reg + 0, bo, offset + 0, predicated);
5810 iris_store_register_mem32(batch, reg + 4, bo, offset + 4, predicated);
5811 }
5812
5813 static void
5814 iris_store_data_imm32(struct iris_batch *batch,
5815 struct iris_bo *bo, uint32_t offset,
5816 uint32_t imm)
5817 {
5818 iris_emit_cmd(batch, GENX(MI_STORE_DATA_IMM), sdi) {
5819 sdi.Address = rw_bo(bo, offset);
5820 sdi.ImmediateData = imm;
5821 }
5822 }
5823
5824 static void
5825 iris_store_data_imm64(struct iris_batch *batch,
5826 struct iris_bo *bo, uint32_t offset,
5827 uint64_t imm)
5828 {
5829 /* Can't use iris_emit_cmd because MI_STORE_DATA_IMM has a length of
5830 * 2 in genxml but it's actually variable length and we need 5 DWords.
5831 */
5832 void *map = iris_get_command_space(batch, 4 * 5);
5833 _iris_pack_command(batch, GENX(MI_STORE_DATA_IMM), map, sdi) {
5834 sdi.DWordLength = 5 - 2;
5835 sdi.Address = rw_bo(bo, offset);
5836 sdi.ImmediateData = imm;
5837 }
5838 }
5839
5840 static void
5841 iris_copy_mem_mem(struct iris_batch *batch,
5842 struct iris_bo *dst_bo, uint32_t dst_offset,
5843 struct iris_bo *src_bo, uint32_t src_offset,
5844 unsigned bytes)
5845 {
5846 /* MI_COPY_MEM_MEM operates on DWords. */
5847 assert(bytes % 4 == 0);
5848 assert(dst_offset % 4 == 0);
5849 assert(src_offset % 4 == 0);
5850
5851 for (unsigned i = 0; i < bytes; i += 4) {
5852 iris_emit_cmd(batch, GENX(MI_COPY_MEM_MEM), cp) {
5853 cp.DestinationMemoryAddress = rw_bo(dst_bo, dst_offset + i);
5854 cp.SourceMemoryAddress = ro_bo(src_bo, src_offset + i);
5855 }
5856 }
5857 }
5858
5859 /* ------------------------------------------------------------------- */
5860
5861 static unsigned
5862 flags_to_post_sync_op(uint32_t flags)
5863 {
5864 if (flags & PIPE_CONTROL_WRITE_IMMEDIATE)
5865 return WriteImmediateData;
5866
5867 if (flags & PIPE_CONTROL_WRITE_DEPTH_COUNT)
5868 return WritePSDepthCount;
5869
5870 if (flags & PIPE_CONTROL_WRITE_TIMESTAMP)
5871 return WriteTimestamp;
5872
5873 return 0;
5874 }
5875
5876 /**
5877 * Do the given flags have a Post Sync or LRI Post Sync operation?
5878 */
5879 static enum pipe_control_flags
5880 get_post_sync_flags(enum pipe_control_flags flags)
5881 {
5882 flags &= PIPE_CONTROL_WRITE_IMMEDIATE |
5883 PIPE_CONTROL_WRITE_DEPTH_COUNT |
5884 PIPE_CONTROL_WRITE_TIMESTAMP |
5885 PIPE_CONTROL_LRI_POST_SYNC_OP;
5886
5887 /* Only one "Post Sync Op" is allowed, and it's mutually exclusive with
5888 * "LRI Post Sync Operation". So more than one bit set would be illegal.
5889 */
5890 assert(util_bitcount(flags) <= 1);
5891
5892 return flags;
5893 }
5894
5895 #define IS_COMPUTE_PIPELINE(batch) (batch->name == IRIS_BATCH_COMPUTE)
5896
5897 /**
5898 * Emit a series of PIPE_CONTROL commands, taking into account any
5899 * workarounds necessary to actually accomplish the caller's request.
5900 *
5901 * Unless otherwise noted, spec quotations in this function come from:
5902 *
5903 * Synchronization of the 3D Pipeline > PIPE_CONTROL Command > Programming
5904 * Restrictions for PIPE_CONTROL.
5905 *
5906 * You should not use this function directly. Use the helpers in
5907 * iris_pipe_control.c instead, which may split the pipe control further.
5908 */
5909 static void
5910 iris_emit_raw_pipe_control(struct iris_batch *batch,
5911 const char *reason,
5912 uint32_t flags,
5913 struct iris_bo *bo,
5914 uint32_t offset,
5915 uint64_t imm)
5916 {
5917 UNUSED const struct gen_device_info *devinfo = &batch->screen->devinfo;
5918 enum pipe_control_flags post_sync_flags = get_post_sync_flags(flags);
5919 enum pipe_control_flags non_lri_post_sync_flags =
5920 post_sync_flags & ~PIPE_CONTROL_LRI_POST_SYNC_OP;
5921
5922 /* Recursive PIPE_CONTROL workarounds --------------------------------
5923 * (http://knowyourmeme.com/memes/xzibit-yo-dawg)
5924 *
5925 * We do these first because we want to look at the original operation,
5926 * rather than any workarounds we set.
5927 */
5928 if (GEN_GEN == 9 && (flags & PIPE_CONTROL_VF_CACHE_INVALIDATE)) {
5929 /* The PIPE_CONTROL "VF Cache Invalidation Enable" bit description
5930 * lists several workarounds:
5931 *
5932 * "Project: SKL, KBL, BXT
5933 *
5934 * If the VF Cache Invalidation Enable is set to a 1 in a
5935 * PIPE_CONTROL, a separate Null PIPE_CONTROL, all bitfields
5936 * sets to 0, with the VF Cache Invalidation Enable set to 0
5937 * needs to be sent prior to the PIPE_CONTROL with VF Cache
5938 * Invalidation Enable set to a 1."
5939 */
5940 iris_emit_raw_pipe_control(batch,
5941 "workaround: recursive VF cache invalidate",
5942 0, NULL, 0, 0);
5943 }
5944
5945 if (GEN_GEN == 9 && IS_COMPUTE_PIPELINE(batch) && post_sync_flags) {
5946 /* Project: SKL / Argument: LRI Post Sync Operation [23]
5947 *
5948 * "PIPECONTROL command with “Command Streamer Stall Enable” must be
5949 * programmed prior to programming a PIPECONTROL command with "LRI
5950 * Post Sync Operation" in GPGPU mode of operation (i.e when
5951 * PIPELINE_SELECT command is set to GPGPU mode of operation)."
5952 *
5953 * The same text exists a few rows below for Post Sync Op.
5954 */
5955 iris_emit_raw_pipe_control(batch,
5956 "workaround: CS stall before gpgpu post-sync",
5957 PIPE_CONTROL_CS_STALL, bo, offset, imm);
5958 }
5959
5960 if (GEN_GEN == 10 && (flags & PIPE_CONTROL_RENDER_TARGET_FLUSH)) {
5961 /* Cannonlake:
5962 * "Before sending a PIPE_CONTROL command with bit 12 set, SW must issue
5963 * another PIPE_CONTROL with Render Target Cache Flush Enable (bit 12)
5964 * = 0 and Pipe Control Flush Enable (bit 7) = 1"
5965 */
5966 iris_emit_raw_pipe_control(batch,
5967 "workaround: PC flush before RT flush",
5968 PIPE_CONTROL_FLUSH_ENABLE, bo, offset, imm);
5969 }
5970
5971 /* "Flush Types" workarounds ---------------------------------------------
5972 * We do these now because they may add post-sync operations or CS stalls.
5973 */
5974
5975 if (GEN_GEN < 11 && flags & PIPE_CONTROL_VF_CACHE_INVALIDATE) {
5976 /* Project: BDW, SKL+ (stopping at CNL) / Argument: VF Invalidate
5977 *
5978 * "'Post Sync Operation' must be enabled to 'Write Immediate Data' or
5979 * 'Write PS Depth Count' or 'Write Timestamp'."
5980 */
5981 if (!bo) {
5982 flags |= PIPE_CONTROL_WRITE_IMMEDIATE;
5983 post_sync_flags |= PIPE_CONTROL_WRITE_IMMEDIATE;
5984 non_lri_post_sync_flags |= PIPE_CONTROL_WRITE_IMMEDIATE;
5985 bo = batch->screen->workaround_bo;
5986 }
5987 }
5988
5989 /* #1130 from Gen10 workarounds page:
5990 *
5991 * "Enable Depth Stall on every Post Sync Op if Render target Cache
5992 * Flush is not enabled in same PIPE CONTROL and Enable Pixel score
5993 * board stall if Render target cache flush is enabled."
5994 *
5995 * Applicable to CNL B0 and C0 steppings only.
5996 *
5997 * The wording here is unclear, and this workaround doesn't look anything
5998 * like the internal bug report recommendations, but leave it be for now...
5999 */
6000 if (GEN_GEN == 10) {
6001 if (flags & PIPE_CONTROL_RENDER_TARGET_FLUSH) {
6002 flags |= PIPE_CONTROL_STALL_AT_SCOREBOARD;
6003 } else if (flags & non_lri_post_sync_flags) {
6004 flags |= PIPE_CONTROL_DEPTH_STALL;
6005 }
6006 }
6007
6008 if (flags & PIPE_CONTROL_DEPTH_STALL) {
6009 /* From the PIPE_CONTROL instruction table, bit 13 (Depth Stall Enable):
6010 *
6011 * "This bit must be DISABLED for operations other than writing
6012 * PS_DEPTH_COUNT."
6013 *
6014 * This seems like nonsense. An Ivybridge workaround requires us to
6015 * emit a PIPE_CONTROL with a depth stall and write immediate post-sync
6016 * operation. Gen8+ requires us to emit depth stalls and depth cache
6017 * flushes together. So, it's hard to imagine this means anything other
6018 * than "we originally intended this to be used for PS_DEPTH_COUNT".
6019 *
6020 * We ignore the supposed restriction and do nothing.
6021 */
6022 }
6023
6024 if (flags & (PIPE_CONTROL_RENDER_TARGET_FLUSH |
6025 PIPE_CONTROL_STALL_AT_SCOREBOARD)) {
6026 /* From the PIPE_CONTROL instruction table, bit 12 and bit 1:
6027 *
6028 * "This bit must be DISABLED for End-of-pipe (Read) fences,
6029 * PS_DEPTH_COUNT or TIMESTAMP queries."
6030 *
6031 * TODO: Implement end-of-pipe checking.
6032 */
6033 assert(!(post_sync_flags & (PIPE_CONTROL_WRITE_DEPTH_COUNT |
6034 PIPE_CONTROL_WRITE_TIMESTAMP)));
6035 }
6036
6037 if (GEN_GEN < 11 && (flags & PIPE_CONTROL_STALL_AT_SCOREBOARD)) {
6038 /* From the PIPE_CONTROL instruction table, bit 1:
6039 *
6040 * "This bit is ignored if Depth Stall Enable is set.
6041 * Further, the render cache is not flushed even if Write Cache
6042 * Flush Enable bit is set."
6043 *
6044 * We assert that the caller doesn't do this combination, to try and
6045 * prevent mistakes. It shouldn't hurt the GPU, though.
6046 *
6047 * We skip this check on Gen11+ as the "Stall at Pixel Scoreboard"
6048 * and "Render Target Flush" combo is explicitly required for BTI
6049 * update workarounds.
6050 */
6051 assert(!(flags & (PIPE_CONTROL_DEPTH_STALL |
6052 PIPE_CONTROL_RENDER_TARGET_FLUSH)));
6053 }
6054
6055 /* PIPE_CONTROL page workarounds ------------------------------------- */
6056
6057 if (GEN_GEN <= 8 && (flags & PIPE_CONTROL_STATE_CACHE_INVALIDATE)) {
6058 /* From the PIPE_CONTROL page itself:
6059 *
6060 * "IVB, HSW, BDW
6061 * Restriction: Pipe_control with CS-stall bit set must be issued
6062 * before a pipe-control command that has the State Cache
6063 * Invalidate bit set."
6064 */
6065 flags |= PIPE_CONTROL_CS_STALL;
6066 }
6067
6068 if (flags & PIPE_CONTROL_FLUSH_LLC) {
6069 /* From the PIPE_CONTROL instruction table, bit 26 (Flush LLC):
6070 *
6071 * "Project: ALL
6072 * SW must always program Post-Sync Operation to "Write Immediate
6073 * Data" when Flush LLC is set."
6074 *
6075 * For now, we just require the caller to do it.
6076 */
6077 assert(flags & PIPE_CONTROL_WRITE_IMMEDIATE);
6078 }
6079
6080 /* "Post-Sync Operation" workarounds -------------------------------- */
6081
6082 /* Project: All / Argument: Global Snapshot Count Reset [19]
6083 *
6084 * "This bit must not be exercised on any product.
6085 * Requires stall bit ([20] of DW1) set."
6086 *
6087 * We don't use this, so we just assert that it isn't used. The
6088 * PIPE_CONTROL instruction page indicates that they intended this
6089 * as a debug feature and don't think it is useful in production,
6090 * but it may actually be usable, should we ever want to.
6091 */
6092 assert((flags & PIPE_CONTROL_GLOBAL_SNAPSHOT_COUNT_RESET) == 0);
6093
6094 if (flags & (PIPE_CONTROL_MEDIA_STATE_CLEAR |
6095 PIPE_CONTROL_INDIRECT_STATE_POINTERS_DISABLE)) {
6096 /* Project: All / Arguments:
6097 *
6098 * - Generic Media State Clear [16]
6099 * - Indirect State Pointers Disable [16]
6100 *
6101 * "Requires stall bit ([20] of DW1) set."
6102 *
6103 * Also, the PIPE_CONTROL instruction table, bit 16 (Generic Media
6104 * State Clear) says:
6105 *
6106 * "PIPECONTROL command with “Command Streamer Stall Enable” must be
6107 * programmed prior to programming a PIPECONTROL command with "Media
6108 * State Clear" set in GPGPU mode of operation"
6109 *
6110 * This is a subset of the earlier rule, so there's nothing to do.
6111 */
6112 flags |= PIPE_CONTROL_CS_STALL;
6113 }
6114
6115 if (flags & PIPE_CONTROL_STORE_DATA_INDEX) {
6116 /* Project: All / Argument: Store Data Index
6117 *
6118 * "Post-Sync Operation ([15:14] of DW1) must be set to something other
6119 * than '0'."
6120 *
6121 * For now, we just assert that the caller does this. We might want to
6122 * automatically add a write to the workaround BO...
6123 */
6124 assert(non_lri_post_sync_flags != 0);
6125 }
6126
6127 if (flags & PIPE_CONTROL_SYNC_GFDT) {
6128 /* Project: All / Argument: Sync GFDT
6129 *
6130 * "Post-Sync Operation ([15:14] of DW1) must be set to something other
6131 * than '0' or 0x2520[13] must be set."
6132 *
6133 * For now, we just assert that the caller does this.
6134 */
6135 assert(non_lri_post_sync_flags != 0);
6136 }
6137
6138 if (flags & PIPE_CONTROL_TLB_INVALIDATE) {
6139 /* Project: IVB+ / Argument: TLB inv
6140 *
6141 * "Requires stall bit ([20] of DW1) set."
6142 *
6143 * Also, from the PIPE_CONTROL instruction table:
6144 *
6145 * "Project: SKL+
6146 * Post Sync Operation or CS stall must be set to ensure a TLB
6147 * invalidation occurs. Otherwise no cycle will occur to the TLB
6148 * cache to invalidate."
6149 *
6150 * This is not a subset of the earlier rule, so there's nothing to do.
6151 */
6152 flags |= PIPE_CONTROL_CS_STALL;
6153 }
6154
6155 if (GEN_GEN == 9 && devinfo->gt == 4) {
6156 /* TODO: The big Skylake GT4 post sync op workaround */
6157 }
6158
6159 /* "GPGPU specific workarounds" (both post-sync and flush) ------------ */
6160
6161 if (IS_COMPUTE_PIPELINE(batch)) {
6162 if (GEN_GEN >= 9 && (flags & PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE)) {
6163 /* Project: SKL+ / Argument: Tex Invalidate
6164 * "Requires stall bit ([20] of DW) set for all GPGPU Workloads."
6165 */
6166 flags |= PIPE_CONTROL_CS_STALL;
6167 }
6168
6169 if (GEN_GEN == 8 && (post_sync_flags ||
6170 (flags & (PIPE_CONTROL_NOTIFY_ENABLE |
6171 PIPE_CONTROL_DEPTH_STALL |
6172 PIPE_CONTROL_RENDER_TARGET_FLUSH |
6173 PIPE_CONTROL_DEPTH_CACHE_FLUSH |
6174 PIPE_CONTROL_DATA_CACHE_FLUSH)))) {
6175 /* Project: BDW / Arguments:
6176 *
6177 * - LRI Post Sync Operation [23]
6178 * - Post Sync Op [15:14]
6179 * - Notify En [8]
6180 * - Depth Stall [13]
6181 * - Render Target Cache Flush [12]
6182 * - Depth Cache Flush [0]
6183 * - DC Flush Enable [5]
6184 *
6185 * "Requires stall bit ([20] of DW) set for all GPGPU and Media
6186 * Workloads."
6187 */
6188 flags |= PIPE_CONTROL_CS_STALL;
6189
6190 /* Also, from the PIPE_CONTROL instruction table, bit 20:
6191 *
6192 * "Project: BDW
6193 * This bit must be always set when PIPE_CONTROL command is
6194 * programmed by GPGPU and MEDIA workloads, except for the cases
6195 * when only Read Only Cache Invalidation bits are set (State
6196 * Cache Invalidation Enable, Instruction cache Invalidation
6197 * Enable, Texture Cache Invalidation Enable, Constant Cache
6198 * Invalidation Enable). This is to WA FFDOP CG issue, this WA
6199 * need not implemented when FF_DOP_CG is disable via "Fixed
6200 * Function DOP Clock Gate Disable" bit in RC_PSMI_CTRL register."
6201 *
6202 * It sounds like we could avoid CS stalls in some cases, but we
6203 * don't currently bother. This list isn't exactly the list above,
6204 * either...
6205 */
6206 }
6207 }
6208
6209 /* "Stall" workarounds ----------------------------------------------
6210 * These have to come after the earlier ones because we may have added
6211 * some additional CS stalls above.
6212 */
6213
6214 if (GEN_GEN < 9 && (flags & PIPE_CONTROL_CS_STALL)) {
6215 /* Project: PRE-SKL, VLV, CHV
6216 *
6217 * "[All Stepping][All SKUs]:
6218 *
6219 * One of the following must also be set:
6220 *
6221 * - Render Target Cache Flush Enable ([12] of DW1)
6222 * - Depth Cache Flush Enable ([0] of DW1)
6223 * - Stall at Pixel Scoreboard ([1] of DW1)
6224 * - Depth Stall ([13] of DW1)
6225 * - Post-Sync Operation ([13] of DW1)
6226 * - DC Flush Enable ([5] of DW1)"
6227 *
6228 * If we don't already have one of those bits set, we choose to add
6229 * "Stall at Pixel Scoreboard". Some of the other bits require a
6230 * CS stall as a workaround (see above), which would send us into
6231 * an infinite recursion of PIPE_CONTROLs. "Stall at Pixel Scoreboard"
6232 * appears to be safe, so we choose that.
6233 */
6234 const uint32_t wa_bits = PIPE_CONTROL_RENDER_TARGET_FLUSH |
6235 PIPE_CONTROL_DEPTH_CACHE_FLUSH |
6236 PIPE_CONTROL_WRITE_IMMEDIATE |
6237 PIPE_CONTROL_WRITE_DEPTH_COUNT |
6238 PIPE_CONTROL_WRITE_TIMESTAMP |
6239 PIPE_CONTROL_STALL_AT_SCOREBOARD |
6240 PIPE_CONTROL_DEPTH_STALL |
6241 PIPE_CONTROL_DATA_CACHE_FLUSH;
6242 if (!(flags & wa_bits))
6243 flags |= PIPE_CONTROL_STALL_AT_SCOREBOARD;
6244 }
6245
6246 /* Emit --------------------------------------------------------------- */
6247
6248 if (INTEL_DEBUG & DEBUG_PIPE_CONTROL) {
6249 fprintf(stderr,
6250 " PC [%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%"PRIx64"]: %s\n",
6251 (flags & PIPE_CONTROL_FLUSH_ENABLE) ? "PipeCon " : "",
6252 (flags & PIPE_CONTROL_CS_STALL) ? "CS " : "",
6253 (flags & PIPE_CONTROL_STALL_AT_SCOREBOARD) ? "Scoreboard " : "",
6254 (flags & PIPE_CONTROL_VF_CACHE_INVALIDATE) ? "VF " : "",
6255 (flags & PIPE_CONTROL_RENDER_TARGET_FLUSH) ? "RT " : "",
6256 (flags & PIPE_CONTROL_CONST_CACHE_INVALIDATE) ? "Const " : "",
6257 (flags & PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE) ? "TC " : "",
6258 (flags & PIPE_CONTROL_DATA_CACHE_FLUSH) ? "DC " : "",
6259 (flags & PIPE_CONTROL_DEPTH_CACHE_FLUSH) ? "ZFlush " : "",
6260 (flags & PIPE_CONTROL_DEPTH_STALL) ? "ZStall " : "",
6261 (flags & PIPE_CONTROL_STATE_CACHE_INVALIDATE) ? "State " : "",
6262 (flags & PIPE_CONTROL_TLB_INVALIDATE) ? "TLB " : "",
6263 (flags & PIPE_CONTROL_INSTRUCTION_INVALIDATE) ? "Inst " : "",
6264 (flags & PIPE_CONTROL_MEDIA_STATE_CLEAR) ? "MediaClear " : "",
6265 (flags & PIPE_CONTROL_NOTIFY_ENABLE) ? "Notify " : "",
6266 (flags & PIPE_CONTROL_GLOBAL_SNAPSHOT_COUNT_RESET) ?
6267 "SnapRes" : "",
6268 (flags & PIPE_CONTROL_INDIRECT_STATE_POINTERS_DISABLE) ?
6269 "ISPDis" : "",
6270 (flags & PIPE_CONTROL_WRITE_IMMEDIATE) ? "WriteImm " : "",
6271 (flags & PIPE_CONTROL_WRITE_DEPTH_COUNT) ? "WriteZCount " : "",
6272 (flags & PIPE_CONTROL_WRITE_TIMESTAMP) ? "WriteTimestamp " : "",
6273 imm, reason);
6274 }
6275
6276 iris_emit_cmd(batch, GENX(PIPE_CONTROL), pc) {
6277 pc.LRIPostSyncOperation = NoLRIOperation;
6278 pc.PipeControlFlushEnable = flags & PIPE_CONTROL_FLUSH_ENABLE;
6279 pc.DCFlushEnable = flags & PIPE_CONTROL_DATA_CACHE_FLUSH;
6280 pc.StoreDataIndex = 0;
6281 pc.CommandStreamerStallEnable = flags & PIPE_CONTROL_CS_STALL;
6282 pc.GlobalSnapshotCountReset =
6283 flags & PIPE_CONTROL_GLOBAL_SNAPSHOT_COUNT_RESET;
6284 pc.TLBInvalidate = flags & PIPE_CONTROL_TLB_INVALIDATE;
6285 pc.GenericMediaStateClear = flags & PIPE_CONTROL_MEDIA_STATE_CLEAR;
6286 pc.StallAtPixelScoreboard = flags & PIPE_CONTROL_STALL_AT_SCOREBOARD;
6287 pc.RenderTargetCacheFlushEnable =
6288 flags & PIPE_CONTROL_RENDER_TARGET_FLUSH;
6289 pc.DepthCacheFlushEnable = flags & PIPE_CONTROL_DEPTH_CACHE_FLUSH;
6290 pc.StateCacheInvalidationEnable =
6291 flags & PIPE_CONTROL_STATE_CACHE_INVALIDATE;
6292 pc.VFCacheInvalidationEnable = flags & PIPE_CONTROL_VF_CACHE_INVALIDATE;
6293 pc.ConstantCacheInvalidationEnable =
6294 flags & PIPE_CONTROL_CONST_CACHE_INVALIDATE;
6295 pc.PostSyncOperation = flags_to_post_sync_op(flags);
6296 pc.DepthStallEnable = flags & PIPE_CONTROL_DEPTH_STALL;
6297 pc.InstructionCacheInvalidateEnable =
6298 flags & PIPE_CONTROL_INSTRUCTION_INVALIDATE;
6299 pc.NotifyEnable = flags & PIPE_CONTROL_NOTIFY_ENABLE;
6300 pc.IndirectStatePointersDisable =
6301 flags & PIPE_CONTROL_INDIRECT_STATE_POINTERS_DISABLE;
6302 pc.TextureCacheInvalidationEnable =
6303 flags & PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE;
6304 pc.Address = rw_bo(bo, offset);
6305 pc.ImmediateData = imm;
6306 }
6307 }
6308
6309 void
6310 genX(emit_urb_setup)(struct iris_context *ice,
6311 struct iris_batch *batch,
6312 const unsigned size[4],
6313 bool tess_present, bool gs_present)
6314 {
6315 const struct gen_device_info *devinfo = &batch->screen->devinfo;
6316 const unsigned push_size_kB = 32;
6317 unsigned entries[4];
6318 unsigned start[4];
6319
6320 ice->shaders.last_vs_entry_size = size[MESA_SHADER_VERTEX];
6321
6322 gen_get_urb_config(devinfo, 1024 * push_size_kB,
6323 1024 * ice->shaders.urb_size,
6324 tess_present, gs_present,
6325 size, entries, start);
6326
6327 for (int i = MESA_SHADER_VERTEX; i <= MESA_SHADER_GEOMETRY; i++) {
6328 iris_emit_cmd(batch, GENX(3DSTATE_URB_VS), urb) {
6329 urb._3DCommandSubOpcode += i;
6330 urb.VSURBStartingAddress = start[i];
6331 urb.VSURBEntryAllocationSize = size[i] - 1;
6332 urb.VSNumberofURBEntries = entries[i];
6333 }
6334 }
6335 }
6336
6337 #if GEN_GEN == 9
6338 /**
6339 * Preemption on Gen9 has to be enabled or disabled in various cases.
6340 *
6341 * See these workarounds for preemption:
6342 * - WaDisableMidObjectPreemptionForGSLineStripAdj
6343 * - WaDisableMidObjectPreemptionForTrifanOrPolygon
6344 * - WaDisableMidObjectPreemptionForLineLoop
6345 * - WA#0798
6346 *
6347 * We don't put this in the vtable because it's only used on Gen9.
6348 */
6349 void
6350 gen9_toggle_preemption(struct iris_context *ice,
6351 struct iris_batch *batch,
6352 const struct pipe_draw_info *draw)
6353 {
6354 struct iris_genx_state *genx = ice->state.genx;
6355 bool object_preemption = true;
6356
6357 /* WaDisableMidObjectPreemptionForGSLineStripAdj
6358 *
6359 * "WA: Disable mid-draw preemption when draw-call is a linestrip_adj
6360 * and GS is enabled."
6361 */
6362 if (draw->mode == PIPE_PRIM_LINE_STRIP_ADJACENCY &&
6363 ice->shaders.prog[MESA_SHADER_GEOMETRY])
6364 object_preemption = false;
6365
6366 /* WaDisableMidObjectPreemptionForTrifanOrPolygon
6367 *
6368 * "TriFan miscompare in Execlist Preemption test. Cut index that is
6369 * on a previous context. End the previous, the resume another context
6370 * with a tri-fan or polygon, and the vertex count is corrupted. If we
6371 * prempt again we will cause corruption.
6372 *
6373 * WA: Disable mid-draw preemption when draw-call has a tri-fan."
6374 */
6375 if (draw->mode == PIPE_PRIM_TRIANGLE_FAN)
6376 object_preemption = false;
6377
6378 /* WaDisableMidObjectPreemptionForLineLoop
6379 *
6380 * "VF Stats Counters Missing a vertex when preemption enabled.
6381 *
6382 * WA: Disable mid-draw preemption when the draw uses a lineloop
6383 * topology."
6384 */
6385 if (draw->mode == PIPE_PRIM_LINE_LOOP)
6386 object_preemption = false;
6387
6388 /* WA#0798
6389 *
6390 * "VF is corrupting GAFS data when preempted on an instance boundary
6391 * and replayed with instancing enabled.
6392 *
6393 * WA: Disable preemption when using instanceing."
6394 */
6395 if (draw->instance_count > 1)
6396 object_preemption = false;
6397
6398 if (genx->object_preemption != object_preemption) {
6399 iris_enable_obj_preemption(batch, object_preemption);
6400 genx->object_preemption = object_preemption;
6401 }
6402 }
6403 #endif
6404
6405 static void
6406 iris_lost_genx_state(struct iris_context *ice, struct iris_batch *batch)
6407 {
6408 struct iris_genx_state *genx = ice->state.genx;
6409
6410 memset(genx->last_index_buffer, 0, sizeof(genx->last_index_buffer));
6411 }
6412
6413 void
6414 genX(init_state)(struct iris_context *ice)
6415 {
6416 struct pipe_context *ctx = &ice->ctx;
6417 struct iris_screen *screen = (struct iris_screen *)ctx->screen;
6418
6419 ctx->create_blend_state = iris_create_blend_state;
6420 ctx->create_depth_stencil_alpha_state = iris_create_zsa_state;
6421 ctx->create_rasterizer_state = iris_create_rasterizer_state;
6422 ctx->create_sampler_state = iris_create_sampler_state;
6423 ctx->create_sampler_view = iris_create_sampler_view;
6424 ctx->create_surface = iris_create_surface;
6425 ctx->create_vertex_elements_state = iris_create_vertex_elements;
6426 ctx->bind_blend_state = iris_bind_blend_state;
6427 ctx->bind_depth_stencil_alpha_state = iris_bind_zsa_state;
6428 ctx->bind_sampler_states = iris_bind_sampler_states;
6429 ctx->bind_rasterizer_state = iris_bind_rasterizer_state;
6430 ctx->bind_vertex_elements_state = iris_bind_vertex_elements_state;
6431 ctx->delete_blend_state = iris_delete_state;
6432 ctx->delete_depth_stencil_alpha_state = iris_delete_state;
6433 ctx->delete_rasterizer_state = iris_delete_state;
6434 ctx->delete_sampler_state = iris_delete_state;
6435 ctx->delete_vertex_elements_state = iris_delete_state;
6436 ctx->set_blend_color = iris_set_blend_color;
6437 ctx->set_clip_state = iris_set_clip_state;
6438 ctx->set_constant_buffer = iris_set_constant_buffer;
6439 ctx->set_shader_buffers = iris_set_shader_buffers;
6440 ctx->set_shader_images = iris_set_shader_images;
6441 ctx->set_sampler_views = iris_set_sampler_views;
6442 ctx->set_tess_state = iris_set_tess_state;
6443 ctx->set_framebuffer_state = iris_set_framebuffer_state;
6444 ctx->set_polygon_stipple = iris_set_polygon_stipple;
6445 ctx->set_sample_mask = iris_set_sample_mask;
6446 ctx->set_scissor_states = iris_set_scissor_states;
6447 ctx->set_stencil_ref = iris_set_stencil_ref;
6448 ctx->set_vertex_buffers = iris_set_vertex_buffers;
6449 ctx->set_viewport_states = iris_set_viewport_states;
6450 ctx->sampler_view_destroy = iris_sampler_view_destroy;
6451 ctx->surface_destroy = iris_surface_destroy;
6452 ctx->draw_vbo = iris_draw_vbo;
6453 ctx->launch_grid = iris_launch_grid;
6454 ctx->create_stream_output_target = iris_create_stream_output_target;
6455 ctx->stream_output_target_destroy = iris_stream_output_target_destroy;
6456 ctx->set_stream_output_targets = iris_set_stream_output_targets;
6457
6458 ice->vtbl.destroy_state = iris_destroy_state;
6459 ice->vtbl.init_render_context = iris_init_render_context;
6460 ice->vtbl.init_compute_context = iris_init_compute_context;
6461 ice->vtbl.upload_render_state = iris_upload_render_state;
6462 ice->vtbl.update_surface_base_address = iris_update_surface_base_address;
6463 ice->vtbl.upload_compute_state = iris_upload_compute_state;
6464 ice->vtbl.emit_raw_pipe_control = iris_emit_raw_pipe_control;
6465 ice->vtbl.rebind_buffer = iris_rebind_buffer;
6466 ice->vtbl.load_register_reg32 = iris_load_register_reg32;
6467 ice->vtbl.load_register_reg64 = iris_load_register_reg64;
6468 ice->vtbl.load_register_imm32 = iris_load_register_imm32;
6469 ice->vtbl.load_register_imm64 = iris_load_register_imm64;
6470 ice->vtbl.load_register_mem32 = iris_load_register_mem32;
6471 ice->vtbl.load_register_mem64 = iris_load_register_mem64;
6472 ice->vtbl.store_register_mem32 = iris_store_register_mem32;
6473 ice->vtbl.store_register_mem64 = iris_store_register_mem64;
6474 ice->vtbl.store_data_imm32 = iris_store_data_imm32;
6475 ice->vtbl.store_data_imm64 = iris_store_data_imm64;
6476 ice->vtbl.copy_mem_mem = iris_copy_mem_mem;
6477 ice->vtbl.derived_program_state_size = iris_derived_program_state_size;
6478 ice->vtbl.store_derived_program_state = iris_store_derived_program_state;
6479 ice->vtbl.create_so_decl_list = iris_create_so_decl_list;
6480 ice->vtbl.populate_vs_key = iris_populate_vs_key;
6481 ice->vtbl.populate_tcs_key = iris_populate_tcs_key;
6482 ice->vtbl.populate_tes_key = iris_populate_tes_key;
6483 ice->vtbl.populate_gs_key = iris_populate_gs_key;
6484 ice->vtbl.populate_fs_key = iris_populate_fs_key;
6485 ice->vtbl.populate_cs_key = iris_populate_cs_key;
6486 ice->vtbl.mocs = mocs;
6487 ice->vtbl.lost_genx_state = iris_lost_genx_state;
6488
6489 ice->state.dirty = ~0ull;
6490
6491 ice->state.statistics_counters_enabled = true;
6492
6493 ice->state.sample_mask = 0xffff;
6494 ice->state.num_viewports = 1;
6495 ice->state.genx = calloc(1, sizeof(struct iris_genx_state));
6496
6497 /* Make a 1x1x1 null surface for unbound textures */
6498 void *null_surf_map =
6499 upload_state(ice->state.surface_uploader, &ice->state.unbound_tex,
6500 4 * GENX(RENDER_SURFACE_STATE_length), 64);
6501 isl_null_fill_state(&screen->isl_dev, null_surf_map, isl_extent3d(1, 1, 1));
6502 ice->state.unbound_tex.offset +=
6503 iris_bo_offset_from_base_address(iris_resource_bo(ice->state.unbound_tex.res));
6504
6505 /* Default all scissor rectangles to be empty regions. */
6506 for (int i = 0; i < IRIS_MAX_VIEWPORTS; i++) {
6507 ice->state.scissors[i] = (struct pipe_scissor_state) {
6508 .minx = 1, .maxx = 0, .miny = 1, .maxy = 0,
6509 };
6510 }
6511 }