nir: Fix num_ssbos when lowering atomic counters
[mesa.git] / src / gallium / drivers / iris / iris_program.c
1 /*
2 * Copyright © 2017 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included
12 * in all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
15 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
19 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
20 * DEALINGS IN THE SOFTWARE.
21 */
22
23 /**
24 * @file iris_program.c
25 *
26 * This file contains the driver interface for compiling shaders.
27 *
28 * See iris_program_cache.c for the in-memory program cache where the
29 * compiled shaders are stored.
30 */
31
32 #include <stdio.h>
33 #include <errno.h>
34 #include "pipe/p_defines.h"
35 #include "pipe/p_state.h"
36 #include "pipe/p_context.h"
37 #include "pipe/p_screen.h"
38 #include "util/u_atomic.h"
39 #include "util/u_upload_mgr.h"
40 #include "util/debug.h"
41 #include "compiler/nir/nir.h"
42 #include "compiler/nir/nir_builder.h"
43 #include "compiler/nir/nir_serialize.h"
44 #include "intel/compiler/brw_compiler.h"
45 #include "intel/compiler/brw_nir.h"
46 #include "iris_context.h"
47 #include "nir/tgsi_to_nir.h"
48
49 #define KEY_INIT_NO_ID(gen) \
50 .base.subgroup_size_type = BRW_SUBGROUP_SIZE_UNIFORM, \
51 .base.tex.swizzles[0 ... MAX_SAMPLERS - 1] = 0x688, \
52 .base.tex.compressed_multisample_layout_mask = ~0, \
53 .base.tex.msaa_16 = (gen >= 9 ? ~0 : 0)
54 #define KEY_INIT(gen) .base.program_string_id = ish->program_id, KEY_INIT_NO_ID(gen)
55
56 static unsigned
57 get_new_program_id(struct iris_screen *screen)
58 {
59 return p_atomic_inc_return(&screen->program_id);
60 }
61
62 static void *
63 upload_state(struct u_upload_mgr *uploader,
64 struct iris_state_ref *ref,
65 unsigned size,
66 unsigned alignment)
67 {
68 void *p = NULL;
69 u_upload_alloc(uploader, 0, size, alignment, &ref->offset, &ref->res, &p);
70 return p;
71 }
72
73 void
74 iris_upload_ubo_ssbo_surf_state(struct iris_context *ice,
75 struct pipe_shader_buffer *buf,
76 struct iris_state_ref *surf_state,
77 bool ssbo)
78 {
79 struct pipe_context *ctx = &ice->ctx;
80 struct iris_screen *screen = (struct iris_screen *) ctx->screen;
81
82 void *map =
83 upload_state(ice->state.surface_uploader, surf_state,
84 screen->isl_dev.ss.size, 64);
85 if (!unlikely(map)) {
86 surf_state->res = NULL;
87 return;
88 }
89
90 struct iris_resource *res = (void *) buf->buffer;
91 struct iris_bo *surf_bo = iris_resource_bo(surf_state->res);
92 surf_state->offset += iris_bo_offset_from_base_address(surf_bo);
93
94 isl_buffer_fill_state(&screen->isl_dev, map,
95 .address = res->bo->gtt_offset + res->offset +
96 buf->buffer_offset,
97 .size_B = buf->buffer_size - res->offset,
98 .format = ssbo ? ISL_FORMAT_RAW
99 : ISL_FORMAT_R32G32B32A32_FLOAT,
100 .swizzle = ISL_SWIZZLE_IDENTITY,
101 .stride_B = 1,
102 .mocs = ice->vtbl.mocs(res->bo));
103 }
104
105 static nir_ssa_def *
106 get_aoa_deref_offset(nir_builder *b,
107 nir_deref_instr *deref,
108 unsigned elem_size)
109 {
110 unsigned array_size = elem_size;
111 nir_ssa_def *offset = nir_imm_int(b, 0);
112
113 while (deref->deref_type != nir_deref_type_var) {
114 assert(deref->deref_type == nir_deref_type_array);
115
116 /* This level's element size is the previous level's array size */
117 nir_ssa_def *index = nir_ssa_for_src(b, deref->arr.index, 1);
118 assert(deref->arr.index.ssa);
119 offset = nir_iadd(b, offset,
120 nir_imul(b, index, nir_imm_int(b, array_size)));
121
122 deref = nir_deref_instr_parent(deref);
123 assert(glsl_type_is_array(deref->type));
124 array_size *= glsl_get_length(deref->type);
125 }
126
127 /* Accessing an invalid surface index with the dataport can result in a
128 * hang. According to the spec "if the index used to select an individual
129 * element is negative or greater than or equal to the size of the array,
130 * the results of the operation are undefined but may not lead to
131 * termination" -- which is one of the possible outcomes of the hang.
132 * Clamp the index to prevent access outside of the array bounds.
133 */
134 return nir_umin(b, offset, nir_imm_int(b, array_size - elem_size));
135 }
136
137 static void
138 iris_lower_storage_image_derefs(nir_shader *nir)
139 {
140 nir_function_impl *impl = nir_shader_get_entrypoint(nir);
141
142 nir_builder b;
143 nir_builder_init(&b, impl);
144
145 nir_foreach_block(block, impl) {
146 nir_foreach_instr_safe(instr, block) {
147 if (instr->type != nir_instr_type_intrinsic)
148 continue;
149
150 nir_intrinsic_instr *intrin = nir_instr_as_intrinsic(instr);
151 switch (intrin->intrinsic) {
152 case nir_intrinsic_image_deref_load:
153 case nir_intrinsic_image_deref_store:
154 case nir_intrinsic_image_deref_atomic_add:
155 case nir_intrinsic_image_deref_atomic_imin:
156 case nir_intrinsic_image_deref_atomic_umin:
157 case nir_intrinsic_image_deref_atomic_imax:
158 case nir_intrinsic_image_deref_atomic_umax:
159 case nir_intrinsic_image_deref_atomic_and:
160 case nir_intrinsic_image_deref_atomic_or:
161 case nir_intrinsic_image_deref_atomic_xor:
162 case nir_intrinsic_image_deref_atomic_exchange:
163 case nir_intrinsic_image_deref_atomic_comp_swap:
164 case nir_intrinsic_image_deref_size:
165 case nir_intrinsic_image_deref_samples:
166 case nir_intrinsic_image_deref_load_raw_intel:
167 case nir_intrinsic_image_deref_store_raw_intel: {
168 nir_deref_instr *deref = nir_src_as_deref(intrin->src[0]);
169 nir_variable *var = nir_deref_instr_get_variable(deref);
170
171 b.cursor = nir_before_instr(&intrin->instr);
172 nir_ssa_def *index =
173 nir_iadd(&b, nir_imm_int(&b, var->data.driver_location),
174 get_aoa_deref_offset(&b, deref, 1));
175 nir_rewrite_image_intrinsic(intrin, index, false);
176 break;
177 }
178
179 default:
180 break;
181 }
182 }
183 }
184 }
185
186 // XXX: need unify_interfaces() at link time...
187
188 /**
189 * Fix an uncompiled shader's stream output info.
190 *
191 * Core Gallium stores output->register_index as a "slot" number, where
192 * slots are assigned consecutively to all outputs in info->outputs_written.
193 * This naive packing of outputs doesn't work for us - we too have slots,
194 * but the layout is defined by the VUE map, which we won't have until we
195 * compile a specific shader variant. So, we remap these and simply store
196 * VARYING_SLOT_* in our copy's output->register_index fields.
197 *
198 * We also fix up VARYING_SLOT_{LAYER,VIEWPORT,PSIZ} to select the Y/Z/W
199 * components of our VUE header. See brw_vue_map.c for the layout.
200 */
201 static void
202 update_so_info(struct pipe_stream_output_info *so_info,
203 uint64_t outputs_written)
204 {
205 uint8_t reverse_map[64] = {};
206 unsigned slot = 0;
207 while (outputs_written) {
208 reverse_map[slot++] = u_bit_scan64(&outputs_written);
209 }
210
211 for (unsigned i = 0; i < so_info->num_outputs; i++) {
212 struct pipe_stream_output *output = &so_info->output[i];
213
214 /* Map Gallium's condensed "slots" back to real VARYING_SLOT_* enums */
215 output->register_index = reverse_map[output->register_index];
216
217 /* The VUE header contains three scalar fields packed together:
218 * - gl_PointSize is stored in VARYING_SLOT_PSIZ.w
219 * - gl_Layer is stored in VARYING_SLOT_PSIZ.y
220 * - gl_ViewportIndex is stored in VARYING_SLOT_PSIZ.z
221 */
222 switch (output->register_index) {
223 case VARYING_SLOT_LAYER:
224 assert(output->num_components == 1);
225 output->register_index = VARYING_SLOT_PSIZ;
226 output->start_component = 1;
227 break;
228 case VARYING_SLOT_VIEWPORT:
229 assert(output->num_components == 1);
230 output->register_index = VARYING_SLOT_PSIZ;
231 output->start_component = 2;
232 break;
233 case VARYING_SLOT_PSIZ:
234 assert(output->num_components == 1);
235 output->start_component = 3;
236 break;
237 }
238
239 //info->outputs_written |= 1ull << output->register_index;
240 }
241 }
242
243 static void
244 setup_vec4_image_sysval(uint32_t *sysvals, uint32_t idx,
245 unsigned offset, unsigned n)
246 {
247 assert(offset % sizeof(uint32_t) == 0);
248
249 for (unsigned i = 0; i < n; ++i)
250 sysvals[i] = BRW_PARAM_IMAGE(idx, offset / sizeof(uint32_t) + i);
251
252 for (unsigned i = n; i < 4; ++i)
253 sysvals[i] = BRW_PARAM_BUILTIN_ZERO;
254 }
255
256 /**
257 * Associate NIR uniform variables with the prog_data->param[] mechanism
258 * used by the backend. Also, decide which UBOs we'd like to push in an
259 * ideal situation (though the backend can reduce this).
260 */
261 static void
262 iris_setup_uniforms(const struct brw_compiler *compiler,
263 void *mem_ctx,
264 nir_shader *nir,
265 struct brw_stage_prog_data *prog_data,
266 enum brw_param_builtin **out_system_values,
267 unsigned *out_num_system_values,
268 unsigned *out_num_cbufs)
269 {
270 UNUSED const struct gen_device_info *devinfo = compiler->devinfo;
271
272 /* The intel compiler assumes that num_uniforms is in bytes. For
273 * scalar that means 4 bytes per uniform slot.
274 *
275 * Ref: brw_nir_lower_uniforms, type_size_scalar_bytes.
276 */
277 nir->num_uniforms *= 4;
278
279 const unsigned IRIS_MAX_SYSTEM_VALUES =
280 PIPE_MAX_SHADER_IMAGES * BRW_IMAGE_PARAM_SIZE;
281 enum brw_param_builtin *system_values =
282 rzalloc_array(mem_ctx, enum brw_param_builtin, IRIS_MAX_SYSTEM_VALUES);
283 unsigned num_system_values = 0;
284
285 unsigned patch_vert_idx = -1;
286 unsigned ucp_idx[IRIS_MAX_CLIP_PLANES];
287 unsigned img_idx[PIPE_MAX_SHADER_IMAGES];
288 memset(ucp_idx, -1, sizeof(ucp_idx));
289 memset(img_idx, -1, sizeof(img_idx));
290
291 nir_function_impl *impl = nir_shader_get_entrypoint(nir);
292
293 nir_builder b;
294 nir_builder_init(&b, impl);
295
296 b.cursor = nir_before_block(nir_start_block(impl));
297 nir_ssa_def *temp_ubo_name = nir_ssa_undef(&b, 1, 32);
298 nir_ssa_def *temp_const_ubo_name = NULL;
299
300 /* Turn system value intrinsics into uniforms */
301 nir_foreach_block(block, impl) {
302 nir_foreach_instr_safe(instr, block) {
303 if (instr->type != nir_instr_type_intrinsic)
304 continue;
305
306 nir_intrinsic_instr *intrin = nir_instr_as_intrinsic(instr);
307 nir_ssa_def *offset;
308
309 switch (intrin->intrinsic) {
310 case nir_intrinsic_load_constant: {
311 /* This one is special because it reads from the shader constant
312 * data and not cbuf0 which gallium uploads for us.
313 */
314 b.cursor = nir_before_instr(instr);
315 nir_ssa_def *offset =
316 nir_iadd_imm(&b, nir_ssa_for_src(&b, intrin->src[0], 1),
317 nir_intrinsic_base(intrin));
318
319 if (temp_const_ubo_name == NULL)
320 temp_const_ubo_name = nir_imm_int(&b, 0);
321
322 nir_intrinsic_instr *load_ubo =
323 nir_intrinsic_instr_create(b.shader, nir_intrinsic_load_ubo);
324 load_ubo->num_components = intrin->num_components;
325 load_ubo->src[0] = nir_src_for_ssa(temp_const_ubo_name);
326 load_ubo->src[1] = nir_src_for_ssa(offset);
327 nir_ssa_dest_init(&load_ubo->instr, &load_ubo->dest,
328 intrin->dest.ssa.num_components,
329 intrin->dest.ssa.bit_size,
330 intrin->dest.ssa.name);
331 nir_builder_instr_insert(&b, &load_ubo->instr);
332
333 nir_ssa_def_rewrite_uses(&intrin->dest.ssa,
334 nir_src_for_ssa(&load_ubo->dest.ssa));
335 nir_instr_remove(&intrin->instr);
336 continue;
337 }
338 case nir_intrinsic_load_user_clip_plane: {
339 unsigned ucp = nir_intrinsic_ucp_id(intrin);
340
341 if (ucp_idx[ucp] == -1) {
342 ucp_idx[ucp] = num_system_values;
343 num_system_values += 4;
344 }
345
346 for (int i = 0; i < 4; i++) {
347 system_values[ucp_idx[ucp] + i] =
348 BRW_PARAM_BUILTIN_CLIP_PLANE(ucp, i);
349 }
350
351 b.cursor = nir_before_instr(instr);
352 offset = nir_imm_int(&b, ucp_idx[ucp] * sizeof(uint32_t));
353 break;
354 }
355 case nir_intrinsic_load_patch_vertices_in:
356 if (patch_vert_idx == -1)
357 patch_vert_idx = num_system_values++;
358
359 system_values[patch_vert_idx] =
360 BRW_PARAM_BUILTIN_PATCH_VERTICES_IN;
361
362 b.cursor = nir_before_instr(instr);
363 offset = nir_imm_int(&b, patch_vert_idx * sizeof(uint32_t));
364 break;
365 case nir_intrinsic_image_deref_load_param_intel: {
366 assert(devinfo->gen < 9);
367 nir_deref_instr *deref = nir_src_as_deref(intrin->src[0]);
368 nir_variable *var = nir_deref_instr_get_variable(deref);
369
370 if (img_idx[var->data.binding] == -1) {
371 /* GL only allows arrays of arrays of images. */
372 assert(glsl_type_is_image(glsl_without_array(var->type)));
373 unsigned num_images = MAX2(1, glsl_get_aoa_size(var->type));
374
375 for (int i = 0; i < num_images; i++) {
376 const unsigned img = var->data.binding + i;
377
378 img_idx[img] = num_system_values;
379 num_system_values += BRW_IMAGE_PARAM_SIZE;
380
381 uint32_t *img_sv = &system_values[img_idx[img]];
382
383 setup_vec4_image_sysval(
384 img_sv + BRW_IMAGE_PARAM_OFFSET_OFFSET, img,
385 offsetof(struct brw_image_param, offset), 2);
386 setup_vec4_image_sysval(
387 img_sv + BRW_IMAGE_PARAM_SIZE_OFFSET, img,
388 offsetof(struct brw_image_param, size), 3);
389 setup_vec4_image_sysval(
390 img_sv + BRW_IMAGE_PARAM_STRIDE_OFFSET, img,
391 offsetof(struct brw_image_param, stride), 4);
392 setup_vec4_image_sysval(
393 img_sv + BRW_IMAGE_PARAM_TILING_OFFSET, img,
394 offsetof(struct brw_image_param, tiling), 3);
395 setup_vec4_image_sysval(
396 img_sv + BRW_IMAGE_PARAM_SWIZZLING_OFFSET, img,
397 offsetof(struct brw_image_param, swizzling), 2);
398 }
399 }
400
401 b.cursor = nir_before_instr(instr);
402 offset = nir_iadd(&b,
403 get_aoa_deref_offset(&b, deref, BRW_IMAGE_PARAM_SIZE * 4),
404 nir_imm_int(&b, img_idx[var->data.binding] * 4 +
405 nir_intrinsic_base(intrin) * 16));
406 break;
407 }
408 default:
409 continue;
410 }
411
412 unsigned comps = nir_intrinsic_dest_components(intrin);
413
414 nir_intrinsic_instr *load =
415 nir_intrinsic_instr_create(nir, nir_intrinsic_load_ubo);
416 load->num_components = comps;
417 load->src[0] = nir_src_for_ssa(temp_ubo_name);
418 load->src[1] = nir_src_for_ssa(offset);
419 nir_ssa_dest_init(&load->instr, &load->dest, comps, 32, NULL);
420 nir_builder_instr_insert(&b, &load->instr);
421 nir_ssa_def_rewrite_uses(&intrin->dest.ssa,
422 nir_src_for_ssa(&load->dest.ssa));
423 nir_instr_remove(instr);
424 }
425 }
426
427 nir_validate_shader(nir, "before remapping");
428
429 /* Uniforms are stored in constant buffer 0, the
430 * user-facing UBOs are indexed by one. So if any constant buffer is
431 * needed, the constant buffer 0 will be needed, so account for it.
432 */
433 unsigned num_cbufs = nir->info.num_ubos;
434 if (num_cbufs || nir->num_uniforms)
435 num_cbufs++;
436
437 /* Place the new params in a new cbuf. */
438 if (num_system_values > 0) {
439 unsigned sysval_cbuf_index = num_cbufs;
440 num_cbufs++;
441
442 system_values = reralloc(mem_ctx, system_values, enum brw_param_builtin,
443 num_system_values);
444
445 nir_foreach_block(block, impl) {
446 nir_foreach_instr_safe(instr, block) {
447 if (instr->type != nir_instr_type_intrinsic)
448 continue;
449
450 nir_intrinsic_instr *load = nir_instr_as_intrinsic(instr);
451
452 if (load->intrinsic != nir_intrinsic_load_ubo)
453 continue;
454
455 b.cursor = nir_before_instr(instr);
456
457 assert(load->src[0].is_ssa);
458
459 if (load->src[0].ssa == temp_ubo_name) {
460 nir_ssa_def *imm = nir_imm_int(&b, sysval_cbuf_index);
461 nir_instr_rewrite_src(instr, &load->src[0],
462 nir_src_for_ssa(imm));
463 }
464 }
465 }
466
467 /* We need to fold the new iadds for brw_nir_analyze_ubo_ranges */
468 nir_opt_constant_folding(nir);
469 } else {
470 ralloc_free(system_values);
471 system_values = NULL;
472 }
473
474 assert(num_cbufs < PIPE_MAX_CONSTANT_BUFFERS);
475 nir_validate_shader(nir, "after remap");
476
477 /* We don't use params[], but fs_visitor::nir_setup_uniforms() asserts
478 * about it for compute shaders, so go ahead and make some fake ones
479 * which the backend will dead code eliminate.
480 */
481 prog_data->nr_params = nir->num_uniforms / 4;
482 prog_data->param = rzalloc_array(mem_ctx, uint32_t, prog_data->nr_params);
483
484 /* Constant loads (if any) need to go at the end of the constant buffers so
485 * we need to know num_cbufs before we can lower to them.
486 */
487 if (temp_const_ubo_name != NULL) {
488 nir_load_const_instr *const_ubo_index =
489 nir_instr_as_load_const(temp_const_ubo_name->parent_instr);
490 assert(const_ubo_index->def.bit_size == 32);
491 const_ubo_index->value[0].u32 = num_cbufs;
492 }
493
494 *out_system_values = system_values;
495 *out_num_system_values = num_system_values;
496 *out_num_cbufs = num_cbufs;
497 }
498
499 static const char *surface_group_names[] = {
500 [IRIS_SURFACE_GROUP_RENDER_TARGET] = "render target",
501 [IRIS_SURFACE_GROUP_RENDER_TARGET_READ] = "non-coherent render target read",
502 [IRIS_SURFACE_GROUP_CS_WORK_GROUPS] = "CS work groups",
503 [IRIS_SURFACE_GROUP_TEXTURE] = "texture",
504 [IRIS_SURFACE_GROUP_UBO] = "ubo",
505 [IRIS_SURFACE_GROUP_SSBO] = "ssbo",
506 [IRIS_SURFACE_GROUP_IMAGE] = "image",
507 };
508
509 static void
510 iris_print_binding_table(FILE *fp, const char *name,
511 const struct iris_binding_table *bt)
512 {
513 STATIC_ASSERT(ARRAY_SIZE(surface_group_names) == IRIS_SURFACE_GROUP_COUNT);
514
515 uint32_t total = 0;
516 uint32_t compacted = 0;
517
518 for (int i = 0; i < IRIS_SURFACE_GROUP_COUNT; i++) {
519 uint32_t size = bt->sizes[i];
520 total += size;
521 if (size)
522 compacted += util_bitcount64(bt->used_mask[i]);
523 }
524
525 if (total == 0) {
526 fprintf(fp, "Binding table for %s is empty\n\n", name);
527 return;
528 }
529
530 if (total != compacted) {
531 fprintf(fp, "Binding table for %s "
532 "(compacted to %u entries from %u entries)\n",
533 name, compacted, total);
534 } else {
535 fprintf(fp, "Binding table for %s (%u entries)\n", name, total);
536 }
537
538 uint32_t entry = 0;
539 for (int i = 0; i < IRIS_SURFACE_GROUP_COUNT; i++) {
540 uint64_t mask = bt->used_mask[i];
541 while (mask) {
542 int index = u_bit_scan64(&mask);
543 fprintf(fp, " [%u] %s #%d\n", entry++, surface_group_names[i], index);
544 }
545 }
546 fprintf(fp, "\n");
547 }
548
549 enum {
550 /* Max elements in a surface group. */
551 SURFACE_GROUP_MAX_ELEMENTS = 64,
552 };
553
554 /**
555 * Map a <group, index> pair to a binding table index.
556 *
557 * For example: <UBO, 5> => binding table index 12
558 */
559 uint32_t
560 iris_group_index_to_bti(const struct iris_binding_table *bt,
561 enum iris_surface_group group, uint32_t index)
562 {
563 assert(index < bt->sizes[group]);
564 uint64_t mask = bt->used_mask[group];
565 uint64_t bit = 1ull << index;
566 if (bit & mask) {
567 return bt->offsets[group] + util_bitcount64((bit - 1) & mask);
568 } else {
569 return IRIS_SURFACE_NOT_USED;
570 }
571 }
572
573 /**
574 * Map a binding table index back to a <group, index> pair.
575 *
576 * For example: binding table index 12 => <UBO, 5>
577 */
578 uint32_t
579 iris_bti_to_group_index(const struct iris_binding_table *bt,
580 enum iris_surface_group group, uint32_t bti)
581 {
582 uint64_t used_mask = bt->used_mask[group];
583 assert(bti >= bt->offsets[group]);
584
585 uint32_t c = bti - bt->offsets[group];
586 while (used_mask) {
587 int i = u_bit_scan64(&used_mask);
588 if (c == 0)
589 return i;
590 c--;
591 }
592
593 return IRIS_SURFACE_NOT_USED;
594 }
595
596 static void
597 rewrite_src_with_bti(nir_builder *b, struct iris_binding_table *bt,
598 nir_instr *instr, nir_src *src,
599 enum iris_surface_group group)
600 {
601 assert(bt->sizes[group] > 0);
602
603 b->cursor = nir_before_instr(instr);
604 nir_ssa_def *bti;
605 if (nir_src_is_const(*src)) {
606 uint32_t index = nir_src_as_uint(*src);
607 bti = nir_imm_intN_t(b, iris_group_index_to_bti(bt, group, index),
608 src->ssa->bit_size);
609 } else {
610 /* Indirect usage makes all the surfaces of the group to be available,
611 * so we can just add the base.
612 */
613 assert(bt->used_mask[group] == BITFIELD64_MASK(bt->sizes[group]));
614 bti = nir_iadd_imm(b, src->ssa, bt->offsets[group]);
615 }
616 nir_instr_rewrite_src(instr, src, nir_src_for_ssa(bti));
617 }
618
619 static void
620 mark_used_with_src(struct iris_binding_table *bt, nir_src *src,
621 enum iris_surface_group group)
622 {
623 assert(bt->sizes[group] > 0);
624
625 if (nir_src_is_const(*src)) {
626 uint64_t index = nir_src_as_uint(*src);
627 assert(index < bt->sizes[group]);
628 bt->used_mask[group] |= 1ull << index;
629 } else {
630 /* There's an indirect usage, we need all the surfaces. */
631 bt->used_mask[group] = BITFIELD64_MASK(bt->sizes[group]);
632 }
633 }
634
635 static bool
636 skip_compacting_binding_tables(void)
637 {
638 static int skip = -1;
639 if (skip < 0)
640 skip = env_var_as_boolean("INTEL_DISABLE_COMPACT_BINDING_TABLE", false);
641 return skip;
642 }
643
644 /**
645 * Set up the binding table indices and apply to the shader.
646 */
647 static void
648 iris_setup_binding_table(const struct gen_device_info *devinfo,
649 struct nir_shader *nir,
650 struct iris_binding_table *bt,
651 unsigned num_render_targets,
652 unsigned num_system_values,
653 unsigned num_cbufs)
654 {
655 const struct shader_info *info = &nir->info;
656
657 memset(bt, 0, sizeof(*bt));
658
659 /* Set the sizes for each surface group. For some groups, we already know
660 * upfront how many will be used, so mark them.
661 */
662 if (info->stage == MESA_SHADER_FRAGMENT) {
663 bt->sizes[IRIS_SURFACE_GROUP_RENDER_TARGET] = num_render_targets;
664 /* All render targets used. */
665 bt->used_mask[IRIS_SURFACE_GROUP_RENDER_TARGET] =
666 BITFIELD64_MASK(num_render_targets);
667
668 /* Setup render target read surface group inorder to support non-coherent
669 * framebuffer fetch on Gen8
670 */
671 if (devinfo->gen == 8 && info->outputs_read) {
672 bt->sizes[IRIS_SURFACE_GROUP_RENDER_TARGET_READ] = num_render_targets;
673 bt->used_mask[IRIS_SURFACE_GROUP_RENDER_TARGET_READ] =
674 BITFIELD64_MASK(num_render_targets);
675 }
676 } else if (info->stage == MESA_SHADER_COMPUTE) {
677 bt->sizes[IRIS_SURFACE_GROUP_CS_WORK_GROUPS] = 1;
678 }
679
680 bt->sizes[IRIS_SURFACE_GROUP_TEXTURE] = util_last_bit(info->textures_used);
681 bt->used_mask[IRIS_SURFACE_GROUP_TEXTURE] = info->textures_used;
682
683 bt->sizes[IRIS_SURFACE_GROUP_IMAGE] = info->num_images;
684
685 /* Allocate an extra slot in the UBO section for NIR constants.
686 * Binding table compaction will remove it if unnecessary.
687 *
688 * We don't include them in iris_compiled_shader::num_cbufs because
689 * they are uploaded separately from shs->constbuf[], but from a shader
690 * point of view, they're another UBO (at the end of the section).
691 */
692 bt->sizes[IRIS_SURFACE_GROUP_UBO] = num_cbufs + 1;
693
694 bt->sizes[IRIS_SURFACE_GROUP_SSBO] = info->num_ssbos;
695
696 for (int i = 0; i < IRIS_SURFACE_GROUP_COUNT; i++)
697 assert(bt->sizes[i] <= SURFACE_GROUP_MAX_ELEMENTS);
698
699 /* Mark surfaces used for the cases we don't have the information available
700 * upfront.
701 */
702 nir_function_impl *impl = nir_shader_get_entrypoint(nir);
703 nir_foreach_block (block, impl) {
704 nir_foreach_instr (instr, block) {
705 if (instr->type != nir_instr_type_intrinsic)
706 continue;
707
708 nir_intrinsic_instr *intrin = nir_instr_as_intrinsic(instr);
709 switch (intrin->intrinsic) {
710 case nir_intrinsic_load_num_work_groups:
711 bt->used_mask[IRIS_SURFACE_GROUP_CS_WORK_GROUPS] = 1;
712 break;
713
714 case nir_intrinsic_load_output:
715 if (devinfo->gen == 8) {
716 mark_used_with_src(bt, &intrin->src[0],
717 IRIS_SURFACE_GROUP_RENDER_TARGET_READ);
718 }
719 break;
720
721 case nir_intrinsic_image_size:
722 case nir_intrinsic_image_load:
723 case nir_intrinsic_image_store:
724 case nir_intrinsic_image_atomic_add:
725 case nir_intrinsic_image_atomic_imin:
726 case nir_intrinsic_image_atomic_umin:
727 case nir_intrinsic_image_atomic_imax:
728 case nir_intrinsic_image_atomic_umax:
729 case nir_intrinsic_image_atomic_and:
730 case nir_intrinsic_image_atomic_or:
731 case nir_intrinsic_image_atomic_xor:
732 case nir_intrinsic_image_atomic_exchange:
733 case nir_intrinsic_image_atomic_comp_swap:
734 case nir_intrinsic_image_load_raw_intel:
735 case nir_intrinsic_image_store_raw_intel:
736 mark_used_with_src(bt, &intrin->src[0], IRIS_SURFACE_GROUP_IMAGE);
737 break;
738
739 case nir_intrinsic_load_ubo:
740 mark_used_with_src(bt, &intrin->src[0], IRIS_SURFACE_GROUP_UBO);
741 break;
742
743 case nir_intrinsic_store_ssbo:
744 mark_used_with_src(bt, &intrin->src[1], IRIS_SURFACE_GROUP_SSBO);
745 break;
746
747 case nir_intrinsic_get_buffer_size:
748 case nir_intrinsic_ssbo_atomic_add:
749 case nir_intrinsic_ssbo_atomic_imin:
750 case nir_intrinsic_ssbo_atomic_umin:
751 case nir_intrinsic_ssbo_atomic_imax:
752 case nir_intrinsic_ssbo_atomic_umax:
753 case nir_intrinsic_ssbo_atomic_and:
754 case nir_intrinsic_ssbo_atomic_or:
755 case nir_intrinsic_ssbo_atomic_xor:
756 case nir_intrinsic_ssbo_atomic_exchange:
757 case nir_intrinsic_ssbo_atomic_comp_swap:
758 case nir_intrinsic_ssbo_atomic_fmin:
759 case nir_intrinsic_ssbo_atomic_fmax:
760 case nir_intrinsic_ssbo_atomic_fcomp_swap:
761 case nir_intrinsic_load_ssbo:
762 mark_used_with_src(bt, &intrin->src[0], IRIS_SURFACE_GROUP_SSBO);
763 break;
764
765 default:
766 break;
767 }
768 }
769 }
770
771 /* When disable we just mark everything as used. */
772 if (unlikely(skip_compacting_binding_tables())) {
773 for (int i = 0; i < IRIS_SURFACE_GROUP_COUNT; i++)
774 bt->used_mask[i] = BITFIELD64_MASK(bt->sizes[i]);
775 }
776
777 /* Calculate the offsets and the binding table size based on the used
778 * surfaces. After this point, the functions to go between "group indices"
779 * and binding table indices can be used.
780 */
781 uint32_t next = 0;
782 for (int i = 0; i < IRIS_SURFACE_GROUP_COUNT; i++) {
783 if (bt->used_mask[i] != 0) {
784 bt->offsets[i] = next;
785 next += util_bitcount64(bt->used_mask[i]);
786 }
787 }
788 bt->size_bytes = next * 4;
789
790 if (unlikely(INTEL_DEBUG & DEBUG_BT)) {
791 iris_print_binding_table(stderr, gl_shader_stage_name(info->stage), bt);
792 }
793
794 /* Apply the binding table indices. The backend compiler is not expected
795 * to change those, as we haven't set any of the *_start entries in brw
796 * binding_table.
797 */
798 nir_builder b;
799 nir_builder_init(&b, impl);
800
801 nir_foreach_block (block, impl) {
802 nir_foreach_instr (instr, block) {
803 if (instr->type == nir_instr_type_tex) {
804 nir_tex_instr *tex = nir_instr_as_tex(instr);
805 tex->texture_index =
806 iris_group_index_to_bti(bt, IRIS_SURFACE_GROUP_TEXTURE,
807 tex->texture_index);
808 continue;
809 }
810
811 if (instr->type != nir_instr_type_intrinsic)
812 continue;
813
814 nir_intrinsic_instr *intrin = nir_instr_as_intrinsic(instr);
815 switch (intrin->intrinsic) {
816 case nir_intrinsic_image_size:
817 case nir_intrinsic_image_load:
818 case nir_intrinsic_image_store:
819 case nir_intrinsic_image_atomic_add:
820 case nir_intrinsic_image_atomic_imin:
821 case nir_intrinsic_image_atomic_umin:
822 case nir_intrinsic_image_atomic_imax:
823 case nir_intrinsic_image_atomic_umax:
824 case nir_intrinsic_image_atomic_and:
825 case nir_intrinsic_image_atomic_or:
826 case nir_intrinsic_image_atomic_xor:
827 case nir_intrinsic_image_atomic_exchange:
828 case nir_intrinsic_image_atomic_comp_swap:
829 case nir_intrinsic_image_load_raw_intel:
830 case nir_intrinsic_image_store_raw_intel:
831 rewrite_src_with_bti(&b, bt, instr, &intrin->src[0],
832 IRIS_SURFACE_GROUP_IMAGE);
833 break;
834
835 case nir_intrinsic_load_ubo:
836 rewrite_src_with_bti(&b, bt, instr, &intrin->src[0],
837 IRIS_SURFACE_GROUP_UBO);
838 break;
839
840 case nir_intrinsic_store_ssbo:
841 rewrite_src_with_bti(&b, bt, instr, &intrin->src[1],
842 IRIS_SURFACE_GROUP_SSBO);
843 break;
844
845 case nir_intrinsic_load_output:
846 if (devinfo->gen == 8) {
847 rewrite_src_with_bti(&b, bt, instr, &intrin->src[0],
848 IRIS_SURFACE_GROUP_RENDER_TARGET_READ);
849 }
850 break;
851
852 case nir_intrinsic_get_buffer_size:
853 case nir_intrinsic_ssbo_atomic_add:
854 case nir_intrinsic_ssbo_atomic_imin:
855 case nir_intrinsic_ssbo_atomic_umin:
856 case nir_intrinsic_ssbo_atomic_imax:
857 case nir_intrinsic_ssbo_atomic_umax:
858 case nir_intrinsic_ssbo_atomic_and:
859 case nir_intrinsic_ssbo_atomic_or:
860 case nir_intrinsic_ssbo_atomic_xor:
861 case nir_intrinsic_ssbo_atomic_exchange:
862 case nir_intrinsic_ssbo_atomic_comp_swap:
863 case nir_intrinsic_ssbo_atomic_fmin:
864 case nir_intrinsic_ssbo_atomic_fmax:
865 case nir_intrinsic_ssbo_atomic_fcomp_swap:
866 case nir_intrinsic_load_ssbo:
867 rewrite_src_with_bti(&b, bt, instr, &intrin->src[0],
868 IRIS_SURFACE_GROUP_SSBO);
869 break;
870
871 default:
872 break;
873 }
874 }
875 }
876 }
877
878 static void
879 iris_debug_recompile(struct iris_context *ice,
880 struct shader_info *info,
881 const struct brw_base_prog_key *key)
882 {
883 struct iris_screen *screen = (struct iris_screen *) ice->ctx.screen;
884 const struct brw_compiler *c = screen->compiler;
885
886 if (!info)
887 return;
888
889 c->shader_perf_log(&ice->dbg, "Recompiling %s shader for program %s: %s\n",
890 _mesa_shader_stage_to_string(info->stage),
891 info->name ? info->name : "(no identifier)",
892 info->label ? info->label : "");
893
894 const void *old_key =
895 iris_find_previous_compile(ice, info->stage, key->program_string_id);
896
897 brw_debug_key_recompile(c, &ice->dbg, info->stage, old_key, key);
898 }
899
900 /**
901 * Get the shader for the last enabled geometry stage.
902 *
903 * This stage is the one which will feed stream output and the rasterizer.
904 */
905 static gl_shader_stage
906 last_vue_stage(struct iris_context *ice)
907 {
908 if (ice->shaders.uncompiled[MESA_SHADER_GEOMETRY])
909 return MESA_SHADER_GEOMETRY;
910
911 if (ice->shaders.uncompiled[MESA_SHADER_TESS_EVAL])
912 return MESA_SHADER_TESS_EVAL;
913
914 return MESA_SHADER_VERTEX;
915 }
916
917 /**
918 * Compile a vertex shader, and upload the assembly.
919 */
920 static struct iris_compiled_shader *
921 iris_compile_vs(struct iris_context *ice,
922 struct iris_uncompiled_shader *ish,
923 const struct brw_vs_prog_key *key)
924 {
925 struct iris_screen *screen = (struct iris_screen *)ice->ctx.screen;
926 const struct brw_compiler *compiler = screen->compiler;
927 const struct gen_device_info *devinfo = &screen->devinfo;
928 void *mem_ctx = ralloc_context(NULL);
929 struct brw_vs_prog_data *vs_prog_data =
930 rzalloc(mem_ctx, struct brw_vs_prog_data);
931 struct brw_vue_prog_data *vue_prog_data = &vs_prog_data->base;
932 struct brw_stage_prog_data *prog_data = &vue_prog_data->base;
933 enum brw_param_builtin *system_values;
934 unsigned num_system_values;
935 unsigned num_cbufs;
936
937 nir_shader *nir = nir_shader_clone(mem_ctx, ish->nir);
938
939 if (key->nr_userclip_plane_consts) {
940 nir_function_impl *impl = nir_shader_get_entrypoint(nir);
941 nir_lower_clip_vs(nir, (1 << key->nr_userclip_plane_consts) - 1, true);
942 nir_lower_io_to_temporaries(nir, impl, true, false);
943 nir_lower_global_vars_to_local(nir);
944 nir_lower_vars_to_ssa(nir);
945 nir_shader_gather_info(nir, impl);
946 }
947
948 prog_data->use_alt_mode = ish->use_alt_mode;
949
950 iris_setup_uniforms(compiler, mem_ctx, nir, prog_data, &system_values,
951 &num_system_values, &num_cbufs);
952
953 struct iris_binding_table bt;
954 iris_setup_binding_table(devinfo, nir, &bt, /* num_render_targets */ 0,
955 num_system_values, num_cbufs);
956
957 brw_nir_analyze_ubo_ranges(compiler, nir, NULL, prog_data->ubo_ranges);
958
959 brw_compute_vue_map(devinfo,
960 &vue_prog_data->vue_map, nir->info.outputs_written,
961 nir->info.separate_shader);
962
963 /* Don't tell the backend about our clip plane constants, we've already
964 * lowered them in NIR and we don't want it doing it again.
965 */
966 struct brw_vs_prog_key key_no_ucp = *key;
967 key_no_ucp.nr_userclip_plane_consts = 0;
968
969 char *error_str = NULL;
970 const unsigned *program =
971 brw_compile_vs(compiler, &ice->dbg, mem_ctx, &key_no_ucp, vs_prog_data,
972 nir, -1, NULL, &error_str);
973 if (program == NULL) {
974 dbg_printf("Failed to compile vertex shader: %s\n", error_str);
975 ralloc_free(mem_ctx);
976 return false;
977 }
978
979 if (ish->compiled_once) {
980 iris_debug_recompile(ice, &nir->info, &key->base);
981 } else {
982 ish->compiled_once = true;
983 }
984
985 uint32_t *so_decls =
986 ice->vtbl.create_so_decl_list(&ish->stream_output,
987 &vue_prog_data->vue_map);
988
989 struct iris_compiled_shader *shader =
990 iris_upload_shader(ice, IRIS_CACHE_VS, sizeof(*key), key, program,
991 prog_data, so_decls, system_values, num_system_values,
992 num_cbufs, &bt);
993
994 iris_disk_cache_store(screen->disk_cache, ish, shader, key, sizeof(*key));
995
996 ralloc_free(mem_ctx);
997 return shader;
998 }
999
1000 /**
1001 * Update the current vertex shader variant.
1002 *
1003 * Fill out the key, look in the cache, compile and bind if needed.
1004 */
1005 static void
1006 iris_update_compiled_vs(struct iris_context *ice)
1007 {
1008 struct iris_shader_state *shs = &ice->state.shaders[MESA_SHADER_VERTEX];
1009 struct iris_uncompiled_shader *ish =
1010 ice->shaders.uncompiled[MESA_SHADER_VERTEX];
1011 struct iris_screen *screen = (struct iris_screen *)ice->ctx.screen;
1012 const struct gen_device_info *devinfo = &screen->devinfo;
1013
1014 struct brw_vs_prog_key key = { KEY_INIT(devinfo->gen) };
1015 ice->vtbl.populate_vs_key(ice, &ish->nir->info, last_vue_stage(ice), &key);
1016
1017 struct iris_compiled_shader *old = ice->shaders.prog[IRIS_CACHE_VS];
1018 struct iris_compiled_shader *shader =
1019 iris_find_cached_shader(ice, IRIS_CACHE_VS, sizeof(key), &key);
1020
1021 if (!shader)
1022 shader = iris_disk_cache_retrieve(ice, ish, &key, sizeof(key));
1023
1024 if (!shader)
1025 shader = iris_compile_vs(ice, ish, &key);
1026
1027 if (old != shader) {
1028 ice->shaders.prog[IRIS_CACHE_VS] = shader;
1029 ice->state.dirty |= IRIS_DIRTY_VS |
1030 IRIS_DIRTY_BINDINGS_VS |
1031 IRIS_DIRTY_CONSTANTS_VS |
1032 IRIS_DIRTY_VF_SGVS;
1033 shs->sysvals_need_upload = true;
1034
1035 const struct brw_vs_prog_data *vs_prog_data =
1036 (void *) shader->prog_data;
1037 const bool uses_draw_params = vs_prog_data->uses_firstvertex ||
1038 vs_prog_data->uses_baseinstance;
1039 const bool uses_derived_draw_params = vs_prog_data->uses_drawid ||
1040 vs_prog_data->uses_is_indexed_draw;
1041 const bool needs_sgvs_element = uses_draw_params ||
1042 vs_prog_data->uses_instanceid ||
1043 vs_prog_data->uses_vertexid;
1044 bool needs_edge_flag = false;
1045 nir_foreach_variable(var, &ish->nir->inputs) {
1046 if (var->data.location == VERT_ATTRIB_EDGEFLAG)
1047 needs_edge_flag = true;
1048 }
1049
1050 if (ice->state.vs_uses_draw_params != uses_draw_params ||
1051 ice->state.vs_uses_derived_draw_params != uses_derived_draw_params ||
1052 ice->state.vs_needs_edge_flag != needs_edge_flag) {
1053 ice->state.dirty |= IRIS_DIRTY_VERTEX_BUFFERS |
1054 IRIS_DIRTY_VERTEX_ELEMENTS;
1055 }
1056 ice->state.vs_uses_draw_params = uses_draw_params;
1057 ice->state.vs_uses_derived_draw_params = uses_derived_draw_params;
1058 ice->state.vs_needs_sgvs_element = needs_sgvs_element;
1059 ice->state.vs_needs_edge_flag = needs_edge_flag;
1060 }
1061 }
1062
1063 /**
1064 * Get the shader_info for a given stage, or NULL if the stage is disabled.
1065 */
1066 const struct shader_info *
1067 iris_get_shader_info(const struct iris_context *ice, gl_shader_stage stage)
1068 {
1069 const struct iris_uncompiled_shader *ish = ice->shaders.uncompiled[stage];
1070
1071 if (!ish)
1072 return NULL;
1073
1074 const nir_shader *nir = ish->nir;
1075 return &nir->info;
1076 }
1077
1078 /**
1079 * Get the union of TCS output and TES input slots.
1080 *
1081 * TCS and TES need to agree on a common URB entry layout. In particular,
1082 * the data for all patch vertices is stored in a single URB entry (unlike
1083 * GS which has one entry per input vertex). This means that per-vertex
1084 * array indexing needs a stride.
1085 *
1086 * SSO requires locations to match, but doesn't require the number of
1087 * outputs/inputs to match (in fact, the TCS often has extra outputs).
1088 * So, we need to take the extra step of unifying these on the fly.
1089 */
1090 static void
1091 get_unified_tess_slots(const struct iris_context *ice,
1092 uint64_t *per_vertex_slots,
1093 uint32_t *per_patch_slots)
1094 {
1095 const struct shader_info *tcs =
1096 iris_get_shader_info(ice, MESA_SHADER_TESS_CTRL);
1097 const struct shader_info *tes =
1098 iris_get_shader_info(ice, MESA_SHADER_TESS_EVAL);
1099
1100 *per_vertex_slots = tes->inputs_read;
1101 *per_patch_slots = tes->patch_inputs_read;
1102
1103 if (tcs) {
1104 *per_vertex_slots |= tcs->outputs_written;
1105 *per_patch_slots |= tcs->patch_outputs_written;
1106 }
1107 }
1108
1109 /**
1110 * Compile a tessellation control shader, and upload the assembly.
1111 */
1112 static struct iris_compiled_shader *
1113 iris_compile_tcs(struct iris_context *ice,
1114 struct iris_uncompiled_shader *ish,
1115 const struct brw_tcs_prog_key *key)
1116 {
1117 struct iris_screen *screen = (struct iris_screen *)ice->ctx.screen;
1118 const struct brw_compiler *compiler = screen->compiler;
1119 const struct nir_shader_compiler_options *options =
1120 compiler->glsl_compiler_options[MESA_SHADER_TESS_CTRL].NirOptions;
1121 void *mem_ctx = ralloc_context(NULL);
1122 struct brw_tcs_prog_data *tcs_prog_data =
1123 rzalloc(mem_ctx, struct brw_tcs_prog_data);
1124 struct brw_vue_prog_data *vue_prog_data = &tcs_prog_data->base;
1125 struct brw_stage_prog_data *prog_data = &vue_prog_data->base;
1126 const struct gen_device_info *devinfo = &screen->devinfo;
1127 enum brw_param_builtin *system_values = NULL;
1128 unsigned num_system_values = 0;
1129 unsigned num_cbufs = 0;
1130
1131 nir_shader *nir;
1132
1133 struct iris_binding_table bt;
1134
1135 if (ish) {
1136 nir = nir_shader_clone(mem_ctx, ish->nir);
1137
1138 iris_setup_uniforms(compiler, mem_ctx, nir, prog_data, &system_values,
1139 &num_system_values, &num_cbufs);
1140 iris_setup_binding_table(devinfo, nir, &bt, /* num_render_targets */ 0,
1141 num_system_values, num_cbufs);
1142 brw_nir_analyze_ubo_ranges(compiler, nir, NULL, prog_data->ubo_ranges);
1143 } else {
1144 nir = brw_nir_create_passthrough_tcs(mem_ctx, compiler, options, key);
1145
1146 /* Reserve space for passing the default tess levels as constants. */
1147 num_cbufs = 1;
1148 num_system_values = 8;
1149 system_values =
1150 rzalloc_array(mem_ctx, enum brw_param_builtin, num_system_values);
1151 prog_data->param = rzalloc_array(mem_ctx, uint32_t, num_system_values);
1152 prog_data->nr_params = num_system_values;
1153
1154 if (key->tes_primitive_mode == GL_QUADS) {
1155 for (int i = 0; i < 4; i++)
1156 system_values[7 - i] = BRW_PARAM_BUILTIN_TESS_LEVEL_OUTER_X + i;
1157
1158 system_values[3] = BRW_PARAM_BUILTIN_TESS_LEVEL_INNER_X;
1159 system_values[2] = BRW_PARAM_BUILTIN_TESS_LEVEL_INNER_Y;
1160 } else if (key->tes_primitive_mode == GL_TRIANGLES) {
1161 for (int i = 0; i < 3; i++)
1162 system_values[7 - i] = BRW_PARAM_BUILTIN_TESS_LEVEL_OUTER_X + i;
1163
1164 system_values[4] = BRW_PARAM_BUILTIN_TESS_LEVEL_INNER_X;
1165 } else {
1166 assert(key->tes_primitive_mode == GL_ISOLINES);
1167 system_values[7] = BRW_PARAM_BUILTIN_TESS_LEVEL_OUTER_Y;
1168 system_values[6] = BRW_PARAM_BUILTIN_TESS_LEVEL_OUTER_X;
1169 }
1170
1171 /* Manually setup the TCS binding table. */
1172 memset(&bt, 0, sizeof(bt));
1173 bt.sizes[IRIS_SURFACE_GROUP_UBO] = 1;
1174 bt.used_mask[IRIS_SURFACE_GROUP_UBO] = 1;
1175 bt.size_bytes = 4;
1176
1177 prog_data->ubo_ranges[0].length = 1;
1178 }
1179
1180 char *error_str = NULL;
1181 const unsigned *program =
1182 brw_compile_tcs(compiler, &ice->dbg, mem_ctx, key, tcs_prog_data, nir,
1183 -1, NULL, &error_str);
1184 if (program == NULL) {
1185 dbg_printf("Failed to compile control shader: %s\n", error_str);
1186 ralloc_free(mem_ctx);
1187 return false;
1188 }
1189
1190 if (ish) {
1191 if (ish->compiled_once) {
1192 iris_debug_recompile(ice, &nir->info, &key->base);
1193 } else {
1194 ish->compiled_once = true;
1195 }
1196 }
1197
1198 struct iris_compiled_shader *shader =
1199 iris_upload_shader(ice, IRIS_CACHE_TCS, sizeof(*key), key, program,
1200 prog_data, NULL, system_values, num_system_values,
1201 num_cbufs, &bt);
1202
1203 if (ish)
1204 iris_disk_cache_store(screen->disk_cache, ish, shader, key, sizeof(*key));
1205
1206 ralloc_free(mem_ctx);
1207 return shader;
1208 }
1209
1210 /**
1211 * Update the current tessellation control shader variant.
1212 *
1213 * Fill out the key, look in the cache, compile and bind if needed.
1214 */
1215 static void
1216 iris_update_compiled_tcs(struct iris_context *ice)
1217 {
1218 struct iris_shader_state *shs = &ice->state.shaders[MESA_SHADER_TESS_CTRL];
1219 struct iris_uncompiled_shader *tcs =
1220 ice->shaders.uncompiled[MESA_SHADER_TESS_CTRL];
1221 struct iris_screen *screen = (struct iris_screen *)ice->ctx.screen;
1222 const struct brw_compiler *compiler = screen->compiler;
1223 const struct gen_device_info *devinfo = &screen->devinfo;
1224
1225 const struct shader_info *tes_info =
1226 iris_get_shader_info(ice, MESA_SHADER_TESS_EVAL);
1227 struct brw_tcs_prog_key key = {
1228 KEY_INIT_NO_ID(devinfo->gen),
1229 .base.program_string_id = tcs ? tcs->program_id : 0,
1230 .tes_primitive_mode = tes_info->tess.primitive_mode,
1231 .input_vertices =
1232 !tcs || compiler->use_tcs_8_patch ? ice->state.vertices_per_patch : 0,
1233 };
1234 get_unified_tess_slots(ice, &key.outputs_written,
1235 &key.patch_outputs_written);
1236 ice->vtbl.populate_tcs_key(ice, &key);
1237
1238 struct iris_compiled_shader *old = ice->shaders.prog[IRIS_CACHE_TCS];
1239 struct iris_compiled_shader *shader =
1240 iris_find_cached_shader(ice, IRIS_CACHE_TCS, sizeof(key), &key);
1241
1242 if (tcs && !shader)
1243 shader = iris_disk_cache_retrieve(ice, tcs, &key, sizeof(key));
1244
1245 if (!shader)
1246 shader = iris_compile_tcs(ice, tcs, &key);
1247
1248 if (old != shader) {
1249 ice->shaders.prog[IRIS_CACHE_TCS] = shader;
1250 ice->state.dirty |= IRIS_DIRTY_TCS |
1251 IRIS_DIRTY_BINDINGS_TCS |
1252 IRIS_DIRTY_CONSTANTS_TCS;
1253 shs->sysvals_need_upload = true;
1254 }
1255 }
1256
1257 /**
1258 * Compile a tessellation evaluation shader, and upload the assembly.
1259 */
1260 static struct iris_compiled_shader *
1261 iris_compile_tes(struct iris_context *ice,
1262 struct iris_uncompiled_shader *ish,
1263 const struct brw_tes_prog_key *key)
1264 {
1265 struct iris_screen *screen = (struct iris_screen *)ice->ctx.screen;
1266 const struct brw_compiler *compiler = screen->compiler;
1267 void *mem_ctx = ralloc_context(NULL);
1268 struct brw_tes_prog_data *tes_prog_data =
1269 rzalloc(mem_ctx, struct brw_tes_prog_data);
1270 struct brw_vue_prog_data *vue_prog_data = &tes_prog_data->base;
1271 struct brw_stage_prog_data *prog_data = &vue_prog_data->base;
1272 enum brw_param_builtin *system_values;
1273 const struct gen_device_info *devinfo = &screen->devinfo;
1274 unsigned num_system_values;
1275 unsigned num_cbufs;
1276
1277 nir_shader *nir = nir_shader_clone(mem_ctx, ish->nir);
1278
1279 if (key->nr_userclip_plane_consts) {
1280 nir_function_impl *impl = nir_shader_get_entrypoint(nir);
1281 nir_lower_clip_vs(nir, (1 << key->nr_userclip_plane_consts) - 1, true);
1282 nir_lower_io_to_temporaries(nir, impl, true, false);
1283 nir_lower_global_vars_to_local(nir);
1284 nir_lower_vars_to_ssa(nir);
1285 nir_shader_gather_info(nir, impl);
1286 }
1287
1288 iris_setup_uniforms(compiler, mem_ctx, nir, prog_data, &system_values,
1289 &num_system_values, &num_cbufs);
1290
1291 struct iris_binding_table bt;
1292 iris_setup_binding_table(devinfo, nir, &bt, /* num_render_targets */ 0,
1293 num_system_values, num_cbufs);
1294
1295 brw_nir_analyze_ubo_ranges(compiler, nir, NULL, prog_data->ubo_ranges);
1296
1297 struct brw_vue_map input_vue_map;
1298 brw_compute_tess_vue_map(&input_vue_map, key->inputs_read,
1299 key->patch_inputs_read);
1300
1301 char *error_str = NULL;
1302 const unsigned *program =
1303 brw_compile_tes(compiler, &ice->dbg, mem_ctx, key, &input_vue_map,
1304 tes_prog_data, nir, -1, NULL, &error_str);
1305 if (program == NULL) {
1306 dbg_printf("Failed to compile evaluation shader: %s\n", error_str);
1307 ralloc_free(mem_ctx);
1308 return false;
1309 }
1310
1311 if (ish->compiled_once) {
1312 iris_debug_recompile(ice, &nir->info, &key->base);
1313 } else {
1314 ish->compiled_once = true;
1315 }
1316
1317 uint32_t *so_decls =
1318 ice->vtbl.create_so_decl_list(&ish->stream_output,
1319 &vue_prog_data->vue_map);
1320
1321
1322 struct iris_compiled_shader *shader =
1323 iris_upload_shader(ice, IRIS_CACHE_TES, sizeof(*key), key, program,
1324 prog_data, so_decls, system_values, num_system_values,
1325 num_cbufs, &bt);
1326
1327 iris_disk_cache_store(screen->disk_cache, ish, shader, key, sizeof(*key));
1328
1329 ralloc_free(mem_ctx);
1330 return shader;
1331 }
1332
1333 /**
1334 * Update the current tessellation evaluation shader variant.
1335 *
1336 * Fill out the key, look in the cache, compile and bind if needed.
1337 */
1338 static void
1339 iris_update_compiled_tes(struct iris_context *ice)
1340 {
1341 struct iris_shader_state *shs = &ice->state.shaders[MESA_SHADER_TESS_EVAL];
1342 struct iris_uncompiled_shader *ish =
1343 ice->shaders.uncompiled[MESA_SHADER_TESS_EVAL];
1344 struct iris_screen *screen = (struct iris_screen *)ice->ctx.screen;
1345 const struct gen_device_info *devinfo = &screen->devinfo;
1346
1347 struct brw_tes_prog_key key = { KEY_INIT(devinfo->gen) };
1348 get_unified_tess_slots(ice, &key.inputs_read, &key.patch_inputs_read);
1349 ice->vtbl.populate_tes_key(ice, &ish->nir->info, last_vue_stage(ice), &key);
1350
1351 struct iris_compiled_shader *old = ice->shaders.prog[IRIS_CACHE_TES];
1352 struct iris_compiled_shader *shader =
1353 iris_find_cached_shader(ice, IRIS_CACHE_TES, sizeof(key), &key);
1354
1355 if (!shader)
1356 shader = iris_disk_cache_retrieve(ice, ish, &key, sizeof(key));
1357
1358 if (!shader)
1359 shader = iris_compile_tes(ice, ish, &key);
1360
1361 if (old != shader) {
1362 ice->shaders.prog[IRIS_CACHE_TES] = shader;
1363 ice->state.dirty |= IRIS_DIRTY_TES |
1364 IRIS_DIRTY_BINDINGS_TES |
1365 IRIS_DIRTY_CONSTANTS_TES;
1366 shs->sysvals_need_upload = true;
1367 }
1368
1369 /* TODO: Could compare and avoid flagging this. */
1370 const struct shader_info *tes_info = &ish->nir->info;
1371 if (tes_info->system_values_read & (1ull << SYSTEM_VALUE_VERTICES_IN)) {
1372 ice->state.dirty |= IRIS_DIRTY_CONSTANTS_TES;
1373 ice->state.shaders[MESA_SHADER_TESS_EVAL].sysvals_need_upload = true;
1374 }
1375 }
1376
1377 /**
1378 * Compile a geometry shader, and upload the assembly.
1379 */
1380 static struct iris_compiled_shader *
1381 iris_compile_gs(struct iris_context *ice,
1382 struct iris_uncompiled_shader *ish,
1383 const struct brw_gs_prog_key *key)
1384 {
1385 struct iris_screen *screen = (struct iris_screen *)ice->ctx.screen;
1386 const struct brw_compiler *compiler = screen->compiler;
1387 const struct gen_device_info *devinfo = &screen->devinfo;
1388 void *mem_ctx = ralloc_context(NULL);
1389 struct brw_gs_prog_data *gs_prog_data =
1390 rzalloc(mem_ctx, struct brw_gs_prog_data);
1391 struct brw_vue_prog_data *vue_prog_data = &gs_prog_data->base;
1392 struct brw_stage_prog_data *prog_data = &vue_prog_data->base;
1393 enum brw_param_builtin *system_values;
1394 unsigned num_system_values;
1395 unsigned num_cbufs;
1396
1397 nir_shader *nir = nir_shader_clone(mem_ctx, ish->nir);
1398
1399 if (key->nr_userclip_plane_consts) {
1400 nir_function_impl *impl = nir_shader_get_entrypoint(nir);
1401 nir_lower_clip_gs(nir, (1 << key->nr_userclip_plane_consts) - 1);
1402 nir_lower_io_to_temporaries(nir, impl, true, false);
1403 nir_lower_global_vars_to_local(nir);
1404 nir_lower_vars_to_ssa(nir);
1405 nir_shader_gather_info(nir, impl);
1406 }
1407
1408 iris_setup_uniforms(compiler, mem_ctx, nir, prog_data, &system_values,
1409 &num_system_values, &num_cbufs);
1410
1411 struct iris_binding_table bt;
1412 iris_setup_binding_table(devinfo, nir, &bt, /* num_render_targets */ 0,
1413 num_system_values, num_cbufs);
1414
1415 brw_nir_analyze_ubo_ranges(compiler, nir, NULL, prog_data->ubo_ranges);
1416
1417 brw_compute_vue_map(devinfo,
1418 &vue_prog_data->vue_map, nir->info.outputs_written,
1419 nir->info.separate_shader);
1420
1421 char *error_str = NULL;
1422 const unsigned *program =
1423 brw_compile_gs(compiler, &ice->dbg, mem_ctx, key, gs_prog_data, nir,
1424 NULL, -1, NULL, &error_str);
1425 if (program == NULL) {
1426 dbg_printf("Failed to compile geometry shader: %s\n", error_str);
1427 ralloc_free(mem_ctx);
1428 return false;
1429 }
1430
1431 if (ish->compiled_once) {
1432 iris_debug_recompile(ice, &nir->info, &key->base);
1433 } else {
1434 ish->compiled_once = true;
1435 }
1436
1437 uint32_t *so_decls =
1438 ice->vtbl.create_so_decl_list(&ish->stream_output,
1439 &vue_prog_data->vue_map);
1440
1441 struct iris_compiled_shader *shader =
1442 iris_upload_shader(ice, IRIS_CACHE_GS, sizeof(*key), key, program,
1443 prog_data, so_decls, system_values, num_system_values,
1444 num_cbufs, &bt);
1445
1446 iris_disk_cache_store(screen->disk_cache, ish, shader, key, sizeof(*key));
1447
1448 ralloc_free(mem_ctx);
1449 return shader;
1450 }
1451
1452 /**
1453 * Update the current geometry shader variant.
1454 *
1455 * Fill out the key, look in the cache, compile and bind if needed.
1456 */
1457 static void
1458 iris_update_compiled_gs(struct iris_context *ice)
1459 {
1460 struct iris_shader_state *shs = &ice->state.shaders[MESA_SHADER_GEOMETRY];
1461 struct iris_uncompiled_shader *ish =
1462 ice->shaders.uncompiled[MESA_SHADER_GEOMETRY];
1463 struct iris_compiled_shader *old = ice->shaders.prog[IRIS_CACHE_GS];
1464 struct iris_compiled_shader *shader = NULL;
1465
1466 if (ish) {
1467 struct iris_screen *screen = (struct iris_screen *)ice->ctx.screen;
1468 const struct gen_device_info *devinfo = &screen->devinfo;
1469 struct brw_gs_prog_key key = { KEY_INIT(devinfo->gen) };
1470 ice->vtbl.populate_gs_key(ice, &ish->nir->info, last_vue_stage(ice), &key);
1471
1472 shader =
1473 iris_find_cached_shader(ice, IRIS_CACHE_GS, sizeof(key), &key);
1474
1475 if (!shader)
1476 shader = iris_disk_cache_retrieve(ice, ish, &key, sizeof(key));
1477
1478 if (!shader)
1479 shader = iris_compile_gs(ice, ish, &key);
1480 }
1481
1482 if (old != shader) {
1483 ice->shaders.prog[IRIS_CACHE_GS] = shader;
1484 ice->state.dirty |= IRIS_DIRTY_GS |
1485 IRIS_DIRTY_BINDINGS_GS |
1486 IRIS_DIRTY_CONSTANTS_GS;
1487 shs->sysvals_need_upload = true;
1488 }
1489 }
1490
1491 /**
1492 * Compile a fragment (pixel) shader, and upload the assembly.
1493 */
1494 static struct iris_compiled_shader *
1495 iris_compile_fs(struct iris_context *ice,
1496 struct iris_uncompiled_shader *ish,
1497 const struct brw_wm_prog_key *key,
1498 struct brw_vue_map *vue_map)
1499 {
1500 struct iris_screen *screen = (struct iris_screen *)ice->ctx.screen;
1501 const struct brw_compiler *compiler = screen->compiler;
1502 void *mem_ctx = ralloc_context(NULL);
1503 struct brw_wm_prog_data *fs_prog_data =
1504 rzalloc(mem_ctx, struct brw_wm_prog_data);
1505 struct brw_stage_prog_data *prog_data = &fs_prog_data->base;
1506 enum brw_param_builtin *system_values;
1507 const struct gen_device_info *devinfo = &screen->devinfo;
1508 unsigned num_system_values;
1509 unsigned num_cbufs;
1510
1511 nir_shader *nir = nir_shader_clone(mem_ctx, ish->nir);
1512
1513 prog_data->use_alt_mode = ish->use_alt_mode;
1514
1515 iris_setup_uniforms(compiler, mem_ctx, nir, prog_data, &system_values,
1516 &num_system_values, &num_cbufs);
1517
1518 /* Lower output variables to load_output intrinsics before setting up
1519 * binding tables, so iris_setup_binding_table can map any load_output
1520 * intrinsics to IRIS_SURFACE_GROUP_RENDER_TARGET_READ on Gen8 for
1521 * non-coherent framebuffer fetches.
1522 */
1523 brw_nir_lower_fs_outputs(nir);
1524
1525 struct iris_binding_table bt;
1526 iris_setup_binding_table(devinfo, nir, &bt, MAX2(key->nr_color_regions, 1),
1527 num_system_values, num_cbufs);
1528
1529 brw_nir_analyze_ubo_ranges(compiler, nir, NULL, prog_data->ubo_ranges);
1530
1531 char *error_str = NULL;
1532 const unsigned *program =
1533 brw_compile_fs(compiler, &ice->dbg, mem_ctx, key, fs_prog_data,
1534 nir, -1, -1, -1, true, false, vue_map,
1535 NULL, &error_str);
1536 if (program == NULL) {
1537 dbg_printf("Failed to compile fragment shader: %s\n", error_str);
1538 ralloc_free(mem_ctx);
1539 return false;
1540 }
1541
1542 if (ish->compiled_once) {
1543 iris_debug_recompile(ice, &nir->info, &key->base);
1544 } else {
1545 ish->compiled_once = true;
1546 }
1547
1548 struct iris_compiled_shader *shader =
1549 iris_upload_shader(ice, IRIS_CACHE_FS, sizeof(*key), key, program,
1550 prog_data, NULL, system_values, num_system_values,
1551 num_cbufs, &bt);
1552
1553 iris_disk_cache_store(screen->disk_cache, ish, shader, key, sizeof(*key));
1554
1555 ralloc_free(mem_ctx);
1556 return shader;
1557 }
1558
1559 /**
1560 * Update the current fragment shader variant.
1561 *
1562 * Fill out the key, look in the cache, compile and bind if needed.
1563 */
1564 static void
1565 iris_update_compiled_fs(struct iris_context *ice)
1566 {
1567 struct iris_shader_state *shs = &ice->state.shaders[MESA_SHADER_FRAGMENT];
1568 struct iris_uncompiled_shader *ish =
1569 ice->shaders.uncompiled[MESA_SHADER_FRAGMENT];
1570 struct iris_screen *screen = (struct iris_screen *)ice->ctx.screen;
1571 const struct gen_device_info *devinfo = &screen->devinfo;
1572 struct brw_wm_prog_key key = { KEY_INIT(devinfo->gen) };
1573 ice->vtbl.populate_fs_key(ice, &ish->nir->info, &key);
1574
1575 if (ish->nos & (1ull << IRIS_NOS_LAST_VUE_MAP))
1576 key.input_slots_valid = ice->shaders.last_vue_map->slots_valid;
1577
1578 struct iris_compiled_shader *old = ice->shaders.prog[IRIS_CACHE_FS];
1579 struct iris_compiled_shader *shader =
1580 iris_find_cached_shader(ice, IRIS_CACHE_FS, sizeof(key), &key);
1581
1582 if (!shader)
1583 shader = iris_disk_cache_retrieve(ice, ish, &key, sizeof(key));
1584
1585 if (!shader)
1586 shader = iris_compile_fs(ice, ish, &key, ice->shaders.last_vue_map);
1587
1588 if (old != shader) {
1589 // XXX: only need to flag CLIP if barycentric has NONPERSPECTIVE
1590 // toggles. might be able to avoid flagging SBE too.
1591 ice->shaders.prog[IRIS_CACHE_FS] = shader;
1592 ice->state.dirty |= IRIS_DIRTY_FS |
1593 IRIS_DIRTY_BINDINGS_FS |
1594 IRIS_DIRTY_CONSTANTS_FS |
1595 IRIS_DIRTY_WM |
1596 IRIS_DIRTY_CLIP |
1597 IRIS_DIRTY_SBE;
1598 shs->sysvals_need_upload = true;
1599 }
1600 }
1601
1602 /**
1603 * Update the last enabled stage's VUE map.
1604 *
1605 * When the shader feeding the rasterizer's output interface changes, we
1606 * need to re-emit various packets.
1607 */
1608 static void
1609 update_last_vue_map(struct iris_context *ice,
1610 struct brw_stage_prog_data *prog_data)
1611 {
1612 struct brw_vue_prog_data *vue_prog_data = (void *) prog_data;
1613 struct brw_vue_map *vue_map = &vue_prog_data->vue_map;
1614 struct brw_vue_map *old_map = ice->shaders.last_vue_map;
1615 const uint64_t changed_slots =
1616 (old_map ? old_map->slots_valid : 0ull) ^ vue_map->slots_valid;
1617
1618 if (changed_slots & VARYING_BIT_VIEWPORT) {
1619 ice->state.num_viewports =
1620 (vue_map->slots_valid & VARYING_BIT_VIEWPORT) ? IRIS_MAX_VIEWPORTS : 1;
1621 ice->state.dirty |= IRIS_DIRTY_CLIP |
1622 IRIS_DIRTY_SF_CL_VIEWPORT |
1623 IRIS_DIRTY_CC_VIEWPORT |
1624 IRIS_DIRTY_SCISSOR_RECT |
1625 IRIS_DIRTY_UNCOMPILED_FS |
1626 ice->state.dirty_for_nos[IRIS_NOS_LAST_VUE_MAP];
1627 }
1628
1629 if (changed_slots || (old_map && old_map->separate != vue_map->separate)) {
1630 ice->state.dirty |= IRIS_DIRTY_SBE;
1631 }
1632
1633 ice->shaders.last_vue_map = &vue_prog_data->vue_map;
1634 }
1635
1636 /**
1637 * Get the prog_data for a given stage, or NULL if the stage is disabled.
1638 */
1639 static struct brw_vue_prog_data *
1640 get_vue_prog_data(struct iris_context *ice, gl_shader_stage stage)
1641 {
1642 if (!ice->shaders.prog[stage])
1643 return NULL;
1644
1645 return (void *) ice->shaders.prog[stage]->prog_data;
1646 }
1647
1648 // XXX: iris_compiled_shaders are space-leaking :(
1649 // XXX: do remember to unbind them if deleting them.
1650
1651 /**
1652 * Update the current shader variants for the given state.
1653 *
1654 * This should be called on every draw call to ensure that the correct
1655 * shaders are bound. It will also flag any dirty state triggered by
1656 * swapping out those shaders.
1657 */
1658 void
1659 iris_update_compiled_shaders(struct iris_context *ice)
1660 {
1661 const uint64_t dirty = ice->state.dirty;
1662
1663 struct brw_vue_prog_data *old_prog_datas[4];
1664 if (!(dirty & IRIS_DIRTY_URB)) {
1665 for (int i = MESA_SHADER_VERTEX; i <= MESA_SHADER_GEOMETRY; i++)
1666 old_prog_datas[i] = get_vue_prog_data(ice, i);
1667 }
1668
1669 if (dirty & (IRIS_DIRTY_UNCOMPILED_TCS | IRIS_DIRTY_UNCOMPILED_TES)) {
1670 struct iris_uncompiled_shader *tes =
1671 ice->shaders.uncompiled[MESA_SHADER_TESS_EVAL];
1672 if (tes) {
1673 iris_update_compiled_tcs(ice);
1674 iris_update_compiled_tes(ice);
1675 } else {
1676 ice->shaders.prog[IRIS_CACHE_TCS] = NULL;
1677 ice->shaders.prog[IRIS_CACHE_TES] = NULL;
1678 ice->state.dirty |=
1679 IRIS_DIRTY_TCS | IRIS_DIRTY_TES |
1680 IRIS_DIRTY_BINDINGS_TCS | IRIS_DIRTY_BINDINGS_TES |
1681 IRIS_DIRTY_CONSTANTS_TCS | IRIS_DIRTY_CONSTANTS_TES;
1682 }
1683 }
1684
1685 if (dirty & IRIS_DIRTY_UNCOMPILED_VS)
1686 iris_update_compiled_vs(ice);
1687 if (dirty & IRIS_DIRTY_UNCOMPILED_GS)
1688 iris_update_compiled_gs(ice);
1689
1690 if (dirty & (IRIS_DIRTY_UNCOMPILED_GS | IRIS_DIRTY_UNCOMPILED_TES)) {
1691 const struct iris_compiled_shader *gs =
1692 ice->shaders.prog[MESA_SHADER_GEOMETRY];
1693 const struct iris_compiled_shader *tes =
1694 ice->shaders.prog[MESA_SHADER_TESS_EVAL];
1695
1696 bool points_or_lines = false;
1697
1698 if (gs) {
1699 const struct brw_gs_prog_data *gs_prog_data = (void *) gs->prog_data;
1700 points_or_lines =
1701 gs_prog_data->output_topology == _3DPRIM_POINTLIST ||
1702 gs_prog_data->output_topology == _3DPRIM_LINESTRIP;
1703 } else if (tes) {
1704 const struct brw_tes_prog_data *tes_data = (void *) tes->prog_data;
1705 points_or_lines =
1706 tes_data->output_topology == BRW_TESS_OUTPUT_TOPOLOGY_LINE ||
1707 tes_data->output_topology == BRW_TESS_OUTPUT_TOPOLOGY_POINT;
1708 }
1709
1710 if (ice->shaders.output_topology_is_points_or_lines != points_or_lines) {
1711 /* Outbound to XY Clip enables */
1712 ice->shaders.output_topology_is_points_or_lines = points_or_lines;
1713 ice->state.dirty |= IRIS_DIRTY_CLIP;
1714 }
1715 }
1716
1717 gl_shader_stage last_stage = last_vue_stage(ice);
1718 struct iris_compiled_shader *shader = ice->shaders.prog[last_stage];
1719 struct iris_uncompiled_shader *ish = ice->shaders.uncompiled[last_stage];
1720 update_last_vue_map(ice, shader->prog_data);
1721 if (ice->state.streamout != shader->streamout) {
1722 ice->state.streamout = shader->streamout;
1723 ice->state.dirty |= IRIS_DIRTY_SO_DECL_LIST | IRIS_DIRTY_STREAMOUT;
1724 }
1725
1726 if (ice->state.streamout_active) {
1727 for (int i = 0; i < PIPE_MAX_SO_BUFFERS; i++) {
1728 struct iris_stream_output_target *so =
1729 (void *) ice->state.so_target[i];
1730 if (so)
1731 so->stride = ish->stream_output.stride[i] * sizeof(uint32_t);
1732 }
1733 }
1734
1735 if (dirty & IRIS_DIRTY_UNCOMPILED_FS)
1736 iris_update_compiled_fs(ice);
1737
1738 /* Changing shader interfaces may require a URB configuration. */
1739 if (!(dirty & IRIS_DIRTY_URB)) {
1740 for (int i = MESA_SHADER_VERTEX; i <= MESA_SHADER_GEOMETRY; i++) {
1741 struct brw_vue_prog_data *old = old_prog_datas[i];
1742 struct brw_vue_prog_data *new = get_vue_prog_data(ice, i);
1743 if (!!old != !!new ||
1744 (new && new->urb_entry_size != old->urb_entry_size)) {
1745 ice->state.dirty |= IRIS_DIRTY_URB;
1746 break;
1747 }
1748 }
1749 }
1750 }
1751
1752 static struct iris_compiled_shader *
1753 iris_compile_cs(struct iris_context *ice,
1754 struct iris_uncompiled_shader *ish,
1755 const struct brw_cs_prog_key *key)
1756 {
1757 struct iris_screen *screen = (struct iris_screen *)ice->ctx.screen;
1758 const struct brw_compiler *compiler = screen->compiler;
1759 void *mem_ctx = ralloc_context(NULL);
1760 struct brw_cs_prog_data *cs_prog_data =
1761 rzalloc(mem_ctx, struct brw_cs_prog_data);
1762 struct brw_stage_prog_data *prog_data = &cs_prog_data->base;
1763 enum brw_param_builtin *system_values;
1764 const struct gen_device_info *devinfo = &screen->devinfo;
1765 unsigned num_system_values;
1766 unsigned num_cbufs;
1767
1768 nir_shader *nir = nir_shader_clone(mem_ctx, ish->nir);
1769
1770 iris_setup_uniforms(compiler, mem_ctx, nir, prog_data, &system_values,
1771 &num_system_values, &num_cbufs);
1772
1773 struct iris_binding_table bt;
1774 iris_setup_binding_table(devinfo, nir, &bt, /* num_render_targets */ 0,
1775 num_system_values, num_cbufs);
1776
1777 char *error_str = NULL;
1778 const unsigned *program =
1779 brw_compile_cs(compiler, &ice->dbg, mem_ctx, key, cs_prog_data,
1780 nir, -1, NULL, &error_str);
1781 if (program == NULL) {
1782 dbg_printf("Failed to compile compute shader: %s\n", error_str);
1783 ralloc_free(mem_ctx);
1784 return false;
1785 }
1786
1787 if (ish->compiled_once) {
1788 iris_debug_recompile(ice, &nir->info, &key->base);
1789 } else {
1790 ish->compiled_once = true;
1791 }
1792
1793 struct iris_compiled_shader *shader =
1794 iris_upload_shader(ice, IRIS_CACHE_CS, sizeof(*key), key, program,
1795 prog_data, NULL, system_values, num_system_values,
1796 num_cbufs, &bt);
1797
1798 iris_disk_cache_store(screen->disk_cache, ish, shader, key, sizeof(*key));
1799
1800 ralloc_free(mem_ctx);
1801 return shader;
1802 }
1803
1804 void
1805 iris_update_compiled_compute_shader(struct iris_context *ice)
1806 {
1807 struct iris_shader_state *shs = &ice->state.shaders[MESA_SHADER_COMPUTE];
1808 struct iris_uncompiled_shader *ish =
1809 ice->shaders.uncompiled[MESA_SHADER_COMPUTE];
1810
1811 struct iris_screen *screen = (struct iris_screen *)ice->ctx.screen;
1812 const struct gen_device_info *devinfo = &screen->devinfo;
1813 struct brw_cs_prog_key key = { KEY_INIT(devinfo->gen) };
1814 ice->vtbl.populate_cs_key(ice, &key);
1815
1816 struct iris_compiled_shader *old = ice->shaders.prog[IRIS_CACHE_CS];
1817 struct iris_compiled_shader *shader =
1818 iris_find_cached_shader(ice, IRIS_CACHE_CS, sizeof(key), &key);
1819
1820 if (!shader)
1821 shader = iris_disk_cache_retrieve(ice, ish, &key, sizeof(key));
1822
1823 if (!shader)
1824 shader = iris_compile_cs(ice, ish, &key);
1825
1826 if (old != shader) {
1827 ice->shaders.prog[IRIS_CACHE_CS] = shader;
1828 ice->state.dirty |= IRIS_DIRTY_CS |
1829 IRIS_DIRTY_BINDINGS_CS |
1830 IRIS_DIRTY_CONSTANTS_CS;
1831 shs->sysvals_need_upload = true;
1832 }
1833 }
1834
1835 void
1836 iris_fill_cs_push_const_buffer(struct brw_cs_prog_data *cs_prog_data,
1837 uint32_t *dst)
1838 {
1839 assert(cs_prog_data->push.total.size > 0);
1840 assert(cs_prog_data->push.cross_thread.size == 0);
1841 assert(cs_prog_data->push.per_thread.dwords == 1);
1842 assert(cs_prog_data->base.param[0] == BRW_PARAM_BUILTIN_SUBGROUP_ID);
1843 for (unsigned t = 0; t < cs_prog_data->threads; t++)
1844 dst[8 * t] = t;
1845 }
1846
1847 /**
1848 * Allocate scratch BOs as needed for the given per-thread size and stage.
1849 */
1850 struct iris_bo *
1851 iris_get_scratch_space(struct iris_context *ice,
1852 unsigned per_thread_scratch,
1853 gl_shader_stage stage)
1854 {
1855 struct iris_screen *screen = (struct iris_screen *)ice->ctx.screen;
1856 struct iris_bufmgr *bufmgr = screen->bufmgr;
1857 const struct gen_device_info *devinfo = &screen->devinfo;
1858
1859 unsigned encoded_size = ffs(per_thread_scratch) - 11;
1860 assert(encoded_size < (1 << 16));
1861
1862 struct iris_bo **bop = &ice->shaders.scratch_bos[encoded_size][stage];
1863
1864 /* The documentation for 3DSTATE_PS "Scratch Space Base Pointer" says:
1865 *
1866 * "Scratch Space per slice is computed based on 4 sub-slices. SW
1867 * must allocate scratch space enough so that each slice has 4
1868 * slices allowed."
1869 *
1870 * According to the other driver team, this applies to compute shaders
1871 * as well. This is not currently documented at all.
1872 *
1873 * This hack is no longer necessary on Gen11+.
1874 */
1875 unsigned subslice_total = screen->subslice_total;
1876 if (devinfo->gen < 11)
1877 subslice_total = 4 * devinfo->num_slices;
1878 assert(subslice_total >= screen->subslice_total);
1879
1880 if (!*bop) {
1881 unsigned scratch_ids_per_subslice = devinfo->max_cs_threads;
1882 uint32_t max_threads[] = {
1883 [MESA_SHADER_VERTEX] = devinfo->max_vs_threads,
1884 [MESA_SHADER_TESS_CTRL] = devinfo->max_tcs_threads,
1885 [MESA_SHADER_TESS_EVAL] = devinfo->max_tes_threads,
1886 [MESA_SHADER_GEOMETRY] = devinfo->max_gs_threads,
1887 [MESA_SHADER_FRAGMENT] = devinfo->max_wm_threads,
1888 [MESA_SHADER_COMPUTE] = scratch_ids_per_subslice * subslice_total,
1889 };
1890
1891 uint32_t size = per_thread_scratch * max_threads[stage];
1892
1893 *bop = iris_bo_alloc(bufmgr, "scratch", size, IRIS_MEMZONE_SHADER);
1894 }
1895
1896 return *bop;
1897 }
1898
1899 /* ------------------------------------------------------------------- */
1900
1901 /**
1902 * The pipe->create_[stage]_state() driver hooks.
1903 *
1904 * Performs basic NIR preprocessing, records any state dependencies, and
1905 * returns an iris_uncompiled_shader as the Gallium CSO.
1906 *
1907 * Actual shader compilation to assembly happens later, at first use.
1908 */
1909 static void *
1910 iris_create_uncompiled_shader(struct pipe_context *ctx,
1911 nir_shader *nir,
1912 const struct pipe_stream_output_info *so_info)
1913 {
1914 struct iris_context *ice = (void *)ctx;
1915 struct iris_screen *screen = (struct iris_screen *)ctx->screen;
1916 const struct gen_device_info *devinfo = &screen->devinfo;
1917
1918 struct iris_uncompiled_shader *ish =
1919 calloc(1, sizeof(struct iris_uncompiled_shader));
1920 if (!ish)
1921 return NULL;
1922
1923 brw_preprocess_nir(screen->compiler, nir, NULL);
1924
1925 NIR_PASS_V(nir, brw_nir_lower_image_load_store, devinfo);
1926 NIR_PASS_V(nir, iris_lower_storage_image_derefs);
1927
1928 nir_sweep(nir);
1929
1930 if (nir->constant_data_size > 0) {
1931 unsigned data_offset;
1932 u_upload_data(ice->shaders.uploader, 0, nir->constant_data_size,
1933 32, nir->constant_data, &data_offset, &ish->const_data);
1934
1935 struct pipe_shader_buffer psb = {
1936 .buffer = ish->const_data,
1937 .buffer_offset = data_offset,
1938 .buffer_size = nir->constant_data_size,
1939 };
1940 iris_upload_ubo_ssbo_surf_state(ice, &psb, &ish->const_data_state, false);
1941 }
1942
1943 ish->program_id = get_new_program_id(screen);
1944 ish->nir = nir;
1945 if (so_info) {
1946 memcpy(&ish->stream_output, so_info, sizeof(*so_info));
1947 update_so_info(&ish->stream_output, nir->info.outputs_written);
1948 }
1949
1950 /* Save this now before potentially dropping nir->info.name */
1951 if (nir->info.name && strncmp(nir->info.name, "ARB", 3) == 0)
1952 ish->use_alt_mode = true;
1953
1954 if (screen->disk_cache) {
1955 /* Serialize the NIR to a binary blob that we can hash for the disk
1956 * cache. First, drop unnecessary information (like variable names)
1957 * so the serialized NIR is smaller, and also to let us detect more
1958 * isomorphic shaders when hashing, increasing cache hits. We clone
1959 * the NIR before stripping away this info because it can be useful
1960 * when inspecting and debugging shaders.
1961 */
1962 nir_shader *clone = nir_shader_clone(NULL, nir);
1963 nir_strip(clone);
1964
1965 struct blob blob;
1966 blob_init(&blob);
1967 nir_serialize(&blob, clone);
1968 _mesa_sha1_compute(blob.data, blob.size, ish->nir_sha1);
1969 blob_finish(&blob);
1970
1971 ralloc_free(clone);
1972 }
1973
1974 return ish;
1975 }
1976
1977 static struct iris_uncompiled_shader *
1978 iris_create_shader_state(struct pipe_context *ctx,
1979 const struct pipe_shader_state *state)
1980 {
1981 struct nir_shader *nir;
1982
1983 if (state->type == PIPE_SHADER_IR_TGSI)
1984 nir = tgsi_to_nir(state->tokens, ctx->screen);
1985 else
1986 nir = state->ir.nir;
1987
1988 return iris_create_uncompiled_shader(ctx, nir, &state->stream_output);
1989 }
1990
1991 static void *
1992 iris_create_vs_state(struct pipe_context *ctx,
1993 const struct pipe_shader_state *state)
1994 {
1995 struct iris_context *ice = (void *) ctx;
1996 struct iris_screen *screen = (void *) ctx->screen;
1997 struct iris_uncompiled_shader *ish = iris_create_shader_state(ctx, state);
1998
1999 /* User clip planes */
2000 if (ish->nir->info.clip_distance_array_size == 0)
2001 ish->nos |= (1ull << IRIS_NOS_RASTERIZER);
2002
2003 if (screen->precompile) {
2004 const struct gen_device_info *devinfo = &screen->devinfo;
2005 struct brw_vs_prog_key key = { KEY_INIT(devinfo->gen) };
2006
2007 if (!iris_disk_cache_retrieve(ice, ish, &key, sizeof(key)))
2008 iris_compile_vs(ice, ish, &key);
2009 }
2010
2011 return ish;
2012 }
2013
2014 static void *
2015 iris_create_tcs_state(struct pipe_context *ctx,
2016 const struct pipe_shader_state *state)
2017 {
2018 struct iris_context *ice = (void *) ctx;
2019 struct iris_screen *screen = (void *) ctx->screen;
2020 const struct brw_compiler *compiler = screen->compiler;
2021 struct iris_uncompiled_shader *ish = iris_create_shader_state(ctx, state);
2022 struct shader_info *info = &ish->nir->info;
2023
2024 if (screen->precompile) {
2025 const unsigned _GL_TRIANGLES = 0x0004;
2026 const struct gen_device_info *devinfo = &screen->devinfo;
2027 struct brw_tcs_prog_key key = {
2028 KEY_INIT(devinfo->gen),
2029 // XXX: make sure the linker fills this out from the TES...
2030 .tes_primitive_mode =
2031 info->tess.primitive_mode ? info->tess.primitive_mode
2032 : _GL_TRIANGLES,
2033 .outputs_written = info->outputs_written,
2034 .patch_outputs_written = info->patch_outputs_written,
2035 };
2036
2037 /* 8_PATCH mode needs the key to contain the input patch dimensionality.
2038 * We don't have that information, so we randomly guess that the input
2039 * and output patches are the same size. This is a bad guess, but we
2040 * can't do much better.
2041 */
2042 if (compiler->use_tcs_8_patch)
2043 key.input_vertices = info->tess.tcs_vertices_out;
2044
2045 if (!iris_disk_cache_retrieve(ice, ish, &key, sizeof(key)))
2046 iris_compile_tcs(ice, ish, &key);
2047 }
2048
2049 return ish;
2050 }
2051
2052 static void *
2053 iris_create_tes_state(struct pipe_context *ctx,
2054 const struct pipe_shader_state *state)
2055 {
2056 struct iris_context *ice = (void *) ctx;
2057 struct iris_screen *screen = (void *) ctx->screen;
2058 struct iris_uncompiled_shader *ish = iris_create_shader_state(ctx, state);
2059 struct shader_info *info = &ish->nir->info;
2060
2061 /* User clip planes */
2062 if (ish->nir->info.clip_distance_array_size == 0)
2063 ish->nos |= (1ull << IRIS_NOS_RASTERIZER);
2064
2065 if (screen->precompile) {
2066 const struct gen_device_info *devinfo = &screen->devinfo;
2067 struct brw_tes_prog_key key = {
2068 KEY_INIT(devinfo->gen),
2069 // XXX: not ideal, need TCS output/TES input unification
2070 .inputs_read = info->inputs_read,
2071 .patch_inputs_read = info->patch_inputs_read,
2072 };
2073
2074 if (!iris_disk_cache_retrieve(ice, ish, &key, sizeof(key)))
2075 iris_compile_tes(ice, ish, &key);
2076 }
2077
2078 return ish;
2079 }
2080
2081 static void *
2082 iris_create_gs_state(struct pipe_context *ctx,
2083 const struct pipe_shader_state *state)
2084 {
2085 struct iris_context *ice = (void *) ctx;
2086 struct iris_screen *screen = (void *) ctx->screen;
2087 struct iris_uncompiled_shader *ish = iris_create_shader_state(ctx, state);
2088
2089 /* User clip planes */
2090 if (ish->nir->info.clip_distance_array_size == 0)
2091 ish->nos |= (1ull << IRIS_NOS_RASTERIZER);
2092
2093 if (screen->precompile) {
2094 const struct gen_device_info *devinfo = &screen->devinfo;
2095 struct brw_gs_prog_key key = { KEY_INIT(devinfo->gen) };
2096
2097 if (!iris_disk_cache_retrieve(ice, ish, &key, sizeof(key)))
2098 iris_compile_gs(ice, ish, &key);
2099 }
2100
2101 return ish;
2102 }
2103
2104 static void *
2105 iris_create_fs_state(struct pipe_context *ctx,
2106 const struct pipe_shader_state *state)
2107 {
2108 struct iris_context *ice = (void *) ctx;
2109 struct iris_screen *screen = (void *) ctx->screen;
2110 struct iris_uncompiled_shader *ish = iris_create_shader_state(ctx, state);
2111 struct shader_info *info = &ish->nir->info;
2112
2113 ish->nos |= (1ull << IRIS_NOS_FRAMEBUFFER) |
2114 (1ull << IRIS_NOS_DEPTH_STENCIL_ALPHA) |
2115 (1ull << IRIS_NOS_RASTERIZER) |
2116 (1ull << IRIS_NOS_BLEND);
2117
2118 /* The program key needs the VUE map if there are > 16 inputs */
2119 if (util_bitcount64(ish->nir->info.inputs_read &
2120 BRW_FS_VARYING_INPUT_MASK) > 16) {
2121 ish->nos |= (1ull << IRIS_NOS_LAST_VUE_MAP);
2122 }
2123
2124 if (screen->precompile) {
2125 const uint64_t color_outputs = info->outputs_written &
2126 ~(BITFIELD64_BIT(FRAG_RESULT_DEPTH) |
2127 BITFIELD64_BIT(FRAG_RESULT_STENCIL) |
2128 BITFIELD64_BIT(FRAG_RESULT_SAMPLE_MASK));
2129
2130 bool can_rearrange_varyings =
2131 util_bitcount64(info->inputs_read & BRW_FS_VARYING_INPUT_MASK) <= 16;
2132
2133 const struct gen_device_info *devinfo = &screen->devinfo;
2134 struct brw_wm_prog_key key = {
2135 KEY_INIT(devinfo->gen),
2136 .nr_color_regions = util_bitcount(color_outputs),
2137 .coherent_fb_fetch = devinfo->gen >= 9,
2138 .input_slots_valid =
2139 can_rearrange_varyings ? 0 : info->inputs_read | VARYING_BIT_POS,
2140 };
2141
2142 if (!iris_disk_cache_retrieve(ice, ish, &key, sizeof(key)))
2143 iris_compile_fs(ice, ish, &key, NULL);
2144 }
2145
2146 return ish;
2147 }
2148
2149 static void *
2150 iris_create_compute_state(struct pipe_context *ctx,
2151 const struct pipe_compute_state *state)
2152 {
2153 assert(state->ir_type == PIPE_SHADER_IR_NIR);
2154
2155 struct iris_context *ice = (void *) ctx;
2156 struct iris_screen *screen = (void *) ctx->screen;
2157 struct iris_uncompiled_shader *ish =
2158 iris_create_uncompiled_shader(ctx, (void *) state->prog, NULL);
2159
2160 // XXX: disallow more than 64KB of shared variables
2161
2162 if (screen->precompile) {
2163 const struct gen_device_info *devinfo = &screen->devinfo;
2164 struct brw_cs_prog_key key = { KEY_INIT(devinfo->gen) };
2165
2166 if (!iris_disk_cache_retrieve(ice, ish, &key, sizeof(key)))
2167 iris_compile_cs(ice, ish, &key);
2168 }
2169
2170 return ish;
2171 }
2172
2173 /**
2174 * The pipe->delete_[stage]_state() driver hooks.
2175 *
2176 * Frees the iris_uncompiled_shader.
2177 */
2178 static void
2179 iris_delete_shader_state(struct pipe_context *ctx, void *state, gl_shader_stage stage)
2180 {
2181 struct iris_uncompiled_shader *ish = state;
2182 struct iris_context *ice = (void *) ctx;
2183
2184 if (ice->shaders.uncompiled[stage] == ish) {
2185 ice->shaders.uncompiled[stage] = NULL;
2186 ice->state.dirty |= IRIS_DIRTY_UNCOMPILED_VS << stage;
2187 }
2188
2189 if (ish->const_data) {
2190 pipe_resource_reference(&ish->const_data, NULL);
2191 pipe_resource_reference(&ish->const_data_state.res, NULL);
2192 }
2193
2194 ralloc_free(ish->nir);
2195 free(ish);
2196 }
2197
2198 static void
2199 iris_delete_vs_state(struct pipe_context *ctx, void *state)
2200 {
2201 iris_delete_shader_state(ctx, state, MESA_SHADER_VERTEX);
2202 }
2203
2204 static void
2205 iris_delete_tcs_state(struct pipe_context *ctx, void *state)
2206 {
2207 iris_delete_shader_state(ctx, state, MESA_SHADER_TESS_CTRL);
2208 }
2209
2210 static void
2211 iris_delete_tes_state(struct pipe_context *ctx, void *state)
2212 {
2213 iris_delete_shader_state(ctx, state, MESA_SHADER_TESS_EVAL);
2214 }
2215
2216 static void
2217 iris_delete_gs_state(struct pipe_context *ctx, void *state)
2218 {
2219 iris_delete_shader_state(ctx, state, MESA_SHADER_GEOMETRY);
2220 }
2221
2222 static void
2223 iris_delete_fs_state(struct pipe_context *ctx, void *state)
2224 {
2225 iris_delete_shader_state(ctx, state, MESA_SHADER_FRAGMENT);
2226 }
2227
2228 static void
2229 iris_delete_cs_state(struct pipe_context *ctx, void *state)
2230 {
2231 iris_delete_shader_state(ctx, state, MESA_SHADER_COMPUTE);
2232 }
2233
2234 /**
2235 * The pipe->bind_[stage]_state() driver hook.
2236 *
2237 * Binds an uncompiled shader as the current one for a particular stage.
2238 * Updates dirty tracking to account for the shader's NOS.
2239 */
2240 static void
2241 bind_shader_state(struct iris_context *ice,
2242 struct iris_uncompiled_shader *ish,
2243 gl_shader_stage stage)
2244 {
2245 uint64_t dirty_bit = IRIS_DIRTY_UNCOMPILED_VS << stage;
2246 const uint64_t nos = ish ? ish->nos : 0;
2247
2248 const struct shader_info *old_info = iris_get_shader_info(ice, stage);
2249 const struct shader_info *new_info = ish ? &ish->nir->info : NULL;
2250
2251 if ((old_info ? util_last_bit(old_info->textures_used) : 0) !=
2252 (new_info ? util_last_bit(new_info->textures_used) : 0)) {
2253 ice->state.dirty |= IRIS_DIRTY_SAMPLER_STATES_VS << stage;
2254 }
2255
2256 ice->shaders.uncompiled[stage] = ish;
2257 ice->state.dirty |= dirty_bit;
2258
2259 /* Record that CSOs need to mark IRIS_DIRTY_UNCOMPILED_XS when they change
2260 * (or that they no longer need to do so).
2261 */
2262 for (int i = 0; i < IRIS_NOS_COUNT; i++) {
2263 if (nos & (1 << i))
2264 ice->state.dirty_for_nos[i] |= dirty_bit;
2265 else
2266 ice->state.dirty_for_nos[i] &= ~dirty_bit;
2267 }
2268 }
2269
2270 static void
2271 iris_bind_vs_state(struct pipe_context *ctx, void *state)
2272 {
2273 struct iris_context *ice = (struct iris_context *)ctx;
2274 struct iris_uncompiled_shader *new_ish = state;
2275
2276 if (new_ish &&
2277 ice->state.window_space_position !=
2278 new_ish->nir->info.vs.window_space_position) {
2279 ice->state.window_space_position =
2280 new_ish->nir->info.vs.window_space_position;
2281
2282 ice->state.dirty |= IRIS_DIRTY_CLIP |
2283 IRIS_DIRTY_RASTER |
2284 IRIS_DIRTY_CC_VIEWPORT;
2285 }
2286
2287 bind_shader_state((void *) ctx, state, MESA_SHADER_VERTEX);
2288 }
2289
2290 static void
2291 iris_bind_tcs_state(struct pipe_context *ctx, void *state)
2292 {
2293 bind_shader_state((void *) ctx, state, MESA_SHADER_TESS_CTRL);
2294 }
2295
2296 static void
2297 iris_bind_tes_state(struct pipe_context *ctx, void *state)
2298 {
2299 struct iris_context *ice = (struct iris_context *)ctx;
2300
2301 /* Enabling/disabling optional stages requires a URB reconfiguration. */
2302 if (!!state != !!ice->shaders.uncompiled[MESA_SHADER_TESS_EVAL])
2303 ice->state.dirty |= IRIS_DIRTY_URB;
2304
2305 bind_shader_state((void *) ctx, state, MESA_SHADER_TESS_EVAL);
2306 }
2307
2308 static void
2309 iris_bind_gs_state(struct pipe_context *ctx, void *state)
2310 {
2311 struct iris_context *ice = (struct iris_context *)ctx;
2312
2313 /* Enabling/disabling optional stages requires a URB reconfiguration. */
2314 if (!!state != !!ice->shaders.uncompiled[MESA_SHADER_GEOMETRY])
2315 ice->state.dirty |= IRIS_DIRTY_URB;
2316
2317 bind_shader_state((void *) ctx, state, MESA_SHADER_GEOMETRY);
2318 }
2319
2320 static void
2321 iris_bind_fs_state(struct pipe_context *ctx, void *state)
2322 {
2323 struct iris_context *ice = (struct iris_context *) ctx;
2324 struct iris_uncompiled_shader *old_ish =
2325 ice->shaders.uncompiled[MESA_SHADER_FRAGMENT];
2326 struct iris_uncompiled_shader *new_ish = state;
2327
2328 const unsigned color_bits =
2329 BITFIELD64_BIT(FRAG_RESULT_COLOR) |
2330 BITFIELD64_RANGE(FRAG_RESULT_DATA0, BRW_MAX_DRAW_BUFFERS);
2331
2332 /* Fragment shader outputs influence HasWriteableRT */
2333 if (!old_ish || !new_ish ||
2334 (old_ish->nir->info.outputs_written & color_bits) !=
2335 (new_ish->nir->info.outputs_written & color_bits))
2336 ice->state.dirty |= IRIS_DIRTY_PS_BLEND;
2337
2338 bind_shader_state((void *) ctx, state, MESA_SHADER_FRAGMENT);
2339 }
2340
2341 static void
2342 iris_bind_cs_state(struct pipe_context *ctx, void *state)
2343 {
2344 bind_shader_state((void *) ctx, state, MESA_SHADER_COMPUTE);
2345 }
2346
2347 void
2348 iris_init_program_functions(struct pipe_context *ctx)
2349 {
2350 ctx->create_vs_state = iris_create_vs_state;
2351 ctx->create_tcs_state = iris_create_tcs_state;
2352 ctx->create_tes_state = iris_create_tes_state;
2353 ctx->create_gs_state = iris_create_gs_state;
2354 ctx->create_fs_state = iris_create_fs_state;
2355 ctx->create_compute_state = iris_create_compute_state;
2356
2357 ctx->delete_vs_state = iris_delete_vs_state;
2358 ctx->delete_tcs_state = iris_delete_tcs_state;
2359 ctx->delete_tes_state = iris_delete_tes_state;
2360 ctx->delete_gs_state = iris_delete_gs_state;
2361 ctx->delete_fs_state = iris_delete_fs_state;
2362 ctx->delete_compute_state = iris_delete_cs_state;
2363
2364 ctx->bind_vs_state = iris_bind_vs_state;
2365 ctx->bind_tcs_state = iris_bind_tcs_state;
2366 ctx->bind_tes_state = iris_bind_tes_state;
2367 ctx->bind_gs_state = iris_bind_gs_state;
2368 ctx->bind_fs_state = iris_bind_fs_state;
2369 ctx->bind_compute_state = iris_bind_cs_state;
2370 }