fc4f2d2949df709b228e917a302c6b0837933af9
[mesa.git] / src / gallium / drivers / iris / iris_program.c
1 /*
2 * Copyright © 2017 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included
12 * in all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
15 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
19 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
20 * DEALINGS IN THE SOFTWARE.
21 */
22
23 /**
24 * @file iris_program.c
25 *
26 * This file contains the driver interface for compiling shaders.
27 *
28 * See iris_program_cache.c for the in-memory program cache where the
29 * compiled shaders are stored.
30 */
31
32 #include <stdio.h>
33 #include <errno.h>
34 #include "pipe/p_defines.h"
35 #include "pipe/p_state.h"
36 #include "pipe/p_context.h"
37 #include "pipe/p_screen.h"
38 #include "util/u_atomic.h"
39 #include "util/u_upload_mgr.h"
40 #include "util/debug.h"
41 #include "compiler/nir/nir.h"
42 #include "compiler/nir/nir_builder.h"
43 #include "compiler/nir/nir_serialize.h"
44 #include "intel/compiler/brw_compiler.h"
45 #include "intel/compiler/brw_nir.h"
46 #include "iris_context.h"
47 #include "nir/tgsi_to_nir.h"
48
49 #define KEY_INIT_NO_ID(gen) \
50 .base.subgroup_size_type = BRW_SUBGROUP_SIZE_UNIFORM, \
51 .base.tex.swizzles[0 ... MAX_SAMPLERS - 1] = 0x688, \
52 .base.tex.compressed_multisample_layout_mask = ~0, \
53 .base.tex.msaa_16 = (gen >= 9 ? ~0 : 0)
54 #define KEY_INIT(gen) .base.program_string_id = ish->program_id, KEY_INIT_NO_ID(gen)
55
56 static unsigned
57 get_new_program_id(struct iris_screen *screen)
58 {
59 return p_atomic_inc_return(&screen->program_id);
60 }
61
62 static void *
63 upload_state(struct u_upload_mgr *uploader,
64 struct iris_state_ref *ref,
65 unsigned size,
66 unsigned alignment)
67 {
68 void *p = NULL;
69 u_upload_alloc(uploader, 0, size, alignment, &ref->offset, &ref->res, &p);
70 return p;
71 }
72
73 void
74 iris_upload_ubo_ssbo_surf_state(struct iris_context *ice,
75 struct pipe_shader_buffer *buf,
76 struct iris_state_ref *surf_state,
77 bool ssbo)
78 {
79 struct pipe_context *ctx = &ice->ctx;
80 struct iris_screen *screen = (struct iris_screen *) ctx->screen;
81
82 void *map =
83 upload_state(ice->state.surface_uploader, surf_state,
84 screen->isl_dev.ss.size, 64);
85 if (!unlikely(map)) {
86 surf_state->res = NULL;
87 return;
88 }
89
90 struct iris_resource *res = (void *) buf->buffer;
91 struct iris_bo *surf_bo = iris_resource_bo(surf_state->res);
92 surf_state->offset += iris_bo_offset_from_base_address(surf_bo);
93
94 isl_buffer_fill_state(&screen->isl_dev, map,
95 .address = res->bo->gtt_offset + res->offset +
96 buf->buffer_offset,
97 .size_B = buf->buffer_size - res->offset,
98 .format = ssbo ? ISL_FORMAT_RAW
99 : ISL_FORMAT_R32G32B32A32_FLOAT,
100 .swizzle = ISL_SWIZZLE_IDENTITY,
101 .stride_B = 1,
102 .mocs = ice->vtbl.mocs(res->bo));
103 }
104
105 static nir_ssa_def *
106 get_aoa_deref_offset(nir_builder *b,
107 nir_deref_instr *deref,
108 unsigned elem_size)
109 {
110 unsigned array_size = elem_size;
111 nir_ssa_def *offset = nir_imm_int(b, 0);
112
113 while (deref->deref_type != nir_deref_type_var) {
114 assert(deref->deref_type == nir_deref_type_array);
115
116 /* This level's element size is the previous level's array size */
117 nir_ssa_def *index = nir_ssa_for_src(b, deref->arr.index, 1);
118 assert(deref->arr.index.ssa);
119 offset = nir_iadd(b, offset,
120 nir_imul(b, index, nir_imm_int(b, array_size)));
121
122 deref = nir_deref_instr_parent(deref);
123 assert(glsl_type_is_array(deref->type));
124 array_size *= glsl_get_length(deref->type);
125 }
126
127 /* Accessing an invalid surface index with the dataport can result in a
128 * hang. According to the spec "if the index used to select an individual
129 * element is negative or greater than or equal to the size of the array,
130 * the results of the operation are undefined but may not lead to
131 * termination" -- which is one of the possible outcomes of the hang.
132 * Clamp the index to prevent access outside of the array bounds.
133 */
134 return nir_umin(b, offset, nir_imm_int(b, array_size - elem_size));
135 }
136
137 static void
138 iris_lower_storage_image_derefs(nir_shader *nir)
139 {
140 nir_function_impl *impl = nir_shader_get_entrypoint(nir);
141
142 nir_builder b;
143 nir_builder_init(&b, impl);
144
145 nir_foreach_block(block, impl) {
146 nir_foreach_instr_safe(instr, block) {
147 if (instr->type != nir_instr_type_intrinsic)
148 continue;
149
150 nir_intrinsic_instr *intrin = nir_instr_as_intrinsic(instr);
151 switch (intrin->intrinsic) {
152 case nir_intrinsic_image_deref_load:
153 case nir_intrinsic_image_deref_store:
154 case nir_intrinsic_image_deref_atomic_add:
155 case nir_intrinsic_image_deref_atomic_imin:
156 case nir_intrinsic_image_deref_atomic_umin:
157 case nir_intrinsic_image_deref_atomic_imax:
158 case nir_intrinsic_image_deref_atomic_umax:
159 case nir_intrinsic_image_deref_atomic_and:
160 case nir_intrinsic_image_deref_atomic_or:
161 case nir_intrinsic_image_deref_atomic_xor:
162 case nir_intrinsic_image_deref_atomic_exchange:
163 case nir_intrinsic_image_deref_atomic_comp_swap:
164 case nir_intrinsic_image_deref_size:
165 case nir_intrinsic_image_deref_samples:
166 case nir_intrinsic_image_deref_load_raw_intel:
167 case nir_intrinsic_image_deref_store_raw_intel: {
168 nir_deref_instr *deref = nir_src_as_deref(intrin->src[0]);
169 nir_variable *var = nir_deref_instr_get_variable(deref);
170
171 b.cursor = nir_before_instr(&intrin->instr);
172 nir_ssa_def *index =
173 nir_iadd(&b, nir_imm_int(&b, var->data.driver_location),
174 get_aoa_deref_offset(&b, deref, 1));
175 nir_rewrite_image_intrinsic(intrin, index, false);
176 break;
177 }
178
179 default:
180 break;
181 }
182 }
183 }
184 }
185
186 // XXX: need unify_interfaces() at link time...
187
188 /**
189 * Fix an uncompiled shader's stream output info.
190 *
191 * Core Gallium stores output->register_index as a "slot" number, where
192 * slots are assigned consecutively to all outputs in info->outputs_written.
193 * This naive packing of outputs doesn't work for us - we too have slots,
194 * but the layout is defined by the VUE map, which we won't have until we
195 * compile a specific shader variant. So, we remap these and simply store
196 * VARYING_SLOT_* in our copy's output->register_index fields.
197 *
198 * We also fix up VARYING_SLOT_{LAYER,VIEWPORT,PSIZ} to select the Y/Z/W
199 * components of our VUE header. See brw_vue_map.c for the layout.
200 */
201 static void
202 update_so_info(struct pipe_stream_output_info *so_info,
203 uint64_t outputs_written)
204 {
205 uint8_t reverse_map[64] = {};
206 unsigned slot = 0;
207 while (outputs_written) {
208 reverse_map[slot++] = u_bit_scan64(&outputs_written);
209 }
210
211 for (unsigned i = 0; i < so_info->num_outputs; i++) {
212 struct pipe_stream_output *output = &so_info->output[i];
213
214 /* Map Gallium's condensed "slots" back to real VARYING_SLOT_* enums */
215 output->register_index = reverse_map[output->register_index];
216
217 /* The VUE header contains three scalar fields packed together:
218 * - gl_PointSize is stored in VARYING_SLOT_PSIZ.w
219 * - gl_Layer is stored in VARYING_SLOT_PSIZ.y
220 * - gl_ViewportIndex is stored in VARYING_SLOT_PSIZ.z
221 */
222 switch (output->register_index) {
223 case VARYING_SLOT_LAYER:
224 assert(output->num_components == 1);
225 output->register_index = VARYING_SLOT_PSIZ;
226 output->start_component = 1;
227 break;
228 case VARYING_SLOT_VIEWPORT:
229 assert(output->num_components == 1);
230 output->register_index = VARYING_SLOT_PSIZ;
231 output->start_component = 2;
232 break;
233 case VARYING_SLOT_PSIZ:
234 assert(output->num_components == 1);
235 output->start_component = 3;
236 break;
237 }
238
239 //info->outputs_written |= 1ull << output->register_index;
240 }
241 }
242
243 static void
244 setup_vec4_image_sysval(uint32_t *sysvals, uint32_t idx,
245 unsigned offset, unsigned n)
246 {
247 assert(offset % sizeof(uint32_t) == 0);
248
249 for (unsigned i = 0; i < n; ++i)
250 sysvals[i] = BRW_PARAM_IMAGE(idx, offset / sizeof(uint32_t) + i);
251
252 for (unsigned i = n; i < 4; ++i)
253 sysvals[i] = BRW_PARAM_BUILTIN_ZERO;
254 }
255
256 /**
257 * Associate NIR uniform variables with the prog_data->param[] mechanism
258 * used by the backend. Also, decide which UBOs we'd like to push in an
259 * ideal situation (though the backend can reduce this).
260 */
261 static void
262 iris_setup_uniforms(const struct brw_compiler *compiler,
263 void *mem_ctx,
264 nir_shader *nir,
265 struct brw_stage_prog_data *prog_data,
266 enum brw_param_builtin **out_system_values,
267 unsigned *out_num_system_values,
268 unsigned *out_num_cbufs)
269 {
270 UNUSED const struct gen_device_info *devinfo = compiler->devinfo;
271
272 /* The intel compiler assumes that num_uniforms is in bytes. For
273 * scalar that means 4 bytes per uniform slot.
274 *
275 * Ref: brw_nir_lower_uniforms, type_size_scalar_bytes.
276 */
277 nir->num_uniforms *= 4;
278
279 const unsigned IRIS_MAX_SYSTEM_VALUES =
280 PIPE_MAX_SHADER_IMAGES * BRW_IMAGE_PARAM_SIZE;
281 enum brw_param_builtin *system_values =
282 rzalloc_array(mem_ctx, enum brw_param_builtin, IRIS_MAX_SYSTEM_VALUES);
283 unsigned num_system_values = 0;
284
285 unsigned patch_vert_idx = -1;
286 unsigned ucp_idx[IRIS_MAX_CLIP_PLANES];
287 unsigned img_idx[PIPE_MAX_SHADER_IMAGES];
288 memset(ucp_idx, -1, sizeof(ucp_idx));
289 memset(img_idx, -1, sizeof(img_idx));
290
291 nir_function_impl *impl = nir_shader_get_entrypoint(nir);
292
293 nir_builder b;
294 nir_builder_init(&b, impl);
295
296 b.cursor = nir_before_block(nir_start_block(impl));
297 nir_ssa_def *temp_ubo_name = nir_ssa_undef(&b, 1, 32);
298 nir_ssa_def *temp_const_ubo_name = NULL;
299
300 /* Turn system value intrinsics into uniforms */
301 nir_foreach_block(block, impl) {
302 nir_foreach_instr_safe(instr, block) {
303 if (instr->type != nir_instr_type_intrinsic)
304 continue;
305
306 nir_intrinsic_instr *intrin = nir_instr_as_intrinsic(instr);
307 nir_ssa_def *offset;
308
309 switch (intrin->intrinsic) {
310 case nir_intrinsic_load_constant: {
311 /* This one is special because it reads from the shader constant
312 * data and not cbuf0 which gallium uploads for us.
313 */
314 b.cursor = nir_before_instr(instr);
315 nir_ssa_def *offset =
316 nir_iadd_imm(&b, nir_ssa_for_src(&b, intrin->src[0], 1),
317 nir_intrinsic_base(intrin));
318
319 if (temp_const_ubo_name == NULL)
320 temp_const_ubo_name = nir_imm_int(&b, 0);
321
322 nir_intrinsic_instr *load_ubo =
323 nir_intrinsic_instr_create(b.shader, nir_intrinsic_load_ubo);
324 load_ubo->num_components = intrin->num_components;
325 load_ubo->src[0] = nir_src_for_ssa(temp_const_ubo_name);
326 load_ubo->src[1] = nir_src_for_ssa(offset);
327 nir_ssa_dest_init(&load_ubo->instr, &load_ubo->dest,
328 intrin->dest.ssa.num_components,
329 intrin->dest.ssa.bit_size,
330 intrin->dest.ssa.name);
331 nir_builder_instr_insert(&b, &load_ubo->instr);
332
333 nir_ssa_def_rewrite_uses(&intrin->dest.ssa,
334 nir_src_for_ssa(&load_ubo->dest.ssa));
335 nir_instr_remove(&intrin->instr);
336 continue;
337 }
338 case nir_intrinsic_load_user_clip_plane: {
339 unsigned ucp = nir_intrinsic_ucp_id(intrin);
340
341 if (ucp_idx[ucp] == -1) {
342 ucp_idx[ucp] = num_system_values;
343 num_system_values += 4;
344 }
345
346 for (int i = 0; i < 4; i++) {
347 system_values[ucp_idx[ucp] + i] =
348 BRW_PARAM_BUILTIN_CLIP_PLANE(ucp, i);
349 }
350
351 b.cursor = nir_before_instr(instr);
352 offset = nir_imm_int(&b, ucp_idx[ucp] * sizeof(uint32_t));
353 break;
354 }
355 case nir_intrinsic_load_patch_vertices_in:
356 if (patch_vert_idx == -1)
357 patch_vert_idx = num_system_values++;
358
359 system_values[patch_vert_idx] =
360 BRW_PARAM_BUILTIN_PATCH_VERTICES_IN;
361
362 b.cursor = nir_before_instr(instr);
363 offset = nir_imm_int(&b, patch_vert_idx * sizeof(uint32_t));
364 break;
365 case nir_intrinsic_image_deref_load_param_intel: {
366 assert(devinfo->gen < 9);
367 nir_deref_instr *deref = nir_src_as_deref(intrin->src[0]);
368 nir_variable *var = nir_deref_instr_get_variable(deref);
369
370 if (img_idx[var->data.binding] == -1) {
371 /* GL only allows arrays of arrays of images. */
372 assert(glsl_type_is_image(glsl_without_array(var->type)));
373 unsigned num_images = MAX2(1, glsl_get_aoa_size(var->type));
374
375 for (int i = 0; i < num_images; i++) {
376 const unsigned img = var->data.binding + i;
377
378 img_idx[img] = num_system_values;
379 num_system_values += BRW_IMAGE_PARAM_SIZE;
380
381 uint32_t *img_sv = &system_values[img_idx[img]];
382
383 setup_vec4_image_sysval(
384 img_sv + BRW_IMAGE_PARAM_OFFSET_OFFSET, img,
385 offsetof(struct brw_image_param, offset), 2);
386 setup_vec4_image_sysval(
387 img_sv + BRW_IMAGE_PARAM_SIZE_OFFSET, img,
388 offsetof(struct brw_image_param, size), 3);
389 setup_vec4_image_sysval(
390 img_sv + BRW_IMAGE_PARAM_STRIDE_OFFSET, img,
391 offsetof(struct brw_image_param, stride), 4);
392 setup_vec4_image_sysval(
393 img_sv + BRW_IMAGE_PARAM_TILING_OFFSET, img,
394 offsetof(struct brw_image_param, tiling), 3);
395 setup_vec4_image_sysval(
396 img_sv + BRW_IMAGE_PARAM_SWIZZLING_OFFSET, img,
397 offsetof(struct brw_image_param, swizzling), 2);
398 }
399 }
400
401 b.cursor = nir_before_instr(instr);
402 offset = nir_iadd(&b,
403 get_aoa_deref_offset(&b, deref, BRW_IMAGE_PARAM_SIZE * 4),
404 nir_imm_int(&b, img_idx[var->data.binding] * 4 +
405 nir_intrinsic_base(intrin) * 16));
406 break;
407 }
408 default:
409 continue;
410 }
411
412 unsigned comps = nir_intrinsic_dest_components(intrin);
413
414 nir_intrinsic_instr *load =
415 nir_intrinsic_instr_create(nir, nir_intrinsic_load_ubo);
416 load->num_components = comps;
417 load->src[0] = nir_src_for_ssa(temp_ubo_name);
418 load->src[1] = nir_src_for_ssa(offset);
419 nir_ssa_dest_init(&load->instr, &load->dest, comps, 32, NULL);
420 nir_builder_instr_insert(&b, &load->instr);
421 nir_ssa_def_rewrite_uses(&intrin->dest.ssa,
422 nir_src_for_ssa(&load->dest.ssa));
423 nir_instr_remove(instr);
424 }
425 }
426
427 nir_validate_shader(nir, "before remapping");
428
429 /* Uniforms are stored in constant buffer 0, the
430 * user-facing UBOs are indexed by one. So if any constant buffer is
431 * needed, the constant buffer 0 will be needed, so account for it.
432 */
433 unsigned num_cbufs = nir->info.num_ubos;
434 if (num_cbufs || nir->num_uniforms)
435 num_cbufs++;
436
437 /* Place the new params in a new cbuf. */
438 if (num_system_values > 0) {
439 unsigned sysval_cbuf_index = num_cbufs;
440 num_cbufs++;
441
442 system_values = reralloc(mem_ctx, system_values, enum brw_param_builtin,
443 num_system_values);
444
445 nir_foreach_block(block, impl) {
446 nir_foreach_instr_safe(instr, block) {
447 if (instr->type != nir_instr_type_intrinsic)
448 continue;
449
450 nir_intrinsic_instr *load = nir_instr_as_intrinsic(instr);
451
452 if (load->intrinsic != nir_intrinsic_load_ubo)
453 continue;
454
455 b.cursor = nir_before_instr(instr);
456
457 assert(load->src[0].is_ssa);
458
459 if (load->src[0].ssa == temp_ubo_name) {
460 nir_ssa_def *imm = nir_imm_int(&b, sysval_cbuf_index);
461 nir_instr_rewrite_src(instr, &load->src[0],
462 nir_src_for_ssa(imm));
463 }
464 }
465 }
466
467 /* We need to fold the new iadds for brw_nir_analyze_ubo_ranges */
468 nir_opt_constant_folding(nir);
469 } else {
470 ralloc_free(system_values);
471 system_values = NULL;
472 }
473
474 assert(num_cbufs < PIPE_MAX_CONSTANT_BUFFERS);
475 nir_validate_shader(nir, "after remap");
476
477 /* We don't use params[], but fs_visitor::nir_setup_uniforms() asserts
478 * about it for compute shaders, so go ahead and make some fake ones
479 * which the backend will dead code eliminate.
480 */
481 prog_data->nr_params = nir->num_uniforms / 4;
482 prog_data->param = rzalloc_array(mem_ctx, uint32_t, prog_data->nr_params);
483
484 /* Constant loads (if any) need to go at the end of the constant buffers so
485 * we need to know num_cbufs before we can lower to them.
486 */
487 if (temp_const_ubo_name != NULL) {
488 nir_load_const_instr *const_ubo_index =
489 nir_instr_as_load_const(temp_const_ubo_name->parent_instr);
490 assert(const_ubo_index->def.bit_size == 32);
491 const_ubo_index->value[0].u32 = num_cbufs;
492 }
493
494 *out_system_values = system_values;
495 *out_num_system_values = num_system_values;
496 *out_num_cbufs = num_cbufs;
497 }
498
499 static const char *surface_group_names[] = {
500 [IRIS_SURFACE_GROUP_RENDER_TARGET] = "render target",
501 [IRIS_SURFACE_GROUP_RENDER_TARGET_READ] = "non-coherent render target read",
502 [IRIS_SURFACE_GROUP_CS_WORK_GROUPS] = "CS work groups",
503 [IRIS_SURFACE_GROUP_TEXTURE] = "texture",
504 [IRIS_SURFACE_GROUP_UBO] = "ubo",
505 [IRIS_SURFACE_GROUP_SSBO] = "ssbo",
506 [IRIS_SURFACE_GROUP_IMAGE] = "image",
507 };
508
509 static void
510 iris_print_binding_table(FILE *fp, const char *name,
511 const struct iris_binding_table *bt)
512 {
513 STATIC_ASSERT(ARRAY_SIZE(surface_group_names) == IRIS_SURFACE_GROUP_COUNT);
514
515 uint32_t total = 0;
516 uint32_t compacted = 0;
517
518 for (int i = 0; i < IRIS_SURFACE_GROUP_COUNT; i++) {
519 uint32_t size = bt->sizes[i];
520 total += size;
521 if (size)
522 compacted += util_bitcount64(bt->used_mask[i]);
523 }
524
525 if (total == 0) {
526 fprintf(fp, "Binding table for %s is empty\n\n", name);
527 return;
528 }
529
530 if (total != compacted) {
531 fprintf(fp, "Binding table for %s "
532 "(compacted to %u entries from %u entries)\n",
533 name, compacted, total);
534 } else {
535 fprintf(fp, "Binding table for %s (%u entries)\n", name, total);
536 }
537
538 uint32_t entry = 0;
539 for (int i = 0; i < IRIS_SURFACE_GROUP_COUNT; i++) {
540 uint64_t mask = bt->used_mask[i];
541 while (mask) {
542 int index = u_bit_scan64(&mask);
543 fprintf(fp, " [%u] %s #%d\n", entry++, surface_group_names[i], index);
544 }
545 }
546 fprintf(fp, "\n");
547 }
548
549 enum {
550 /* Max elements in a surface group. */
551 SURFACE_GROUP_MAX_ELEMENTS = 64,
552 };
553
554 /**
555 * Map a <group, index> pair to a binding table index.
556 *
557 * For example: <UBO, 5> => binding table index 12
558 */
559 uint32_t
560 iris_group_index_to_bti(const struct iris_binding_table *bt,
561 enum iris_surface_group group, uint32_t index)
562 {
563 assert(index < bt->sizes[group]);
564 uint64_t mask = bt->used_mask[group];
565 uint64_t bit = 1ull << index;
566 if (bit & mask) {
567 return bt->offsets[group] + util_bitcount64((bit - 1) & mask);
568 } else {
569 return IRIS_SURFACE_NOT_USED;
570 }
571 }
572
573 /**
574 * Map a binding table index back to a <group, index> pair.
575 *
576 * For example: binding table index 12 => <UBO, 5>
577 */
578 uint32_t
579 iris_bti_to_group_index(const struct iris_binding_table *bt,
580 enum iris_surface_group group, uint32_t bti)
581 {
582 uint64_t used_mask = bt->used_mask[group];
583 assert(bti >= bt->offsets[group]);
584
585 uint32_t c = bti - bt->offsets[group];
586 while (used_mask) {
587 int i = u_bit_scan64(&used_mask);
588 if (c == 0)
589 return i;
590 c--;
591 }
592
593 return IRIS_SURFACE_NOT_USED;
594 }
595
596 static void
597 rewrite_src_with_bti(nir_builder *b, struct iris_binding_table *bt,
598 nir_instr *instr, nir_src *src,
599 enum iris_surface_group group)
600 {
601 assert(bt->sizes[group] > 0);
602
603 b->cursor = nir_before_instr(instr);
604 nir_ssa_def *bti;
605 if (nir_src_is_const(*src)) {
606 uint32_t index = nir_src_as_uint(*src);
607 bti = nir_imm_intN_t(b, iris_group_index_to_bti(bt, group, index),
608 src->ssa->bit_size);
609 } else {
610 /* Indirect usage makes all the surfaces of the group to be available,
611 * so we can just add the base.
612 */
613 assert(bt->used_mask[group] == BITFIELD64_MASK(bt->sizes[group]));
614 bti = nir_iadd_imm(b, src->ssa, bt->offsets[group]);
615 }
616 nir_instr_rewrite_src(instr, src, nir_src_for_ssa(bti));
617 }
618
619 static void
620 mark_used_with_src(struct iris_binding_table *bt, nir_src *src,
621 enum iris_surface_group group)
622 {
623 assert(bt->sizes[group] > 0);
624
625 if (nir_src_is_const(*src)) {
626 uint64_t index = nir_src_as_uint(*src);
627 assert(index < bt->sizes[group]);
628 bt->used_mask[group] |= 1ull << index;
629 } else {
630 /* There's an indirect usage, we need all the surfaces. */
631 bt->used_mask[group] = BITFIELD64_MASK(bt->sizes[group]);
632 }
633 }
634
635 static bool
636 skip_compacting_binding_tables(void)
637 {
638 static int skip = -1;
639 if (skip < 0)
640 skip = env_var_as_boolean("INTEL_DISABLE_COMPACT_BINDING_TABLE", false);
641 return skip;
642 }
643
644 /**
645 * Set up the binding table indices and apply to the shader.
646 */
647 static void
648 iris_setup_binding_table(const struct gen_device_info *devinfo,
649 struct nir_shader *nir,
650 struct iris_binding_table *bt,
651 unsigned num_render_targets,
652 unsigned num_system_values,
653 unsigned num_cbufs)
654 {
655 const struct shader_info *info = &nir->info;
656
657 memset(bt, 0, sizeof(*bt));
658
659 /* Set the sizes for each surface group. For some groups, we already know
660 * upfront how many will be used, so mark them.
661 */
662 if (info->stage == MESA_SHADER_FRAGMENT) {
663 bt->sizes[IRIS_SURFACE_GROUP_RENDER_TARGET] = num_render_targets;
664 /* All render targets used. */
665 bt->used_mask[IRIS_SURFACE_GROUP_RENDER_TARGET] =
666 BITFIELD64_MASK(num_render_targets);
667
668 /* Setup render target read surface group inorder to support non-coherent
669 * framebuffer fetch on Gen8
670 */
671 if (devinfo->gen == 8 && info->outputs_read) {
672 bt->sizes[IRIS_SURFACE_GROUP_RENDER_TARGET_READ] = num_render_targets;
673 bt->used_mask[IRIS_SURFACE_GROUP_RENDER_TARGET_READ] =
674 BITFIELD64_MASK(num_render_targets);
675 }
676 } else if (info->stage == MESA_SHADER_COMPUTE) {
677 bt->sizes[IRIS_SURFACE_GROUP_CS_WORK_GROUPS] = 1;
678 }
679
680 bt->sizes[IRIS_SURFACE_GROUP_TEXTURE] = util_last_bit(info->textures_used);
681 bt->used_mask[IRIS_SURFACE_GROUP_TEXTURE] = info->textures_used;
682
683 bt->sizes[IRIS_SURFACE_GROUP_IMAGE] = info->num_images;
684
685 /* Allocate an extra slot in the UBO section for NIR constants.
686 * Binding table compaction will remove it if unnecessary.
687 *
688 * We don't include them in iris_compiled_shader::num_cbufs because
689 * they are uploaded separately from shs->constbuf[], but from a shader
690 * point of view, they're another UBO (at the end of the section).
691 */
692 bt->sizes[IRIS_SURFACE_GROUP_UBO] = num_cbufs + 1;
693
694 bt->sizes[IRIS_SURFACE_GROUP_SSBO] = info->num_ssbos;
695
696 for (int i = 0; i < IRIS_SURFACE_GROUP_COUNT; i++)
697 assert(bt->sizes[i] <= SURFACE_GROUP_MAX_ELEMENTS);
698
699 /* Mark surfaces used for the cases we don't have the information available
700 * upfront.
701 */
702 nir_function_impl *impl = nir_shader_get_entrypoint(nir);
703 nir_foreach_block (block, impl) {
704 nir_foreach_instr (instr, block) {
705 if (instr->type != nir_instr_type_intrinsic)
706 continue;
707
708 nir_intrinsic_instr *intrin = nir_instr_as_intrinsic(instr);
709 switch (intrin->intrinsic) {
710 case nir_intrinsic_load_num_work_groups:
711 bt->used_mask[IRIS_SURFACE_GROUP_CS_WORK_GROUPS] = 1;
712 break;
713
714 case nir_intrinsic_load_output:
715 if (devinfo->gen == 8) {
716 mark_used_with_src(bt, &intrin->src[0],
717 IRIS_SURFACE_GROUP_RENDER_TARGET_READ);
718 }
719 break;
720
721 case nir_intrinsic_image_size:
722 case nir_intrinsic_image_load:
723 case nir_intrinsic_image_store:
724 case nir_intrinsic_image_atomic_add:
725 case nir_intrinsic_image_atomic_imin:
726 case nir_intrinsic_image_atomic_umin:
727 case nir_intrinsic_image_atomic_imax:
728 case nir_intrinsic_image_atomic_umax:
729 case nir_intrinsic_image_atomic_and:
730 case nir_intrinsic_image_atomic_or:
731 case nir_intrinsic_image_atomic_xor:
732 case nir_intrinsic_image_atomic_exchange:
733 case nir_intrinsic_image_atomic_comp_swap:
734 case nir_intrinsic_image_load_raw_intel:
735 case nir_intrinsic_image_store_raw_intel:
736 mark_used_with_src(bt, &intrin->src[0], IRIS_SURFACE_GROUP_IMAGE);
737 break;
738
739 case nir_intrinsic_load_ubo:
740 mark_used_with_src(bt, &intrin->src[0], IRIS_SURFACE_GROUP_UBO);
741 break;
742
743 case nir_intrinsic_store_ssbo:
744 mark_used_with_src(bt, &intrin->src[1], IRIS_SURFACE_GROUP_SSBO);
745 break;
746
747 case nir_intrinsic_get_buffer_size:
748 case nir_intrinsic_ssbo_atomic_add:
749 case nir_intrinsic_ssbo_atomic_imin:
750 case nir_intrinsic_ssbo_atomic_umin:
751 case nir_intrinsic_ssbo_atomic_imax:
752 case nir_intrinsic_ssbo_atomic_umax:
753 case nir_intrinsic_ssbo_atomic_and:
754 case nir_intrinsic_ssbo_atomic_or:
755 case nir_intrinsic_ssbo_atomic_xor:
756 case nir_intrinsic_ssbo_atomic_exchange:
757 case nir_intrinsic_ssbo_atomic_comp_swap:
758 case nir_intrinsic_ssbo_atomic_fmin:
759 case nir_intrinsic_ssbo_atomic_fmax:
760 case nir_intrinsic_ssbo_atomic_fcomp_swap:
761 case nir_intrinsic_load_ssbo:
762 mark_used_with_src(bt, &intrin->src[0], IRIS_SURFACE_GROUP_SSBO);
763 break;
764
765 default:
766 break;
767 }
768 }
769 }
770
771 /* When disable we just mark everything as used. */
772 if (unlikely(skip_compacting_binding_tables())) {
773 for (int i = 0; i < IRIS_SURFACE_GROUP_COUNT; i++)
774 bt->used_mask[i] = BITFIELD64_MASK(bt->sizes[i]);
775 }
776
777 /* Calculate the offsets and the binding table size based on the used
778 * surfaces. After this point, the functions to go between "group indices"
779 * and binding table indices can be used.
780 */
781 uint32_t next = 0;
782 for (int i = 0; i < IRIS_SURFACE_GROUP_COUNT; i++) {
783 if (bt->used_mask[i] != 0) {
784 bt->offsets[i] = next;
785 next += util_bitcount64(bt->used_mask[i]);
786 }
787 }
788 bt->size_bytes = next * 4;
789
790 if (unlikely(INTEL_DEBUG & DEBUG_BT)) {
791 iris_print_binding_table(stderr, gl_shader_stage_name(info->stage), bt);
792 }
793
794 /* Apply the binding table indices. The backend compiler is not expected
795 * to change those, as we haven't set any of the *_start entries in brw
796 * binding_table.
797 */
798 nir_builder b;
799 nir_builder_init(&b, impl);
800
801 nir_foreach_block (block, impl) {
802 nir_foreach_instr (instr, block) {
803 if (instr->type == nir_instr_type_tex) {
804 nir_tex_instr *tex = nir_instr_as_tex(instr);
805 tex->texture_index =
806 iris_group_index_to_bti(bt, IRIS_SURFACE_GROUP_TEXTURE,
807 tex->texture_index);
808 continue;
809 }
810
811 if (instr->type != nir_instr_type_intrinsic)
812 continue;
813
814 nir_intrinsic_instr *intrin = nir_instr_as_intrinsic(instr);
815 switch (intrin->intrinsic) {
816 case nir_intrinsic_image_size:
817 case nir_intrinsic_image_load:
818 case nir_intrinsic_image_store:
819 case nir_intrinsic_image_atomic_add:
820 case nir_intrinsic_image_atomic_imin:
821 case nir_intrinsic_image_atomic_umin:
822 case nir_intrinsic_image_atomic_imax:
823 case nir_intrinsic_image_atomic_umax:
824 case nir_intrinsic_image_atomic_and:
825 case nir_intrinsic_image_atomic_or:
826 case nir_intrinsic_image_atomic_xor:
827 case nir_intrinsic_image_atomic_exchange:
828 case nir_intrinsic_image_atomic_comp_swap:
829 case nir_intrinsic_image_load_raw_intel:
830 case nir_intrinsic_image_store_raw_intel:
831 rewrite_src_with_bti(&b, bt, instr, &intrin->src[0],
832 IRIS_SURFACE_GROUP_IMAGE);
833 break;
834
835 case nir_intrinsic_load_ubo:
836 rewrite_src_with_bti(&b, bt, instr, &intrin->src[0],
837 IRIS_SURFACE_GROUP_UBO);
838 break;
839
840 case nir_intrinsic_store_ssbo:
841 rewrite_src_with_bti(&b, bt, instr, &intrin->src[1],
842 IRIS_SURFACE_GROUP_SSBO);
843 break;
844
845 case nir_intrinsic_load_output:
846 if (devinfo->gen == 8) {
847 rewrite_src_with_bti(&b, bt, instr, &intrin->src[0],
848 IRIS_SURFACE_GROUP_RENDER_TARGET_READ);
849 }
850 break;
851
852 case nir_intrinsic_get_buffer_size:
853 case nir_intrinsic_ssbo_atomic_add:
854 case nir_intrinsic_ssbo_atomic_imin:
855 case nir_intrinsic_ssbo_atomic_umin:
856 case nir_intrinsic_ssbo_atomic_imax:
857 case nir_intrinsic_ssbo_atomic_umax:
858 case nir_intrinsic_ssbo_atomic_and:
859 case nir_intrinsic_ssbo_atomic_or:
860 case nir_intrinsic_ssbo_atomic_xor:
861 case nir_intrinsic_ssbo_atomic_exchange:
862 case nir_intrinsic_ssbo_atomic_comp_swap:
863 case nir_intrinsic_ssbo_atomic_fmin:
864 case nir_intrinsic_ssbo_atomic_fmax:
865 case nir_intrinsic_ssbo_atomic_fcomp_swap:
866 case nir_intrinsic_load_ssbo:
867 rewrite_src_with_bti(&b, bt, instr, &intrin->src[0],
868 IRIS_SURFACE_GROUP_SSBO);
869 break;
870
871 default:
872 break;
873 }
874 }
875 }
876 }
877
878 static void
879 iris_debug_recompile(struct iris_context *ice,
880 struct shader_info *info,
881 const struct brw_base_prog_key *key)
882 {
883 struct iris_screen *screen = (struct iris_screen *) ice->ctx.screen;
884 const struct brw_compiler *c = screen->compiler;
885
886 if (!info)
887 return;
888
889 c->shader_perf_log(&ice->dbg, "Recompiling %s shader for program %s: %s\n",
890 _mesa_shader_stage_to_string(info->stage),
891 info->name ? info->name : "(no identifier)",
892 info->label ? info->label : "");
893
894 const void *old_key =
895 iris_find_previous_compile(ice, info->stage, key->program_string_id);
896
897 brw_debug_key_recompile(c, &ice->dbg, info->stage, old_key, key);
898 }
899
900 /**
901 * Get the shader for the last enabled geometry stage.
902 *
903 * This stage is the one which will feed stream output and the rasterizer.
904 */
905 static gl_shader_stage
906 last_vue_stage(struct iris_context *ice)
907 {
908 if (ice->shaders.uncompiled[MESA_SHADER_GEOMETRY])
909 return MESA_SHADER_GEOMETRY;
910
911 if (ice->shaders.uncompiled[MESA_SHADER_TESS_EVAL])
912 return MESA_SHADER_TESS_EVAL;
913
914 return MESA_SHADER_VERTEX;
915 }
916
917 /**
918 * Compile a vertex shader, and upload the assembly.
919 */
920 static struct iris_compiled_shader *
921 iris_compile_vs(struct iris_context *ice,
922 struct iris_uncompiled_shader *ish,
923 const struct brw_vs_prog_key *key)
924 {
925 struct iris_screen *screen = (struct iris_screen *)ice->ctx.screen;
926 const struct brw_compiler *compiler = screen->compiler;
927 const struct gen_device_info *devinfo = &screen->devinfo;
928 void *mem_ctx = ralloc_context(NULL);
929 struct brw_vs_prog_data *vs_prog_data =
930 rzalloc(mem_ctx, struct brw_vs_prog_data);
931 struct brw_vue_prog_data *vue_prog_data = &vs_prog_data->base;
932 struct brw_stage_prog_data *prog_data = &vue_prog_data->base;
933 enum brw_param_builtin *system_values;
934 unsigned num_system_values;
935 unsigned num_cbufs;
936
937 nir_shader *nir = nir_shader_clone(mem_ctx, ish->nir);
938
939 if (key->nr_userclip_plane_consts) {
940 nir_function_impl *impl = nir_shader_get_entrypoint(nir);
941 nir_lower_clip_vs(nir, (1 << key->nr_userclip_plane_consts) - 1, true,
942 false);
943 nir_lower_io_to_temporaries(nir, impl, true, false);
944 nir_lower_global_vars_to_local(nir);
945 nir_lower_vars_to_ssa(nir);
946 nir_shader_gather_info(nir, impl);
947 }
948
949 prog_data->use_alt_mode = ish->use_alt_mode;
950
951 iris_setup_uniforms(compiler, mem_ctx, nir, prog_data, &system_values,
952 &num_system_values, &num_cbufs);
953
954 struct iris_binding_table bt;
955 iris_setup_binding_table(devinfo, nir, &bt, /* num_render_targets */ 0,
956 num_system_values, num_cbufs);
957
958 brw_nir_analyze_ubo_ranges(compiler, nir, NULL, prog_data->ubo_ranges);
959
960 brw_compute_vue_map(devinfo,
961 &vue_prog_data->vue_map, nir->info.outputs_written,
962 nir->info.separate_shader);
963
964 /* Don't tell the backend about our clip plane constants, we've already
965 * lowered them in NIR and we don't want it doing it again.
966 */
967 struct brw_vs_prog_key key_no_ucp = *key;
968 key_no_ucp.nr_userclip_plane_consts = 0;
969
970 char *error_str = NULL;
971 const unsigned *program =
972 brw_compile_vs(compiler, &ice->dbg, mem_ctx, &key_no_ucp, vs_prog_data,
973 nir, -1, NULL, &error_str);
974 if (program == NULL) {
975 dbg_printf("Failed to compile vertex shader: %s\n", error_str);
976 ralloc_free(mem_ctx);
977 return false;
978 }
979
980 if (ish->compiled_once) {
981 iris_debug_recompile(ice, &nir->info, &key->base);
982 } else {
983 ish->compiled_once = true;
984 }
985
986 uint32_t *so_decls =
987 ice->vtbl.create_so_decl_list(&ish->stream_output,
988 &vue_prog_data->vue_map);
989
990 struct iris_compiled_shader *shader =
991 iris_upload_shader(ice, IRIS_CACHE_VS, sizeof(*key), key, program,
992 prog_data, so_decls, system_values, num_system_values,
993 num_cbufs, &bt);
994
995 iris_disk_cache_store(screen->disk_cache, ish, shader, key, sizeof(*key));
996
997 ralloc_free(mem_ctx);
998 return shader;
999 }
1000
1001 /**
1002 * Update the current vertex shader variant.
1003 *
1004 * Fill out the key, look in the cache, compile and bind if needed.
1005 */
1006 static void
1007 iris_update_compiled_vs(struct iris_context *ice)
1008 {
1009 struct iris_shader_state *shs = &ice->state.shaders[MESA_SHADER_VERTEX];
1010 struct iris_uncompiled_shader *ish =
1011 ice->shaders.uncompiled[MESA_SHADER_VERTEX];
1012 struct iris_screen *screen = (struct iris_screen *)ice->ctx.screen;
1013 const struct gen_device_info *devinfo = &screen->devinfo;
1014
1015 struct brw_vs_prog_key key = { KEY_INIT(devinfo->gen) };
1016 ice->vtbl.populate_vs_key(ice, &ish->nir->info, last_vue_stage(ice), &key);
1017
1018 struct iris_compiled_shader *old = ice->shaders.prog[IRIS_CACHE_VS];
1019 struct iris_compiled_shader *shader =
1020 iris_find_cached_shader(ice, IRIS_CACHE_VS, sizeof(key), &key);
1021
1022 if (!shader)
1023 shader = iris_disk_cache_retrieve(ice, ish, &key, sizeof(key));
1024
1025 if (!shader)
1026 shader = iris_compile_vs(ice, ish, &key);
1027
1028 if (old != shader) {
1029 ice->shaders.prog[IRIS_CACHE_VS] = shader;
1030 ice->state.dirty |= IRIS_DIRTY_VS |
1031 IRIS_DIRTY_BINDINGS_VS |
1032 IRIS_DIRTY_CONSTANTS_VS |
1033 IRIS_DIRTY_VF_SGVS;
1034 shs->sysvals_need_upload = true;
1035
1036 const struct brw_vs_prog_data *vs_prog_data =
1037 (void *) shader->prog_data;
1038 const bool uses_draw_params = vs_prog_data->uses_firstvertex ||
1039 vs_prog_data->uses_baseinstance;
1040 const bool uses_derived_draw_params = vs_prog_data->uses_drawid ||
1041 vs_prog_data->uses_is_indexed_draw;
1042 const bool needs_sgvs_element = uses_draw_params ||
1043 vs_prog_data->uses_instanceid ||
1044 vs_prog_data->uses_vertexid;
1045 bool needs_edge_flag = false;
1046 nir_foreach_variable(var, &ish->nir->inputs) {
1047 if (var->data.location == VERT_ATTRIB_EDGEFLAG)
1048 needs_edge_flag = true;
1049 }
1050
1051 if (ice->state.vs_uses_draw_params != uses_draw_params ||
1052 ice->state.vs_uses_derived_draw_params != uses_derived_draw_params ||
1053 ice->state.vs_needs_edge_flag != needs_edge_flag) {
1054 ice->state.dirty |= IRIS_DIRTY_VERTEX_BUFFERS |
1055 IRIS_DIRTY_VERTEX_ELEMENTS;
1056 }
1057 ice->state.vs_uses_draw_params = uses_draw_params;
1058 ice->state.vs_uses_derived_draw_params = uses_derived_draw_params;
1059 ice->state.vs_needs_sgvs_element = needs_sgvs_element;
1060 ice->state.vs_needs_edge_flag = needs_edge_flag;
1061 }
1062 }
1063
1064 /**
1065 * Get the shader_info for a given stage, or NULL if the stage is disabled.
1066 */
1067 const struct shader_info *
1068 iris_get_shader_info(const struct iris_context *ice, gl_shader_stage stage)
1069 {
1070 const struct iris_uncompiled_shader *ish = ice->shaders.uncompiled[stage];
1071
1072 if (!ish)
1073 return NULL;
1074
1075 const nir_shader *nir = ish->nir;
1076 return &nir->info;
1077 }
1078
1079 /**
1080 * Get the union of TCS output and TES input slots.
1081 *
1082 * TCS and TES need to agree on a common URB entry layout. In particular,
1083 * the data for all patch vertices is stored in a single URB entry (unlike
1084 * GS which has one entry per input vertex). This means that per-vertex
1085 * array indexing needs a stride.
1086 *
1087 * SSO requires locations to match, but doesn't require the number of
1088 * outputs/inputs to match (in fact, the TCS often has extra outputs).
1089 * So, we need to take the extra step of unifying these on the fly.
1090 */
1091 static void
1092 get_unified_tess_slots(const struct iris_context *ice,
1093 uint64_t *per_vertex_slots,
1094 uint32_t *per_patch_slots)
1095 {
1096 const struct shader_info *tcs =
1097 iris_get_shader_info(ice, MESA_SHADER_TESS_CTRL);
1098 const struct shader_info *tes =
1099 iris_get_shader_info(ice, MESA_SHADER_TESS_EVAL);
1100
1101 *per_vertex_slots = tes->inputs_read;
1102 *per_patch_slots = tes->patch_inputs_read;
1103
1104 if (tcs) {
1105 *per_vertex_slots |= tcs->outputs_written;
1106 *per_patch_slots |= tcs->patch_outputs_written;
1107 }
1108 }
1109
1110 /**
1111 * Compile a tessellation control shader, and upload the assembly.
1112 */
1113 static struct iris_compiled_shader *
1114 iris_compile_tcs(struct iris_context *ice,
1115 struct iris_uncompiled_shader *ish,
1116 const struct brw_tcs_prog_key *key)
1117 {
1118 struct iris_screen *screen = (struct iris_screen *)ice->ctx.screen;
1119 const struct brw_compiler *compiler = screen->compiler;
1120 const struct nir_shader_compiler_options *options =
1121 compiler->glsl_compiler_options[MESA_SHADER_TESS_CTRL].NirOptions;
1122 void *mem_ctx = ralloc_context(NULL);
1123 struct brw_tcs_prog_data *tcs_prog_data =
1124 rzalloc(mem_ctx, struct brw_tcs_prog_data);
1125 struct brw_vue_prog_data *vue_prog_data = &tcs_prog_data->base;
1126 struct brw_stage_prog_data *prog_data = &vue_prog_data->base;
1127 const struct gen_device_info *devinfo = &screen->devinfo;
1128 enum brw_param_builtin *system_values = NULL;
1129 unsigned num_system_values = 0;
1130 unsigned num_cbufs = 0;
1131
1132 nir_shader *nir;
1133
1134 struct iris_binding_table bt;
1135
1136 if (ish) {
1137 nir = nir_shader_clone(mem_ctx, ish->nir);
1138
1139 iris_setup_uniforms(compiler, mem_ctx, nir, prog_data, &system_values,
1140 &num_system_values, &num_cbufs);
1141 iris_setup_binding_table(devinfo, nir, &bt, /* num_render_targets */ 0,
1142 num_system_values, num_cbufs);
1143 brw_nir_analyze_ubo_ranges(compiler, nir, NULL, prog_data->ubo_ranges);
1144 } else {
1145 nir = brw_nir_create_passthrough_tcs(mem_ctx, compiler, options, key);
1146
1147 /* Reserve space for passing the default tess levels as constants. */
1148 num_cbufs = 1;
1149 num_system_values = 8;
1150 system_values =
1151 rzalloc_array(mem_ctx, enum brw_param_builtin, num_system_values);
1152 prog_data->param = rzalloc_array(mem_ctx, uint32_t, num_system_values);
1153 prog_data->nr_params = num_system_values;
1154
1155 if (key->tes_primitive_mode == GL_QUADS) {
1156 for (int i = 0; i < 4; i++)
1157 system_values[7 - i] = BRW_PARAM_BUILTIN_TESS_LEVEL_OUTER_X + i;
1158
1159 system_values[3] = BRW_PARAM_BUILTIN_TESS_LEVEL_INNER_X;
1160 system_values[2] = BRW_PARAM_BUILTIN_TESS_LEVEL_INNER_Y;
1161 } else if (key->tes_primitive_mode == GL_TRIANGLES) {
1162 for (int i = 0; i < 3; i++)
1163 system_values[7 - i] = BRW_PARAM_BUILTIN_TESS_LEVEL_OUTER_X + i;
1164
1165 system_values[4] = BRW_PARAM_BUILTIN_TESS_LEVEL_INNER_X;
1166 } else {
1167 assert(key->tes_primitive_mode == GL_ISOLINES);
1168 system_values[7] = BRW_PARAM_BUILTIN_TESS_LEVEL_OUTER_Y;
1169 system_values[6] = BRW_PARAM_BUILTIN_TESS_LEVEL_OUTER_X;
1170 }
1171
1172 /* Manually setup the TCS binding table. */
1173 memset(&bt, 0, sizeof(bt));
1174 bt.sizes[IRIS_SURFACE_GROUP_UBO] = 1;
1175 bt.used_mask[IRIS_SURFACE_GROUP_UBO] = 1;
1176 bt.size_bytes = 4;
1177
1178 prog_data->ubo_ranges[0].length = 1;
1179 }
1180
1181 char *error_str = NULL;
1182 const unsigned *program =
1183 brw_compile_tcs(compiler, &ice->dbg, mem_ctx, key, tcs_prog_data, nir,
1184 -1, NULL, &error_str);
1185 if (program == NULL) {
1186 dbg_printf("Failed to compile control shader: %s\n", error_str);
1187 ralloc_free(mem_ctx);
1188 return false;
1189 }
1190
1191 if (ish) {
1192 if (ish->compiled_once) {
1193 iris_debug_recompile(ice, &nir->info, &key->base);
1194 } else {
1195 ish->compiled_once = true;
1196 }
1197 }
1198
1199 struct iris_compiled_shader *shader =
1200 iris_upload_shader(ice, IRIS_CACHE_TCS, sizeof(*key), key, program,
1201 prog_data, NULL, system_values, num_system_values,
1202 num_cbufs, &bt);
1203
1204 if (ish)
1205 iris_disk_cache_store(screen->disk_cache, ish, shader, key, sizeof(*key));
1206
1207 ralloc_free(mem_ctx);
1208 return shader;
1209 }
1210
1211 /**
1212 * Update the current tessellation control shader variant.
1213 *
1214 * Fill out the key, look in the cache, compile and bind if needed.
1215 */
1216 static void
1217 iris_update_compiled_tcs(struct iris_context *ice)
1218 {
1219 struct iris_shader_state *shs = &ice->state.shaders[MESA_SHADER_TESS_CTRL];
1220 struct iris_uncompiled_shader *tcs =
1221 ice->shaders.uncompiled[MESA_SHADER_TESS_CTRL];
1222 struct iris_screen *screen = (struct iris_screen *)ice->ctx.screen;
1223 const struct brw_compiler *compiler = screen->compiler;
1224 const struct gen_device_info *devinfo = &screen->devinfo;
1225
1226 const struct shader_info *tes_info =
1227 iris_get_shader_info(ice, MESA_SHADER_TESS_EVAL);
1228 struct brw_tcs_prog_key key = {
1229 KEY_INIT_NO_ID(devinfo->gen),
1230 .base.program_string_id = tcs ? tcs->program_id : 0,
1231 .tes_primitive_mode = tes_info->tess.primitive_mode,
1232 .input_vertices =
1233 !tcs || compiler->use_tcs_8_patch ? ice->state.vertices_per_patch : 0,
1234 .quads_workaround = devinfo->gen < 9 &&
1235 tes_info->tess.primitive_mode == GL_QUADS &&
1236 tes_info->tess.spacing == TESS_SPACING_EQUAL,
1237 };
1238 get_unified_tess_slots(ice, &key.outputs_written,
1239 &key.patch_outputs_written);
1240 ice->vtbl.populate_tcs_key(ice, &key);
1241
1242 struct iris_compiled_shader *old = ice->shaders.prog[IRIS_CACHE_TCS];
1243 struct iris_compiled_shader *shader =
1244 iris_find_cached_shader(ice, IRIS_CACHE_TCS, sizeof(key), &key);
1245
1246 if (tcs && !shader)
1247 shader = iris_disk_cache_retrieve(ice, tcs, &key, sizeof(key));
1248
1249 if (!shader)
1250 shader = iris_compile_tcs(ice, tcs, &key);
1251
1252 if (old != shader) {
1253 ice->shaders.prog[IRIS_CACHE_TCS] = shader;
1254 ice->state.dirty |= IRIS_DIRTY_TCS |
1255 IRIS_DIRTY_BINDINGS_TCS |
1256 IRIS_DIRTY_CONSTANTS_TCS;
1257 shs->sysvals_need_upload = true;
1258 }
1259 }
1260
1261 /**
1262 * Compile a tessellation evaluation shader, and upload the assembly.
1263 */
1264 static struct iris_compiled_shader *
1265 iris_compile_tes(struct iris_context *ice,
1266 struct iris_uncompiled_shader *ish,
1267 const struct brw_tes_prog_key *key)
1268 {
1269 struct iris_screen *screen = (struct iris_screen *)ice->ctx.screen;
1270 const struct brw_compiler *compiler = screen->compiler;
1271 void *mem_ctx = ralloc_context(NULL);
1272 struct brw_tes_prog_data *tes_prog_data =
1273 rzalloc(mem_ctx, struct brw_tes_prog_data);
1274 struct brw_vue_prog_data *vue_prog_data = &tes_prog_data->base;
1275 struct brw_stage_prog_data *prog_data = &vue_prog_data->base;
1276 enum brw_param_builtin *system_values;
1277 const struct gen_device_info *devinfo = &screen->devinfo;
1278 unsigned num_system_values;
1279 unsigned num_cbufs;
1280
1281 nir_shader *nir = nir_shader_clone(mem_ctx, ish->nir);
1282
1283 if (key->nr_userclip_plane_consts) {
1284 nir_function_impl *impl = nir_shader_get_entrypoint(nir);
1285 nir_lower_clip_vs(nir, (1 << key->nr_userclip_plane_consts) - 1, true,
1286 false);
1287 nir_lower_io_to_temporaries(nir, impl, true, false);
1288 nir_lower_global_vars_to_local(nir);
1289 nir_lower_vars_to_ssa(nir);
1290 nir_shader_gather_info(nir, impl);
1291 }
1292
1293 iris_setup_uniforms(compiler, mem_ctx, nir, prog_data, &system_values,
1294 &num_system_values, &num_cbufs);
1295
1296 struct iris_binding_table bt;
1297 iris_setup_binding_table(devinfo, nir, &bt, /* num_render_targets */ 0,
1298 num_system_values, num_cbufs);
1299
1300 brw_nir_analyze_ubo_ranges(compiler, nir, NULL, prog_data->ubo_ranges);
1301
1302 struct brw_vue_map input_vue_map;
1303 brw_compute_tess_vue_map(&input_vue_map, key->inputs_read,
1304 key->patch_inputs_read);
1305
1306 char *error_str = NULL;
1307 const unsigned *program =
1308 brw_compile_tes(compiler, &ice->dbg, mem_ctx, key, &input_vue_map,
1309 tes_prog_data, nir, -1, NULL, &error_str);
1310 if (program == NULL) {
1311 dbg_printf("Failed to compile evaluation shader: %s\n", error_str);
1312 ralloc_free(mem_ctx);
1313 return false;
1314 }
1315
1316 if (ish->compiled_once) {
1317 iris_debug_recompile(ice, &nir->info, &key->base);
1318 } else {
1319 ish->compiled_once = true;
1320 }
1321
1322 uint32_t *so_decls =
1323 ice->vtbl.create_so_decl_list(&ish->stream_output,
1324 &vue_prog_data->vue_map);
1325
1326
1327 struct iris_compiled_shader *shader =
1328 iris_upload_shader(ice, IRIS_CACHE_TES, sizeof(*key), key, program,
1329 prog_data, so_decls, system_values, num_system_values,
1330 num_cbufs, &bt);
1331
1332 iris_disk_cache_store(screen->disk_cache, ish, shader, key, sizeof(*key));
1333
1334 ralloc_free(mem_ctx);
1335 return shader;
1336 }
1337
1338 /**
1339 * Update the current tessellation evaluation shader variant.
1340 *
1341 * Fill out the key, look in the cache, compile and bind if needed.
1342 */
1343 static void
1344 iris_update_compiled_tes(struct iris_context *ice)
1345 {
1346 struct iris_shader_state *shs = &ice->state.shaders[MESA_SHADER_TESS_EVAL];
1347 struct iris_uncompiled_shader *ish =
1348 ice->shaders.uncompiled[MESA_SHADER_TESS_EVAL];
1349 struct iris_screen *screen = (struct iris_screen *)ice->ctx.screen;
1350 const struct gen_device_info *devinfo = &screen->devinfo;
1351
1352 struct brw_tes_prog_key key = { KEY_INIT(devinfo->gen) };
1353 get_unified_tess_slots(ice, &key.inputs_read, &key.patch_inputs_read);
1354 ice->vtbl.populate_tes_key(ice, &ish->nir->info, last_vue_stage(ice), &key);
1355
1356 struct iris_compiled_shader *old = ice->shaders.prog[IRIS_CACHE_TES];
1357 struct iris_compiled_shader *shader =
1358 iris_find_cached_shader(ice, IRIS_CACHE_TES, sizeof(key), &key);
1359
1360 if (!shader)
1361 shader = iris_disk_cache_retrieve(ice, ish, &key, sizeof(key));
1362
1363 if (!shader)
1364 shader = iris_compile_tes(ice, ish, &key);
1365
1366 if (old != shader) {
1367 ice->shaders.prog[IRIS_CACHE_TES] = shader;
1368 ice->state.dirty |= IRIS_DIRTY_TES |
1369 IRIS_DIRTY_BINDINGS_TES |
1370 IRIS_DIRTY_CONSTANTS_TES;
1371 shs->sysvals_need_upload = true;
1372 }
1373
1374 /* TODO: Could compare and avoid flagging this. */
1375 const struct shader_info *tes_info = &ish->nir->info;
1376 if (tes_info->system_values_read & (1ull << SYSTEM_VALUE_VERTICES_IN)) {
1377 ice->state.dirty |= IRIS_DIRTY_CONSTANTS_TES;
1378 ice->state.shaders[MESA_SHADER_TESS_EVAL].sysvals_need_upload = true;
1379 }
1380 }
1381
1382 /**
1383 * Compile a geometry shader, and upload the assembly.
1384 */
1385 static struct iris_compiled_shader *
1386 iris_compile_gs(struct iris_context *ice,
1387 struct iris_uncompiled_shader *ish,
1388 const struct brw_gs_prog_key *key)
1389 {
1390 struct iris_screen *screen = (struct iris_screen *)ice->ctx.screen;
1391 const struct brw_compiler *compiler = screen->compiler;
1392 const struct gen_device_info *devinfo = &screen->devinfo;
1393 void *mem_ctx = ralloc_context(NULL);
1394 struct brw_gs_prog_data *gs_prog_data =
1395 rzalloc(mem_ctx, struct brw_gs_prog_data);
1396 struct brw_vue_prog_data *vue_prog_data = &gs_prog_data->base;
1397 struct brw_stage_prog_data *prog_data = &vue_prog_data->base;
1398 enum brw_param_builtin *system_values;
1399 unsigned num_system_values;
1400 unsigned num_cbufs;
1401
1402 nir_shader *nir = nir_shader_clone(mem_ctx, ish->nir);
1403
1404 if (key->nr_userclip_plane_consts) {
1405 nir_function_impl *impl = nir_shader_get_entrypoint(nir);
1406 nir_lower_clip_gs(nir, (1 << key->nr_userclip_plane_consts) - 1, false);
1407 nir_lower_io_to_temporaries(nir, impl, true, false);
1408 nir_lower_global_vars_to_local(nir);
1409 nir_lower_vars_to_ssa(nir);
1410 nir_shader_gather_info(nir, impl);
1411 }
1412
1413 iris_setup_uniforms(compiler, mem_ctx, nir, prog_data, &system_values,
1414 &num_system_values, &num_cbufs);
1415
1416 struct iris_binding_table bt;
1417 iris_setup_binding_table(devinfo, nir, &bt, /* num_render_targets */ 0,
1418 num_system_values, num_cbufs);
1419
1420 brw_nir_analyze_ubo_ranges(compiler, nir, NULL, prog_data->ubo_ranges);
1421
1422 brw_compute_vue_map(devinfo,
1423 &vue_prog_data->vue_map, nir->info.outputs_written,
1424 nir->info.separate_shader);
1425
1426 char *error_str = NULL;
1427 const unsigned *program =
1428 brw_compile_gs(compiler, &ice->dbg, mem_ctx, key, gs_prog_data, nir,
1429 NULL, -1, NULL, &error_str);
1430 if (program == NULL) {
1431 dbg_printf("Failed to compile geometry shader: %s\n", error_str);
1432 ralloc_free(mem_ctx);
1433 return false;
1434 }
1435
1436 if (ish->compiled_once) {
1437 iris_debug_recompile(ice, &nir->info, &key->base);
1438 } else {
1439 ish->compiled_once = true;
1440 }
1441
1442 uint32_t *so_decls =
1443 ice->vtbl.create_so_decl_list(&ish->stream_output,
1444 &vue_prog_data->vue_map);
1445
1446 struct iris_compiled_shader *shader =
1447 iris_upload_shader(ice, IRIS_CACHE_GS, sizeof(*key), key, program,
1448 prog_data, so_decls, system_values, num_system_values,
1449 num_cbufs, &bt);
1450
1451 iris_disk_cache_store(screen->disk_cache, ish, shader, key, sizeof(*key));
1452
1453 ralloc_free(mem_ctx);
1454 return shader;
1455 }
1456
1457 /**
1458 * Update the current geometry shader variant.
1459 *
1460 * Fill out the key, look in the cache, compile and bind if needed.
1461 */
1462 static void
1463 iris_update_compiled_gs(struct iris_context *ice)
1464 {
1465 struct iris_shader_state *shs = &ice->state.shaders[MESA_SHADER_GEOMETRY];
1466 struct iris_uncompiled_shader *ish =
1467 ice->shaders.uncompiled[MESA_SHADER_GEOMETRY];
1468 struct iris_compiled_shader *old = ice->shaders.prog[IRIS_CACHE_GS];
1469 struct iris_compiled_shader *shader = NULL;
1470
1471 if (ish) {
1472 struct iris_screen *screen = (struct iris_screen *)ice->ctx.screen;
1473 const struct gen_device_info *devinfo = &screen->devinfo;
1474 struct brw_gs_prog_key key = { KEY_INIT(devinfo->gen) };
1475 ice->vtbl.populate_gs_key(ice, &ish->nir->info, last_vue_stage(ice), &key);
1476
1477 shader =
1478 iris_find_cached_shader(ice, IRIS_CACHE_GS, sizeof(key), &key);
1479
1480 if (!shader)
1481 shader = iris_disk_cache_retrieve(ice, ish, &key, sizeof(key));
1482
1483 if (!shader)
1484 shader = iris_compile_gs(ice, ish, &key);
1485 }
1486
1487 if (old != shader) {
1488 ice->shaders.prog[IRIS_CACHE_GS] = shader;
1489 ice->state.dirty |= IRIS_DIRTY_GS |
1490 IRIS_DIRTY_BINDINGS_GS |
1491 IRIS_DIRTY_CONSTANTS_GS;
1492 shs->sysvals_need_upload = true;
1493 }
1494 }
1495
1496 /**
1497 * Compile a fragment (pixel) shader, and upload the assembly.
1498 */
1499 static struct iris_compiled_shader *
1500 iris_compile_fs(struct iris_context *ice,
1501 struct iris_uncompiled_shader *ish,
1502 const struct brw_wm_prog_key *key,
1503 struct brw_vue_map *vue_map)
1504 {
1505 struct iris_screen *screen = (struct iris_screen *)ice->ctx.screen;
1506 const struct brw_compiler *compiler = screen->compiler;
1507 void *mem_ctx = ralloc_context(NULL);
1508 struct brw_wm_prog_data *fs_prog_data =
1509 rzalloc(mem_ctx, struct brw_wm_prog_data);
1510 struct brw_stage_prog_data *prog_data = &fs_prog_data->base;
1511 enum brw_param_builtin *system_values;
1512 const struct gen_device_info *devinfo = &screen->devinfo;
1513 unsigned num_system_values;
1514 unsigned num_cbufs;
1515
1516 nir_shader *nir = nir_shader_clone(mem_ctx, ish->nir);
1517
1518 prog_data->use_alt_mode = ish->use_alt_mode;
1519
1520 iris_setup_uniforms(compiler, mem_ctx, nir, prog_data, &system_values,
1521 &num_system_values, &num_cbufs);
1522
1523 /* Lower output variables to load_output intrinsics before setting up
1524 * binding tables, so iris_setup_binding_table can map any load_output
1525 * intrinsics to IRIS_SURFACE_GROUP_RENDER_TARGET_READ on Gen8 for
1526 * non-coherent framebuffer fetches.
1527 */
1528 brw_nir_lower_fs_outputs(nir);
1529
1530 /* On Gen11+, shader RT write messages have a "Null Render Target" bit
1531 * and do not need a binding table entry with a null surface. Earlier
1532 * generations need an entry for a null surface.
1533 */
1534 int null_rts = devinfo->gen < 11 ? 1 : 0;
1535
1536 struct iris_binding_table bt;
1537 iris_setup_binding_table(devinfo, nir, &bt,
1538 MAX2(key->nr_color_regions, null_rts),
1539 num_system_values, num_cbufs);
1540
1541 brw_nir_analyze_ubo_ranges(compiler, nir, NULL, prog_data->ubo_ranges);
1542
1543 char *error_str = NULL;
1544 const unsigned *program =
1545 brw_compile_fs(compiler, &ice->dbg, mem_ctx, key, fs_prog_data,
1546 nir, -1, -1, -1, true, false, vue_map,
1547 NULL, &error_str);
1548 if (program == NULL) {
1549 dbg_printf("Failed to compile fragment shader: %s\n", error_str);
1550 ralloc_free(mem_ctx);
1551 return false;
1552 }
1553
1554 if (ish->compiled_once) {
1555 iris_debug_recompile(ice, &nir->info, &key->base);
1556 } else {
1557 ish->compiled_once = true;
1558 }
1559
1560 struct iris_compiled_shader *shader =
1561 iris_upload_shader(ice, IRIS_CACHE_FS, sizeof(*key), key, program,
1562 prog_data, NULL, system_values, num_system_values,
1563 num_cbufs, &bt);
1564
1565 iris_disk_cache_store(screen->disk_cache, ish, shader, key, sizeof(*key));
1566
1567 ralloc_free(mem_ctx);
1568 return shader;
1569 }
1570
1571 /**
1572 * Update the current fragment shader variant.
1573 *
1574 * Fill out the key, look in the cache, compile and bind if needed.
1575 */
1576 static void
1577 iris_update_compiled_fs(struct iris_context *ice)
1578 {
1579 struct iris_screen *screen = (struct iris_screen *)ice->ctx.screen;
1580 const struct gen_device_info *devinfo = &screen->devinfo;
1581 struct iris_shader_state *shs = &ice->state.shaders[MESA_SHADER_FRAGMENT];
1582 struct iris_uncompiled_shader *ish =
1583 ice->shaders.uncompiled[MESA_SHADER_FRAGMENT];
1584 struct brw_wm_prog_key key = { KEY_INIT(devinfo->gen) };
1585 ice->vtbl.populate_fs_key(ice, &ish->nir->info, &key);
1586
1587 if (ish->nos & (1ull << IRIS_NOS_LAST_VUE_MAP))
1588 key.input_slots_valid = ice->shaders.last_vue_map->slots_valid;
1589
1590 struct iris_compiled_shader *old = ice->shaders.prog[IRIS_CACHE_FS];
1591 struct iris_compiled_shader *shader =
1592 iris_find_cached_shader(ice, IRIS_CACHE_FS, sizeof(key), &key);
1593
1594 if (!shader)
1595 shader = iris_disk_cache_retrieve(ice, ish, &key, sizeof(key));
1596
1597 if (!shader)
1598 shader = iris_compile_fs(ice, ish, &key, ice->shaders.last_vue_map);
1599
1600 if (old != shader) {
1601 // XXX: only need to flag CLIP if barycentric has NONPERSPECTIVE
1602 // toggles. might be able to avoid flagging SBE too.
1603 ice->shaders.prog[IRIS_CACHE_FS] = shader;
1604 ice->state.dirty |= IRIS_DIRTY_FS |
1605 IRIS_DIRTY_BINDINGS_FS |
1606 IRIS_DIRTY_CONSTANTS_FS |
1607 IRIS_DIRTY_WM |
1608 IRIS_DIRTY_CLIP |
1609 IRIS_DIRTY_SBE;
1610 shs->sysvals_need_upload = true;
1611 }
1612 }
1613
1614 /**
1615 * Update the last enabled stage's VUE map.
1616 *
1617 * When the shader feeding the rasterizer's output interface changes, we
1618 * need to re-emit various packets.
1619 */
1620 static void
1621 update_last_vue_map(struct iris_context *ice,
1622 struct brw_stage_prog_data *prog_data)
1623 {
1624 struct brw_vue_prog_data *vue_prog_data = (void *) prog_data;
1625 struct brw_vue_map *vue_map = &vue_prog_data->vue_map;
1626 struct brw_vue_map *old_map = ice->shaders.last_vue_map;
1627 const uint64_t changed_slots =
1628 (old_map ? old_map->slots_valid : 0ull) ^ vue_map->slots_valid;
1629
1630 if (changed_slots & VARYING_BIT_VIEWPORT) {
1631 ice->state.num_viewports =
1632 (vue_map->slots_valid & VARYING_BIT_VIEWPORT) ? IRIS_MAX_VIEWPORTS : 1;
1633 ice->state.dirty |= IRIS_DIRTY_CLIP |
1634 IRIS_DIRTY_SF_CL_VIEWPORT |
1635 IRIS_DIRTY_CC_VIEWPORT |
1636 IRIS_DIRTY_SCISSOR_RECT |
1637 IRIS_DIRTY_UNCOMPILED_FS |
1638 ice->state.dirty_for_nos[IRIS_NOS_LAST_VUE_MAP];
1639 }
1640
1641 if (changed_slots || (old_map && old_map->separate != vue_map->separate)) {
1642 ice->state.dirty |= IRIS_DIRTY_SBE;
1643 }
1644
1645 ice->shaders.last_vue_map = &vue_prog_data->vue_map;
1646 }
1647
1648 static void
1649 iris_update_pull_constant_descriptors(struct iris_context *ice,
1650 gl_shader_stage stage)
1651 {
1652 struct iris_compiled_shader *shader = ice->shaders.prog[stage];
1653
1654 if (!shader || !shader->prog_data->has_ubo_pull)
1655 return;
1656
1657 struct iris_shader_state *shs = &ice->state.shaders[stage];
1658 bool any_new_descriptors =
1659 shader->num_system_values > 0 && shs->sysvals_need_upload;
1660
1661 unsigned bound_cbufs = shs->bound_cbufs;
1662
1663 while (bound_cbufs) {
1664 const int i = u_bit_scan(&bound_cbufs);
1665 struct pipe_shader_buffer *cbuf = &shs->constbuf[i];
1666 struct iris_state_ref *surf_state = &shs->constbuf_surf_state[i];
1667 if (!surf_state->res && cbuf->buffer) {
1668 iris_upload_ubo_ssbo_surf_state(ice, cbuf, surf_state, false);
1669 any_new_descriptors = true;
1670 }
1671 }
1672
1673 if (any_new_descriptors)
1674 ice->state.dirty |= IRIS_DIRTY_BINDINGS_VS << stage;
1675 }
1676
1677 /**
1678 * Get the prog_data for a given stage, or NULL if the stage is disabled.
1679 */
1680 static struct brw_vue_prog_data *
1681 get_vue_prog_data(struct iris_context *ice, gl_shader_stage stage)
1682 {
1683 if (!ice->shaders.prog[stage])
1684 return NULL;
1685
1686 return (void *) ice->shaders.prog[stage]->prog_data;
1687 }
1688
1689 // XXX: iris_compiled_shaders are space-leaking :(
1690 // XXX: do remember to unbind them if deleting them.
1691
1692 /**
1693 * Update the current shader variants for the given state.
1694 *
1695 * This should be called on every draw call to ensure that the correct
1696 * shaders are bound. It will also flag any dirty state triggered by
1697 * swapping out those shaders.
1698 */
1699 void
1700 iris_update_compiled_shaders(struct iris_context *ice)
1701 {
1702 const uint64_t dirty = ice->state.dirty;
1703
1704 struct brw_vue_prog_data *old_prog_datas[4];
1705 if (!(dirty & IRIS_DIRTY_URB)) {
1706 for (int i = MESA_SHADER_VERTEX; i <= MESA_SHADER_GEOMETRY; i++)
1707 old_prog_datas[i] = get_vue_prog_data(ice, i);
1708 }
1709
1710 if (dirty & (IRIS_DIRTY_UNCOMPILED_TCS | IRIS_DIRTY_UNCOMPILED_TES)) {
1711 struct iris_uncompiled_shader *tes =
1712 ice->shaders.uncompiled[MESA_SHADER_TESS_EVAL];
1713 if (tes) {
1714 iris_update_compiled_tcs(ice);
1715 iris_update_compiled_tes(ice);
1716 } else {
1717 ice->shaders.prog[IRIS_CACHE_TCS] = NULL;
1718 ice->shaders.prog[IRIS_CACHE_TES] = NULL;
1719 ice->state.dirty |=
1720 IRIS_DIRTY_TCS | IRIS_DIRTY_TES |
1721 IRIS_DIRTY_BINDINGS_TCS | IRIS_DIRTY_BINDINGS_TES |
1722 IRIS_DIRTY_CONSTANTS_TCS | IRIS_DIRTY_CONSTANTS_TES;
1723 }
1724 }
1725
1726 if (dirty & IRIS_DIRTY_UNCOMPILED_VS)
1727 iris_update_compiled_vs(ice);
1728 if (dirty & IRIS_DIRTY_UNCOMPILED_GS)
1729 iris_update_compiled_gs(ice);
1730
1731 if (dirty & (IRIS_DIRTY_UNCOMPILED_GS | IRIS_DIRTY_UNCOMPILED_TES)) {
1732 const struct iris_compiled_shader *gs =
1733 ice->shaders.prog[MESA_SHADER_GEOMETRY];
1734 const struct iris_compiled_shader *tes =
1735 ice->shaders.prog[MESA_SHADER_TESS_EVAL];
1736
1737 bool points_or_lines = false;
1738
1739 if (gs) {
1740 const struct brw_gs_prog_data *gs_prog_data = (void *) gs->prog_data;
1741 points_or_lines =
1742 gs_prog_data->output_topology == _3DPRIM_POINTLIST ||
1743 gs_prog_data->output_topology == _3DPRIM_LINESTRIP;
1744 } else if (tes) {
1745 const struct brw_tes_prog_data *tes_data = (void *) tes->prog_data;
1746 points_or_lines =
1747 tes_data->output_topology == BRW_TESS_OUTPUT_TOPOLOGY_LINE ||
1748 tes_data->output_topology == BRW_TESS_OUTPUT_TOPOLOGY_POINT;
1749 }
1750
1751 if (ice->shaders.output_topology_is_points_or_lines != points_or_lines) {
1752 /* Outbound to XY Clip enables */
1753 ice->shaders.output_topology_is_points_or_lines = points_or_lines;
1754 ice->state.dirty |= IRIS_DIRTY_CLIP;
1755 }
1756 }
1757
1758 gl_shader_stage last_stage = last_vue_stage(ice);
1759 struct iris_compiled_shader *shader = ice->shaders.prog[last_stage];
1760 struct iris_uncompiled_shader *ish = ice->shaders.uncompiled[last_stage];
1761 update_last_vue_map(ice, shader->prog_data);
1762 if (ice->state.streamout != shader->streamout) {
1763 ice->state.streamout = shader->streamout;
1764 ice->state.dirty |= IRIS_DIRTY_SO_DECL_LIST | IRIS_DIRTY_STREAMOUT;
1765 }
1766
1767 if (ice->state.streamout_active) {
1768 for (int i = 0; i < PIPE_MAX_SO_BUFFERS; i++) {
1769 struct iris_stream_output_target *so =
1770 (void *) ice->state.so_target[i];
1771 if (so)
1772 so->stride = ish->stream_output.stride[i] * sizeof(uint32_t);
1773 }
1774 }
1775
1776 if (dirty & IRIS_DIRTY_UNCOMPILED_FS)
1777 iris_update_compiled_fs(ice);
1778
1779 /* Changing shader interfaces may require a URB configuration. */
1780 if (!(dirty & IRIS_DIRTY_URB)) {
1781 for (int i = MESA_SHADER_VERTEX; i <= MESA_SHADER_GEOMETRY; i++) {
1782 struct brw_vue_prog_data *old = old_prog_datas[i];
1783 struct brw_vue_prog_data *new = get_vue_prog_data(ice, i);
1784 if (!!old != !!new ||
1785 (new && new->urb_entry_size != old->urb_entry_size)) {
1786 ice->state.dirty |= IRIS_DIRTY_URB;
1787 break;
1788 }
1789 }
1790 }
1791
1792 for (int i = MESA_SHADER_VERTEX; i <= MESA_SHADER_FRAGMENT; i++) {
1793 if (ice->state.dirty & (IRIS_DIRTY_CONSTANTS_VS << i))
1794 iris_update_pull_constant_descriptors(ice, i);
1795 }
1796 }
1797
1798 static struct iris_compiled_shader *
1799 iris_compile_cs(struct iris_context *ice,
1800 struct iris_uncompiled_shader *ish,
1801 const struct brw_cs_prog_key *key)
1802 {
1803 struct iris_screen *screen = (struct iris_screen *)ice->ctx.screen;
1804 const struct brw_compiler *compiler = screen->compiler;
1805 void *mem_ctx = ralloc_context(NULL);
1806 struct brw_cs_prog_data *cs_prog_data =
1807 rzalloc(mem_ctx, struct brw_cs_prog_data);
1808 struct brw_stage_prog_data *prog_data = &cs_prog_data->base;
1809 enum brw_param_builtin *system_values;
1810 const struct gen_device_info *devinfo = &screen->devinfo;
1811 unsigned num_system_values;
1812 unsigned num_cbufs;
1813
1814 nir_shader *nir = nir_shader_clone(mem_ctx, ish->nir);
1815
1816 iris_setup_uniforms(compiler, mem_ctx, nir, prog_data, &system_values,
1817 &num_system_values, &num_cbufs);
1818
1819 struct iris_binding_table bt;
1820 iris_setup_binding_table(devinfo, nir, &bt, /* num_render_targets */ 0,
1821 num_system_values, num_cbufs);
1822
1823 char *error_str = NULL;
1824 const unsigned *program =
1825 brw_compile_cs(compiler, &ice->dbg, mem_ctx, key, cs_prog_data,
1826 nir, -1, NULL, &error_str);
1827 if (program == NULL) {
1828 dbg_printf("Failed to compile compute shader: %s\n", error_str);
1829 ralloc_free(mem_ctx);
1830 return false;
1831 }
1832
1833 if (ish->compiled_once) {
1834 iris_debug_recompile(ice, &nir->info, &key->base);
1835 } else {
1836 ish->compiled_once = true;
1837 }
1838
1839 struct iris_compiled_shader *shader =
1840 iris_upload_shader(ice, IRIS_CACHE_CS, sizeof(*key), key, program,
1841 prog_data, NULL, system_values, num_system_values,
1842 num_cbufs, &bt);
1843
1844 iris_disk_cache_store(screen->disk_cache, ish, shader, key, sizeof(*key));
1845
1846 ralloc_free(mem_ctx);
1847 return shader;
1848 }
1849
1850 static void
1851 iris_update_compiled_cs(struct iris_context *ice)
1852 {
1853 struct iris_shader_state *shs = &ice->state.shaders[MESA_SHADER_COMPUTE];
1854 struct iris_uncompiled_shader *ish =
1855 ice->shaders.uncompiled[MESA_SHADER_COMPUTE];
1856
1857 struct iris_screen *screen = (struct iris_screen *)ice->ctx.screen;
1858 const struct gen_device_info *devinfo = &screen->devinfo;
1859 struct brw_cs_prog_key key = { KEY_INIT(devinfo->gen) };
1860 ice->vtbl.populate_cs_key(ice, &key);
1861
1862 struct iris_compiled_shader *old = ice->shaders.prog[IRIS_CACHE_CS];
1863 struct iris_compiled_shader *shader =
1864 iris_find_cached_shader(ice, IRIS_CACHE_CS, sizeof(key), &key);
1865
1866 if (!shader)
1867 shader = iris_disk_cache_retrieve(ice, ish, &key, sizeof(key));
1868
1869 if (!shader)
1870 shader = iris_compile_cs(ice, ish, &key);
1871
1872 if (old != shader) {
1873 ice->shaders.prog[IRIS_CACHE_CS] = shader;
1874 ice->state.dirty |= IRIS_DIRTY_CS |
1875 IRIS_DIRTY_BINDINGS_CS |
1876 IRIS_DIRTY_CONSTANTS_CS;
1877 shs->sysvals_need_upload = true;
1878 }
1879 }
1880
1881 void
1882 iris_update_compiled_compute_shader(struct iris_context *ice)
1883 {
1884 if (ice->state.dirty & IRIS_DIRTY_UNCOMPILED_CS)
1885 iris_update_compiled_cs(ice);
1886
1887 if (ice->state.dirty & IRIS_DIRTY_CONSTANTS_CS)
1888 iris_update_pull_constant_descriptors(ice, MESA_SHADER_COMPUTE);
1889 }
1890
1891 void
1892 iris_fill_cs_push_const_buffer(struct brw_cs_prog_data *cs_prog_data,
1893 uint32_t *dst)
1894 {
1895 assert(cs_prog_data->push.total.size > 0);
1896 assert(cs_prog_data->push.cross_thread.size == 0);
1897 assert(cs_prog_data->push.per_thread.dwords == 1);
1898 assert(cs_prog_data->base.param[0] == BRW_PARAM_BUILTIN_SUBGROUP_ID);
1899 for (unsigned t = 0; t < cs_prog_data->threads; t++)
1900 dst[8 * t] = t;
1901 }
1902
1903 /**
1904 * Allocate scratch BOs as needed for the given per-thread size and stage.
1905 */
1906 struct iris_bo *
1907 iris_get_scratch_space(struct iris_context *ice,
1908 unsigned per_thread_scratch,
1909 gl_shader_stage stage)
1910 {
1911 struct iris_screen *screen = (struct iris_screen *)ice->ctx.screen;
1912 struct iris_bufmgr *bufmgr = screen->bufmgr;
1913 const struct gen_device_info *devinfo = &screen->devinfo;
1914
1915 unsigned encoded_size = ffs(per_thread_scratch) - 11;
1916 assert(encoded_size < (1 << 16));
1917
1918 struct iris_bo **bop = &ice->shaders.scratch_bos[encoded_size][stage];
1919
1920 /* The documentation for 3DSTATE_PS "Scratch Space Base Pointer" says:
1921 *
1922 * "Scratch Space per slice is computed based on 4 sub-slices. SW
1923 * must allocate scratch space enough so that each slice has 4
1924 * slices allowed."
1925 *
1926 * According to the other driver team, this applies to compute shaders
1927 * as well. This is not currently documented at all.
1928 *
1929 * This hack is no longer necessary on Gen11+.
1930 */
1931 unsigned subslice_total = screen->subslice_total;
1932 if (devinfo->gen < 11)
1933 subslice_total = 4 * devinfo->num_slices;
1934 assert(subslice_total >= screen->subslice_total);
1935
1936 if (!*bop) {
1937 unsigned scratch_ids_per_subslice = devinfo->max_cs_threads;
1938
1939 if (devinfo->gen >= 11) {
1940 /* The MEDIA_VFE_STATE docs say:
1941 *
1942 * "Starting with this configuration, the Maximum Number of
1943 * Threads must be set to (#EU * 8) for GPGPU dispatches.
1944 *
1945 * Although there are only 7 threads per EU in the configuration,
1946 * the FFTID is calculated as if there are 8 threads per EU,
1947 * which in turn requires a larger amount of Scratch Space to be
1948 * allocated by the driver."
1949 */
1950 scratch_ids_per_subslice = 8 * 8;
1951 }
1952
1953 uint32_t max_threads[] = {
1954 [MESA_SHADER_VERTEX] = devinfo->max_vs_threads,
1955 [MESA_SHADER_TESS_CTRL] = devinfo->max_tcs_threads,
1956 [MESA_SHADER_TESS_EVAL] = devinfo->max_tes_threads,
1957 [MESA_SHADER_GEOMETRY] = devinfo->max_gs_threads,
1958 [MESA_SHADER_FRAGMENT] = devinfo->max_wm_threads,
1959 [MESA_SHADER_COMPUTE] = scratch_ids_per_subslice * subslice_total,
1960 };
1961
1962 uint32_t size = per_thread_scratch * max_threads[stage];
1963
1964 *bop = iris_bo_alloc(bufmgr, "scratch", size, IRIS_MEMZONE_SHADER);
1965 }
1966
1967 return *bop;
1968 }
1969
1970 /* ------------------------------------------------------------------- */
1971
1972 /**
1973 * The pipe->create_[stage]_state() driver hooks.
1974 *
1975 * Performs basic NIR preprocessing, records any state dependencies, and
1976 * returns an iris_uncompiled_shader as the Gallium CSO.
1977 *
1978 * Actual shader compilation to assembly happens later, at first use.
1979 */
1980 static void *
1981 iris_create_uncompiled_shader(struct pipe_context *ctx,
1982 nir_shader *nir,
1983 const struct pipe_stream_output_info *so_info)
1984 {
1985 struct iris_context *ice = (void *)ctx;
1986 struct iris_screen *screen = (struct iris_screen *)ctx->screen;
1987 const struct gen_device_info *devinfo = &screen->devinfo;
1988
1989 struct iris_uncompiled_shader *ish =
1990 calloc(1, sizeof(struct iris_uncompiled_shader));
1991 if (!ish)
1992 return NULL;
1993
1994 brw_preprocess_nir(screen->compiler, nir, NULL);
1995
1996 NIR_PASS_V(nir, brw_nir_lower_image_load_store, devinfo);
1997 NIR_PASS_V(nir, iris_lower_storage_image_derefs);
1998
1999 nir_sweep(nir);
2000
2001 if (nir->constant_data_size > 0) {
2002 unsigned data_offset;
2003 u_upload_data(ice->shaders.uploader, 0, nir->constant_data_size,
2004 32, nir->constant_data, &data_offset, &ish->const_data);
2005
2006 struct pipe_shader_buffer psb = {
2007 .buffer = ish->const_data,
2008 .buffer_offset = data_offset,
2009 .buffer_size = nir->constant_data_size,
2010 };
2011 iris_upload_ubo_ssbo_surf_state(ice, &psb, &ish->const_data_state, false);
2012 }
2013
2014 ish->program_id = get_new_program_id(screen);
2015 ish->nir = nir;
2016 if (so_info) {
2017 memcpy(&ish->stream_output, so_info, sizeof(*so_info));
2018 update_so_info(&ish->stream_output, nir->info.outputs_written);
2019 }
2020
2021 /* Save this now before potentially dropping nir->info.name */
2022 if (nir->info.name && strncmp(nir->info.name, "ARB", 3) == 0)
2023 ish->use_alt_mode = true;
2024
2025 if (screen->disk_cache) {
2026 /* Serialize the NIR to a binary blob that we can hash for the disk
2027 * cache. Drop unnecessary information (like variable names)
2028 * so the serialized NIR is smaller, and also to let us detect more
2029 * isomorphic shaders when hashing, increasing cache hits.
2030 */
2031 struct blob blob;
2032 blob_init(&blob);
2033 nir_serialize(&blob, nir, true);
2034 _mesa_sha1_compute(blob.data, blob.size, ish->nir_sha1);
2035 blob_finish(&blob);
2036 }
2037
2038 return ish;
2039 }
2040
2041 static struct iris_uncompiled_shader *
2042 iris_create_shader_state(struct pipe_context *ctx,
2043 const struct pipe_shader_state *state)
2044 {
2045 struct nir_shader *nir;
2046
2047 if (state->type == PIPE_SHADER_IR_TGSI)
2048 nir = tgsi_to_nir(state->tokens, ctx->screen);
2049 else
2050 nir = state->ir.nir;
2051
2052 return iris_create_uncompiled_shader(ctx, nir, &state->stream_output);
2053 }
2054
2055 static void *
2056 iris_create_vs_state(struct pipe_context *ctx,
2057 const struct pipe_shader_state *state)
2058 {
2059 struct iris_context *ice = (void *) ctx;
2060 struct iris_screen *screen = (void *) ctx->screen;
2061 struct iris_uncompiled_shader *ish = iris_create_shader_state(ctx, state);
2062
2063 /* User clip planes */
2064 if (ish->nir->info.clip_distance_array_size == 0)
2065 ish->nos |= (1ull << IRIS_NOS_RASTERIZER);
2066
2067 if (screen->precompile) {
2068 const struct gen_device_info *devinfo = &screen->devinfo;
2069 struct brw_vs_prog_key key = { KEY_INIT(devinfo->gen) };
2070
2071 if (!iris_disk_cache_retrieve(ice, ish, &key, sizeof(key)))
2072 iris_compile_vs(ice, ish, &key);
2073 }
2074
2075 return ish;
2076 }
2077
2078 static void *
2079 iris_create_tcs_state(struct pipe_context *ctx,
2080 const struct pipe_shader_state *state)
2081 {
2082 struct iris_context *ice = (void *) ctx;
2083 struct iris_screen *screen = (void *) ctx->screen;
2084 const struct brw_compiler *compiler = screen->compiler;
2085 struct iris_uncompiled_shader *ish = iris_create_shader_state(ctx, state);
2086 struct shader_info *info = &ish->nir->info;
2087
2088 if (screen->precompile) {
2089 const unsigned _GL_TRIANGLES = 0x0004;
2090 const struct gen_device_info *devinfo = &screen->devinfo;
2091 struct brw_tcs_prog_key key = {
2092 KEY_INIT(devinfo->gen),
2093 // XXX: make sure the linker fills this out from the TES...
2094 .tes_primitive_mode =
2095 info->tess.primitive_mode ? info->tess.primitive_mode
2096 : _GL_TRIANGLES,
2097 .outputs_written = info->outputs_written,
2098 .patch_outputs_written = info->patch_outputs_written,
2099 };
2100
2101 /* 8_PATCH mode needs the key to contain the input patch dimensionality.
2102 * We don't have that information, so we randomly guess that the input
2103 * and output patches are the same size. This is a bad guess, but we
2104 * can't do much better.
2105 */
2106 if (compiler->use_tcs_8_patch)
2107 key.input_vertices = info->tess.tcs_vertices_out;
2108
2109 if (!iris_disk_cache_retrieve(ice, ish, &key, sizeof(key)))
2110 iris_compile_tcs(ice, ish, &key);
2111 }
2112
2113 return ish;
2114 }
2115
2116 static void *
2117 iris_create_tes_state(struct pipe_context *ctx,
2118 const struct pipe_shader_state *state)
2119 {
2120 struct iris_context *ice = (void *) ctx;
2121 struct iris_screen *screen = (void *) ctx->screen;
2122 struct iris_uncompiled_shader *ish = iris_create_shader_state(ctx, state);
2123 struct shader_info *info = &ish->nir->info;
2124
2125 /* User clip planes */
2126 if (ish->nir->info.clip_distance_array_size == 0)
2127 ish->nos |= (1ull << IRIS_NOS_RASTERIZER);
2128
2129 if (screen->precompile) {
2130 const struct gen_device_info *devinfo = &screen->devinfo;
2131 struct brw_tes_prog_key key = {
2132 KEY_INIT(devinfo->gen),
2133 // XXX: not ideal, need TCS output/TES input unification
2134 .inputs_read = info->inputs_read,
2135 .patch_inputs_read = info->patch_inputs_read,
2136 };
2137
2138 if (!iris_disk_cache_retrieve(ice, ish, &key, sizeof(key)))
2139 iris_compile_tes(ice, ish, &key);
2140 }
2141
2142 return ish;
2143 }
2144
2145 static void *
2146 iris_create_gs_state(struct pipe_context *ctx,
2147 const struct pipe_shader_state *state)
2148 {
2149 struct iris_context *ice = (void *) ctx;
2150 struct iris_screen *screen = (void *) ctx->screen;
2151 struct iris_uncompiled_shader *ish = iris_create_shader_state(ctx, state);
2152
2153 /* User clip planes */
2154 if (ish->nir->info.clip_distance_array_size == 0)
2155 ish->nos |= (1ull << IRIS_NOS_RASTERIZER);
2156
2157 if (screen->precompile) {
2158 const struct gen_device_info *devinfo = &screen->devinfo;
2159 struct brw_gs_prog_key key = { KEY_INIT(devinfo->gen) };
2160
2161 if (!iris_disk_cache_retrieve(ice, ish, &key, sizeof(key)))
2162 iris_compile_gs(ice, ish, &key);
2163 }
2164
2165 return ish;
2166 }
2167
2168 static void *
2169 iris_create_fs_state(struct pipe_context *ctx,
2170 const struct pipe_shader_state *state)
2171 {
2172 struct iris_context *ice = (void *) ctx;
2173 struct iris_screen *screen = (void *) ctx->screen;
2174 struct iris_uncompiled_shader *ish = iris_create_shader_state(ctx, state);
2175 struct shader_info *info = &ish->nir->info;
2176
2177 ish->nos |= (1ull << IRIS_NOS_FRAMEBUFFER) |
2178 (1ull << IRIS_NOS_DEPTH_STENCIL_ALPHA) |
2179 (1ull << IRIS_NOS_RASTERIZER) |
2180 (1ull << IRIS_NOS_BLEND);
2181
2182 /* The program key needs the VUE map if there are > 16 inputs */
2183 if (util_bitcount64(ish->nir->info.inputs_read &
2184 BRW_FS_VARYING_INPUT_MASK) > 16) {
2185 ish->nos |= (1ull << IRIS_NOS_LAST_VUE_MAP);
2186 }
2187
2188 if (screen->precompile) {
2189 const uint64_t color_outputs = info->outputs_written &
2190 ~(BITFIELD64_BIT(FRAG_RESULT_DEPTH) |
2191 BITFIELD64_BIT(FRAG_RESULT_STENCIL) |
2192 BITFIELD64_BIT(FRAG_RESULT_SAMPLE_MASK));
2193
2194 bool can_rearrange_varyings =
2195 util_bitcount64(info->inputs_read & BRW_FS_VARYING_INPUT_MASK) <= 16;
2196
2197 const struct gen_device_info *devinfo = &screen->devinfo;
2198 struct brw_wm_prog_key key = {
2199 KEY_INIT(devinfo->gen),
2200 .nr_color_regions = util_bitcount(color_outputs),
2201 .coherent_fb_fetch = devinfo->gen >= 9,
2202 .input_slots_valid =
2203 can_rearrange_varyings ? 0 : info->inputs_read | VARYING_BIT_POS,
2204 };
2205
2206 if (!iris_disk_cache_retrieve(ice, ish, &key, sizeof(key)))
2207 iris_compile_fs(ice, ish, &key, NULL);
2208 }
2209
2210 return ish;
2211 }
2212
2213 static void *
2214 iris_create_compute_state(struct pipe_context *ctx,
2215 const struct pipe_compute_state *state)
2216 {
2217 assert(state->ir_type == PIPE_SHADER_IR_NIR);
2218
2219 struct iris_context *ice = (void *) ctx;
2220 struct iris_screen *screen = (void *) ctx->screen;
2221 struct iris_uncompiled_shader *ish =
2222 iris_create_uncompiled_shader(ctx, (void *) state->prog, NULL);
2223
2224 // XXX: disallow more than 64KB of shared variables
2225
2226 if (screen->precompile) {
2227 const struct gen_device_info *devinfo = &screen->devinfo;
2228 struct brw_cs_prog_key key = { KEY_INIT(devinfo->gen) };
2229
2230 if (!iris_disk_cache_retrieve(ice, ish, &key, sizeof(key)))
2231 iris_compile_cs(ice, ish, &key);
2232 }
2233
2234 return ish;
2235 }
2236
2237 /**
2238 * The pipe->delete_[stage]_state() driver hooks.
2239 *
2240 * Frees the iris_uncompiled_shader.
2241 */
2242 static void
2243 iris_delete_shader_state(struct pipe_context *ctx, void *state, gl_shader_stage stage)
2244 {
2245 struct iris_uncompiled_shader *ish = state;
2246 struct iris_context *ice = (void *) ctx;
2247
2248 if (ice->shaders.uncompiled[stage] == ish) {
2249 ice->shaders.uncompiled[stage] = NULL;
2250 ice->state.dirty |= IRIS_DIRTY_UNCOMPILED_VS << stage;
2251 }
2252
2253 if (ish->const_data) {
2254 pipe_resource_reference(&ish->const_data, NULL);
2255 pipe_resource_reference(&ish->const_data_state.res, NULL);
2256 }
2257
2258 ralloc_free(ish->nir);
2259 free(ish);
2260 }
2261
2262 static void
2263 iris_delete_vs_state(struct pipe_context *ctx, void *state)
2264 {
2265 iris_delete_shader_state(ctx, state, MESA_SHADER_VERTEX);
2266 }
2267
2268 static void
2269 iris_delete_tcs_state(struct pipe_context *ctx, void *state)
2270 {
2271 iris_delete_shader_state(ctx, state, MESA_SHADER_TESS_CTRL);
2272 }
2273
2274 static void
2275 iris_delete_tes_state(struct pipe_context *ctx, void *state)
2276 {
2277 iris_delete_shader_state(ctx, state, MESA_SHADER_TESS_EVAL);
2278 }
2279
2280 static void
2281 iris_delete_gs_state(struct pipe_context *ctx, void *state)
2282 {
2283 iris_delete_shader_state(ctx, state, MESA_SHADER_GEOMETRY);
2284 }
2285
2286 static void
2287 iris_delete_fs_state(struct pipe_context *ctx, void *state)
2288 {
2289 iris_delete_shader_state(ctx, state, MESA_SHADER_FRAGMENT);
2290 }
2291
2292 static void
2293 iris_delete_cs_state(struct pipe_context *ctx, void *state)
2294 {
2295 iris_delete_shader_state(ctx, state, MESA_SHADER_COMPUTE);
2296 }
2297
2298 /**
2299 * The pipe->bind_[stage]_state() driver hook.
2300 *
2301 * Binds an uncompiled shader as the current one for a particular stage.
2302 * Updates dirty tracking to account for the shader's NOS.
2303 */
2304 static void
2305 bind_shader_state(struct iris_context *ice,
2306 struct iris_uncompiled_shader *ish,
2307 gl_shader_stage stage)
2308 {
2309 uint64_t dirty_bit = IRIS_DIRTY_UNCOMPILED_VS << stage;
2310 const uint64_t nos = ish ? ish->nos : 0;
2311
2312 const struct shader_info *old_info = iris_get_shader_info(ice, stage);
2313 const struct shader_info *new_info = ish ? &ish->nir->info : NULL;
2314
2315 if ((old_info ? util_last_bit(old_info->textures_used) : 0) !=
2316 (new_info ? util_last_bit(new_info->textures_used) : 0)) {
2317 ice->state.dirty |= IRIS_DIRTY_SAMPLER_STATES_VS << stage;
2318 }
2319
2320 ice->shaders.uncompiled[stage] = ish;
2321 ice->state.dirty |= dirty_bit;
2322
2323 /* Record that CSOs need to mark IRIS_DIRTY_UNCOMPILED_XS when they change
2324 * (or that they no longer need to do so).
2325 */
2326 for (int i = 0; i < IRIS_NOS_COUNT; i++) {
2327 if (nos & (1 << i))
2328 ice->state.dirty_for_nos[i] |= dirty_bit;
2329 else
2330 ice->state.dirty_for_nos[i] &= ~dirty_bit;
2331 }
2332 }
2333
2334 static void
2335 iris_bind_vs_state(struct pipe_context *ctx, void *state)
2336 {
2337 struct iris_context *ice = (struct iris_context *)ctx;
2338 struct iris_uncompiled_shader *new_ish = state;
2339
2340 if (new_ish &&
2341 ice->state.window_space_position !=
2342 new_ish->nir->info.vs.window_space_position) {
2343 ice->state.window_space_position =
2344 new_ish->nir->info.vs.window_space_position;
2345
2346 ice->state.dirty |= IRIS_DIRTY_CLIP |
2347 IRIS_DIRTY_RASTER |
2348 IRIS_DIRTY_CC_VIEWPORT;
2349 }
2350
2351 bind_shader_state((void *) ctx, state, MESA_SHADER_VERTEX);
2352 }
2353
2354 static void
2355 iris_bind_tcs_state(struct pipe_context *ctx, void *state)
2356 {
2357 bind_shader_state((void *) ctx, state, MESA_SHADER_TESS_CTRL);
2358 }
2359
2360 static void
2361 iris_bind_tes_state(struct pipe_context *ctx, void *state)
2362 {
2363 struct iris_context *ice = (struct iris_context *)ctx;
2364
2365 /* Enabling/disabling optional stages requires a URB reconfiguration. */
2366 if (!!state != !!ice->shaders.uncompiled[MESA_SHADER_TESS_EVAL])
2367 ice->state.dirty |= IRIS_DIRTY_URB;
2368
2369 bind_shader_state((void *) ctx, state, MESA_SHADER_TESS_EVAL);
2370 }
2371
2372 static void
2373 iris_bind_gs_state(struct pipe_context *ctx, void *state)
2374 {
2375 struct iris_context *ice = (struct iris_context *)ctx;
2376
2377 /* Enabling/disabling optional stages requires a URB reconfiguration. */
2378 if (!!state != !!ice->shaders.uncompiled[MESA_SHADER_GEOMETRY])
2379 ice->state.dirty |= IRIS_DIRTY_URB;
2380
2381 bind_shader_state((void *) ctx, state, MESA_SHADER_GEOMETRY);
2382 }
2383
2384 static void
2385 iris_bind_fs_state(struct pipe_context *ctx, void *state)
2386 {
2387 struct iris_context *ice = (struct iris_context *) ctx;
2388 struct iris_screen *screen = (struct iris_screen *) ctx->screen;
2389 const struct gen_device_info *devinfo = &screen->devinfo;
2390 struct iris_uncompiled_shader *old_ish =
2391 ice->shaders.uncompiled[MESA_SHADER_FRAGMENT];
2392 struct iris_uncompiled_shader *new_ish = state;
2393
2394 const unsigned color_bits =
2395 BITFIELD64_BIT(FRAG_RESULT_COLOR) |
2396 BITFIELD64_RANGE(FRAG_RESULT_DATA0, BRW_MAX_DRAW_BUFFERS);
2397
2398 /* Fragment shader outputs influence HasWriteableRT */
2399 if (!old_ish || !new_ish ||
2400 (old_ish->nir->info.outputs_written & color_bits) !=
2401 (new_ish->nir->info.outputs_written & color_bits))
2402 ice->state.dirty |= IRIS_DIRTY_PS_BLEND;
2403
2404 if (devinfo->gen == 8)
2405 ice->state.dirty |= IRIS_DIRTY_PMA_FIX;
2406
2407 bind_shader_state((void *) ctx, state, MESA_SHADER_FRAGMENT);
2408 }
2409
2410 static void
2411 iris_bind_cs_state(struct pipe_context *ctx, void *state)
2412 {
2413 bind_shader_state((void *) ctx, state, MESA_SHADER_COMPUTE);
2414 }
2415
2416 void
2417 iris_init_program_functions(struct pipe_context *ctx)
2418 {
2419 ctx->create_vs_state = iris_create_vs_state;
2420 ctx->create_tcs_state = iris_create_tcs_state;
2421 ctx->create_tes_state = iris_create_tes_state;
2422 ctx->create_gs_state = iris_create_gs_state;
2423 ctx->create_fs_state = iris_create_fs_state;
2424 ctx->create_compute_state = iris_create_compute_state;
2425
2426 ctx->delete_vs_state = iris_delete_vs_state;
2427 ctx->delete_tcs_state = iris_delete_tcs_state;
2428 ctx->delete_tes_state = iris_delete_tes_state;
2429 ctx->delete_gs_state = iris_delete_gs_state;
2430 ctx->delete_fs_state = iris_delete_fs_state;
2431 ctx->delete_compute_state = iris_delete_cs_state;
2432
2433 ctx->bind_vs_state = iris_bind_vs_state;
2434 ctx->bind_tcs_state = iris_bind_tcs_state;
2435 ctx->bind_tes_state = iris_bind_tes_state;
2436 ctx->bind_gs_state = iris_bind_gs_state;
2437 ctx->bind_fs_state = iris_bind_fs_state;
2438 ctx->bind_compute_state = iris_bind_cs_state;
2439 }