projects / mesa.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
history | raw | HEAD
nir: Drop nir_tex_instr::texture_array_size
[mesa.git] / src / compiler / nir / nir.c
1 /*
2 * Copyright © 2014 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 *
23 * Authors:
24 * Connor Abbott (cwabbott0@gmail.com)
25 *
26 */
27
28 #include "nir.h"
29 #include "nir_builder.h"
30 #include "nir_control_flow_private.h"
31 #include "util/half_float.h"
32 #include <limits.h>
33 #include <assert.h>
34 #include <math.h>
35 #include "util/u_math.h"
36
37 #include "main/menums.h" /* BITFIELD64_MASK */
38
39 nir_shader *
40 nir_shader_create(void *mem_ctx,
41 gl_shader_stage stage,
42 const nir_shader_compiler_options *options,
43 shader_info *si)
44 {
45 nir_shader *shader = rzalloc(mem_ctx, nir_shader);
46
47 exec_list_make_empty(&shader->uniforms);
48 exec_list_make_empty(&shader->inputs);
49 exec_list_make_empty(&shader->outputs);
50 exec_list_make_empty(&shader->shared);
51
52 shader->options = options;
53
54 if (si) {
55 assert(si->stage == stage);
56 shader->info = *si;
57 } else {
58 shader->info.stage = stage;
59 }
60
61 exec_list_make_empty(&shader->functions);
62 exec_list_make_empty(&shader->globals);
63 exec_list_make_empty(&shader->system_values);
64
65 shader->num_inputs = 0;
66 shader->num_outputs = 0;
67 shader->num_uniforms = 0;
68 shader->num_shared = 0;
69
70 return shader;
71 }
72
73 static nir_register *
74 reg_create(void *mem_ctx, struct exec_list *list)
75 {
76 nir_register *reg = ralloc(mem_ctx, nir_register);
77
78 list_inithead(&reg->uses);
79 list_inithead(&reg->defs);
80 list_inithead(&reg->if_uses);
81
82 reg->num_components = 0;
83 reg->bit_size = 32;
84 reg->num_array_elems = 0;
85 reg->name = NULL;
86
87 exec_list_push_tail(list, &reg->node);
88
89 return reg;
90 }
91
92 nir_register *
93 nir_local_reg_create(nir_function_impl *impl)
94 {
95 nir_register *reg = reg_create(ralloc_parent(impl), &impl->registers);
96 reg->index = impl->reg_alloc++;
97
98 return reg;
99 }
100
101 void
102 nir_reg_remove(nir_register *reg)
103 {
104 exec_node_remove(&reg->node);
105 }
106
107 void
108 nir_shader_add_variable(nir_shader *shader, nir_variable *var)
109 {
110 switch (var->data.mode) {
111 case nir_num_variable_modes:
112 case nir_var_all:
113 assert(!"invalid mode");
114 break;
115
116 case nir_var_function_temp:
117 assert(!"nir_shader_add_variable cannot be used for local variables");
118 break;
119
120 case nir_var_shader_temp:
121 exec_list_push_tail(&shader->globals, &var->node);
122 break;
123
124 case nir_var_shader_in:
125 exec_list_push_tail(&shader->inputs, &var->node);
126 break;
127
128 case nir_var_shader_out:
129 exec_list_push_tail(&shader->outputs, &var->node);
130 break;
131
132 case nir_var_uniform:
133 case nir_var_mem_ubo:
134 case nir_var_mem_ssbo:
135 exec_list_push_tail(&shader->uniforms, &var->node);
136 break;
137
138 case nir_var_mem_shared:
139 assert(gl_shader_stage_is_compute(shader->info.stage));
140 exec_list_push_tail(&shader->shared, &var->node);
141 break;
142
143 case nir_var_mem_global:
144 assert(!"nir_shader_add_variable cannot be used for global memory");
145 break;
146
147 case nir_var_system_value:
148 exec_list_push_tail(&shader->system_values, &var->node);
149 break;
150
151 case nir_var_mem_push_const:
152 assert(!"nir_var_push_constant is not supposed to be used for variables");
153 break;
154 }
155 }
156
157 nir_variable *
158 nir_variable_create(nir_shader *shader, nir_variable_mode mode,
159 const struct glsl_type *type, const char *name)
160 {
161 nir_variable *var = rzalloc(shader, nir_variable);
162 var->name = ralloc_strdup(var, name);
163 var->type = type;
164 var->data.mode = mode;
165 var->data.how_declared = nir_var_declared_normally;
166
167 if ((mode == nir_var_shader_in &&
168 shader->info.stage != MESA_SHADER_VERTEX) ||
169 (mode == nir_var_shader_out &&
170 shader->info.stage != MESA_SHADER_FRAGMENT))
171 var->data.interpolation = INTERP_MODE_SMOOTH;
172
173 if (mode == nir_var_shader_in || mode == nir_var_uniform)
174 var->data.read_only = true;
175
176 nir_shader_add_variable(shader, var);
177
178 return var;
179 }
180
181 nir_variable *
182 nir_local_variable_create(nir_function_impl *impl,
183 const struct glsl_type *type, const char *name)
184 {
185 nir_variable *var = rzalloc(impl->function->shader, nir_variable);
186 var->name = ralloc_strdup(var, name);
187 var->type = type;
188 var->data.mode = nir_var_function_temp;
189
190 nir_function_impl_add_variable(impl, var);
191
192 return var;
193 }
194
195 nir_function *
196 nir_function_create(nir_shader *shader, const char *name)
197 {
198 nir_function *func = ralloc(shader, nir_function);
199
200 exec_list_push_tail(&shader->functions, &func->node);
201
202 func->name = ralloc_strdup(func, name);
203 func->shader = shader;
204 func->num_params = 0;
205 func->params = NULL;
206 func->impl = NULL;
207 func->is_entrypoint = false;
208
209 return func;
210 }
211
212 /* NOTE: if the instruction you are copying a src to is already added
213 * to the IR, use nir_instr_rewrite_src() instead.
214 */
215 void nir_src_copy(nir_src *dest, const nir_src *src, void *mem_ctx)
216 {
217 dest->is_ssa = src->is_ssa;
218 if (src->is_ssa) {
219 dest->ssa = src->ssa;
220 } else {
221 dest->reg.base_offset = src->reg.base_offset;
222 dest->reg.reg = src->reg.reg;
223 if (src->reg.indirect) {
224 dest->reg.indirect = ralloc(mem_ctx, nir_src);
225 nir_src_copy(dest->reg.indirect, src->reg.indirect, mem_ctx);
226 } else {
227 dest->reg.indirect = NULL;
228 }
229 }
230 }
231
232 void nir_dest_copy(nir_dest *dest, const nir_dest *src, nir_instr *instr)
233 {
234 /* Copying an SSA definition makes no sense whatsoever. */
235 assert(!src->is_ssa);
236
237 dest->is_ssa = false;
238
239 dest->reg.base_offset = src->reg.base_offset;
240 dest->reg.reg = src->reg.reg;
241 if (src->reg.indirect) {
242 dest->reg.indirect = ralloc(instr, nir_src);
243 nir_src_copy(dest->reg.indirect, src->reg.indirect, instr);
244 } else {
245 dest->reg.indirect = NULL;
246 }
247 }
248
249 void
250 nir_alu_src_copy(nir_alu_src *dest, const nir_alu_src *src,
251 nir_alu_instr *instr)
252 {
253 nir_src_copy(&dest->src, &src->src, &instr->instr);
254 dest->abs = src->abs;
255 dest->negate = src->negate;
256 for (unsigned i = 0; i < NIR_MAX_VEC_COMPONENTS; i++)
257 dest->swizzle[i] = src->swizzle[i];
258 }
259
260 void
261 nir_alu_dest_copy(nir_alu_dest *dest, const nir_alu_dest *src,
262 nir_alu_instr *instr)
263 {
264 nir_dest_copy(&dest->dest, &src->dest, &instr->instr);
265 dest->write_mask = src->write_mask;
266 dest->saturate = src->saturate;
267 }
268
269
270 static void
271 cf_init(nir_cf_node *node, nir_cf_node_type type)
272 {
273 exec_node_init(&node->node);
274 node->parent = NULL;
275 node->type = type;
276 }
277
278 nir_function_impl *
279 nir_function_impl_create_bare(nir_shader *shader)
280 {
281 nir_function_impl *impl = ralloc(shader, nir_function_impl);
282
283 impl->function = NULL;
284
285 cf_init(&impl->cf_node, nir_cf_node_function);
286
287 exec_list_make_empty(&impl->body);
288 exec_list_make_empty(&impl->registers);
289 exec_list_make_empty(&impl->locals);
290 impl->reg_alloc = 0;
291 impl->ssa_alloc = 0;
292 impl->valid_metadata = nir_metadata_none;
293
294 /* create start & end blocks */
295 nir_block *start_block = nir_block_create(shader);
296 nir_block *end_block = nir_block_create(shader);
297 start_block->cf_node.parent = &impl->cf_node;
298 end_block->cf_node.parent = &impl->cf_node;
299 impl->end_block = end_block;
300
301 exec_list_push_tail(&impl->body, &start_block->cf_node.node);
302
303 start_block->successors[0] = end_block;
304 _mesa_set_add(end_block->predecessors, start_block);
305 return impl;
306 }
307
308 nir_function_impl *
309 nir_function_impl_create(nir_function *function)
310 {
311 assert(function->impl == NULL);
312
313 nir_function_impl *impl = nir_function_impl_create_bare(function->shader);
314
315 function->impl = impl;
316 impl->function = function;
317
318 return impl;
319 }
320
321 nir_block *
322 nir_block_create(nir_shader *shader)
323 {
324 nir_block *block = rzalloc(shader, nir_block);
325
326 cf_init(&block->cf_node, nir_cf_node_block);
327
328 block->successors[0] = block->successors[1] = NULL;
329 block->predecessors = _mesa_pointer_set_create(block);
330 block->imm_dom = NULL;
331 /* XXX maybe it would be worth it to defer allocation? This
332 * way it doesn't get allocated for shader refs that never run
333 * nir_calc_dominance? For example, state-tracker creates an
334 * initial IR, clones that, runs appropriate lowering pass, passes
335 * to driver which does common lowering/opt, and then stores ref
336 * which is later used to do state specific lowering and futher
337 * opt. Do any of the references not need dominance metadata?
338 */
339 block->dom_frontier = _mesa_pointer_set_create(block);
340
341 exec_list_make_empty(&block->instr_list);
342
343 return block;
344 }
345
346 static inline void
347 src_init(nir_src *src)
348 {
349 src->is_ssa = false;
350 src->reg.reg = NULL;
351 src->reg.indirect = NULL;
352 src->reg.base_offset = 0;
353 }
354
355 nir_if *
356 nir_if_create(nir_shader *shader)
357 {
358 nir_if *if_stmt = ralloc(shader, nir_if);
359
360 if_stmt->control = nir_selection_control_none;
361
362 cf_init(&if_stmt->cf_node, nir_cf_node_if);
363 src_init(&if_stmt->condition);
364
365 nir_block *then = nir_block_create(shader);
366 exec_list_make_empty(&if_stmt->then_list);
367 exec_list_push_tail(&if_stmt->then_list, &then->cf_node.node);
368 then->cf_node.parent = &if_stmt->cf_node;
369
370 nir_block *else_stmt = nir_block_create(shader);
371 exec_list_make_empty(&if_stmt->else_list);
372 exec_list_push_tail(&if_stmt->else_list, &else_stmt->cf_node.node);
373 else_stmt->cf_node.parent = &if_stmt->cf_node;
374
375 return if_stmt;
376 }
377
378 nir_loop *
379 nir_loop_create(nir_shader *shader)
380 {
381 nir_loop *loop = rzalloc(shader, nir_loop);
382
383 cf_init(&loop->cf_node, nir_cf_node_loop);
384
385 nir_block *body = nir_block_create(shader);
386 exec_list_make_empty(&loop->body);
387 exec_list_push_tail(&loop->body, &body->cf_node.node);
388 body->cf_node.parent = &loop->cf_node;
389
390 body->successors[0] = body;
391 _mesa_set_add(body->predecessors, body);
392
393 return loop;
394 }
395
396 static void
397 instr_init(nir_instr *instr, nir_instr_type type)
398 {
399 instr->type = type;
400 instr->block = NULL;
401 exec_node_init(&instr->node);
402 }
403
404 static void
405 dest_init(nir_dest *dest)
406 {
407 dest->is_ssa = false;
408 dest->reg.reg = NULL;
409 dest->reg.indirect = NULL;
410 dest->reg.base_offset = 0;
411 }
412
413 static void
414 alu_dest_init(nir_alu_dest *dest)
415 {
416 dest_init(&dest->dest);
417 dest->saturate = false;
418 dest->write_mask = 0xf;
419 }
420
421 static void
422 alu_src_init(nir_alu_src *src)
423 {
424 src_init(&src->src);
425 src->abs = src->negate = false;
426 for (int i = 0; i < NIR_MAX_VEC_COMPONENTS; ++i)
427 src->swizzle[i] = i;
428 }
429
430 nir_alu_instr *
431 nir_alu_instr_create(nir_shader *shader, nir_op op)
432 {
433 unsigned num_srcs = nir_op_infos[op].num_inputs;
434 /* TODO: don't use rzalloc */
435 nir_alu_instr *instr =
436 rzalloc_size(shader,
437 sizeof(nir_alu_instr) + num_srcs * sizeof(nir_alu_src));
438
439 instr_init(&instr->instr, nir_instr_type_alu);
440 instr->op = op;
441 alu_dest_init(&instr->dest);
442 for (unsigned i = 0; i < num_srcs; i++)
443 alu_src_init(&instr->src[i]);
444
445 return instr;
446 }
447
448 nir_deref_instr *
449 nir_deref_instr_create(nir_shader *shader, nir_deref_type deref_type)
450 {
451 nir_deref_instr *instr =
452 rzalloc_size(shader, sizeof(nir_deref_instr));
453
454 instr_init(&instr->instr, nir_instr_type_deref);
455
456 instr->deref_type = deref_type;
457 if (deref_type != nir_deref_type_var)
458 src_init(&instr->parent);
459
460 if (deref_type == nir_deref_type_array ||
461 deref_type == nir_deref_type_ptr_as_array)
462 src_init(&instr->arr.index);
463
464 dest_init(&instr->dest);
465
466 return instr;
467 }
468
469 nir_jump_instr *
470 nir_jump_instr_create(nir_shader *shader, nir_jump_type type)
471 {
472 nir_jump_instr *instr = ralloc(shader, nir_jump_instr);
473 instr_init(&instr->instr, nir_instr_type_jump);
474 instr->type = type;
475 return instr;
476 }
477
478 nir_load_const_instr *
479 nir_load_const_instr_create(nir_shader *shader, unsigned num_components,
480 unsigned bit_size)
481 {
482 nir_load_const_instr *instr =
483 rzalloc_size(shader, sizeof(*instr) + num_components * sizeof(*instr->value));
484 instr_init(&instr->instr, nir_instr_type_load_const);
485
486 nir_ssa_def_init(&instr->instr, &instr->def, num_components, bit_size, NULL);
487
488 return instr;
489 }
490
491 nir_intrinsic_instr *
492 nir_intrinsic_instr_create(nir_shader *shader, nir_intrinsic_op op)
493 {
494 unsigned num_srcs = nir_intrinsic_infos[op].num_srcs;
495 /* TODO: don't use rzalloc */
496 nir_intrinsic_instr *instr =
497 rzalloc_size(shader,
498 sizeof(nir_intrinsic_instr) + num_srcs * sizeof(nir_src));
499
500 instr_init(&instr->instr, nir_instr_type_intrinsic);
501 instr->intrinsic = op;
502
503 if (nir_intrinsic_infos[op].has_dest)
504 dest_init(&instr->dest);
505
506 for (unsigned i = 0; i < num_srcs; i++)
507 src_init(&instr->src[i]);
508
509 return instr;
510 }
511
512 nir_call_instr *
513 nir_call_instr_create(nir_shader *shader, nir_function *callee)
514 {
515 const unsigned num_params = callee->num_params;
516 nir_call_instr *instr =
517 rzalloc_size(shader, sizeof(*instr) +
518 num_params * sizeof(instr->params[0]));
519
520 instr_init(&instr->instr, nir_instr_type_call);
521 instr->callee = callee;
522 instr->num_params = num_params;
523 for (unsigned i = 0; i < num_params; i++)
524 src_init(&instr->params[i]);
525
526 return instr;
527 }
528
529 static int8_t default_tg4_offsets[4][2] =
530 {
531 { 0, 1 },
532 { 1, 1 },
533 { 1, 0 },
534 { 0, 0 },
535 };
536
537 nir_tex_instr *
538 nir_tex_instr_create(nir_shader *shader, unsigned num_srcs)
539 {
540 nir_tex_instr *instr = rzalloc(shader, nir_tex_instr);
541 instr_init(&instr->instr, nir_instr_type_tex);
542
543 dest_init(&instr->dest);
544
545 instr->num_srcs = num_srcs;
546 instr->src = ralloc_array(instr, nir_tex_src, num_srcs);
547 for (unsigned i = 0; i < num_srcs; i++)
548 src_init(&instr->src[i].src);
549
550 instr->texture_index = 0;
551 instr->sampler_index = 0;
552 memcpy(instr->tg4_offsets, default_tg4_offsets, sizeof(instr->tg4_offsets));
553
554 return instr;
555 }
556
557 void
558 nir_tex_instr_add_src(nir_tex_instr *tex,
559 nir_tex_src_type src_type,
560 nir_src src)
561 {
562 nir_tex_src *new_srcs = rzalloc_array(tex, nir_tex_src,
563 tex->num_srcs + 1);
564
565 for (unsigned i = 0; i < tex->num_srcs; i++) {
566 new_srcs[i].src_type = tex->src[i].src_type;
567 nir_instr_move_src(&tex->instr, &new_srcs[i].src,
568 &tex->src[i].src);
569 }
570
571 ralloc_free(tex->src);
572 tex->src = new_srcs;
573
574 tex->src[tex->num_srcs].src_type = src_type;
575 nir_instr_rewrite_src(&tex->instr, &tex->src[tex->num_srcs].src, src);
576 tex->num_srcs++;
577 }
578
579 void
580 nir_tex_instr_remove_src(nir_tex_instr *tex, unsigned src_idx)
581 {
582 assert(src_idx < tex->num_srcs);
583
584 /* First rewrite the source to NIR_SRC_INIT */
585 nir_instr_rewrite_src(&tex->instr, &tex->src[src_idx].src, NIR_SRC_INIT);
586
587 /* Now, move all of the other sources down */
588 for (unsigned i = src_idx + 1; i < tex->num_srcs; i++) {
589 tex->src[i-1].src_type = tex->src[i].src_type;
590 nir_instr_move_src(&tex->instr, &tex->src[i-1].src, &tex->src[i].src);
591 }
592 tex->num_srcs--;
593 }
594
595 bool
596 nir_tex_instr_has_explicit_tg4_offsets(nir_tex_instr *tex)
597 {
598 if (tex->op != nir_texop_tg4)
599 return false;
600 return memcmp(tex->tg4_offsets, default_tg4_offsets,
601 sizeof(tex->tg4_offsets)) != 0;
602 }
603
604 nir_phi_instr *
605 nir_phi_instr_create(nir_shader *shader)
606 {
607 nir_phi_instr *instr = ralloc(shader, nir_phi_instr);
608 instr_init(&instr->instr, nir_instr_type_phi);
609
610 dest_init(&instr->dest);
611 exec_list_make_empty(&instr->srcs);
612 return instr;
613 }
614
615 nir_parallel_copy_instr *
616 nir_parallel_copy_instr_create(nir_shader *shader)
617 {
618 nir_parallel_copy_instr *instr = ralloc(shader, nir_parallel_copy_instr);
619 instr_init(&instr->instr, nir_instr_type_parallel_copy);
620
621 exec_list_make_empty(&instr->entries);
622
623 return instr;
624 }
625
626 nir_ssa_undef_instr *
627 nir_ssa_undef_instr_create(nir_shader *shader,
628 unsigned num_components,
629 unsigned bit_size)
630 {
631 nir_ssa_undef_instr *instr = ralloc(shader, nir_ssa_undef_instr);
632 instr_init(&instr->instr, nir_instr_type_ssa_undef);
633
634 nir_ssa_def_init(&instr->instr, &instr->def, num_components, bit_size, NULL);
635
636 return instr;
637 }
638
639 static nir_const_value
640 const_value_float(double d, unsigned bit_size)
641 {
642 nir_const_value v;
643 memset(&v, 0, sizeof(v));
644 switch (bit_size) {
645 case 16: v.u16 = _mesa_float_to_half(d); break;
646 case 32: v.f32 = d; break;
647 case 64: v.f64 = d; break;
648 default:
649 unreachable("Invalid bit size");
650 }
651 return v;
652 }
653
654 static nir_const_value
655 const_value_int(int64_t i, unsigned bit_size)
656 {
657 nir_const_value v;
658 memset(&v, 0, sizeof(v));
659 switch (bit_size) {
660 case 1: v.b = i & 1; break;
661 case 8: v.i8 = i; break;
662 case 16: v.i16 = i; break;
663 case 32: v.i32 = i; break;
664 case 64: v.i64 = i; break;
665 default:
666 unreachable("Invalid bit size");
667 }
668 return v;
669 }
670
671 nir_const_value
672 nir_alu_binop_identity(nir_op binop, unsigned bit_size)
673 {
674 const int64_t max_int = (1ull << (bit_size - 1)) - 1;
675 const int64_t min_int = -max_int - 1;
676 switch (binop) {
677 case nir_op_iadd:
678 return const_value_int(0, bit_size);
679 case nir_op_fadd:
680 return const_value_float(0, bit_size);
681 case nir_op_imul:
682 return const_value_int(1, bit_size);
683 case nir_op_fmul:
684 return const_value_float(1, bit_size);
685 case nir_op_imin:
686 return const_value_int(max_int, bit_size);
687 case nir_op_umin:
688 return const_value_int(~0ull, bit_size);
689 case nir_op_fmin:
690 return const_value_float(INFINITY, bit_size);
691 case nir_op_imax:
692 return const_value_int(min_int, bit_size);
693 case nir_op_umax:
694 return const_value_int(0, bit_size);
695 case nir_op_fmax:
696 return const_value_float(-INFINITY, bit_size);
697 case nir_op_iand:
698 return const_value_int(~0ull, bit_size);
699 case nir_op_ior:
700 return const_value_int(0, bit_size);
701 case nir_op_ixor:
702 return const_value_int(0, bit_size);
703 default:
704 unreachable("Invalid reduction operation");
705 }
706 }
707
708 nir_function_impl *
709 nir_cf_node_get_function(nir_cf_node *node)
710 {
711 while (node->type != nir_cf_node_function) {
712 node = node->parent;
713 }
714
715 return nir_cf_node_as_function(node);
716 }
717
718 /* Reduces a cursor by trying to convert everything to after and trying to
719 * go up to block granularity when possible.
720 */
721 static nir_cursor
722 reduce_cursor(nir_cursor cursor)
723 {
724 switch (cursor.option) {
725 case nir_cursor_before_block:
726 assert(nir_cf_node_prev(&cursor.block->cf_node) == NULL ||
727 nir_cf_node_prev(&cursor.block->cf_node)->type != nir_cf_node_block);
728 if (exec_list_is_empty(&cursor.block->instr_list)) {
729 /* Empty block. After is as good as before. */
730 cursor.option = nir_cursor_after_block;
731 }
732 return cursor;
733
734 case nir_cursor_after_block:
735 return cursor;
736
737 case nir_cursor_before_instr: {
738 nir_instr *prev_instr = nir_instr_prev(cursor.instr);
739 if (prev_instr) {
740 /* Before this instruction is after the previous */
741 cursor.instr = prev_instr;
742 cursor.option = nir_cursor_after_instr;
743 } else {
744 /* No previous instruction. Switch to before block */
745 cursor.block = cursor.instr->block;
746 cursor.option = nir_cursor_before_block;
747 }
748 return reduce_cursor(cursor);
749 }
750
751 case nir_cursor_after_instr:
752 if (nir_instr_next(cursor.instr) == NULL) {
753 /* This is the last instruction, switch to after block */
754 cursor.option = nir_cursor_after_block;
755 cursor.block = cursor.instr->block;
756 }
757 return cursor;
758
759 default:
760 unreachable("Inavlid cursor option");
761 }
762 }
763
764 bool
765 nir_cursors_equal(nir_cursor a, nir_cursor b)
766 {
767 /* Reduced cursors should be unique */
768 a = reduce_cursor(a);
769 b = reduce_cursor(b);
770
771 return a.block == b.block && a.option == b.option;
772 }
773
774 static bool
775 add_use_cb(nir_src *src, void *state)
776 {
777 nir_instr *instr = state;
778
779 src->parent_instr = instr;
780 list_addtail(&src->use_link,
781 src->is_ssa ? &src->ssa->uses : &src->reg.reg->uses);
782
783 return true;
784 }
785
786 static bool
787 add_ssa_def_cb(nir_ssa_def *def, void *state)
788 {
789 nir_instr *instr = state;
790
791 if (instr->block && def->index == UINT_MAX) {
792 nir_function_impl *impl =
793 nir_cf_node_get_function(&instr->block->cf_node);
794
795 def->index = impl->ssa_alloc++;
796 }
797
798 return true;
799 }
800
801 static bool
802 add_reg_def_cb(nir_dest *dest, void *state)
803 {
804 nir_instr *instr = state;
805
806 if (!dest->is_ssa) {
807 dest->reg.parent_instr = instr;
808 list_addtail(&dest->reg.def_link, &dest->reg.reg->defs);
809 }
810
811 return true;
812 }
813
814 static void
815 add_defs_uses(nir_instr *instr)
816 {
817 nir_foreach_src(instr, add_use_cb, instr);
818 nir_foreach_dest(instr, add_reg_def_cb, instr);
819 nir_foreach_ssa_def(instr, add_ssa_def_cb, instr);
820 }
821
822 void
823 nir_instr_insert(nir_cursor cursor, nir_instr *instr)
824 {
825 switch (cursor.option) {
826 case nir_cursor_before_block:
827 /* Only allow inserting jumps into empty blocks. */
828 if (instr->type == nir_instr_type_jump)
829 assert(exec_list_is_empty(&cursor.block->instr_list));
830
831 instr->block = cursor.block;
832 add_defs_uses(instr);
833 exec_list_push_head(&cursor.block->instr_list, &instr->node);
834 break;
835 case nir_cursor_after_block: {
836 /* Inserting instructions after a jump is illegal. */
837 nir_instr *last = nir_block_last_instr(cursor.block);
838 assert(last == NULL || last->type != nir_instr_type_jump);
839 (void) last;
840
841 instr->block = cursor.block;
842 add_defs_uses(instr);
843 exec_list_push_tail(&cursor.block->instr_list, &instr->node);
844 break;
845 }
846 case nir_cursor_before_instr:
847 assert(instr->type != nir_instr_type_jump);
848 instr->block = cursor.instr->block;
849 add_defs_uses(instr);
850 exec_node_insert_node_before(&cursor.instr->node, &instr->node);
851 break;
852 case nir_cursor_after_instr:
853 /* Inserting instructions after a jump is illegal. */
854 assert(cursor.instr->type != nir_instr_type_jump);
855
856 /* Only allow inserting jumps at the end of the block. */
857 if (instr->type == nir_instr_type_jump)
858 assert(cursor.instr == nir_block_last_instr(cursor.instr->block));
859
860 instr->block = cursor.instr->block;
861 add_defs_uses(instr);
862 exec_node_insert_after(&cursor.instr->node, &instr->node);
863 break;
864 }
865
866 if (instr->type == nir_instr_type_jump)
867 nir_handle_add_jump(instr->block);
868 }
869
870 static bool
871 src_is_valid(const nir_src *src)
872 {
873 return src->is_ssa ? (src->ssa != NULL) : (src->reg.reg != NULL);
874 }
875
876 static bool
877 remove_use_cb(nir_src *src, void *state)
878 {
879 (void) state;
880
881 if (src_is_valid(src))
882 list_del(&src->use_link);
883
884 return true;
885 }
886
887 static bool
888 remove_def_cb(nir_dest *dest, void *state)
889 {
890 (void) state;
891
892 if (!dest->is_ssa)
893 list_del(&dest->reg.def_link);
894
895 return true;
896 }
897
898 static void
899 remove_defs_uses(nir_instr *instr)
900 {
901 nir_foreach_dest(instr, remove_def_cb, instr);
902 nir_foreach_src(instr, remove_use_cb, instr);
903 }
904
905 void nir_instr_remove_v(nir_instr *instr)
906 {
907 remove_defs_uses(instr);
908 exec_node_remove(&instr->node);
909
910 if (instr->type == nir_instr_type_jump) {
911 nir_jump_instr *jump_instr = nir_instr_as_jump(instr);
912 nir_handle_remove_jump(instr->block, jump_instr->type);
913 }
914 }
915
916 /*@}*/
917
918 void
919 nir_index_local_regs(nir_function_impl *impl)
920 {
921 unsigned index = 0;
922 foreach_list_typed(nir_register, reg, node, &impl->registers) {
923 reg->index = index++;
924 }
925 impl->reg_alloc = index;
926 }
927
928 static bool
929 visit_alu_dest(nir_alu_instr *instr, nir_foreach_dest_cb cb, void *state)
930 {
931 return cb(&instr->dest.dest, state);
932 }
933
934 static bool
935 visit_deref_dest(nir_deref_instr *instr, nir_foreach_dest_cb cb, void *state)
936 {
937 return cb(&instr->dest, state);
938 }
939
940 static bool
941 visit_intrinsic_dest(nir_intrinsic_instr *instr, nir_foreach_dest_cb cb,
942 void *state)
943 {
944 if (nir_intrinsic_infos[instr->intrinsic].has_dest)
945 return cb(&instr->dest, state);
946
947 return true;
948 }
949
950 static bool
951 visit_texture_dest(nir_tex_instr *instr, nir_foreach_dest_cb cb,
952 void *state)
953 {
954 return cb(&instr->dest, state);
955 }
956
957 static bool
958 visit_phi_dest(nir_phi_instr *instr, nir_foreach_dest_cb cb, void *state)
959 {
960 return cb(&instr->dest, state);
961 }
962
963 static bool
964 visit_parallel_copy_dest(nir_parallel_copy_instr *instr,
965 nir_foreach_dest_cb cb, void *state)
966 {
967 nir_foreach_parallel_copy_entry(entry, instr) {
968 if (!cb(&entry->dest, state))
969 return false;
970 }
971
972 return true;
973 }
974
975 bool
976 nir_foreach_dest(nir_instr *instr, nir_foreach_dest_cb cb, void *state)
977 {
978 switch (instr->type) {
979 case nir_instr_type_alu:
980 return visit_alu_dest(nir_instr_as_alu(instr), cb, state);
981 case nir_instr_type_deref:
982 return visit_deref_dest(nir_instr_as_deref(instr), cb, state);
983 case nir_instr_type_intrinsic:
984 return visit_intrinsic_dest(nir_instr_as_intrinsic(instr), cb, state);
985 case nir_instr_type_tex:
986 return visit_texture_dest(nir_instr_as_tex(instr), cb, state);
987 case nir_instr_type_phi:
988 return visit_phi_dest(nir_instr_as_phi(instr), cb, state);
989 case nir_instr_type_parallel_copy:
990 return visit_parallel_copy_dest(nir_instr_as_parallel_copy(instr),
991 cb, state);
992
993 case nir_instr_type_load_const:
994 case nir_instr_type_ssa_undef:
995 case nir_instr_type_call:
996 case nir_instr_type_jump:
997 break;
998
999 default:
1000 unreachable("Invalid instruction type");
1001 break;
1002 }
1003
1004 return true;
1005 }
1006
1007 struct foreach_ssa_def_state {
1008 nir_foreach_ssa_def_cb cb;
1009 void *client_state;
1010 };
1011
1012 static inline bool
1013 nir_ssa_def_visitor(nir_dest *dest, void *void_state)
1014 {
1015 struct foreach_ssa_def_state *state = void_state;
1016
1017 if (dest->is_ssa)
1018 return state->cb(&dest->ssa, state->client_state);
1019 else
1020 return true;
1021 }
1022
1023 bool
1024 nir_foreach_ssa_def(nir_instr *instr, nir_foreach_ssa_def_cb cb, void *state)
1025 {
1026 switch (instr->type) {
1027 case nir_instr_type_alu:
1028 case nir_instr_type_deref:
1029 case nir_instr_type_tex:
1030 case nir_instr_type_intrinsic:
1031 case nir_instr_type_phi:
1032 case nir_instr_type_parallel_copy: {
1033 struct foreach_ssa_def_state foreach_state = {cb, state};
1034 return nir_foreach_dest(instr, nir_ssa_def_visitor, &foreach_state);
1035 }
1036
1037 case nir_instr_type_load_const:
1038 return cb(&nir_instr_as_load_const(instr)->def, state);
1039 case nir_instr_type_ssa_undef:
1040 return cb(&nir_instr_as_ssa_undef(instr)->def, state);
1041 case nir_instr_type_call:
1042 case nir_instr_type_jump:
1043 return true;
1044 default:
1045 unreachable("Invalid instruction type");
1046 }
1047 }
1048
1049 nir_ssa_def *
1050 nir_instr_ssa_def(nir_instr *instr)
1051 {
1052 switch (instr->type) {
1053 case nir_instr_type_alu:
1054 assert(nir_instr_as_alu(instr)->dest.dest.is_ssa);
1055 return &nir_instr_as_alu(instr)->dest.dest.ssa;
1056
1057 case nir_instr_type_deref:
1058 assert(nir_instr_as_deref(instr)->dest.is_ssa);
1059 return &nir_instr_as_deref(instr)->dest.ssa;
1060
1061 case nir_instr_type_tex:
1062 assert(nir_instr_as_tex(instr)->dest.is_ssa);
1063 return &nir_instr_as_tex(instr)->dest.ssa;
1064
1065 case nir_instr_type_intrinsic: {
1066 nir_intrinsic_instr *intrin = nir_instr_as_intrinsic(instr);
1067 if (nir_intrinsic_infos[intrin->intrinsic].has_dest) {
1068 assert(intrin->dest.is_ssa);
1069 return &intrin->dest.ssa;
1070 } else {
1071 return NULL;
1072 }
1073 }
1074
1075 case nir_instr_type_phi:
1076 assert(nir_instr_as_phi(instr)->dest.is_ssa);
1077 return &nir_instr_as_phi(instr)->dest.ssa;
1078
1079 case nir_instr_type_parallel_copy:
1080 unreachable("Parallel copies are unsupported by this function");
1081
1082 case nir_instr_type_load_const:
1083 return &nir_instr_as_load_const(instr)->def;
1084
1085 case nir_instr_type_ssa_undef:
1086 return &nir_instr_as_ssa_undef(instr)->def;
1087
1088 case nir_instr_type_call:
1089 case nir_instr_type_jump:
1090 return NULL;
1091 }
1092
1093 unreachable("Invalid instruction type");
1094 }
1095
1096 static bool
1097 visit_src(nir_src *src, nir_foreach_src_cb cb, void *state)
1098 {
1099 if (!cb(src, state))
1100 return false;
1101 if (!src->is_ssa && src->reg.indirect)
1102 return cb(src->reg.indirect, state);
1103 return true;
1104 }
1105
1106 static bool
1107 visit_alu_src(nir_alu_instr *instr, nir_foreach_src_cb cb, void *state)
1108 {
1109 for (unsigned i = 0; i < nir_op_infos[instr->op].num_inputs; i++)
1110 if (!visit_src(&instr->src[i].src, cb, state))
1111 return false;
1112
1113 return true;
1114 }
1115
1116 static bool
1117 visit_deref_instr_src(nir_deref_instr *instr,
1118 nir_foreach_src_cb cb, void *state)
1119 {
1120 if (instr->deref_type != nir_deref_type_var) {
1121 if (!visit_src(&instr->parent, cb, state))
1122 return false;
1123 }
1124
1125 if (instr->deref_type == nir_deref_type_array ||
1126 instr->deref_type == nir_deref_type_ptr_as_array) {
1127 if (!visit_src(&instr->arr.index, cb, state))
1128 return false;
1129 }
1130
1131 return true;
1132 }
1133
1134 static bool
1135 visit_tex_src(nir_tex_instr *instr, nir_foreach_src_cb cb, void *state)
1136 {
1137 for (unsigned i = 0; i < instr->num_srcs; i++) {
1138 if (!visit_src(&instr->src[i].src, cb, state))
1139 return false;
1140 }
1141
1142 return true;
1143 }
1144
1145 static bool
1146 visit_intrinsic_src(nir_intrinsic_instr *instr, nir_foreach_src_cb cb,
1147 void *state)
1148 {
1149 unsigned num_srcs = nir_intrinsic_infos[instr->intrinsic].num_srcs;
1150 for (unsigned i = 0; i < num_srcs; i++) {
1151 if (!visit_src(&instr->src[i], cb, state))
1152 return false;
1153 }
1154
1155 return true;
1156 }
1157
1158 static bool
1159 visit_call_src(nir_call_instr *instr, nir_foreach_src_cb cb, void *state)
1160 {
1161 for (unsigned i = 0; i < instr->num_params; i++) {
1162 if (!visit_src(&instr->params[i], cb, state))
1163 return false;
1164 }
1165
1166 return true;
1167 }
1168
1169 static bool
1170 visit_phi_src(nir_phi_instr *instr, nir_foreach_src_cb cb, void *state)
1171 {
1172 nir_foreach_phi_src(src, instr) {
1173 if (!visit_src(&src->src, cb, state))
1174 return false;
1175 }
1176
1177 return true;
1178 }
1179
1180 static bool
1181 visit_parallel_copy_src(nir_parallel_copy_instr *instr,
1182 nir_foreach_src_cb cb, void *state)
1183 {
1184 nir_foreach_parallel_copy_entry(entry, instr) {
1185 if (!visit_src(&entry->src, cb, state))
1186 return false;
1187 }
1188
1189 return true;
1190 }
1191
1192 typedef struct {
1193 void *state;
1194 nir_foreach_src_cb cb;
1195 } visit_dest_indirect_state;
1196
1197 static bool
1198 visit_dest_indirect(nir_dest *dest, void *_state)
1199 {
1200 visit_dest_indirect_state *state = (visit_dest_indirect_state *) _state;
1201
1202 if (!dest->is_ssa && dest->reg.indirect)
1203 return state->cb(dest->reg.indirect, state->state);
1204
1205 return true;
1206 }
1207
1208 bool
1209 nir_foreach_src(nir_instr *instr, nir_foreach_src_cb cb, void *state)
1210 {
1211 switch (instr->type) {
1212 case nir_instr_type_alu:
1213 if (!visit_alu_src(nir_instr_as_alu(instr), cb, state))
1214 return false;
1215 break;
1216 case nir_instr_type_deref:
1217 if (!visit_deref_instr_src(nir_instr_as_deref(instr), cb, state))
1218 return false;
1219 break;
1220 case nir_instr_type_intrinsic:
1221 if (!visit_intrinsic_src(nir_instr_as_intrinsic(instr), cb, state))
1222 return false;
1223 break;
1224 case nir_instr_type_tex:
1225 if (!visit_tex_src(nir_instr_as_tex(instr), cb, state))
1226 return false;
1227 break;
1228 case nir_instr_type_call:
1229 if (!visit_call_src(nir_instr_as_call(instr), cb, state))
1230 return false;
1231 break;
1232 case nir_instr_type_load_const:
1233 /* Constant load instructions have no regular sources */
1234 break;
1235 case nir_instr_type_phi:
1236 if (!visit_phi_src(nir_instr_as_phi(instr), cb, state))
1237 return false;
1238 break;
1239 case nir_instr_type_parallel_copy:
1240 if (!visit_parallel_copy_src(nir_instr_as_parallel_copy(instr),
1241 cb, state))
1242 return false;
1243 break;
1244 case nir_instr_type_jump:
1245 case nir_instr_type_ssa_undef:
1246 return true;
1247
1248 default:
1249 unreachable("Invalid instruction type");
1250 break;
1251 }
1252
1253 visit_dest_indirect_state dest_state;
1254 dest_state.state = state;
1255 dest_state.cb = cb;
1256 return nir_foreach_dest(instr, visit_dest_indirect, &dest_state);
1257 }
1258
1259 nir_const_value
1260 nir_const_value_for_float(double f, unsigned bit_size)
1261 {
1262 nir_const_value v;
1263 memset(&v, 0, sizeof(v));
1264
1265 switch (bit_size) {
1266 case 16:
1267 v.u16 = _mesa_float_to_half(f);
1268 break;
1269 case 32:
1270 v.f32 = f;
1271 break;
1272 case 64:
1273 v.f64 = f;
1274 break;
1275 default:
1276 unreachable("Invalid bit size");
1277 }
1278
1279 return v;
1280 }
1281
1282 double
1283 nir_const_value_as_float(nir_const_value value, unsigned bit_size)
1284 {
1285 switch (bit_size) {
1286 case 16: return _mesa_half_to_float(value.u16);
1287 case 32: return value.f32;
1288 case 64: return value.f64;
1289 default:
1290 unreachable("Invalid bit size");
1291 }
1292 }
1293
1294 nir_const_value *
1295 nir_src_as_const_value(nir_src src)
1296 {
1297 if (!src.is_ssa)
1298 return NULL;
1299
1300 if (src.ssa->parent_instr->type != nir_instr_type_load_const)
1301 return NULL;
1302
1303 nir_load_const_instr *load = nir_instr_as_load_const(src.ssa->parent_instr);
1304
1305 return load->value;
1306 }
1307
1308 /**
1309 * Returns true if the source is known to be dynamically uniform. Otherwise it
1310 * returns false which means it may or may not be dynamically uniform but it
1311 * can't be determined.
1312 */
1313 bool
1314 nir_src_is_dynamically_uniform(nir_src src)
1315 {
1316 if (!src.is_ssa)
1317 return false;
1318
1319 /* Constants are trivially dynamically uniform */
1320 if (src.ssa->parent_instr->type == nir_instr_type_load_const)
1321 return true;
1322
1323 /* As are uniform variables */
1324 if (src.ssa->parent_instr->type == nir_instr_type_intrinsic) {
1325 nir_intrinsic_instr *intr = nir_instr_as_intrinsic(src.ssa->parent_instr);
1326
1327 if (intr->intrinsic == nir_intrinsic_load_uniform)
1328 return true;
1329 }
1330
1331 /* Operating together dynamically uniform expressions produces a
1332 * dynamically uniform result
1333 */
1334 if (src.ssa->parent_instr->type == nir_instr_type_alu) {
1335 nir_alu_instr *alu = nir_instr_as_alu(src.ssa->parent_instr);
1336 for (int i = 0; i < nir_op_infos[alu->op].num_inputs; i++) {
1337 if (!nir_src_is_dynamically_uniform(alu->src[i].src))
1338 return false;
1339 }
1340
1341 return true;
1342 }
1343
1344 /* XXX: this could have many more tests, such as when a sampler function is
1345 * called with dynamically uniform arguments.
1346 */
1347 return false;
1348 }
1349
1350 static void
1351 src_remove_all_uses(nir_src *src)
1352 {
1353 for (; src; src = src->is_ssa ? NULL : src->reg.indirect) {
1354 if (!src_is_valid(src))
1355 continue;
1356
1357 list_del(&src->use_link);
1358 }
1359 }
1360
1361 static void
1362 src_add_all_uses(nir_src *src, nir_instr *parent_instr, nir_if *parent_if)
1363 {
1364 for (; src; src = src->is_ssa ? NULL : src->reg.indirect) {
1365 if (!src_is_valid(src))
1366 continue;
1367
1368 if (parent_instr) {
1369 src->parent_instr = parent_instr;
1370 if (src->is_ssa)
1371 list_addtail(&src->use_link, &src->ssa->uses);
1372 else
1373 list_addtail(&src->use_link, &src->reg.reg->uses);
1374 } else {
1375 assert(parent_if);
1376 src->parent_if = parent_if;
1377 if (src->is_ssa)
1378 list_addtail(&src->use_link, &src->ssa->if_uses);
1379 else
1380 list_addtail(&src->use_link, &src->reg.reg->if_uses);
1381 }
1382 }
1383 }
1384
1385 void
1386 nir_instr_rewrite_src(nir_instr *instr, nir_src *src, nir_src new_src)
1387 {
1388 assert(!src_is_valid(src) || src->parent_instr == instr);
1389
1390 src_remove_all_uses(src);
1391 *src = new_src;
1392 src_add_all_uses(src, instr, NULL);
1393 }
1394
1395 void
1396 nir_instr_move_src(nir_instr *dest_instr, nir_src *dest, nir_src *src)
1397 {
1398 assert(!src_is_valid(dest) || dest->parent_instr == dest_instr);
1399
1400 src_remove_all_uses(dest);
1401 src_remove_all_uses(src);
1402 *dest = *src;
1403 *src = NIR_SRC_INIT;
1404 src_add_all_uses(dest, dest_instr, NULL);
1405 }
1406
1407 void
1408 nir_if_rewrite_condition(nir_if *if_stmt, nir_src new_src)
1409 {
1410 nir_src *src = &if_stmt->condition;
1411 assert(!src_is_valid(src) || src->parent_if == if_stmt);
1412
1413 src_remove_all_uses(src);
1414 *src = new_src;
1415 src_add_all_uses(src, NULL, if_stmt);
1416 }
1417
1418 void
1419 nir_instr_rewrite_dest(nir_instr *instr, nir_dest *dest, nir_dest new_dest)
1420 {
1421 if (dest->is_ssa) {
1422 /* We can only overwrite an SSA destination if it has no uses. */
1423 assert(list_is_empty(&dest->ssa.uses) && list_is_empty(&dest->ssa.if_uses));
1424 } else {
1425 list_del(&dest->reg.def_link);
1426 if (dest->reg.indirect)
1427 src_remove_all_uses(dest->reg.indirect);
1428 }
1429
1430 /* We can't re-write with an SSA def */
1431 assert(!new_dest.is_ssa);
1432
1433 nir_dest_copy(dest, &new_dest, instr);
1434
1435 dest->reg.parent_instr = instr;
1436 list_addtail(&dest->reg.def_link, &new_dest.reg.reg->defs);
1437
1438 if (dest->reg.indirect)
1439 src_add_all_uses(dest->reg.indirect, instr, NULL);
1440 }
1441
1442 /* note: does *not* take ownership of 'name' */
1443 void
1444 nir_ssa_def_init(nir_instr *instr, nir_ssa_def *def,
1445 unsigned num_components,
1446 unsigned bit_size, const char *name)
1447 {
1448 def->name = ralloc_strdup(instr, name);
1449 def->parent_instr = instr;
1450 list_inithead(&def->uses);
1451 list_inithead(&def->if_uses);
1452 def->num_components = num_components;
1453 def->bit_size = bit_size;
1454
1455 if (instr->block) {
1456 nir_function_impl *impl =
1457 nir_cf_node_get_function(&instr->block->cf_node);
1458
1459 def->index = impl->ssa_alloc++;
1460 } else {
1461 def->index = UINT_MAX;
1462 }
1463 }
1464
1465 /* note: does *not* take ownership of 'name' */
1466 void
1467 nir_ssa_dest_init(nir_instr *instr, nir_dest *dest,
1468 unsigned num_components, unsigned bit_size,
1469 const char *name)
1470 {
1471 dest->is_ssa = true;
1472 nir_ssa_def_init(instr, &dest->ssa, num_components, bit_size, name);
1473 }
1474
1475 void
1476 nir_ssa_def_rewrite_uses(nir_ssa_def *def, nir_src new_src)
1477 {
1478 assert(!new_src.is_ssa || def != new_src.ssa);
1479
1480 nir_foreach_use_safe(use_src, def)
1481 nir_instr_rewrite_src(use_src->parent_instr, use_src, new_src);
1482
1483 nir_foreach_if_use_safe(use_src, def)
1484 nir_if_rewrite_condition(use_src->parent_if, new_src);
1485 }
1486
1487 static bool
1488 is_instr_between(nir_instr *start, nir_instr *end, nir_instr *between)
1489 {
1490 assert(start->block == end->block);
1491
1492 if (between->block != start->block)
1493 return false;
1494
1495 /* Search backwards looking for "between" */
1496 while (start != end) {
1497 if (between == end)
1498 return true;
1499
1500 end = nir_instr_prev(end);
1501 assert(end);
1502 }
1503
1504 return false;
1505 }
1506
1507 /* Replaces all uses of the given SSA def with the given source but only if
1508 * the use comes after the after_me instruction. This can be useful if you
1509 * are emitting code to fix up the result of some instruction: you can freely
1510 * use the result in that code and then call rewrite_uses_after and pass the
1511 * last fixup instruction as after_me and it will replace all of the uses you
1512 * want without touching the fixup code.
1513 *
1514 * This function assumes that after_me is in the same block as
1515 * def->parent_instr and that after_me comes after def->parent_instr.
1516 */
1517 void
1518 nir_ssa_def_rewrite_uses_after(nir_ssa_def *def, nir_src new_src,
1519 nir_instr *after_me)
1520 {
1521 if (new_src.is_ssa && def == new_src.ssa)
1522 return;
1523
1524 nir_foreach_use_safe(use_src, def) {
1525 assert(use_src->parent_instr != def->parent_instr);
1526 /* Since def already dominates all of its uses, the only way a use can
1527 * not be dominated by after_me is if it is between def and after_me in
1528 * the instruction list.
1529 */
1530 if (!is_instr_between(def->parent_instr, after_me, use_src->parent_instr))
1531 nir_instr_rewrite_src(use_src->parent_instr, use_src, new_src);
1532 }
1533
1534 nir_foreach_if_use_safe(use_src, def)
1535 nir_if_rewrite_condition(use_src->parent_if, new_src);
1536 }
1537
1538 nir_component_mask_t
1539 nir_ssa_def_components_read(const nir_ssa_def *def)
1540 {
1541 nir_component_mask_t read_mask = 0;
1542 nir_foreach_use(use, def) {
1543 if (use->parent_instr->type == nir_instr_type_alu) {
1544 nir_alu_instr *alu = nir_instr_as_alu(use->parent_instr);
1545 nir_alu_src *alu_src = exec_node_data(nir_alu_src, use, src);
1546 int src_idx = alu_src - &alu->src[0];
1547 assert(src_idx >= 0 && src_idx < nir_op_infos[alu->op].num_inputs);
1548 read_mask |= nir_alu_instr_src_read_mask(alu, src_idx);
1549 } else {
1550 return (1 << def->num_components) - 1;
1551 }
1552 }
1553
1554 if (!list_is_empty(&def->if_uses))
1555 read_mask |= 1;
1556
1557 return read_mask;
1558 }
1559
1560 nir_block *
1561 nir_block_cf_tree_next(nir_block *block)
1562 {
1563 if (block == NULL) {
1564 /* nir_foreach_block_safe() will call this function on a NULL block
1565 * after the last iteration, but it won't use the result so just return
1566 * NULL here.
1567 */
1568 return NULL;
1569 }
1570
1571 nir_cf_node *cf_next = nir_cf_node_next(&block->cf_node);
1572 if (cf_next)
1573 return nir_cf_node_cf_tree_first(cf_next);
1574
1575 nir_cf_node *parent = block->cf_node.parent;
1576
1577 switch (parent->type) {
1578 case nir_cf_node_if: {
1579 /* Are we at the end of the if? Go to the beginning of the else */
1580 nir_if *if_stmt = nir_cf_node_as_if(parent);
1581 if (block == nir_if_last_then_block(if_stmt))
1582 return nir_if_first_else_block(if_stmt);
1583
1584 assert(block == nir_if_last_else_block(if_stmt));
1585 /* fall through */
1586 }
1587
1588 case nir_cf_node_loop:
1589 return nir_cf_node_as_block(nir_cf_node_next(parent));
1590
1591 case nir_cf_node_function:
1592 return NULL;
1593
1594 default:
1595 unreachable("unknown cf node type");
1596 }
1597 }
1598
1599 nir_block *
1600 nir_block_cf_tree_prev(nir_block *block)
1601 {
1602 if (block == NULL) {
1603 /* do this for consistency with nir_block_cf_tree_next() */
1604 return NULL;
1605 }
1606
1607 nir_cf_node *cf_prev = nir_cf_node_prev(&block->cf_node);
1608 if (cf_prev)
1609 return nir_cf_node_cf_tree_last(cf_prev);
1610
1611 nir_cf_node *parent = block->cf_node.parent;
1612
1613 switch (parent->type) {
1614 case nir_cf_node_if: {
1615 /* Are we at the beginning of the else? Go to the end of the if */
1616 nir_if *if_stmt = nir_cf_node_as_if(parent);
1617 if (block == nir_if_first_else_block(if_stmt))
1618 return nir_if_last_then_block(if_stmt);
1619
1620 assert(block == nir_if_first_then_block(if_stmt));
1621 /* fall through */
1622 }
1623
1624 case nir_cf_node_loop:
1625 return nir_cf_node_as_block(nir_cf_node_prev(parent));
1626
1627 case nir_cf_node_function:
1628 return NULL;
1629
1630 default:
1631 unreachable("unknown cf node type");
1632 }
1633 }
1634
1635 nir_block *nir_cf_node_cf_tree_first(nir_cf_node *node)
1636 {
1637 switch (node->type) {
1638 case nir_cf_node_function: {
1639 nir_function_impl *impl = nir_cf_node_as_function(node);
1640 return nir_start_block(impl);
1641 }
1642
1643 case nir_cf_node_if: {
1644 nir_if *if_stmt = nir_cf_node_as_if(node);
1645 return nir_if_first_then_block(if_stmt);
1646 }
1647
1648 case nir_cf_node_loop: {
1649 nir_loop *loop = nir_cf_node_as_loop(node);
1650 return nir_loop_first_block(loop);
1651 }
1652
1653 case nir_cf_node_block: {
1654 return nir_cf_node_as_block(node);
1655 }
1656
1657 default:
1658 unreachable("unknown node type");
1659 }
1660 }
1661
1662 nir_block *nir_cf_node_cf_tree_last(nir_cf_node *node)
1663 {
1664 switch (node->type) {
1665 case nir_cf_node_function: {
1666 nir_function_impl *impl = nir_cf_node_as_function(node);
1667 return nir_impl_last_block(impl);
1668 }
1669
1670 case nir_cf_node_if: {
1671 nir_if *if_stmt = nir_cf_node_as_if(node);
1672 return nir_if_last_else_block(if_stmt);
1673 }
1674
1675 case nir_cf_node_loop: {
1676 nir_loop *loop = nir_cf_node_as_loop(node);
1677 return nir_loop_last_block(loop);
1678 }
1679
1680 case nir_cf_node_block: {
1681 return nir_cf_node_as_block(node);
1682 }
1683
1684 default:
1685 unreachable("unknown node type");
1686 }
1687 }
1688
1689 nir_block *nir_cf_node_cf_tree_next(nir_cf_node *node)
1690 {
1691 if (node->type == nir_cf_node_block)
1692 return nir_block_cf_tree_next(nir_cf_node_as_block(node));
1693 else if (node->type == nir_cf_node_function)
1694 return NULL;
1695 else
1696 return nir_cf_node_as_block(nir_cf_node_next(node));
1697 }
1698
1699 nir_if *
1700 nir_block_get_following_if(nir_block *block)
1701 {
1702 if (exec_node_is_tail_sentinel(&block->cf_node.node))
1703 return NULL;
1704
1705 if (nir_cf_node_is_last(&block->cf_node))
1706 return NULL;
1707
1708 nir_cf_node *next_node = nir_cf_node_next(&block->cf_node);
1709
1710 if (next_node->type != nir_cf_node_if)
1711 return NULL;
1712
1713 return nir_cf_node_as_if(next_node);
1714 }
1715
1716 nir_loop *
1717 nir_block_get_following_loop(nir_block *block)
1718 {
1719 if (exec_node_is_tail_sentinel(&block->cf_node.node))
1720 return NULL;
1721
1722 if (nir_cf_node_is_last(&block->cf_node))
1723 return NULL;
1724
1725 nir_cf_node *next_node = nir_cf_node_next(&block->cf_node);
1726
1727 if (next_node->type != nir_cf_node_loop)
1728 return NULL;
1729
1730 return nir_cf_node_as_loop(next_node);
1731 }
1732
1733 void
1734 nir_index_blocks(nir_function_impl *impl)
1735 {
1736 unsigned index = 0;
1737
1738 if (impl->valid_metadata & nir_metadata_block_index)
1739 return;
1740
1741 nir_foreach_block(block, impl) {
1742 block->index = index++;
1743 }
1744
1745 /* The end_block isn't really part of the program, which is why its index
1746 * is >= num_blocks.
1747 */
1748 impl->num_blocks = impl->end_block->index = index;
1749 }
1750
1751 static bool
1752 index_ssa_def_cb(nir_ssa_def *def, void *state)
1753 {
1754 unsigned *index = (unsigned *) state;
1755 def->index = (*index)++;
1756
1757 return true;
1758 }
1759
1760 /**
1761 * The indices are applied top-to-bottom which has the very nice property
1762 * that, if A dominates B, then A->index <= B->index.
1763 */
1764 void
1765 nir_index_ssa_defs(nir_function_impl *impl)
1766 {
1767 unsigned index = 0;
1768
1769 nir_foreach_block(block, impl) {
1770 nir_foreach_instr(instr, block)
1771 nir_foreach_ssa_def(instr, index_ssa_def_cb, &index);
1772 }
1773
1774 impl->ssa_alloc = index;
1775 }
1776
1777 /**
1778 * The indices are applied top-to-bottom which has the very nice property
1779 * that, if A dominates B, then A->index <= B->index.
1780 */
1781 unsigned
1782 nir_index_instrs(nir_function_impl *impl)
1783 {
1784 unsigned index = 0;
1785
1786 nir_foreach_block(block, impl) {
1787 nir_foreach_instr(instr, block)
1788 instr->index = index++;
1789 }
1790
1791 return index;
1792 }
1793
1794 static void
1795 index_var_list(struct exec_list *list)
1796 {
1797 unsigned next_index = 0;
1798 nir_foreach_variable(var, list)
1799 var->index = next_index++;
1800 }
1801
1802 void
1803 nir_index_vars(nir_shader *shader, nir_function_impl *impl, nir_variable_mode modes)
1804 {
1805 if ((modes & nir_var_function_temp) && impl)
1806 index_var_list(&impl->locals);
1807
1808 if (modes & nir_var_shader_temp)
1809 index_var_list(&shader->globals);
1810
1811 if (modes & nir_var_shader_in)
1812 index_var_list(&shader->inputs);
1813
1814 if (modes & nir_var_shader_out)
1815 index_var_list(&shader->outputs);
1816
1817 if (modes & (nir_var_uniform | nir_var_mem_ubo | nir_var_mem_ssbo))
1818 index_var_list(&shader->uniforms);
1819
1820 if (modes & nir_var_mem_shared)
1821 index_var_list(&shader->shared);
1822
1823 if (modes & nir_var_system_value)
1824 index_var_list(&shader->system_values);
1825 }
1826
1827 static nir_instr *
1828 cursor_next_instr(nir_cursor cursor)
1829 {
1830 switch (cursor.option) {
1831 case nir_cursor_before_block:
1832 for (nir_block *block = cursor.block; block;
1833 block = nir_block_cf_tree_next(block)) {
1834 nir_instr *instr = nir_block_first_instr(block);
1835 if (instr)
1836 return instr;
1837 }
1838 return NULL;
1839
1840 case nir_cursor_after_block:
1841 cursor.block = nir_block_cf_tree_next(cursor.block);
1842 if (cursor.block == NULL)
1843 return NULL;
1844
1845 cursor.option = nir_cursor_before_block;
1846 return cursor_next_instr(cursor);
1847
1848 case nir_cursor_before_instr:
1849 return cursor.instr;
1850
1851 case nir_cursor_after_instr:
1852 if (nir_instr_next(cursor.instr))
1853 return nir_instr_next(cursor.instr);
1854
1855 cursor.option = nir_cursor_after_block;
1856 cursor.block = cursor.instr->block;
1857 return cursor_next_instr(cursor);
1858 }
1859
1860 unreachable("Inavlid cursor option");
1861 }
1862
1863 ASSERTED static bool
1864 dest_is_ssa(nir_dest *dest, void *_state)
1865 {
1866 (void) _state;
1867 return dest->is_ssa;
1868 }
1869
1870 bool
1871 nir_function_impl_lower_instructions(nir_function_impl *impl,
1872 nir_instr_filter_cb filter,
1873 nir_lower_instr_cb lower,
1874 void *cb_data)
1875 {
1876 nir_builder b;
1877 nir_builder_init(&b, impl);
1878
1879 nir_metadata preserved = nir_metadata_block_index |
1880 nir_metadata_dominance;
1881
1882 bool progress = false;
1883 nir_cursor iter = nir_before_cf_list(&impl->body);
1884 nir_instr *instr;
1885 while ((instr = cursor_next_instr(iter)) != NULL) {
1886 if (filter && !filter(instr, cb_data)) {
1887 iter = nir_after_instr(instr);
1888 continue;
1889 }
1890
1891 assert(nir_foreach_dest(instr, dest_is_ssa, NULL));
1892 nir_ssa_def *old_def = nir_instr_ssa_def(instr);
1893 if (old_def == NULL) {
1894 iter = nir_after_instr(instr);
1895 continue;
1896 }
1897
1898 /* We're about to ask the callback to generate a replacement for instr.
1899 * Save off the uses from instr's SSA def so we know what uses to
1900 * rewrite later. If we use nir_ssa_def_rewrite_uses, it fails in the
1901 * case where the generated replacement code uses the result of instr
1902 * itself. If we use nir_ssa_def_rewrite_uses_after (which is the
1903 * normal solution to this problem), it doesn't work well if control-
1904 * flow is inserted as part of the replacement, doesn't handle cases
1905 * where the replacement is something consumed by instr, and suffers
1906 * from performance issues. This is the only way to 100% guarantee
1907 * that we rewrite the correct set efficiently.
1908 */
1909 struct list_head old_uses, old_if_uses;
1910 list_replace(&old_def->uses, &old_uses);
1911 list_inithead(&old_def->uses);
1912 list_replace(&old_def->if_uses, &old_if_uses);
1913 list_inithead(&old_def->if_uses);
1914
1915 b.cursor = nir_after_instr(instr);
1916 nir_ssa_def *new_def = lower(&b, instr, cb_data);
1917 if (new_def && new_def != NIR_LOWER_INSTR_PROGRESS) {
1918 assert(old_def != NULL);
1919 if (new_def->parent_instr->block != instr->block)
1920 preserved = nir_metadata_none;
1921
1922 nir_src new_src = nir_src_for_ssa(new_def);
1923 list_for_each_entry_safe(nir_src, use_src, &old_uses, use_link)
1924 nir_instr_rewrite_src(use_src->parent_instr, use_src, new_src);
1925
1926 list_for_each_entry_safe(nir_src, use_src, &old_if_uses, use_link)
1927 nir_if_rewrite_condition(use_src->parent_if, new_src);
1928
1929 if (list_is_empty(&old_def->uses) && list_is_empty(&old_def->if_uses)) {
1930 iter = nir_instr_remove(instr);
1931 } else {
1932 iter = nir_after_instr(instr);
1933 }
1934 progress = true;
1935 } else {
1936 /* We didn't end up lowering after all. Put the uses back */
1937 if (old_def) {
1938 list_replace(&old_uses, &old_def->uses);
1939 list_replace(&old_if_uses, &old_def->if_uses);
1940 }
1941 iter = nir_after_instr(instr);
1942
1943 if (new_def == NIR_LOWER_INSTR_PROGRESS)
1944 progress = true;
1945 }
1946 }
1947
1948 if (progress) {
1949 nir_metadata_preserve(impl, preserved);
1950 } else {
1951 #ifndef NDEBUG
1952 impl->valid_metadata &= ~nir_metadata_not_properly_reset;
1953 #endif
1954 }
1955
1956 return progress;
1957 }
1958
1959 bool
1960 nir_shader_lower_instructions(nir_shader *shader,
1961 nir_instr_filter_cb filter,
1962 nir_lower_instr_cb lower,
1963 void *cb_data)
1964 {
1965 bool progress = false;
1966
1967 nir_foreach_function(function, shader) {
1968 if (function->impl &&
1969 nir_function_impl_lower_instructions(function->impl,
1970 filter, lower, cb_data))
1971 progress = true;
1972 }
1973
1974 return progress;
1975 }
1976
1977 nir_intrinsic_op
1978 nir_intrinsic_from_system_value(gl_system_value val)
1979 {
1980 switch (val) {
1981 case SYSTEM_VALUE_VERTEX_ID:
1982 return nir_intrinsic_load_vertex_id;
1983 case SYSTEM_VALUE_INSTANCE_ID:
1984 return nir_intrinsic_load_instance_id;
1985 case SYSTEM_VALUE_DRAW_ID:
1986 return nir_intrinsic_load_draw_id;
1987 case SYSTEM_VALUE_BASE_INSTANCE:
1988 return nir_intrinsic_load_base_instance;
1989 case SYSTEM_VALUE_VERTEX_ID_ZERO_BASE:
1990 return nir_intrinsic_load_vertex_id_zero_base;
1991 case SYSTEM_VALUE_IS_INDEXED_DRAW:
1992 return nir_intrinsic_load_is_indexed_draw;
1993 case SYSTEM_VALUE_FIRST_VERTEX:
1994 return nir_intrinsic_load_first_vertex;
1995 case SYSTEM_VALUE_BASE_VERTEX:
1996 return nir_intrinsic_load_base_vertex;
1997 case SYSTEM_VALUE_INVOCATION_ID:
1998 return nir_intrinsic_load_invocation_id;
1999 case SYSTEM_VALUE_FRAG_COORD:
2000 return nir_intrinsic_load_frag_coord;
2001 case SYSTEM_VALUE_POINT_COORD:
2002 return nir_intrinsic_load_point_coord;
2003 case SYSTEM_VALUE_FRONT_FACE:
2004 return nir_intrinsic_load_front_face;
2005 case SYSTEM_VALUE_SAMPLE_ID:
2006 return nir_intrinsic_load_sample_id;
2007 case SYSTEM_VALUE_SAMPLE_POS:
2008 return nir_intrinsic_load_sample_pos;
2009 case SYSTEM_VALUE_SAMPLE_MASK_IN:
2010 return nir_intrinsic_load_sample_mask_in;
2011 case SYSTEM_VALUE_LOCAL_INVOCATION_ID:
2012 return nir_intrinsic_load_local_invocation_id;
2013 case SYSTEM_VALUE_LOCAL_INVOCATION_INDEX:
2014 return nir_intrinsic_load_local_invocation_index;
2015 case SYSTEM_VALUE_WORK_GROUP_ID:
2016 return nir_intrinsic_load_work_group_id;
2017 case SYSTEM_VALUE_NUM_WORK_GROUPS:
2018 return nir_intrinsic_load_num_work_groups;
2019 case SYSTEM_VALUE_PRIMITIVE_ID:
2020 return nir_intrinsic_load_primitive_id;
2021 case SYSTEM_VALUE_TESS_COORD:
2022 return nir_intrinsic_load_tess_coord;
2023 case SYSTEM_VALUE_TESS_LEVEL_OUTER:
2024 return nir_intrinsic_load_tess_level_outer;
2025 case SYSTEM_VALUE_TESS_LEVEL_INNER:
2026 return nir_intrinsic_load_tess_level_inner;
2027 case SYSTEM_VALUE_TESS_LEVEL_OUTER_DEFAULT:
2028 return nir_intrinsic_load_tess_level_outer_default;
2029 case SYSTEM_VALUE_TESS_LEVEL_INNER_DEFAULT:
2030 return nir_intrinsic_load_tess_level_inner_default;
2031 case SYSTEM_VALUE_VERTICES_IN:
2032 return nir_intrinsic_load_patch_vertices_in;
2033 case SYSTEM_VALUE_HELPER_INVOCATION:
2034 return nir_intrinsic_load_helper_invocation;
2035 case SYSTEM_VALUE_COLOR0:
2036 return nir_intrinsic_load_color0;
2037 case SYSTEM_VALUE_COLOR1:
2038 return nir_intrinsic_load_color1;
2039 case SYSTEM_VALUE_VIEW_INDEX:
2040 return nir_intrinsic_load_view_index;
2041 case SYSTEM_VALUE_SUBGROUP_SIZE:
2042 return nir_intrinsic_load_subgroup_size;
2043 case SYSTEM_VALUE_SUBGROUP_INVOCATION:
2044 return nir_intrinsic_load_subgroup_invocation;
2045 case SYSTEM_VALUE_SUBGROUP_EQ_MASK:
2046 return nir_intrinsic_load_subgroup_eq_mask;
2047 case SYSTEM_VALUE_SUBGROUP_GE_MASK:
2048 return nir_intrinsic_load_subgroup_ge_mask;
2049 case SYSTEM_VALUE_SUBGROUP_GT_MASK:
2050 return nir_intrinsic_load_subgroup_gt_mask;
2051 case SYSTEM_VALUE_SUBGROUP_LE_MASK:
2052 return nir_intrinsic_load_subgroup_le_mask;
2053 case SYSTEM_VALUE_SUBGROUP_LT_MASK:
2054 return nir_intrinsic_load_subgroup_lt_mask;
2055 case SYSTEM_VALUE_NUM_SUBGROUPS:
2056 return nir_intrinsic_load_num_subgroups;
2057 case SYSTEM_VALUE_SUBGROUP_ID:
2058 return nir_intrinsic_load_subgroup_id;
2059 case SYSTEM_VALUE_LOCAL_GROUP_SIZE:
2060 return nir_intrinsic_load_local_group_size;
2061 case SYSTEM_VALUE_GLOBAL_INVOCATION_ID:
2062 return nir_intrinsic_load_global_invocation_id;
2063 case SYSTEM_VALUE_GLOBAL_INVOCATION_INDEX:
2064 return nir_intrinsic_load_global_invocation_index;
2065 case SYSTEM_VALUE_WORK_DIM:
2066 return nir_intrinsic_load_work_dim;
2067 case SYSTEM_VALUE_USER_DATA_AMD:
2068 return nir_intrinsic_load_user_data_amd;
2069 default:
2070 unreachable("system value does not directly correspond to intrinsic");
2071 }
2072 }
2073
2074 gl_system_value
2075 nir_system_value_from_intrinsic(nir_intrinsic_op intrin)
2076 {
2077 switch (intrin) {
2078 case nir_intrinsic_load_vertex_id:
2079 return SYSTEM_VALUE_VERTEX_ID;
2080 case nir_intrinsic_load_instance_id:
2081 return SYSTEM_VALUE_INSTANCE_ID;
2082 case nir_intrinsic_load_draw_id:
2083 return SYSTEM_VALUE_DRAW_ID;
2084 case nir_intrinsic_load_base_instance:
2085 return SYSTEM_VALUE_BASE_INSTANCE;
2086 case nir_intrinsic_load_vertex_id_zero_base:
2087 return SYSTEM_VALUE_VERTEX_ID_ZERO_BASE;
2088 case nir_intrinsic_load_first_vertex:
2089 return SYSTEM_VALUE_FIRST_VERTEX;
2090 case nir_intrinsic_load_is_indexed_draw:
2091 return SYSTEM_VALUE_IS_INDEXED_DRAW;
2092 case nir_intrinsic_load_base_vertex:
2093 return SYSTEM_VALUE_BASE_VERTEX;
2094 case nir_intrinsic_load_invocation_id:
2095 return SYSTEM_VALUE_INVOCATION_ID;
2096 case nir_intrinsic_load_frag_coord:
2097 return SYSTEM_VALUE_FRAG_COORD;
2098 case nir_intrinsic_load_point_coord:
2099 return SYSTEM_VALUE_POINT_COORD;
2100 case nir_intrinsic_load_front_face:
2101 return SYSTEM_VALUE_FRONT_FACE;
2102 case nir_intrinsic_load_sample_id:
2103 return SYSTEM_VALUE_SAMPLE_ID;
2104 case nir_intrinsic_load_sample_pos:
2105 return SYSTEM_VALUE_SAMPLE_POS;
2106 case nir_intrinsic_load_sample_mask_in:
2107 return SYSTEM_VALUE_SAMPLE_MASK_IN;
2108 case nir_intrinsic_load_local_invocation_id:
2109 return SYSTEM_VALUE_LOCAL_INVOCATION_ID;
2110 case nir_intrinsic_load_local_invocation_index:
2111 return SYSTEM_VALUE_LOCAL_INVOCATION_INDEX;
2112 case nir_intrinsic_load_num_work_groups:
2113 return SYSTEM_VALUE_NUM_WORK_GROUPS;
2114 case nir_intrinsic_load_work_group_id:
2115 return SYSTEM_VALUE_WORK_GROUP_ID;
2116 case nir_intrinsic_load_primitive_id:
2117 return SYSTEM_VALUE_PRIMITIVE_ID;
2118 case nir_intrinsic_load_tess_coord:
2119 return SYSTEM_VALUE_TESS_COORD;
2120 case nir_intrinsic_load_tess_level_outer:
2121 return SYSTEM_VALUE_TESS_LEVEL_OUTER;
2122 case nir_intrinsic_load_tess_level_inner:
2123 return SYSTEM_VALUE_TESS_LEVEL_INNER;
2124 case nir_intrinsic_load_tess_level_outer_default:
2125 return SYSTEM_VALUE_TESS_LEVEL_OUTER_DEFAULT;
2126 case nir_intrinsic_load_tess_level_inner_default:
2127 return SYSTEM_VALUE_TESS_LEVEL_INNER_DEFAULT;
2128 case nir_intrinsic_load_patch_vertices_in:
2129 return SYSTEM_VALUE_VERTICES_IN;
2130 case nir_intrinsic_load_helper_invocation:
2131 return SYSTEM_VALUE_HELPER_INVOCATION;
2132 case nir_intrinsic_load_color0:
2133 return SYSTEM_VALUE_COLOR0;
2134 case nir_intrinsic_load_color1:
2135 return SYSTEM_VALUE_COLOR1;
2136 case nir_intrinsic_load_view_index:
2137 return SYSTEM_VALUE_VIEW_INDEX;
2138 case nir_intrinsic_load_subgroup_size:
2139 return SYSTEM_VALUE_SUBGROUP_SIZE;
2140 case nir_intrinsic_load_subgroup_invocation:
2141 return SYSTEM_VALUE_SUBGROUP_INVOCATION;
2142 case nir_intrinsic_load_subgroup_eq_mask:
2143 return SYSTEM_VALUE_SUBGROUP_EQ_MASK;
2144 case nir_intrinsic_load_subgroup_ge_mask:
2145 return SYSTEM_VALUE_SUBGROUP_GE_MASK;
2146 case nir_intrinsic_load_subgroup_gt_mask:
2147 return SYSTEM_VALUE_SUBGROUP_GT_MASK;
2148 case nir_intrinsic_load_subgroup_le_mask:
2149 return SYSTEM_VALUE_SUBGROUP_LE_MASK;
2150 case nir_intrinsic_load_subgroup_lt_mask:
2151 return SYSTEM_VALUE_SUBGROUP_LT_MASK;
2152 case nir_intrinsic_load_num_subgroups:
2153 return SYSTEM_VALUE_NUM_SUBGROUPS;
2154 case nir_intrinsic_load_subgroup_id:
2155 return SYSTEM_VALUE_SUBGROUP_ID;
2156 case nir_intrinsic_load_local_group_size:
2157 return SYSTEM_VALUE_LOCAL_GROUP_SIZE;
2158 case nir_intrinsic_load_global_invocation_id:
2159 return SYSTEM_VALUE_GLOBAL_INVOCATION_ID;
2160 case nir_intrinsic_load_user_data_amd:
2161 return SYSTEM_VALUE_USER_DATA_AMD;
2162 default:
2163 unreachable("intrinsic doesn't produce a system value");
2164 }
2165 }
2166
2167 /* OpenGL utility method that remaps the location attributes if they are
2168 * doubles. Not needed for vulkan due the differences on the input location
2169 * count for doubles on vulkan vs OpenGL
2170 *
2171 * The bitfield returned in dual_slot is one bit for each double input slot in
2172 * the original OpenGL single-slot input numbering. The mapping from old
2173 * locations to new locations is as follows:
2174 *
2175 * new_loc = loc + util_bitcount(dual_slot & BITFIELD64_MASK(loc))
2176 */
2177 void
2178 nir_remap_dual_slot_attributes(nir_shader *shader, uint64_t *dual_slot)
2179 {
2180 assert(shader->info.stage == MESA_SHADER_VERTEX);
2181
2182 *dual_slot = 0;
2183 nir_foreach_variable(var, &shader->inputs) {
2184 if (glsl_type_is_dual_slot(glsl_without_array(var->type))) {
2185 unsigned slots = glsl_count_attribute_slots(var->type, true);
2186 *dual_slot |= BITFIELD64_MASK(slots) << var->data.location;
2187 }
2188 }
2189
2190 nir_foreach_variable(var, &shader->inputs) {
2191 var->data.location +=
2192 util_bitcount64(*dual_slot & BITFIELD64_MASK(var->data.location));
2193 }
2194 }
2195
2196 /* Returns an attribute mask that has been re-compacted using the given
2197 * dual_slot mask.
2198 */
2199 uint64_t
2200 nir_get_single_slot_attribs_mask(uint64_t attribs, uint64_t dual_slot)
2201 {
2202 while (dual_slot) {
2203 unsigned loc = u_bit_scan64(&dual_slot);
2204 /* mask of all bits up to and including loc */
2205 uint64_t mask = BITFIELD64_MASK(loc + 1);
2206 attribs = (attribs & mask) | ((attribs & ~mask) >> 1);
2207 }
2208 return attribs;
2209 }
2210
2211 void
2212 nir_rewrite_image_intrinsic(nir_intrinsic_instr *intrin, nir_ssa_def *src,
2213 bool bindless)
2214 {
2215 enum gl_access_qualifier access = nir_intrinsic_access(intrin);
2216
2217 switch (intrin->intrinsic) {
2218 #define CASE(op) \
2219 case nir_intrinsic_image_deref_##op: \
2220 intrin->intrinsic = bindless ? nir_intrinsic_bindless_image_##op \
2221 : nir_intrinsic_image_##op; \
2222 break;
2223 CASE(load)
2224 CASE(store)
2225 CASE(atomic_add)
2226 CASE(atomic_imin)
2227 CASE(atomic_umin)
2228 CASE(atomic_imax)
2229 CASE(atomic_umax)
2230 CASE(atomic_and)
2231 CASE(atomic_or)
2232 CASE(atomic_xor)
2233 CASE(atomic_exchange)
2234 CASE(atomic_comp_swap)
2235 CASE(atomic_fadd)
2236 CASE(size)
2237 CASE(samples)
2238 CASE(load_raw_intel)
2239 CASE(store_raw_intel)
2240 #undef CASE
2241 default:
2242 unreachable("Unhanded image intrinsic");
2243 }
2244
2245 nir_deref_instr *deref = nir_src_as_deref(intrin->src[0]);
2246 nir_variable *var = nir_deref_instr_get_variable(deref);
2247
2248 nir_intrinsic_set_image_dim(intrin, glsl_get_sampler_dim(deref->type));
2249 nir_intrinsic_set_image_array(intrin, glsl_sampler_type_is_array(deref->type));
2250 nir_intrinsic_set_access(intrin, access | var->data.access);
2251 nir_intrinsic_set_format(intrin, var->data.image.format);
2252
2253 nir_instr_rewrite_src(&intrin->instr, &intrin->src[0],
2254 nir_src_for_ssa(src));
2255 }
2256
2257 unsigned
2258 nir_image_intrinsic_coord_components(const nir_intrinsic_instr *instr)
2259 {
2260 enum glsl_sampler_dim dim = nir_intrinsic_image_dim(instr);
2261 int coords = glsl_get_sampler_dim_coordinate_components(dim);
2262 if (dim == GLSL_SAMPLER_DIM_CUBE)
2263 return coords;
2264 else
2265 return coords + nir_intrinsic_image_array(instr);
2266 }