projects / mesa.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
history | raw | HEAD
nir: Add a new nir_var_mem_constant variable mode
[mesa.git] / src / compiler / nir / nir.c
1 /*
2 * Copyright © 2014 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 *
23 * Authors:
24 * Connor Abbott (cwabbott0@gmail.com)
25 *
26 */
27
28 #include "nir.h"
29 #include "nir_builder.h"
30 #include "nir_control_flow_private.h"
31 #include "util/half_float.h"
32 #include <limits.h>
33 #include <assert.h>
34 #include <math.h>
35 #include "util/u_math.h"
36
37 #include "main/menums.h" /* BITFIELD64_MASK */
38
39 nir_shader *
40 nir_shader_create(void *mem_ctx,
41 gl_shader_stage stage,
42 const nir_shader_compiler_options *options,
43 shader_info *si)
44 {
45 nir_shader *shader = rzalloc(mem_ctx, nir_shader);
46
47 exec_list_make_empty(&shader->variables);
48
49 shader->options = options;
50
51 if (si) {
52 assert(si->stage == stage);
53 shader->info = *si;
54 } else {
55 shader->info.stage = stage;
56 }
57
58 exec_list_make_empty(&shader->functions);
59
60 shader->num_inputs = 0;
61 shader->num_outputs = 0;
62 shader->num_uniforms = 0;
63 shader->shared_size = 0;
64
65 return shader;
66 }
67
68 static nir_register *
69 reg_create(void *mem_ctx, struct exec_list *list)
70 {
71 nir_register *reg = ralloc(mem_ctx, nir_register);
72
73 list_inithead(&reg->uses);
74 list_inithead(&reg->defs);
75 list_inithead(&reg->if_uses);
76
77 reg->num_components = 0;
78 reg->bit_size = 32;
79 reg->num_array_elems = 0;
80 reg->name = NULL;
81
82 exec_list_push_tail(list, &reg->node);
83
84 return reg;
85 }
86
87 nir_register *
88 nir_local_reg_create(nir_function_impl *impl)
89 {
90 nir_register *reg = reg_create(ralloc_parent(impl), &impl->registers);
91 reg->index = impl->reg_alloc++;
92
93 return reg;
94 }
95
96 void
97 nir_reg_remove(nir_register *reg)
98 {
99 exec_node_remove(&reg->node);
100 }
101
102 void
103 nir_shader_add_variable(nir_shader *shader, nir_variable *var)
104 {
105 switch (var->data.mode) {
106 case nir_var_function_temp:
107 assert(!"nir_shader_add_variable cannot be used for local variables");
108 return;
109
110 case nir_var_shader_temp:
111 case nir_var_shader_in:
112 case nir_var_shader_out:
113 case nir_var_uniform:
114 case nir_var_mem_ubo:
115 case nir_var_mem_ssbo:
116 case nir_var_mem_shared:
117 case nir_var_system_value:
118 case nir_var_mem_constant:
119 break;
120
121 case nir_var_mem_global:
122 assert(!"nir_shader_add_variable cannot be used for global memory");
123 return;
124
125 case nir_var_mem_push_const:
126 assert(!"nir_var_push_constant is not supposed to be used for variables");
127 return;
128
129 default:
130 assert(!"invalid mode");
131 return;
132 }
133
134 exec_list_push_tail(&shader->variables, &var->node);
135 }
136
137 nir_variable *
138 nir_variable_create(nir_shader *shader, nir_variable_mode mode,
139 const struct glsl_type *type, const char *name)
140 {
141 nir_variable *var = rzalloc(shader, nir_variable);
142 var->name = ralloc_strdup(var, name);
143 var->type = type;
144 var->data.mode = mode;
145 var->data.how_declared = nir_var_declared_normally;
146
147 if ((mode == nir_var_shader_in &&
148 shader->info.stage != MESA_SHADER_VERTEX &&
149 shader->info.stage != MESA_SHADER_KERNEL) ||
150 (mode == nir_var_shader_out &&
151 shader->info.stage != MESA_SHADER_FRAGMENT))
152 var->data.interpolation = INTERP_MODE_SMOOTH;
153
154 if (mode == nir_var_shader_in || mode == nir_var_uniform)
155 var->data.read_only = true;
156
157 nir_shader_add_variable(shader, var);
158
159 return var;
160 }
161
162 nir_variable *
163 nir_local_variable_create(nir_function_impl *impl,
164 const struct glsl_type *type, const char *name)
165 {
166 nir_variable *var = rzalloc(impl->function->shader, nir_variable);
167 var->name = ralloc_strdup(var, name);
168 var->type = type;
169 var->data.mode = nir_var_function_temp;
170
171 nir_function_impl_add_variable(impl, var);
172
173 return var;
174 }
175
176 nir_variable *
177 nir_find_variable_with_location(nir_shader *shader,
178 nir_variable_mode mode,
179 unsigned location)
180 {
181 assert(util_bitcount(mode) == 1 && mode != nir_var_function_temp);
182 nir_foreach_variable_with_modes(var, shader, mode) {
183 if (var->data.location == location)
184 return var;
185 }
186 return NULL;
187 }
188
189 nir_variable *
190 nir_find_variable_with_driver_location(nir_shader *shader,
191 nir_variable_mode mode,
192 unsigned location)
193 {
194 assert(util_bitcount(mode) == 1 && mode != nir_var_function_temp);
195 nir_foreach_variable_with_modes(var, shader, mode) {
196 if (var->data.driver_location == location)
197 return var;
198 }
199 return NULL;
200 }
201
202 nir_function *
203 nir_function_create(nir_shader *shader, const char *name)
204 {
205 nir_function *func = ralloc(shader, nir_function);
206
207 exec_list_push_tail(&shader->functions, &func->node);
208
209 func->name = ralloc_strdup(func, name);
210 func->shader = shader;
211 func->num_params = 0;
212 func->params = NULL;
213 func->impl = NULL;
214 func->is_entrypoint = false;
215
216 return func;
217 }
218
219 /* NOTE: if the instruction you are copying a src to is already added
220 * to the IR, use nir_instr_rewrite_src() instead.
221 */
222 void nir_src_copy(nir_src *dest, const nir_src *src, void *mem_ctx)
223 {
224 dest->is_ssa = src->is_ssa;
225 if (src->is_ssa) {
226 dest->ssa = src->ssa;
227 } else {
228 dest->reg.base_offset = src->reg.base_offset;
229 dest->reg.reg = src->reg.reg;
230 if (src->reg.indirect) {
231 dest->reg.indirect = ralloc(mem_ctx, nir_src);
232 nir_src_copy(dest->reg.indirect, src->reg.indirect, mem_ctx);
233 } else {
234 dest->reg.indirect = NULL;
235 }
236 }
237 }
238
239 void nir_dest_copy(nir_dest *dest, const nir_dest *src, nir_instr *instr)
240 {
241 /* Copying an SSA definition makes no sense whatsoever. */
242 assert(!src->is_ssa);
243
244 dest->is_ssa = false;
245
246 dest->reg.base_offset = src->reg.base_offset;
247 dest->reg.reg = src->reg.reg;
248 if (src->reg.indirect) {
249 dest->reg.indirect = ralloc(instr, nir_src);
250 nir_src_copy(dest->reg.indirect, src->reg.indirect, instr);
251 } else {
252 dest->reg.indirect = NULL;
253 }
254 }
255
256 void
257 nir_alu_src_copy(nir_alu_src *dest, const nir_alu_src *src,
258 nir_alu_instr *instr)
259 {
260 nir_src_copy(&dest->src, &src->src, &instr->instr);
261 dest->abs = src->abs;
262 dest->negate = src->negate;
263 for (unsigned i = 0; i < NIR_MAX_VEC_COMPONENTS; i++)
264 dest->swizzle[i] = src->swizzle[i];
265 }
266
267 void
268 nir_alu_dest_copy(nir_alu_dest *dest, const nir_alu_dest *src,
269 nir_alu_instr *instr)
270 {
271 nir_dest_copy(&dest->dest, &src->dest, &instr->instr);
272 dest->write_mask = src->write_mask;
273 dest->saturate = src->saturate;
274 }
275
276
277 static void
278 cf_init(nir_cf_node *node, nir_cf_node_type type)
279 {
280 exec_node_init(&node->node);
281 node->parent = NULL;
282 node->type = type;
283 }
284
285 nir_function_impl *
286 nir_function_impl_create_bare(nir_shader *shader)
287 {
288 nir_function_impl *impl = ralloc(shader, nir_function_impl);
289
290 impl->function = NULL;
291
292 cf_init(&impl->cf_node, nir_cf_node_function);
293
294 exec_list_make_empty(&impl->body);
295 exec_list_make_empty(&impl->registers);
296 exec_list_make_empty(&impl->locals);
297 impl->reg_alloc = 0;
298 impl->ssa_alloc = 0;
299 impl->valid_metadata = nir_metadata_none;
300 impl->structured = true;
301
302 /* create start & end blocks */
303 nir_block *start_block = nir_block_create(shader);
304 nir_block *end_block = nir_block_create(shader);
305 start_block->cf_node.parent = &impl->cf_node;
306 end_block->cf_node.parent = &impl->cf_node;
307 impl->end_block = end_block;
308
309 exec_list_push_tail(&impl->body, &start_block->cf_node.node);
310
311 start_block->successors[0] = end_block;
312 _mesa_set_add(end_block->predecessors, start_block);
313 return impl;
314 }
315
316 nir_function_impl *
317 nir_function_impl_create(nir_function *function)
318 {
319 assert(function->impl == NULL);
320
321 nir_function_impl *impl = nir_function_impl_create_bare(function->shader);
322
323 function->impl = impl;
324 impl->function = function;
325
326 return impl;
327 }
328
329 nir_block *
330 nir_block_create(nir_shader *shader)
331 {
332 nir_block *block = rzalloc(shader, nir_block);
333
334 cf_init(&block->cf_node, nir_cf_node_block);
335
336 block->successors[0] = block->successors[1] = NULL;
337 block->predecessors = _mesa_pointer_set_create(block);
338 block->imm_dom = NULL;
339 /* XXX maybe it would be worth it to defer allocation? This
340 * way it doesn't get allocated for shader refs that never run
341 * nir_calc_dominance? For example, state-tracker creates an
342 * initial IR, clones that, runs appropriate lowering pass, passes
343 * to driver which does common lowering/opt, and then stores ref
344 * which is later used to do state specific lowering and futher
345 * opt. Do any of the references not need dominance metadata?
346 */
347 block->dom_frontier = _mesa_pointer_set_create(block);
348
349 exec_list_make_empty(&block->instr_list);
350
351 return block;
352 }
353
354 static inline void
355 src_init(nir_src *src)
356 {
357 src->is_ssa = false;
358 src->reg.reg = NULL;
359 src->reg.indirect = NULL;
360 src->reg.base_offset = 0;
361 }
362
363 nir_if *
364 nir_if_create(nir_shader *shader)
365 {
366 nir_if *if_stmt = ralloc(shader, nir_if);
367
368 if_stmt->control = nir_selection_control_none;
369
370 cf_init(&if_stmt->cf_node, nir_cf_node_if);
371 src_init(&if_stmt->condition);
372
373 nir_block *then = nir_block_create(shader);
374 exec_list_make_empty(&if_stmt->then_list);
375 exec_list_push_tail(&if_stmt->then_list, &then->cf_node.node);
376 then->cf_node.parent = &if_stmt->cf_node;
377
378 nir_block *else_stmt = nir_block_create(shader);
379 exec_list_make_empty(&if_stmt->else_list);
380 exec_list_push_tail(&if_stmt->else_list, &else_stmt->cf_node.node);
381 else_stmt->cf_node.parent = &if_stmt->cf_node;
382
383 return if_stmt;
384 }
385
386 nir_loop *
387 nir_loop_create(nir_shader *shader)
388 {
389 nir_loop *loop = rzalloc(shader, nir_loop);
390
391 cf_init(&loop->cf_node, nir_cf_node_loop);
392
393 nir_block *body = nir_block_create(shader);
394 exec_list_make_empty(&loop->body);
395 exec_list_push_tail(&loop->body, &body->cf_node.node);
396 body->cf_node.parent = &loop->cf_node;
397
398 body->successors[0] = body;
399 _mesa_set_add(body->predecessors, body);
400
401 return loop;
402 }
403
404 static void
405 instr_init(nir_instr *instr, nir_instr_type type)
406 {
407 instr->type = type;
408 instr->block = NULL;
409 exec_node_init(&instr->node);
410 }
411
412 static void
413 dest_init(nir_dest *dest)
414 {
415 dest->is_ssa = false;
416 dest->reg.reg = NULL;
417 dest->reg.indirect = NULL;
418 dest->reg.base_offset = 0;
419 }
420
421 static void
422 alu_dest_init(nir_alu_dest *dest)
423 {
424 dest_init(&dest->dest);
425 dest->saturate = false;
426 dest->write_mask = 0xf;
427 }
428
429 static void
430 alu_src_init(nir_alu_src *src)
431 {
432 src_init(&src->src);
433 src->abs = src->negate = false;
434 for (int i = 0; i < NIR_MAX_VEC_COMPONENTS; ++i)
435 src->swizzle[i] = i;
436 }
437
438 nir_alu_instr *
439 nir_alu_instr_create(nir_shader *shader, nir_op op)
440 {
441 unsigned num_srcs = nir_op_infos[op].num_inputs;
442 /* TODO: don't use rzalloc */
443 nir_alu_instr *instr =
444 rzalloc_size(shader,
445 sizeof(nir_alu_instr) + num_srcs * sizeof(nir_alu_src));
446
447 instr_init(&instr->instr, nir_instr_type_alu);
448 instr->op = op;
449 alu_dest_init(&instr->dest);
450 for (unsigned i = 0; i < num_srcs; i++)
451 alu_src_init(&instr->src[i]);
452
453 return instr;
454 }
455
456 nir_deref_instr *
457 nir_deref_instr_create(nir_shader *shader, nir_deref_type deref_type)
458 {
459 nir_deref_instr *instr =
460 rzalloc_size(shader, sizeof(nir_deref_instr));
461
462 instr_init(&instr->instr, nir_instr_type_deref);
463
464 instr->deref_type = deref_type;
465 if (deref_type != nir_deref_type_var)
466 src_init(&instr->parent);
467
468 if (deref_type == nir_deref_type_array ||
469 deref_type == nir_deref_type_ptr_as_array)
470 src_init(&instr->arr.index);
471
472 dest_init(&instr->dest);
473
474 return instr;
475 }
476
477 nir_jump_instr *
478 nir_jump_instr_create(nir_shader *shader, nir_jump_type type)
479 {
480 nir_jump_instr *instr = ralloc(shader, nir_jump_instr);
481 instr_init(&instr->instr, nir_instr_type_jump);
482 src_init(&instr->condition);
483 instr->type = type;
484 instr->target = NULL;
485 instr->else_target = NULL;
486 return instr;
487 }
488
489 nir_load_const_instr *
490 nir_load_const_instr_create(nir_shader *shader, unsigned num_components,
491 unsigned bit_size)
492 {
493 nir_load_const_instr *instr =
494 rzalloc_size(shader, sizeof(*instr) + num_components * sizeof(*instr->value));
495 instr_init(&instr->instr, nir_instr_type_load_const);
496
497 nir_ssa_def_init(&instr->instr, &instr->def, num_components, bit_size, NULL);
498
499 return instr;
500 }
501
502 nir_intrinsic_instr *
503 nir_intrinsic_instr_create(nir_shader *shader, nir_intrinsic_op op)
504 {
505 unsigned num_srcs = nir_intrinsic_infos[op].num_srcs;
506 /* TODO: don't use rzalloc */
507 nir_intrinsic_instr *instr =
508 rzalloc_size(shader,
509 sizeof(nir_intrinsic_instr) + num_srcs * sizeof(nir_src));
510
511 instr_init(&instr->instr, nir_instr_type_intrinsic);
512 instr->intrinsic = op;
513
514 if (nir_intrinsic_infos[op].has_dest)
515 dest_init(&instr->dest);
516
517 for (unsigned i = 0; i < num_srcs; i++)
518 src_init(&instr->src[i]);
519
520 return instr;
521 }
522
523 nir_call_instr *
524 nir_call_instr_create(nir_shader *shader, nir_function *callee)
525 {
526 const unsigned num_params = callee->num_params;
527 nir_call_instr *instr =
528 rzalloc_size(shader, sizeof(*instr) +
529 num_params * sizeof(instr->params[0]));
530
531 instr_init(&instr->instr, nir_instr_type_call);
532 instr->callee = callee;
533 instr->num_params = num_params;
534 for (unsigned i = 0; i < num_params; i++)
535 src_init(&instr->params[i]);
536
537 return instr;
538 }
539
540 static int8_t default_tg4_offsets[4][2] =
541 {
542 { 0, 1 },
543 { 1, 1 },
544 { 1, 0 },
545 { 0, 0 },
546 };
547
548 nir_tex_instr *
549 nir_tex_instr_create(nir_shader *shader, unsigned num_srcs)
550 {
551 nir_tex_instr *instr = rzalloc(shader, nir_tex_instr);
552 instr_init(&instr->instr, nir_instr_type_tex);
553
554 dest_init(&instr->dest);
555
556 instr->num_srcs = num_srcs;
557 instr->src = ralloc_array(instr, nir_tex_src, num_srcs);
558 for (unsigned i = 0; i < num_srcs; i++)
559 src_init(&instr->src[i].src);
560
561 instr->texture_index = 0;
562 instr->sampler_index = 0;
563 memcpy(instr->tg4_offsets, default_tg4_offsets, sizeof(instr->tg4_offsets));
564
565 return instr;
566 }
567
568 void
569 nir_tex_instr_add_src(nir_tex_instr *tex,
570 nir_tex_src_type src_type,
571 nir_src src)
572 {
573 nir_tex_src *new_srcs = rzalloc_array(tex, nir_tex_src,
574 tex->num_srcs + 1);
575
576 for (unsigned i = 0; i < tex->num_srcs; i++) {
577 new_srcs[i].src_type = tex->src[i].src_type;
578 nir_instr_move_src(&tex->instr, &new_srcs[i].src,
579 &tex->src[i].src);
580 }
581
582 ralloc_free(tex->src);
583 tex->src = new_srcs;
584
585 tex->src[tex->num_srcs].src_type = src_type;
586 nir_instr_rewrite_src(&tex->instr, &tex->src[tex->num_srcs].src, src);
587 tex->num_srcs++;
588 }
589
590 void
591 nir_tex_instr_remove_src(nir_tex_instr *tex, unsigned src_idx)
592 {
593 assert(src_idx < tex->num_srcs);
594
595 /* First rewrite the source to NIR_SRC_INIT */
596 nir_instr_rewrite_src(&tex->instr, &tex->src[src_idx].src, NIR_SRC_INIT);
597
598 /* Now, move all of the other sources down */
599 for (unsigned i = src_idx + 1; i < tex->num_srcs; i++) {
600 tex->src[i-1].src_type = tex->src[i].src_type;
601 nir_instr_move_src(&tex->instr, &tex->src[i-1].src, &tex->src[i].src);
602 }
603 tex->num_srcs--;
604 }
605
606 bool
607 nir_tex_instr_has_explicit_tg4_offsets(nir_tex_instr *tex)
608 {
609 if (tex->op != nir_texop_tg4)
610 return false;
611 return memcmp(tex->tg4_offsets, default_tg4_offsets,
612 sizeof(tex->tg4_offsets)) != 0;
613 }
614
615 nir_phi_instr *
616 nir_phi_instr_create(nir_shader *shader)
617 {
618 nir_phi_instr *instr = ralloc(shader, nir_phi_instr);
619 instr_init(&instr->instr, nir_instr_type_phi);
620
621 dest_init(&instr->dest);
622 exec_list_make_empty(&instr->srcs);
623 return instr;
624 }
625
626 nir_parallel_copy_instr *
627 nir_parallel_copy_instr_create(nir_shader *shader)
628 {
629 nir_parallel_copy_instr *instr = ralloc(shader, nir_parallel_copy_instr);
630 instr_init(&instr->instr, nir_instr_type_parallel_copy);
631
632 exec_list_make_empty(&instr->entries);
633
634 return instr;
635 }
636
637 nir_ssa_undef_instr *
638 nir_ssa_undef_instr_create(nir_shader *shader,
639 unsigned num_components,
640 unsigned bit_size)
641 {
642 nir_ssa_undef_instr *instr = ralloc(shader, nir_ssa_undef_instr);
643 instr_init(&instr->instr, nir_instr_type_ssa_undef);
644
645 nir_ssa_def_init(&instr->instr, &instr->def, num_components, bit_size, NULL);
646
647 return instr;
648 }
649
650 static nir_const_value
651 const_value_float(double d, unsigned bit_size)
652 {
653 nir_const_value v;
654 memset(&v, 0, sizeof(v));
655 switch (bit_size) {
656 case 16: v.u16 = _mesa_float_to_half(d); break;
657 case 32: v.f32 = d; break;
658 case 64: v.f64 = d; break;
659 default:
660 unreachable("Invalid bit size");
661 }
662 return v;
663 }
664
665 static nir_const_value
666 const_value_int(int64_t i, unsigned bit_size)
667 {
668 nir_const_value v;
669 memset(&v, 0, sizeof(v));
670 switch (bit_size) {
671 case 1: v.b = i & 1; break;
672 case 8: v.i8 = i; break;
673 case 16: v.i16 = i; break;
674 case 32: v.i32 = i; break;
675 case 64: v.i64 = i; break;
676 default:
677 unreachable("Invalid bit size");
678 }
679 return v;
680 }
681
682 nir_const_value
683 nir_alu_binop_identity(nir_op binop, unsigned bit_size)
684 {
685 const int64_t max_int = (1ull << (bit_size - 1)) - 1;
686 const int64_t min_int = -max_int - 1;
687 switch (binop) {
688 case nir_op_iadd:
689 return const_value_int(0, bit_size);
690 case nir_op_fadd:
691 return const_value_float(0, bit_size);
692 case nir_op_imul:
693 return const_value_int(1, bit_size);
694 case nir_op_fmul:
695 return const_value_float(1, bit_size);
696 case nir_op_imin:
697 return const_value_int(max_int, bit_size);
698 case nir_op_umin:
699 return const_value_int(~0ull, bit_size);
700 case nir_op_fmin:
701 return const_value_float(INFINITY, bit_size);
702 case nir_op_imax:
703 return const_value_int(min_int, bit_size);
704 case nir_op_umax:
705 return const_value_int(0, bit_size);
706 case nir_op_fmax:
707 return const_value_float(-INFINITY, bit_size);
708 case nir_op_iand:
709 return const_value_int(~0ull, bit_size);
710 case nir_op_ior:
711 return const_value_int(0, bit_size);
712 case nir_op_ixor:
713 return const_value_int(0, bit_size);
714 default:
715 unreachable("Invalid reduction operation");
716 }
717 }
718
719 nir_function_impl *
720 nir_cf_node_get_function(nir_cf_node *node)
721 {
722 while (node->type != nir_cf_node_function) {
723 node = node->parent;
724 }
725
726 return nir_cf_node_as_function(node);
727 }
728
729 /* Reduces a cursor by trying to convert everything to after and trying to
730 * go up to block granularity when possible.
731 */
732 static nir_cursor
733 reduce_cursor(nir_cursor cursor)
734 {
735 switch (cursor.option) {
736 case nir_cursor_before_block:
737 if (exec_list_is_empty(&cursor.block->instr_list)) {
738 /* Empty block. After is as good as before. */
739 cursor.option = nir_cursor_after_block;
740 }
741 return cursor;
742
743 case nir_cursor_after_block:
744 return cursor;
745
746 case nir_cursor_before_instr: {
747 nir_instr *prev_instr = nir_instr_prev(cursor.instr);
748 if (prev_instr) {
749 /* Before this instruction is after the previous */
750 cursor.instr = prev_instr;
751 cursor.option = nir_cursor_after_instr;
752 } else {
753 /* No previous instruction. Switch to before block */
754 cursor.block = cursor.instr->block;
755 cursor.option = nir_cursor_before_block;
756 }
757 return reduce_cursor(cursor);
758 }
759
760 case nir_cursor_after_instr:
761 if (nir_instr_next(cursor.instr) == NULL) {
762 /* This is the last instruction, switch to after block */
763 cursor.option = nir_cursor_after_block;
764 cursor.block = cursor.instr->block;
765 }
766 return cursor;
767
768 default:
769 unreachable("Inavlid cursor option");
770 }
771 }
772
773 bool
774 nir_cursors_equal(nir_cursor a, nir_cursor b)
775 {
776 /* Reduced cursors should be unique */
777 a = reduce_cursor(a);
778 b = reduce_cursor(b);
779
780 return a.block == b.block && a.option == b.option;
781 }
782
783 static bool
784 add_use_cb(nir_src *src, void *state)
785 {
786 nir_instr *instr = state;
787
788 src->parent_instr = instr;
789 list_addtail(&src->use_link,
790 src->is_ssa ? &src->ssa->uses : &src->reg.reg->uses);
791
792 return true;
793 }
794
795 static bool
796 add_ssa_def_cb(nir_ssa_def *def, void *state)
797 {
798 nir_instr *instr = state;
799
800 if (instr->block && def->index == UINT_MAX) {
801 nir_function_impl *impl =
802 nir_cf_node_get_function(&instr->block->cf_node);
803
804 def->index = impl->ssa_alloc++;
805
806 impl->valid_metadata &= ~nir_metadata_live_ssa_defs;
807 }
808
809 return true;
810 }
811
812 static bool
813 add_reg_def_cb(nir_dest *dest, void *state)
814 {
815 nir_instr *instr = state;
816
817 if (!dest->is_ssa) {
818 dest->reg.parent_instr = instr;
819 list_addtail(&dest->reg.def_link, &dest->reg.reg->defs);
820 }
821
822 return true;
823 }
824
825 static void
826 add_defs_uses(nir_instr *instr)
827 {
828 nir_foreach_src(instr, add_use_cb, instr);
829 nir_foreach_dest(instr, add_reg_def_cb, instr);
830 nir_foreach_ssa_def(instr, add_ssa_def_cb, instr);
831 }
832
833 void
834 nir_instr_insert(nir_cursor cursor, nir_instr *instr)
835 {
836 switch (cursor.option) {
837 case nir_cursor_before_block:
838 /* Only allow inserting jumps into empty blocks. */
839 if (instr->type == nir_instr_type_jump)
840 assert(exec_list_is_empty(&cursor.block->instr_list));
841
842 instr->block = cursor.block;
843 add_defs_uses(instr);
844 exec_list_push_head(&cursor.block->instr_list, &instr->node);
845 break;
846 case nir_cursor_after_block: {
847 /* Inserting instructions after a jump is illegal. */
848 nir_instr *last = nir_block_last_instr(cursor.block);
849 assert(last == NULL || last->type != nir_instr_type_jump);
850 (void) last;
851
852 instr->block = cursor.block;
853 add_defs_uses(instr);
854 exec_list_push_tail(&cursor.block->instr_list, &instr->node);
855 break;
856 }
857 case nir_cursor_before_instr:
858 assert(instr->type != nir_instr_type_jump);
859 instr->block = cursor.instr->block;
860 add_defs_uses(instr);
861 exec_node_insert_node_before(&cursor.instr->node, &instr->node);
862 break;
863 case nir_cursor_after_instr:
864 /* Inserting instructions after a jump is illegal. */
865 assert(cursor.instr->type != nir_instr_type_jump);
866
867 /* Only allow inserting jumps at the end of the block. */
868 if (instr->type == nir_instr_type_jump)
869 assert(cursor.instr == nir_block_last_instr(cursor.instr->block));
870
871 instr->block = cursor.instr->block;
872 add_defs_uses(instr);
873 exec_node_insert_after(&cursor.instr->node, &instr->node);
874 break;
875 }
876
877 if (instr->type == nir_instr_type_jump)
878 nir_handle_add_jump(instr->block);
879 }
880
881 static bool
882 src_is_valid(const nir_src *src)
883 {
884 return src->is_ssa ? (src->ssa != NULL) : (src->reg.reg != NULL);
885 }
886
887 static bool
888 remove_use_cb(nir_src *src, void *state)
889 {
890 (void) state;
891
892 if (src_is_valid(src))
893 list_del(&src->use_link);
894
895 return true;
896 }
897
898 static bool
899 remove_def_cb(nir_dest *dest, void *state)
900 {
901 (void) state;
902
903 if (!dest->is_ssa)
904 list_del(&dest->reg.def_link);
905
906 return true;
907 }
908
909 static void
910 remove_defs_uses(nir_instr *instr)
911 {
912 nir_foreach_dest(instr, remove_def_cb, instr);
913 nir_foreach_src(instr, remove_use_cb, instr);
914 }
915
916 void nir_instr_remove_v(nir_instr *instr)
917 {
918 remove_defs_uses(instr);
919 exec_node_remove(&instr->node);
920
921 if (instr->type == nir_instr_type_jump) {
922 nir_jump_instr *jump_instr = nir_instr_as_jump(instr);
923 nir_handle_remove_jump(instr->block, jump_instr->type);
924 }
925 }
926
927 /*@}*/
928
929 void
930 nir_index_local_regs(nir_function_impl *impl)
931 {
932 unsigned index = 0;
933 foreach_list_typed(nir_register, reg, node, &impl->registers) {
934 reg->index = index++;
935 }
936 impl->reg_alloc = index;
937 }
938
939 static bool
940 visit_alu_dest(nir_alu_instr *instr, nir_foreach_dest_cb cb, void *state)
941 {
942 return cb(&instr->dest.dest, state);
943 }
944
945 static bool
946 visit_deref_dest(nir_deref_instr *instr, nir_foreach_dest_cb cb, void *state)
947 {
948 return cb(&instr->dest, state);
949 }
950
951 static bool
952 visit_intrinsic_dest(nir_intrinsic_instr *instr, nir_foreach_dest_cb cb,
953 void *state)
954 {
955 if (nir_intrinsic_infos[instr->intrinsic].has_dest)
956 return cb(&instr->dest, state);
957
958 return true;
959 }
960
961 static bool
962 visit_texture_dest(nir_tex_instr *instr, nir_foreach_dest_cb cb,
963 void *state)
964 {
965 return cb(&instr->dest, state);
966 }
967
968 static bool
969 visit_phi_dest(nir_phi_instr *instr, nir_foreach_dest_cb cb, void *state)
970 {
971 return cb(&instr->dest, state);
972 }
973
974 static bool
975 visit_parallel_copy_dest(nir_parallel_copy_instr *instr,
976 nir_foreach_dest_cb cb, void *state)
977 {
978 nir_foreach_parallel_copy_entry(entry, instr) {
979 if (!cb(&entry->dest, state))
980 return false;
981 }
982
983 return true;
984 }
985
986 bool
987 nir_foreach_dest(nir_instr *instr, nir_foreach_dest_cb cb, void *state)
988 {
989 switch (instr->type) {
990 case nir_instr_type_alu:
991 return visit_alu_dest(nir_instr_as_alu(instr), cb, state);
992 case nir_instr_type_deref:
993 return visit_deref_dest(nir_instr_as_deref(instr), cb, state);
994 case nir_instr_type_intrinsic:
995 return visit_intrinsic_dest(nir_instr_as_intrinsic(instr), cb, state);
996 case nir_instr_type_tex:
997 return visit_texture_dest(nir_instr_as_tex(instr), cb, state);
998 case nir_instr_type_phi:
999 return visit_phi_dest(nir_instr_as_phi(instr), cb, state);
1000 case nir_instr_type_parallel_copy:
1001 return visit_parallel_copy_dest(nir_instr_as_parallel_copy(instr),
1002 cb, state);
1003
1004 case nir_instr_type_load_const:
1005 case nir_instr_type_ssa_undef:
1006 case nir_instr_type_call:
1007 case nir_instr_type_jump:
1008 break;
1009
1010 default:
1011 unreachable("Invalid instruction type");
1012 break;
1013 }
1014
1015 return true;
1016 }
1017
1018 struct foreach_ssa_def_state {
1019 nir_foreach_ssa_def_cb cb;
1020 void *client_state;
1021 };
1022
1023 static inline bool
1024 nir_ssa_def_visitor(nir_dest *dest, void *void_state)
1025 {
1026 struct foreach_ssa_def_state *state = void_state;
1027
1028 if (dest->is_ssa)
1029 return state->cb(&dest->ssa, state->client_state);
1030 else
1031 return true;
1032 }
1033
1034 bool
1035 nir_foreach_ssa_def(nir_instr *instr, nir_foreach_ssa_def_cb cb, void *state)
1036 {
1037 switch (instr->type) {
1038 case nir_instr_type_alu:
1039 case nir_instr_type_deref:
1040 case nir_instr_type_tex:
1041 case nir_instr_type_intrinsic:
1042 case nir_instr_type_phi:
1043 case nir_instr_type_parallel_copy: {
1044 struct foreach_ssa_def_state foreach_state = {cb, state};
1045 return nir_foreach_dest(instr, nir_ssa_def_visitor, &foreach_state);
1046 }
1047
1048 case nir_instr_type_load_const:
1049 return cb(&nir_instr_as_load_const(instr)->def, state);
1050 case nir_instr_type_ssa_undef:
1051 return cb(&nir_instr_as_ssa_undef(instr)->def, state);
1052 case nir_instr_type_call:
1053 case nir_instr_type_jump:
1054 return true;
1055 default:
1056 unreachable("Invalid instruction type");
1057 }
1058 }
1059
1060 nir_ssa_def *
1061 nir_instr_ssa_def(nir_instr *instr)
1062 {
1063 switch (instr->type) {
1064 case nir_instr_type_alu:
1065 assert(nir_instr_as_alu(instr)->dest.dest.is_ssa);
1066 return &nir_instr_as_alu(instr)->dest.dest.ssa;
1067
1068 case nir_instr_type_deref:
1069 assert(nir_instr_as_deref(instr)->dest.is_ssa);
1070 return &nir_instr_as_deref(instr)->dest.ssa;
1071
1072 case nir_instr_type_tex:
1073 assert(nir_instr_as_tex(instr)->dest.is_ssa);
1074 return &nir_instr_as_tex(instr)->dest.ssa;
1075
1076 case nir_instr_type_intrinsic: {
1077 nir_intrinsic_instr *intrin = nir_instr_as_intrinsic(instr);
1078 if (nir_intrinsic_infos[intrin->intrinsic].has_dest) {
1079 assert(intrin->dest.is_ssa);
1080 return &intrin->dest.ssa;
1081 } else {
1082 return NULL;
1083 }
1084 }
1085
1086 case nir_instr_type_phi:
1087 assert(nir_instr_as_phi(instr)->dest.is_ssa);
1088 return &nir_instr_as_phi(instr)->dest.ssa;
1089
1090 case nir_instr_type_parallel_copy:
1091 unreachable("Parallel copies are unsupported by this function");
1092
1093 case nir_instr_type_load_const:
1094 return &nir_instr_as_load_const(instr)->def;
1095
1096 case nir_instr_type_ssa_undef:
1097 return &nir_instr_as_ssa_undef(instr)->def;
1098
1099 case nir_instr_type_call:
1100 case nir_instr_type_jump:
1101 return NULL;
1102 }
1103
1104 unreachable("Invalid instruction type");
1105 }
1106
1107 static bool
1108 visit_src(nir_src *src, nir_foreach_src_cb cb, void *state)
1109 {
1110 if (!cb(src, state))
1111 return false;
1112 if (!src->is_ssa && src->reg.indirect)
1113 return cb(src->reg.indirect, state);
1114 return true;
1115 }
1116
1117 static bool
1118 visit_alu_src(nir_alu_instr *instr, nir_foreach_src_cb cb, void *state)
1119 {
1120 for (unsigned i = 0; i < nir_op_infos[instr->op].num_inputs; i++)
1121 if (!visit_src(&instr->src[i].src, cb, state))
1122 return false;
1123
1124 return true;
1125 }
1126
1127 static bool
1128 visit_deref_instr_src(nir_deref_instr *instr,
1129 nir_foreach_src_cb cb, void *state)
1130 {
1131 if (instr->deref_type != nir_deref_type_var) {
1132 if (!visit_src(&instr->parent, cb, state))
1133 return false;
1134 }
1135
1136 if (instr->deref_type == nir_deref_type_array ||
1137 instr->deref_type == nir_deref_type_ptr_as_array) {
1138 if (!visit_src(&instr->arr.index, cb, state))
1139 return false;
1140 }
1141
1142 return true;
1143 }
1144
1145 static bool
1146 visit_tex_src(nir_tex_instr *instr, nir_foreach_src_cb cb, void *state)
1147 {
1148 for (unsigned i = 0; i < instr->num_srcs; i++) {
1149 if (!visit_src(&instr->src[i].src, cb, state))
1150 return false;
1151 }
1152
1153 return true;
1154 }
1155
1156 static bool
1157 visit_intrinsic_src(nir_intrinsic_instr *instr, nir_foreach_src_cb cb,
1158 void *state)
1159 {
1160 unsigned num_srcs = nir_intrinsic_infos[instr->intrinsic].num_srcs;
1161 for (unsigned i = 0; i < num_srcs; i++) {
1162 if (!visit_src(&instr->src[i], cb, state))
1163 return false;
1164 }
1165
1166 return true;
1167 }
1168
1169 static bool
1170 visit_call_src(nir_call_instr *instr, nir_foreach_src_cb cb, void *state)
1171 {
1172 for (unsigned i = 0; i < instr->num_params; i++) {
1173 if (!visit_src(&instr->params[i], cb, state))
1174 return false;
1175 }
1176
1177 return true;
1178 }
1179
1180 static bool
1181 visit_phi_src(nir_phi_instr *instr, nir_foreach_src_cb cb, void *state)
1182 {
1183 nir_foreach_phi_src(src, instr) {
1184 if (!visit_src(&src->src, cb, state))
1185 return false;
1186 }
1187
1188 return true;
1189 }
1190
1191 static bool
1192 visit_parallel_copy_src(nir_parallel_copy_instr *instr,
1193 nir_foreach_src_cb cb, void *state)
1194 {
1195 nir_foreach_parallel_copy_entry(entry, instr) {
1196 if (!visit_src(&entry->src, cb, state))
1197 return false;
1198 }
1199
1200 return true;
1201 }
1202
1203 static bool
1204 visit_jump_src(nir_jump_instr *instr, nir_foreach_src_cb cb, void *state)
1205 {
1206 if (instr->type != nir_jump_goto_if)
1207 return true;
1208
1209 return visit_src(&instr->condition, cb, state);
1210 }
1211
1212 typedef struct {
1213 void *state;
1214 nir_foreach_src_cb cb;
1215 } visit_dest_indirect_state;
1216
1217 static bool
1218 visit_dest_indirect(nir_dest *dest, void *_state)
1219 {
1220 visit_dest_indirect_state *state = (visit_dest_indirect_state *) _state;
1221
1222 if (!dest->is_ssa && dest->reg.indirect)
1223 return state->cb(dest->reg.indirect, state->state);
1224
1225 return true;
1226 }
1227
1228 bool
1229 nir_foreach_src(nir_instr *instr, nir_foreach_src_cb cb, void *state)
1230 {
1231 switch (instr->type) {
1232 case nir_instr_type_alu:
1233 if (!visit_alu_src(nir_instr_as_alu(instr), cb, state))
1234 return false;
1235 break;
1236 case nir_instr_type_deref:
1237 if (!visit_deref_instr_src(nir_instr_as_deref(instr), cb, state))
1238 return false;
1239 break;
1240 case nir_instr_type_intrinsic:
1241 if (!visit_intrinsic_src(nir_instr_as_intrinsic(instr), cb, state))
1242 return false;
1243 break;
1244 case nir_instr_type_tex:
1245 if (!visit_tex_src(nir_instr_as_tex(instr), cb, state))
1246 return false;
1247 break;
1248 case nir_instr_type_call:
1249 if (!visit_call_src(nir_instr_as_call(instr), cb, state))
1250 return false;
1251 break;
1252 case nir_instr_type_load_const:
1253 /* Constant load instructions have no regular sources */
1254 break;
1255 case nir_instr_type_phi:
1256 if (!visit_phi_src(nir_instr_as_phi(instr), cb, state))
1257 return false;
1258 break;
1259 case nir_instr_type_parallel_copy:
1260 if (!visit_parallel_copy_src(nir_instr_as_parallel_copy(instr),
1261 cb, state))
1262 return false;
1263 break;
1264 case nir_instr_type_jump:
1265 return visit_jump_src(nir_instr_as_jump(instr), cb, state);
1266 case nir_instr_type_ssa_undef:
1267 return true;
1268
1269 default:
1270 unreachable("Invalid instruction type");
1271 break;
1272 }
1273
1274 visit_dest_indirect_state dest_state;
1275 dest_state.state = state;
1276 dest_state.cb = cb;
1277 return nir_foreach_dest(instr, visit_dest_indirect, &dest_state);
1278 }
1279
1280 bool
1281 nir_foreach_phi_src_leaving_block(nir_block *block,
1282 nir_foreach_src_cb cb,
1283 void *state)
1284 {
1285 for (unsigned i = 0; i < ARRAY_SIZE(block->successors); i++) {
1286 if (block->successors[i] == NULL)
1287 continue;
1288
1289 nir_foreach_instr(instr, block->successors[i]) {
1290 if (instr->type != nir_instr_type_phi)
1291 break;
1292
1293 nir_phi_instr *phi = nir_instr_as_phi(instr);
1294 nir_foreach_phi_src(phi_src, phi) {
1295 if (phi_src->pred == block) {
1296 if (!cb(&phi_src->src, state))
1297 return false;
1298 }
1299 }
1300 }
1301 }
1302
1303 return true;
1304 }
1305
1306 nir_const_value
1307 nir_const_value_for_float(double f, unsigned bit_size)
1308 {
1309 nir_const_value v;
1310 memset(&v, 0, sizeof(v));
1311
1312 switch (bit_size) {
1313 case 16:
1314 v.u16 = _mesa_float_to_half(f);
1315 break;
1316 case 32:
1317 v.f32 = f;
1318 break;
1319 case 64:
1320 v.f64 = f;
1321 break;
1322 default:
1323 unreachable("Invalid bit size");
1324 }
1325
1326 return v;
1327 }
1328
1329 double
1330 nir_const_value_as_float(nir_const_value value, unsigned bit_size)
1331 {
1332 switch (bit_size) {
1333 case 16: return _mesa_half_to_float(value.u16);
1334 case 32: return value.f32;
1335 case 64: return value.f64;
1336 default:
1337 unreachable("Invalid bit size");
1338 }
1339 }
1340
1341 nir_const_value *
1342 nir_src_as_const_value(nir_src src)
1343 {
1344 if (!src.is_ssa)
1345 return NULL;
1346
1347 if (src.ssa->parent_instr->type != nir_instr_type_load_const)
1348 return NULL;
1349
1350 nir_load_const_instr *load = nir_instr_as_load_const(src.ssa->parent_instr);
1351
1352 return load->value;
1353 }
1354
1355 /**
1356 * Returns true if the source is known to be dynamically uniform. Otherwise it
1357 * returns false which means it may or may not be dynamically uniform but it
1358 * can't be determined.
1359 */
1360 bool
1361 nir_src_is_dynamically_uniform(nir_src src)
1362 {
1363 if (!src.is_ssa)
1364 return false;
1365
1366 /* Constants are trivially dynamically uniform */
1367 if (src.ssa->parent_instr->type == nir_instr_type_load_const)
1368 return true;
1369
1370 /* As are uniform variables */
1371 if (src.ssa->parent_instr->type == nir_instr_type_intrinsic) {
1372 nir_intrinsic_instr *intr = nir_instr_as_intrinsic(src.ssa->parent_instr);
1373 if (intr->intrinsic == nir_intrinsic_load_uniform &&
1374 nir_src_is_dynamically_uniform(intr->src[0]))
1375 return true;
1376 }
1377
1378 /* Operating together dynamically uniform expressions produces a
1379 * dynamically uniform result
1380 */
1381 if (src.ssa->parent_instr->type == nir_instr_type_alu) {
1382 nir_alu_instr *alu = nir_instr_as_alu(src.ssa->parent_instr);
1383 for (int i = 0; i < nir_op_infos[alu->op].num_inputs; i++) {
1384 if (!nir_src_is_dynamically_uniform(alu->src[i].src))
1385 return false;
1386 }
1387
1388 return true;
1389 }
1390
1391 /* XXX: this could have many more tests, such as when a sampler function is
1392 * called with dynamically uniform arguments.
1393 */
1394 return false;
1395 }
1396
1397 static void
1398 src_remove_all_uses(nir_src *src)
1399 {
1400 for (; src; src = src->is_ssa ? NULL : src->reg.indirect) {
1401 if (!src_is_valid(src))
1402 continue;
1403
1404 list_del(&src->use_link);
1405 }
1406 }
1407
1408 static void
1409 src_add_all_uses(nir_src *src, nir_instr *parent_instr, nir_if *parent_if)
1410 {
1411 for (; src; src = src->is_ssa ? NULL : src->reg.indirect) {
1412 if (!src_is_valid(src))
1413 continue;
1414
1415 if (parent_instr) {
1416 src->parent_instr = parent_instr;
1417 if (src->is_ssa)
1418 list_addtail(&src->use_link, &src->ssa->uses);
1419 else
1420 list_addtail(&src->use_link, &src->reg.reg->uses);
1421 } else {
1422 assert(parent_if);
1423 src->parent_if = parent_if;
1424 if (src->is_ssa)
1425 list_addtail(&src->use_link, &src->ssa->if_uses);
1426 else
1427 list_addtail(&src->use_link, &src->reg.reg->if_uses);
1428 }
1429 }
1430 }
1431
1432 void
1433 nir_instr_rewrite_src(nir_instr *instr, nir_src *src, nir_src new_src)
1434 {
1435 assert(!src_is_valid(src) || src->parent_instr == instr);
1436
1437 src_remove_all_uses(src);
1438 *src = new_src;
1439 src_add_all_uses(src, instr, NULL);
1440 }
1441
1442 void
1443 nir_instr_move_src(nir_instr *dest_instr, nir_src *dest, nir_src *src)
1444 {
1445 assert(!src_is_valid(dest) || dest->parent_instr == dest_instr);
1446
1447 src_remove_all_uses(dest);
1448 src_remove_all_uses(src);
1449 *dest = *src;
1450 *src = NIR_SRC_INIT;
1451 src_add_all_uses(dest, dest_instr, NULL);
1452 }
1453
1454 void
1455 nir_if_rewrite_condition(nir_if *if_stmt, nir_src new_src)
1456 {
1457 nir_src *src = &if_stmt->condition;
1458 assert(!src_is_valid(src) || src->parent_if == if_stmt);
1459
1460 src_remove_all_uses(src);
1461 *src = new_src;
1462 src_add_all_uses(src, NULL, if_stmt);
1463 }
1464
1465 void
1466 nir_instr_rewrite_dest(nir_instr *instr, nir_dest *dest, nir_dest new_dest)
1467 {
1468 if (dest->is_ssa) {
1469 /* We can only overwrite an SSA destination if it has no uses. */
1470 assert(list_is_empty(&dest->ssa.uses) && list_is_empty(&dest->ssa.if_uses));
1471 } else {
1472 list_del(&dest->reg.def_link);
1473 if (dest->reg.indirect)
1474 src_remove_all_uses(dest->reg.indirect);
1475 }
1476
1477 /* We can't re-write with an SSA def */
1478 assert(!new_dest.is_ssa);
1479
1480 nir_dest_copy(dest, &new_dest, instr);
1481
1482 dest->reg.parent_instr = instr;
1483 list_addtail(&dest->reg.def_link, &new_dest.reg.reg->defs);
1484
1485 if (dest->reg.indirect)
1486 src_add_all_uses(dest->reg.indirect, instr, NULL);
1487 }
1488
1489 /* note: does *not* take ownership of 'name' */
1490 void
1491 nir_ssa_def_init(nir_instr *instr, nir_ssa_def *def,
1492 unsigned num_components,
1493 unsigned bit_size, const char *name)
1494 {
1495 def->name = ralloc_strdup(instr, name);
1496 def->live_index = UINT_MAX; /* Something clearly OOB */
1497 def->parent_instr = instr;
1498 list_inithead(&def->uses);
1499 list_inithead(&def->if_uses);
1500 def->num_components = num_components;
1501 def->bit_size = bit_size;
1502 def->divergent = true; /* This is the safer default */
1503
1504 if (instr->block) {
1505 nir_function_impl *impl =
1506 nir_cf_node_get_function(&instr->block->cf_node);
1507
1508 def->index = impl->ssa_alloc++;
1509
1510 impl->valid_metadata &= ~nir_metadata_live_ssa_defs;
1511 } else {
1512 def->index = UINT_MAX;
1513 }
1514 }
1515
1516 /* note: does *not* take ownership of 'name' */
1517 void
1518 nir_ssa_dest_init(nir_instr *instr, nir_dest *dest,
1519 unsigned num_components, unsigned bit_size,
1520 const char *name)
1521 {
1522 dest->is_ssa = true;
1523 nir_ssa_def_init(instr, &dest->ssa, num_components, bit_size, name);
1524 }
1525
1526 void
1527 nir_ssa_def_rewrite_uses(nir_ssa_def *def, nir_src new_src)
1528 {
1529 assert(!new_src.is_ssa || def != new_src.ssa);
1530
1531 nir_foreach_use_safe(use_src, def)
1532 nir_instr_rewrite_src(use_src->parent_instr, use_src, new_src);
1533
1534 nir_foreach_if_use_safe(use_src, def)
1535 nir_if_rewrite_condition(use_src->parent_if, new_src);
1536 }
1537
1538 static bool
1539 is_instr_between(nir_instr *start, nir_instr *end, nir_instr *between)
1540 {
1541 assert(start->block == end->block);
1542
1543 if (between->block != start->block)
1544 return false;
1545
1546 /* Search backwards looking for "between" */
1547 while (start != end) {
1548 if (between == end)
1549 return true;
1550
1551 end = nir_instr_prev(end);
1552 assert(end);
1553 }
1554
1555 return false;
1556 }
1557
1558 /* Replaces all uses of the given SSA def with the given source but only if
1559 * the use comes after the after_me instruction. This can be useful if you
1560 * are emitting code to fix up the result of some instruction: you can freely
1561 * use the result in that code and then call rewrite_uses_after and pass the
1562 * last fixup instruction as after_me and it will replace all of the uses you
1563 * want without touching the fixup code.
1564 *
1565 * This function assumes that after_me is in the same block as
1566 * def->parent_instr and that after_me comes after def->parent_instr.
1567 */
1568 void
1569 nir_ssa_def_rewrite_uses_after(nir_ssa_def *def, nir_src new_src,
1570 nir_instr *after_me)
1571 {
1572 if (new_src.is_ssa && def == new_src.ssa)
1573 return;
1574
1575 nir_foreach_use_safe(use_src, def) {
1576 assert(use_src->parent_instr != def->parent_instr);
1577 /* Since def already dominates all of its uses, the only way a use can
1578 * not be dominated by after_me is if it is between def and after_me in
1579 * the instruction list.
1580 */
1581 if (!is_instr_between(def->parent_instr, after_me, use_src->parent_instr))
1582 nir_instr_rewrite_src(use_src->parent_instr, use_src, new_src);
1583 }
1584
1585 nir_foreach_if_use_safe(use_src, def)
1586 nir_if_rewrite_condition(use_src->parent_if, new_src);
1587 }
1588
1589 nir_component_mask_t
1590 nir_ssa_def_components_read(const nir_ssa_def *def)
1591 {
1592 nir_component_mask_t read_mask = 0;
1593 nir_foreach_use(use, def) {
1594 if (use->parent_instr->type == nir_instr_type_alu) {
1595 nir_alu_instr *alu = nir_instr_as_alu(use->parent_instr);
1596 nir_alu_src *alu_src = exec_node_data(nir_alu_src, use, src);
1597 int src_idx = alu_src - &alu->src[0];
1598 assert(src_idx >= 0 && src_idx < nir_op_infos[alu->op].num_inputs);
1599 read_mask |= nir_alu_instr_src_read_mask(alu, src_idx);
1600 } else {
1601 return (1 << def->num_components) - 1;
1602 }
1603 }
1604
1605 if (!list_is_empty(&def->if_uses))
1606 read_mask |= 1;
1607
1608 return read_mask;
1609 }
1610
1611 nir_block *
1612 nir_block_unstructured_next(nir_block *block)
1613 {
1614 if (block == NULL) {
1615 /* nir_foreach_block_unstructured_safe() will call this function on a
1616 * NULL block after the last iteration, but it won't use the result so
1617 * just return NULL here.
1618 */
1619 return NULL;
1620 }
1621
1622 nir_cf_node *cf_next = nir_cf_node_next(&block->cf_node);
1623 if (cf_next == NULL && block->cf_node.parent->type == nir_cf_node_function)
1624 return NULL;
1625
1626 if (cf_next && cf_next->type == nir_cf_node_block)
1627 return nir_cf_node_as_block(cf_next);
1628
1629 return nir_block_cf_tree_next(block);
1630 }
1631
1632 nir_block *
1633 nir_unstructured_start_block(nir_function_impl *impl)
1634 {
1635 return nir_start_block(impl);
1636 }
1637
1638 nir_block *
1639 nir_block_cf_tree_next(nir_block *block)
1640 {
1641 if (block == NULL) {
1642 /* nir_foreach_block_safe() will call this function on a NULL block
1643 * after the last iteration, but it won't use the result so just return
1644 * NULL here.
1645 */
1646 return NULL;
1647 }
1648
1649 assert(nir_cf_node_get_function(&block->cf_node)->structured);
1650
1651 nir_cf_node *cf_next = nir_cf_node_next(&block->cf_node);
1652 if (cf_next)
1653 return nir_cf_node_cf_tree_first(cf_next);
1654
1655 nir_cf_node *parent = block->cf_node.parent;
1656
1657 switch (parent->type) {
1658 case nir_cf_node_if: {
1659 /* Are we at the end of the if? Go to the beginning of the else */
1660 nir_if *if_stmt = nir_cf_node_as_if(parent);
1661 if (block == nir_if_last_then_block(if_stmt))
1662 return nir_if_first_else_block(if_stmt);
1663
1664 assert(block == nir_if_last_else_block(if_stmt));
1665 }
1666 /* fallthrough */
1667
1668 case nir_cf_node_loop:
1669 return nir_cf_node_as_block(nir_cf_node_next(parent));
1670
1671 case nir_cf_node_function:
1672 return NULL;
1673
1674 default:
1675 unreachable("unknown cf node type");
1676 }
1677 }
1678
1679 nir_block *
1680 nir_block_cf_tree_prev(nir_block *block)
1681 {
1682 if (block == NULL) {
1683 /* do this for consistency with nir_block_cf_tree_next() */
1684 return NULL;
1685 }
1686
1687 assert(nir_cf_node_get_function(&block->cf_node)->structured);
1688
1689 nir_cf_node *cf_prev = nir_cf_node_prev(&block->cf_node);
1690 if (cf_prev)
1691 return nir_cf_node_cf_tree_last(cf_prev);
1692
1693 nir_cf_node *parent = block->cf_node.parent;
1694
1695 switch (parent->type) {
1696 case nir_cf_node_if: {
1697 /* Are we at the beginning of the else? Go to the end of the if */
1698 nir_if *if_stmt = nir_cf_node_as_if(parent);
1699 if (block == nir_if_first_else_block(if_stmt))
1700 return nir_if_last_then_block(if_stmt);
1701
1702 assert(block == nir_if_first_then_block(if_stmt));
1703 }
1704 /* fallthrough */
1705
1706 case nir_cf_node_loop:
1707 return nir_cf_node_as_block(nir_cf_node_prev(parent));
1708
1709 case nir_cf_node_function:
1710 return NULL;
1711
1712 default:
1713 unreachable("unknown cf node type");
1714 }
1715 }
1716
1717 nir_block *nir_cf_node_cf_tree_first(nir_cf_node *node)
1718 {
1719 switch (node->type) {
1720 case nir_cf_node_function: {
1721 nir_function_impl *impl = nir_cf_node_as_function(node);
1722 return nir_start_block(impl);
1723 }
1724
1725 case nir_cf_node_if: {
1726 nir_if *if_stmt = nir_cf_node_as_if(node);
1727 return nir_if_first_then_block(if_stmt);
1728 }
1729
1730 case nir_cf_node_loop: {
1731 nir_loop *loop = nir_cf_node_as_loop(node);
1732 return nir_loop_first_block(loop);
1733 }
1734
1735 case nir_cf_node_block: {
1736 return nir_cf_node_as_block(node);
1737 }
1738
1739 default:
1740 unreachable("unknown node type");
1741 }
1742 }
1743
1744 nir_block *nir_cf_node_cf_tree_last(nir_cf_node *node)
1745 {
1746 switch (node->type) {
1747 case nir_cf_node_function: {
1748 nir_function_impl *impl = nir_cf_node_as_function(node);
1749 return nir_impl_last_block(impl);
1750 }
1751
1752 case nir_cf_node_if: {
1753 nir_if *if_stmt = nir_cf_node_as_if(node);
1754 return nir_if_last_else_block(if_stmt);
1755 }
1756
1757 case nir_cf_node_loop: {
1758 nir_loop *loop = nir_cf_node_as_loop(node);
1759 return nir_loop_last_block(loop);
1760 }
1761
1762 case nir_cf_node_block: {
1763 return nir_cf_node_as_block(node);
1764 }
1765
1766 default:
1767 unreachable("unknown node type");
1768 }
1769 }
1770
1771 nir_block *nir_cf_node_cf_tree_next(nir_cf_node *node)
1772 {
1773 if (node->type == nir_cf_node_block)
1774 return nir_block_cf_tree_next(nir_cf_node_as_block(node));
1775 else if (node->type == nir_cf_node_function)
1776 return NULL;
1777 else
1778 return nir_cf_node_as_block(nir_cf_node_next(node));
1779 }
1780
1781 nir_if *
1782 nir_block_get_following_if(nir_block *block)
1783 {
1784 if (exec_node_is_tail_sentinel(&block->cf_node.node))
1785 return NULL;
1786
1787 if (nir_cf_node_is_last(&block->cf_node))
1788 return NULL;
1789
1790 nir_cf_node *next_node = nir_cf_node_next(&block->cf_node);
1791
1792 if (next_node->type != nir_cf_node_if)
1793 return NULL;
1794
1795 return nir_cf_node_as_if(next_node);
1796 }
1797
1798 nir_loop *
1799 nir_block_get_following_loop(nir_block *block)
1800 {
1801 if (exec_node_is_tail_sentinel(&block->cf_node.node))
1802 return NULL;
1803
1804 if (nir_cf_node_is_last(&block->cf_node))
1805 return NULL;
1806
1807 nir_cf_node *next_node = nir_cf_node_next(&block->cf_node);
1808
1809 if (next_node->type != nir_cf_node_loop)
1810 return NULL;
1811
1812 return nir_cf_node_as_loop(next_node);
1813 }
1814
1815 void
1816 nir_index_blocks(nir_function_impl *impl)
1817 {
1818 unsigned index = 0;
1819
1820 if (impl->valid_metadata & nir_metadata_block_index)
1821 return;
1822
1823 nir_foreach_block_unstructured(block, impl) {
1824 block->index = index++;
1825 }
1826
1827 /* The end_block isn't really part of the program, which is why its index
1828 * is >= num_blocks.
1829 */
1830 impl->num_blocks = impl->end_block->index = index;
1831 }
1832
1833 static bool
1834 index_ssa_def_cb(nir_ssa_def *def, void *state)
1835 {
1836 unsigned *index = (unsigned *) state;
1837 def->index = (*index)++;
1838
1839 return true;
1840 }
1841
1842 /**
1843 * The indices are applied top-to-bottom which has the very nice property
1844 * that, if A dominates B, then A->index <= B->index.
1845 */
1846 void
1847 nir_index_ssa_defs(nir_function_impl *impl)
1848 {
1849 unsigned index = 0;
1850
1851 impl->valid_metadata &= ~nir_metadata_live_ssa_defs;
1852
1853 nir_foreach_block_unstructured(block, impl) {
1854 nir_foreach_instr(instr, block)
1855 nir_foreach_ssa_def(instr, index_ssa_def_cb, &index);
1856 }
1857
1858 impl->ssa_alloc = index;
1859 }
1860
1861 /**
1862 * The indices are applied top-to-bottom which has the very nice property
1863 * that, if A dominates B, then A->index <= B->index.
1864 */
1865 unsigned
1866 nir_index_instrs(nir_function_impl *impl)
1867 {
1868 unsigned index = 0;
1869
1870 nir_foreach_block(block, impl) {
1871 nir_foreach_instr(instr, block)
1872 instr->index = index++;
1873 }
1874
1875 return index;
1876 }
1877
1878 unsigned
1879 nir_shader_index_vars(nir_shader *shader, nir_variable_mode modes)
1880 {
1881 unsigned count = 0;
1882 nir_foreach_variable_with_modes(var, shader, modes)
1883 var->index = count++;
1884 return count;
1885 }
1886
1887 unsigned
1888 nir_function_impl_index_vars(nir_function_impl *impl)
1889 {
1890 unsigned count = 0;
1891 nir_foreach_function_temp_variable(var, impl)
1892 var->index = count++;
1893 return count;
1894 }
1895
1896 static nir_instr *
1897 cursor_next_instr(nir_cursor cursor)
1898 {
1899 switch (cursor.option) {
1900 case nir_cursor_before_block:
1901 for (nir_block *block = cursor.block; block;
1902 block = nir_block_cf_tree_next(block)) {
1903 nir_instr *instr = nir_block_first_instr(block);
1904 if (instr)
1905 return instr;
1906 }
1907 return NULL;
1908
1909 case nir_cursor_after_block:
1910 cursor.block = nir_block_cf_tree_next(cursor.block);
1911 if (cursor.block == NULL)
1912 return NULL;
1913
1914 cursor.option = nir_cursor_before_block;
1915 return cursor_next_instr(cursor);
1916
1917 case nir_cursor_before_instr:
1918 return cursor.instr;
1919
1920 case nir_cursor_after_instr:
1921 if (nir_instr_next(cursor.instr))
1922 return nir_instr_next(cursor.instr);
1923
1924 cursor.option = nir_cursor_after_block;
1925 cursor.block = cursor.instr->block;
1926 return cursor_next_instr(cursor);
1927 }
1928
1929 unreachable("Inavlid cursor option");
1930 }
1931
1932 ASSERTED static bool
1933 dest_is_ssa(nir_dest *dest, void *_state)
1934 {
1935 (void) _state;
1936 return dest->is_ssa;
1937 }
1938
1939 bool
1940 nir_function_impl_lower_instructions(nir_function_impl *impl,
1941 nir_instr_filter_cb filter,
1942 nir_lower_instr_cb lower,
1943 void *cb_data)
1944 {
1945 nir_builder b;
1946 nir_builder_init(&b, impl);
1947
1948 nir_metadata preserved = nir_metadata_block_index |
1949 nir_metadata_dominance;
1950
1951 bool progress = false;
1952 nir_cursor iter = nir_before_cf_list(&impl->body);
1953 nir_instr *instr;
1954 while ((instr = cursor_next_instr(iter)) != NULL) {
1955 if (filter && !filter(instr, cb_data)) {
1956 iter = nir_after_instr(instr);
1957 continue;
1958 }
1959
1960 assert(nir_foreach_dest(instr, dest_is_ssa, NULL));
1961 nir_ssa_def *old_def = nir_instr_ssa_def(instr);
1962 if (old_def == NULL) {
1963 iter = nir_after_instr(instr);
1964 continue;
1965 }
1966
1967 /* We're about to ask the callback to generate a replacement for instr.
1968 * Save off the uses from instr's SSA def so we know what uses to
1969 * rewrite later. If we use nir_ssa_def_rewrite_uses, it fails in the
1970 * case where the generated replacement code uses the result of instr
1971 * itself. If we use nir_ssa_def_rewrite_uses_after (which is the
1972 * normal solution to this problem), it doesn't work well if control-
1973 * flow is inserted as part of the replacement, doesn't handle cases
1974 * where the replacement is something consumed by instr, and suffers
1975 * from performance issues. This is the only way to 100% guarantee
1976 * that we rewrite the correct set efficiently.
1977 */
1978 struct list_head old_uses, old_if_uses;
1979 list_replace(&old_def->uses, &old_uses);
1980 list_inithead(&old_def->uses);
1981 list_replace(&old_def->if_uses, &old_if_uses);
1982 list_inithead(&old_def->if_uses);
1983
1984 b.cursor = nir_after_instr(instr);
1985 nir_ssa_def *new_def = lower(&b, instr, cb_data);
1986 if (new_def && new_def != NIR_LOWER_INSTR_PROGRESS) {
1987 assert(old_def != NULL);
1988 if (new_def->parent_instr->block != instr->block)
1989 preserved = nir_metadata_none;
1990
1991 nir_src new_src = nir_src_for_ssa(new_def);
1992 list_for_each_entry_safe(nir_src, use_src, &old_uses, use_link)
1993 nir_instr_rewrite_src(use_src->parent_instr, use_src, new_src);
1994
1995 list_for_each_entry_safe(nir_src, use_src, &old_if_uses, use_link)
1996 nir_if_rewrite_condition(use_src->parent_if, new_src);
1997
1998 if (list_is_empty(&old_def->uses) && list_is_empty(&old_def->if_uses)) {
1999 iter = nir_instr_remove(instr);
2000 } else {
2001 iter = nir_after_instr(instr);
2002 }
2003 progress = true;
2004 } else {
2005 /* We didn't end up lowering after all. Put the uses back */
2006 if (old_def) {
2007 list_replace(&old_uses, &old_def->uses);
2008 list_replace(&old_if_uses, &old_def->if_uses);
2009 }
2010 iter = nir_after_instr(instr);
2011
2012 if (new_def == NIR_LOWER_INSTR_PROGRESS)
2013 progress = true;
2014 }
2015 }
2016
2017 if (progress) {
2018 nir_metadata_preserve(impl, preserved);
2019 } else {
2020 nir_metadata_preserve(impl, nir_metadata_all);
2021 }
2022
2023 return progress;
2024 }
2025
2026 bool
2027 nir_shader_lower_instructions(nir_shader *shader,
2028 nir_instr_filter_cb filter,
2029 nir_lower_instr_cb lower,
2030 void *cb_data)
2031 {
2032 bool progress = false;
2033
2034 nir_foreach_function(function, shader) {
2035 if (function->impl &&
2036 nir_function_impl_lower_instructions(function->impl,
2037 filter, lower, cb_data))
2038 progress = true;
2039 }
2040
2041 return progress;
2042 }
2043
2044 nir_intrinsic_op
2045 nir_intrinsic_from_system_value(gl_system_value val)
2046 {
2047 switch (val) {
2048 case SYSTEM_VALUE_VERTEX_ID:
2049 return nir_intrinsic_load_vertex_id;
2050 case SYSTEM_VALUE_INSTANCE_ID:
2051 return nir_intrinsic_load_instance_id;
2052 case SYSTEM_VALUE_DRAW_ID:
2053 return nir_intrinsic_load_draw_id;
2054 case SYSTEM_VALUE_BASE_INSTANCE:
2055 return nir_intrinsic_load_base_instance;
2056 case SYSTEM_VALUE_VERTEX_ID_ZERO_BASE:
2057 return nir_intrinsic_load_vertex_id_zero_base;
2058 case SYSTEM_VALUE_IS_INDEXED_DRAW:
2059 return nir_intrinsic_load_is_indexed_draw;
2060 case SYSTEM_VALUE_FIRST_VERTEX:
2061 return nir_intrinsic_load_first_vertex;
2062 case SYSTEM_VALUE_BASE_VERTEX:
2063 return nir_intrinsic_load_base_vertex;
2064 case SYSTEM_VALUE_INVOCATION_ID:
2065 return nir_intrinsic_load_invocation_id;
2066 case SYSTEM_VALUE_FRAG_COORD:
2067 return nir_intrinsic_load_frag_coord;
2068 case SYSTEM_VALUE_POINT_COORD:
2069 return nir_intrinsic_load_point_coord;
2070 case SYSTEM_VALUE_LINE_COORD:
2071 return nir_intrinsic_load_line_coord;
2072 case SYSTEM_VALUE_FRONT_FACE:
2073 return nir_intrinsic_load_front_face;
2074 case SYSTEM_VALUE_SAMPLE_ID:
2075 return nir_intrinsic_load_sample_id;
2076 case SYSTEM_VALUE_SAMPLE_POS:
2077 return nir_intrinsic_load_sample_pos;
2078 case SYSTEM_VALUE_SAMPLE_MASK_IN:
2079 return nir_intrinsic_load_sample_mask_in;
2080 case SYSTEM_VALUE_LOCAL_INVOCATION_ID:
2081 return nir_intrinsic_load_local_invocation_id;
2082 case SYSTEM_VALUE_LOCAL_INVOCATION_INDEX:
2083 return nir_intrinsic_load_local_invocation_index;
2084 case SYSTEM_VALUE_WORK_GROUP_ID:
2085 return nir_intrinsic_load_work_group_id;
2086 case SYSTEM_VALUE_NUM_WORK_GROUPS:
2087 return nir_intrinsic_load_num_work_groups;
2088 case SYSTEM_VALUE_PRIMITIVE_ID:
2089 return nir_intrinsic_load_primitive_id;
2090 case SYSTEM_VALUE_TESS_COORD:
2091 return nir_intrinsic_load_tess_coord;
2092 case SYSTEM_VALUE_TESS_LEVEL_OUTER:
2093 return nir_intrinsic_load_tess_level_outer;
2094 case SYSTEM_VALUE_TESS_LEVEL_INNER:
2095 return nir_intrinsic_load_tess_level_inner;
2096 case SYSTEM_VALUE_TESS_LEVEL_OUTER_DEFAULT:
2097 return nir_intrinsic_load_tess_level_outer_default;
2098 case SYSTEM_VALUE_TESS_LEVEL_INNER_DEFAULT:
2099 return nir_intrinsic_load_tess_level_inner_default;
2100 case SYSTEM_VALUE_VERTICES_IN:
2101 return nir_intrinsic_load_patch_vertices_in;
2102 case SYSTEM_VALUE_HELPER_INVOCATION:
2103 return nir_intrinsic_load_helper_invocation;
2104 case SYSTEM_VALUE_COLOR0:
2105 return nir_intrinsic_load_color0;
2106 case SYSTEM_VALUE_COLOR1:
2107 return nir_intrinsic_load_color1;
2108 case SYSTEM_VALUE_VIEW_INDEX:
2109 return nir_intrinsic_load_view_index;
2110 case SYSTEM_VALUE_SUBGROUP_SIZE:
2111 return nir_intrinsic_load_subgroup_size;
2112 case SYSTEM_VALUE_SUBGROUP_INVOCATION:
2113 return nir_intrinsic_load_subgroup_invocation;
2114 case SYSTEM_VALUE_SUBGROUP_EQ_MASK:
2115 return nir_intrinsic_load_subgroup_eq_mask;
2116 case SYSTEM_VALUE_SUBGROUP_GE_MASK:
2117 return nir_intrinsic_load_subgroup_ge_mask;
2118 case SYSTEM_VALUE_SUBGROUP_GT_MASK:
2119 return nir_intrinsic_load_subgroup_gt_mask;
2120 case SYSTEM_VALUE_SUBGROUP_LE_MASK:
2121 return nir_intrinsic_load_subgroup_le_mask;
2122 case SYSTEM_VALUE_SUBGROUP_LT_MASK:
2123 return nir_intrinsic_load_subgroup_lt_mask;
2124 case SYSTEM_VALUE_NUM_SUBGROUPS:
2125 return nir_intrinsic_load_num_subgroups;
2126 case SYSTEM_VALUE_SUBGROUP_ID:
2127 return nir_intrinsic_load_subgroup_id;
2128 case SYSTEM_VALUE_LOCAL_GROUP_SIZE:
2129 return nir_intrinsic_load_local_group_size;
2130 case SYSTEM_VALUE_GLOBAL_INVOCATION_ID:
2131 return nir_intrinsic_load_global_invocation_id;
2132 case SYSTEM_VALUE_BASE_GLOBAL_INVOCATION_ID:
2133 return nir_intrinsic_load_base_global_invocation_id;
2134 case SYSTEM_VALUE_GLOBAL_INVOCATION_INDEX:
2135 return nir_intrinsic_load_global_invocation_index;
2136 case SYSTEM_VALUE_WORK_DIM:
2137 return nir_intrinsic_load_work_dim;
2138 case SYSTEM_VALUE_USER_DATA_AMD:
2139 return nir_intrinsic_load_user_data_amd;
2140 default:
2141 unreachable("system value does not directly correspond to intrinsic");
2142 }
2143 }
2144
2145 gl_system_value
2146 nir_system_value_from_intrinsic(nir_intrinsic_op intrin)
2147 {
2148 switch (intrin) {
2149 case nir_intrinsic_load_vertex_id:
2150 return SYSTEM_VALUE_VERTEX_ID;
2151 case nir_intrinsic_load_instance_id:
2152 return SYSTEM_VALUE_INSTANCE_ID;
2153 case nir_intrinsic_load_draw_id:
2154 return SYSTEM_VALUE_DRAW_ID;
2155 case nir_intrinsic_load_base_instance:
2156 return SYSTEM_VALUE_BASE_INSTANCE;
2157 case nir_intrinsic_load_vertex_id_zero_base:
2158 return SYSTEM_VALUE_VERTEX_ID_ZERO_BASE;
2159 case nir_intrinsic_load_first_vertex:
2160 return SYSTEM_VALUE_FIRST_VERTEX;
2161 case nir_intrinsic_load_is_indexed_draw:
2162 return SYSTEM_VALUE_IS_INDEXED_DRAW;
2163 case nir_intrinsic_load_base_vertex:
2164 return SYSTEM_VALUE_BASE_VERTEX;
2165 case nir_intrinsic_load_invocation_id:
2166 return SYSTEM_VALUE_INVOCATION_ID;
2167 case nir_intrinsic_load_frag_coord:
2168 return SYSTEM_VALUE_FRAG_COORD;
2169 case nir_intrinsic_load_point_coord:
2170 return SYSTEM_VALUE_POINT_COORD;
2171 case nir_intrinsic_load_line_coord:
2172 return SYSTEM_VALUE_LINE_COORD;
2173 case nir_intrinsic_load_front_face:
2174 return SYSTEM_VALUE_FRONT_FACE;
2175 case nir_intrinsic_load_sample_id:
2176 return SYSTEM_VALUE_SAMPLE_ID;
2177 case nir_intrinsic_load_sample_pos:
2178 return SYSTEM_VALUE_SAMPLE_POS;
2179 case nir_intrinsic_load_sample_mask_in:
2180 return SYSTEM_VALUE_SAMPLE_MASK_IN;
2181 case nir_intrinsic_load_local_invocation_id:
2182 return SYSTEM_VALUE_LOCAL_INVOCATION_ID;
2183 case nir_intrinsic_load_local_invocation_index:
2184 return SYSTEM_VALUE_LOCAL_INVOCATION_INDEX;
2185 case nir_intrinsic_load_num_work_groups:
2186 return SYSTEM_VALUE_NUM_WORK_GROUPS;
2187 case nir_intrinsic_load_work_group_id:
2188 return SYSTEM_VALUE_WORK_GROUP_ID;
2189 case nir_intrinsic_load_primitive_id:
2190 return SYSTEM_VALUE_PRIMITIVE_ID;
2191 case nir_intrinsic_load_tess_coord:
2192 return SYSTEM_VALUE_TESS_COORD;
2193 case nir_intrinsic_load_tess_level_outer:
2194 return SYSTEM_VALUE_TESS_LEVEL_OUTER;
2195 case nir_intrinsic_load_tess_level_inner:
2196 return SYSTEM_VALUE_TESS_LEVEL_INNER;
2197 case nir_intrinsic_load_tess_level_outer_default:
2198 return SYSTEM_VALUE_TESS_LEVEL_OUTER_DEFAULT;
2199 case nir_intrinsic_load_tess_level_inner_default:
2200 return SYSTEM_VALUE_TESS_LEVEL_INNER_DEFAULT;
2201 case nir_intrinsic_load_patch_vertices_in:
2202 return SYSTEM_VALUE_VERTICES_IN;
2203 case nir_intrinsic_load_helper_invocation:
2204 return SYSTEM_VALUE_HELPER_INVOCATION;
2205 case nir_intrinsic_load_color0:
2206 return SYSTEM_VALUE_COLOR0;
2207 case nir_intrinsic_load_color1:
2208 return SYSTEM_VALUE_COLOR1;
2209 case nir_intrinsic_load_view_index:
2210 return SYSTEM_VALUE_VIEW_INDEX;
2211 case nir_intrinsic_load_subgroup_size:
2212 return SYSTEM_VALUE_SUBGROUP_SIZE;
2213 case nir_intrinsic_load_subgroup_invocation:
2214 return SYSTEM_VALUE_SUBGROUP_INVOCATION;
2215 case nir_intrinsic_load_subgroup_eq_mask:
2216 return SYSTEM_VALUE_SUBGROUP_EQ_MASK;
2217 case nir_intrinsic_load_subgroup_ge_mask:
2218 return SYSTEM_VALUE_SUBGROUP_GE_MASK;
2219 case nir_intrinsic_load_subgroup_gt_mask:
2220 return SYSTEM_VALUE_SUBGROUP_GT_MASK;
2221 case nir_intrinsic_load_subgroup_le_mask:
2222 return SYSTEM_VALUE_SUBGROUP_LE_MASK;
2223 case nir_intrinsic_load_subgroup_lt_mask:
2224 return SYSTEM_VALUE_SUBGROUP_LT_MASK;
2225 case nir_intrinsic_load_num_subgroups:
2226 return SYSTEM_VALUE_NUM_SUBGROUPS;
2227 case nir_intrinsic_load_subgroup_id:
2228 return SYSTEM_VALUE_SUBGROUP_ID;
2229 case nir_intrinsic_load_local_group_size:
2230 return SYSTEM_VALUE_LOCAL_GROUP_SIZE;
2231 case nir_intrinsic_load_global_invocation_id:
2232 return SYSTEM_VALUE_GLOBAL_INVOCATION_ID;
2233 case nir_intrinsic_load_base_global_invocation_id:
2234 return SYSTEM_VALUE_BASE_GLOBAL_INVOCATION_ID;
2235 case nir_intrinsic_load_global_invocation_index:
2236 return SYSTEM_VALUE_GLOBAL_INVOCATION_INDEX;
2237 case nir_intrinsic_load_work_dim:
2238 return SYSTEM_VALUE_WORK_DIM;
2239 case nir_intrinsic_load_user_data_amd:
2240 return SYSTEM_VALUE_USER_DATA_AMD;
2241 default:
2242 unreachable("intrinsic doesn't produce a system value");
2243 }
2244 }
2245
2246 /* OpenGL utility method that remaps the location attributes if they are
2247 * doubles. Not needed for vulkan due the differences on the input location
2248 * count for doubles on vulkan vs OpenGL
2249 *
2250 * The bitfield returned in dual_slot is one bit for each double input slot in
2251 * the original OpenGL single-slot input numbering. The mapping from old
2252 * locations to new locations is as follows:
2253 *
2254 * new_loc = loc + util_bitcount(dual_slot & BITFIELD64_MASK(loc))
2255 */
2256 void
2257 nir_remap_dual_slot_attributes(nir_shader *shader, uint64_t *dual_slot)
2258 {
2259 assert(shader->info.stage == MESA_SHADER_VERTEX);
2260
2261 *dual_slot = 0;
2262 nir_foreach_shader_in_variable(var, shader) {
2263 if (glsl_type_is_dual_slot(glsl_without_array(var->type))) {
2264 unsigned slots = glsl_count_attribute_slots(var->type, true);
2265 *dual_slot |= BITFIELD64_MASK(slots) << var->data.location;
2266 }
2267 }
2268
2269 nir_foreach_shader_in_variable(var, shader) {
2270 var->data.location +=
2271 util_bitcount64(*dual_slot & BITFIELD64_MASK(var->data.location));
2272 }
2273 }
2274
2275 /* Returns an attribute mask that has been re-compacted using the given
2276 * dual_slot mask.
2277 */
2278 uint64_t
2279 nir_get_single_slot_attribs_mask(uint64_t attribs, uint64_t dual_slot)
2280 {
2281 while (dual_slot) {
2282 unsigned loc = u_bit_scan64(&dual_slot);
2283 /* mask of all bits up to and including loc */
2284 uint64_t mask = BITFIELD64_MASK(loc + 1);
2285 attribs = (attribs & mask) | ((attribs & ~mask) >> 1);
2286 }
2287 return attribs;
2288 }
2289
2290 void
2291 nir_rewrite_image_intrinsic(nir_intrinsic_instr *intrin, nir_ssa_def *src,
2292 bool bindless)
2293 {
2294 enum gl_access_qualifier access = nir_intrinsic_access(intrin);
2295 nir_alu_type type = nir_type_invalid;
2296 if (nir_intrinsic_infos[intrin->intrinsic].index_map[NIR_INTRINSIC_TYPE])
2297 type = nir_intrinsic_type(intrin);
2298
2299 switch (intrin->intrinsic) {
2300 #define CASE(op) \
2301 case nir_intrinsic_image_deref_##op: \
2302 intrin->intrinsic = bindless ? nir_intrinsic_bindless_image_##op \
2303 : nir_intrinsic_image_##op; \
2304 break;
2305 CASE(load)
2306 CASE(store)
2307 CASE(atomic_add)
2308 CASE(atomic_imin)
2309 CASE(atomic_umin)
2310 CASE(atomic_imax)
2311 CASE(atomic_umax)
2312 CASE(atomic_and)
2313 CASE(atomic_or)
2314 CASE(atomic_xor)
2315 CASE(atomic_exchange)
2316 CASE(atomic_comp_swap)
2317 CASE(atomic_fadd)
2318 CASE(atomic_inc_wrap)
2319 CASE(atomic_dec_wrap)
2320 CASE(size)
2321 CASE(samples)
2322 CASE(load_raw_intel)
2323 CASE(store_raw_intel)
2324 #undef CASE
2325 default:
2326 unreachable("Unhanded image intrinsic");
2327 }
2328
2329 nir_deref_instr *deref = nir_src_as_deref(intrin->src[0]);
2330 nir_variable *var = nir_deref_instr_get_variable(deref);
2331
2332 nir_intrinsic_set_image_dim(intrin, glsl_get_sampler_dim(deref->type));
2333 nir_intrinsic_set_image_array(intrin, glsl_sampler_type_is_array(deref->type));
2334 nir_intrinsic_set_access(intrin, access | var->data.access);
2335 nir_intrinsic_set_format(intrin, var->data.image.format);
2336 if (nir_intrinsic_infos[intrin->intrinsic].index_map[NIR_INTRINSIC_TYPE])
2337 nir_intrinsic_set_type(intrin, type);
2338
2339 nir_instr_rewrite_src(&intrin->instr, &intrin->src[0],
2340 nir_src_for_ssa(src));
2341 }
2342
2343 unsigned
2344 nir_image_intrinsic_coord_components(const nir_intrinsic_instr *instr)
2345 {
2346 enum glsl_sampler_dim dim = nir_intrinsic_image_dim(instr);
2347 int coords = glsl_get_sampler_dim_coordinate_components(dim);
2348 if (dim == GLSL_SAMPLER_DIM_CUBE)
2349 return coords;
2350 else
2351 return coords + nir_intrinsic_image_array(instr);
2352 }