nir/builder: Add load/store_global helpers
[mesa.git] / src / compiler / nir / nir_builder.h
1 /*
2 * Copyright © 2014-2015 Broadcom
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 */
23
24 #ifndef NIR_BUILDER_H
25 #define NIR_BUILDER_H
26
27 #include "nir_control_flow.h"
28 #include "util/bitscan.h"
29 #include "util/half_float.h"
30
31 struct exec_list;
32
33 typedef struct nir_builder {
34 nir_cursor cursor;
35
36 /* Whether new ALU instructions will be marked "exact" */
37 bool exact;
38
39 nir_shader *shader;
40 nir_function_impl *impl;
41 } nir_builder;
42
43 static inline void
44 nir_builder_init(nir_builder *build, nir_function_impl *impl)
45 {
46 memset(build, 0, sizeof(*build));
47 build->exact = false;
48 build->impl = impl;
49 build->shader = impl->function->shader;
50 }
51
52 static inline void
53 nir_builder_init_simple_shader(nir_builder *build, void *mem_ctx,
54 gl_shader_stage stage,
55 const nir_shader_compiler_options *options)
56 {
57 build->shader = nir_shader_create(mem_ctx, stage, options, NULL);
58 nir_function *func = nir_function_create(build->shader, "main");
59 func->is_entrypoint = true;
60 build->exact = false;
61 build->impl = nir_function_impl_create(func);
62 build->cursor = nir_after_cf_list(&build->impl->body);
63 }
64
65 typedef bool (*nir_instr_pass_cb)(struct nir_builder *, nir_instr *, void *);
66
67 /**
68 * Iterates over all the instructions in a NIR shader and calls the given pass
69 * on them.
70 *
71 * The pass should return true if it modified the shader. In that case, only
72 * the preserved metadata flags will be preserved in the function impl.
73 *
74 * The builder will be initialized to point at the function impl, but its
75 * cursor is unset.
76 */
77 static inline bool
78 nir_shader_instructions_pass(nir_shader *shader,
79 nir_instr_pass_cb pass,
80 nir_metadata preserved,
81 void *cb_data)
82 {
83 bool progress = false;
84
85 nir_foreach_function(function, shader) {
86 if (!function->impl)
87 continue;
88
89 nir_builder b;
90 nir_builder_init(&b, function->impl);
91
92 nir_foreach_block_safe(block, function->impl) {
93 nir_foreach_instr_safe(instr, block) {
94 progress |= pass(&b, instr, cb_data);
95 }
96 }
97
98 if (progress) {
99 nir_metadata_preserve(function->impl, preserved);
100 } else {
101 nir_metadata_preserve(function->impl, nir_metadata_all);
102 }
103 }
104
105 return progress;
106 }
107
108 static inline void
109 nir_builder_instr_insert(nir_builder *build, nir_instr *instr)
110 {
111 nir_instr_insert(build->cursor, instr);
112
113 /* Move the cursor forward. */
114 build->cursor = nir_after_instr(instr);
115 }
116
117 static inline nir_instr *
118 nir_builder_last_instr(nir_builder *build)
119 {
120 assert(build->cursor.option == nir_cursor_after_instr);
121 return build->cursor.instr;
122 }
123
124 static inline void
125 nir_builder_cf_insert(nir_builder *build, nir_cf_node *cf)
126 {
127 nir_cf_node_insert(build->cursor, cf);
128 }
129
130 static inline bool
131 nir_builder_is_inside_cf(nir_builder *build, nir_cf_node *cf_node)
132 {
133 nir_block *block = nir_cursor_current_block(build->cursor);
134 for (nir_cf_node *n = &block->cf_node; n; n = n->parent) {
135 if (n == cf_node)
136 return true;
137 }
138 return false;
139 }
140
141 static inline nir_if *
142 nir_push_if_src(nir_builder *build, nir_src condition)
143 {
144 nir_if *nif = nir_if_create(build->shader);
145 nif->condition = condition;
146 nir_builder_cf_insert(build, &nif->cf_node);
147 build->cursor = nir_before_cf_list(&nif->then_list);
148 return nif;
149 }
150
151 static inline nir_if *
152 nir_push_if(nir_builder *build, nir_ssa_def *condition)
153 {
154 return nir_push_if_src(build, nir_src_for_ssa(condition));
155 }
156
157 static inline nir_if *
158 nir_push_else(nir_builder *build, nir_if *nif)
159 {
160 if (nif) {
161 assert(nir_builder_is_inside_cf(build, &nif->cf_node));
162 } else {
163 nir_block *block = nir_cursor_current_block(build->cursor);
164 nif = nir_cf_node_as_if(block->cf_node.parent);
165 }
166 build->cursor = nir_before_cf_list(&nif->else_list);
167 return nif;
168 }
169
170 static inline void
171 nir_pop_if(nir_builder *build, nir_if *nif)
172 {
173 if (nif) {
174 assert(nir_builder_is_inside_cf(build, &nif->cf_node));
175 } else {
176 nir_block *block = nir_cursor_current_block(build->cursor);
177 nif = nir_cf_node_as_if(block->cf_node.parent);
178 }
179 build->cursor = nir_after_cf_node(&nif->cf_node);
180 }
181
182 static inline nir_ssa_def *
183 nir_if_phi(nir_builder *build, nir_ssa_def *then_def, nir_ssa_def *else_def)
184 {
185 nir_block *block = nir_cursor_current_block(build->cursor);
186 nir_if *nif = nir_cf_node_as_if(nir_cf_node_prev(&block->cf_node));
187
188 nir_phi_instr *phi = nir_phi_instr_create(build->shader);
189
190 nir_phi_src *src = ralloc(phi, nir_phi_src);
191 src->pred = nir_if_last_then_block(nif);
192 src->src = nir_src_for_ssa(then_def);
193 exec_list_push_tail(&phi->srcs, &src->node);
194
195 src = ralloc(phi, nir_phi_src);
196 src->pred = nir_if_last_else_block(nif);
197 src->src = nir_src_for_ssa(else_def);
198 exec_list_push_tail(&phi->srcs, &src->node);
199
200 assert(then_def->num_components == else_def->num_components);
201 assert(then_def->bit_size == else_def->bit_size);
202 nir_ssa_dest_init(&phi->instr, &phi->dest,
203 then_def->num_components, then_def->bit_size, NULL);
204
205 nir_builder_instr_insert(build, &phi->instr);
206
207 return &phi->dest.ssa;
208 }
209
210 static inline nir_loop *
211 nir_push_loop(nir_builder *build)
212 {
213 nir_loop *loop = nir_loop_create(build->shader);
214 nir_builder_cf_insert(build, &loop->cf_node);
215 build->cursor = nir_before_cf_list(&loop->body);
216 return loop;
217 }
218
219 static inline void
220 nir_pop_loop(nir_builder *build, nir_loop *loop)
221 {
222 if (loop) {
223 assert(nir_builder_is_inside_cf(build, &loop->cf_node));
224 } else {
225 nir_block *block = nir_cursor_current_block(build->cursor);
226 loop = nir_cf_node_as_loop(block->cf_node.parent);
227 }
228 build->cursor = nir_after_cf_node(&loop->cf_node);
229 }
230
231 static inline nir_ssa_def *
232 nir_ssa_undef(nir_builder *build, unsigned num_components, unsigned bit_size)
233 {
234 nir_ssa_undef_instr *undef =
235 nir_ssa_undef_instr_create(build->shader, num_components, bit_size);
236 if (!undef)
237 return NULL;
238
239 nir_instr_insert(nir_before_cf_list(&build->impl->body), &undef->instr);
240
241 return &undef->def;
242 }
243
244 static inline nir_ssa_def *
245 nir_build_imm(nir_builder *build, unsigned num_components,
246 unsigned bit_size, const nir_const_value *value)
247 {
248 nir_load_const_instr *load_const =
249 nir_load_const_instr_create(build->shader, num_components, bit_size);
250 if (!load_const)
251 return NULL;
252
253 memcpy(load_const->value, value, sizeof(nir_const_value) * num_components);
254
255 nir_builder_instr_insert(build, &load_const->instr);
256
257 return &load_const->def;
258 }
259
260 static inline nir_ssa_def *
261 nir_imm_zero(nir_builder *build, unsigned num_components, unsigned bit_size)
262 {
263 nir_load_const_instr *load_const =
264 nir_load_const_instr_create(build->shader, num_components, bit_size);
265
266 /* nir_load_const_instr_create uses rzalloc so it's already zero */
267
268 nir_builder_instr_insert(build, &load_const->instr);
269
270 return &load_const->def;
271 }
272
273 static inline nir_ssa_def *
274 nir_imm_boolN_t(nir_builder *build, bool x, unsigned bit_size)
275 {
276 nir_const_value v = nir_const_value_for_bool(x, bit_size);
277 return nir_build_imm(build, 1, bit_size, &v);
278 }
279
280 static inline nir_ssa_def *
281 nir_imm_bool(nir_builder *build, bool x)
282 {
283 return nir_imm_boolN_t(build, x, 1);
284 }
285
286 static inline nir_ssa_def *
287 nir_imm_true(nir_builder *build)
288 {
289 return nir_imm_bool(build, true);
290 }
291
292 static inline nir_ssa_def *
293 nir_imm_false(nir_builder *build)
294 {
295 return nir_imm_bool(build, false);
296 }
297
298 static inline nir_ssa_def *
299 nir_imm_floatN_t(nir_builder *build, double x, unsigned bit_size)
300 {
301 nir_const_value v = nir_const_value_for_float(x, bit_size);
302 return nir_build_imm(build, 1, bit_size, &v);
303 }
304
305 static inline nir_ssa_def *
306 nir_imm_float16(nir_builder *build, float x)
307 {
308 return nir_imm_floatN_t(build, x, 16);
309 }
310
311 static inline nir_ssa_def *
312 nir_imm_float(nir_builder *build, float x)
313 {
314 return nir_imm_floatN_t(build, x, 32);
315 }
316
317 static inline nir_ssa_def *
318 nir_imm_double(nir_builder *build, double x)
319 {
320 return nir_imm_floatN_t(build, x, 64);
321 }
322
323 static inline nir_ssa_def *
324 nir_imm_vec2(nir_builder *build, float x, float y)
325 {
326 nir_const_value v[2] = {
327 nir_const_value_for_float(x, 32),
328 nir_const_value_for_float(y, 32),
329 };
330 return nir_build_imm(build, 2, 32, v);
331 }
332
333 static inline nir_ssa_def *
334 nir_imm_vec4(nir_builder *build, float x, float y, float z, float w)
335 {
336 nir_const_value v[4] = {
337 nir_const_value_for_float(x, 32),
338 nir_const_value_for_float(y, 32),
339 nir_const_value_for_float(z, 32),
340 nir_const_value_for_float(w, 32),
341 };
342
343 return nir_build_imm(build, 4, 32, v);
344 }
345
346 static inline nir_ssa_def *
347 nir_imm_vec4_16(nir_builder *build, float x, float y, float z, float w)
348 {
349 nir_const_value v[4] = {
350 nir_const_value_for_float(x, 16),
351 nir_const_value_for_float(y, 16),
352 nir_const_value_for_float(z, 16),
353 nir_const_value_for_float(w, 16),
354 };
355
356 return nir_build_imm(build, 4, 16, v);
357 }
358
359 static inline nir_ssa_def *
360 nir_imm_intN_t(nir_builder *build, uint64_t x, unsigned bit_size)
361 {
362 nir_const_value v = nir_const_value_for_raw_uint(x, bit_size);
363 return nir_build_imm(build, 1, bit_size, &v);
364 }
365
366 static inline nir_ssa_def *
367 nir_imm_int(nir_builder *build, int x)
368 {
369 return nir_imm_intN_t(build, x, 32);
370 }
371
372 static inline nir_ssa_def *
373 nir_imm_int64(nir_builder *build, int64_t x)
374 {
375 return nir_imm_intN_t(build, x, 64);
376 }
377
378 static inline nir_ssa_def *
379 nir_imm_ivec2(nir_builder *build, int x, int y)
380 {
381 nir_const_value v[2] = {
382 nir_const_value_for_int(x, 32),
383 nir_const_value_for_int(y, 32),
384 };
385
386 return nir_build_imm(build, 2, 32, v);
387 }
388
389 static inline nir_ssa_def *
390 nir_imm_ivec4(nir_builder *build, int x, int y, int z, int w)
391 {
392 nir_const_value v[4] = {
393 nir_const_value_for_int(x, 32),
394 nir_const_value_for_int(y, 32),
395 nir_const_value_for_int(z, 32),
396 nir_const_value_for_int(w, 32),
397 };
398
399 return nir_build_imm(build, 4, 32, v);
400 }
401
402 static inline nir_ssa_def *
403 nir_builder_alu_instr_finish_and_insert(nir_builder *build, nir_alu_instr *instr)
404 {
405 const nir_op_info *op_info = &nir_op_infos[instr->op];
406
407 instr->exact = build->exact;
408
409 /* Guess the number of components the destination temporary should have
410 * based on our input sizes, if it's not fixed for the op.
411 */
412 unsigned num_components = op_info->output_size;
413 if (num_components == 0) {
414 for (unsigned i = 0; i < op_info->num_inputs; i++) {
415 if (op_info->input_sizes[i] == 0)
416 num_components = MAX2(num_components,
417 instr->src[i].src.ssa->num_components);
418 }
419 }
420 assert(num_components != 0);
421
422 /* Figure out the bitwidth based on the source bitwidth if the instruction
423 * is variable-width.
424 */
425 unsigned bit_size = nir_alu_type_get_type_size(op_info->output_type);
426 if (bit_size == 0) {
427 for (unsigned i = 0; i < op_info->num_inputs; i++) {
428 unsigned src_bit_size = instr->src[i].src.ssa->bit_size;
429 if (nir_alu_type_get_type_size(op_info->input_types[i]) == 0) {
430 if (bit_size)
431 assert(src_bit_size == bit_size);
432 else
433 bit_size = src_bit_size;
434 } else {
435 assert(src_bit_size ==
436 nir_alu_type_get_type_size(op_info->input_types[i]));
437 }
438 }
439 }
440
441 /* When in doubt, assume 32. */
442 if (bit_size == 0)
443 bit_size = 32;
444
445 /* Make sure we don't swizzle from outside of our source vector (like if a
446 * scalar value was passed into a multiply with a vector).
447 */
448 for (unsigned i = 0; i < op_info->num_inputs; i++) {
449 for (unsigned j = instr->src[i].src.ssa->num_components;
450 j < NIR_MAX_VEC_COMPONENTS; j++) {
451 instr->src[i].swizzle[j] = instr->src[i].src.ssa->num_components - 1;
452 }
453 }
454
455 nir_ssa_dest_init(&instr->instr, &instr->dest.dest, num_components,
456 bit_size, NULL);
457 instr->dest.write_mask = (1 << num_components) - 1;
458
459 nir_builder_instr_insert(build, &instr->instr);
460
461 return &instr->dest.dest.ssa;
462 }
463
464 static inline nir_ssa_def *
465 nir_build_alu(nir_builder *build, nir_op op, nir_ssa_def *src0,
466 nir_ssa_def *src1, nir_ssa_def *src2, nir_ssa_def *src3)
467 {
468 nir_alu_instr *instr = nir_alu_instr_create(build->shader, op);
469 if (!instr)
470 return NULL;
471
472 instr->src[0].src = nir_src_for_ssa(src0);
473 if (src1)
474 instr->src[1].src = nir_src_for_ssa(src1);
475 if (src2)
476 instr->src[2].src = nir_src_for_ssa(src2);
477 if (src3)
478 instr->src[3].src = nir_src_for_ssa(src3);
479
480 return nir_builder_alu_instr_finish_and_insert(build, instr);
481 }
482
483 /* for the couple special cases with more than 4 src args: */
484 static inline nir_ssa_def *
485 nir_build_alu_src_arr(nir_builder *build, nir_op op, nir_ssa_def **srcs)
486 {
487 const nir_op_info *op_info = &nir_op_infos[op];
488 nir_alu_instr *instr = nir_alu_instr_create(build->shader, op);
489 if (!instr)
490 return NULL;
491
492 for (unsigned i = 0; i < op_info->num_inputs; i++)
493 instr->src[i].src = nir_src_for_ssa(srcs[i]);
494
495 return nir_builder_alu_instr_finish_and_insert(build, instr);
496 }
497
498 #include "nir_builder_opcodes.h"
499
500 static inline nir_ssa_def *
501 nir_vec(nir_builder *build, nir_ssa_def **comp, unsigned num_components)
502 {
503 return nir_build_alu_src_arr(build, nir_op_vec(num_components), comp);
504 }
505
506 static inline nir_ssa_def *
507 nir_mov_alu(nir_builder *build, nir_alu_src src, unsigned num_components)
508 {
509 assert(!src.abs && !src.negate);
510 if (src.src.is_ssa && src.src.ssa->num_components == num_components) {
511 bool any_swizzles = false;
512 for (unsigned i = 0; i < num_components; i++) {
513 if (src.swizzle[i] != i)
514 any_swizzles = true;
515 }
516 if (!any_swizzles)
517 return src.src.ssa;
518 }
519
520 nir_alu_instr *mov = nir_alu_instr_create(build->shader, nir_op_mov);
521 nir_ssa_dest_init(&mov->instr, &mov->dest.dest, num_components,
522 nir_src_bit_size(src.src), NULL);
523 mov->exact = build->exact;
524 mov->dest.write_mask = (1 << num_components) - 1;
525 mov->src[0] = src;
526 nir_builder_instr_insert(build, &mov->instr);
527
528 return &mov->dest.dest.ssa;
529 }
530
531 /**
532 * Construct an fmov or imov that reswizzles the source's components.
533 */
534 static inline nir_ssa_def *
535 nir_swizzle(nir_builder *build, nir_ssa_def *src, const unsigned *swiz,
536 unsigned num_components)
537 {
538 assert(num_components <= NIR_MAX_VEC_COMPONENTS);
539 nir_alu_src alu_src = { NIR_SRC_INIT };
540 alu_src.src = nir_src_for_ssa(src);
541
542 bool is_identity_swizzle = true;
543 for (unsigned i = 0; i < num_components && i < NIR_MAX_VEC_COMPONENTS; i++) {
544 if (swiz[i] != i)
545 is_identity_swizzle = false;
546 alu_src.swizzle[i] = swiz[i];
547 }
548
549 if (num_components == src->num_components && is_identity_swizzle)
550 return src;
551
552 return nir_mov_alu(build, alu_src, num_components);
553 }
554
555 /* Selects the right fdot given the number of components in each source. */
556 static inline nir_ssa_def *
557 nir_fdot(nir_builder *build, nir_ssa_def *src0, nir_ssa_def *src1)
558 {
559 assert(src0->num_components == src1->num_components);
560 switch (src0->num_components) {
561 case 1: return nir_fmul(build, src0, src1);
562 case 2: return nir_fdot2(build, src0, src1);
563 case 3: return nir_fdot3(build, src0, src1);
564 case 4: return nir_fdot4(build, src0, src1);
565 case 8: return nir_fdot8(build, src0, src1);
566 case 16: return nir_fdot16(build, src0, src1);
567 default:
568 unreachable("bad component size");
569 }
570
571 return NULL;
572 }
573
574 static inline nir_ssa_def *
575 nir_ball_iequal(nir_builder *b, nir_ssa_def *src0, nir_ssa_def *src1)
576 {
577 switch (src0->num_components) {
578 case 1: return nir_ieq(b, src0, src1);
579 case 2: return nir_ball_iequal2(b, src0, src1);
580 case 3: return nir_ball_iequal3(b, src0, src1);
581 case 4: return nir_ball_iequal4(b, src0, src1);
582 case 8: return nir_ball_iequal8(b, src0, src1);
583 case 16: return nir_ball_iequal16(b, src0, src1);
584 default:
585 unreachable("bad component size");
586 }
587 }
588
589 static inline nir_ssa_def *
590 nir_ball(nir_builder *b, nir_ssa_def *src)
591 {
592 return nir_ball_iequal(b, src, nir_imm_true(b));
593 }
594
595 static inline nir_ssa_def *
596 nir_bany_inequal(nir_builder *b, nir_ssa_def *src0, nir_ssa_def *src1)
597 {
598 switch (src0->num_components) {
599 case 1: return nir_ine(b, src0, src1);
600 case 2: return nir_bany_inequal2(b, src0, src1);
601 case 3: return nir_bany_inequal3(b, src0, src1);
602 case 4: return nir_bany_inequal4(b, src0, src1);
603 case 8: return nir_bany_inequal8(b, src0, src1);
604 case 16: return nir_bany_inequal16(b, src0, src1);
605 default:
606 unreachable("bad component size");
607 }
608 }
609
610 static inline nir_ssa_def *
611 nir_bany(nir_builder *b, nir_ssa_def *src)
612 {
613 return nir_bany_inequal(b, src, nir_imm_false(b));
614 }
615
616 static inline nir_ssa_def *
617 nir_channel(nir_builder *b, nir_ssa_def *def, unsigned c)
618 {
619 return nir_swizzle(b, def, &c, 1);
620 }
621
622 static inline nir_ssa_def *
623 nir_channels(nir_builder *b, nir_ssa_def *def, nir_component_mask_t mask)
624 {
625 unsigned num_channels = 0, swizzle[NIR_MAX_VEC_COMPONENTS] = { 0 };
626
627 for (unsigned i = 0; i < NIR_MAX_VEC_COMPONENTS; i++) {
628 if ((mask & (1 << i)) == 0)
629 continue;
630 swizzle[num_channels++] = i;
631 }
632
633 return nir_swizzle(b, def, swizzle, num_channels);
634 }
635
636 static inline nir_ssa_def *
637 _nir_vector_extract_helper(nir_builder *b, nir_ssa_def *vec, nir_ssa_def *c,
638 unsigned start, unsigned end)
639 {
640 if (start == end - 1) {
641 return nir_channel(b, vec, start);
642 } else {
643 unsigned mid = start + (end - start) / 2;
644 return nir_bcsel(b, nir_ilt(b, c, nir_imm_intN_t(b, mid, c->bit_size)),
645 _nir_vector_extract_helper(b, vec, c, start, mid),
646 _nir_vector_extract_helper(b, vec, c, mid, end));
647 }
648 }
649
650 static inline nir_ssa_def *
651 nir_vector_extract(nir_builder *b, nir_ssa_def *vec, nir_ssa_def *c)
652 {
653 nir_src c_src = nir_src_for_ssa(c);
654 if (nir_src_is_const(c_src)) {
655 uint64_t c_const = nir_src_as_uint(c_src);
656 if (c_const < vec->num_components)
657 return nir_channel(b, vec, c_const);
658 else
659 return nir_ssa_undef(b, 1, vec->bit_size);
660 } else {
661 return _nir_vector_extract_helper(b, vec, c, 0, vec->num_components);
662 }
663 }
664
665 /** Replaces the component of `vec` specified by `c` with `scalar` */
666 static inline nir_ssa_def *
667 nir_vector_insert_imm(nir_builder *b, nir_ssa_def *vec,
668 nir_ssa_def *scalar, unsigned c)
669 {
670 assert(scalar->num_components == 1);
671 assert(c < vec->num_components);
672
673 nir_op vec_op = nir_op_vec(vec->num_components);
674 nir_alu_instr *vec_instr = nir_alu_instr_create(b->shader, vec_op);
675
676 for (unsigned i = 0; i < vec->num_components; i++) {
677 if (i == c) {
678 vec_instr->src[i].src = nir_src_for_ssa(scalar);
679 vec_instr->src[i].swizzle[0] = 0;
680 } else {
681 vec_instr->src[i].src = nir_src_for_ssa(vec);
682 vec_instr->src[i].swizzle[0] = i;
683 }
684 }
685
686 return nir_builder_alu_instr_finish_and_insert(b, vec_instr);
687 }
688
689 /** Replaces the component of `vec` specified by `c` with `scalar` */
690 static inline nir_ssa_def *
691 nir_vector_insert(nir_builder *b, nir_ssa_def *vec, nir_ssa_def *scalar,
692 nir_ssa_def *c)
693 {
694 assert(scalar->num_components == 1);
695 assert(c->num_components == 1);
696
697 nir_src c_src = nir_src_for_ssa(c);
698 if (nir_src_is_const(c_src)) {
699 uint64_t c_const = nir_src_as_uint(c_src);
700 if (c_const < vec->num_components)
701 return nir_vector_insert_imm(b, vec, scalar, c_const);
702 else
703 return vec;
704 } else {
705 nir_const_value per_comp_idx_const[NIR_MAX_VEC_COMPONENTS];
706 for (unsigned i = 0; i < NIR_MAX_VEC_COMPONENTS; i++)
707 per_comp_idx_const[i] = nir_const_value_for_int(i, c->bit_size);
708 nir_ssa_def *per_comp_idx =
709 nir_build_imm(b, vec->num_components,
710 c->bit_size, per_comp_idx_const);
711
712 /* nir_builder will automatically splat out scalars to vectors so an
713 * insert is as simple as "if I'm the channel, replace me with the
714 * scalar."
715 */
716 return nir_bcsel(b, nir_ieq(b, c, per_comp_idx), scalar, vec);
717 }
718 }
719
720 static inline nir_ssa_def *
721 nir_i2i(nir_builder *build, nir_ssa_def *x, unsigned dest_bit_size)
722 {
723 if (x->bit_size == dest_bit_size)
724 return x;
725
726 switch (dest_bit_size) {
727 case 64: return nir_i2i64(build, x);
728 case 32: return nir_i2i32(build, x);
729 case 16: return nir_i2i16(build, x);
730 case 8: return nir_i2i8(build, x);
731 default: unreachable("Invalid bit size");
732 }
733 }
734
735 static inline nir_ssa_def *
736 nir_u2u(nir_builder *build, nir_ssa_def *x, unsigned dest_bit_size)
737 {
738 if (x->bit_size == dest_bit_size)
739 return x;
740
741 switch (dest_bit_size) {
742 case 64: return nir_u2u64(build, x);
743 case 32: return nir_u2u32(build, x);
744 case 16: return nir_u2u16(build, x);
745 case 8: return nir_u2u8(build, x);
746 default: unreachable("Invalid bit size");
747 }
748 }
749
750 static inline nir_ssa_def *
751 nir_iadd_imm(nir_builder *build, nir_ssa_def *x, uint64_t y)
752 {
753 assert(x->bit_size <= 64);
754 y &= BITFIELD64_MASK(x->bit_size);
755
756 if (y == 0) {
757 return x;
758 } else {
759 return nir_iadd(build, x, nir_imm_intN_t(build, y, x->bit_size));
760 }
761 }
762
763 static inline nir_ssa_def *
764 _nir_mul_imm(nir_builder *build, nir_ssa_def *x, uint64_t y, bool amul)
765 {
766 assert(x->bit_size <= 64);
767 y &= BITFIELD64_MASK(x->bit_size);
768
769 if (y == 0) {
770 return nir_imm_intN_t(build, 0, x->bit_size);
771 } else if (y == 1) {
772 return x;
773 } else if (!build->shader->options->lower_bitops &&
774 util_is_power_of_two_or_zero64(y)) {
775 return nir_ishl(build, x, nir_imm_int(build, ffsll(y) - 1));
776 } else if (amul) {
777 return nir_amul(build, x, nir_imm_intN_t(build, y, x->bit_size));
778 } else {
779 return nir_imul(build, x, nir_imm_intN_t(build, y, x->bit_size));
780 }
781 }
782
783 static inline nir_ssa_def *
784 nir_imul_imm(nir_builder *build, nir_ssa_def *x, uint64_t y)
785 {
786 return _nir_mul_imm(build, x, y, false);
787 }
788
789 static inline nir_ssa_def *
790 nir_amul_imm(nir_builder *build, nir_ssa_def *x, uint64_t y)
791 {
792 return _nir_mul_imm(build, x, y, true);
793 }
794
795 static inline nir_ssa_def *
796 nir_fadd_imm(nir_builder *build, nir_ssa_def *x, double y)
797 {
798 return nir_fadd(build, x, nir_imm_floatN_t(build, y, x->bit_size));
799 }
800
801 static inline nir_ssa_def *
802 nir_fmul_imm(nir_builder *build, nir_ssa_def *x, double y)
803 {
804 return nir_fmul(build, x, nir_imm_floatN_t(build, y, x->bit_size));
805 }
806
807 static inline nir_ssa_def *
808 nir_iand_imm(nir_builder *build, nir_ssa_def *x, uint64_t y)
809 {
810 assert(x->bit_size <= 64);
811 y &= BITFIELD64_MASK(x->bit_size);
812
813 if (y == 0) {
814 return nir_imm_intN_t(build, 0, x->bit_size);
815 } else if (y == BITFIELD64_MASK(x->bit_size)) {
816 return x;
817 } else {
818 return nir_iand(build, x, nir_imm_intN_t(build, y, x->bit_size));
819 }
820 }
821
822 static inline nir_ssa_def *
823 nir_ishr_imm(nir_builder *build, nir_ssa_def *x, uint32_t y)
824 {
825 if (y == 0) {
826 return x;
827 } else {
828 return nir_ishr(build, x, nir_imm_int(build, y));
829 }
830 }
831
832 static inline nir_ssa_def *
833 nir_ushr_imm(nir_builder *build, nir_ssa_def *x, uint32_t y)
834 {
835 if (y == 0) {
836 return x;
837 } else {
838 return nir_ushr(build, x, nir_imm_int(build, y));
839 }
840 }
841
842 static inline nir_ssa_def *
843 nir_udiv_imm(nir_builder *build, nir_ssa_def *x, uint64_t y)
844 {
845 assert(x->bit_size <= 64);
846 y &= BITFIELD64_MASK(x->bit_size);
847
848 if (y == 1) {
849 return x;
850 } else if (util_is_power_of_two_nonzero(y)) {
851 return nir_ushr_imm(build, x, ffsll(y) - 1);
852 } else {
853 return nir_udiv(build, x, nir_imm_intN_t(build, y, x->bit_size));
854 }
855 }
856
857 static inline nir_ssa_def *
858 nir_pack_bits(nir_builder *b, nir_ssa_def *src, unsigned dest_bit_size)
859 {
860 assert(src->num_components * src->bit_size == dest_bit_size);
861
862 switch (dest_bit_size) {
863 case 64:
864 switch (src->bit_size) {
865 case 32: return nir_pack_64_2x32(b, src);
866 case 16: return nir_pack_64_4x16(b, src);
867 default: break;
868 }
869 break;
870
871 case 32:
872 if (src->bit_size == 16)
873 return nir_pack_32_2x16(b, src);
874 break;
875
876 default:
877 break;
878 }
879
880 /* If we got here, we have no dedicated unpack opcode. */
881 nir_ssa_def *dest = nir_imm_intN_t(b, 0, dest_bit_size);
882 for (unsigned i = 0; i < src->num_components; i++) {
883 nir_ssa_def *val = nir_u2u(b, nir_channel(b, src, i), dest_bit_size);
884 val = nir_ishl(b, val, nir_imm_int(b, i * src->bit_size));
885 dest = nir_ior(b, dest, val);
886 }
887 return dest;
888 }
889
890 static inline nir_ssa_def *
891 nir_unpack_bits(nir_builder *b, nir_ssa_def *src, unsigned dest_bit_size)
892 {
893 assert(src->num_components == 1);
894 assert(src->bit_size > dest_bit_size);
895 const unsigned dest_num_components = src->bit_size / dest_bit_size;
896 assert(dest_num_components <= NIR_MAX_VEC_COMPONENTS);
897
898 switch (src->bit_size) {
899 case 64:
900 switch (dest_bit_size) {
901 case 32: return nir_unpack_64_2x32(b, src);
902 case 16: return nir_unpack_64_4x16(b, src);
903 default: break;
904 }
905 break;
906
907 case 32:
908 if (dest_bit_size == 16)
909 return nir_unpack_32_2x16(b, src);
910 break;
911
912 default:
913 break;
914 }
915
916 /* If we got here, we have no dedicated unpack opcode. */
917 nir_ssa_def *dest_comps[NIR_MAX_VEC_COMPONENTS];
918 for (unsigned i = 0; i < dest_num_components; i++) {
919 nir_ssa_def *val = nir_ushr_imm(b, src, i * dest_bit_size);
920 dest_comps[i] = nir_u2u(b, val, dest_bit_size);
921 }
922 return nir_vec(b, dest_comps, dest_num_components);
923 }
924
925 /**
926 * Treats srcs as if it's one big blob of bits and extracts the range of bits
927 * given by
928 *
929 * [first_bit, first_bit + dest_num_components * dest_bit_size)
930 *
931 * The range can have any alignment or size as long as it's an integer number
932 * of destination components and fits inside the concatenated sources.
933 *
934 * TODO: The one caveat here is that we can't handle byte alignment if 64-bit
935 * values are involved because that would require pack/unpack to/from a vec8
936 * which NIR currently does not support.
937 */
938 static inline nir_ssa_def *
939 nir_extract_bits(nir_builder *b, nir_ssa_def **srcs, unsigned num_srcs,
940 unsigned first_bit,
941 unsigned dest_num_components, unsigned dest_bit_size)
942 {
943 const unsigned num_bits = dest_num_components * dest_bit_size;
944
945 /* Figure out the common bit size */
946 unsigned common_bit_size = dest_bit_size;
947 for (unsigned i = 0; i < num_srcs; i++)
948 common_bit_size = MIN2(common_bit_size, srcs[i]->bit_size);
949 if (first_bit > 0)
950 common_bit_size = MIN2(common_bit_size, (1u << (ffs(first_bit) - 1)));
951
952 /* We don't want to have to deal with 1-bit values */
953 assert(common_bit_size >= 8);
954
955 nir_ssa_def *common_comps[NIR_MAX_VEC_COMPONENTS * sizeof(uint64_t)];
956 assert(num_bits / common_bit_size <= ARRAY_SIZE(common_comps));
957
958 /* First, unpack to the common bit size and select the components from the
959 * source.
960 */
961 int src_idx = -1;
962 unsigned src_start_bit = 0;
963 unsigned src_end_bit = 0;
964 for (unsigned i = 0; i < num_bits / common_bit_size; i++) {
965 const unsigned bit = first_bit + (i * common_bit_size);
966 while (bit >= src_end_bit) {
967 src_idx++;
968 assert(src_idx < (int) num_srcs);
969 src_start_bit = src_end_bit;
970 src_end_bit += srcs[src_idx]->bit_size *
971 srcs[src_idx]->num_components;
972 }
973 assert(bit >= src_start_bit);
974 assert(bit + common_bit_size <= src_end_bit);
975 const unsigned rel_bit = bit - src_start_bit;
976 const unsigned src_bit_size = srcs[src_idx]->bit_size;
977
978 nir_ssa_def *comp = nir_channel(b, srcs[src_idx],
979 rel_bit / src_bit_size);
980 if (srcs[src_idx]->bit_size > common_bit_size) {
981 nir_ssa_def *unpacked = nir_unpack_bits(b, comp, common_bit_size);
982 comp = nir_channel(b, unpacked, (rel_bit % src_bit_size) /
983 common_bit_size);
984 }
985 common_comps[i] = comp;
986 }
987
988 /* Now, re-pack the destination if we have to */
989 if (dest_bit_size > common_bit_size) {
990 unsigned common_per_dest = dest_bit_size / common_bit_size;
991 nir_ssa_def *dest_comps[NIR_MAX_VEC_COMPONENTS];
992 for (unsigned i = 0; i < dest_num_components; i++) {
993 nir_ssa_def *unpacked = nir_vec(b, common_comps + i * common_per_dest,
994 common_per_dest);
995 dest_comps[i] = nir_pack_bits(b, unpacked, dest_bit_size);
996 }
997 return nir_vec(b, dest_comps, dest_num_components);
998 } else {
999 assert(dest_bit_size == common_bit_size);
1000 return nir_vec(b, common_comps, dest_num_components);
1001 }
1002 }
1003
1004 static inline nir_ssa_def *
1005 nir_bitcast_vector(nir_builder *b, nir_ssa_def *src, unsigned dest_bit_size)
1006 {
1007 assert((src->bit_size * src->num_components) % dest_bit_size == 0);
1008 const unsigned dest_num_components =
1009 (src->bit_size * src->num_components) / dest_bit_size;
1010 assert(dest_num_components <= NIR_MAX_VEC_COMPONENTS);
1011
1012 return nir_extract_bits(b, &src, 1, 0, dest_num_components, dest_bit_size);
1013 }
1014
1015 /**
1016 * Turns a nir_src into a nir_ssa_def * so it can be passed to
1017 * nir_build_alu()-based builder calls.
1018 *
1019 * See nir_ssa_for_alu_src() for alu instructions.
1020 */
1021 static inline nir_ssa_def *
1022 nir_ssa_for_src(nir_builder *build, nir_src src, int num_components)
1023 {
1024 if (src.is_ssa && src.ssa->num_components == num_components)
1025 return src.ssa;
1026
1027 nir_alu_src alu = { NIR_SRC_INIT };
1028 alu.src = src;
1029 for (int j = 0; j < 4; j++)
1030 alu.swizzle[j] = j;
1031
1032 return nir_mov_alu(build, alu, num_components);
1033 }
1034
1035 /**
1036 * Similar to nir_ssa_for_src(), but for alu srcs, respecting the
1037 * nir_alu_src's swizzle.
1038 */
1039 static inline nir_ssa_def *
1040 nir_ssa_for_alu_src(nir_builder *build, nir_alu_instr *instr, unsigned srcn)
1041 {
1042 static uint8_t trivial_swizzle[] = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 };
1043 STATIC_ASSERT(ARRAY_SIZE(trivial_swizzle) == NIR_MAX_VEC_COMPONENTS);
1044
1045 nir_alu_src *src = &instr->src[srcn];
1046 unsigned num_components = nir_ssa_alu_instr_src_components(instr, srcn);
1047
1048 if (src->src.is_ssa && (src->src.ssa->num_components == num_components) &&
1049 !src->abs && !src->negate &&
1050 (memcmp(src->swizzle, trivial_swizzle, num_components) == 0))
1051 return src->src.ssa;
1052
1053 return nir_mov_alu(build, *src, num_components);
1054 }
1055
1056 static inline unsigned
1057 nir_get_ptr_bitsize(nir_shader *shader)
1058 {
1059 if (shader->info.stage == MESA_SHADER_KERNEL)
1060 return shader->info.cs.ptr_size;
1061 return 32;
1062 }
1063
1064 static inline nir_deref_instr *
1065 nir_build_deref_var(nir_builder *build, nir_variable *var)
1066 {
1067 nir_deref_instr *deref =
1068 nir_deref_instr_create(build->shader, nir_deref_type_var);
1069
1070 deref->mode = (nir_variable_mode)var->data.mode;
1071 deref->type = var->type;
1072 deref->var = var;
1073
1074 nir_ssa_dest_init(&deref->instr, &deref->dest, 1,
1075 nir_get_ptr_bitsize(build->shader), NULL);
1076
1077 nir_builder_instr_insert(build, &deref->instr);
1078
1079 return deref;
1080 }
1081
1082 static inline nir_deref_instr *
1083 nir_build_deref_array(nir_builder *build, nir_deref_instr *parent,
1084 nir_ssa_def *index)
1085 {
1086 assert(glsl_type_is_array(parent->type) ||
1087 glsl_type_is_matrix(parent->type) ||
1088 glsl_type_is_vector(parent->type));
1089
1090 assert(index->bit_size == parent->dest.ssa.bit_size);
1091
1092 nir_deref_instr *deref =
1093 nir_deref_instr_create(build->shader, nir_deref_type_array);
1094
1095 deref->mode = parent->mode;
1096 deref->type = glsl_get_array_element(parent->type);
1097 deref->parent = nir_src_for_ssa(&parent->dest.ssa);
1098 deref->arr.index = nir_src_for_ssa(index);
1099
1100 nir_ssa_dest_init(&deref->instr, &deref->dest,
1101 parent->dest.ssa.num_components,
1102 parent->dest.ssa.bit_size, NULL);
1103
1104 nir_builder_instr_insert(build, &deref->instr);
1105
1106 return deref;
1107 }
1108
1109 static inline nir_deref_instr *
1110 nir_build_deref_array_imm(nir_builder *build, nir_deref_instr *parent,
1111 int64_t index)
1112 {
1113 assert(parent->dest.is_ssa);
1114 nir_ssa_def *idx_ssa = nir_imm_intN_t(build, index,
1115 parent->dest.ssa.bit_size);
1116
1117 return nir_build_deref_array(build, parent, idx_ssa);
1118 }
1119
1120 static inline nir_deref_instr *
1121 nir_build_deref_ptr_as_array(nir_builder *build, nir_deref_instr *parent,
1122 nir_ssa_def *index)
1123 {
1124 assert(parent->deref_type == nir_deref_type_array ||
1125 parent->deref_type == nir_deref_type_ptr_as_array ||
1126 parent->deref_type == nir_deref_type_cast);
1127
1128 assert(index->bit_size == parent->dest.ssa.bit_size);
1129
1130 nir_deref_instr *deref =
1131 nir_deref_instr_create(build->shader, nir_deref_type_ptr_as_array);
1132
1133 deref->mode = parent->mode;
1134 deref->type = parent->type;
1135 deref->parent = nir_src_for_ssa(&parent->dest.ssa);
1136 deref->arr.index = nir_src_for_ssa(index);
1137
1138 nir_ssa_dest_init(&deref->instr, &deref->dest,
1139 parent->dest.ssa.num_components,
1140 parent->dest.ssa.bit_size, NULL);
1141
1142 nir_builder_instr_insert(build, &deref->instr);
1143
1144 return deref;
1145 }
1146
1147 static inline nir_deref_instr *
1148 nir_build_deref_array_wildcard(nir_builder *build, nir_deref_instr *parent)
1149 {
1150 assert(glsl_type_is_array(parent->type) ||
1151 glsl_type_is_matrix(parent->type));
1152
1153 nir_deref_instr *deref =
1154 nir_deref_instr_create(build->shader, nir_deref_type_array_wildcard);
1155
1156 deref->mode = parent->mode;
1157 deref->type = glsl_get_array_element(parent->type);
1158 deref->parent = nir_src_for_ssa(&parent->dest.ssa);
1159
1160 nir_ssa_dest_init(&deref->instr, &deref->dest,
1161 parent->dest.ssa.num_components,
1162 parent->dest.ssa.bit_size, NULL);
1163
1164 nir_builder_instr_insert(build, &deref->instr);
1165
1166 return deref;
1167 }
1168
1169 static inline nir_deref_instr *
1170 nir_build_deref_struct(nir_builder *build, nir_deref_instr *parent,
1171 unsigned index)
1172 {
1173 assert(glsl_type_is_struct_or_ifc(parent->type));
1174
1175 nir_deref_instr *deref =
1176 nir_deref_instr_create(build->shader, nir_deref_type_struct);
1177
1178 deref->mode = parent->mode;
1179 deref->type = glsl_get_struct_field(parent->type, index);
1180 deref->parent = nir_src_for_ssa(&parent->dest.ssa);
1181 deref->strct.index = index;
1182
1183 nir_ssa_dest_init(&deref->instr, &deref->dest,
1184 parent->dest.ssa.num_components,
1185 parent->dest.ssa.bit_size, NULL);
1186
1187 nir_builder_instr_insert(build, &deref->instr);
1188
1189 return deref;
1190 }
1191
1192 static inline nir_deref_instr *
1193 nir_build_deref_cast(nir_builder *build, nir_ssa_def *parent,
1194 nir_variable_mode mode, const struct glsl_type *type,
1195 unsigned ptr_stride)
1196 {
1197 nir_deref_instr *deref =
1198 nir_deref_instr_create(build->shader, nir_deref_type_cast);
1199
1200 deref->mode = mode;
1201 deref->type = type;
1202 deref->parent = nir_src_for_ssa(parent);
1203 deref->cast.ptr_stride = ptr_stride;
1204
1205 nir_ssa_dest_init(&deref->instr, &deref->dest,
1206 parent->num_components, parent->bit_size, NULL);
1207
1208 nir_builder_instr_insert(build, &deref->instr);
1209
1210 return deref;
1211 }
1212
1213 /** Returns a deref that follows another but starting from the given parent
1214 *
1215 * The new deref will be the same type and take the same array or struct index
1216 * as the leader deref but it may have a different parent. This is very
1217 * useful for walking deref paths.
1218 */
1219 static inline nir_deref_instr *
1220 nir_build_deref_follower(nir_builder *b, nir_deref_instr *parent,
1221 nir_deref_instr *leader)
1222 {
1223 /* If the derefs would have the same parent, don't make a new one */
1224 assert(leader->parent.is_ssa);
1225 if (leader->parent.ssa == &parent->dest.ssa)
1226 return leader;
1227
1228 UNUSED nir_deref_instr *leader_parent = nir_src_as_deref(leader->parent);
1229
1230 switch (leader->deref_type) {
1231 case nir_deref_type_var:
1232 unreachable("A var dereference cannot have a parent");
1233 break;
1234
1235 case nir_deref_type_array:
1236 case nir_deref_type_array_wildcard:
1237 assert(glsl_type_is_matrix(parent->type) ||
1238 glsl_type_is_array(parent->type) ||
1239 (leader->deref_type == nir_deref_type_array &&
1240 glsl_type_is_vector(parent->type)));
1241 assert(glsl_get_length(parent->type) ==
1242 glsl_get_length(leader_parent->type));
1243
1244 if (leader->deref_type == nir_deref_type_array) {
1245 assert(leader->arr.index.is_ssa);
1246 nir_ssa_def *index = nir_i2i(b, leader->arr.index.ssa,
1247 parent->dest.ssa.bit_size);
1248 return nir_build_deref_array(b, parent, index);
1249 } else {
1250 return nir_build_deref_array_wildcard(b, parent);
1251 }
1252
1253 case nir_deref_type_struct:
1254 assert(glsl_type_is_struct_or_ifc(parent->type));
1255 assert(glsl_get_length(parent->type) ==
1256 glsl_get_length(leader_parent->type));
1257
1258 return nir_build_deref_struct(b, parent, leader->strct.index);
1259
1260 default:
1261 unreachable("Invalid deref instruction type");
1262 }
1263 }
1264
1265 static inline nir_ssa_def *
1266 nir_load_reg(nir_builder *build, nir_register *reg)
1267 {
1268 return nir_ssa_for_src(build, nir_src_for_reg(reg), reg->num_components);
1269 }
1270
1271 static inline void
1272 nir_store_reg(nir_builder *build, nir_register *reg,
1273 nir_ssa_def *def, nir_component_mask_t write_mask)
1274 {
1275 assert(reg->num_components == def->num_components);
1276 assert(reg->bit_size == def->bit_size);
1277
1278 nir_alu_instr *mov = nir_alu_instr_create(build->shader, nir_op_mov);
1279 mov->src[0].src = nir_src_for_ssa(def);
1280 mov->dest.dest = nir_dest_for_reg(reg);
1281 mov->dest.write_mask = write_mask & BITFIELD_MASK(reg->num_components);
1282 nir_builder_instr_insert(build, &mov->instr);
1283 }
1284
1285 static inline nir_ssa_def *
1286 nir_load_deref_with_access(nir_builder *build, nir_deref_instr *deref,
1287 enum gl_access_qualifier access)
1288 {
1289 nir_intrinsic_instr *load =
1290 nir_intrinsic_instr_create(build->shader, nir_intrinsic_load_deref);
1291 load->num_components = glsl_get_vector_elements(deref->type);
1292 load->src[0] = nir_src_for_ssa(&deref->dest.ssa);
1293 nir_ssa_dest_init(&load->instr, &load->dest, load->num_components,
1294 glsl_get_bit_size(deref->type), NULL);
1295 nir_intrinsic_set_access(load, access);
1296 nir_builder_instr_insert(build, &load->instr);
1297 return &load->dest.ssa;
1298 }
1299
1300 static inline nir_ssa_def *
1301 nir_load_deref(nir_builder *build, nir_deref_instr *deref)
1302 {
1303 return nir_load_deref_with_access(build, deref, (enum gl_access_qualifier)0);
1304 }
1305
1306 static inline void
1307 nir_store_deref_with_access(nir_builder *build, nir_deref_instr *deref,
1308 nir_ssa_def *value, unsigned writemask,
1309 enum gl_access_qualifier access)
1310 {
1311 nir_intrinsic_instr *store =
1312 nir_intrinsic_instr_create(build->shader, nir_intrinsic_store_deref);
1313 store->num_components = glsl_get_vector_elements(deref->type);
1314 store->src[0] = nir_src_for_ssa(&deref->dest.ssa);
1315 store->src[1] = nir_src_for_ssa(value);
1316 nir_intrinsic_set_write_mask(store,
1317 writemask & ((1 << store->num_components) - 1));
1318 nir_intrinsic_set_access(store, access);
1319 nir_builder_instr_insert(build, &store->instr);
1320 }
1321
1322 static inline void
1323 nir_store_deref(nir_builder *build, nir_deref_instr *deref,
1324 nir_ssa_def *value, unsigned writemask)
1325 {
1326 nir_store_deref_with_access(build, deref, value, writemask,
1327 (enum gl_access_qualifier)0);
1328 }
1329
1330 static inline void
1331 nir_copy_deref_with_access(nir_builder *build, nir_deref_instr *dest,
1332 nir_deref_instr *src,
1333 enum gl_access_qualifier dest_access,
1334 enum gl_access_qualifier src_access)
1335 {
1336 nir_intrinsic_instr *copy =
1337 nir_intrinsic_instr_create(build->shader, nir_intrinsic_copy_deref);
1338 copy->src[0] = nir_src_for_ssa(&dest->dest.ssa);
1339 copy->src[1] = nir_src_for_ssa(&src->dest.ssa);
1340 nir_intrinsic_set_dst_access(copy, dest_access);
1341 nir_intrinsic_set_src_access(copy, src_access);
1342 nir_builder_instr_insert(build, &copy->instr);
1343 }
1344
1345 static inline void
1346 nir_copy_deref(nir_builder *build, nir_deref_instr *dest, nir_deref_instr *src)
1347 {
1348 nir_copy_deref_with_access(build, dest, src,
1349 (enum gl_access_qualifier) 0,
1350 (enum gl_access_qualifier) 0);
1351 }
1352
1353 static inline nir_ssa_def *
1354 nir_load_var(nir_builder *build, nir_variable *var)
1355 {
1356 return nir_load_deref(build, nir_build_deref_var(build, var));
1357 }
1358
1359 static inline void
1360 nir_store_var(nir_builder *build, nir_variable *var, nir_ssa_def *value,
1361 unsigned writemask)
1362 {
1363 nir_store_deref(build, nir_build_deref_var(build, var), value, writemask);
1364 }
1365
1366 static inline void
1367 nir_copy_var(nir_builder *build, nir_variable *dest, nir_variable *src)
1368 {
1369 nir_copy_deref(build, nir_build_deref_var(build, dest),
1370 nir_build_deref_var(build, src));
1371 }
1372
1373 static inline nir_ssa_def *
1374 nir_load_global(nir_builder *build, nir_ssa_def *addr, unsigned align,
1375 unsigned num_components, unsigned bit_size)
1376 {
1377 nir_intrinsic_instr *load =
1378 nir_intrinsic_instr_create(build->shader, nir_intrinsic_load_global);
1379 load->num_components = num_components;
1380 load->src[0] = nir_src_for_ssa(addr);
1381 nir_intrinsic_set_align(load, align, 0);
1382 nir_ssa_dest_init(&load->instr, &load->dest,
1383 num_components, bit_size, NULL);
1384 nir_builder_instr_insert(build, &load->instr);
1385 return &load->dest.ssa;
1386 }
1387
1388 static inline void
1389 nir_store_global(nir_builder *build, nir_ssa_def *addr, unsigned align,
1390 nir_ssa_def *value, nir_component_mask_t write_mask)
1391 {
1392 nir_intrinsic_instr *store =
1393 nir_intrinsic_instr_create(build->shader, nir_intrinsic_store_global);
1394 store->num_components = value->num_components;
1395 store->src[0] = nir_src_for_ssa(value);
1396 store->src[1] = nir_src_for_ssa(addr);
1397 nir_intrinsic_set_write_mask(store,
1398 write_mask & BITFIELD_MASK(value->num_components));
1399 nir_intrinsic_set_align(store, align, 0);
1400 nir_builder_instr_insert(build, &store->instr);
1401 }
1402
1403 static inline nir_ssa_def *
1404 nir_load_param(nir_builder *build, uint32_t param_idx)
1405 {
1406 assert(param_idx < build->impl->function->num_params);
1407 nir_parameter *param = &build->impl->function->params[param_idx];
1408
1409 nir_intrinsic_instr *load =
1410 nir_intrinsic_instr_create(build->shader, nir_intrinsic_load_param);
1411 nir_intrinsic_set_param_idx(load, param_idx);
1412 load->num_components = param->num_components;
1413 nir_ssa_dest_init(&load->instr, &load->dest,
1414 param->num_components, param->bit_size, NULL);
1415 nir_builder_instr_insert(build, &load->instr);
1416 return &load->dest.ssa;
1417 }
1418
1419 static inline nir_ssa_def *
1420 nir_load_reloc_const_intel(nir_builder *b, uint32_t id)
1421 {
1422 nir_intrinsic_instr *load =
1423 nir_intrinsic_instr_create(b->shader,
1424 nir_intrinsic_load_reloc_const_intel);
1425 nir_intrinsic_set_param_idx(load, id);
1426 nir_ssa_dest_init(&load->instr, &load->dest, 1, 32, NULL);
1427 nir_builder_instr_insert(b, &load->instr);
1428 return &load->dest.ssa;
1429 }
1430
1431 #include "nir_builder_opcodes.h"
1432
1433 static inline nir_ssa_def *
1434 nir_f2b(nir_builder *build, nir_ssa_def *f)
1435 {
1436 return nir_f2b1(build, f);
1437 }
1438
1439 static inline nir_ssa_def *
1440 nir_i2b(nir_builder *build, nir_ssa_def *i)
1441 {
1442 return nir_i2b1(build, i);
1443 }
1444
1445 static inline nir_ssa_def *
1446 nir_b2f(nir_builder *build, nir_ssa_def *b, uint32_t bit_size)
1447 {
1448 switch (bit_size) {
1449 case 64: return nir_b2f64(build, b);
1450 case 32: return nir_b2f32(build, b);
1451 case 16: return nir_b2f16(build, b);
1452 default:
1453 unreachable("Invalid bit-size");
1454 };
1455 }
1456
1457 static inline nir_ssa_def *
1458 nir_b2i(nir_builder *build, nir_ssa_def *b, uint32_t bit_size)
1459 {
1460 switch (bit_size) {
1461 case 64: return nir_b2i64(build, b);
1462 case 32: return nir_b2i32(build, b);
1463 case 16: return nir_b2i16(build, b);
1464 case 8: return nir_b2i8(build, b);
1465 default:
1466 unreachable("Invalid bit-size");
1467 };
1468 }
1469 static inline nir_ssa_def *
1470 nir_load_barycentric(nir_builder *build, nir_intrinsic_op op,
1471 unsigned interp_mode)
1472 {
1473 unsigned num_components = op == nir_intrinsic_load_barycentric_model ? 3 : 2;
1474 nir_intrinsic_instr *bary = nir_intrinsic_instr_create(build->shader, op);
1475 nir_ssa_dest_init(&bary->instr, &bary->dest, num_components, 32, NULL);
1476 nir_intrinsic_set_interp_mode(bary, interp_mode);
1477 nir_builder_instr_insert(build, &bary->instr);
1478 return &bary->dest.ssa;
1479 }
1480
1481 static inline void
1482 nir_jump(nir_builder *build, nir_jump_type jump_type)
1483 {
1484 assert(jump_type != nir_jump_goto && jump_type != nir_jump_goto_if);
1485 nir_jump_instr *jump = nir_jump_instr_create(build->shader, jump_type);
1486 nir_builder_instr_insert(build, &jump->instr);
1487 }
1488
1489 static inline void
1490 nir_goto(nir_builder *build, struct nir_block *target)
1491 {
1492 assert(!build->impl->structured);
1493 nir_jump_instr *jump = nir_jump_instr_create(build->shader, nir_jump_goto);
1494 jump->target = target;
1495 nir_builder_instr_insert(build, &jump->instr);
1496 }
1497
1498 static inline void
1499 nir_goto_if(nir_builder *build, struct nir_block *target, nir_src cond,
1500 struct nir_block *else_target)
1501 {
1502 assert(!build->impl->structured);
1503 nir_jump_instr *jump = nir_jump_instr_create(build->shader, nir_jump_goto_if);
1504 jump->condition = cond;
1505 jump->target = target;
1506 jump->else_target = else_target;
1507 nir_builder_instr_insert(build, &jump->instr);
1508 }
1509
1510 static inline nir_ssa_def *
1511 nir_compare_func(nir_builder *b, enum compare_func func,
1512 nir_ssa_def *src0, nir_ssa_def *src1)
1513 {
1514 switch (func) {
1515 case COMPARE_FUNC_NEVER:
1516 return nir_imm_int(b, 0);
1517 case COMPARE_FUNC_ALWAYS:
1518 return nir_imm_int(b, ~0);
1519 case COMPARE_FUNC_EQUAL:
1520 return nir_feq(b, src0, src1);
1521 case COMPARE_FUNC_NOTEQUAL:
1522 return nir_fneu(b, src0, src1);
1523 case COMPARE_FUNC_GREATER:
1524 return nir_flt(b, src1, src0);
1525 case COMPARE_FUNC_GEQUAL:
1526 return nir_fge(b, src0, src1);
1527 case COMPARE_FUNC_LESS:
1528 return nir_flt(b, src0, src1);
1529 case COMPARE_FUNC_LEQUAL:
1530 return nir_fge(b, src1, src0);
1531 }
1532 unreachable("bad compare func");
1533 }
1534
1535 static inline void
1536 nir_scoped_barrier(nir_builder *b,
1537 nir_scope exec_scope,
1538 nir_scope mem_scope,
1539 nir_memory_semantics mem_semantics,
1540 nir_variable_mode mem_modes)
1541 {
1542 nir_intrinsic_instr *intrin =
1543 nir_intrinsic_instr_create(b->shader, nir_intrinsic_scoped_barrier);
1544 nir_intrinsic_set_execution_scope(intrin, exec_scope);
1545 nir_intrinsic_set_memory_scope(intrin, mem_scope);
1546 nir_intrinsic_set_memory_semantics(intrin, mem_semantics);
1547 nir_intrinsic_set_memory_modes(intrin, mem_modes);
1548 nir_builder_instr_insert(b, &intrin->instr);
1549 }
1550
1551 static inline void
1552 nir_scoped_memory_barrier(nir_builder *b,
1553 nir_scope scope,
1554 nir_memory_semantics semantics,
1555 nir_variable_mode modes)
1556 {
1557 nir_scoped_barrier(b, NIR_SCOPE_NONE, scope, semantics, modes);
1558 }
1559
1560 static inline nir_ssa_def *
1561 nir_convert_to_bit_size(nir_builder *b,
1562 nir_ssa_def *src,
1563 nir_alu_type type,
1564 unsigned bit_size)
1565 {
1566 nir_alu_type base_type = nir_alu_type_get_base_type(type);
1567 nir_alu_type dst_type = (nir_alu_type)(bit_size | base_type);
1568
1569 nir_op opcode =
1570 nir_type_conversion_op(type, dst_type, nir_rounding_mode_undef);
1571
1572 return nir_build_alu(b, opcode, src, NULL, NULL, NULL);
1573 }
1574
1575 static inline nir_ssa_def *
1576 nir_i2iN(nir_builder *b, nir_ssa_def *src, unsigned bit_size)
1577 {
1578 return nir_convert_to_bit_size(b, src, nir_type_int, bit_size);
1579 }
1580
1581 static inline nir_ssa_def *
1582 nir_u2uN(nir_builder *b, nir_ssa_def *src, unsigned bit_size)
1583 {
1584 return nir_convert_to_bit_size(b, src, nir_type_uint, bit_size);
1585 }
1586
1587 static inline nir_ssa_def *
1588 nir_b2bN(nir_builder *b, nir_ssa_def *src, unsigned bit_size)
1589 {
1590 return nir_convert_to_bit_size(b, src, nir_type_bool, bit_size);
1591 }
1592
1593 static inline nir_ssa_def *
1594 nir_f2fN(nir_builder *b, nir_ssa_def *src, unsigned bit_size)
1595 {
1596 return nir_convert_to_bit_size(b, src, nir_type_float, bit_size);
1597 }
1598
1599 #endif /* NIR_BUILDER_H */