2 * Copyright © 2014-2015 Broadcom
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
27 #include "nir_control_flow.h"
28 #include "util/half_float.h"
32 typedef struct nir_builder
{
35 /* Whether new ALU instructions will be marked "exact" */
39 nir_function_impl
*impl
;
43 nir_builder_init(nir_builder
*build
, nir_function_impl
*impl
)
45 memset(build
, 0, sizeof(*build
));
48 build
->shader
= impl
->function
->shader
;
52 nir_builder_init_simple_shader(nir_builder
*build
, void *mem_ctx
,
53 gl_shader_stage stage
,
54 const nir_shader_compiler_options
*options
)
56 build
->shader
= nir_shader_create(mem_ctx
, stage
, options
, NULL
);
57 nir_function
*func
= nir_function_create(build
->shader
, "main");
58 func
->is_entrypoint
= true;
60 build
->impl
= nir_function_impl_create(func
);
61 build
->cursor
= nir_after_cf_list(&build
->impl
->body
);
65 nir_builder_instr_insert(nir_builder
*build
, nir_instr
*instr
)
67 nir_instr_insert(build
->cursor
, instr
);
69 /* Move the cursor forward. */
70 build
->cursor
= nir_after_instr(instr
);
73 static inline nir_instr
*
74 nir_builder_last_instr(nir_builder
*build
)
76 assert(build
->cursor
.option
== nir_cursor_after_instr
);
77 return build
->cursor
.instr
;
81 nir_builder_cf_insert(nir_builder
*build
, nir_cf_node
*cf
)
83 nir_cf_node_insert(build
->cursor
, cf
);
87 nir_builder_is_inside_cf(nir_builder
*build
, nir_cf_node
*cf_node
)
89 nir_block
*block
= nir_cursor_current_block(build
->cursor
);
90 for (nir_cf_node
*n
= &block
->cf_node
; n
; n
= n
->parent
) {
97 static inline nir_if
*
98 nir_push_if(nir_builder
*build
, nir_ssa_def
*condition
)
100 nir_if
*nif
= nir_if_create(build
->shader
);
101 nif
->condition
= nir_src_for_ssa(condition
);
102 nir_builder_cf_insert(build
, &nif
->cf_node
);
103 build
->cursor
= nir_before_cf_list(&nif
->then_list
);
107 static inline nir_if
*
108 nir_push_else(nir_builder
*build
, nir_if
*nif
)
111 assert(nir_builder_is_inside_cf(build
, &nif
->cf_node
));
113 nir_block
*block
= nir_cursor_current_block(build
->cursor
);
114 nif
= nir_cf_node_as_if(block
->cf_node
.parent
);
116 build
->cursor
= nir_before_cf_list(&nif
->else_list
);
121 nir_pop_if(nir_builder
*build
, nir_if
*nif
)
124 assert(nir_builder_is_inside_cf(build
, &nif
->cf_node
));
126 nir_block
*block
= nir_cursor_current_block(build
->cursor
);
127 nif
= nir_cf_node_as_if(block
->cf_node
.parent
);
129 build
->cursor
= nir_after_cf_node(&nif
->cf_node
);
132 static inline nir_ssa_def
*
133 nir_if_phi(nir_builder
*build
, nir_ssa_def
*then_def
, nir_ssa_def
*else_def
)
135 nir_block
*block
= nir_cursor_current_block(build
->cursor
);
136 nir_if
*nif
= nir_cf_node_as_if(nir_cf_node_prev(&block
->cf_node
));
138 nir_phi_instr
*phi
= nir_phi_instr_create(build
->shader
);
140 nir_phi_src
*src
= ralloc(phi
, nir_phi_src
);
141 src
->pred
= nir_if_last_then_block(nif
);
142 src
->src
= nir_src_for_ssa(then_def
);
143 exec_list_push_tail(&phi
->srcs
, &src
->node
);
145 src
= ralloc(phi
, nir_phi_src
);
146 src
->pred
= nir_if_last_else_block(nif
);
147 src
->src
= nir_src_for_ssa(else_def
);
148 exec_list_push_tail(&phi
->srcs
, &src
->node
);
150 assert(then_def
->num_components
== else_def
->num_components
);
151 assert(then_def
->bit_size
== else_def
->bit_size
);
152 nir_ssa_dest_init(&phi
->instr
, &phi
->dest
,
153 then_def
->num_components
, then_def
->bit_size
, NULL
);
155 nir_builder_instr_insert(build
, &phi
->instr
);
157 return &phi
->dest
.ssa
;
160 static inline nir_loop
*
161 nir_push_loop(nir_builder
*build
)
163 nir_loop
*loop
= nir_loop_create(build
->shader
);
164 nir_builder_cf_insert(build
, &loop
->cf_node
);
165 build
->cursor
= nir_before_cf_list(&loop
->body
);
170 nir_pop_loop(nir_builder
*build
, nir_loop
*loop
)
173 assert(nir_builder_is_inside_cf(build
, &loop
->cf_node
));
175 nir_block
*block
= nir_cursor_current_block(build
->cursor
);
176 loop
= nir_cf_node_as_loop(block
->cf_node
.parent
);
178 build
->cursor
= nir_after_cf_node(&loop
->cf_node
);
181 static inline nir_ssa_def
*
182 nir_ssa_undef(nir_builder
*build
, unsigned num_components
, unsigned bit_size
)
184 nir_ssa_undef_instr
*undef
=
185 nir_ssa_undef_instr_create(build
->shader
, num_components
, bit_size
);
189 nir_instr_insert(nir_before_cf_list(&build
->impl
->body
), &undef
->instr
);
194 static inline nir_ssa_def
*
195 nir_build_imm(nir_builder
*build
, unsigned num_components
,
196 unsigned bit_size
, nir_const_value value
)
198 nir_load_const_instr
*load_const
=
199 nir_load_const_instr_create(build
->shader
, num_components
, bit_size
);
203 load_const
->value
= value
;
205 nir_builder_instr_insert(build
, &load_const
->instr
);
207 return &load_const
->def
;
210 static inline nir_ssa_def
*
211 nir_imm_bool(nir_builder
*build
, bool x
)
215 memset(&v
, 0, sizeof(v
));
218 return nir_build_imm(build
, 1, 1, v
);
221 static inline nir_ssa_def
*
222 nir_imm_true(nir_builder
*build
)
224 return nir_imm_bool(build
, true);
227 static inline nir_ssa_def
*
228 nir_imm_false(nir_builder
*build
)
230 return nir_imm_bool(build
, false);
233 static inline nir_ssa_def
*
234 nir_imm_float16(nir_builder
*build
, float x
)
238 memset(&v
, 0, sizeof(v
));
239 v
.u16
[0] = _mesa_float_to_half(x
);
241 return nir_build_imm(build
, 1, 16, v
);
244 static inline nir_ssa_def
*
245 nir_imm_float(nir_builder
*build
, float x
)
249 memset(&v
, 0, sizeof(v
));
252 return nir_build_imm(build
, 1, 32, v
);
255 static inline nir_ssa_def
*
256 nir_imm_double(nir_builder
*build
, double x
)
260 memset(&v
, 0, sizeof(v
));
263 return nir_build_imm(build
, 1, 64, v
);
266 static inline nir_ssa_def
*
267 nir_imm_floatN_t(nir_builder
*build
, double x
, unsigned bit_size
)
271 return nir_imm_float16(build
, x
);
273 return nir_imm_float(build
, x
);
275 return nir_imm_double(build
, x
);
278 unreachable("unknown float immediate bit size");
281 static inline nir_ssa_def
*
282 nir_imm_vec4(nir_builder
*build
, float x
, float y
, float z
, float w
)
286 memset(&v
, 0, sizeof(v
));
292 return nir_build_imm(build
, 4, 32, v
);
295 static inline nir_ssa_def
*
296 nir_imm_ivec2(nir_builder
*build
, int x
, int y
)
300 memset(&v
, 0, sizeof(v
));
304 return nir_build_imm(build
, 2, 32, v
);
307 static inline nir_ssa_def
*
308 nir_imm_int(nir_builder
*build
, int x
)
312 memset(&v
, 0, sizeof(v
));
315 return nir_build_imm(build
, 1, 32, v
);
318 static inline nir_ssa_def
*
319 nir_imm_int64(nir_builder
*build
, int64_t x
)
323 memset(&v
, 0, sizeof(v
));
326 return nir_build_imm(build
, 1, 64, v
);
329 static inline nir_ssa_def
*
330 nir_imm_intN_t(nir_builder
*build
, uint64_t x
, unsigned bit_size
)
334 memset(&v
, 0, sizeof(v
));
335 assert(bit_size
<= 64);
339 v
.i64
[0] = x
& (~0ull >> (64 - bit_size
));
341 return nir_build_imm(build
, 1, bit_size
, v
);
344 static inline nir_ssa_def
*
345 nir_imm_ivec4(nir_builder
*build
, int x
, int y
, int z
, int w
)
349 memset(&v
, 0, sizeof(v
));
355 return nir_build_imm(build
, 4, 32, v
);
358 static inline nir_ssa_def
*
359 nir_imm_boolN_t(nir_builder
*build
, bool x
, unsigned bit_size
)
361 /* We use a 0/-1 convention for all booleans regardless of size */
362 return nir_imm_intN_t(build
, -(int)x
, bit_size
);
365 static inline nir_ssa_def
*
366 nir_build_alu(nir_builder
*build
, nir_op op
, nir_ssa_def
*src0
,
367 nir_ssa_def
*src1
, nir_ssa_def
*src2
, nir_ssa_def
*src3
)
369 const nir_op_info
*op_info
= &nir_op_infos
[op
];
370 nir_alu_instr
*instr
= nir_alu_instr_create(build
->shader
, op
);
374 instr
->exact
= build
->exact
;
376 instr
->src
[0].src
= nir_src_for_ssa(src0
);
378 instr
->src
[1].src
= nir_src_for_ssa(src1
);
380 instr
->src
[2].src
= nir_src_for_ssa(src2
);
382 instr
->src
[3].src
= nir_src_for_ssa(src3
);
384 /* Guess the number of components the destination temporary should have
385 * based on our input sizes, if it's not fixed for the op.
387 unsigned num_components
= op_info
->output_size
;
388 if (num_components
== 0) {
389 for (unsigned i
= 0; i
< op_info
->num_inputs
; i
++) {
390 if (op_info
->input_sizes
[i
] == 0)
391 num_components
= MAX2(num_components
,
392 instr
->src
[i
].src
.ssa
->num_components
);
395 assert(num_components
!= 0);
397 /* Figure out the bitwidth based on the source bitwidth if the instruction
400 unsigned bit_size
= nir_alu_type_get_type_size(op_info
->output_type
);
402 for (unsigned i
= 0; i
< op_info
->num_inputs
; i
++) {
403 unsigned src_bit_size
= instr
->src
[i
].src
.ssa
->bit_size
;
404 if (nir_alu_type_get_type_size(op_info
->input_types
[i
]) == 0) {
406 assert(src_bit_size
== bit_size
);
408 bit_size
= src_bit_size
;
410 assert(src_bit_size
==
411 nir_alu_type_get_type_size(op_info
->input_types
[i
]));
416 /* When in doubt, assume 32. */
420 /* Make sure we don't swizzle from outside of our source vector (like if a
421 * scalar value was passed into a multiply with a vector).
423 for (unsigned i
= 0; i
< op_info
->num_inputs
; i
++) {
424 for (unsigned j
= instr
->src
[i
].src
.ssa
->num_components
;
425 j
< NIR_MAX_VEC_COMPONENTS
; j
++) {
426 instr
->src
[i
].swizzle
[j
] = instr
->src
[i
].src
.ssa
->num_components
- 1;
430 nir_ssa_dest_init(&instr
->instr
, &instr
->dest
.dest
, num_components
,
432 instr
->dest
.write_mask
= (1 << num_components
) - 1;
434 nir_builder_instr_insert(build
, &instr
->instr
);
436 return &instr
->dest
.dest
.ssa
;
439 #include "nir_builder_opcodes.h"
441 static inline nir_ssa_def
*
442 nir_vec(nir_builder
*build
, nir_ssa_def
**comp
, unsigned num_components
)
444 switch (num_components
) {
446 return nir_vec4(build
, comp
[0], comp
[1], comp
[2], comp
[3]);
448 return nir_vec3(build
, comp
[0], comp
[1], comp
[2]);
450 return nir_vec2(build
, comp
[0], comp
[1]);
454 unreachable("bad component count");
460 * Similar to nir_fmov, but takes a nir_alu_src instead of a nir_ssa_def.
462 static inline nir_ssa_def
*
463 nir_fmov_alu(nir_builder
*build
, nir_alu_src src
, unsigned num_components
)
465 nir_alu_instr
*mov
= nir_alu_instr_create(build
->shader
, nir_op_fmov
);
466 nir_ssa_dest_init(&mov
->instr
, &mov
->dest
.dest
, num_components
,
467 nir_src_bit_size(src
.src
), NULL
);
468 mov
->exact
= build
->exact
;
469 mov
->dest
.write_mask
= (1 << num_components
) - 1;
471 nir_builder_instr_insert(build
, &mov
->instr
);
473 return &mov
->dest
.dest
.ssa
;
476 static inline nir_ssa_def
*
477 nir_imov_alu(nir_builder
*build
, nir_alu_src src
, unsigned num_components
)
479 nir_alu_instr
*mov
= nir_alu_instr_create(build
->shader
, nir_op_imov
);
480 nir_ssa_dest_init(&mov
->instr
, &mov
->dest
.dest
, num_components
,
481 nir_src_bit_size(src
.src
), NULL
);
482 mov
->exact
= build
->exact
;
483 mov
->dest
.write_mask
= (1 << num_components
) - 1;
485 nir_builder_instr_insert(build
, &mov
->instr
);
487 return &mov
->dest
.dest
.ssa
;
491 * Construct an fmov or imov that reswizzles the source's components.
493 static inline nir_ssa_def
*
494 nir_swizzle(nir_builder
*build
, nir_ssa_def
*src
, const unsigned *swiz
,
495 unsigned num_components
, bool use_fmov
)
497 assert(num_components
<= NIR_MAX_VEC_COMPONENTS
);
498 nir_alu_src alu_src
= { NIR_SRC_INIT
};
499 alu_src
.src
= nir_src_for_ssa(src
);
501 bool is_identity_swizzle
= true;
502 for (unsigned i
= 0; i
< num_components
&& i
< NIR_MAX_VEC_COMPONENTS
; i
++) {
504 is_identity_swizzle
= false;
505 alu_src
.swizzle
[i
] = swiz
[i
];
508 if (num_components
== src
->num_components
&& is_identity_swizzle
)
511 return use_fmov
? nir_fmov_alu(build
, alu_src
, num_components
) :
512 nir_imov_alu(build
, alu_src
, num_components
);
515 /* Selects the right fdot given the number of components in each source. */
516 static inline nir_ssa_def
*
517 nir_fdot(nir_builder
*build
, nir_ssa_def
*src0
, nir_ssa_def
*src1
)
519 assert(src0
->num_components
== src1
->num_components
);
520 switch (src0
->num_components
) {
521 case 1: return nir_fmul(build
, src0
, src1
);
522 case 2: return nir_fdot2(build
, src0
, src1
);
523 case 3: return nir_fdot3(build
, src0
, src1
);
524 case 4: return nir_fdot4(build
, src0
, src1
);
526 unreachable("bad component size");
532 static inline nir_ssa_def
*
533 nir_bany_inequal(nir_builder
*b
, nir_ssa_def
*src0
, nir_ssa_def
*src1
)
535 switch (src0
->num_components
) {
536 case 1: return nir_ine(b
, src0
, src1
);
537 case 2: return nir_bany_inequal2(b
, src0
, src1
);
538 case 3: return nir_bany_inequal3(b
, src0
, src1
);
539 case 4: return nir_bany_inequal4(b
, src0
, src1
);
541 unreachable("bad component size");
545 static inline nir_ssa_def
*
546 nir_bany(nir_builder
*b
, nir_ssa_def
*src
)
548 return nir_bany_inequal(b
, src
, nir_imm_false(b
));
551 static inline nir_ssa_def
*
552 nir_channel(nir_builder
*b
, nir_ssa_def
*def
, unsigned c
)
554 return nir_swizzle(b
, def
, &c
, 1, false);
557 static inline nir_ssa_def
*
558 nir_channels(nir_builder
*b
, nir_ssa_def
*def
, nir_component_mask_t mask
)
560 unsigned num_channels
= 0, swizzle
[NIR_MAX_VEC_COMPONENTS
] = { 0 };
562 for (unsigned i
= 0; i
< NIR_MAX_VEC_COMPONENTS
; i
++) {
563 if ((mask
& (1 << i
)) == 0)
565 swizzle
[num_channels
++] = i
;
568 return nir_swizzle(b
, def
, swizzle
, num_channels
, false);
571 static inline nir_ssa_def
*
572 nir_i2i(nir_builder
*build
, nir_ssa_def
*x
, unsigned dest_bit_size
)
574 if (x
->bit_size
== dest_bit_size
)
577 switch (dest_bit_size
) {
578 case 64: return nir_i2i64(build
, x
);
579 case 32: return nir_i2i32(build
, x
);
580 case 16: return nir_i2i16(build
, x
);
581 case 8: return nir_i2i8(build
, x
);
582 default: unreachable("Invalid bit size");
586 static inline nir_ssa_def
*
587 nir_u2u(nir_builder
*build
, nir_ssa_def
*x
, unsigned dest_bit_size
)
589 if (x
->bit_size
== dest_bit_size
)
592 switch (dest_bit_size
) {
593 case 64: return nir_u2u64(build
, x
);
594 case 32: return nir_u2u32(build
, x
);
595 case 16: return nir_u2u16(build
, x
);
596 case 8: return nir_u2u8(build
, x
);
597 default: unreachable("Invalid bit size");
601 static inline nir_ssa_def
*
602 nir_iadd_imm(nir_builder
*build
, nir_ssa_def
*x
, uint64_t y
)
604 return nir_iadd(build
, x
, nir_imm_intN_t(build
, y
, x
->bit_size
));
607 static inline nir_ssa_def
*
608 nir_imul_imm(nir_builder
*build
, nir_ssa_def
*x
, uint64_t y
)
610 return nir_imul(build
, x
, nir_imm_intN_t(build
, y
, x
->bit_size
));
613 static inline nir_ssa_def
*
614 nir_fadd_imm(nir_builder
*build
, nir_ssa_def
*x
, double y
)
616 return nir_fadd(build
, x
, nir_imm_floatN_t(build
, y
, x
->bit_size
));
619 static inline nir_ssa_def
*
620 nir_fmul_imm(nir_builder
*build
, nir_ssa_def
*x
, double y
)
622 return nir_fmul(build
, x
, nir_imm_floatN_t(build
, y
, x
->bit_size
));
625 static inline nir_ssa_def
*
626 nir_pack_bits(nir_builder
*b
, nir_ssa_def
*src
, unsigned dest_bit_size
)
628 assert(src
->num_components
* src
->bit_size
== dest_bit_size
);
630 switch (dest_bit_size
) {
632 switch (src
->bit_size
) {
633 case 32: return nir_pack_64_2x32(b
, src
);
634 case 16: return nir_pack_64_4x16(b
, src
);
640 if (src
->bit_size
== 16)
641 return nir_pack_32_2x16(b
, src
);
648 /* If we got here, we have no dedicated unpack opcode. */
649 nir_ssa_def
*dest
= nir_imm_intN_t(b
, 0, dest_bit_size
);
650 for (unsigned i
= 0; i
< src
->num_components
; i
++) {
651 nir_ssa_def
*val
= nir_u2u(b
, nir_channel(b
, src
, i
), dest_bit_size
);
652 val
= nir_ishl(b
, val
, nir_imm_int(b
, i
* src
->bit_size
));
653 dest
= nir_ior(b
, dest
, val
);
658 static inline nir_ssa_def
*
659 nir_unpack_bits(nir_builder
*b
, nir_ssa_def
*src
, unsigned dest_bit_size
)
661 assert(src
->num_components
== 1);
662 assert(src
->bit_size
> dest_bit_size
);
663 const unsigned dest_num_components
= src
->bit_size
/ dest_bit_size
;
664 assert(dest_num_components
<= NIR_MAX_VEC_COMPONENTS
);
666 switch (src
->bit_size
) {
668 switch (dest_bit_size
) {
669 case 32: return nir_unpack_64_2x32(b
, src
);
670 case 16: return nir_unpack_64_4x16(b
, src
);
676 if (dest_bit_size
== 16)
677 return nir_unpack_32_2x16(b
, src
);
684 /* If we got here, we have no dedicated unpack opcode. */
685 nir_ssa_def
*dest_comps
[NIR_MAX_VEC_COMPONENTS
];
686 for (unsigned i
= 0; i
< dest_num_components
; i
++) {
687 nir_ssa_def
*val
= nir_ushr(b
, src
, nir_imm_int(b
, i
* dest_bit_size
));
688 dest_comps
[i
] = nir_u2u(b
, val
, dest_bit_size
);
690 return nir_vec(b
, dest_comps
, dest_num_components
);
693 static inline nir_ssa_def
*
694 nir_bitcast_vector(nir_builder
*b
, nir_ssa_def
*src
, unsigned dest_bit_size
)
696 assert((src
->bit_size
* src
->num_components
) % dest_bit_size
== 0);
697 const unsigned dest_num_components
=
698 (src
->bit_size
* src
->num_components
) / dest_bit_size
;
699 assert(dest_num_components
<= NIR_MAX_VEC_COMPONENTS
);
701 if (src
->bit_size
> dest_bit_size
) {
702 assert(src
->bit_size
% dest_bit_size
== 0);
703 if (src
->num_components
== 1) {
704 return nir_unpack_bits(b
, src
, dest_bit_size
);
706 const unsigned divisor
= src
->bit_size
/ dest_bit_size
;
707 assert(src
->num_components
* divisor
== dest_num_components
);
708 nir_ssa_def
*dest
[NIR_MAX_VEC_COMPONENTS
];
709 for (unsigned i
= 0; i
< src
->num_components
; i
++) {
710 nir_ssa_def
*unpacked
=
711 nir_unpack_bits(b
, nir_channel(b
, src
, i
), dest_bit_size
);
712 assert(unpacked
->num_components
== divisor
);
713 for (unsigned j
= 0; j
< divisor
; j
++)
714 dest
[i
* divisor
+ j
] = nir_channel(b
, unpacked
, j
);
716 return nir_vec(b
, dest
, dest_num_components
);
718 } else if (src
->bit_size
< dest_bit_size
) {
719 assert(dest_bit_size
% src
->bit_size
== 0);
720 if (dest_num_components
== 1) {
721 return nir_pack_bits(b
, src
, dest_bit_size
);
723 const unsigned divisor
= dest_bit_size
/ src
->bit_size
;
724 assert(src
->num_components
== dest_num_components
* divisor
);
725 nir_ssa_def
*dest
[NIR_MAX_VEC_COMPONENTS
];
726 for (unsigned i
= 0; i
< dest_num_components
; i
++) {
727 nir_component_mask_t src_mask
=
728 ((1 << divisor
) - 1) << (i
* divisor
);
729 dest
[i
] = nir_pack_bits(b
, nir_channels(b
, src
, src_mask
),
732 return nir_vec(b
, dest
, dest_num_components
);
735 assert(src
->bit_size
== dest_bit_size
);
741 * Turns a nir_src into a nir_ssa_def * so it can be passed to
742 * nir_build_alu()-based builder calls.
744 * See nir_ssa_for_alu_src() for alu instructions.
746 static inline nir_ssa_def
*
747 nir_ssa_for_src(nir_builder
*build
, nir_src src
, int num_components
)
749 if (src
.is_ssa
&& src
.ssa
->num_components
== num_components
)
752 nir_alu_src alu
= { NIR_SRC_INIT
};
754 for (int j
= 0; j
< 4; j
++)
757 return nir_imov_alu(build
, alu
, num_components
);
761 * Similar to nir_ssa_for_src(), but for alu srcs, respecting the
762 * nir_alu_src's swizzle.
764 static inline nir_ssa_def
*
765 nir_ssa_for_alu_src(nir_builder
*build
, nir_alu_instr
*instr
, unsigned srcn
)
767 static uint8_t trivial_swizzle
[NIR_MAX_VEC_COMPONENTS
];
768 for (int i
= 0; i
< NIR_MAX_VEC_COMPONENTS
; ++i
)
769 trivial_swizzle
[i
] = i
;
770 nir_alu_src
*src
= &instr
->src
[srcn
];
771 unsigned num_components
= nir_ssa_alu_instr_src_components(instr
, srcn
);
773 if (src
->src
.is_ssa
&& (src
->src
.ssa
->num_components
== num_components
) &&
774 !src
->abs
&& !src
->negate
&&
775 (memcmp(src
->swizzle
, trivial_swizzle
, num_components
) == 0))
778 return nir_imov_alu(build
, *src
, num_components
);
781 static inline nir_deref_instr
*
782 nir_build_deref_var(nir_builder
*build
, nir_variable
*var
)
784 nir_deref_instr
*deref
=
785 nir_deref_instr_create(build
->shader
, nir_deref_type_var
);
787 deref
->mode
= var
->data
.mode
;
788 deref
->type
= var
->type
;
791 nir_ssa_dest_init(&deref
->instr
, &deref
->dest
, 1, 32, NULL
);
793 nir_builder_instr_insert(build
, &deref
->instr
);
798 static inline nir_deref_instr
*
799 nir_build_deref_array(nir_builder
*build
, nir_deref_instr
*parent
,
802 assert(glsl_type_is_array(parent
->type
) ||
803 glsl_type_is_matrix(parent
->type
) ||
804 glsl_type_is_vector(parent
->type
));
806 nir_deref_instr
*deref
=
807 nir_deref_instr_create(build
->shader
, nir_deref_type_array
);
809 deref
->mode
= parent
->mode
;
810 deref
->type
= glsl_get_array_element(parent
->type
);
811 deref
->parent
= nir_src_for_ssa(&parent
->dest
.ssa
);
812 deref
->arr
.index
= nir_src_for_ssa(index
);
814 nir_ssa_dest_init(&deref
->instr
, &deref
->dest
,
815 parent
->dest
.ssa
.num_components
,
816 parent
->dest
.ssa
.bit_size
, NULL
);
818 nir_builder_instr_insert(build
, &deref
->instr
);
823 static inline nir_deref_instr
*
824 nir_build_deref_ptr_as_array(nir_builder
*build
, nir_deref_instr
*parent
,
827 assert(parent
->deref_type
== nir_deref_type_array
||
828 parent
->deref_type
== nir_deref_type_ptr_as_array
||
829 parent
->deref_type
== nir_deref_type_cast
);
831 nir_deref_instr
*deref
=
832 nir_deref_instr_create(build
->shader
, nir_deref_type_ptr_as_array
);
834 deref
->mode
= parent
->mode
;
835 deref
->type
= parent
->type
;
836 deref
->parent
= nir_src_for_ssa(&parent
->dest
.ssa
);
837 deref
->arr
.index
= nir_src_for_ssa(index
);
839 nir_ssa_dest_init(&deref
->instr
, &deref
->dest
,
840 parent
->dest
.ssa
.num_components
,
841 parent
->dest
.ssa
.bit_size
, NULL
);
843 nir_builder_instr_insert(build
, &deref
->instr
);
848 static inline nir_deref_instr
*
849 nir_build_deref_array_wildcard(nir_builder
*build
, nir_deref_instr
*parent
)
851 assert(glsl_type_is_array(parent
->type
) ||
852 glsl_type_is_matrix(parent
->type
));
854 nir_deref_instr
*deref
=
855 nir_deref_instr_create(build
->shader
, nir_deref_type_array_wildcard
);
857 deref
->mode
= parent
->mode
;
858 deref
->type
= glsl_get_array_element(parent
->type
);
859 deref
->parent
= nir_src_for_ssa(&parent
->dest
.ssa
);
861 nir_ssa_dest_init(&deref
->instr
, &deref
->dest
,
862 parent
->dest
.ssa
.num_components
,
863 parent
->dest
.ssa
.bit_size
, NULL
);
865 nir_builder_instr_insert(build
, &deref
->instr
);
870 static inline nir_deref_instr
*
871 nir_build_deref_struct(nir_builder
*build
, nir_deref_instr
*parent
,
874 assert(glsl_type_is_struct(parent
->type
));
876 nir_deref_instr
*deref
=
877 nir_deref_instr_create(build
->shader
, nir_deref_type_struct
);
879 deref
->mode
= parent
->mode
;
880 deref
->type
= glsl_get_struct_field(parent
->type
, index
);
881 deref
->parent
= nir_src_for_ssa(&parent
->dest
.ssa
);
882 deref
->strct
.index
= index
;
884 nir_ssa_dest_init(&deref
->instr
, &deref
->dest
,
885 parent
->dest
.ssa
.num_components
,
886 parent
->dest
.ssa
.bit_size
, NULL
);
888 nir_builder_instr_insert(build
, &deref
->instr
);
893 static inline nir_deref_instr
*
894 nir_build_deref_cast(nir_builder
*build
, nir_ssa_def
*parent
,
895 nir_variable_mode mode
, const struct glsl_type
*type
,
898 nir_deref_instr
*deref
=
899 nir_deref_instr_create(build
->shader
, nir_deref_type_cast
);
903 deref
->parent
= nir_src_for_ssa(parent
);
904 deref
->cast
.ptr_stride
= ptr_stride
;
906 nir_ssa_dest_init(&deref
->instr
, &deref
->dest
,
907 parent
->num_components
, parent
->bit_size
, NULL
);
909 nir_builder_instr_insert(build
, &deref
->instr
);
914 /** Returns a deref that follows another but starting from the given parent
916 * The new deref will be the same type and take the same array or struct index
917 * as the leader deref but it may have a different parent. This is very
918 * useful for walking deref paths.
920 static inline nir_deref_instr
*
921 nir_build_deref_follower(nir_builder
*b
, nir_deref_instr
*parent
,
922 nir_deref_instr
*leader
)
924 /* If the derefs would have the same parent, don't make a new one */
925 assert(leader
->parent
.is_ssa
);
926 if (leader
->parent
.ssa
== &parent
->dest
.ssa
)
929 UNUSED nir_deref_instr
*leader_parent
= nir_src_as_deref(leader
->parent
);
931 switch (leader
->deref_type
) {
932 case nir_deref_type_var
:
933 unreachable("A var dereference cannot have a parent");
936 case nir_deref_type_array
:
937 case nir_deref_type_array_wildcard
:
938 assert(glsl_type_is_matrix(parent
->type
) ||
939 glsl_type_is_array(parent
->type
) ||
940 (leader
->deref_type
== nir_deref_type_array
&&
941 glsl_type_is_vector(parent
->type
)));
942 assert(glsl_get_length(parent
->type
) ==
943 glsl_get_length(leader_parent
->type
));
945 if (leader
->deref_type
== nir_deref_type_array
) {
946 assert(leader
->arr
.index
.is_ssa
);
947 return nir_build_deref_array(b
, parent
, leader
->arr
.index
.ssa
);
949 return nir_build_deref_array_wildcard(b
, parent
);
952 case nir_deref_type_struct
:
953 assert(glsl_type_is_struct(parent
->type
));
954 assert(glsl_get_length(parent
->type
) ==
955 glsl_get_length(leader_parent
->type
));
957 return nir_build_deref_struct(b
, parent
, leader
->strct
.index
);
960 unreachable("Invalid deref instruction type");
964 static inline nir_ssa_def
*
965 nir_load_reg(nir_builder
*build
, nir_register
*reg
)
967 return nir_ssa_for_src(build
, nir_src_for_reg(reg
), reg
->num_components
);
970 static inline nir_ssa_def
*
971 nir_load_deref(nir_builder
*build
, nir_deref_instr
*deref
)
973 nir_intrinsic_instr
*load
=
974 nir_intrinsic_instr_create(build
->shader
, nir_intrinsic_load_deref
);
975 load
->num_components
= glsl_get_vector_elements(deref
->type
);
976 load
->src
[0] = nir_src_for_ssa(&deref
->dest
.ssa
);
977 nir_ssa_dest_init(&load
->instr
, &load
->dest
, load
->num_components
,
978 glsl_get_bit_size(deref
->type
), NULL
);
979 nir_builder_instr_insert(build
, &load
->instr
);
980 return &load
->dest
.ssa
;
984 nir_store_deref(nir_builder
*build
, nir_deref_instr
*deref
,
985 nir_ssa_def
*value
, unsigned writemask
)
987 nir_intrinsic_instr
*store
=
988 nir_intrinsic_instr_create(build
->shader
, nir_intrinsic_store_deref
);
989 store
->num_components
= glsl_get_vector_elements(deref
->type
);
990 store
->src
[0] = nir_src_for_ssa(&deref
->dest
.ssa
);
991 store
->src
[1] = nir_src_for_ssa(value
);
992 nir_intrinsic_set_write_mask(store
,
993 writemask
& ((1 << store
->num_components
) - 1));
994 nir_builder_instr_insert(build
, &store
->instr
);
998 nir_copy_deref(nir_builder
*build
, nir_deref_instr
*dest
, nir_deref_instr
*src
)
1000 nir_intrinsic_instr
*copy
=
1001 nir_intrinsic_instr_create(build
->shader
, nir_intrinsic_copy_deref
);
1002 copy
->src
[0] = nir_src_for_ssa(&dest
->dest
.ssa
);
1003 copy
->src
[1] = nir_src_for_ssa(&src
->dest
.ssa
);
1004 nir_builder_instr_insert(build
, ©
->instr
);
1007 static inline nir_ssa_def
*
1008 nir_load_var(nir_builder
*build
, nir_variable
*var
)
1010 return nir_load_deref(build
, nir_build_deref_var(build
, var
));
1014 nir_store_var(nir_builder
*build
, nir_variable
*var
, nir_ssa_def
*value
,
1017 nir_store_deref(build
, nir_build_deref_var(build
, var
), value
, writemask
);
1021 nir_copy_var(nir_builder
*build
, nir_variable
*dest
, nir_variable
*src
)
1023 nir_copy_deref(build
, nir_build_deref_var(build
, dest
),
1024 nir_build_deref_var(build
, src
));
1027 static inline nir_ssa_def
*
1028 nir_load_param(nir_builder
*build
, uint32_t param_idx
)
1030 assert(param_idx
< build
->impl
->function
->num_params
);
1031 nir_parameter
*param
= &build
->impl
->function
->params
[param_idx
];
1033 nir_intrinsic_instr
*load
=
1034 nir_intrinsic_instr_create(build
->shader
, nir_intrinsic_load_param
);
1035 nir_intrinsic_set_param_idx(load
, param_idx
);
1036 load
->num_components
= param
->num_components
;
1037 nir_ssa_dest_init(&load
->instr
, &load
->dest
,
1038 param
->num_components
, param
->bit_size
, NULL
);
1039 nir_builder_instr_insert(build
, &load
->instr
);
1040 return &load
->dest
.ssa
;
1043 #include "nir_builder_opcodes.h"
1045 static inline nir_ssa_def
*
1046 nir_f2b(nir_builder
*build
, nir_ssa_def
*f
)
1048 return nir_f2b1(build
, f
);
1051 static inline nir_ssa_def
*
1052 nir_i2b(nir_builder
*build
, nir_ssa_def
*i
)
1054 return nir_i2b1(build
, i
);
1057 static inline nir_ssa_def
*
1058 nir_b2f(nir_builder
*build
, nir_ssa_def
*b
, uint32_t bit_size
)
1061 case 64: return nir_b2f64(build
, b
);
1062 case 32: return nir_b2f32(build
, b
);
1063 case 16: return nir_b2f16(build
, b
);
1065 unreachable("Invalid bit-size");
1069 static inline nir_ssa_def
*
1070 nir_load_barycentric(nir_builder
*build
, nir_intrinsic_op op
,
1071 unsigned interp_mode
)
1073 nir_intrinsic_instr
*bary
= nir_intrinsic_instr_create(build
->shader
, op
);
1074 nir_ssa_dest_init(&bary
->instr
, &bary
->dest
, 2, 32, NULL
);
1075 nir_intrinsic_set_interp_mode(bary
, interp_mode
);
1076 nir_builder_instr_insert(build
, &bary
->instr
);
1077 return &bary
->dest
.ssa
;
1081 nir_jump(nir_builder
*build
, nir_jump_type jump_type
)
1083 nir_jump_instr
*jump
= nir_jump_instr_create(build
->shader
, jump_type
);
1084 nir_builder_instr_insert(build
, &jump
->instr
);
1087 static inline nir_ssa_def
*
1088 nir_compare_func(nir_builder
*b
, enum compare_func func
,
1089 nir_ssa_def
*src0
, nir_ssa_def
*src1
)
1092 case COMPARE_FUNC_NEVER
:
1093 return nir_imm_int(b
, 0);
1094 case COMPARE_FUNC_ALWAYS
:
1095 return nir_imm_int(b
, ~0);
1096 case COMPARE_FUNC_EQUAL
:
1097 return nir_feq(b
, src0
, src1
);
1098 case COMPARE_FUNC_NOTEQUAL
:
1099 return nir_fne(b
, src0
, src1
);
1100 case COMPARE_FUNC_GREATER
:
1101 return nir_flt(b
, src1
, src0
);
1102 case COMPARE_FUNC_GEQUAL
:
1103 return nir_fge(b
, src0
, src1
);
1104 case COMPARE_FUNC_LESS
:
1105 return nir_flt(b
, src0
, src1
);
1106 case COMPARE_FUNC_LEQUAL
:
1107 return nir_fge(b
, src1
, src0
);
1109 unreachable("bad compare func");
1112 #endif /* NIR_BUILDER_H */