2 * Copyright © 2014-2015 Broadcom
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
27 #include "nir_control_flow.h"
28 #include "util/bitscan.h"
29 #include "util/half_float.h"
33 typedef struct nir_builder
{
36 /* Whether new ALU instructions will be marked "exact" */
40 nir_function_impl
*impl
;
44 nir_builder_init(nir_builder
*build
, nir_function_impl
*impl
)
46 memset(build
, 0, sizeof(*build
));
49 build
->shader
= impl
->function
->shader
;
53 nir_builder_init_simple_shader(nir_builder
*build
, void *mem_ctx
,
54 gl_shader_stage stage
,
55 const nir_shader_compiler_options
*options
)
57 build
->shader
= nir_shader_create(mem_ctx
, stage
, options
, NULL
);
58 nir_function
*func
= nir_function_create(build
->shader
, "main");
59 func
->is_entrypoint
= true;
61 build
->impl
= nir_function_impl_create(func
);
62 build
->cursor
= nir_after_cf_list(&build
->impl
->body
);
65 typedef bool (*nir_instr_pass_cb
)(struct nir_builder
*, nir_instr
*, void *);
68 * Iterates over all the instructions in a NIR shader and calls the given pass
71 * The pass should return true if it modified the shader. In that case, only
72 * the preserved metadata flags will be preserved in the function impl.
74 * The builder will be initialized to point at the function impl, but its
78 nir_shader_instructions_pass(nir_shader
*shader
,
79 nir_instr_pass_cb pass
,
80 nir_metadata preserved
,
83 bool progress
= false;
85 nir_foreach_function(function
, shader
) {
90 nir_builder_init(&b
, function
->impl
);
92 nir_foreach_block_safe(block
, function
->impl
) {
93 nir_foreach_instr_safe(instr
, block
) {
94 progress
|= pass(&b
, instr
, cb_data
);
99 nir_metadata_preserve(function
->impl
, preserved
);
101 nir_metadata_preserve(function
->impl
, nir_metadata_all
);
109 nir_builder_instr_insert(nir_builder
*build
, nir_instr
*instr
)
111 nir_instr_insert(build
->cursor
, instr
);
113 /* Move the cursor forward. */
114 build
->cursor
= nir_after_instr(instr
);
117 static inline nir_instr
*
118 nir_builder_last_instr(nir_builder
*build
)
120 assert(build
->cursor
.option
== nir_cursor_after_instr
);
121 return build
->cursor
.instr
;
125 nir_builder_cf_insert(nir_builder
*build
, nir_cf_node
*cf
)
127 nir_cf_node_insert(build
->cursor
, cf
);
131 nir_builder_is_inside_cf(nir_builder
*build
, nir_cf_node
*cf_node
)
133 nir_block
*block
= nir_cursor_current_block(build
->cursor
);
134 for (nir_cf_node
*n
= &block
->cf_node
; n
; n
= n
->parent
) {
141 static inline nir_if
*
142 nir_push_if_src(nir_builder
*build
, nir_src condition
)
144 nir_if
*nif
= nir_if_create(build
->shader
);
145 nif
->condition
= condition
;
146 nir_builder_cf_insert(build
, &nif
->cf_node
);
147 build
->cursor
= nir_before_cf_list(&nif
->then_list
);
151 static inline nir_if
*
152 nir_push_if(nir_builder
*build
, nir_ssa_def
*condition
)
154 return nir_push_if_src(build
, nir_src_for_ssa(condition
));
157 static inline nir_if
*
158 nir_push_else(nir_builder
*build
, nir_if
*nif
)
161 assert(nir_builder_is_inside_cf(build
, &nif
->cf_node
));
163 nir_block
*block
= nir_cursor_current_block(build
->cursor
);
164 nif
= nir_cf_node_as_if(block
->cf_node
.parent
);
166 build
->cursor
= nir_before_cf_list(&nif
->else_list
);
171 nir_pop_if(nir_builder
*build
, nir_if
*nif
)
174 assert(nir_builder_is_inside_cf(build
, &nif
->cf_node
));
176 nir_block
*block
= nir_cursor_current_block(build
->cursor
);
177 nif
= nir_cf_node_as_if(block
->cf_node
.parent
);
179 build
->cursor
= nir_after_cf_node(&nif
->cf_node
);
182 static inline nir_ssa_def
*
183 nir_if_phi(nir_builder
*build
, nir_ssa_def
*then_def
, nir_ssa_def
*else_def
)
185 nir_block
*block
= nir_cursor_current_block(build
->cursor
);
186 nir_if
*nif
= nir_cf_node_as_if(nir_cf_node_prev(&block
->cf_node
));
188 nir_phi_instr
*phi
= nir_phi_instr_create(build
->shader
);
190 nir_phi_src
*src
= ralloc(phi
, nir_phi_src
);
191 src
->pred
= nir_if_last_then_block(nif
);
192 src
->src
= nir_src_for_ssa(then_def
);
193 exec_list_push_tail(&phi
->srcs
, &src
->node
);
195 src
= ralloc(phi
, nir_phi_src
);
196 src
->pred
= nir_if_last_else_block(nif
);
197 src
->src
= nir_src_for_ssa(else_def
);
198 exec_list_push_tail(&phi
->srcs
, &src
->node
);
200 assert(then_def
->num_components
== else_def
->num_components
);
201 assert(then_def
->bit_size
== else_def
->bit_size
);
202 nir_ssa_dest_init(&phi
->instr
, &phi
->dest
,
203 then_def
->num_components
, then_def
->bit_size
, NULL
);
205 nir_builder_instr_insert(build
, &phi
->instr
);
207 return &phi
->dest
.ssa
;
210 static inline nir_loop
*
211 nir_push_loop(nir_builder
*build
)
213 nir_loop
*loop
= nir_loop_create(build
->shader
);
214 nir_builder_cf_insert(build
, &loop
->cf_node
);
215 build
->cursor
= nir_before_cf_list(&loop
->body
);
220 nir_pop_loop(nir_builder
*build
, nir_loop
*loop
)
223 assert(nir_builder_is_inside_cf(build
, &loop
->cf_node
));
225 nir_block
*block
= nir_cursor_current_block(build
->cursor
);
226 loop
= nir_cf_node_as_loop(block
->cf_node
.parent
);
228 build
->cursor
= nir_after_cf_node(&loop
->cf_node
);
231 static inline nir_ssa_def
*
232 nir_ssa_undef(nir_builder
*build
, unsigned num_components
, unsigned bit_size
)
234 nir_ssa_undef_instr
*undef
=
235 nir_ssa_undef_instr_create(build
->shader
, num_components
, bit_size
);
239 nir_instr_insert(nir_before_cf_list(&build
->impl
->body
), &undef
->instr
);
244 static inline nir_ssa_def
*
245 nir_build_imm(nir_builder
*build
, unsigned num_components
,
246 unsigned bit_size
, const nir_const_value
*value
)
248 nir_load_const_instr
*load_const
=
249 nir_load_const_instr_create(build
->shader
, num_components
, bit_size
);
253 memcpy(load_const
->value
, value
, sizeof(nir_const_value
) * num_components
);
255 nir_builder_instr_insert(build
, &load_const
->instr
);
257 return &load_const
->def
;
260 static inline nir_ssa_def
*
261 nir_imm_zero(nir_builder
*build
, unsigned num_components
, unsigned bit_size
)
263 nir_load_const_instr
*load_const
=
264 nir_load_const_instr_create(build
->shader
, num_components
, bit_size
);
266 /* nir_load_const_instr_create uses rzalloc so it's already zero */
268 nir_builder_instr_insert(build
, &load_const
->instr
);
270 return &load_const
->def
;
273 static inline nir_ssa_def
*
274 nir_imm_boolN_t(nir_builder
*build
, bool x
, unsigned bit_size
)
276 nir_const_value v
= nir_const_value_for_bool(x
, bit_size
);
277 return nir_build_imm(build
, 1, bit_size
, &v
);
280 static inline nir_ssa_def
*
281 nir_imm_bool(nir_builder
*build
, bool x
)
283 return nir_imm_boolN_t(build
, x
, 1);
286 static inline nir_ssa_def
*
287 nir_imm_true(nir_builder
*build
)
289 return nir_imm_bool(build
, true);
292 static inline nir_ssa_def
*
293 nir_imm_false(nir_builder
*build
)
295 return nir_imm_bool(build
, false);
298 static inline nir_ssa_def
*
299 nir_imm_floatN_t(nir_builder
*build
, double x
, unsigned bit_size
)
301 nir_const_value v
= nir_const_value_for_float(x
, bit_size
);
302 return nir_build_imm(build
, 1, bit_size
, &v
);
305 static inline nir_ssa_def
*
306 nir_imm_float16(nir_builder
*build
, float x
)
308 return nir_imm_floatN_t(build
, x
, 16);
311 static inline nir_ssa_def
*
312 nir_imm_float(nir_builder
*build
, float x
)
314 return nir_imm_floatN_t(build
, x
, 32);
317 static inline nir_ssa_def
*
318 nir_imm_double(nir_builder
*build
, double x
)
320 return nir_imm_floatN_t(build
, x
, 64);
323 static inline nir_ssa_def
*
324 nir_imm_vec2(nir_builder
*build
, float x
, float y
)
326 nir_const_value v
[2] = {
327 nir_const_value_for_float(x
, 32),
328 nir_const_value_for_float(y
, 32),
330 return nir_build_imm(build
, 2, 32, v
);
333 static inline nir_ssa_def
*
334 nir_imm_vec4(nir_builder
*build
, float x
, float y
, float z
, float w
)
336 nir_const_value v
[4] = {
337 nir_const_value_for_float(x
, 32),
338 nir_const_value_for_float(y
, 32),
339 nir_const_value_for_float(z
, 32),
340 nir_const_value_for_float(w
, 32),
343 return nir_build_imm(build
, 4, 32, v
);
346 static inline nir_ssa_def
*
347 nir_imm_vec4_16(nir_builder
*build
, float x
, float y
, float z
, float w
)
349 nir_const_value v
[4] = {
350 nir_const_value_for_float(x
, 16),
351 nir_const_value_for_float(y
, 16),
352 nir_const_value_for_float(z
, 16),
353 nir_const_value_for_float(w
, 16),
356 return nir_build_imm(build
, 4, 16, v
);
359 static inline nir_ssa_def
*
360 nir_imm_intN_t(nir_builder
*build
, uint64_t x
, unsigned bit_size
)
362 nir_const_value v
= nir_const_value_for_raw_uint(x
, bit_size
);
363 return nir_build_imm(build
, 1, bit_size
, &v
);
366 static inline nir_ssa_def
*
367 nir_imm_int(nir_builder
*build
, int x
)
369 return nir_imm_intN_t(build
, x
, 32);
372 static inline nir_ssa_def
*
373 nir_imm_int64(nir_builder
*build
, int64_t x
)
375 return nir_imm_intN_t(build
, x
, 64);
378 static inline nir_ssa_def
*
379 nir_imm_ivec2(nir_builder
*build
, int x
, int y
)
381 nir_const_value v
[2] = {
382 nir_const_value_for_int(x
, 32),
383 nir_const_value_for_int(y
, 32),
386 return nir_build_imm(build
, 2, 32, v
);
389 static inline nir_ssa_def
*
390 nir_imm_ivec4(nir_builder
*build
, int x
, int y
, int z
, int w
)
392 nir_const_value v
[4] = {
393 nir_const_value_for_int(x
, 32),
394 nir_const_value_for_int(y
, 32),
395 nir_const_value_for_int(z
, 32),
396 nir_const_value_for_int(w
, 32),
399 return nir_build_imm(build
, 4, 32, v
);
402 static inline nir_ssa_def
*
403 nir_builder_alu_instr_finish_and_insert(nir_builder
*build
, nir_alu_instr
*instr
)
405 const nir_op_info
*op_info
= &nir_op_infos
[instr
->op
];
407 instr
->exact
= build
->exact
;
409 /* Guess the number of components the destination temporary should have
410 * based on our input sizes, if it's not fixed for the op.
412 unsigned num_components
= op_info
->output_size
;
413 if (num_components
== 0) {
414 for (unsigned i
= 0; i
< op_info
->num_inputs
; i
++) {
415 if (op_info
->input_sizes
[i
] == 0)
416 num_components
= MAX2(num_components
,
417 instr
->src
[i
].src
.ssa
->num_components
);
420 assert(num_components
!= 0);
422 /* Figure out the bitwidth based on the source bitwidth if the instruction
425 unsigned bit_size
= nir_alu_type_get_type_size(op_info
->output_type
);
427 for (unsigned i
= 0; i
< op_info
->num_inputs
; i
++) {
428 unsigned src_bit_size
= instr
->src
[i
].src
.ssa
->bit_size
;
429 if (nir_alu_type_get_type_size(op_info
->input_types
[i
]) == 0) {
431 assert(src_bit_size
== bit_size
);
433 bit_size
= src_bit_size
;
435 assert(src_bit_size
==
436 nir_alu_type_get_type_size(op_info
->input_types
[i
]));
441 /* When in doubt, assume 32. */
445 /* Make sure we don't swizzle from outside of our source vector (like if a
446 * scalar value was passed into a multiply with a vector).
448 for (unsigned i
= 0; i
< op_info
->num_inputs
; i
++) {
449 for (unsigned j
= instr
->src
[i
].src
.ssa
->num_components
;
450 j
< NIR_MAX_VEC_COMPONENTS
; j
++) {
451 instr
->src
[i
].swizzle
[j
] = instr
->src
[i
].src
.ssa
->num_components
- 1;
455 nir_ssa_dest_init(&instr
->instr
, &instr
->dest
.dest
, num_components
,
457 instr
->dest
.write_mask
= (1 << num_components
) - 1;
459 nir_builder_instr_insert(build
, &instr
->instr
);
461 return &instr
->dest
.dest
.ssa
;
464 static inline nir_ssa_def
*
465 nir_build_alu(nir_builder
*build
, nir_op op
, nir_ssa_def
*src0
,
466 nir_ssa_def
*src1
, nir_ssa_def
*src2
, nir_ssa_def
*src3
)
468 nir_alu_instr
*instr
= nir_alu_instr_create(build
->shader
, op
);
472 instr
->src
[0].src
= nir_src_for_ssa(src0
);
474 instr
->src
[1].src
= nir_src_for_ssa(src1
);
476 instr
->src
[2].src
= nir_src_for_ssa(src2
);
478 instr
->src
[3].src
= nir_src_for_ssa(src3
);
480 return nir_builder_alu_instr_finish_and_insert(build
, instr
);
483 /* for the couple special cases with more than 4 src args: */
484 static inline nir_ssa_def
*
485 nir_build_alu_src_arr(nir_builder
*build
, nir_op op
, nir_ssa_def
**srcs
)
487 const nir_op_info
*op_info
= &nir_op_infos
[op
];
488 nir_alu_instr
*instr
= nir_alu_instr_create(build
->shader
, op
);
492 for (unsigned i
= 0; i
< op_info
->num_inputs
; i
++)
493 instr
->src
[i
].src
= nir_src_for_ssa(srcs
[i
]);
495 return nir_builder_alu_instr_finish_and_insert(build
, instr
);
498 #include "nir_builder_opcodes.h"
500 static inline nir_ssa_def
*
501 nir_vec(nir_builder
*build
, nir_ssa_def
**comp
, unsigned num_components
)
503 return nir_build_alu_src_arr(build
, nir_op_vec(num_components
), comp
);
506 static inline nir_ssa_def
*
507 nir_mov_alu(nir_builder
*build
, nir_alu_src src
, unsigned num_components
)
509 assert(!src
.abs
&& !src
.negate
);
510 if (src
.src
.is_ssa
&& src
.src
.ssa
->num_components
== num_components
) {
511 bool any_swizzles
= false;
512 for (unsigned i
= 0; i
< num_components
; i
++) {
513 if (src
.swizzle
[i
] != i
)
520 nir_alu_instr
*mov
= nir_alu_instr_create(build
->shader
, nir_op_mov
);
521 nir_ssa_dest_init(&mov
->instr
, &mov
->dest
.dest
, num_components
,
522 nir_src_bit_size(src
.src
), NULL
);
523 mov
->exact
= build
->exact
;
524 mov
->dest
.write_mask
= (1 << num_components
) - 1;
526 nir_builder_instr_insert(build
, &mov
->instr
);
528 return &mov
->dest
.dest
.ssa
;
532 * Construct an fmov or imov that reswizzles the source's components.
534 static inline nir_ssa_def
*
535 nir_swizzle(nir_builder
*build
, nir_ssa_def
*src
, const unsigned *swiz
,
536 unsigned num_components
)
538 assert(num_components
<= NIR_MAX_VEC_COMPONENTS
);
539 nir_alu_src alu_src
= { NIR_SRC_INIT
};
540 alu_src
.src
= nir_src_for_ssa(src
);
542 bool is_identity_swizzle
= true;
543 for (unsigned i
= 0; i
< num_components
&& i
< NIR_MAX_VEC_COMPONENTS
; i
++) {
545 is_identity_swizzle
= false;
546 alu_src
.swizzle
[i
] = swiz
[i
];
549 if (num_components
== src
->num_components
&& is_identity_swizzle
)
552 return nir_mov_alu(build
, alu_src
, num_components
);
555 /* Selects the right fdot given the number of components in each source. */
556 static inline nir_ssa_def
*
557 nir_fdot(nir_builder
*build
, nir_ssa_def
*src0
, nir_ssa_def
*src1
)
559 assert(src0
->num_components
== src1
->num_components
);
560 switch (src0
->num_components
) {
561 case 1: return nir_fmul(build
, src0
, src1
);
562 case 2: return nir_fdot2(build
, src0
, src1
);
563 case 3: return nir_fdot3(build
, src0
, src1
);
564 case 4: return nir_fdot4(build
, src0
, src1
);
565 case 8: return nir_fdot8(build
, src0
, src1
);
566 case 16: return nir_fdot16(build
, src0
, src1
);
568 unreachable("bad component size");
574 static inline nir_ssa_def
*
575 nir_ball_iequal(nir_builder
*b
, nir_ssa_def
*src0
, nir_ssa_def
*src1
)
577 switch (src0
->num_components
) {
578 case 1: return nir_ieq(b
, src0
, src1
);
579 case 2: return nir_ball_iequal2(b
, src0
, src1
);
580 case 3: return nir_ball_iequal3(b
, src0
, src1
);
581 case 4: return nir_ball_iequal4(b
, src0
, src1
);
582 case 8: return nir_ball_iequal8(b
, src0
, src1
);
583 case 16: return nir_ball_iequal16(b
, src0
, src1
);
585 unreachable("bad component size");
589 static inline nir_ssa_def
*
590 nir_ball(nir_builder
*b
, nir_ssa_def
*src
)
592 return nir_ball_iequal(b
, src
, nir_imm_true(b
));
595 static inline nir_ssa_def
*
596 nir_bany_inequal(nir_builder
*b
, nir_ssa_def
*src0
, nir_ssa_def
*src1
)
598 switch (src0
->num_components
) {
599 case 1: return nir_ine(b
, src0
, src1
);
600 case 2: return nir_bany_inequal2(b
, src0
, src1
);
601 case 3: return nir_bany_inequal3(b
, src0
, src1
);
602 case 4: return nir_bany_inequal4(b
, src0
, src1
);
603 case 8: return nir_bany_inequal8(b
, src0
, src1
);
604 case 16: return nir_bany_inequal16(b
, src0
, src1
);
606 unreachable("bad component size");
610 static inline nir_ssa_def
*
611 nir_bany(nir_builder
*b
, nir_ssa_def
*src
)
613 return nir_bany_inequal(b
, src
, nir_imm_false(b
));
616 static inline nir_ssa_def
*
617 nir_channel(nir_builder
*b
, nir_ssa_def
*def
, unsigned c
)
619 return nir_swizzle(b
, def
, &c
, 1);
622 static inline nir_ssa_def
*
623 nir_channels(nir_builder
*b
, nir_ssa_def
*def
, nir_component_mask_t mask
)
625 unsigned num_channels
= 0, swizzle
[NIR_MAX_VEC_COMPONENTS
] = { 0 };
627 for (unsigned i
= 0; i
< NIR_MAX_VEC_COMPONENTS
; i
++) {
628 if ((mask
& (1 << i
)) == 0)
630 swizzle
[num_channels
++] = i
;
633 return nir_swizzle(b
, def
, swizzle
, num_channels
);
636 static inline nir_ssa_def
*
637 _nir_vector_extract_helper(nir_builder
*b
, nir_ssa_def
*vec
, nir_ssa_def
*c
,
638 unsigned start
, unsigned end
)
640 if (start
== end
- 1) {
641 return nir_channel(b
, vec
, start
);
643 unsigned mid
= start
+ (end
- start
) / 2;
644 return nir_bcsel(b
, nir_ilt(b
, c
, nir_imm_intN_t(b
, mid
, c
->bit_size
)),
645 _nir_vector_extract_helper(b
, vec
, c
, start
, mid
),
646 _nir_vector_extract_helper(b
, vec
, c
, mid
, end
));
650 static inline nir_ssa_def
*
651 nir_vector_extract(nir_builder
*b
, nir_ssa_def
*vec
, nir_ssa_def
*c
)
653 nir_src c_src
= nir_src_for_ssa(c
);
654 if (nir_src_is_const(c_src
)) {
655 uint64_t c_const
= nir_src_as_uint(c_src
);
656 if (c_const
< vec
->num_components
)
657 return nir_channel(b
, vec
, c_const
);
659 return nir_ssa_undef(b
, 1, vec
->bit_size
);
661 return _nir_vector_extract_helper(b
, vec
, c
, 0, vec
->num_components
);
665 /** Replaces the component of `vec` specified by `c` with `scalar` */
666 static inline nir_ssa_def
*
667 nir_vector_insert_imm(nir_builder
*b
, nir_ssa_def
*vec
,
668 nir_ssa_def
*scalar
, unsigned c
)
670 assert(scalar
->num_components
== 1);
671 assert(c
< vec
->num_components
);
673 nir_op vec_op
= nir_op_vec(vec
->num_components
);
674 nir_alu_instr
*vec_instr
= nir_alu_instr_create(b
->shader
, vec_op
);
676 for (unsigned i
= 0; i
< vec
->num_components
; i
++) {
678 vec_instr
->src
[i
].src
= nir_src_for_ssa(scalar
);
679 vec_instr
->src
[i
].swizzle
[0] = 0;
681 vec_instr
->src
[i
].src
= nir_src_for_ssa(vec
);
682 vec_instr
->src
[i
].swizzle
[0] = i
;
686 return nir_builder_alu_instr_finish_and_insert(b
, vec_instr
);
689 /** Replaces the component of `vec` specified by `c` with `scalar` */
690 static inline nir_ssa_def
*
691 nir_vector_insert(nir_builder
*b
, nir_ssa_def
*vec
, nir_ssa_def
*scalar
,
694 assert(scalar
->num_components
== 1);
695 assert(c
->num_components
== 1);
697 nir_src c_src
= nir_src_for_ssa(c
);
698 if (nir_src_is_const(c_src
)) {
699 uint64_t c_const
= nir_src_as_uint(c_src
);
700 if (c_const
< vec
->num_components
)
701 return nir_vector_insert_imm(b
, vec
, scalar
, c_const
);
705 nir_const_value per_comp_idx_const
[NIR_MAX_VEC_COMPONENTS
];
706 for (unsigned i
= 0; i
< NIR_MAX_VEC_COMPONENTS
; i
++)
707 per_comp_idx_const
[i
] = nir_const_value_for_int(i
, c
->bit_size
);
708 nir_ssa_def
*per_comp_idx
=
709 nir_build_imm(b
, vec
->num_components
,
710 c
->bit_size
, per_comp_idx_const
);
712 /* nir_builder will automatically splat out scalars to vectors so an
713 * insert is as simple as "if I'm the channel, replace me with the
716 return nir_bcsel(b
, nir_ieq(b
, c
, per_comp_idx
), scalar
, vec
);
720 static inline nir_ssa_def
*
721 nir_i2i(nir_builder
*build
, nir_ssa_def
*x
, unsigned dest_bit_size
)
723 if (x
->bit_size
== dest_bit_size
)
726 switch (dest_bit_size
) {
727 case 64: return nir_i2i64(build
, x
);
728 case 32: return nir_i2i32(build
, x
);
729 case 16: return nir_i2i16(build
, x
);
730 case 8: return nir_i2i8(build
, x
);
731 default: unreachable("Invalid bit size");
735 static inline nir_ssa_def
*
736 nir_u2u(nir_builder
*build
, nir_ssa_def
*x
, unsigned dest_bit_size
)
738 if (x
->bit_size
== dest_bit_size
)
741 switch (dest_bit_size
) {
742 case 64: return nir_u2u64(build
, x
);
743 case 32: return nir_u2u32(build
, x
);
744 case 16: return nir_u2u16(build
, x
);
745 case 8: return nir_u2u8(build
, x
);
746 default: unreachable("Invalid bit size");
750 static inline nir_ssa_def
*
751 nir_iadd_imm(nir_builder
*build
, nir_ssa_def
*x
, uint64_t y
)
753 assert(x
->bit_size
<= 64);
754 y
&= BITFIELD64_MASK(x
->bit_size
);
759 return nir_iadd(build
, x
, nir_imm_intN_t(build
, y
, x
->bit_size
));
763 static inline nir_ssa_def
*
764 _nir_mul_imm(nir_builder
*build
, nir_ssa_def
*x
, uint64_t y
, bool amul
)
766 assert(x
->bit_size
<= 64);
767 y
&= BITFIELD64_MASK(x
->bit_size
);
770 return nir_imm_intN_t(build
, 0, x
->bit_size
);
773 } else if (!build
->shader
->options
->lower_bitops
&&
774 util_is_power_of_two_or_zero64(y
)) {
775 return nir_ishl(build
, x
, nir_imm_int(build
, ffsll(y
) - 1));
777 return nir_amul(build
, x
, nir_imm_intN_t(build
, y
, x
->bit_size
));
779 return nir_imul(build
, x
, nir_imm_intN_t(build
, y
, x
->bit_size
));
783 static inline nir_ssa_def
*
784 nir_imul_imm(nir_builder
*build
, nir_ssa_def
*x
, uint64_t y
)
786 return _nir_mul_imm(build
, x
, y
, false);
789 static inline nir_ssa_def
*
790 nir_amul_imm(nir_builder
*build
, nir_ssa_def
*x
, uint64_t y
)
792 return _nir_mul_imm(build
, x
, y
, true);
795 static inline nir_ssa_def
*
796 nir_fadd_imm(nir_builder
*build
, nir_ssa_def
*x
, double y
)
798 return nir_fadd(build
, x
, nir_imm_floatN_t(build
, y
, x
->bit_size
));
801 static inline nir_ssa_def
*
802 nir_fmul_imm(nir_builder
*build
, nir_ssa_def
*x
, double y
)
804 return nir_fmul(build
, x
, nir_imm_floatN_t(build
, y
, x
->bit_size
));
807 static inline nir_ssa_def
*
808 nir_iand_imm(nir_builder
*build
, nir_ssa_def
*x
, uint64_t y
)
810 assert(x
->bit_size
<= 64);
811 y
&= BITFIELD64_MASK(x
->bit_size
);
814 return nir_imm_intN_t(build
, 0, x
->bit_size
);
815 } else if (y
== BITFIELD64_MASK(x
->bit_size
)) {
818 return nir_iand(build
, x
, nir_imm_intN_t(build
, y
, x
->bit_size
));
822 static inline nir_ssa_def
*
823 nir_ishr_imm(nir_builder
*build
, nir_ssa_def
*x
, uint32_t y
)
828 return nir_ishr(build
, x
, nir_imm_int(build
, y
));
832 static inline nir_ssa_def
*
833 nir_ushr_imm(nir_builder
*build
, nir_ssa_def
*x
, uint32_t y
)
838 return nir_ushr(build
, x
, nir_imm_int(build
, y
));
842 static inline nir_ssa_def
*
843 nir_udiv_imm(nir_builder
*build
, nir_ssa_def
*x
, uint64_t y
)
845 assert(x
->bit_size
<= 64);
846 y
&= BITFIELD64_MASK(x
->bit_size
);
850 } else if (util_is_power_of_two_nonzero(y
)) {
851 return nir_ushr_imm(build
, x
, ffsll(y
) - 1);
853 return nir_udiv(build
, x
, nir_imm_intN_t(build
, y
, x
->bit_size
));
857 static inline nir_ssa_def
*
858 nir_pack_bits(nir_builder
*b
, nir_ssa_def
*src
, unsigned dest_bit_size
)
860 assert(src
->num_components
* src
->bit_size
== dest_bit_size
);
862 switch (dest_bit_size
) {
864 switch (src
->bit_size
) {
865 case 32: return nir_pack_64_2x32(b
, src
);
866 case 16: return nir_pack_64_4x16(b
, src
);
872 if (src
->bit_size
== 16)
873 return nir_pack_32_2x16(b
, src
);
880 /* If we got here, we have no dedicated unpack opcode. */
881 nir_ssa_def
*dest
= nir_imm_intN_t(b
, 0, dest_bit_size
);
882 for (unsigned i
= 0; i
< src
->num_components
; i
++) {
883 nir_ssa_def
*val
= nir_u2u(b
, nir_channel(b
, src
, i
), dest_bit_size
);
884 val
= nir_ishl(b
, val
, nir_imm_int(b
, i
* src
->bit_size
));
885 dest
= nir_ior(b
, dest
, val
);
890 static inline nir_ssa_def
*
891 nir_unpack_bits(nir_builder
*b
, nir_ssa_def
*src
, unsigned dest_bit_size
)
893 assert(src
->num_components
== 1);
894 assert(src
->bit_size
> dest_bit_size
);
895 const unsigned dest_num_components
= src
->bit_size
/ dest_bit_size
;
896 assert(dest_num_components
<= NIR_MAX_VEC_COMPONENTS
);
898 switch (src
->bit_size
) {
900 switch (dest_bit_size
) {
901 case 32: return nir_unpack_64_2x32(b
, src
);
902 case 16: return nir_unpack_64_4x16(b
, src
);
908 if (dest_bit_size
== 16)
909 return nir_unpack_32_2x16(b
, src
);
916 /* If we got here, we have no dedicated unpack opcode. */
917 nir_ssa_def
*dest_comps
[NIR_MAX_VEC_COMPONENTS
];
918 for (unsigned i
= 0; i
< dest_num_components
; i
++) {
919 nir_ssa_def
*val
= nir_ushr_imm(b
, src
, i
* dest_bit_size
);
920 dest_comps
[i
] = nir_u2u(b
, val
, dest_bit_size
);
922 return nir_vec(b
, dest_comps
, dest_num_components
);
926 * Treats srcs as if it's one big blob of bits and extracts the range of bits
929 * [first_bit, first_bit + dest_num_components * dest_bit_size)
931 * The range can have any alignment or size as long as it's an integer number
932 * of destination components and fits inside the concatenated sources.
934 * TODO: The one caveat here is that we can't handle byte alignment if 64-bit
935 * values are involved because that would require pack/unpack to/from a vec8
936 * which NIR currently does not support.
938 static inline nir_ssa_def
*
939 nir_extract_bits(nir_builder
*b
, nir_ssa_def
**srcs
, unsigned num_srcs
,
941 unsigned dest_num_components
, unsigned dest_bit_size
)
943 const unsigned num_bits
= dest_num_components
* dest_bit_size
;
945 /* Figure out the common bit size */
946 unsigned common_bit_size
= dest_bit_size
;
947 for (unsigned i
= 0; i
< num_srcs
; i
++)
948 common_bit_size
= MIN2(common_bit_size
, srcs
[i
]->bit_size
);
950 common_bit_size
= MIN2(common_bit_size
, (1u << (ffs(first_bit
) - 1)));
952 /* We don't want to have to deal with 1-bit values */
953 assert(common_bit_size
>= 8);
955 nir_ssa_def
*common_comps
[NIR_MAX_VEC_COMPONENTS
* sizeof(uint64_t)];
956 assert(num_bits
/ common_bit_size
<= ARRAY_SIZE(common_comps
));
958 /* First, unpack to the common bit size and select the components from the
962 unsigned src_start_bit
= 0;
963 unsigned src_end_bit
= 0;
964 for (unsigned i
= 0; i
< num_bits
/ common_bit_size
; i
++) {
965 const unsigned bit
= first_bit
+ (i
* common_bit_size
);
966 while (bit
>= src_end_bit
) {
968 assert(src_idx
< (int) num_srcs
);
969 src_start_bit
= src_end_bit
;
970 src_end_bit
+= srcs
[src_idx
]->bit_size
*
971 srcs
[src_idx
]->num_components
;
973 assert(bit
>= src_start_bit
);
974 assert(bit
+ common_bit_size
<= src_end_bit
);
975 const unsigned rel_bit
= bit
- src_start_bit
;
976 const unsigned src_bit_size
= srcs
[src_idx
]->bit_size
;
978 nir_ssa_def
*comp
= nir_channel(b
, srcs
[src_idx
],
979 rel_bit
/ src_bit_size
);
980 if (srcs
[src_idx
]->bit_size
> common_bit_size
) {
981 nir_ssa_def
*unpacked
= nir_unpack_bits(b
, comp
, common_bit_size
);
982 comp
= nir_channel(b
, unpacked
, (rel_bit
% src_bit_size
) /
985 common_comps
[i
] = comp
;
988 /* Now, re-pack the destination if we have to */
989 if (dest_bit_size
> common_bit_size
) {
990 unsigned common_per_dest
= dest_bit_size
/ common_bit_size
;
991 nir_ssa_def
*dest_comps
[NIR_MAX_VEC_COMPONENTS
];
992 for (unsigned i
= 0; i
< dest_num_components
; i
++) {
993 nir_ssa_def
*unpacked
= nir_vec(b
, common_comps
+ i
* common_per_dest
,
995 dest_comps
[i
] = nir_pack_bits(b
, unpacked
, dest_bit_size
);
997 return nir_vec(b
, dest_comps
, dest_num_components
);
999 assert(dest_bit_size
== common_bit_size
);
1000 return nir_vec(b
, common_comps
, dest_num_components
);
1004 static inline nir_ssa_def
*
1005 nir_bitcast_vector(nir_builder
*b
, nir_ssa_def
*src
, unsigned dest_bit_size
)
1007 assert((src
->bit_size
* src
->num_components
) % dest_bit_size
== 0);
1008 const unsigned dest_num_components
=
1009 (src
->bit_size
* src
->num_components
) / dest_bit_size
;
1010 assert(dest_num_components
<= NIR_MAX_VEC_COMPONENTS
);
1012 return nir_extract_bits(b
, &src
, 1, 0, dest_num_components
, dest_bit_size
);
1016 * Turns a nir_src into a nir_ssa_def * so it can be passed to
1017 * nir_build_alu()-based builder calls.
1019 * See nir_ssa_for_alu_src() for alu instructions.
1021 static inline nir_ssa_def
*
1022 nir_ssa_for_src(nir_builder
*build
, nir_src src
, int num_components
)
1024 if (src
.is_ssa
&& src
.ssa
->num_components
== num_components
)
1027 nir_alu_src alu
= { NIR_SRC_INIT
};
1029 for (int j
= 0; j
< 4; j
++)
1032 return nir_mov_alu(build
, alu
, num_components
);
1036 * Similar to nir_ssa_for_src(), but for alu srcs, respecting the
1037 * nir_alu_src's swizzle.
1039 static inline nir_ssa_def
*
1040 nir_ssa_for_alu_src(nir_builder
*build
, nir_alu_instr
*instr
, unsigned srcn
)
1042 static uint8_t trivial_swizzle
[] = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 };
1043 STATIC_ASSERT(ARRAY_SIZE(trivial_swizzle
) == NIR_MAX_VEC_COMPONENTS
);
1045 nir_alu_src
*src
= &instr
->src
[srcn
];
1046 unsigned num_components
= nir_ssa_alu_instr_src_components(instr
, srcn
);
1048 if (src
->src
.is_ssa
&& (src
->src
.ssa
->num_components
== num_components
) &&
1049 !src
->abs
&& !src
->negate
&&
1050 (memcmp(src
->swizzle
, trivial_swizzle
, num_components
) == 0))
1051 return src
->src
.ssa
;
1053 return nir_mov_alu(build
, *src
, num_components
);
1056 static inline unsigned
1057 nir_get_ptr_bitsize(nir_shader
*shader
)
1059 if (shader
->info
.stage
== MESA_SHADER_KERNEL
)
1060 return shader
->info
.cs
.ptr_size
;
1064 static inline nir_deref_instr
*
1065 nir_build_deref_var(nir_builder
*build
, nir_variable
*var
)
1067 nir_deref_instr
*deref
=
1068 nir_deref_instr_create(build
->shader
, nir_deref_type_var
);
1070 deref
->mode
= (nir_variable_mode
)var
->data
.mode
;
1071 deref
->type
= var
->type
;
1074 nir_ssa_dest_init(&deref
->instr
, &deref
->dest
, 1,
1075 nir_get_ptr_bitsize(build
->shader
), NULL
);
1077 nir_builder_instr_insert(build
, &deref
->instr
);
1082 static inline nir_deref_instr
*
1083 nir_build_deref_array(nir_builder
*build
, nir_deref_instr
*parent
,
1086 assert(glsl_type_is_array(parent
->type
) ||
1087 glsl_type_is_matrix(parent
->type
) ||
1088 glsl_type_is_vector(parent
->type
));
1090 assert(index
->bit_size
== parent
->dest
.ssa
.bit_size
);
1092 nir_deref_instr
*deref
=
1093 nir_deref_instr_create(build
->shader
, nir_deref_type_array
);
1095 deref
->mode
= parent
->mode
;
1096 deref
->type
= glsl_get_array_element(parent
->type
);
1097 deref
->parent
= nir_src_for_ssa(&parent
->dest
.ssa
);
1098 deref
->arr
.index
= nir_src_for_ssa(index
);
1100 nir_ssa_dest_init(&deref
->instr
, &deref
->dest
,
1101 parent
->dest
.ssa
.num_components
,
1102 parent
->dest
.ssa
.bit_size
, NULL
);
1104 nir_builder_instr_insert(build
, &deref
->instr
);
1109 static inline nir_deref_instr
*
1110 nir_build_deref_array_imm(nir_builder
*build
, nir_deref_instr
*parent
,
1113 assert(parent
->dest
.is_ssa
);
1114 nir_ssa_def
*idx_ssa
= nir_imm_intN_t(build
, index
,
1115 parent
->dest
.ssa
.bit_size
);
1117 return nir_build_deref_array(build
, parent
, idx_ssa
);
1120 static inline nir_deref_instr
*
1121 nir_build_deref_ptr_as_array(nir_builder
*build
, nir_deref_instr
*parent
,
1124 assert(parent
->deref_type
== nir_deref_type_array
||
1125 parent
->deref_type
== nir_deref_type_ptr_as_array
||
1126 parent
->deref_type
== nir_deref_type_cast
);
1128 assert(index
->bit_size
== parent
->dest
.ssa
.bit_size
);
1130 nir_deref_instr
*deref
=
1131 nir_deref_instr_create(build
->shader
, nir_deref_type_ptr_as_array
);
1133 deref
->mode
= parent
->mode
;
1134 deref
->type
= parent
->type
;
1135 deref
->parent
= nir_src_for_ssa(&parent
->dest
.ssa
);
1136 deref
->arr
.index
= nir_src_for_ssa(index
);
1138 nir_ssa_dest_init(&deref
->instr
, &deref
->dest
,
1139 parent
->dest
.ssa
.num_components
,
1140 parent
->dest
.ssa
.bit_size
, NULL
);
1142 nir_builder_instr_insert(build
, &deref
->instr
);
1147 static inline nir_deref_instr
*
1148 nir_build_deref_array_wildcard(nir_builder
*build
, nir_deref_instr
*parent
)
1150 assert(glsl_type_is_array(parent
->type
) ||
1151 glsl_type_is_matrix(parent
->type
));
1153 nir_deref_instr
*deref
=
1154 nir_deref_instr_create(build
->shader
, nir_deref_type_array_wildcard
);
1156 deref
->mode
= parent
->mode
;
1157 deref
->type
= glsl_get_array_element(parent
->type
);
1158 deref
->parent
= nir_src_for_ssa(&parent
->dest
.ssa
);
1160 nir_ssa_dest_init(&deref
->instr
, &deref
->dest
,
1161 parent
->dest
.ssa
.num_components
,
1162 parent
->dest
.ssa
.bit_size
, NULL
);
1164 nir_builder_instr_insert(build
, &deref
->instr
);
1169 static inline nir_deref_instr
*
1170 nir_build_deref_struct(nir_builder
*build
, nir_deref_instr
*parent
,
1173 assert(glsl_type_is_struct_or_ifc(parent
->type
));
1175 nir_deref_instr
*deref
=
1176 nir_deref_instr_create(build
->shader
, nir_deref_type_struct
);
1178 deref
->mode
= parent
->mode
;
1179 deref
->type
= glsl_get_struct_field(parent
->type
, index
);
1180 deref
->parent
= nir_src_for_ssa(&parent
->dest
.ssa
);
1181 deref
->strct
.index
= index
;
1183 nir_ssa_dest_init(&deref
->instr
, &deref
->dest
,
1184 parent
->dest
.ssa
.num_components
,
1185 parent
->dest
.ssa
.bit_size
, NULL
);
1187 nir_builder_instr_insert(build
, &deref
->instr
);
1192 static inline nir_deref_instr
*
1193 nir_build_deref_cast(nir_builder
*build
, nir_ssa_def
*parent
,
1194 nir_variable_mode mode
, const struct glsl_type
*type
,
1195 unsigned ptr_stride
)
1197 nir_deref_instr
*deref
=
1198 nir_deref_instr_create(build
->shader
, nir_deref_type_cast
);
1202 deref
->parent
= nir_src_for_ssa(parent
);
1203 deref
->cast
.ptr_stride
= ptr_stride
;
1205 nir_ssa_dest_init(&deref
->instr
, &deref
->dest
,
1206 parent
->num_components
, parent
->bit_size
, NULL
);
1208 nir_builder_instr_insert(build
, &deref
->instr
);
1213 /** Returns a deref that follows another but starting from the given parent
1215 * The new deref will be the same type and take the same array or struct index
1216 * as the leader deref but it may have a different parent. This is very
1217 * useful for walking deref paths.
1219 static inline nir_deref_instr
*
1220 nir_build_deref_follower(nir_builder
*b
, nir_deref_instr
*parent
,
1221 nir_deref_instr
*leader
)
1223 /* If the derefs would have the same parent, don't make a new one */
1224 assert(leader
->parent
.is_ssa
);
1225 if (leader
->parent
.ssa
== &parent
->dest
.ssa
)
1228 UNUSED nir_deref_instr
*leader_parent
= nir_src_as_deref(leader
->parent
);
1230 switch (leader
->deref_type
) {
1231 case nir_deref_type_var
:
1232 unreachable("A var dereference cannot have a parent");
1235 case nir_deref_type_array
:
1236 case nir_deref_type_array_wildcard
:
1237 assert(glsl_type_is_matrix(parent
->type
) ||
1238 glsl_type_is_array(parent
->type
) ||
1239 (leader
->deref_type
== nir_deref_type_array
&&
1240 glsl_type_is_vector(parent
->type
)));
1241 assert(glsl_get_length(parent
->type
) ==
1242 glsl_get_length(leader_parent
->type
));
1244 if (leader
->deref_type
== nir_deref_type_array
) {
1245 assert(leader
->arr
.index
.is_ssa
);
1246 nir_ssa_def
*index
= nir_i2i(b
, leader
->arr
.index
.ssa
,
1247 parent
->dest
.ssa
.bit_size
);
1248 return nir_build_deref_array(b
, parent
, index
);
1250 return nir_build_deref_array_wildcard(b
, parent
);
1253 case nir_deref_type_struct
:
1254 assert(glsl_type_is_struct_or_ifc(parent
->type
));
1255 assert(glsl_get_length(parent
->type
) ==
1256 glsl_get_length(leader_parent
->type
));
1258 return nir_build_deref_struct(b
, parent
, leader
->strct
.index
);
1261 unreachable("Invalid deref instruction type");
1265 static inline nir_ssa_def
*
1266 nir_load_reg(nir_builder
*build
, nir_register
*reg
)
1268 return nir_ssa_for_src(build
, nir_src_for_reg(reg
), reg
->num_components
);
1272 nir_store_reg(nir_builder
*build
, nir_register
*reg
,
1273 nir_ssa_def
*def
, nir_component_mask_t write_mask
)
1275 assert(reg
->num_components
== def
->num_components
);
1276 assert(reg
->bit_size
== def
->bit_size
);
1278 nir_alu_instr
*mov
= nir_alu_instr_create(build
->shader
, nir_op_mov
);
1279 mov
->src
[0].src
= nir_src_for_ssa(def
);
1280 mov
->dest
.dest
= nir_dest_for_reg(reg
);
1281 mov
->dest
.write_mask
= write_mask
& BITFIELD_MASK(reg
->num_components
);
1282 nir_builder_instr_insert(build
, &mov
->instr
);
1285 static inline nir_ssa_def
*
1286 nir_load_deref_with_access(nir_builder
*build
, nir_deref_instr
*deref
,
1287 enum gl_access_qualifier access
)
1289 nir_intrinsic_instr
*load
=
1290 nir_intrinsic_instr_create(build
->shader
, nir_intrinsic_load_deref
);
1291 load
->num_components
= glsl_get_vector_elements(deref
->type
);
1292 load
->src
[0] = nir_src_for_ssa(&deref
->dest
.ssa
);
1293 nir_ssa_dest_init(&load
->instr
, &load
->dest
, load
->num_components
,
1294 glsl_get_bit_size(deref
->type
), NULL
);
1295 nir_intrinsic_set_access(load
, access
);
1296 nir_builder_instr_insert(build
, &load
->instr
);
1297 return &load
->dest
.ssa
;
1300 static inline nir_ssa_def
*
1301 nir_load_deref(nir_builder
*build
, nir_deref_instr
*deref
)
1303 return nir_load_deref_with_access(build
, deref
, (enum gl_access_qualifier
)0);
1307 nir_store_deref_with_access(nir_builder
*build
, nir_deref_instr
*deref
,
1308 nir_ssa_def
*value
, unsigned writemask
,
1309 enum gl_access_qualifier access
)
1311 nir_intrinsic_instr
*store
=
1312 nir_intrinsic_instr_create(build
->shader
, nir_intrinsic_store_deref
);
1313 store
->num_components
= glsl_get_vector_elements(deref
->type
);
1314 store
->src
[0] = nir_src_for_ssa(&deref
->dest
.ssa
);
1315 store
->src
[1] = nir_src_for_ssa(value
);
1316 nir_intrinsic_set_write_mask(store
,
1317 writemask
& ((1 << store
->num_components
) - 1));
1318 nir_intrinsic_set_access(store
, access
);
1319 nir_builder_instr_insert(build
, &store
->instr
);
1323 nir_store_deref(nir_builder
*build
, nir_deref_instr
*deref
,
1324 nir_ssa_def
*value
, unsigned writemask
)
1326 nir_store_deref_with_access(build
, deref
, value
, writemask
,
1327 (enum gl_access_qualifier
)0);
1331 nir_copy_deref_with_access(nir_builder
*build
, nir_deref_instr
*dest
,
1332 nir_deref_instr
*src
,
1333 enum gl_access_qualifier dest_access
,
1334 enum gl_access_qualifier src_access
)
1336 nir_intrinsic_instr
*copy
=
1337 nir_intrinsic_instr_create(build
->shader
, nir_intrinsic_copy_deref
);
1338 copy
->src
[0] = nir_src_for_ssa(&dest
->dest
.ssa
);
1339 copy
->src
[1] = nir_src_for_ssa(&src
->dest
.ssa
);
1340 nir_intrinsic_set_dst_access(copy
, dest_access
);
1341 nir_intrinsic_set_src_access(copy
, src_access
);
1342 nir_builder_instr_insert(build
, ©
->instr
);
1346 nir_copy_deref(nir_builder
*build
, nir_deref_instr
*dest
, nir_deref_instr
*src
)
1348 nir_copy_deref_with_access(build
, dest
, src
,
1349 (enum gl_access_qualifier
) 0,
1350 (enum gl_access_qualifier
) 0);
1353 static inline nir_ssa_def
*
1354 nir_load_var(nir_builder
*build
, nir_variable
*var
)
1356 return nir_load_deref(build
, nir_build_deref_var(build
, var
));
1360 nir_store_var(nir_builder
*build
, nir_variable
*var
, nir_ssa_def
*value
,
1363 nir_store_deref(build
, nir_build_deref_var(build
, var
), value
, writemask
);
1367 nir_copy_var(nir_builder
*build
, nir_variable
*dest
, nir_variable
*src
)
1369 nir_copy_deref(build
, nir_build_deref_var(build
, dest
),
1370 nir_build_deref_var(build
, src
));
1373 static inline nir_ssa_def
*
1374 nir_load_param(nir_builder
*build
, uint32_t param_idx
)
1376 assert(param_idx
< build
->impl
->function
->num_params
);
1377 nir_parameter
*param
= &build
->impl
->function
->params
[param_idx
];
1379 nir_intrinsic_instr
*load
=
1380 nir_intrinsic_instr_create(build
->shader
, nir_intrinsic_load_param
);
1381 nir_intrinsic_set_param_idx(load
, param_idx
);
1382 load
->num_components
= param
->num_components
;
1383 nir_ssa_dest_init(&load
->instr
, &load
->dest
,
1384 param
->num_components
, param
->bit_size
, NULL
);
1385 nir_builder_instr_insert(build
, &load
->instr
);
1386 return &load
->dest
.ssa
;
1389 #include "nir_builder_opcodes.h"
1391 static inline nir_ssa_def
*
1392 nir_f2b(nir_builder
*build
, nir_ssa_def
*f
)
1394 return nir_f2b1(build
, f
);
1397 static inline nir_ssa_def
*
1398 nir_i2b(nir_builder
*build
, nir_ssa_def
*i
)
1400 return nir_i2b1(build
, i
);
1403 static inline nir_ssa_def
*
1404 nir_b2f(nir_builder
*build
, nir_ssa_def
*b
, uint32_t bit_size
)
1407 case 64: return nir_b2f64(build
, b
);
1408 case 32: return nir_b2f32(build
, b
);
1409 case 16: return nir_b2f16(build
, b
);
1411 unreachable("Invalid bit-size");
1415 static inline nir_ssa_def
*
1416 nir_b2i(nir_builder
*build
, nir_ssa_def
*b
, uint32_t bit_size
)
1419 case 64: return nir_b2i64(build
, b
);
1420 case 32: return nir_b2i32(build
, b
);
1421 case 16: return nir_b2i16(build
, b
);
1422 case 8: return nir_b2i8(build
, b
);
1424 unreachable("Invalid bit-size");
1427 static inline nir_ssa_def
*
1428 nir_load_barycentric(nir_builder
*build
, nir_intrinsic_op op
,
1429 unsigned interp_mode
)
1431 unsigned num_components
= op
== nir_intrinsic_load_barycentric_model
? 3 : 2;
1432 nir_intrinsic_instr
*bary
= nir_intrinsic_instr_create(build
->shader
, op
);
1433 nir_ssa_dest_init(&bary
->instr
, &bary
->dest
, num_components
, 32, NULL
);
1434 nir_intrinsic_set_interp_mode(bary
, interp_mode
);
1435 nir_builder_instr_insert(build
, &bary
->instr
);
1436 return &bary
->dest
.ssa
;
1440 nir_jump(nir_builder
*build
, nir_jump_type jump_type
)
1442 assert(jump_type
!= nir_jump_goto
&& jump_type
!= nir_jump_goto_if
);
1443 nir_jump_instr
*jump
= nir_jump_instr_create(build
->shader
, jump_type
);
1444 nir_builder_instr_insert(build
, &jump
->instr
);
1448 nir_goto(nir_builder
*build
, struct nir_block
*target
)
1450 assert(!build
->impl
->structured
);
1451 nir_jump_instr
*jump
= nir_jump_instr_create(build
->shader
, nir_jump_goto
);
1452 jump
->target
= target
;
1453 nir_builder_instr_insert(build
, &jump
->instr
);
1457 nir_goto_if(nir_builder
*build
, struct nir_block
*target
, nir_src cond
,
1458 struct nir_block
*else_target
)
1460 assert(!build
->impl
->structured
);
1461 nir_jump_instr
*jump
= nir_jump_instr_create(build
->shader
, nir_jump_goto_if
);
1462 jump
->condition
= cond
;
1463 jump
->target
= target
;
1464 jump
->else_target
= else_target
;
1465 nir_builder_instr_insert(build
, &jump
->instr
);
1468 static inline nir_ssa_def
*
1469 nir_compare_func(nir_builder
*b
, enum compare_func func
,
1470 nir_ssa_def
*src0
, nir_ssa_def
*src1
)
1473 case COMPARE_FUNC_NEVER
:
1474 return nir_imm_int(b
, 0);
1475 case COMPARE_FUNC_ALWAYS
:
1476 return nir_imm_int(b
, ~0);
1477 case COMPARE_FUNC_EQUAL
:
1478 return nir_feq(b
, src0
, src1
);
1479 case COMPARE_FUNC_NOTEQUAL
:
1480 return nir_fneu(b
, src0
, src1
);
1481 case COMPARE_FUNC_GREATER
:
1482 return nir_flt(b
, src1
, src0
);
1483 case COMPARE_FUNC_GEQUAL
:
1484 return nir_fge(b
, src0
, src1
);
1485 case COMPARE_FUNC_LESS
:
1486 return nir_flt(b
, src0
, src1
);
1487 case COMPARE_FUNC_LEQUAL
:
1488 return nir_fge(b
, src1
, src0
);
1490 unreachable("bad compare func");
1494 nir_scoped_barrier(nir_builder
*b
,
1495 nir_scope exec_scope
,
1496 nir_scope mem_scope
,
1497 nir_memory_semantics mem_semantics
,
1498 nir_variable_mode mem_modes
)
1500 nir_intrinsic_instr
*intrin
=
1501 nir_intrinsic_instr_create(b
->shader
, nir_intrinsic_scoped_barrier
);
1502 nir_intrinsic_set_execution_scope(intrin
, exec_scope
);
1503 nir_intrinsic_set_memory_scope(intrin
, mem_scope
);
1504 nir_intrinsic_set_memory_semantics(intrin
, mem_semantics
);
1505 nir_intrinsic_set_memory_modes(intrin
, mem_modes
);
1506 nir_builder_instr_insert(b
, &intrin
->instr
);
1510 nir_scoped_memory_barrier(nir_builder
*b
,
1512 nir_memory_semantics semantics
,
1513 nir_variable_mode modes
)
1515 nir_scoped_barrier(b
, NIR_SCOPE_NONE
, scope
, semantics
, modes
);
1518 static inline nir_ssa_def
*
1519 nir_convert_to_bit_size(nir_builder
*b
,
1524 nir_alu_type base_type
= nir_alu_type_get_base_type(type
);
1525 nir_alu_type dst_type
= (nir_alu_type
)(bit_size
| base_type
);
1528 nir_type_conversion_op(type
, dst_type
, nir_rounding_mode_undef
);
1530 return nir_build_alu(b
, opcode
, src
, NULL
, NULL
, NULL
);
1533 static inline nir_ssa_def
*
1534 nir_i2iN(nir_builder
*b
, nir_ssa_def
*src
, unsigned bit_size
)
1536 return nir_convert_to_bit_size(b
, src
, nir_type_int
, bit_size
);
1539 static inline nir_ssa_def
*
1540 nir_u2uN(nir_builder
*b
, nir_ssa_def
*src
, unsigned bit_size
)
1542 return nir_convert_to_bit_size(b
, src
, nir_type_uint
, bit_size
);
1545 static inline nir_ssa_def
*
1546 nir_b2bN(nir_builder
*b
, nir_ssa_def
*src
, unsigned bit_size
)
1548 return nir_convert_to_bit_size(b
, src
, nir_type_bool
, bit_size
);
1551 static inline nir_ssa_def
*
1552 nir_f2fN(nir_builder
*b
, nir_ssa_def
*src
, unsigned bit_size
)
1554 return nir_convert_to_bit_size(b
, src
, nir_type_float
, bit_size
);
1557 #endif /* NIR_BUILDER_H */