1 /**************************************************************************
3 * Copyright 2019 Red Hat.
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the "Software"),
8 * to deal in the Software without restriction, including without limitation
9 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
10 * and/or sell copies of the Software, and to permit persons to whom the
11 * Software is furnished to do so, subject to the following conditions:
13 * The above copyright notice and this permission notice shall be included
14 * in all copies or substantial portions of the Software.
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
17 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
24 **************************************************************************/
26 #include "lp_bld_nir.h"
27 #include "lp_bld_arit.h"
28 #include "lp_bld_bitarit.h"
29 #include "lp_bld_const.h"
30 #include "lp_bld_gather.h"
31 #include "lp_bld_logic.h"
32 #include "lp_bld_quad.h"
33 #include "lp_bld_flow.h"
34 #include "lp_bld_struct.h"
35 #include "lp_bld_debug.h"
36 #include "lp_bld_printf.h"
37 #include "nir_deref.h"
39 static void visit_cf_list(struct lp_build_nir_context
*bld_base
,
40 struct exec_list
*list
);
42 static LLVMValueRef
cast_type(struct lp_build_nir_context
*bld_base
, LLVMValueRef val
,
43 nir_alu_type alu_type
, unsigned bit_size
)
45 LLVMBuilderRef builder
= bld_base
->base
.gallivm
->builder
;
50 return LLVMBuildBitCast(builder
, val
, LLVMVectorType(LLVMHalfTypeInContext(bld_base
->base
.gallivm
->context
), bld_base
->base
.type
.length
), "");
52 return LLVMBuildBitCast(builder
, val
, bld_base
->base
.vec_type
, "");
54 return LLVMBuildBitCast(builder
, val
, bld_base
->dbl_bld
.vec_type
, "");
63 return LLVMBuildBitCast(builder
, val
, bld_base
->int8_bld
.vec_type
, "");
65 return LLVMBuildBitCast(builder
, val
, bld_base
->int16_bld
.vec_type
, "");
67 return LLVMBuildBitCast(builder
, val
, bld_base
->int_bld
.vec_type
, "");
69 return LLVMBuildBitCast(builder
, val
, bld_base
->int64_bld
.vec_type
, "");
78 return LLVMBuildBitCast(builder
, val
, bld_base
->uint8_bld
.vec_type
, "");
80 return LLVMBuildBitCast(builder
, val
, bld_base
->uint16_bld
.vec_type
, "");
82 return LLVMBuildBitCast(builder
, val
, bld_base
->uint_bld
.vec_type
, "");
84 return LLVMBuildBitCast(builder
, val
, bld_base
->uint64_bld
.vec_type
, "");
91 return LLVMBuildBitCast(builder
, val
, bld_base
->uint_bld
.vec_type
, "");
99 static struct lp_build_context
*get_flt_bld(struct lp_build_nir_context
*bld_base
,
100 unsigned op_bit_size
)
102 if (op_bit_size
== 64)
103 return &bld_base
->dbl_bld
;
105 return &bld_base
->base
;
108 static unsigned glsl_sampler_to_pipe(int sampler_dim
, bool is_array
)
110 unsigned pipe_target
= PIPE_BUFFER
;
111 switch (sampler_dim
) {
112 case GLSL_SAMPLER_DIM_1D
:
113 pipe_target
= is_array
? PIPE_TEXTURE_1D_ARRAY
: PIPE_TEXTURE_1D
;
115 case GLSL_SAMPLER_DIM_2D
:
116 case GLSL_SAMPLER_DIM_SUBPASS
:
117 pipe_target
= is_array
? PIPE_TEXTURE_2D_ARRAY
: PIPE_TEXTURE_2D
;
119 case GLSL_SAMPLER_DIM_3D
:
120 pipe_target
= PIPE_TEXTURE_3D
;
122 case GLSL_SAMPLER_DIM_MS
:
123 case GLSL_SAMPLER_DIM_SUBPASS_MS
:
124 pipe_target
= is_array
? PIPE_TEXTURE_2D_ARRAY
: PIPE_TEXTURE_2D
;
126 case GLSL_SAMPLER_DIM_CUBE
:
127 pipe_target
= is_array
? PIPE_TEXTURE_CUBE_ARRAY
: PIPE_TEXTURE_CUBE
;
129 case GLSL_SAMPLER_DIM_RECT
:
130 pipe_target
= PIPE_TEXTURE_RECT
;
132 case GLSL_SAMPLER_DIM_BUF
:
133 pipe_target
= PIPE_BUFFER
;
141 static LLVMValueRef
get_ssa_src(struct lp_build_nir_context
*bld_base
, nir_ssa_def
*ssa
)
143 return bld_base
->ssa_defs
[ssa
->index
];
146 static LLVMValueRef
get_src(struct lp_build_nir_context
*bld_base
, nir_src src
);
148 static LLVMValueRef
get_reg_src(struct lp_build_nir_context
*bld_base
, nir_reg_src src
)
150 struct hash_entry
*entry
= _mesa_hash_table_search(bld_base
->regs
, src
.reg
);
151 LLVMValueRef reg_storage
= (LLVMValueRef
)entry
->data
;
152 struct lp_build_context
*reg_bld
= get_int_bld(bld_base
, true, src
.reg
->bit_size
);
153 LLVMValueRef indir_src
= NULL
;
155 indir_src
= get_src(bld_base
, *src
.indirect
);
156 return bld_base
->load_reg(bld_base
, reg_bld
, &src
, indir_src
, reg_storage
);
159 static LLVMValueRef
get_src(struct lp_build_nir_context
*bld_base
, nir_src src
)
162 return get_ssa_src(bld_base
, src
.ssa
);
164 return get_reg_src(bld_base
, src
.reg
);
167 static void assign_ssa(struct lp_build_nir_context
*bld_base
, int idx
, LLVMValueRef ptr
)
169 bld_base
->ssa_defs
[idx
] = ptr
;
172 static void assign_ssa_dest(struct lp_build_nir_context
*bld_base
, const nir_ssa_def
*ssa
,
173 LLVMValueRef vals
[NIR_MAX_VEC_COMPONENTS
])
175 assign_ssa(bld_base
, ssa
->index
, ssa
->num_components
== 1 ? vals
[0] : lp_nir_array_build_gather_values(bld_base
->base
.gallivm
->builder
, vals
, ssa
->num_components
));
178 static void assign_reg(struct lp_build_nir_context
*bld_base
, const nir_reg_dest
*reg
,
180 LLVMValueRef vals
[NIR_MAX_VEC_COMPONENTS
])
182 struct hash_entry
*entry
= _mesa_hash_table_search(bld_base
->regs
, reg
->reg
);
183 LLVMValueRef reg_storage
= (LLVMValueRef
)entry
->data
;
184 struct lp_build_context
*reg_bld
= get_int_bld(bld_base
, true, reg
->reg
->bit_size
);
185 LLVMValueRef indir_src
= NULL
;
187 indir_src
= get_src(bld_base
, *reg
->indirect
);
188 bld_base
->store_reg(bld_base
, reg_bld
, reg
, write_mask
? write_mask
: 0xf, indir_src
, reg_storage
, vals
);
191 static void assign_dest(struct lp_build_nir_context
*bld_base
, const nir_dest
*dest
, LLVMValueRef vals
[NIR_MAX_VEC_COMPONENTS
])
194 assign_ssa_dest(bld_base
, &dest
->ssa
, vals
);
196 assign_reg(bld_base
, &dest
->reg
, 0, vals
);
199 static void assign_alu_dest(struct lp_build_nir_context
*bld_base
, const nir_alu_dest
*dest
, LLVMValueRef vals
[NIR_MAX_VEC_COMPONENTS
])
201 if (dest
->dest
.is_ssa
)
202 assign_ssa_dest(bld_base
, &dest
->dest
.ssa
, vals
);
204 assign_reg(bld_base
, &dest
->dest
.reg
, dest
->write_mask
, vals
);
207 static LLVMValueRef
int_to_bool32(struct lp_build_nir_context
*bld_base
,
208 uint32_t src_bit_size
,
212 LLVMBuilderRef builder
= bld_base
->base
.gallivm
->builder
;
213 struct lp_build_context
*int_bld
= get_int_bld(bld_base
, is_unsigned
, src_bit_size
);
214 LLVMValueRef result
= lp_build_compare(bld_base
->base
.gallivm
, int_bld
->type
, PIPE_FUNC_NOTEQUAL
, val
, int_bld
->zero
);
215 if (src_bit_size
== 64)
216 result
= LLVMBuildTrunc(builder
, result
, bld_base
->int_bld
.vec_type
, "");
220 static LLVMValueRef
flt_to_bool32(struct lp_build_nir_context
*bld_base
,
221 uint32_t src_bit_size
,
224 LLVMBuilderRef builder
= bld_base
->base
.gallivm
->builder
;
225 struct lp_build_context
*flt_bld
= get_flt_bld(bld_base
, src_bit_size
);
226 LLVMValueRef result
= lp_build_cmp(flt_bld
, PIPE_FUNC_NOTEQUAL
, val
, flt_bld
->zero
);
227 if (src_bit_size
== 64)
228 result
= LLVMBuildTrunc(builder
, result
, bld_base
->int_bld
.vec_type
, "");
232 static LLVMValueRef
fcmp32(struct lp_build_nir_context
*bld_base
,
233 enum pipe_compare_func compare
,
234 uint32_t src_bit_size
,
235 LLVMValueRef src
[NIR_MAX_VEC_COMPONENTS
])
237 LLVMBuilderRef builder
= bld_base
->base
.gallivm
->builder
;
238 struct lp_build_context
*flt_bld
= get_flt_bld(bld_base
, src_bit_size
);
241 if (compare
!= PIPE_FUNC_NOTEQUAL
)
242 result
= lp_build_cmp_ordered(flt_bld
, compare
, src
[0], src
[1]);
244 result
= lp_build_cmp(flt_bld
, compare
, src
[0], src
[1]);
245 if (src_bit_size
== 64)
246 result
= LLVMBuildTrunc(builder
, result
, bld_base
->int_bld
.vec_type
, "");
250 static LLVMValueRef
icmp32(struct lp_build_nir_context
*bld_base
,
251 enum pipe_compare_func compare
,
253 uint32_t src_bit_size
,
254 LLVMValueRef src
[NIR_MAX_VEC_COMPONENTS
])
256 LLVMBuilderRef builder
= bld_base
->base
.gallivm
->builder
;
257 struct lp_build_context
*i_bld
= get_int_bld(bld_base
, is_unsigned
, src_bit_size
);
258 LLVMValueRef result
= lp_build_cmp(i_bld
, compare
, src
[0], src
[1]);
259 if (src_bit_size
< 32)
260 result
= LLVMBuildSExt(builder
, result
, bld_base
->int_bld
.vec_type
, "");
261 else if (src_bit_size
== 64)
262 result
= LLVMBuildTrunc(builder
, result
, bld_base
->int_bld
.vec_type
, "");
266 static LLVMValueRef
get_alu_src(struct lp_build_nir_context
*bld_base
,
268 unsigned num_components
)
270 LLVMBuilderRef builder
= bld_base
->base
.gallivm
->builder
;
271 struct gallivm_state
*gallivm
= bld_base
->base
.gallivm
;
272 LLVMValueRef value
= get_src(bld_base
, src
.src
);
273 bool need_swizzle
= false;
276 unsigned src_components
= nir_src_num_components(src
.src
);
277 for (unsigned i
= 0; i
< num_components
; ++i
) {
278 assert(src
.swizzle
[i
] < src_components
);
279 if (src
.swizzle
[i
] != i
)
283 if (need_swizzle
|| num_components
!= src_components
) {
284 if (src_components
> 1 && num_components
== 1) {
285 value
= LLVMBuildExtractValue(gallivm
->builder
, value
,
287 } else if (src_components
== 1 && num_components
> 1) {
288 LLVMValueRef values
[] = {value
, value
, value
, value
, value
, value
, value
, value
, value
, value
, value
, value
, value
, value
, value
, value
};
289 value
= lp_nir_array_build_gather_values(builder
, values
, num_components
);
291 LLVMValueRef arr
= LLVMGetUndef(LLVMArrayType(LLVMTypeOf(LLVMBuildExtractValue(builder
, value
, 0, "")), num_components
));
292 for (unsigned i
= 0; i
< num_components
; i
++)
293 arr
= LLVMBuildInsertValue(builder
, arr
, LLVMBuildExtractValue(builder
, value
, src
.swizzle
[i
], ""), i
, "");
302 static LLVMValueRef
emit_b2f(struct lp_build_nir_context
*bld_base
,
306 LLVMBuilderRef builder
= bld_base
->base
.gallivm
->builder
;
307 LLVMValueRef result
= LLVMBuildAnd(builder
, cast_type(bld_base
, src0
, nir_type_int
, 32),
308 LLVMBuildBitCast(builder
, lp_build_const_vec(bld_base
->base
.gallivm
, bld_base
->base
.type
,
309 1.0), bld_base
->int_bld
.vec_type
, ""),
311 result
= LLVMBuildBitCast(builder
, result
, bld_base
->base
.vec_type
, "");
316 result
= LLVMBuildFPExt(builder
, result
, bld_base
->dbl_bld
.vec_type
, "");
319 unreachable("unsupported bit size.");
324 static LLVMValueRef
emit_b2i(struct lp_build_nir_context
*bld_base
,
328 LLVMBuilderRef builder
= bld_base
->base
.gallivm
->builder
;
329 LLVMValueRef result
= LLVMBuildAnd(builder
, cast_type(bld_base
, src0
, nir_type_int
, 32),
330 lp_build_const_int_vec(bld_base
->base
.gallivm
, bld_base
->base
.type
, 1), "");
335 return LLVMBuildZExt(builder
, result
, bld_base
->int64_bld
.vec_type
, "");
337 unreachable("unsupported bit size.");
341 static LLVMValueRef
emit_b32csel(struct lp_build_nir_context
*bld_base
,
342 unsigned src_bit_size
[NIR_MAX_VEC_COMPONENTS
],
343 LLVMValueRef src
[NIR_MAX_VEC_COMPONENTS
])
345 LLVMValueRef sel
= cast_type(bld_base
, src
[0], nir_type_int
, 32);
346 LLVMValueRef v
= lp_build_compare(bld_base
->base
.gallivm
, bld_base
->int_bld
.type
, PIPE_FUNC_NOTEQUAL
, sel
, bld_base
->int_bld
.zero
);
347 struct lp_build_context
*bld
= get_int_bld(bld_base
, false, src_bit_size
[1]);
348 return lp_build_select(bld
, v
, src
[1], src
[2]);
351 static LLVMValueRef
split_64bit(struct lp_build_nir_context
*bld_base
,
355 struct gallivm_state
*gallivm
= bld_base
->base
.gallivm
;
356 LLVMValueRef shuffles
[LP_MAX_VECTOR_WIDTH
/32];
357 LLVMValueRef shuffles2
[LP_MAX_VECTOR_WIDTH
/32];
358 int len
= bld_base
->base
.type
.length
* 2;
359 for (unsigned i
= 0; i
< bld_base
->base
.type
.length
; i
++) {
360 #if UTIL_ARCH_LITTLE_ENDIAN
361 shuffles
[i
] = lp_build_const_int32(gallivm
, i
* 2);
362 shuffles2
[i
] = lp_build_const_int32(gallivm
, (i
* 2) + 1);
364 shuffles
[i
] = lp_build_const_int32(gallivm
, (i
* 2) + 1);
365 shuffles2
[i
] = lp_build_const_int32(gallivm
, (i
* 2));
369 src
= LLVMBuildBitCast(gallivm
->builder
, src
, LLVMVectorType(LLVMInt32TypeInContext(gallivm
->context
), len
), "");
370 return LLVMBuildShuffleVector(gallivm
->builder
, src
,
371 LLVMGetUndef(LLVMTypeOf(src
)),
372 LLVMConstVector(hi
? shuffles2
: shuffles
,
373 bld_base
->base
.type
.length
),
378 merge_64bit(struct lp_build_nir_context
*bld_base
,
382 struct gallivm_state
*gallivm
= bld_base
->base
.gallivm
;
383 LLVMBuilderRef builder
= gallivm
->builder
;
385 LLVMValueRef shuffles
[2 * (LP_MAX_VECTOR_WIDTH
/32)];
386 int len
= bld_base
->base
.type
.length
* 2;
387 assert(len
<= (2 * (LP_MAX_VECTOR_WIDTH
/32)));
389 for (i
= 0; i
< bld_base
->base
.type
.length
* 2; i
+=2) {
390 #if UTIL_ARCH_LITTLE_ENDIAN
391 shuffles
[i
] = lp_build_const_int32(gallivm
, i
/ 2);
392 shuffles
[i
+ 1] = lp_build_const_int32(gallivm
, i
/ 2 + bld_base
->base
.type
.length
);
394 shuffles
[i
] = lp_build_const_int32(gallivm
, i
/ 2 + bld_base
->base
.type
.length
);
395 shuffles
[i
+ 1] = lp_build_const_int32(gallivm
, i
/ 2);
398 return LLVMBuildShuffleVector(builder
, input
, input2
, LLVMConstVector(shuffles
, len
), "");
402 do_int_divide(struct lp_build_nir_context
*bld_base
,
403 bool is_unsigned
, unsigned src_bit_size
,
404 LLVMValueRef src
, LLVMValueRef src2
)
406 struct gallivm_state
*gallivm
= bld_base
->base
.gallivm
;
407 LLVMBuilderRef builder
= gallivm
->builder
;
408 struct lp_build_context
*int_bld
= get_int_bld(bld_base
, is_unsigned
, src_bit_size
);
409 struct lp_build_context
*mask_bld
= get_int_bld(bld_base
, true, src_bit_size
);
410 LLVMValueRef div_mask
= lp_build_cmp(mask_bld
, PIPE_FUNC_EQUAL
, src2
,
414 /* INT_MIN (0x80000000) / -1 (0xffffffff) causes sigfpe, seen with blender. */
415 div_mask
= LLVMBuildAnd(builder
, div_mask
, lp_build_const_int_vec(gallivm
, int_bld
->type
, 0x7fffffff), "");
417 LLVMValueRef divisor
= LLVMBuildOr(builder
,
420 LLVMValueRef result
= lp_build_div(int_bld
, src
, divisor
);
423 LLVMValueRef not_div_mask
= LLVMBuildNot(builder
, div_mask
, "");
424 return LLVMBuildAnd(builder
, not_div_mask
, result
, "");
426 /* udiv by zero is guaranteed to return 0xffffffff at least with d3d10
427 * may as well do same for idiv */
428 return LLVMBuildOr(builder
, div_mask
, result
, "");
432 do_int_mod(struct lp_build_nir_context
*bld_base
,
433 bool is_unsigned
, unsigned src_bit_size
,
434 LLVMValueRef src
, LLVMValueRef src2
)
436 struct gallivm_state
*gallivm
= bld_base
->base
.gallivm
;
437 LLVMBuilderRef builder
= gallivm
->builder
;
438 struct lp_build_context
*int_bld
= get_int_bld(bld_base
, is_unsigned
, src_bit_size
);
439 LLVMValueRef div_mask
= lp_build_cmp(int_bld
, PIPE_FUNC_EQUAL
, src2
,
441 LLVMValueRef divisor
= LLVMBuildOr(builder
,
444 LLVMValueRef result
= lp_build_mod(int_bld
, src
, divisor
);
445 return LLVMBuildOr(builder
, div_mask
, result
, "");
449 do_quantize_to_f16(struct lp_build_nir_context
*bld_base
,
452 struct gallivm_state
*gallivm
= bld_base
->base
.gallivm
;
453 LLVMBuilderRef builder
= gallivm
->builder
;
455 result
= LLVMBuildFPTrunc(builder
, src
, LLVMVectorType(LLVMHalfTypeInContext(gallivm
->context
), bld_base
->base
.type
.length
), "");
456 result
= LLVMBuildFPExt(builder
, result
, bld_base
->base
.vec_type
, "");
460 static LLVMValueRef
do_alu_action(struct lp_build_nir_context
*bld_base
,
461 nir_op op
, unsigned src_bit_size
[NIR_MAX_VEC_COMPONENTS
], LLVMValueRef src
[NIR_MAX_VEC_COMPONENTS
])
463 struct gallivm_state
*gallivm
= bld_base
->base
.gallivm
;
464 LLVMBuilderRef builder
= gallivm
->builder
;
468 result
= emit_b2f(bld_base
, src
[0], 32);
471 result
= emit_b2f(bld_base
, src
[0], 64);
474 result
= emit_b2i(bld_base
, src
[0], 32);
477 result
= emit_b2i(bld_base
, src
[0], 64);
480 result
= emit_b32csel(bld_base
, src_bit_size
, src
);
482 case nir_op_bit_count
:
483 result
= lp_build_popcount(get_int_bld(bld_base
, false, src_bit_size
[0]), src
[0]);
485 case nir_op_bitfield_select
:
486 result
= lp_build_xor(&bld_base
->uint_bld
, src
[2], lp_build_and(&bld_base
->uint_bld
, src
[0], lp_build_xor(&bld_base
->uint_bld
, src
[1], src
[2])));
488 case nir_op_bitfield_reverse
:
489 result
= lp_build_bitfield_reverse(get_int_bld(bld_base
, false, src_bit_size
[0]), src
[0]);
492 result
= flt_to_bool32(bld_base
, src_bit_size
[0], src
[0]);
495 result
= LLVMBuildFPTrunc(builder
, src
[0],
496 LLVMVectorType(LLVMHalfTypeInContext(gallivm
->context
), bld_base
->base
.type
.length
), "");
499 if (src_bit_size
[0] < 32)
500 result
= LLVMBuildFPExt(builder
, src
[0],
501 bld_base
->base
.vec_type
, "");
503 result
= LLVMBuildFPTrunc(builder
, src
[0],
504 bld_base
->base
.vec_type
, "");
507 result
= LLVMBuildFPExt(builder
, src
[0],
508 bld_base
->dbl_bld
.vec_type
, "");
511 result
= LLVMBuildFPToSI(builder
, src
[0], bld_base
->base
.int_vec_type
, "");
514 result
= LLVMBuildFPToUI(builder
,
516 bld_base
->base
.int_vec_type
, "");
519 result
= LLVMBuildFPToSI(builder
,
521 bld_base
->int64_bld
.vec_type
, "");
524 result
= LLVMBuildFPToUI(builder
,
526 bld_base
->uint64_bld
.vec_type
, "");
529 result
= lp_build_abs(get_flt_bld(bld_base
, src_bit_size
[0]), src
[0]);
532 result
= lp_build_add(get_flt_bld(bld_base
, src_bit_size
[0]),
536 result
= lp_build_ceil(get_flt_bld(bld_base
, src_bit_size
[0]), src
[0]);
539 result
= lp_build_cos(&bld_base
->base
, src
[0]);
542 case nir_op_fddx_coarse
:
543 case nir_op_fddx_fine
:
544 result
= lp_build_ddx(&bld_base
->base
, src
[0]);
547 case nir_op_fddy_coarse
:
548 case nir_op_fddy_fine
:
549 result
= lp_build_ddy(&bld_base
->base
, src
[0]);
552 result
= lp_build_div(get_flt_bld(bld_base
, src_bit_size
[0]),
556 result
= fcmp32(bld_base
, PIPE_FUNC_EQUAL
, src_bit_size
[0], src
);
559 result
= lp_build_exp2(&bld_base
->base
, src
[0]);
562 result
= lp_build_floor(get_flt_bld(bld_base
, src_bit_size
[0]), src
[0]);
565 result
= lp_build_fmuladd(builder
, src
[0], src
[1], src
[2]);
567 case nir_op_ffract
: {
568 struct lp_build_context
*flt_bld
= get_flt_bld(bld_base
, src_bit_size
[0]);
569 LLVMValueRef tmp
= lp_build_floor(flt_bld
, src
[0]);
570 result
= lp_build_sub(flt_bld
, src
[0], tmp
);
574 result
= fcmp32(bld_base
, PIPE_FUNC_GEQUAL
, src_bit_size
[0], src
);
576 case nir_op_find_lsb
:
577 result
= lp_build_cttz(get_int_bld(bld_base
, false, src_bit_size
[0]), src
[0]);
580 result
= lp_build_log2_safe(&bld_base
->base
, src
[0]);
583 result
= fcmp32(bld_base
, PIPE_FUNC_LESS
, src_bit_size
[0], src
);
586 result
= lp_build_min(get_flt_bld(bld_base
, src_bit_size
[0]), src
[0], src
[1]);
589 struct lp_build_context
*flt_bld
= get_flt_bld(bld_base
, src_bit_size
[0]);
590 result
= lp_build_div(flt_bld
, src
[0], src
[1]);
591 result
= lp_build_floor(flt_bld
, result
);
592 result
= lp_build_mul(flt_bld
, src
[1], result
);
593 result
= lp_build_sub(flt_bld
, src
[0], result
);
597 result
= lp_build_mul(get_flt_bld(bld_base
, src_bit_size
[0]),
601 result
= lp_build_max(get_flt_bld(bld_base
, src_bit_size
[0]), src
[0], src
[1]);
604 result
= fcmp32(bld_base
, PIPE_FUNC_NOTEQUAL
, src_bit_size
[0], src
);
607 result
= lp_build_negate(get_flt_bld(bld_base
, src_bit_size
[0]), src
[0]);
610 result
= lp_build_pow(&bld_base
->base
, src
[0], src
[1]);
612 case nir_op_fquantize2f16
:
613 result
= do_quantize_to_f16(bld_base
, src
[0]);
616 result
= lp_build_rcp(get_flt_bld(bld_base
, src_bit_size
[0]), src
[0]);
618 case nir_op_fround_even
:
619 result
= lp_build_round(get_flt_bld(bld_base
, src_bit_size
[0]), src
[0]);
622 result
= lp_build_rsqrt(get_flt_bld(bld_base
, src_bit_size
[0]), src
[0]);
625 result
= lp_build_clamp_zero_one_nanzero(get_flt_bld(bld_base
, src_bit_size
[0]), src
[0]);
628 result
= lp_build_sgn(get_flt_bld(bld_base
, src_bit_size
[0]), src
[0]);
631 result
= lp_build_sin(&bld_base
->base
, src
[0]);
634 result
= lp_build_sqrt(get_flt_bld(bld_base
, src_bit_size
[0]), src
[0]);
637 result
= lp_build_trunc(get_flt_bld(bld_base
, src_bit_size
[0]), src
[0]);
640 result
= int_to_bool32(bld_base
, src_bit_size
[0], false, src
[0]);
643 result
= lp_build_int_to_float(&bld_base
->base
, src
[0]);
646 result
= lp_build_int_to_float(&bld_base
->dbl_bld
, src
[0]);
649 result
= LLVMBuildTrunc(builder
, src
[0], bld_base
->int8_bld
.vec_type
, "");
652 if (src_bit_size
[0] < 16)
653 result
= LLVMBuildSExt(builder
, src
[0], bld_base
->int16_bld
.vec_type
, "");
655 result
= LLVMBuildTrunc(builder
, src
[0], bld_base
->int16_bld
.vec_type
, "");
658 if (src_bit_size
[0] < 32)
659 result
= LLVMBuildSExt(builder
, src
[0], bld_base
->int_bld
.vec_type
, "");
661 result
= LLVMBuildTrunc(builder
, src
[0], bld_base
->int_bld
.vec_type
, "");
664 result
= LLVMBuildSExt(builder
, src
[0], bld_base
->int64_bld
.vec_type
, "");
667 result
= lp_build_abs(get_int_bld(bld_base
, false, src_bit_size
[0]), src
[0]);
670 result
= lp_build_add(get_int_bld(bld_base
, false, src_bit_size
[0]),
674 result
= lp_build_and(get_int_bld(bld_base
, false, src_bit_size
[0]),
678 result
= do_int_divide(bld_base
, false, src_bit_size
[0], src
[0], src
[1]);
681 result
= icmp32(bld_base
, PIPE_FUNC_EQUAL
, false, src_bit_size
[0], src
);
684 result
= icmp32(bld_base
, PIPE_FUNC_GEQUAL
, false, src_bit_size
[0], src
);
687 result
= icmp32(bld_base
, PIPE_FUNC_LESS
, false, src_bit_size
[0], src
);
690 result
= lp_build_max(get_int_bld(bld_base
, false, src_bit_size
[0]), src
[0], src
[1]);
693 result
= lp_build_min(get_int_bld(bld_base
, false, src_bit_size
[0]), src
[0], src
[1]);
697 result
= lp_build_mul(get_int_bld(bld_base
, false, src_bit_size
[0]),
700 case nir_op_imul_high
: {
701 LLVMValueRef hi_bits
;
702 lp_build_mul_32_lohi(&bld_base
->int_bld
, src
[0], src
[1], &hi_bits
);
707 result
= icmp32(bld_base
, PIPE_FUNC_NOTEQUAL
, false, src_bit_size
[0], src
);
710 result
= lp_build_negate(get_int_bld(bld_base
, false, src_bit_size
[0]), src
[0]);
713 result
= lp_build_not(get_int_bld(bld_base
, false, src_bit_size
[0]), src
[0]);
716 result
= lp_build_or(get_int_bld(bld_base
, false, src_bit_size
[0]),
721 result
= do_int_mod(bld_base
, false, src_bit_size
[0], src
[0], src
[1]);
724 struct lp_build_context
*uint_bld
= get_int_bld(bld_base
, true, src_bit_size
[0]);
725 struct lp_build_context
*int_bld
= get_int_bld(bld_base
, false, src_bit_size
[0]);
726 if (src_bit_size
[0] == 64)
727 src
[1] = LLVMBuildZExt(builder
, src
[1], uint_bld
->vec_type
, "");
728 if (src_bit_size
[0] < 32)
729 src
[1] = LLVMBuildTrunc(builder
, src
[1], uint_bld
->vec_type
, "");
730 src
[1] = lp_build_and(uint_bld
, src
[1], lp_build_const_int_vec(gallivm
, uint_bld
->type
, (src_bit_size
[0] - 1)));
731 result
= lp_build_shl(int_bld
, src
[0], src
[1]);
735 struct lp_build_context
*uint_bld
= get_int_bld(bld_base
, true, src_bit_size
[0]);
736 struct lp_build_context
*int_bld
= get_int_bld(bld_base
, false, src_bit_size
[0]);
737 if (src_bit_size
[0] == 64)
738 src
[1] = LLVMBuildZExt(builder
, src
[1], uint_bld
->vec_type
, "");
739 if (src_bit_size
[0] < 32)
740 src
[1] = LLVMBuildTrunc(builder
, src
[1], uint_bld
->vec_type
, "");
741 src
[1] = lp_build_and(uint_bld
, src
[1], lp_build_const_int_vec(gallivm
, uint_bld
->type
, (src_bit_size
[0] - 1)));
742 result
= lp_build_shr(int_bld
, src
[0], src
[1]);
746 result
= lp_build_sgn(get_int_bld(bld_base
, false, src_bit_size
[0]), src
[0]);
749 result
= lp_build_sub(get_int_bld(bld_base
, false, src_bit_size
[0]),
753 result
= lp_build_xor(get_int_bld(bld_base
, false, src_bit_size
[0]),
759 case nir_op_unpack_64_2x32_split_x
:
760 result
= split_64bit(bld_base
, src
[0], false);
762 case nir_op_unpack_64_2x32_split_y
:
763 result
= split_64bit(bld_base
, src
[0], true);
766 case nir_op_pack_64_2x32_split
: {
767 LLVMValueRef tmp
= merge_64bit(bld_base
, src
[0], src
[1]);
768 result
= LLVMBuildBitCast(builder
, tmp
, bld_base
->dbl_bld
.vec_type
, "");
772 result
= LLVMBuildUIToFP(builder
, src
[0], bld_base
->base
.vec_type
, "");
775 result
= LLVMBuildUIToFP(builder
, src
[0], bld_base
->dbl_bld
.vec_type
, "");
778 result
= LLVMBuildTrunc(builder
, src
[0], bld_base
->uint8_bld
.vec_type
, "");
781 if (src_bit_size
[0] < 16)
782 result
= LLVMBuildZExt(builder
, src
[0], bld_base
->uint16_bld
.vec_type
, "");
784 result
= LLVMBuildTrunc(builder
, src
[0], bld_base
->uint16_bld
.vec_type
, "");
787 if (src_bit_size
[0] < 32)
788 result
= LLVMBuildZExt(builder
, src
[0], bld_base
->uint_bld
.vec_type
, "");
790 result
= LLVMBuildTrunc(builder
, src
[0], bld_base
->uint_bld
.vec_type
, "");
793 result
= LLVMBuildZExt(builder
, src
[0], bld_base
->uint64_bld
.vec_type
, "");
796 result
= do_int_divide(bld_base
, true, src_bit_size
[0], src
[0], src
[1]);
798 case nir_op_ufind_msb
: {
799 struct lp_build_context
*uint_bld
= get_int_bld(bld_base
, true, src_bit_size
[0]);
800 result
= lp_build_ctlz(uint_bld
, src
[0]);
801 result
= lp_build_sub(uint_bld
, lp_build_const_int_vec(gallivm
, uint_bld
->type
, src_bit_size
[0] - 1), result
);
805 result
= icmp32(bld_base
, PIPE_FUNC_GEQUAL
, true, src_bit_size
[0], src
);
808 result
= icmp32(bld_base
, PIPE_FUNC_LESS
, true, src_bit_size
[0], src
);
811 result
= lp_build_max(get_int_bld(bld_base
, true, src_bit_size
[0]), src
[0], src
[1]);
814 result
= lp_build_min(get_int_bld(bld_base
, true, src_bit_size
[0]), src
[0], src
[1]);
817 result
= do_int_mod(bld_base
, true, src_bit_size
[0], src
[0], src
[1]);
819 case nir_op_umul_high
: {
820 LLVMValueRef hi_bits
;
821 lp_build_mul_32_lohi(&bld_base
->uint_bld
, src
[0], src
[1], &hi_bits
);
826 struct lp_build_context
*uint_bld
= get_int_bld(bld_base
, true, src_bit_size
[0]);
827 if (src_bit_size
[0] == 64)
828 src
[1] = LLVMBuildZExt(builder
, src
[1], uint_bld
->vec_type
, "");
829 if (src_bit_size
[0] < 32)
830 src
[1] = LLVMBuildTrunc(builder
, src
[1], uint_bld
->vec_type
, "");
831 src
[1] = lp_build_and(uint_bld
, src
[1], lp_build_const_int_vec(gallivm
, uint_bld
->type
, (src_bit_size
[0] - 1)));
832 result
= lp_build_shr(uint_bld
, src
[0], src
[1]);
842 static void visit_alu(struct lp_build_nir_context
*bld_base
, const nir_alu_instr
*instr
)
844 struct gallivm_state
*gallivm
= bld_base
->base
.gallivm
;
845 LLVMValueRef src
[NIR_MAX_VEC_COMPONENTS
];
846 unsigned src_bit_size
[NIR_MAX_VEC_COMPONENTS
];
847 unsigned num_components
= nir_dest_num_components(instr
->dest
.dest
);
848 unsigned src_components
;
857 case nir_op_pack_half_2x16
:
860 case nir_op_unpack_half_2x16
:
863 case nir_op_cube_face_coord
:
864 case nir_op_cube_face_index
:
868 src_components
= num_components
;
871 for (unsigned i
= 0; i
< nir_op_infos
[instr
->op
].num_inputs
; i
++) {
872 src
[i
] = get_alu_src(bld_base
, instr
->src
[i
], src_components
);
873 src_bit_size
[i
] = nir_src_bit_size(instr
->src
[i
].src
);
876 LLVMValueRef result
[NIR_MAX_VEC_COMPONENTS
];
877 if (instr
->op
== nir_op_vec4
|| instr
->op
== nir_op_vec3
|| instr
->op
== nir_op_vec2
|| instr
->op
== nir_op_vec8
|| instr
->op
== nir_op_vec16
) {
878 for (unsigned i
= 0; i
< nir_op_infos
[instr
->op
].num_inputs
; i
++) {
879 result
[i
] = cast_type(bld_base
, src
[i
], nir_op_infos
[instr
->op
].input_types
[i
], src_bit_size
[i
]);
882 for (unsigned c
= 0; c
< num_components
; c
++) {
883 LLVMValueRef src_chan
[NIR_MAX_VEC_COMPONENTS
];
885 for (unsigned i
= 0; i
< nir_op_infos
[instr
->op
].num_inputs
; i
++) {
886 if (num_components
> 1) {
887 src_chan
[i
] = LLVMBuildExtractValue(gallivm
->builder
,
890 src_chan
[i
] = src
[i
];
891 src_chan
[i
] = cast_type(bld_base
, src_chan
[i
], nir_op_infos
[instr
->op
].input_types
[i
], src_bit_size
[i
]);
893 result
[c
] = do_alu_action(bld_base
, instr
->op
, src_bit_size
, src_chan
);
894 result
[c
] = cast_type(bld_base
, result
[c
], nir_op_infos
[instr
->op
].output_type
, nir_dest_bit_size(instr
->dest
.dest
));
897 assign_alu_dest(bld_base
, &instr
->dest
, result
);
900 static void visit_load_const(struct lp_build_nir_context
*bld_base
,
901 const nir_load_const_instr
*instr
)
903 LLVMValueRef result
[NIR_MAX_VEC_COMPONENTS
];
904 struct lp_build_context
*int_bld
= get_int_bld(bld_base
, true, instr
->def
.bit_size
);
905 for (unsigned i
= 0; i
< instr
->def
.num_components
; i
++)
906 result
[i
] = lp_build_const_int_vec(bld_base
->base
.gallivm
, int_bld
->type
, instr
->def
.bit_size
== 32 ? instr
->value
[i
].u32
: instr
->value
[i
].u64
);
907 assign_ssa_dest(bld_base
, &instr
->def
, result
);
911 get_deref_offset(struct lp_build_nir_context
*bld_base
, nir_deref_instr
*instr
,
912 bool vs_in
, unsigned *vertex_index_out
,
913 LLVMValueRef
*vertex_index_ref
,
914 unsigned *const_out
, LLVMValueRef
*indir_out
)
916 LLVMBuilderRef builder
= bld_base
->base
.gallivm
->builder
;
917 nir_variable
*var
= nir_deref_instr_get_variable(instr
);
919 unsigned idx_lvl
= 1;
921 nir_deref_path_init(&path
, instr
, NULL
);
923 if (vertex_index_out
!= NULL
|| vertex_index_ref
!= NULL
) {
924 if (vertex_index_ref
) {
925 *vertex_index_ref
= get_src(bld_base
, path
.path
[idx_lvl
]->arr
.index
);
926 if (vertex_index_out
)
927 *vertex_index_out
= 0;
929 *vertex_index_out
= nir_src_as_uint(path
.path
[idx_lvl
]->arr
.index
);
934 uint32_t const_offset
= 0;
935 LLVMValueRef offset
= NULL
;
937 if (var
->data
.compact
&& nir_src_is_const(instr
->arr
.index
)) {
938 assert(instr
->deref_type
== nir_deref_type_array
);
939 const_offset
= nir_src_as_uint(instr
->arr
.index
);
943 for (; path
.path
[idx_lvl
]; ++idx_lvl
) {
944 const struct glsl_type
*parent_type
= path
.path
[idx_lvl
- 1]->type
;
945 if (path
.path
[idx_lvl
]->deref_type
== nir_deref_type_struct
) {
946 unsigned index
= path
.path
[idx_lvl
]->strct
.index
;
948 for (unsigned i
= 0; i
< index
; i
++) {
949 const struct glsl_type
*ft
= glsl_get_struct_field(parent_type
, i
);
950 const_offset
+= glsl_count_attribute_slots(ft
, vs_in
);
952 } else if(path
.path
[idx_lvl
]->deref_type
== nir_deref_type_array
) {
953 unsigned size
= glsl_count_attribute_slots(path
.path
[idx_lvl
]->type
, vs_in
);
954 if (nir_src_is_const(path
.path
[idx_lvl
]->arr
.index
)) {
955 const_offset
+= nir_src_comp_as_int(path
.path
[idx_lvl
]->arr
.index
, 0) * size
;
957 LLVMValueRef idx_src
= get_src(bld_base
, path
.path
[idx_lvl
]->arr
.index
);
958 idx_src
= cast_type(bld_base
, idx_src
, nir_type_uint
, 32);
959 LLVMValueRef array_off
= lp_build_mul(&bld_base
->uint_bld
, lp_build_const_int_vec(bld_base
->base
.gallivm
, bld_base
->base
.type
, size
),
962 offset
= lp_build_add(&bld_base
->uint_bld
, offset
, array_off
);
967 unreachable("Uhandled deref type in get_deref_instr_offset");
971 nir_deref_path_finish(&path
);
973 if (const_offset
&& offset
)
974 offset
= LLVMBuildAdd(builder
, offset
,
975 lp_build_const_int_vec(bld_base
->base
.gallivm
, bld_base
->uint_bld
.type
, const_offset
),
977 *const_out
= const_offset
;
981 static void visit_load_var(struct lp_build_nir_context
*bld_base
,
982 nir_intrinsic_instr
*instr
,
983 LLVMValueRef result
[NIR_MAX_VEC_COMPONENTS
])
985 nir_deref_instr
*deref
= nir_instr_as_deref(instr
->src
[0].ssa
->parent_instr
);
986 nir_variable
*var
= nir_deref_instr_get_variable(deref
);
987 nir_variable_mode mode
= deref
->mode
;
988 unsigned const_index
;
989 LLVMValueRef indir_index
;
990 LLVMValueRef indir_vertex_index
= NULL
;
991 unsigned vertex_index
= 0;
992 unsigned nc
= nir_dest_num_components(instr
->dest
);
993 unsigned bit_size
= nir_dest_bit_size(instr
->dest
);
995 bool vs_in
= bld_base
->shader
->info
.stage
== MESA_SHADER_VERTEX
&&
996 var
->data
.mode
== nir_var_shader_in
;
997 bool gs_in
= bld_base
->shader
->info
.stage
== MESA_SHADER_GEOMETRY
&&
998 var
->data
.mode
== nir_var_shader_in
;
999 bool tcs_in
= bld_base
->shader
->info
.stage
== MESA_SHADER_TESS_CTRL
&&
1000 var
->data
.mode
== nir_var_shader_in
;
1001 bool tcs_out
= bld_base
->shader
->info
.stage
== MESA_SHADER_TESS_CTRL
&&
1002 var
->data
.mode
== nir_var_shader_out
&& !var
->data
.patch
;
1003 bool tes_in
= bld_base
->shader
->info
.stage
== MESA_SHADER_TESS_EVAL
&&
1004 var
->data
.mode
== nir_var_shader_in
&& !var
->data
.patch
;
1006 mode
= var
->data
.mode
;
1008 get_deref_offset(bld_base
, deref
, vs_in
, gs_in
? &vertex_index
: NULL
, (tcs_in
|| tcs_out
|| tes_in
) ? &indir_vertex_index
: NULL
,
1009 &const_index
, &indir_index
);
1011 bld_base
->load_var(bld_base
, mode
, nc
, bit_size
, var
, vertex_index
, indir_vertex_index
, const_index
, indir_index
, result
);
1015 visit_store_var(struct lp_build_nir_context
*bld_base
,
1016 nir_intrinsic_instr
*instr
)
1018 nir_deref_instr
*deref
= nir_instr_as_deref(instr
->src
[0].ssa
->parent_instr
);
1019 nir_variable
*var
= nir_deref_instr_get_variable(deref
);
1020 nir_variable_mode mode
= deref
->mode
;
1021 int writemask
= instr
->const_index
[0];
1022 unsigned bit_size
= nir_src_bit_size(instr
->src
[1]);
1023 LLVMValueRef src
= get_src(bld_base
, instr
->src
[1]);
1024 unsigned const_index
= 0;
1025 LLVMValueRef indir_index
, indir_vertex_index
= NULL
;
1027 bool tcs_out
= bld_base
->shader
->info
.stage
== MESA_SHADER_TESS_CTRL
&&
1028 var
->data
.mode
== nir_var_shader_out
&& !var
->data
.patch
;
1029 get_deref_offset(bld_base
, deref
, false, NULL
, tcs_out
? &indir_vertex_index
: NULL
,
1030 &const_index
, &indir_index
);
1032 bld_base
->store_var(bld_base
, mode
, instr
->num_components
, bit_size
, var
, writemask
, indir_vertex_index
, const_index
, indir_index
, src
);
1035 static void visit_load_ubo(struct lp_build_nir_context
*bld_base
,
1036 nir_intrinsic_instr
*instr
,
1037 LLVMValueRef result
[NIR_MAX_VEC_COMPONENTS
])
1039 struct gallivm_state
*gallivm
= bld_base
->base
.gallivm
;
1040 LLVMBuilderRef builder
= gallivm
->builder
;
1041 LLVMValueRef idx
= get_src(bld_base
, instr
->src
[0]);
1042 LLVMValueRef offset
= get_src(bld_base
, instr
->src
[1]);
1044 bool offset_is_uniform
= nir_src_is_dynamically_uniform(instr
->src
[1]);
1045 idx
= LLVMBuildExtractElement(builder
, idx
, lp_build_const_int32(gallivm
, 0), "");
1046 bld_base
->load_ubo(bld_base
, nir_dest_num_components(instr
->dest
), nir_dest_bit_size(instr
->dest
),
1047 offset_is_uniform
, idx
, offset
, result
);
1050 static void visit_load_push_constant(struct lp_build_nir_context
*bld_base
,
1051 nir_intrinsic_instr
*instr
,
1052 LLVMValueRef result
[4])
1054 struct gallivm_state
*gallivm
= bld_base
->base
.gallivm
;
1055 LLVMValueRef offset
= get_src(bld_base
, instr
->src
[0]);
1056 LLVMValueRef idx
= lp_build_const_int32(gallivm
, 0);
1057 bool offset_is_uniform
= nir_src_is_dynamically_uniform(instr
->src
[0]);
1059 bld_base
->load_ubo(bld_base
, nir_dest_num_components(instr
->dest
), nir_dest_bit_size(instr
->dest
),
1060 offset_is_uniform
, idx
, offset
, result
);
1064 static void visit_load_ssbo(struct lp_build_nir_context
*bld_base
,
1065 nir_intrinsic_instr
*instr
,
1066 LLVMValueRef result
[NIR_MAX_VEC_COMPONENTS
])
1068 LLVMValueRef idx
= get_src(bld_base
, instr
->src
[0]);
1069 LLVMValueRef offset
= get_src(bld_base
, instr
->src
[1]);
1070 bld_base
->load_mem(bld_base
, nir_dest_num_components(instr
->dest
), nir_dest_bit_size(instr
->dest
),
1071 idx
, offset
, result
);
1074 static void visit_store_ssbo(struct lp_build_nir_context
*bld_base
,
1075 nir_intrinsic_instr
*instr
)
1077 LLVMValueRef val
= get_src(bld_base
, instr
->src
[0]);
1078 LLVMValueRef idx
= get_src(bld_base
, instr
->src
[1]);
1079 LLVMValueRef offset
= get_src(bld_base
, instr
->src
[2]);
1080 int writemask
= instr
->const_index
[0];
1081 int nc
= nir_src_num_components(instr
->src
[0]);
1082 int bitsize
= nir_src_bit_size(instr
->src
[0]);
1083 bld_base
->store_mem(bld_base
, writemask
, nc
, bitsize
, idx
, offset
, val
);
1086 static void visit_get_buffer_size(struct lp_build_nir_context
*bld_base
,
1087 nir_intrinsic_instr
*instr
,
1088 LLVMValueRef result
[NIR_MAX_VEC_COMPONENTS
])
1090 LLVMValueRef idx
= get_src(bld_base
, instr
->src
[0]);
1091 result
[0] = bld_base
->get_buffer_size(bld_base
, idx
);
1094 static void visit_ssbo_atomic(struct lp_build_nir_context
*bld_base
,
1095 nir_intrinsic_instr
*instr
,
1096 LLVMValueRef result
[NIR_MAX_VEC_COMPONENTS
])
1098 LLVMValueRef idx
= get_src(bld_base
, instr
->src
[0]);
1099 LLVMValueRef offset
= get_src(bld_base
, instr
->src
[1]);
1100 LLVMValueRef val
= get_src(bld_base
, instr
->src
[2]);
1101 LLVMValueRef val2
= NULL
;
1102 if (instr
->intrinsic
== nir_intrinsic_ssbo_atomic_comp_swap
)
1103 val2
= get_src(bld_base
, instr
->src
[3]);
1105 bld_base
->atomic_mem(bld_base
, instr
->intrinsic
, idx
, offset
, val
, val2
, &result
[0]);
1109 static void visit_load_image(struct lp_build_nir_context
*bld_base
,
1110 nir_intrinsic_instr
*instr
,
1111 LLVMValueRef result
[NIR_MAX_VEC_COMPONENTS
])
1113 struct gallivm_state
*gallivm
= bld_base
->base
.gallivm
;
1114 LLVMBuilderRef builder
= gallivm
->builder
;
1115 nir_deref_instr
*deref
= nir_instr_as_deref(instr
->src
[0].ssa
->parent_instr
);
1116 nir_variable
*var
= nir_deref_instr_get_variable(deref
);
1117 LLVMValueRef coord_val
= get_src(bld_base
, instr
->src
[1]);
1118 LLVMValueRef coords
[5];
1119 struct lp_img_params params
;
1120 const struct glsl_type
*type
= glsl_without_array(var
->type
);
1121 unsigned const_index
;
1122 LLVMValueRef indir_index
;
1123 get_deref_offset(bld_base
, deref
, false, NULL
, NULL
,
1124 &const_index
, &indir_index
);
1126 memset(¶ms
, 0, sizeof(params
));
1127 params
.target
= glsl_sampler_to_pipe(glsl_get_sampler_dim(type
), glsl_sampler_type_is_array(type
));
1128 for (unsigned i
= 0; i
< 4; i
++)
1129 coords
[i
] = LLVMBuildExtractValue(builder
, coord_val
, i
, "");
1130 if (params
.target
== PIPE_TEXTURE_1D_ARRAY
)
1131 coords
[2] = coords
[1];
1133 params
.coords
= coords
;
1134 params
.outdata
= result
;
1135 params
.img_op
= LP_IMG_LOAD
;
1136 if (glsl_get_sampler_dim(type
) == GLSL_SAMPLER_DIM_MS
|| glsl_get_sampler_dim(type
) == GLSL_SAMPLER_DIM_SUBPASS_MS
)
1137 params
.ms_index
= cast_type(bld_base
, get_src(bld_base
, instr
->src
[2]), nir_type_uint
, 32);
1138 params
.image_index
= var
->data
.binding
+ (indir_index
? 0 : const_index
);
1139 params
.image_index_offset
= indir_index
;
1140 bld_base
->image_op(bld_base
, ¶ms
);
1143 static void visit_store_image(struct lp_build_nir_context
*bld_base
,
1144 nir_intrinsic_instr
*instr
)
1146 struct gallivm_state
*gallivm
= bld_base
->base
.gallivm
;
1147 LLVMBuilderRef builder
= gallivm
->builder
;
1148 nir_deref_instr
*deref
= nir_instr_as_deref(instr
->src
[0].ssa
->parent_instr
);
1149 nir_variable
*var
= nir_deref_instr_get_variable(deref
);
1150 LLVMValueRef coord_val
= get_src(bld_base
, instr
->src
[1]);
1151 LLVMValueRef in_val
= get_src(bld_base
, instr
->src
[3]);
1152 LLVMValueRef coords
[5];
1153 struct lp_img_params params
;
1154 const struct glsl_type
*type
= glsl_without_array(var
->type
);
1155 unsigned const_index
;
1156 LLVMValueRef indir_index
;
1157 get_deref_offset(bld_base
, deref
, false, NULL
, NULL
,
1158 &const_index
, &indir_index
);
1160 memset(¶ms
, 0, sizeof(params
));
1161 params
.target
= glsl_sampler_to_pipe(glsl_get_sampler_dim(type
), glsl_sampler_type_is_array(type
));
1162 for (unsigned i
= 0; i
< 4; i
++)
1163 coords
[i
] = LLVMBuildExtractValue(builder
, coord_val
, i
, "");
1164 if (params
.target
== PIPE_TEXTURE_1D_ARRAY
)
1165 coords
[2] = coords
[1];
1166 params
.coords
= coords
;
1168 for (unsigned i
= 0; i
< 4; i
++) {
1169 params
.indata
[i
] = LLVMBuildExtractValue(builder
, in_val
, i
, "");
1170 params
.indata
[i
] = LLVMBuildBitCast(builder
, params
.indata
[i
], bld_base
->base
.vec_type
, "");
1172 if (glsl_get_sampler_dim(type
) == GLSL_SAMPLER_DIM_MS
)
1173 params
.ms_index
= get_src(bld_base
, instr
->src
[2]);
1174 params
.img_op
= LP_IMG_STORE
;
1175 params
.image_index
= var
->data
.binding
+ (indir_index
? 0 : const_index
);
1176 params
.image_index_offset
= indir_index
;
1178 if (params
.target
== PIPE_TEXTURE_1D_ARRAY
)
1179 coords
[2] = coords
[1];
1180 bld_base
->image_op(bld_base
, ¶ms
);
1183 static void visit_atomic_image(struct lp_build_nir_context
*bld_base
,
1184 nir_intrinsic_instr
*instr
,
1185 LLVMValueRef result
[NIR_MAX_VEC_COMPONENTS
])
1187 struct gallivm_state
*gallivm
= bld_base
->base
.gallivm
;
1188 LLVMBuilderRef builder
= gallivm
->builder
;
1189 nir_deref_instr
*deref
= nir_instr_as_deref(instr
->src
[0].ssa
->parent_instr
);
1190 nir_variable
*var
= nir_deref_instr_get_variable(deref
);
1191 struct lp_img_params params
;
1192 LLVMValueRef coord_val
= get_src(bld_base
, instr
->src
[1]);
1193 LLVMValueRef in_val
= get_src(bld_base
, instr
->src
[3]);
1194 LLVMValueRef coords
[5];
1195 const struct glsl_type
*type
= glsl_without_array(var
->type
);
1196 unsigned const_index
;
1197 LLVMValueRef indir_index
;
1198 get_deref_offset(bld_base
, deref
, false, NULL
, NULL
,
1199 &const_index
, &indir_index
);
1201 memset(¶ms
, 0, sizeof(params
));
1203 switch (instr
->intrinsic
) {
1204 case nir_intrinsic_image_deref_atomic_add
:
1205 params
.op
= LLVMAtomicRMWBinOpAdd
;
1207 case nir_intrinsic_image_deref_atomic_exchange
:
1208 params
.op
= LLVMAtomicRMWBinOpXchg
;
1210 case nir_intrinsic_image_deref_atomic_and
:
1211 params
.op
= LLVMAtomicRMWBinOpAnd
;
1213 case nir_intrinsic_image_deref_atomic_or
:
1214 params
.op
= LLVMAtomicRMWBinOpOr
;
1216 case nir_intrinsic_image_deref_atomic_xor
:
1217 params
.op
= LLVMAtomicRMWBinOpXor
;
1219 case nir_intrinsic_image_deref_atomic_umin
:
1220 params
.op
= LLVMAtomicRMWBinOpUMin
;
1222 case nir_intrinsic_image_deref_atomic_umax
:
1223 params
.op
= LLVMAtomicRMWBinOpUMax
;
1225 case nir_intrinsic_image_deref_atomic_imin
:
1226 params
.op
= LLVMAtomicRMWBinOpMin
;
1228 case nir_intrinsic_image_deref_atomic_imax
:
1229 params
.op
= LLVMAtomicRMWBinOpMax
;
1235 params
.target
= glsl_sampler_to_pipe(glsl_get_sampler_dim(type
), glsl_sampler_type_is_array(type
));
1236 for (unsigned i
= 0; i
< 4; i
++)
1237 coords
[i
] = LLVMBuildExtractValue(builder
, coord_val
, i
, "");
1238 if (params
.target
== PIPE_TEXTURE_1D_ARRAY
)
1239 coords
[2] = coords
[1];
1240 params
.coords
= coords
;
1241 if (glsl_get_sampler_dim(type
) == GLSL_SAMPLER_DIM_MS
)
1242 params
.ms_index
= get_src(bld_base
, instr
->src
[2]);
1243 if (instr
->intrinsic
== nir_intrinsic_image_deref_atomic_comp_swap
) {
1244 LLVMValueRef cas_val
= get_src(bld_base
, instr
->src
[4]);
1245 params
.indata
[0] = in_val
;
1246 params
.indata2
[0] = cas_val
;
1248 params
.indata
[0] = in_val
;
1250 params
.outdata
= result
;
1251 params
.img_op
= (instr
->intrinsic
== nir_intrinsic_image_deref_atomic_comp_swap
) ? LP_IMG_ATOMIC_CAS
: LP_IMG_ATOMIC
;
1252 params
.image_index
= var
->data
.binding
+ (indir_index
? 0 : const_index
);
1253 params
.image_index_offset
= indir_index
;
1255 bld_base
->image_op(bld_base
, ¶ms
);
1259 static void visit_image_size(struct lp_build_nir_context
*bld_base
,
1260 nir_intrinsic_instr
*instr
,
1261 LLVMValueRef result
[NIR_MAX_VEC_COMPONENTS
])
1263 nir_deref_instr
*deref
= nir_instr_as_deref(instr
->src
[0].ssa
->parent_instr
);
1264 nir_variable
*var
= nir_deref_instr_get_variable(deref
);
1265 struct lp_sampler_size_query_params params
= { 0 };
1266 unsigned const_index
;
1267 LLVMValueRef indir_index
;
1268 const struct glsl_type
*type
= glsl_without_array(var
->type
);
1269 get_deref_offset(bld_base
, deref
, false, NULL
, NULL
,
1270 &const_index
, &indir_index
);
1271 params
.texture_unit
= var
->data
.binding
+ (indir_index
? 0 : const_index
);
1272 params
.texture_unit_offset
= indir_index
;
1273 params
.target
= glsl_sampler_to_pipe(glsl_get_sampler_dim(type
), glsl_sampler_type_is_array(type
));
1274 params
.sizes_out
= result
;
1276 bld_base
->image_size(bld_base
, ¶ms
);
1279 static void visit_image_samples(struct lp_build_nir_context
*bld_base
,
1280 nir_intrinsic_instr
*instr
,
1281 LLVMValueRef result
[NIR_MAX_VEC_COMPONENTS
])
1283 nir_deref_instr
*deref
= nir_instr_as_deref(instr
->src
[0].ssa
->parent_instr
);
1284 nir_variable
*var
= nir_deref_instr_get_variable(deref
);
1285 struct lp_sampler_size_query_params params
= { 0 };
1286 unsigned const_index
;
1287 LLVMValueRef indir_index
;
1288 const struct glsl_type
*type
= glsl_without_array(var
->type
);
1289 get_deref_offset(bld_base
, deref
, false, NULL
, NULL
,
1290 &const_index
, &indir_index
);
1292 params
.texture_unit
= var
->data
.binding
+ (indir_index
? 0 : const_index
);
1293 params
.texture_unit_offset
= indir_index
;
1294 params
.target
= glsl_sampler_to_pipe(glsl_get_sampler_dim(type
), glsl_sampler_type_is_array(type
));
1295 params
.sizes_out
= result
;
1296 params
.samples_only
= true;
1298 bld_base
->image_size(bld_base
, ¶ms
);
1301 static void visit_shared_load(struct lp_build_nir_context
*bld_base
,
1302 nir_intrinsic_instr
*instr
,
1303 LLVMValueRef result
[NIR_MAX_VEC_COMPONENTS
])
1305 LLVMValueRef offset
= get_src(bld_base
, instr
->src
[0]);
1306 bld_base
->load_mem(bld_base
, nir_dest_num_components(instr
->dest
), nir_dest_bit_size(instr
->dest
),
1307 NULL
, offset
, result
);
1310 static void visit_shared_store(struct lp_build_nir_context
*bld_base
,
1311 nir_intrinsic_instr
*instr
)
1313 LLVMValueRef val
= get_src(bld_base
, instr
->src
[0]);
1314 LLVMValueRef offset
= get_src(bld_base
, instr
->src
[1]);
1315 int writemask
= instr
->const_index
[1];
1316 int nc
= nir_src_num_components(instr
->src
[0]);
1317 int bitsize
= nir_src_bit_size(instr
->src
[0]);
1318 bld_base
->store_mem(bld_base
, writemask
, nc
, bitsize
, NULL
, offset
, val
);
1321 static void visit_shared_atomic(struct lp_build_nir_context
*bld_base
,
1322 nir_intrinsic_instr
*instr
,
1323 LLVMValueRef result
[NIR_MAX_VEC_COMPONENTS
])
1325 LLVMValueRef offset
= get_src(bld_base
, instr
->src
[0]);
1326 LLVMValueRef val
= get_src(bld_base
, instr
->src
[1]);
1327 LLVMValueRef val2
= NULL
;
1328 if (instr
->intrinsic
== nir_intrinsic_shared_atomic_comp_swap
)
1329 val2
= get_src(bld_base
, instr
->src
[2]);
1331 bld_base
->atomic_mem(bld_base
, instr
->intrinsic
, NULL
, offset
, val
, val2
, &result
[0]);
1335 static void visit_barrier(struct lp_build_nir_context
*bld_base
)
1337 bld_base
->barrier(bld_base
);
1340 static void visit_discard(struct lp_build_nir_context
*bld_base
,
1341 nir_intrinsic_instr
*instr
)
1343 LLVMValueRef cond
= NULL
;
1344 if (instr
->intrinsic
== nir_intrinsic_discard_if
) {
1345 cond
= get_src(bld_base
, instr
->src
[0]);
1346 cond
= cast_type(bld_base
, cond
, nir_type_int
, 32);
1348 bld_base
->discard(bld_base
, cond
);
1351 static void visit_load_kernel_input(struct lp_build_nir_context
*bld_base
,
1352 nir_intrinsic_instr
*instr
, LLVMValueRef result
[NIR_MAX_VEC_COMPONENTS
])
1354 LLVMValueRef offset
= get_src(bld_base
, instr
->src
[0]);
1356 bool offset_is_uniform
= nir_src_is_dynamically_uniform(instr
->src
[0]);
1357 bld_base
->load_kernel_arg(bld_base
, nir_dest_num_components(instr
->dest
), nir_dest_bit_size(instr
->dest
),
1358 nir_src_bit_size(instr
->src
[0]),
1359 offset_is_uniform
, offset
, result
);
1362 static void visit_load_global(struct lp_build_nir_context
*bld_base
,
1363 nir_intrinsic_instr
*instr
, LLVMValueRef result
[NIR_MAX_VEC_COMPONENTS
])
1365 LLVMValueRef addr
= get_src(bld_base
, instr
->src
[0]);
1366 bld_base
->load_global(bld_base
, nir_dest_num_components(instr
->dest
), nir_dest_bit_size(instr
->dest
),
1367 nir_src_bit_size(instr
->src
[0]),
1371 static void visit_store_global(struct lp_build_nir_context
*bld_base
,
1372 nir_intrinsic_instr
*instr
)
1374 LLVMValueRef val
= get_src(bld_base
, instr
->src
[0]);
1375 int nc
= nir_src_num_components(instr
->src
[0]);
1376 int bitsize
= nir_src_bit_size(instr
->src
[0]);
1377 LLVMValueRef addr
= get_src(bld_base
, instr
->src
[1]);
1378 int addr_bitsize
= nir_src_bit_size(instr
->src
[1]);
1379 int writemask
= instr
->const_index
[0];
1380 bld_base
->store_global(bld_base
, writemask
, nc
, bitsize
, addr_bitsize
, addr
, val
);
1383 static void visit_global_atomic(struct lp_build_nir_context
*bld_base
,
1384 nir_intrinsic_instr
*instr
,
1385 LLVMValueRef result
[NIR_MAX_VEC_COMPONENTS
])
1387 LLVMValueRef addr
= get_src(bld_base
, instr
->src
[0]);
1388 LLVMValueRef val
= get_src(bld_base
, instr
->src
[1]);
1389 LLVMValueRef val2
= NULL
;
1390 int addr_bitsize
= nir_src_bit_size(instr
->src
[0]);
1391 if (instr
->intrinsic
== nir_intrinsic_global_atomic_comp_swap
)
1392 val2
= get_src(bld_base
, instr
->src
[2]);
1394 bld_base
->atomic_global(bld_base
, instr
->intrinsic
, addr_bitsize
, addr
, val
, val2
, &result
[0]);
1397 static void visit_interp(struct lp_build_nir_context
*bld_base
,
1398 nir_intrinsic_instr
*instr
,
1399 LLVMValueRef result
[NIR_MAX_VEC_COMPONENTS
])
1401 struct gallivm_state
*gallivm
= bld_base
->base
.gallivm
;
1402 LLVMBuilderRef builder
= gallivm
->builder
;
1403 nir_deref_instr
*deref
= nir_instr_as_deref(instr
->src
[0].ssa
->parent_instr
);
1404 unsigned num_components
= nir_dest_num_components(instr
->dest
);
1405 nir_variable
*var
= nir_deref_instr_get_variable(deref
);
1406 unsigned const_index
;
1407 LLVMValueRef indir_index
;
1408 LLVMValueRef offsets
[2] = { NULL
, NULL
};
1409 get_deref_offset(bld_base
, deref
, false, NULL
, NULL
,
1410 &const_index
, &indir_index
);
1411 bool centroid
= instr
->intrinsic
== nir_intrinsic_interp_deref_at_centroid
;
1412 bool sample
= false;
1413 if (instr
->intrinsic
== nir_intrinsic_interp_deref_at_offset
) {
1414 for (unsigned i
= 0; i
< 2; i
++) {
1415 offsets
[i
] = LLVMBuildExtractValue(builder
, get_src(bld_base
, instr
->src
[1]), i
, "");
1416 offsets
[i
] = cast_type(bld_base
, offsets
[i
], nir_type_float
, 32);
1418 } else if (instr
->intrinsic
== nir_intrinsic_interp_deref_at_sample
) {
1419 offsets
[0] = get_src(bld_base
, instr
->src
[1]);
1420 offsets
[0] = cast_type(bld_base
, offsets
[0], nir_type_int
, 32);
1423 bld_base
->interp_at(bld_base
, num_components
, var
, centroid
, sample
, const_index
, indir_index
, offsets
, result
);
1426 static void visit_intrinsic(struct lp_build_nir_context
*bld_base
,
1427 nir_intrinsic_instr
*instr
)
1429 LLVMValueRef result
[NIR_MAX_VEC_COMPONENTS
] = {0};
1430 switch (instr
->intrinsic
) {
1431 case nir_intrinsic_load_deref
:
1432 visit_load_var(bld_base
, instr
, result
);
1434 case nir_intrinsic_store_deref
:
1435 visit_store_var(bld_base
, instr
);
1437 case nir_intrinsic_load_ubo
:
1438 visit_load_ubo(bld_base
, instr
, result
);
1440 case nir_intrinsic_load_push_constant
:
1441 visit_load_push_constant(bld_base
, instr
, result
);
1443 case nir_intrinsic_load_ssbo
:
1444 visit_load_ssbo(bld_base
, instr
, result
);
1446 case nir_intrinsic_store_ssbo
:
1447 visit_store_ssbo(bld_base
, instr
);
1449 case nir_intrinsic_get_buffer_size
:
1450 visit_get_buffer_size(bld_base
, instr
, result
);
1452 case nir_intrinsic_load_vertex_id
:
1453 case nir_intrinsic_load_primitive_id
:
1454 case nir_intrinsic_load_instance_id
:
1455 case nir_intrinsic_load_base_instance
:
1456 case nir_intrinsic_load_base_vertex
:
1457 case nir_intrinsic_load_work_group_id
:
1458 case nir_intrinsic_load_local_invocation_id
:
1459 case nir_intrinsic_load_num_work_groups
:
1460 case nir_intrinsic_load_invocation_id
:
1461 case nir_intrinsic_load_front_face
:
1462 case nir_intrinsic_load_draw_id
:
1463 case nir_intrinsic_load_local_group_size
:
1464 case nir_intrinsic_load_work_dim
:
1465 case nir_intrinsic_load_tess_coord
:
1466 case nir_intrinsic_load_tess_level_outer
:
1467 case nir_intrinsic_load_tess_level_inner
:
1468 case nir_intrinsic_load_patch_vertices_in
:
1469 case nir_intrinsic_load_sample_id
:
1470 case nir_intrinsic_load_sample_pos
:
1471 case nir_intrinsic_load_sample_mask_in
:
1472 bld_base
->sysval_intrin(bld_base
, instr
, result
);
1474 case nir_intrinsic_load_helper_invocation
:
1475 bld_base
->helper_invocation(bld_base
, &result
[0]);
1477 case nir_intrinsic_discard_if
:
1478 case nir_intrinsic_discard
:
1479 visit_discard(bld_base
, instr
);
1481 case nir_intrinsic_emit_vertex
:
1482 bld_base
->emit_vertex(bld_base
, nir_intrinsic_stream_id(instr
));
1484 case nir_intrinsic_end_primitive
:
1485 bld_base
->end_primitive(bld_base
, nir_intrinsic_stream_id(instr
));
1487 case nir_intrinsic_ssbo_atomic_add
:
1488 case nir_intrinsic_ssbo_atomic_imin
:
1489 case nir_intrinsic_ssbo_atomic_imax
:
1490 case nir_intrinsic_ssbo_atomic_umin
:
1491 case nir_intrinsic_ssbo_atomic_umax
:
1492 case nir_intrinsic_ssbo_atomic_and
:
1493 case nir_intrinsic_ssbo_atomic_or
:
1494 case nir_intrinsic_ssbo_atomic_xor
:
1495 case nir_intrinsic_ssbo_atomic_exchange
:
1496 case nir_intrinsic_ssbo_atomic_comp_swap
:
1497 visit_ssbo_atomic(bld_base
, instr
, result
);
1499 case nir_intrinsic_image_deref_load
:
1500 visit_load_image(bld_base
, instr
, result
);
1502 case nir_intrinsic_image_deref_store
:
1503 visit_store_image(bld_base
, instr
);
1505 case nir_intrinsic_image_deref_atomic_add
:
1506 case nir_intrinsic_image_deref_atomic_imin
:
1507 case nir_intrinsic_image_deref_atomic_imax
:
1508 case nir_intrinsic_image_deref_atomic_umin
:
1509 case nir_intrinsic_image_deref_atomic_umax
:
1510 case nir_intrinsic_image_deref_atomic_and
:
1511 case nir_intrinsic_image_deref_atomic_or
:
1512 case nir_intrinsic_image_deref_atomic_xor
:
1513 case nir_intrinsic_image_deref_atomic_exchange
:
1514 case nir_intrinsic_image_deref_atomic_comp_swap
:
1515 visit_atomic_image(bld_base
, instr
, result
);
1517 case nir_intrinsic_image_deref_size
:
1518 visit_image_size(bld_base
, instr
, result
);
1520 case nir_intrinsic_image_deref_samples
:
1521 visit_image_samples(bld_base
, instr
, result
);
1523 case nir_intrinsic_load_shared
:
1524 visit_shared_load(bld_base
, instr
, result
);
1526 case nir_intrinsic_store_shared
:
1527 visit_shared_store(bld_base
, instr
);
1529 case nir_intrinsic_shared_atomic_add
:
1530 case nir_intrinsic_shared_atomic_imin
:
1531 case nir_intrinsic_shared_atomic_umin
:
1532 case nir_intrinsic_shared_atomic_imax
:
1533 case nir_intrinsic_shared_atomic_umax
:
1534 case nir_intrinsic_shared_atomic_and
:
1535 case nir_intrinsic_shared_atomic_or
:
1536 case nir_intrinsic_shared_atomic_xor
:
1537 case nir_intrinsic_shared_atomic_exchange
:
1538 case nir_intrinsic_shared_atomic_comp_swap
:
1539 visit_shared_atomic(bld_base
, instr
, result
);
1541 case nir_intrinsic_control_barrier
:
1542 visit_barrier(bld_base
);
1544 case nir_intrinsic_group_memory_barrier
:
1545 case nir_intrinsic_memory_barrier
:
1546 case nir_intrinsic_memory_barrier_shared
:
1547 case nir_intrinsic_memory_barrier_buffer
:
1548 case nir_intrinsic_memory_barrier_image
:
1549 case nir_intrinsic_memory_barrier_tcs_patch
:
1551 case nir_intrinsic_load_kernel_input
:
1552 visit_load_kernel_input(bld_base
, instr
, result
);
1554 case nir_intrinsic_load_global
:
1555 case nir_intrinsic_load_global_constant
:
1556 visit_load_global(bld_base
, instr
, result
);
1558 case nir_intrinsic_store_global
:
1559 visit_store_global(bld_base
, instr
);
1561 case nir_intrinsic_global_atomic_add
:
1562 case nir_intrinsic_global_atomic_imin
:
1563 case nir_intrinsic_global_atomic_umin
:
1564 case nir_intrinsic_global_atomic_imax
:
1565 case nir_intrinsic_global_atomic_umax
:
1566 case nir_intrinsic_global_atomic_and
:
1567 case nir_intrinsic_global_atomic_or
:
1568 case nir_intrinsic_global_atomic_xor
:
1569 case nir_intrinsic_global_atomic_exchange
:
1570 case nir_intrinsic_global_atomic_comp_swap
:
1571 visit_global_atomic(bld_base
, instr
, result
);
1573 case nir_intrinsic_vote_all
:
1574 case nir_intrinsic_vote_any
:
1575 case nir_intrinsic_vote_ieq
:
1576 bld_base
->vote(bld_base
, cast_type(bld_base
, get_src(bld_base
, instr
->src
[0]), nir_type_int
, 32), instr
, result
);
1578 case nir_intrinsic_interp_deref_at_offset
:
1579 case nir_intrinsic_interp_deref_at_centroid
:
1580 case nir_intrinsic_interp_deref_at_sample
:
1581 visit_interp(bld_base
, instr
, result
);
1588 assign_dest(bld_base
, &instr
->dest
, result
);
1592 static void visit_txs(struct lp_build_nir_context
*bld_base
, nir_tex_instr
*instr
)
1594 struct lp_sampler_size_query_params params
= { 0 };
1595 LLVMValueRef sizes_out
[NIR_MAX_VEC_COMPONENTS
];
1596 LLVMValueRef explicit_lod
= NULL
;
1597 LLVMValueRef texture_unit_offset
= NULL
;
1598 for (unsigned i
= 0; i
< instr
->num_srcs
; i
++) {
1599 switch (instr
->src
[i
].src_type
) {
1600 case nir_tex_src_lod
:
1601 explicit_lod
= cast_type(bld_base
, get_src(bld_base
, instr
->src
[i
].src
), nir_type_int
, 32);
1603 case nir_tex_src_texture_offset
:
1604 texture_unit_offset
= get_src(bld_base
, instr
->src
[i
].src
);
1611 params
.target
= glsl_sampler_to_pipe(instr
->sampler_dim
, instr
->is_array
);
1612 params
.texture_unit
= instr
->texture_index
;
1613 params
.explicit_lod
= explicit_lod
;
1614 params
.is_sviewinfo
= TRUE
;
1615 params
.sizes_out
= sizes_out
;
1616 params
.samples_only
= (instr
->op
== nir_texop_texture_samples
);
1617 params
.texture_unit_offset
= texture_unit_offset
;
1619 if (instr
->op
== nir_texop_query_levels
)
1620 params
.explicit_lod
= bld_base
->uint_bld
.zero
;
1621 bld_base
->tex_size(bld_base
, ¶ms
);
1622 assign_dest(bld_base
, &instr
->dest
, &sizes_out
[instr
->op
== nir_texop_query_levels
? 3 : 0]);
1625 static enum lp_sampler_lod_property
lp_build_nir_lod_property(struct lp_build_nir_context
*bld_base
,
1628 enum lp_sampler_lod_property lod_property
;
1630 if (nir_src_is_dynamically_uniform(lod_src
))
1631 lod_property
= LP_SAMPLER_LOD_SCALAR
;
1632 else if (bld_base
->shader
->info
.stage
== MESA_SHADER_FRAGMENT
) {
1633 if (gallivm_perf
& GALLIVM_PERF_NO_QUAD_LOD
)
1634 lod_property
= LP_SAMPLER_LOD_PER_ELEMENT
;
1636 lod_property
= LP_SAMPLER_LOD_PER_QUAD
;
1639 lod_property
= LP_SAMPLER_LOD_PER_ELEMENT
;
1640 return lod_property
;
1643 static void visit_tex(struct lp_build_nir_context
*bld_base
, nir_tex_instr
*instr
)
1645 struct gallivm_state
*gallivm
= bld_base
->base
.gallivm
;
1646 LLVMBuilderRef builder
= gallivm
->builder
;
1647 LLVMValueRef coords
[5];
1648 LLVMValueRef offsets
[3] = { NULL
};
1649 LLVMValueRef explicit_lod
= NULL
, projector
= NULL
, ms_index
= NULL
;
1650 struct lp_sampler_params params
;
1651 struct lp_derivatives derivs
;
1652 unsigned sample_key
= 0;
1653 nir_deref_instr
*texture_deref_instr
= NULL
;
1654 nir_deref_instr
*sampler_deref_instr
= NULL
;
1655 LLVMValueRef texture_unit_offset
= NULL
;
1656 LLVMValueRef texel
[NIR_MAX_VEC_COMPONENTS
];
1657 unsigned lod_src
= 0;
1658 LLVMValueRef coord_undef
= LLVMGetUndef(bld_base
->base
.int_vec_type
);
1660 memset(¶ms
, 0, sizeof(params
));
1661 enum lp_sampler_lod_property lod_property
= LP_SAMPLER_LOD_SCALAR
;
1663 if (instr
->op
== nir_texop_txs
|| instr
->op
== nir_texop_query_levels
|| instr
->op
== nir_texop_texture_samples
) {
1664 visit_txs(bld_base
, instr
);
1667 if (instr
->op
== nir_texop_txf
|| instr
->op
== nir_texop_txf_ms
)
1668 sample_key
|= LP_SAMPLER_OP_FETCH
<< LP_SAMPLER_OP_TYPE_SHIFT
;
1669 else if (instr
->op
== nir_texop_tg4
) {
1670 sample_key
|= LP_SAMPLER_OP_GATHER
<< LP_SAMPLER_OP_TYPE_SHIFT
;
1671 sample_key
|= (instr
->component
<< LP_SAMPLER_GATHER_COMP_SHIFT
);
1672 } else if (instr
->op
== nir_texop_lod
)
1673 sample_key
|= LP_SAMPLER_OP_LODQ
<< LP_SAMPLER_OP_TYPE_SHIFT
;
1674 for (unsigned i
= 0; i
< instr
->num_srcs
; i
++) {
1675 switch (instr
->src
[i
].src_type
) {
1676 case nir_tex_src_coord
: {
1677 LLVMValueRef coord
= get_src(bld_base
, instr
->src
[i
].src
);
1678 if (instr
->coord_components
== 1)
1681 for (unsigned chan
= 0; chan
< instr
->coord_components
; ++chan
)
1682 coords
[chan
] = LLVMBuildExtractValue(builder
, coord
,
1685 for (unsigned chan
= instr
->coord_components
; chan
< 5; chan
++)
1686 coords
[chan
] = coord_undef
;
1690 case nir_tex_src_texture_deref
:
1691 texture_deref_instr
= nir_src_as_deref(instr
->src
[i
].src
);
1693 case nir_tex_src_sampler_deref
:
1694 sampler_deref_instr
= nir_src_as_deref(instr
->src
[i
].src
);
1696 case nir_tex_src_projector
:
1697 projector
= lp_build_rcp(&bld_base
->base
, cast_type(bld_base
, get_src(bld_base
, instr
->src
[i
].src
), nir_type_float
, 32));
1699 case nir_tex_src_comparator
:
1700 sample_key
|= LP_SAMPLER_SHADOW
;
1701 coords
[4] = get_src(bld_base
, instr
->src
[i
].src
);
1702 coords
[4] = cast_type(bld_base
, coords
[4], nir_type_float
, 32);
1704 case nir_tex_src_bias
:
1705 sample_key
|= LP_SAMPLER_LOD_BIAS
<< LP_SAMPLER_LOD_CONTROL_SHIFT
;
1707 explicit_lod
= cast_type(bld_base
, get_src(bld_base
, instr
->src
[i
].src
), nir_type_float
, 32);
1709 case nir_tex_src_lod
:
1710 sample_key
|= LP_SAMPLER_LOD_EXPLICIT
<< LP_SAMPLER_LOD_CONTROL_SHIFT
;
1712 if (instr
->op
== nir_texop_txf
)
1713 explicit_lod
= cast_type(bld_base
, get_src(bld_base
, instr
->src
[i
].src
), nir_type_int
, 32);
1715 explicit_lod
= cast_type(bld_base
, get_src(bld_base
, instr
->src
[i
].src
), nir_type_float
, 32);
1717 case nir_tex_src_ddx
: {
1718 int deriv_cnt
= instr
->coord_components
;
1719 if (instr
->is_array
)
1721 LLVMValueRef deriv_val
= get_src(bld_base
, instr
->src
[i
].src
);
1723 derivs
.ddx
[0] = deriv_val
;
1725 for (unsigned chan
= 0; chan
< deriv_cnt
; ++chan
)
1726 derivs
.ddx
[chan
] = LLVMBuildExtractValue(builder
, deriv_val
,
1728 for (unsigned chan
= 0; chan
< deriv_cnt
; ++chan
)
1729 derivs
.ddx
[chan
] = cast_type(bld_base
, derivs
.ddx
[chan
], nir_type_float
, 32);
1732 case nir_tex_src_ddy
: {
1733 int deriv_cnt
= instr
->coord_components
;
1734 if (instr
->is_array
)
1736 LLVMValueRef deriv_val
= get_src(bld_base
, instr
->src
[i
].src
);
1738 derivs
.ddy
[0] = deriv_val
;
1740 for (unsigned chan
= 0; chan
< deriv_cnt
; ++chan
)
1741 derivs
.ddy
[chan
] = LLVMBuildExtractValue(builder
, deriv_val
,
1743 for (unsigned chan
= 0; chan
< deriv_cnt
; ++chan
)
1744 derivs
.ddy
[chan
] = cast_type(bld_base
, derivs
.ddy
[chan
], nir_type_float
, 32);
1747 case nir_tex_src_offset
: {
1748 int offset_cnt
= instr
->coord_components
;
1749 if (instr
->is_array
)
1751 LLVMValueRef offset_val
= get_src(bld_base
, instr
->src
[i
].src
);
1752 sample_key
|= LP_SAMPLER_OFFSETS
;
1753 if (offset_cnt
== 1)
1754 offsets
[0] = cast_type(bld_base
, offset_val
, nir_type_int
, 32);
1756 for (unsigned chan
= 0; chan
< offset_cnt
; ++chan
) {
1757 offsets
[chan
] = LLVMBuildExtractValue(builder
, offset_val
,
1759 offsets
[chan
] = cast_type(bld_base
, offsets
[chan
], nir_type_int
, 32);
1764 case nir_tex_src_ms_index
:
1765 sample_key
|= LP_SAMPLER_FETCH_MS
;
1766 ms_index
= cast_type(bld_base
, get_src(bld_base
, instr
->src
[i
].src
), nir_type_int
, 32);
1769 case nir_tex_src_texture_offset
:
1770 texture_unit_offset
= get_src(bld_base
, instr
->src
[i
].src
);
1772 case nir_tex_src_sampler_offset
:
1779 if (!sampler_deref_instr
)
1780 sampler_deref_instr
= texture_deref_instr
;
1783 lod_property
= lp_build_nir_lod_property(bld_base
, instr
->src
[lod_src
].src
);
1785 if (instr
->op
== nir_texop_tex
|| instr
->op
== nir_texop_tg4
|| instr
->op
== nir_texop_txb
||
1786 instr
->op
== nir_texop_txl
|| instr
->op
== nir_texop_txd
|| instr
->op
== nir_texop_lod
)
1787 for (unsigned chan
= 0; chan
< instr
->coord_components
; ++chan
)
1788 coords
[chan
] = cast_type(bld_base
, coords
[chan
], nir_type_float
, 32);
1789 else if (instr
->op
== nir_texop_txf
|| instr
->op
== nir_texop_txf_ms
)
1790 for (unsigned chan
= 0; chan
< instr
->coord_components
; ++chan
)
1791 coords
[chan
] = cast_type(bld_base
, coords
[chan
], nir_type_int
, 32);
1793 if (instr
->is_array
&& instr
->sampler_dim
== GLSL_SAMPLER_DIM_1D
) {
1794 /* move layer coord for 1d arrays. */
1795 coords
[2] = coords
[1];
1796 coords
[1] = coord_undef
;
1800 for (unsigned chan
= 0; chan
< instr
->coord_components
; ++chan
)
1801 coords
[chan
] = lp_build_mul(&bld_base
->base
, coords
[chan
], projector
);
1802 if (sample_key
& LP_SAMPLER_SHADOW
)
1803 coords
[4] = lp_build_mul(&bld_base
->base
, coords
[4], projector
);
1806 uint32_t samp_base_index
= 0, tex_base_index
= 0;
1807 if (!sampler_deref_instr
) {
1808 int samp_src_index
= nir_tex_instr_src_index(instr
, nir_tex_src_sampler_handle
);
1809 if (samp_src_index
== -1) {
1810 samp_base_index
= instr
->sampler_index
;
1813 if (!texture_deref_instr
) {
1814 int tex_src_index
= nir_tex_instr_src_index(instr
, nir_tex_src_texture_handle
);
1815 if (tex_src_index
== -1) {
1816 tex_base_index
= instr
->texture_index
;
1820 if (instr
->op
== nir_texop_txd
) {
1821 sample_key
|= LP_SAMPLER_LOD_DERIVATIVES
<< LP_SAMPLER_LOD_CONTROL_SHIFT
;
1822 params
.derivs
= &derivs
;
1823 if (bld_base
->shader
->info
.stage
== MESA_SHADER_FRAGMENT
) {
1824 if (gallivm_perf
& GALLIVM_PERF_NO_QUAD_LOD
)
1825 lod_property
= LP_SAMPLER_LOD_PER_ELEMENT
;
1827 lod_property
= LP_SAMPLER_LOD_PER_QUAD
;
1829 lod_property
= LP_SAMPLER_LOD_PER_ELEMENT
;
1832 sample_key
|= lod_property
<< LP_SAMPLER_LOD_PROPERTY_SHIFT
;
1833 params
.sample_key
= sample_key
;
1834 params
.offsets
= offsets
;
1835 params
.texture_index
= tex_base_index
;
1836 params
.texture_index_offset
= texture_unit_offset
;
1837 params
.sampler_index
= samp_base_index
;
1838 params
.coords
= coords
;
1839 params
.texel
= texel
;
1840 params
.lod
= explicit_lod
;
1841 params
.ms_index
= ms_index
;
1842 bld_base
->tex(bld_base
, ¶ms
);
1843 assign_dest(bld_base
, &instr
->dest
, texel
);
1846 static void visit_ssa_undef(struct lp_build_nir_context
*bld_base
,
1847 const nir_ssa_undef_instr
*instr
)
1849 unsigned num_components
= instr
->def
.num_components
;
1850 LLVMValueRef undef
[NIR_MAX_VEC_COMPONENTS
];
1851 struct lp_build_context
*undef_bld
= get_int_bld(bld_base
, true, instr
->def
.bit_size
);
1852 for (unsigned i
= 0; i
< num_components
; i
++)
1853 undef
[i
] = LLVMGetUndef(undef_bld
->vec_type
);
1854 assign_ssa_dest(bld_base
, &instr
->def
, undef
);
1857 static void visit_jump(struct lp_build_nir_context
*bld_base
,
1858 const nir_jump_instr
*instr
)
1860 switch (instr
->type
) {
1861 case nir_jump_break
:
1862 bld_base
->break_stmt(bld_base
);
1864 case nir_jump_continue
:
1865 bld_base
->continue_stmt(bld_base
);
1868 unreachable("Unknown jump instr\n");
1872 static void visit_deref(struct lp_build_nir_context
*bld_base
,
1873 nir_deref_instr
*instr
)
1875 if (instr
->mode
!= nir_var_mem_shared
&&
1876 instr
->mode
!= nir_var_mem_global
)
1878 LLVMValueRef result
= NULL
;
1879 switch(instr
->deref_type
) {
1880 case nir_deref_type_var
: {
1881 struct hash_entry
*entry
= _mesa_hash_table_search(bld_base
->vars
, instr
->var
);
1882 result
= entry
->data
;
1886 unreachable("Unhandled deref_instr deref type");
1889 assign_ssa(bld_base
, instr
->dest
.ssa
.index
, result
);
1892 static void visit_block(struct lp_build_nir_context
*bld_base
, nir_block
*block
)
1894 nir_foreach_instr(instr
, block
)
1896 switch (instr
->type
) {
1897 case nir_instr_type_alu
:
1898 visit_alu(bld_base
, nir_instr_as_alu(instr
));
1900 case nir_instr_type_load_const
:
1901 visit_load_const(bld_base
, nir_instr_as_load_const(instr
));
1903 case nir_instr_type_intrinsic
:
1904 visit_intrinsic(bld_base
, nir_instr_as_intrinsic(instr
));
1906 case nir_instr_type_tex
:
1907 visit_tex(bld_base
, nir_instr_as_tex(instr
));
1909 case nir_instr_type_phi
:
1912 case nir_instr_type_ssa_undef
:
1913 visit_ssa_undef(bld_base
, nir_instr_as_ssa_undef(instr
));
1915 case nir_instr_type_jump
:
1916 visit_jump(bld_base
, nir_instr_as_jump(instr
));
1918 case nir_instr_type_deref
:
1919 visit_deref(bld_base
, nir_instr_as_deref(instr
));
1922 fprintf(stderr
, "Unknown NIR instr type: ");
1923 nir_print_instr(instr
, stderr
);
1924 fprintf(stderr
, "\n");
1930 static void visit_if(struct lp_build_nir_context
*bld_base
, nir_if
*if_stmt
)
1932 LLVMValueRef cond
= get_src(bld_base
, if_stmt
->condition
);
1934 bld_base
->if_cond(bld_base
, cond
);
1935 visit_cf_list(bld_base
, &if_stmt
->then_list
);
1937 if (!exec_list_is_empty(&if_stmt
->else_list
)) {
1938 bld_base
->else_stmt(bld_base
);
1939 visit_cf_list(bld_base
, &if_stmt
->else_list
);
1941 bld_base
->endif_stmt(bld_base
);
1944 static void visit_loop(struct lp_build_nir_context
*bld_base
, nir_loop
*loop
)
1946 bld_base
->bgnloop(bld_base
);
1947 visit_cf_list(bld_base
, &loop
->body
);
1948 bld_base
->endloop(bld_base
);
1951 static void visit_cf_list(struct lp_build_nir_context
*bld_base
,
1952 struct exec_list
*list
)
1954 foreach_list_typed(nir_cf_node
, node
, node
, list
)
1956 switch (node
->type
) {
1957 case nir_cf_node_block
:
1958 visit_block(bld_base
, nir_cf_node_as_block(node
));
1961 case nir_cf_node_if
:
1962 visit_if(bld_base
, nir_cf_node_as_if(node
));
1965 case nir_cf_node_loop
:
1966 visit_loop(bld_base
, nir_cf_node_as_loop(node
));
1976 handle_shader_output_decl(struct lp_build_nir_context
*bld_base
,
1977 struct nir_shader
*nir
,
1978 struct nir_variable
*variable
)
1980 bld_base
->emit_var_decl(bld_base
, variable
);
1983 /* vector registers are stored as arrays in LLVM side,
1984 so we can use GEP on them, as to do exec mask stores
1985 we need to operate on a single components.
1991 static LLVMTypeRef
get_register_type(struct lp_build_nir_context
*bld_base
,
1994 struct lp_build_context
*int_bld
= get_int_bld(bld_base
, true, reg
->bit_size
);
1996 LLVMTypeRef type
= int_bld
->vec_type
;
1997 if (reg
->num_array_elems
)
1998 type
= LLVMArrayType(type
, reg
->num_array_elems
);
1999 if (reg
->num_components
> 1)
2000 type
= LLVMArrayType(type
, reg
->num_components
);
2006 bool lp_build_nir_llvm(
2007 struct lp_build_nir_context
*bld_base
,
2008 struct nir_shader
*nir
)
2010 struct nir_function
*func
;
2012 nir_convert_from_ssa(nir
, true);
2013 nir_lower_locals_to_regs(nir
);
2014 nir_remove_dead_derefs(nir
);
2015 nir_remove_dead_variables(nir
, nir_var_function_temp
, NULL
);
2017 nir_foreach_shader_out_variable(variable
, nir
)
2018 handle_shader_output_decl(bld_base
, nir
, variable
);
2020 bld_base
->regs
= _mesa_hash_table_create(NULL
, _mesa_hash_pointer
,
2021 _mesa_key_pointer_equal
);
2022 bld_base
->vars
= _mesa_hash_table_create(NULL
, _mesa_hash_pointer
,
2023 _mesa_key_pointer_equal
);
2025 func
= (struct nir_function
*)exec_list_get_head(&nir
->functions
);
2027 nir_foreach_register(reg
, &func
->impl
->registers
) {
2028 LLVMTypeRef type
= get_register_type(bld_base
, reg
);
2029 LLVMValueRef reg_alloc
= lp_build_alloca_undef(bld_base
->base
.gallivm
,
2031 _mesa_hash_table_insert(bld_base
->regs
, reg
, reg_alloc
);
2033 nir_index_ssa_defs(func
->impl
);
2034 bld_base
->ssa_defs
= calloc(func
->impl
->ssa_alloc
, sizeof(LLVMValueRef
));
2035 visit_cf_list(bld_base
, &func
->impl
->body
);
2037 free(bld_base
->ssa_defs
);
2038 ralloc_free(bld_base
->vars
);
2039 ralloc_free(bld_base
->regs
);
2043 /* do some basic opts to remove some things we don't want to see. */
2044 void lp_build_opt_nir(struct nir_shader
*nir
)
2048 static const struct nir_lower_tex_options lower_tex_options
= {
2049 .lower_tg4_offsets
= true,
2051 NIR_PASS_V(nir
, nir_lower_tex
, &lower_tex_options
);
2052 NIR_PASS_V(nir
, nir_lower_frexp
);
2056 NIR_PASS_V(nir
, nir_opt_constant_folding
);
2057 NIR_PASS_V(nir
, nir_opt_algebraic
);
2058 NIR_PASS_V(nir
, nir_lower_pack
);
2060 nir_lower_tex_options options
= { .lower_tex_without_implicit_lod
= true };
2061 NIR_PASS_V(nir
, nir_lower_tex
, &options
);
2063 nir_lower_bool_to_int32(nir
);