1 /**************************************************************************
3 * Copyright 2019 Red Hat.
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the "Software"),
8 * to deal in the Software without restriction, including without limitation
9 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
10 * and/or sell copies of the Software, and to permit persons to whom the
11 * Software is furnished to do so, subject to the following conditions:
13 * The above copyright notice and this permission notice shall be included
14 * in all copies or substantial portions of the Software.
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
17 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
24 **************************************************************************/
26 #include "lp_bld_nir.h"
27 #include "lp_bld_arit.h"
28 #include "lp_bld_bitarit.h"
29 #include "lp_bld_const.h"
30 #include "lp_bld_gather.h"
31 #include "lp_bld_logic.h"
32 #include "lp_bld_quad.h"
33 #include "lp_bld_flow.h"
34 #include "lp_bld_struct.h"
35 #include "lp_bld_debug.h"
36 #include "lp_bld_printf.h"
37 #include "nir_deref.h"
39 static void visit_cf_list(struct lp_build_nir_context
*bld_base
,
40 struct exec_list
*list
);
42 static LLVMValueRef
cast_type(struct lp_build_nir_context
*bld_base
, LLVMValueRef val
,
43 nir_alu_type alu_type
, unsigned bit_size
)
45 LLVMBuilderRef builder
= bld_base
->base
.gallivm
->builder
;
50 return LLVMBuildBitCast(builder
, val
, bld_base
->base
.vec_type
, "");
52 return LLVMBuildBitCast(builder
, val
, bld_base
->dbl_bld
.vec_type
, "");
61 return LLVMBuildBitCast(builder
, val
, bld_base
->int8_bld
.vec_type
, "");
63 return LLVMBuildBitCast(builder
, val
, bld_base
->int16_bld
.vec_type
, "");
65 return LLVMBuildBitCast(builder
, val
, bld_base
->int_bld
.vec_type
, "");
67 return LLVMBuildBitCast(builder
, val
, bld_base
->int64_bld
.vec_type
, "");
76 return LLVMBuildBitCast(builder
, val
, bld_base
->uint8_bld
.vec_type
, "");
78 return LLVMBuildBitCast(builder
, val
, bld_base
->uint16_bld
.vec_type
, "");
80 return LLVMBuildBitCast(builder
, val
, bld_base
->uint_bld
.vec_type
, "");
82 return LLVMBuildBitCast(builder
, val
, bld_base
->uint64_bld
.vec_type
, "");
89 return LLVMBuildBitCast(builder
, val
, bld_base
->uint_bld
.vec_type
, "");
97 static struct lp_build_context
*get_flt_bld(struct lp_build_nir_context
*bld_base
,
100 if (op_bit_size
== 64)
101 return &bld_base
->dbl_bld
;
103 return &bld_base
->base
;
106 static unsigned glsl_sampler_to_pipe(int sampler_dim
, bool is_array
)
108 unsigned pipe_target
= PIPE_BUFFER
;
109 switch (sampler_dim
) {
110 case GLSL_SAMPLER_DIM_1D
:
111 pipe_target
= is_array
? PIPE_TEXTURE_1D_ARRAY
: PIPE_TEXTURE_1D
;
113 case GLSL_SAMPLER_DIM_2D
:
114 pipe_target
= is_array
? PIPE_TEXTURE_2D_ARRAY
: PIPE_TEXTURE_2D
;
116 case GLSL_SAMPLER_DIM_3D
:
117 pipe_target
= PIPE_TEXTURE_3D
;
119 case GLSL_SAMPLER_DIM_MS
:
120 pipe_target
= is_array
? PIPE_TEXTURE_2D_ARRAY
: PIPE_TEXTURE_2D
;
122 case GLSL_SAMPLER_DIM_CUBE
:
123 pipe_target
= is_array
? PIPE_TEXTURE_CUBE_ARRAY
: PIPE_TEXTURE_CUBE
;
125 case GLSL_SAMPLER_DIM_RECT
:
126 pipe_target
= PIPE_TEXTURE_RECT
;
128 case GLSL_SAMPLER_DIM_BUF
:
129 pipe_target
= PIPE_BUFFER
;
137 static LLVMValueRef
get_ssa_src(struct lp_build_nir_context
*bld_base
, nir_ssa_def
*ssa
)
139 return bld_base
->ssa_defs
[ssa
->index
];
142 static LLVMValueRef
get_src(struct lp_build_nir_context
*bld_base
, nir_src src
);
144 static LLVMValueRef
get_reg_src(struct lp_build_nir_context
*bld_base
, nir_reg_src src
)
146 struct hash_entry
*entry
= _mesa_hash_table_search(bld_base
->regs
, src
.reg
);
147 LLVMValueRef reg_storage
= (LLVMValueRef
)entry
->data
;
148 struct lp_build_context
*reg_bld
= get_int_bld(bld_base
, true, src
.reg
->bit_size
);
149 LLVMValueRef indir_src
= NULL
;
151 indir_src
= get_src(bld_base
, *src
.indirect
);
152 return bld_base
->load_reg(bld_base
, reg_bld
, &src
, indir_src
, reg_storage
);
155 static LLVMValueRef
get_src(struct lp_build_nir_context
*bld_base
, nir_src src
)
158 return get_ssa_src(bld_base
, src
.ssa
);
160 return get_reg_src(bld_base
, src
.reg
);
163 static void assign_ssa(struct lp_build_nir_context
*bld_base
, int idx
, LLVMValueRef ptr
)
165 bld_base
->ssa_defs
[idx
] = ptr
;
168 static void assign_ssa_dest(struct lp_build_nir_context
*bld_base
, const nir_ssa_def
*ssa
,
169 LLVMValueRef vals
[NIR_MAX_VEC_COMPONENTS
])
171 assign_ssa(bld_base
, ssa
->index
, ssa
->num_components
== 1 ? vals
[0] : lp_nir_array_build_gather_values(bld_base
->base
.gallivm
->builder
, vals
, ssa
->num_components
));
174 static void assign_reg(struct lp_build_nir_context
*bld_base
, const nir_reg_dest
*reg
,
176 LLVMValueRef vals
[NIR_MAX_VEC_COMPONENTS
])
178 struct hash_entry
*entry
= _mesa_hash_table_search(bld_base
->regs
, reg
->reg
);
179 LLVMValueRef reg_storage
= (LLVMValueRef
)entry
->data
;
180 struct lp_build_context
*reg_bld
= get_int_bld(bld_base
, true, reg
->reg
->bit_size
);
181 LLVMValueRef indir_src
= NULL
;
183 indir_src
= get_src(bld_base
, *reg
->indirect
);
184 bld_base
->store_reg(bld_base
, reg_bld
, reg
, write_mask
? write_mask
: 0xf, indir_src
, reg_storage
, vals
);
187 static void assign_dest(struct lp_build_nir_context
*bld_base
, const nir_dest
*dest
, LLVMValueRef vals
[NIR_MAX_VEC_COMPONENTS
])
190 assign_ssa_dest(bld_base
, &dest
->ssa
, vals
);
192 assign_reg(bld_base
, &dest
->reg
, 0, vals
);
195 static void assign_alu_dest(struct lp_build_nir_context
*bld_base
, const nir_alu_dest
*dest
, LLVMValueRef vals
[NIR_MAX_VEC_COMPONENTS
])
197 if (dest
->dest
.is_ssa
)
198 assign_ssa_dest(bld_base
, &dest
->dest
.ssa
, vals
);
200 assign_reg(bld_base
, &dest
->dest
.reg
, dest
->write_mask
, vals
);
203 static LLVMValueRef
int_to_bool32(struct lp_build_nir_context
*bld_base
,
204 uint32_t src_bit_size
,
208 LLVMBuilderRef builder
= bld_base
->base
.gallivm
->builder
;
209 struct lp_build_context
*int_bld
= get_int_bld(bld_base
, is_unsigned
, src_bit_size
);
210 LLVMValueRef result
= lp_build_compare(bld_base
->base
.gallivm
, int_bld
->type
, PIPE_FUNC_NOTEQUAL
, val
, int_bld
->zero
);
211 if (src_bit_size
== 64)
212 result
= LLVMBuildTrunc(builder
, result
, bld_base
->int_bld
.vec_type
, "");
216 static LLVMValueRef
flt_to_bool32(struct lp_build_nir_context
*bld_base
,
217 uint32_t src_bit_size
,
220 LLVMBuilderRef builder
= bld_base
->base
.gallivm
->builder
;
221 struct lp_build_context
*flt_bld
= get_flt_bld(bld_base
, src_bit_size
);
222 LLVMValueRef result
= lp_build_cmp(flt_bld
, PIPE_FUNC_NOTEQUAL
, val
, flt_bld
->zero
);
223 if (src_bit_size
== 64)
224 result
= LLVMBuildTrunc(builder
, result
, bld_base
->int_bld
.vec_type
, "");
228 static LLVMValueRef
fcmp32(struct lp_build_nir_context
*bld_base
,
229 enum pipe_compare_func compare
,
230 uint32_t src_bit_size
,
231 LLVMValueRef src
[NIR_MAX_VEC_COMPONENTS
])
233 LLVMBuilderRef builder
= bld_base
->base
.gallivm
->builder
;
234 struct lp_build_context
*flt_bld
= get_flt_bld(bld_base
, src_bit_size
);
237 if (compare
!= PIPE_FUNC_NOTEQUAL
)
238 result
= lp_build_cmp_ordered(flt_bld
, compare
, src
[0], src
[1]);
240 result
= lp_build_cmp(flt_bld
, compare
, src
[0], src
[1]);
241 if (src_bit_size
== 64)
242 result
= LLVMBuildTrunc(builder
, result
, bld_base
->int_bld
.vec_type
, "");
246 static LLVMValueRef
icmp32(struct lp_build_nir_context
*bld_base
,
247 enum pipe_compare_func compare
,
249 uint32_t src_bit_size
,
250 LLVMValueRef src
[NIR_MAX_VEC_COMPONENTS
])
252 LLVMBuilderRef builder
= bld_base
->base
.gallivm
->builder
;
253 struct lp_build_context
*i_bld
= get_int_bld(bld_base
, is_unsigned
, src_bit_size
);
254 LLVMValueRef result
= lp_build_cmp(i_bld
, compare
, src
[0], src
[1]);
255 if (src_bit_size
< 32)
256 result
= LLVMBuildSExt(builder
, result
, bld_base
->int_bld
.vec_type
, "");
257 else if (src_bit_size
== 64)
258 result
= LLVMBuildTrunc(builder
, result
, bld_base
->int_bld
.vec_type
, "");
262 static LLVMValueRef
get_alu_src(struct lp_build_nir_context
*bld_base
,
264 unsigned num_components
)
266 LLVMBuilderRef builder
= bld_base
->base
.gallivm
->builder
;
267 struct gallivm_state
*gallivm
= bld_base
->base
.gallivm
;
268 LLVMValueRef value
= get_src(bld_base
, src
.src
);
269 bool need_swizzle
= false;
272 unsigned src_components
= nir_src_num_components(src
.src
);
273 for (unsigned i
= 0; i
< num_components
; ++i
) {
274 assert(src
.swizzle
[i
] < src_components
);
275 if (src
.swizzle
[i
] != i
)
279 if (need_swizzle
|| num_components
!= src_components
) {
280 if (src_components
> 1 && num_components
== 1) {
281 value
= LLVMBuildExtractValue(gallivm
->builder
, value
,
283 } else if (src_components
== 1 && num_components
> 1) {
284 LLVMValueRef values
[] = {value
, value
, value
, value
, value
, value
, value
, value
, value
, value
, value
, value
, value
, value
, value
, value
};
285 value
= lp_nir_array_build_gather_values(builder
, values
, num_components
);
287 LLVMValueRef arr
= LLVMGetUndef(LLVMArrayType(LLVMTypeOf(LLVMBuildExtractValue(builder
, value
, 0, "")), num_components
));
288 for (unsigned i
= 0; i
< num_components
; i
++)
289 arr
= LLVMBuildInsertValue(builder
, arr
, LLVMBuildExtractValue(builder
, value
, src
.swizzle
[i
], ""), i
, "");
298 static LLVMValueRef
emit_b2f(struct lp_build_nir_context
*bld_base
,
302 LLVMBuilderRef builder
= bld_base
->base
.gallivm
->builder
;
303 LLVMValueRef result
= LLVMBuildAnd(builder
, cast_type(bld_base
, src0
, nir_type_int
, 32),
304 LLVMBuildBitCast(builder
, lp_build_const_vec(bld_base
->base
.gallivm
, bld_base
->base
.type
,
305 1.0), bld_base
->int_bld
.vec_type
, ""),
307 result
= LLVMBuildBitCast(builder
, result
, bld_base
->base
.vec_type
, "");
312 result
= LLVMBuildFPExt(builder
, result
, bld_base
->dbl_bld
.vec_type
, "");
315 unreachable("unsupported bit size.");
320 static LLVMValueRef
emit_b2i(struct lp_build_nir_context
*bld_base
,
324 LLVMBuilderRef builder
= bld_base
->base
.gallivm
->builder
;
325 LLVMValueRef result
= LLVMBuildAnd(builder
, cast_type(bld_base
, src0
, nir_type_int
, 32),
326 lp_build_const_int_vec(bld_base
->base
.gallivm
, bld_base
->base
.type
, 1), "");
331 return LLVMBuildZExt(builder
, result
, bld_base
->int64_bld
.vec_type
, "");
333 unreachable("unsupported bit size.");
337 static LLVMValueRef
emit_b32csel(struct lp_build_nir_context
*bld_base
,
338 unsigned src_bit_size
[NIR_MAX_VEC_COMPONENTS
],
339 LLVMValueRef src
[NIR_MAX_VEC_COMPONENTS
])
341 LLVMValueRef sel
= cast_type(bld_base
, src
[0], nir_type_int
, 32);
342 LLVMValueRef v
= lp_build_compare(bld_base
->base
.gallivm
, bld_base
->int_bld
.type
, PIPE_FUNC_NOTEQUAL
, sel
, bld_base
->int_bld
.zero
);
343 struct lp_build_context
*bld
= get_int_bld(bld_base
, false, src_bit_size
[1]);
344 return lp_build_select(bld
, v
, src
[1], src
[2]);
347 static LLVMValueRef
split_64bit(struct lp_build_nir_context
*bld_base
,
351 struct gallivm_state
*gallivm
= bld_base
->base
.gallivm
;
352 LLVMValueRef shuffles
[LP_MAX_VECTOR_WIDTH
/32];
353 LLVMValueRef shuffles2
[LP_MAX_VECTOR_WIDTH
/32];
354 int len
= bld_base
->base
.type
.length
* 2;
355 for (unsigned i
= 0; i
< bld_base
->base
.type
.length
; i
++) {
356 shuffles
[i
] = lp_build_const_int32(gallivm
, i
* 2);
357 shuffles2
[i
] = lp_build_const_int32(gallivm
, (i
* 2) + 1);
360 src
= LLVMBuildBitCast(gallivm
->builder
, src
, LLVMVectorType(LLVMInt32TypeInContext(gallivm
->context
), len
), "");
361 return LLVMBuildShuffleVector(gallivm
->builder
, src
,
362 LLVMGetUndef(LLVMTypeOf(src
)),
363 LLVMConstVector(hi
? shuffles2
: shuffles
,
364 bld_base
->base
.type
.length
),
369 merge_64bit(struct lp_build_nir_context
*bld_base
,
373 struct gallivm_state
*gallivm
= bld_base
->base
.gallivm
;
374 LLVMBuilderRef builder
= gallivm
->builder
;
376 LLVMValueRef shuffles
[2 * (LP_MAX_VECTOR_WIDTH
/32)];
377 int len
= bld_base
->base
.type
.length
* 2;
378 assert(len
<= (2 * (LP_MAX_VECTOR_WIDTH
/32)));
380 for (i
= 0; i
< bld_base
->base
.type
.length
* 2; i
+=2) {
381 shuffles
[i
] = lp_build_const_int32(gallivm
, i
/ 2);
382 shuffles
[i
+ 1] = lp_build_const_int32(gallivm
, i
/ 2 + bld_base
->base
.type
.length
);
384 return LLVMBuildShuffleVector(builder
, input
, input2
, LLVMConstVector(shuffles
, len
), "");
388 do_int_divide(struct lp_build_nir_context
*bld_base
,
389 bool is_unsigned
, unsigned src_bit_size
,
390 LLVMValueRef src
, LLVMValueRef src2
)
392 struct gallivm_state
*gallivm
= bld_base
->base
.gallivm
;
393 LLVMBuilderRef builder
= gallivm
->builder
;
394 struct lp_build_context
*int_bld
= get_int_bld(bld_base
, is_unsigned
, src_bit_size
);
395 struct lp_build_context
*mask_bld
= get_int_bld(bld_base
, true, src_bit_size
);
396 LLVMValueRef div_mask
= lp_build_cmp(mask_bld
, PIPE_FUNC_EQUAL
, src2
,
400 /* INT_MIN (0x80000000) / -1 (0xffffffff) causes sigfpe, seen with blender. */
401 div_mask
= LLVMBuildAnd(builder
, div_mask
, lp_build_const_int_vec(gallivm
, int_bld
->type
, 0x7fffffff), "");
403 LLVMValueRef divisor
= LLVMBuildOr(builder
,
406 LLVMValueRef result
= lp_build_div(int_bld
, src
, divisor
);
409 LLVMValueRef not_div_mask
= LLVMBuildNot(builder
, div_mask
, "");
410 return LLVMBuildAnd(builder
, not_div_mask
, result
, "");
412 /* udiv by zero is guaranteed to return 0xffffffff at least with d3d10
413 * may as well do same for idiv */
414 return LLVMBuildOr(builder
, div_mask
, result
, "");
418 do_int_mod(struct lp_build_nir_context
*bld_base
,
419 bool is_unsigned
, unsigned src_bit_size
,
420 LLVMValueRef src
, LLVMValueRef src2
)
422 struct gallivm_state
*gallivm
= bld_base
->base
.gallivm
;
423 LLVMBuilderRef builder
= gallivm
->builder
;
424 struct lp_build_context
*int_bld
= get_int_bld(bld_base
, is_unsigned
, src_bit_size
);
425 LLVMValueRef div_mask
= lp_build_cmp(int_bld
, PIPE_FUNC_EQUAL
, src2
,
427 LLVMValueRef divisor
= LLVMBuildOr(builder
,
430 LLVMValueRef result
= lp_build_mod(int_bld
, src
, divisor
);
431 return LLVMBuildOr(builder
, div_mask
, result
, "");
434 static LLVMValueRef
do_alu_action(struct lp_build_nir_context
*bld_base
,
435 nir_op op
, unsigned src_bit_size
[NIR_MAX_VEC_COMPONENTS
], LLVMValueRef src
[NIR_MAX_VEC_COMPONENTS
])
437 struct gallivm_state
*gallivm
= bld_base
->base
.gallivm
;
438 LLVMBuilderRef builder
= gallivm
->builder
;
442 result
= emit_b2f(bld_base
, src
[0], 32);
445 result
= emit_b2f(bld_base
, src
[0], 64);
448 result
= emit_b2i(bld_base
, src
[0], 32);
451 result
= emit_b2i(bld_base
, src
[0], 64);
454 result
= emit_b32csel(bld_base
, src_bit_size
, src
);
456 case nir_op_bit_count
:
457 result
= lp_build_popcount(get_int_bld(bld_base
, false, src_bit_size
[0]), src
[0]);
459 case nir_op_bitfield_select
:
460 result
= lp_build_xor(&bld_base
->uint_bld
, src
[2], lp_build_and(&bld_base
->uint_bld
, src
[0], lp_build_xor(&bld_base
->uint_bld
, src
[1], src
[2])));
462 case nir_op_bitfield_reverse
:
463 result
= lp_build_bitfield_reverse(get_int_bld(bld_base
, false, src_bit_size
[0]), src
[0]);
466 result
= flt_to_bool32(bld_base
, src_bit_size
[0], src
[0]);
469 result
= LLVMBuildFPTrunc(builder
, src
[0],
470 bld_base
->base
.vec_type
, "");
473 result
= LLVMBuildFPExt(builder
, src
[0],
474 bld_base
->dbl_bld
.vec_type
, "");
477 result
= LLVMBuildFPToSI(builder
, src
[0], bld_base
->base
.int_vec_type
, "");
480 result
= LLVMBuildFPToUI(builder
,
482 bld_base
->base
.int_vec_type
, "");
485 result
= LLVMBuildFPToSI(builder
,
487 bld_base
->int64_bld
.vec_type
, "");
490 result
= LLVMBuildFPToUI(builder
,
492 bld_base
->uint64_bld
.vec_type
, "");
495 result
= lp_build_abs(get_flt_bld(bld_base
, src_bit_size
[0]), src
[0]);
498 result
= lp_build_add(get_flt_bld(bld_base
, src_bit_size
[0]),
502 result
= lp_build_ceil(get_flt_bld(bld_base
, src_bit_size
[0]), src
[0]);
505 result
= lp_build_cos(&bld_base
->base
, src
[0]);
508 case nir_op_fddx_coarse
:
509 case nir_op_fddx_fine
:
510 result
= lp_build_ddx(&bld_base
->base
, src
[0]);
513 case nir_op_fddy_coarse
:
514 case nir_op_fddy_fine
:
515 result
= lp_build_ddy(&bld_base
->base
, src
[0]);
518 result
= lp_build_div(get_flt_bld(bld_base
, src_bit_size
[0]),
522 result
= fcmp32(bld_base
, PIPE_FUNC_EQUAL
, src_bit_size
[0], src
);
525 result
= lp_build_exp2(&bld_base
->base
, src
[0]);
528 result
= lp_build_floor(get_flt_bld(bld_base
, src_bit_size
[0]), src
[0]);
531 result
= lp_build_fmuladd(builder
, src
[0], src
[1], src
[2]);
533 case nir_op_ffract
: {
534 struct lp_build_context
*flt_bld
= get_flt_bld(bld_base
, src_bit_size
[0]);
535 LLVMValueRef tmp
= lp_build_floor(flt_bld
, src
[0]);
536 result
= lp_build_sub(flt_bld
, src
[0], tmp
);
540 result
= fcmp32(bld_base
, PIPE_FUNC_GEQUAL
, src_bit_size
[0], src
);
542 case nir_op_find_lsb
:
543 result
= lp_build_cttz(get_int_bld(bld_base
, false, src_bit_size
[0]), src
[0]);
546 result
= lp_build_log2_safe(&bld_base
->base
, src
[0]);
549 result
= fcmp32(bld_base
, PIPE_FUNC_LESS
, src_bit_size
[0], src
);
552 result
= lp_build_min(get_flt_bld(bld_base
, src_bit_size
[0]), src
[0], src
[1]);
555 struct lp_build_context
*flt_bld
= get_flt_bld(bld_base
, src_bit_size
[0]);
556 result
= lp_build_div(flt_bld
, src
[0], src
[1]);
557 result
= lp_build_floor(flt_bld
, result
);
558 result
= lp_build_mul(flt_bld
, src
[1], result
);
559 result
= lp_build_sub(flt_bld
, src
[0], result
);
563 result
= lp_build_mul(get_flt_bld(bld_base
, src_bit_size
[0]),
567 result
= lp_build_max(get_flt_bld(bld_base
, src_bit_size
[0]), src
[0], src
[1]);
570 result
= fcmp32(bld_base
, PIPE_FUNC_NOTEQUAL
, src_bit_size
[0], src
);
573 result
= lp_build_negate(get_flt_bld(bld_base
, src_bit_size
[0]), src
[0]);
576 result
= lp_build_pow(&bld_base
->base
, src
[0], src
[1]);
579 result
= lp_build_rcp(get_flt_bld(bld_base
, src_bit_size
[0]), src
[0]);
581 case nir_op_fround_even
:
582 result
= lp_build_round(get_flt_bld(bld_base
, src_bit_size
[0]), src
[0]);
585 result
= lp_build_rsqrt(get_flt_bld(bld_base
, src_bit_size
[0]), src
[0]);
588 result
= lp_build_clamp_zero_one_nanzero(get_flt_bld(bld_base
, src_bit_size
[0]), src
[0]);
591 result
= lp_build_sgn(get_flt_bld(bld_base
, src_bit_size
[0]), src
[0]);
594 result
= lp_build_sin(&bld_base
->base
, src
[0]);
597 result
= lp_build_sqrt(get_flt_bld(bld_base
, src_bit_size
[0]), src
[0]);
600 result
= lp_build_trunc(get_flt_bld(bld_base
, src_bit_size
[0]), src
[0]);
603 result
= int_to_bool32(bld_base
, src_bit_size
[0], false, src
[0]);
606 result
= lp_build_int_to_float(&bld_base
->base
, src
[0]);
609 result
= lp_build_int_to_float(&bld_base
->dbl_bld
, src
[0]);
612 result
= LLVMBuildTrunc(builder
, src
[0], bld_base
->int8_bld
.vec_type
, "");
615 if (src_bit_size
[0] < 16)
616 result
= LLVMBuildSExt(builder
, src
[0], bld_base
->int16_bld
.vec_type
, "");
618 result
= LLVMBuildTrunc(builder
, src
[0], bld_base
->int16_bld
.vec_type
, "");
621 if (src_bit_size
[0] < 32)
622 result
= LLVMBuildSExt(builder
, src
[0], bld_base
->int_bld
.vec_type
, "");
624 result
= LLVMBuildTrunc(builder
, src
[0], bld_base
->int_bld
.vec_type
, "");
627 result
= LLVMBuildSExt(builder
, src
[0], bld_base
->int64_bld
.vec_type
, "");
630 result
= lp_build_abs(get_int_bld(bld_base
, false, src_bit_size
[0]), src
[0]);
633 result
= lp_build_add(get_int_bld(bld_base
, false, src_bit_size
[0]),
637 result
= lp_build_and(get_int_bld(bld_base
, false, src_bit_size
[0]),
641 result
= do_int_divide(bld_base
, false, src_bit_size
[0], src
[0], src
[1]);
644 result
= icmp32(bld_base
, PIPE_FUNC_EQUAL
, false, src_bit_size
[0], src
);
647 result
= icmp32(bld_base
, PIPE_FUNC_GEQUAL
, false, src_bit_size
[0], src
);
650 result
= icmp32(bld_base
, PIPE_FUNC_LESS
, false, src_bit_size
[0], src
);
653 result
= lp_build_max(get_int_bld(bld_base
, false, src_bit_size
[0]), src
[0], src
[1]);
656 result
= lp_build_min(get_int_bld(bld_base
, false, src_bit_size
[0]), src
[0], src
[1]);
660 result
= lp_build_mul(get_int_bld(bld_base
, false, src_bit_size
[0]),
663 case nir_op_imul_high
: {
664 LLVMValueRef hi_bits
;
665 lp_build_mul_32_lohi(&bld_base
->int_bld
, src
[0], src
[1], &hi_bits
);
670 result
= icmp32(bld_base
, PIPE_FUNC_NOTEQUAL
, false, src_bit_size
[0], src
);
673 result
= lp_build_negate(get_int_bld(bld_base
, false, src_bit_size
[0]), src
[0]);
676 result
= lp_build_not(get_int_bld(bld_base
, false, src_bit_size
[0]), src
[0]);
679 result
= lp_build_or(get_int_bld(bld_base
, false, src_bit_size
[0]),
683 result
= do_int_mod(bld_base
, false, src_bit_size
[0], src
[0], src
[1]);
686 struct lp_build_context
*uint_bld
= get_int_bld(bld_base
, true, src_bit_size
[0]);
687 struct lp_build_context
*int_bld
= get_int_bld(bld_base
, false, src_bit_size
[0]);
688 if (src_bit_size
[0] == 64)
689 src
[1] = LLVMBuildZExt(builder
, src
[1], uint_bld
->vec_type
, "");
690 if (src_bit_size
[0] < 32)
691 src
[1] = LLVMBuildTrunc(builder
, src
[1], uint_bld
->vec_type
, "");
692 src
[1] = lp_build_and(uint_bld
, src
[1], lp_build_const_int_vec(gallivm
, uint_bld
->type
, (src_bit_size
[0] - 1)));
693 result
= lp_build_shl(int_bld
, src
[0], src
[1]);
697 struct lp_build_context
*uint_bld
= get_int_bld(bld_base
, true, src_bit_size
[0]);
698 struct lp_build_context
*int_bld
= get_int_bld(bld_base
, false, src_bit_size
[0]);
699 if (src_bit_size
[0] == 64)
700 src
[1] = LLVMBuildZExt(builder
, src
[1], uint_bld
->vec_type
, "");
701 if (src_bit_size
[0] < 32)
702 src
[1] = LLVMBuildTrunc(builder
, src
[1], uint_bld
->vec_type
, "");
703 src
[1] = lp_build_and(uint_bld
, src
[1], lp_build_const_int_vec(gallivm
, uint_bld
->type
, (src_bit_size
[0] - 1)));
704 result
= lp_build_shr(int_bld
, src
[0], src
[1]);
708 result
= lp_build_sgn(get_int_bld(bld_base
, false, src_bit_size
[0]), src
[0]);
711 result
= lp_build_sub(get_int_bld(bld_base
, false, src_bit_size
[0]),
715 result
= lp_build_xor(get_int_bld(bld_base
, false, src_bit_size
[0]),
721 case nir_op_unpack_64_2x32_split_x
:
722 result
= split_64bit(bld_base
, src
[0], false);
724 case nir_op_unpack_64_2x32_split_y
:
725 result
= split_64bit(bld_base
, src
[0], true);
728 case nir_op_pack_64_2x32_split
: {
729 LLVMValueRef tmp
= merge_64bit(bld_base
, src
[0], src
[1]);
730 result
= LLVMBuildBitCast(builder
, tmp
, bld_base
->dbl_bld
.vec_type
, "");
734 result
= LLVMBuildUIToFP(builder
, src
[0], bld_base
->base
.vec_type
, "");
737 result
= LLVMBuildUIToFP(builder
, src
[0], bld_base
->dbl_bld
.vec_type
, "");
740 result
= LLVMBuildTrunc(builder
, src
[0], bld_base
->uint8_bld
.vec_type
, "");
743 if (src_bit_size
[0] < 16)
744 result
= LLVMBuildZExt(builder
, src
[0], bld_base
->uint16_bld
.vec_type
, "");
746 result
= LLVMBuildTrunc(builder
, src
[0], bld_base
->uint16_bld
.vec_type
, "");
749 if (src_bit_size
[0] < 32)
750 result
= LLVMBuildZExt(builder
, src
[0], bld_base
->uint_bld
.vec_type
, "");
752 result
= LLVMBuildTrunc(builder
, src
[0], bld_base
->uint_bld
.vec_type
, "");
755 result
= LLVMBuildZExt(builder
, src
[0], bld_base
->uint64_bld
.vec_type
, "");
758 result
= do_int_divide(bld_base
, true, src_bit_size
[0], src
[0], src
[1]);
760 case nir_op_ufind_msb
: {
761 struct lp_build_context
*uint_bld
= get_int_bld(bld_base
, true, src_bit_size
[0]);
762 result
= lp_build_ctlz(uint_bld
, src
[0]);
763 result
= lp_build_sub(uint_bld
, lp_build_const_int_vec(gallivm
, uint_bld
->type
, src_bit_size
[0] - 1), result
);
767 result
= icmp32(bld_base
, PIPE_FUNC_GEQUAL
, true, src_bit_size
[0], src
);
770 result
= icmp32(bld_base
, PIPE_FUNC_LESS
, true, src_bit_size
[0], src
);
773 result
= lp_build_max(get_int_bld(bld_base
, true, src_bit_size
[0]), src
[0], src
[1]);
776 result
= lp_build_min(get_int_bld(bld_base
, true, src_bit_size
[0]), src
[0], src
[1]);
779 result
= do_int_mod(bld_base
, true, src_bit_size
[0], src
[0], src
[1]);
781 case nir_op_umul_high
: {
782 LLVMValueRef hi_bits
;
783 lp_build_mul_32_lohi(&bld_base
->uint_bld
, src
[0], src
[1], &hi_bits
);
788 struct lp_build_context
*uint_bld
= get_int_bld(bld_base
, true, src_bit_size
[0]);
789 if (src_bit_size
[0] == 64)
790 src
[1] = LLVMBuildZExt(builder
, src
[1], uint_bld
->vec_type
, "");
791 if (src_bit_size
[0] < 32)
792 src
[1] = LLVMBuildTrunc(builder
, src
[1], uint_bld
->vec_type
, "");
793 src
[1] = lp_build_and(uint_bld
, src
[1], lp_build_const_int_vec(gallivm
, uint_bld
->type
, (src_bit_size
[0] - 1)));
794 result
= lp_build_shr(uint_bld
, src
[0], src
[1]);
804 static void visit_alu(struct lp_build_nir_context
*bld_base
, const nir_alu_instr
*instr
)
806 struct gallivm_state
*gallivm
= bld_base
->base
.gallivm
;
807 LLVMValueRef src
[NIR_MAX_VEC_COMPONENTS
];
808 unsigned src_bit_size
[NIR_MAX_VEC_COMPONENTS
];
809 unsigned num_components
= nir_dest_num_components(instr
->dest
.dest
);
810 unsigned src_components
;
819 case nir_op_pack_half_2x16
:
822 case nir_op_unpack_half_2x16
:
825 case nir_op_cube_face_coord
:
826 case nir_op_cube_face_index
:
830 src_components
= num_components
;
833 for (unsigned i
= 0; i
< nir_op_infos
[instr
->op
].num_inputs
; i
++) {
834 src
[i
] = get_alu_src(bld_base
, instr
->src
[i
], src_components
);
835 src_bit_size
[i
] = nir_src_bit_size(instr
->src
[i
].src
);
838 LLVMValueRef result
[NIR_MAX_VEC_COMPONENTS
];
839 if (instr
->op
== nir_op_vec4
|| instr
->op
== nir_op_vec3
|| instr
->op
== nir_op_vec2
|| instr
->op
== nir_op_vec8
|| instr
->op
== nir_op_vec16
) {
840 for (unsigned i
= 0; i
< nir_op_infos
[instr
->op
].num_inputs
; i
++) {
841 result
[i
] = cast_type(bld_base
, src
[i
], nir_op_infos
[instr
->op
].input_types
[i
], src_bit_size
[i
]);
844 for (unsigned c
= 0; c
< num_components
; c
++) {
845 LLVMValueRef src_chan
[NIR_MAX_VEC_COMPONENTS
];
847 for (unsigned i
= 0; i
< nir_op_infos
[instr
->op
].num_inputs
; i
++) {
848 if (num_components
> 1) {
849 src_chan
[i
] = LLVMBuildExtractValue(gallivm
->builder
,
852 src_chan
[i
] = src
[i
];
853 src_chan
[i
] = cast_type(bld_base
, src_chan
[i
], nir_op_infos
[instr
->op
].input_types
[i
], src_bit_size
[i
]);
855 result
[c
] = do_alu_action(bld_base
, instr
->op
, src_bit_size
, src_chan
);
856 result
[c
] = cast_type(bld_base
, result
[c
], nir_op_infos
[instr
->op
].output_type
, nir_dest_bit_size(instr
->dest
.dest
));
859 assign_alu_dest(bld_base
, &instr
->dest
, result
);
862 static void visit_load_const(struct lp_build_nir_context
*bld_base
,
863 const nir_load_const_instr
*instr
)
865 LLVMValueRef result
[NIR_MAX_VEC_COMPONENTS
];
866 struct lp_build_context
*int_bld
= get_int_bld(bld_base
, true, instr
->def
.bit_size
);
867 for (unsigned i
= 0; i
< instr
->def
.num_components
; i
++)
868 result
[i
] = lp_build_const_int_vec(bld_base
->base
.gallivm
, int_bld
->type
, instr
->value
[i
].u64
);
869 assign_ssa_dest(bld_base
, &instr
->def
, result
);
873 get_deref_offset(struct lp_build_nir_context
*bld_base
, nir_deref_instr
*instr
,
874 bool vs_in
, unsigned *vertex_index_out
,
875 LLVMValueRef
*vertex_index_ref
,
876 unsigned *const_out
, LLVMValueRef
*indir_out
)
878 LLVMBuilderRef builder
= bld_base
->base
.gallivm
->builder
;
879 nir_variable
*var
= nir_deref_instr_get_variable(instr
);
881 unsigned idx_lvl
= 1;
883 nir_deref_path_init(&path
, instr
, NULL
);
885 if (vertex_index_out
!= NULL
|| vertex_index_ref
!= NULL
) {
886 if (vertex_index_ref
) {
887 *vertex_index_ref
= get_src(bld_base
, path
.path
[idx_lvl
]->arr
.index
);
888 if (vertex_index_out
)
889 *vertex_index_out
= 0;
891 *vertex_index_out
= nir_src_as_uint(path
.path
[idx_lvl
]->arr
.index
);
896 uint32_t const_offset
= 0;
897 LLVMValueRef offset
= NULL
;
899 if (var
->data
.compact
) {
900 assert(instr
->deref_type
== nir_deref_type_array
);
901 const_offset
= nir_src_as_uint(instr
->arr
.index
);
905 for (; path
.path
[idx_lvl
]; ++idx_lvl
) {
906 const struct glsl_type
*parent_type
= path
.path
[idx_lvl
- 1]->type
;
907 if (path
.path
[idx_lvl
]->deref_type
== nir_deref_type_struct
) {
908 unsigned index
= path
.path
[idx_lvl
]->strct
.index
;
910 for (unsigned i
= 0; i
< index
; i
++) {
911 const struct glsl_type
*ft
= glsl_get_struct_field(parent_type
, i
);
912 const_offset
+= glsl_count_attribute_slots(ft
, vs_in
);
914 } else if(path
.path
[idx_lvl
]->deref_type
== nir_deref_type_array
) {
915 unsigned size
= glsl_count_attribute_slots(path
.path
[idx_lvl
]->type
, vs_in
);
916 if (nir_src_is_const(path
.path
[idx_lvl
]->arr
.index
)) {
917 const_offset
+= nir_src_comp_as_int(path
.path
[idx_lvl
]->arr
.index
, 0) * size
;
919 LLVMValueRef idx_src
= get_src(bld_base
, path
.path
[idx_lvl
]->arr
.index
);
920 idx_src
= cast_type(bld_base
, idx_src
, nir_type_uint
, 32);
921 LLVMValueRef array_off
= lp_build_mul(&bld_base
->uint_bld
, lp_build_const_int_vec(bld_base
->base
.gallivm
, bld_base
->base
.type
, size
),
924 offset
= lp_build_add(&bld_base
->uint_bld
, offset
, array_off
);
929 unreachable("Uhandled deref type in get_deref_instr_offset");
933 nir_deref_path_finish(&path
);
935 if (const_offset
&& offset
)
936 offset
= LLVMBuildAdd(builder
, offset
,
937 lp_build_const_int_vec(bld_base
->base
.gallivm
, bld_base
->uint_bld
.type
, const_offset
),
939 *const_out
= const_offset
;
943 static void visit_load_var(struct lp_build_nir_context
*bld_base
,
944 nir_intrinsic_instr
*instr
,
945 LLVMValueRef result
[NIR_MAX_VEC_COMPONENTS
])
947 nir_deref_instr
*deref
= nir_instr_as_deref(instr
->src
[0].ssa
->parent_instr
);
948 nir_variable
*var
= nir_deref_instr_get_variable(deref
);
949 nir_variable_mode mode
= deref
->mode
;
950 unsigned const_index
;
951 LLVMValueRef indir_index
;
952 LLVMValueRef indir_vertex_index
= NULL
;
953 unsigned vertex_index
= 0;
954 unsigned nc
= nir_dest_num_components(instr
->dest
);
955 unsigned bit_size
= nir_dest_bit_size(instr
->dest
);
957 bool vs_in
= bld_base
->shader
->info
.stage
== MESA_SHADER_VERTEX
&&
958 var
->data
.mode
== nir_var_shader_in
;
959 bool gs_in
= bld_base
->shader
->info
.stage
== MESA_SHADER_GEOMETRY
&&
960 var
->data
.mode
== nir_var_shader_in
;
961 bool tcs_in
= bld_base
->shader
->info
.stage
== MESA_SHADER_TESS_CTRL
&&
962 var
->data
.mode
== nir_var_shader_in
;
963 bool tcs_out
= bld_base
->shader
->info
.stage
== MESA_SHADER_TESS_CTRL
&&
964 var
->data
.mode
== nir_var_shader_out
&& !var
->data
.patch
;
965 bool tes_in
= bld_base
->shader
->info
.stage
== MESA_SHADER_TESS_EVAL
&&
966 var
->data
.mode
== nir_var_shader_in
&& !var
->data
.patch
;
968 mode
= var
->data
.mode
;
970 get_deref_offset(bld_base
, deref
, vs_in
, gs_in
? &vertex_index
: NULL
, (tcs_in
|| tcs_out
|| tes_in
) ? &indir_vertex_index
: NULL
,
971 &const_index
, &indir_index
);
973 bld_base
->load_var(bld_base
, mode
, nc
, bit_size
, var
, vertex_index
, indir_vertex_index
, const_index
, indir_index
, result
);
977 visit_store_var(struct lp_build_nir_context
*bld_base
,
978 nir_intrinsic_instr
*instr
)
980 nir_deref_instr
*deref
= nir_instr_as_deref(instr
->src
[0].ssa
->parent_instr
);
981 nir_variable
*var
= nir_deref_instr_get_variable(deref
);
982 nir_variable_mode mode
= deref
->mode
;
983 int writemask
= instr
->const_index
[0];
984 unsigned bit_size
= nir_src_bit_size(instr
->src
[1]);
985 LLVMValueRef src
= get_src(bld_base
, instr
->src
[1]);
986 unsigned const_index
= 0;
987 LLVMValueRef indir_index
, indir_vertex_index
= NULL
;
989 bool tcs_out
= bld_base
->shader
->info
.stage
== MESA_SHADER_TESS_CTRL
&&
990 var
->data
.mode
== nir_var_shader_out
&& !var
->data
.patch
;
991 get_deref_offset(bld_base
, deref
, false, NULL
, tcs_out
? &indir_vertex_index
: NULL
,
992 &const_index
, &indir_index
);
994 bld_base
->store_var(bld_base
, mode
, instr
->num_components
, bit_size
, var
, writemask
, indir_vertex_index
, const_index
, indir_index
, src
);
997 static void visit_load_ubo(struct lp_build_nir_context
*bld_base
,
998 nir_intrinsic_instr
*instr
,
999 LLVMValueRef result
[NIR_MAX_VEC_COMPONENTS
])
1001 struct gallivm_state
*gallivm
= bld_base
->base
.gallivm
;
1002 LLVMBuilderRef builder
= gallivm
->builder
;
1003 LLVMValueRef idx
= get_src(bld_base
, instr
->src
[0]);
1004 LLVMValueRef offset
= get_src(bld_base
, instr
->src
[1]);
1006 bool offset_is_uniform
= nir_src_is_dynamically_uniform(instr
->src
[1]);
1007 idx
= LLVMBuildExtractElement(builder
, idx
, lp_build_const_int32(gallivm
, 0), "");
1008 bld_base
->load_ubo(bld_base
, nir_dest_num_components(instr
->dest
), nir_dest_bit_size(instr
->dest
),
1009 offset_is_uniform
, idx
, offset
, result
);
1013 static void visit_load_ssbo(struct lp_build_nir_context
*bld_base
,
1014 nir_intrinsic_instr
*instr
,
1015 LLVMValueRef result
[NIR_MAX_VEC_COMPONENTS
])
1017 LLVMValueRef idx
= get_src(bld_base
, instr
->src
[0]);
1018 LLVMValueRef offset
= get_src(bld_base
, instr
->src
[1]);
1019 bld_base
->load_mem(bld_base
, nir_dest_num_components(instr
->dest
), nir_dest_bit_size(instr
->dest
),
1020 idx
, offset
, result
);
1023 static void visit_store_ssbo(struct lp_build_nir_context
*bld_base
,
1024 nir_intrinsic_instr
*instr
)
1026 LLVMValueRef val
= get_src(bld_base
, instr
->src
[0]);
1027 LLVMValueRef idx
= get_src(bld_base
, instr
->src
[1]);
1028 LLVMValueRef offset
= get_src(bld_base
, instr
->src
[2]);
1029 int writemask
= instr
->const_index
[0];
1030 int nc
= nir_src_num_components(instr
->src
[0]);
1031 int bitsize
= nir_src_bit_size(instr
->src
[0]);
1032 bld_base
->store_mem(bld_base
, writemask
, nc
, bitsize
, idx
, offset
, val
);
1035 static void visit_get_buffer_size(struct lp_build_nir_context
*bld_base
,
1036 nir_intrinsic_instr
*instr
,
1037 LLVMValueRef result
[NIR_MAX_VEC_COMPONENTS
])
1039 LLVMValueRef idx
= get_src(bld_base
, instr
->src
[0]);
1040 result
[0] = bld_base
->get_buffer_size(bld_base
, idx
);
1043 static void visit_ssbo_atomic(struct lp_build_nir_context
*bld_base
,
1044 nir_intrinsic_instr
*instr
,
1045 LLVMValueRef result
[NIR_MAX_VEC_COMPONENTS
])
1047 LLVMValueRef idx
= get_src(bld_base
, instr
->src
[0]);
1048 LLVMValueRef offset
= get_src(bld_base
, instr
->src
[1]);
1049 LLVMValueRef val
= get_src(bld_base
, instr
->src
[2]);
1050 LLVMValueRef val2
= NULL
;
1051 if (instr
->intrinsic
== nir_intrinsic_ssbo_atomic_comp_swap
)
1052 val2
= get_src(bld_base
, instr
->src
[3]);
1054 bld_base
->atomic_mem(bld_base
, instr
->intrinsic
, idx
, offset
, val
, val2
, &result
[0]);
1058 static void visit_load_image(struct lp_build_nir_context
*bld_base
,
1059 nir_intrinsic_instr
*instr
,
1060 LLVMValueRef result
[NIR_MAX_VEC_COMPONENTS
])
1062 struct gallivm_state
*gallivm
= bld_base
->base
.gallivm
;
1063 LLVMBuilderRef builder
= gallivm
->builder
;
1064 nir_deref_instr
*deref
= nir_instr_as_deref(instr
->src
[0].ssa
->parent_instr
);
1065 nir_variable
*var
= nir_deref_instr_get_variable(deref
);
1066 LLVMValueRef coord_val
= get_src(bld_base
, instr
->src
[1]);
1067 LLVMValueRef coords
[5];
1068 struct lp_img_params params
;
1069 const struct glsl_type
*type
= glsl_without_array(var
->type
);
1071 memset(¶ms
, 0, sizeof(params
));
1072 params
.target
= glsl_sampler_to_pipe(glsl_get_sampler_dim(type
), glsl_sampler_type_is_array(type
));
1073 for (unsigned i
= 0; i
< 4; i
++)
1074 coords
[i
] = LLVMBuildExtractValue(builder
, coord_val
, i
, "");
1075 if (params
.target
== PIPE_TEXTURE_1D_ARRAY
)
1076 coords
[2] = coords
[1];
1078 params
.coords
= coords
;
1079 params
.outdata
= result
;
1080 params
.img_op
= LP_IMG_LOAD
;
1081 if (glsl_get_sampler_dim(type
) == GLSL_SAMPLER_DIM_MS
)
1082 params
.ms_index
= get_src(bld_base
, instr
->src
[2]);
1083 params
.image_index
= var
->data
.binding
;
1084 bld_base
->image_op(bld_base
, ¶ms
);
1087 static void visit_store_image(struct lp_build_nir_context
*bld_base
,
1088 nir_intrinsic_instr
*instr
)
1090 struct gallivm_state
*gallivm
= bld_base
->base
.gallivm
;
1091 LLVMBuilderRef builder
= gallivm
->builder
;
1092 nir_deref_instr
*deref
= nir_instr_as_deref(instr
->src
[0].ssa
->parent_instr
);
1093 nir_variable
*var
= nir_deref_instr_get_variable(deref
);
1094 LLVMValueRef coord_val
= get_src(bld_base
, instr
->src
[1]);
1095 LLVMValueRef in_val
= get_src(bld_base
, instr
->src
[3]);
1096 LLVMValueRef coords
[5];
1097 struct lp_img_params params
;
1098 const struct glsl_type
*type
= glsl_without_array(var
->type
);
1100 memset(¶ms
, 0, sizeof(params
));
1101 params
.target
= glsl_sampler_to_pipe(glsl_get_sampler_dim(type
), glsl_sampler_type_is_array(type
));
1102 for (unsigned i
= 0; i
< 4; i
++)
1103 coords
[i
] = LLVMBuildExtractValue(builder
, coord_val
, i
, "");
1104 if (params
.target
== PIPE_TEXTURE_1D_ARRAY
)
1105 coords
[2] = coords
[1];
1106 params
.coords
= coords
;
1108 for (unsigned i
= 0; i
< 4; i
++) {
1109 params
.indata
[i
] = LLVMBuildExtractValue(builder
, in_val
, i
, "");
1110 params
.indata
[i
] = LLVMBuildBitCast(builder
, params
.indata
[i
], bld_base
->base
.vec_type
, "");
1112 if (glsl_get_sampler_dim(type
) == GLSL_SAMPLER_DIM_MS
)
1113 params
.ms_index
= get_src(bld_base
, instr
->src
[2]);
1114 params
.img_op
= LP_IMG_STORE
;
1115 params
.image_index
= var
->data
.binding
;
1117 if (params
.target
== PIPE_TEXTURE_1D_ARRAY
)
1118 coords
[2] = coords
[1];
1119 bld_base
->image_op(bld_base
, ¶ms
);
1122 static void visit_atomic_image(struct lp_build_nir_context
*bld_base
,
1123 nir_intrinsic_instr
*instr
,
1124 LLVMValueRef result
[NIR_MAX_VEC_COMPONENTS
])
1126 struct gallivm_state
*gallivm
= bld_base
->base
.gallivm
;
1127 LLVMBuilderRef builder
= gallivm
->builder
;
1128 nir_deref_instr
*deref
= nir_instr_as_deref(instr
->src
[0].ssa
->parent_instr
);
1129 nir_variable
*var
= nir_deref_instr_get_variable(deref
);
1130 struct lp_img_params params
;
1131 LLVMValueRef coord_val
= get_src(bld_base
, instr
->src
[1]);
1132 LLVMValueRef in_val
= get_src(bld_base
, instr
->src
[3]);
1133 LLVMValueRef coords
[5];
1134 const struct glsl_type
*type
= glsl_without_array(var
->type
);
1136 memset(¶ms
, 0, sizeof(params
));
1138 switch (instr
->intrinsic
) {
1139 case nir_intrinsic_image_deref_atomic_add
:
1140 params
.op
= LLVMAtomicRMWBinOpAdd
;
1142 case nir_intrinsic_image_deref_atomic_exchange
:
1143 params
.op
= LLVMAtomicRMWBinOpXchg
;
1145 case nir_intrinsic_image_deref_atomic_and
:
1146 params
.op
= LLVMAtomicRMWBinOpAnd
;
1148 case nir_intrinsic_image_deref_atomic_or
:
1149 params
.op
= LLVMAtomicRMWBinOpOr
;
1151 case nir_intrinsic_image_deref_atomic_xor
:
1152 params
.op
= LLVMAtomicRMWBinOpXor
;
1154 case nir_intrinsic_image_deref_atomic_umin
:
1155 params
.op
= LLVMAtomicRMWBinOpUMin
;
1157 case nir_intrinsic_image_deref_atomic_umax
:
1158 params
.op
= LLVMAtomicRMWBinOpUMax
;
1160 case nir_intrinsic_image_deref_atomic_imin
:
1161 params
.op
= LLVMAtomicRMWBinOpMin
;
1163 case nir_intrinsic_image_deref_atomic_imax
:
1164 params
.op
= LLVMAtomicRMWBinOpMax
;
1170 params
.target
= glsl_sampler_to_pipe(glsl_get_sampler_dim(type
), glsl_sampler_type_is_array(type
));
1171 for (unsigned i
= 0; i
< 4; i
++)
1172 coords
[i
] = LLVMBuildExtractValue(builder
, coord_val
, i
, "");
1173 if (params
.target
== PIPE_TEXTURE_1D_ARRAY
)
1174 coords
[2] = coords
[1];
1175 params
.coords
= coords
;
1176 if (glsl_get_sampler_dim(type
) == GLSL_SAMPLER_DIM_MS
)
1177 params
.ms_index
= get_src(bld_base
, instr
->src
[2]);
1178 if (instr
->intrinsic
== nir_intrinsic_image_deref_atomic_comp_swap
) {
1179 LLVMValueRef cas_val
= get_src(bld_base
, instr
->src
[4]);
1180 params
.indata
[0] = in_val
;
1181 params
.indata2
[0] = cas_val
;
1183 params
.indata
[0] = in_val
;
1185 params
.outdata
= result
;
1186 params
.img_op
= (instr
->intrinsic
== nir_intrinsic_image_deref_atomic_comp_swap
) ? LP_IMG_ATOMIC_CAS
: LP_IMG_ATOMIC
;
1187 params
.image_index
= var
->data
.binding
;
1189 bld_base
->image_op(bld_base
, ¶ms
);
1193 static void visit_image_size(struct lp_build_nir_context
*bld_base
,
1194 nir_intrinsic_instr
*instr
,
1195 LLVMValueRef result
[NIR_MAX_VEC_COMPONENTS
])
1197 nir_deref_instr
*deref
= nir_instr_as_deref(instr
->src
[0].ssa
->parent_instr
);
1198 nir_variable
*var
= nir_deref_instr_get_variable(deref
);
1199 struct lp_sampler_size_query_params params
= { 0 };
1200 params
.texture_unit
= var
->data
.binding
;
1201 params
.target
= glsl_sampler_to_pipe(glsl_get_sampler_dim(var
->type
), glsl_sampler_type_is_array(var
->type
));
1202 params
.sizes_out
= result
;
1204 bld_base
->image_size(bld_base
, ¶ms
);
1207 static void visit_image_samples(struct lp_build_nir_context
*bld_base
,
1208 nir_intrinsic_instr
*instr
,
1209 LLVMValueRef result
[NIR_MAX_VEC_COMPONENTS
])
1211 nir_deref_instr
*deref
= nir_instr_as_deref(instr
->src
[0].ssa
->parent_instr
);
1212 nir_variable
*var
= nir_deref_instr_get_variable(deref
);
1213 struct lp_sampler_size_query_params params
= { 0 };
1214 params
.texture_unit
= var
->data
.binding
;
1215 params
.target
= glsl_sampler_to_pipe(glsl_get_sampler_dim(var
->type
), glsl_sampler_type_is_array(var
->type
));
1216 params
.sizes_out
= result
;
1217 params
.samples_only
= true;
1219 bld_base
->image_size(bld_base
, ¶ms
);
1222 static void visit_shared_load(struct lp_build_nir_context
*bld_base
,
1223 nir_intrinsic_instr
*instr
,
1224 LLVMValueRef result
[NIR_MAX_VEC_COMPONENTS
])
1226 LLVMValueRef offset
= get_src(bld_base
, instr
->src
[0]);
1227 bld_base
->load_mem(bld_base
, nir_dest_num_components(instr
->dest
), nir_dest_bit_size(instr
->dest
),
1228 NULL
, offset
, result
);
1231 static void visit_shared_store(struct lp_build_nir_context
*bld_base
,
1232 nir_intrinsic_instr
*instr
)
1234 LLVMValueRef val
= get_src(bld_base
, instr
->src
[0]);
1235 LLVMValueRef offset
= get_src(bld_base
, instr
->src
[1]);
1236 int writemask
= instr
->const_index
[1];
1237 int nc
= nir_src_num_components(instr
->src
[0]);
1238 int bitsize
= nir_src_bit_size(instr
->src
[0]);
1239 bld_base
->store_mem(bld_base
, writemask
, nc
, bitsize
, NULL
, offset
, val
);
1242 static void visit_shared_atomic(struct lp_build_nir_context
*bld_base
,
1243 nir_intrinsic_instr
*instr
,
1244 LLVMValueRef result
[NIR_MAX_VEC_COMPONENTS
])
1246 LLVMValueRef offset
= get_src(bld_base
, instr
->src
[0]);
1247 LLVMValueRef val
= get_src(bld_base
, instr
->src
[1]);
1248 LLVMValueRef val2
= NULL
;
1249 if (instr
->intrinsic
== nir_intrinsic_shared_atomic_comp_swap
)
1250 val2
= get_src(bld_base
, instr
->src
[2]);
1252 bld_base
->atomic_mem(bld_base
, instr
->intrinsic
, NULL
, offset
, val
, val2
, &result
[0]);
1256 static void visit_barrier(struct lp_build_nir_context
*bld_base
)
1258 bld_base
->barrier(bld_base
);
1261 static void visit_discard(struct lp_build_nir_context
*bld_base
,
1262 nir_intrinsic_instr
*instr
)
1264 LLVMValueRef cond
= NULL
;
1265 if (instr
->intrinsic
== nir_intrinsic_discard_if
) {
1266 cond
= get_src(bld_base
, instr
->src
[0]);
1267 cond
= cast_type(bld_base
, cond
, nir_type_int
, 32);
1269 bld_base
->discard(bld_base
, cond
);
1272 static void visit_load_kernel_input(struct lp_build_nir_context
*bld_base
,
1273 nir_intrinsic_instr
*instr
, LLVMValueRef result
[NIR_MAX_VEC_COMPONENTS
])
1275 LLVMValueRef offset
= get_src(bld_base
, instr
->src
[0]);
1277 bool offset_is_uniform
= nir_src_is_dynamically_uniform(instr
->src
[0]);
1278 bld_base
->load_kernel_arg(bld_base
, nir_dest_num_components(instr
->dest
), nir_dest_bit_size(instr
->dest
),
1279 nir_src_bit_size(instr
->src
[0]),
1280 offset_is_uniform
, offset
, result
);
1283 static void visit_load_global(struct lp_build_nir_context
*bld_base
,
1284 nir_intrinsic_instr
*instr
, LLVMValueRef result
[NIR_MAX_VEC_COMPONENTS
])
1286 LLVMValueRef addr
= get_src(bld_base
, instr
->src
[0]);
1287 bld_base
->load_global(bld_base
, nir_dest_num_components(instr
->dest
), nir_dest_bit_size(instr
->dest
),
1288 nir_src_bit_size(instr
->src
[0]),
1292 static void visit_store_global(struct lp_build_nir_context
*bld_base
,
1293 nir_intrinsic_instr
*instr
)
1295 LLVMValueRef val
= get_src(bld_base
, instr
->src
[0]);
1296 int nc
= nir_src_num_components(instr
->src
[0]);
1297 int bitsize
= nir_src_bit_size(instr
->src
[0]);
1298 LLVMValueRef addr
= get_src(bld_base
, instr
->src
[1]);
1299 int addr_bitsize
= nir_src_bit_size(instr
->src
[1]);
1300 int writemask
= instr
->const_index
[0];
1301 bld_base
->store_global(bld_base
, writemask
, nc
, bitsize
, addr_bitsize
, addr
, val
);
1304 static void visit_global_atomic(struct lp_build_nir_context
*bld_base
,
1305 nir_intrinsic_instr
*instr
,
1306 LLVMValueRef result
[NIR_MAX_VEC_COMPONENTS
])
1308 LLVMValueRef addr
= get_src(bld_base
, instr
->src
[0]);
1309 LLVMValueRef val
= get_src(bld_base
, instr
->src
[1]);
1310 LLVMValueRef val2
= NULL
;
1311 int addr_bitsize
= nir_src_bit_size(instr
->src
[0]);
1312 if (instr
->intrinsic
== nir_intrinsic_global_atomic_comp_swap
)
1313 val2
= get_src(bld_base
, instr
->src
[2]);
1315 bld_base
->atomic_global(bld_base
, instr
->intrinsic
, addr_bitsize
, addr
, val
, val2
, &result
[0]);
1318 static void visit_interp(struct lp_build_nir_context
*bld_base
,
1319 nir_intrinsic_instr
*instr
,
1320 LLVMValueRef result
[NIR_MAX_VEC_COMPONENTS
])
1322 struct gallivm_state
*gallivm
= bld_base
->base
.gallivm
;
1323 LLVMBuilderRef builder
= gallivm
->builder
;
1324 nir_deref_instr
*deref
= nir_instr_as_deref(instr
->src
[0].ssa
->parent_instr
);
1325 unsigned num_components
= nir_dest_num_components(instr
->dest
);
1326 nir_variable
*var
= nir_deref_instr_get_variable(deref
);
1327 unsigned const_index
;
1328 LLVMValueRef indir_index
;
1329 LLVMValueRef offsets
[2] = { NULL
, NULL
};
1330 get_deref_offset(bld_base
, deref
, false, NULL
, NULL
,
1331 &const_index
, &indir_index
);
1332 bool centroid
= instr
->intrinsic
== nir_intrinsic_interp_deref_at_centroid
;
1333 bool sample
= false;
1334 if (instr
->intrinsic
== nir_intrinsic_interp_deref_at_offset
) {
1335 for (unsigned i
= 0; i
< 2; i
++) {
1336 offsets
[i
] = LLVMBuildExtractValue(builder
, get_src(bld_base
, instr
->src
[1]), i
, "");
1337 offsets
[i
] = cast_type(bld_base
, offsets
[i
], nir_type_float
, 32);
1339 } else if (instr
->intrinsic
== nir_intrinsic_interp_deref_at_sample
) {
1340 offsets
[0] = get_src(bld_base
, instr
->src
[1]);
1343 bld_base
->interp_at(bld_base
, num_components
, var
, centroid
, sample
, const_index
, indir_index
, offsets
, result
);
1346 static void visit_intrinsic(struct lp_build_nir_context
*bld_base
,
1347 nir_intrinsic_instr
*instr
)
1349 LLVMValueRef result
[NIR_MAX_VEC_COMPONENTS
] = {0};
1350 switch (instr
->intrinsic
) {
1351 case nir_intrinsic_load_deref
:
1352 visit_load_var(bld_base
, instr
, result
);
1354 case nir_intrinsic_store_deref
:
1355 visit_store_var(bld_base
, instr
);
1357 case nir_intrinsic_load_ubo
:
1358 visit_load_ubo(bld_base
, instr
, result
);
1360 case nir_intrinsic_load_ssbo
:
1361 visit_load_ssbo(bld_base
, instr
, result
);
1363 case nir_intrinsic_store_ssbo
:
1364 visit_store_ssbo(bld_base
, instr
);
1366 case nir_intrinsic_get_buffer_size
:
1367 visit_get_buffer_size(bld_base
, instr
, result
);
1369 case nir_intrinsic_load_vertex_id
:
1370 case nir_intrinsic_load_primitive_id
:
1371 case nir_intrinsic_load_instance_id
:
1372 case nir_intrinsic_load_base_instance
:
1373 case nir_intrinsic_load_base_vertex
:
1374 case nir_intrinsic_load_work_group_id
:
1375 case nir_intrinsic_load_local_invocation_id
:
1376 case nir_intrinsic_load_num_work_groups
:
1377 case nir_intrinsic_load_invocation_id
:
1378 case nir_intrinsic_load_front_face
:
1379 case nir_intrinsic_load_draw_id
:
1380 case nir_intrinsic_load_local_group_size
:
1381 case nir_intrinsic_load_work_dim
:
1382 case nir_intrinsic_load_tess_coord
:
1383 case nir_intrinsic_load_tess_level_outer
:
1384 case nir_intrinsic_load_tess_level_inner
:
1385 case nir_intrinsic_load_patch_vertices_in
:
1386 case nir_intrinsic_load_sample_id
:
1387 case nir_intrinsic_load_sample_pos
:
1388 bld_base
->sysval_intrin(bld_base
, instr
, result
);
1390 case nir_intrinsic_load_helper_invocation
:
1391 bld_base
->helper_invocation(bld_base
, &result
[0]);
1393 case nir_intrinsic_discard_if
:
1394 case nir_intrinsic_discard
:
1395 visit_discard(bld_base
, instr
);
1397 case nir_intrinsic_emit_vertex
:
1398 bld_base
->emit_vertex(bld_base
, nir_intrinsic_stream_id(instr
));
1400 case nir_intrinsic_end_primitive
:
1401 bld_base
->end_primitive(bld_base
, nir_intrinsic_stream_id(instr
));
1403 case nir_intrinsic_ssbo_atomic_add
:
1404 case nir_intrinsic_ssbo_atomic_imin
:
1405 case nir_intrinsic_ssbo_atomic_imax
:
1406 case nir_intrinsic_ssbo_atomic_umin
:
1407 case nir_intrinsic_ssbo_atomic_umax
:
1408 case nir_intrinsic_ssbo_atomic_and
:
1409 case nir_intrinsic_ssbo_atomic_or
:
1410 case nir_intrinsic_ssbo_atomic_xor
:
1411 case nir_intrinsic_ssbo_atomic_exchange
:
1412 case nir_intrinsic_ssbo_atomic_comp_swap
:
1413 visit_ssbo_atomic(bld_base
, instr
, result
);
1415 case nir_intrinsic_image_deref_load
:
1416 visit_load_image(bld_base
, instr
, result
);
1418 case nir_intrinsic_image_deref_store
:
1419 visit_store_image(bld_base
, instr
);
1421 case nir_intrinsic_image_deref_atomic_add
:
1422 case nir_intrinsic_image_deref_atomic_imin
:
1423 case nir_intrinsic_image_deref_atomic_imax
:
1424 case nir_intrinsic_image_deref_atomic_umin
:
1425 case nir_intrinsic_image_deref_atomic_umax
:
1426 case nir_intrinsic_image_deref_atomic_and
:
1427 case nir_intrinsic_image_deref_atomic_or
:
1428 case nir_intrinsic_image_deref_atomic_xor
:
1429 case nir_intrinsic_image_deref_atomic_exchange
:
1430 case nir_intrinsic_image_deref_atomic_comp_swap
:
1431 visit_atomic_image(bld_base
, instr
, result
);
1433 case nir_intrinsic_image_deref_size
:
1434 visit_image_size(bld_base
, instr
, result
);
1436 case nir_intrinsic_image_deref_samples
:
1437 visit_image_samples(bld_base
, instr
, result
);
1439 case nir_intrinsic_load_shared
:
1440 visit_shared_load(bld_base
, instr
, result
);
1442 case nir_intrinsic_store_shared
:
1443 visit_shared_store(bld_base
, instr
);
1445 case nir_intrinsic_shared_atomic_add
:
1446 case nir_intrinsic_shared_atomic_imin
:
1447 case nir_intrinsic_shared_atomic_umin
:
1448 case nir_intrinsic_shared_atomic_imax
:
1449 case nir_intrinsic_shared_atomic_umax
:
1450 case nir_intrinsic_shared_atomic_and
:
1451 case nir_intrinsic_shared_atomic_or
:
1452 case nir_intrinsic_shared_atomic_xor
:
1453 case nir_intrinsic_shared_atomic_exchange
:
1454 case nir_intrinsic_shared_atomic_comp_swap
:
1455 visit_shared_atomic(bld_base
, instr
, result
);
1457 case nir_intrinsic_control_barrier
:
1458 visit_barrier(bld_base
);
1460 case nir_intrinsic_memory_barrier
:
1461 case nir_intrinsic_memory_barrier_shared
:
1462 case nir_intrinsic_memory_barrier_buffer
:
1463 case nir_intrinsic_memory_barrier_image
:
1464 case nir_intrinsic_memory_barrier_tcs_patch
:
1466 case nir_intrinsic_load_kernel_input
:
1467 visit_load_kernel_input(bld_base
, instr
, result
);
1469 case nir_intrinsic_load_global
:
1470 visit_load_global(bld_base
, instr
, result
);
1472 case nir_intrinsic_store_global
:
1473 visit_store_global(bld_base
, instr
);
1475 case nir_intrinsic_global_atomic_add
:
1476 case nir_intrinsic_global_atomic_imin
:
1477 case nir_intrinsic_global_atomic_umin
:
1478 case nir_intrinsic_global_atomic_imax
:
1479 case nir_intrinsic_global_atomic_umax
:
1480 case nir_intrinsic_global_atomic_and
:
1481 case nir_intrinsic_global_atomic_or
:
1482 case nir_intrinsic_global_atomic_xor
:
1483 case nir_intrinsic_global_atomic_exchange
:
1484 case nir_intrinsic_global_atomic_comp_swap
:
1485 visit_global_atomic(bld_base
, instr
, result
);
1486 case nir_intrinsic_vote_all
:
1487 case nir_intrinsic_vote_any
:
1488 case nir_intrinsic_vote_ieq
:
1489 bld_base
->vote(bld_base
, cast_type(bld_base
, get_src(bld_base
, instr
->src
[0]), nir_type_int
, 32), instr
, result
);
1491 case nir_intrinsic_interp_deref_at_offset
:
1492 case nir_intrinsic_interp_deref_at_centroid
:
1493 case nir_intrinsic_interp_deref_at_sample
:
1494 visit_interp(bld_base
, instr
, result
);
1501 assign_dest(bld_base
, &instr
->dest
, result
);
1505 static void visit_txs(struct lp_build_nir_context
*bld_base
, nir_tex_instr
*instr
)
1507 struct lp_sampler_size_query_params params
= { 0 };
1508 LLVMValueRef sizes_out
[NIR_MAX_VEC_COMPONENTS
];
1509 LLVMValueRef explicit_lod
= NULL
;
1511 for (unsigned i
= 0; i
< instr
->num_srcs
; i
++) {
1512 switch (instr
->src
[i
].src_type
) {
1513 case nir_tex_src_lod
:
1514 explicit_lod
= cast_type(bld_base
, get_src(bld_base
, instr
->src
[i
].src
), nir_type_int
, 32);
1521 params
.target
= glsl_sampler_to_pipe(instr
->sampler_dim
, instr
->is_array
);
1522 params
.texture_unit
= instr
->texture_index
;
1523 params
.explicit_lod
= explicit_lod
;
1524 params
.is_sviewinfo
= TRUE
;
1525 params
.sizes_out
= sizes_out
;
1526 params
.samples_only
= (instr
->op
== nir_texop_texture_samples
);
1528 if (instr
->op
== nir_texop_query_levels
)
1529 params
.explicit_lod
= bld_base
->uint_bld
.zero
;
1530 bld_base
->tex_size(bld_base
, ¶ms
);
1531 assign_dest(bld_base
, &instr
->dest
, &sizes_out
[instr
->op
== nir_texop_query_levels
? 3 : 0]);
1534 static enum lp_sampler_lod_property
lp_build_nir_lod_property(struct lp_build_nir_context
*bld_base
,
1537 enum lp_sampler_lod_property lod_property
;
1539 if (nir_src_is_dynamically_uniform(lod_src
))
1540 lod_property
= LP_SAMPLER_LOD_SCALAR
;
1541 else if (bld_base
->shader
->info
.stage
== MESA_SHADER_FRAGMENT
) {
1542 if (gallivm_perf
& GALLIVM_PERF_NO_QUAD_LOD
)
1543 lod_property
= LP_SAMPLER_LOD_PER_ELEMENT
;
1545 lod_property
= LP_SAMPLER_LOD_PER_QUAD
;
1548 lod_property
= LP_SAMPLER_LOD_PER_ELEMENT
;
1549 return lod_property
;
1552 static void visit_tex(struct lp_build_nir_context
*bld_base
, nir_tex_instr
*instr
)
1554 struct gallivm_state
*gallivm
= bld_base
->base
.gallivm
;
1555 LLVMBuilderRef builder
= gallivm
->builder
;
1556 LLVMValueRef coords
[5];
1557 LLVMValueRef offsets
[3] = { NULL
};
1558 LLVMValueRef explicit_lod
= NULL
, projector
= NULL
, ms_index
= NULL
;
1559 struct lp_sampler_params params
;
1560 struct lp_derivatives derivs
;
1561 unsigned sample_key
= 0;
1562 nir_deref_instr
*texture_deref_instr
= NULL
;
1563 nir_deref_instr
*sampler_deref_instr
= NULL
;
1564 LLVMValueRef texel
[NIR_MAX_VEC_COMPONENTS
];
1565 unsigned lod_src
= 0;
1566 LLVMValueRef coord_undef
= LLVMGetUndef(bld_base
->base
.int_vec_type
);
1568 memset(¶ms
, 0, sizeof(params
));
1569 enum lp_sampler_lod_property lod_property
= LP_SAMPLER_LOD_SCALAR
;
1571 if (instr
->op
== nir_texop_txs
|| instr
->op
== nir_texop_query_levels
|| instr
->op
== nir_texop_texture_samples
) {
1572 visit_txs(bld_base
, instr
);
1575 if (instr
->op
== nir_texop_txf
|| instr
->op
== nir_texop_txf_ms
)
1576 sample_key
|= LP_SAMPLER_OP_FETCH
<< LP_SAMPLER_OP_TYPE_SHIFT
;
1577 else if (instr
->op
== nir_texop_tg4
) {
1578 sample_key
|= LP_SAMPLER_OP_GATHER
<< LP_SAMPLER_OP_TYPE_SHIFT
;
1579 sample_key
|= (instr
->component
<< LP_SAMPLER_GATHER_COMP_SHIFT
);
1580 } else if (instr
->op
== nir_texop_lod
)
1581 sample_key
|= LP_SAMPLER_OP_LODQ
<< LP_SAMPLER_OP_TYPE_SHIFT
;
1582 for (unsigned i
= 0; i
< instr
->num_srcs
; i
++) {
1583 switch (instr
->src
[i
].src_type
) {
1584 case nir_tex_src_coord
: {
1585 LLVMValueRef coord
= get_src(bld_base
, instr
->src
[i
].src
);
1586 if (instr
->coord_components
== 1)
1589 for (unsigned chan
= 0; chan
< instr
->coord_components
; ++chan
)
1590 coords
[chan
] = LLVMBuildExtractValue(builder
, coord
,
1593 for (unsigned chan
= instr
->coord_components
; chan
< 5; chan
++)
1594 coords
[chan
] = coord_undef
;
1598 case nir_tex_src_texture_deref
:
1599 texture_deref_instr
= nir_src_as_deref(instr
->src
[i
].src
);
1601 case nir_tex_src_sampler_deref
:
1602 sampler_deref_instr
= nir_src_as_deref(instr
->src
[i
].src
);
1604 case nir_tex_src_projector
:
1605 projector
= lp_build_rcp(&bld_base
->base
, cast_type(bld_base
, get_src(bld_base
, instr
->src
[i
].src
), nir_type_float
, 32));
1607 case nir_tex_src_comparator
:
1608 sample_key
|= LP_SAMPLER_SHADOW
;
1609 coords
[4] = get_src(bld_base
, instr
->src
[i
].src
);
1610 coords
[4] = cast_type(bld_base
, coords
[4], nir_type_float
, 32);
1612 case nir_tex_src_bias
:
1613 sample_key
|= LP_SAMPLER_LOD_BIAS
<< LP_SAMPLER_LOD_CONTROL_SHIFT
;
1615 explicit_lod
= cast_type(bld_base
, get_src(bld_base
, instr
->src
[i
].src
), nir_type_float
, 32);
1617 case nir_tex_src_lod
:
1618 sample_key
|= LP_SAMPLER_LOD_EXPLICIT
<< LP_SAMPLER_LOD_CONTROL_SHIFT
;
1620 if (instr
->op
== nir_texop_txf
)
1621 explicit_lod
= cast_type(bld_base
, get_src(bld_base
, instr
->src
[i
].src
), nir_type_int
, 32);
1623 explicit_lod
= cast_type(bld_base
, get_src(bld_base
, instr
->src
[i
].src
), nir_type_float
, 32);
1625 case nir_tex_src_ddx
: {
1626 int deriv_cnt
= instr
->coord_components
;
1627 if (instr
->is_array
)
1629 LLVMValueRef deriv_val
= get_src(bld_base
, instr
->src
[i
].src
);
1631 derivs
.ddx
[0] = deriv_val
;
1633 for (unsigned chan
= 0; chan
< deriv_cnt
; ++chan
)
1634 derivs
.ddx
[chan
] = LLVMBuildExtractValue(builder
, deriv_val
,
1636 for (unsigned chan
= 0; chan
< deriv_cnt
; ++chan
)
1637 derivs
.ddx
[chan
] = cast_type(bld_base
, derivs
.ddx
[chan
], nir_type_float
, 32);
1640 case nir_tex_src_ddy
: {
1641 int deriv_cnt
= instr
->coord_components
;
1642 if (instr
->is_array
)
1644 LLVMValueRef deriv_val
= get_src(bld_base
, instr
->src
[i
].src
);
1646 derivs
.ddy
[0] = deriv_val
;
1648 for (unsigned chan
= 0; chan
< deriv_cnt
; ++chan
)
1649 derivs
.ddy
[chan
] = LLVMBuildExtractValue(builder
, deriv_val
,
1651 for (unsigned chan
= 0; chan
< deriv_cnt
; ++chan
)
1652 derivs
.ddy
[chan
] = cast_type(bld_base
, derivs
.ddy
[chan
], nir_type_float
, 32);
1655 case nir_tex_src_offset
: {
1656 int offset_cnt
= instr
->coord_components
;
1657 if (instr
->is_array
)
1659 LLVMValueRef offset_val
= get_src(bld_base
, instr
->src
[i
].src
);
1660 sample_key
|= LP_SAMPLER_OFFSETS
;
1661 if (offset_cnt
== 1)
1662 offsets
[0] = cast_type(bld_base
, offset_val
, nir_type_int
, 32);
1664 for (unsigned chan
= 0; chan
< offset_cnt
; ++chan
) {
1665 offsets
[chan
] = LLVMBuildExtractValue(builder
, offset_val
,
1667 offsets
[chan
] = cast_type(bld_base
, offsets
[chan
], nir_type_int
, 32);
1672 case nir_tex_src_ms_index
:
1673 sample_key
|= LP_SAMPLER_FETCH_MS
;
1674 ms_index
= cast_type(bld_base
, get_src(bld_base
, instr
->src
[i
].src
), nir_type_int
, 32);
1681 if (!sampler_deref_instr
)
1682 sampler_deref_instr
= texture_deref_instr
;
1685 lod_property
= lp_build_nir_lod_property(bld_base
, instr
->src
[lod_src
].src
);
1687 if (instr
->op
== nir_texop_tex
|| instr
->op
== nir_texop_tg4
|| instr
->op
== nir_texop_txb
||
1688 instr
->op
== nir_texop_txl
|| instr
->op
== nir_texop_txd
|| instr
->op
== nir_texop_lod
)
1689 for (unsigned chan
= 0; chan
< instr
->coord_components
; ++chan
)
1690 coords
[chan
] = cast_type(bld_base
, coords
[chan
], nir_type_float
, 32);
1691 else if (instr
->op
== nir_texop_txf
|| instr
->op
== nir_texop_txf_ms
)
1692 for (unsigned chan
= 0; chan
< instr
->coord_components
; ++chan
)
1693 coords
[chan
] = cast_type(bld_base
, coords
[chan
], nir_type_int
, 32);
1695 if (instr
->is_array
&& instr
->sampler_dim
== GLSL_SAMPLER_DIM_1D
) {
1696 /* move layer coord for 1d arrays. */
1697 coords
[2] = coords
[1];
1698 coords
[1] = coord_undef
;
1702 for (unsigned chan
= 0; chan
< instr
->coord_components
; ++chan
)
1703 coords
[chan
] = lp_build_mul(&bld_base
->base
, coords
[chan
], projector
);
1704 if (sample_key
& LP_SAMPLER_SHADOW
)
1705 coords
[4] = lp_build_mul(&bld_base
->base
, coords
[4], projector
);
1708 uint32_t base_index
= 0;
1709 if (!texture_deref_instr
) {
1710 int samp_src_index
= nir_tex_instr_src_index(instr
, nir_tex_src_sampler_handle
);
1711 if (samp_src_index
== -1) {
1712 base_index
= instr
->sampler_index
;
1716 if (instr
->op
== nir_texop_txd
) {
1717 sample_key
|= LP_SAMPLER_LOD_DERIVATIVES
<< LP_SAMPLER_LOD_CONTROL_SHIFT
;
1718 params
.derivs
= &derivs
;
1719 if (bld_base
->shader
->info
.stage
== MESA_SHADER_FRAGMENT
) {
1720 if (gallivm_perf
& GALLIVM_PERF_NO_QUAD_LOD
)
1721 lod_property
= LP_SAMPLER_LOD_PER_ELEMENT
;
1723 lod_property
= LP_SAMPLER_LOD_PER_QUAD
;
1725 lod_property
= LP_SAMPLER_LOD_PER_ELEMENT
;
1728 sample_key
|= lod_property
<< LP_SAMPLER_LOD_PROPERTY_SHIFT
;
1729 params
.sample_key
= sample_key
;
1730 params
.offsets
= offsets
;
1731 params
.texture_index
= base_index
;
1732 params
.sampler_index
= base_index
;
1733 params
.coords
= coords
;
1734 params
.texel
= texel
;
1735 params
.lod
= explicit_lod
;
1736 params
.ms_index
= ms_index
;
1737 bld_base
->tex(bld_base
, ¶ms
);
1738 assign_dest(bld_base
, &instr
->dest
, texel
);
1741 static void visit_ssa_undef(struct lp_build_nir_context
*bld_base
,
1742 const nir_ssa_undef_instr
*instr
)
1744 unsigned num_components
= instr
->def
.num_components
;
1745 LLVMValueRef undef
[NIR_MAX_VEC_COMPONENTS
];
1746 struct lp_build_context
*undef_bld
= get_int_bld(bld_base
, true, instr
->def
.bit_size
);
1747 for (unsigned i
= 0; i
< num_components
; i
++)
1748 undef
[i
] = LLVMGetUndef(undef_bld
->vec_type
);
1749 assign_ssa_dest(bld_base
, &instr
->def
, undef
);
1752 static void visit_jump(struct lp_build_nir_context
*bld_base
,
1753 const nir_jump_instr
*instr
)
1755 switch (instr
->type
) {
1756 case nir_jump_break
:
1757 bld_base
->break_stmt(bld_base
);
1759 case nir_jump_continue
:
1760 bld_base
->continue_stmt(bld_base
);
1763 unreachable("Unknown jump instr\n");
1767 static void visit_deref(struct lp_build_nir_context
*bld_base
,
1768 nir_deref_instr
*instr
)
1770 if (instr
->mode
!= nir_var_mem_shared
&&
1771 instr
->mode
!= nir_var_mem_global
)
1773 LLVMValueRef result
= NULL
;
1774 switch(instr
->deref_type
) {
1775 case nir_deref_type_var
: {
1776 struct hash_entry
*entry
= _mesa_hash_table_search(bld_base
->vars
, instr
->var
);
1777 result
= entry
->data
;
1781 unreachable("Unhandled deref_instr deref type");
1784 assign_ssa(bld_base
, instr
->dest
.ssa
.index
, result
);
1787 static void visit_block(struct lp_build_nir_context
*bld_base
, nir_block
*block
)
1789 nir_foreach_instr(instr
, block
)
1791 switch (instr
->type
) {
1792 case nir_instr_type_alu
:
1793 visit_alu(bld_base
, nir_instr_as_alu(instr
));
1795 case nir_instr_type_load_const
:
1796 visit_load_const(bld_base
, nir_instr_as_load_const(instr
));
1798 case nir_instr_type_intrinsic
:
1799 visit_intrinsic(bld_base
, nir_instr_as_intrinsic(instr
));
1801 case nir_instr_type_tex
:
1802 visit_tex(bld_base
, nir_instr_as_tex(instr
));
1804 case nir_instr_type_phi
:
1807 case nir_instr_type_ssa_undef
:
1808 visit_ssa_undef(bld_base
, nir_instr_as_ssa_undef(instr
));
1810 case nir_instr_type_jump
:
1811 visit_jump(bld_base
, nir_instr_as_jump(instr
));
1813 case nir_instr_type_deref
:
1814 visit_deref(bld_base
, nir_instr_as_deref(instr
));
1817 fprintf(stderr
, "Unknown NIR instr type: ");
1818 nir_print_instr(instr
, stderr
);
1819 fprintf(stderr
, "\n");
1825 static void visit_if(struct lp_build_nir_context
*bld_base
, nir_if
*if_stmt
)
1827 LLVMValueRef cond
= get_src(bld_base
, if_stmt
->condition
);
1829 bld_base
->if_cond(bld_base
, cond
);
1830 visit_cf_list(bld_base
, &if_stmt
->then_list
);
1832 if (!exec_list_is_empty(&if_stmt
->else_list
)) {
1833 bld_base
->else_stmt(bld_base
);
1834 visit_cf_list(bld_base
, &if_stmt
->else_list
);
1836 bld_base
->endif_stmt(bld_base
);
1839 static void visit_loop(struct lp_build_nir_context
*bld_base
, nir_loop
*loop
)
1841 bld_base
->bgnloop(bld_base
);
1842 visit_cf_list(bld_base
, &loop
->body
);
1843 bld_base
->endloop(bld_base
);
1846 static void visit_cf_list(struct lp_build_nir_context
*bld_base
,
1847 struct exec_list
*list
)
1849 foreach_list_typed(nir_cf_node
, node
, node
, list
)
1851 switch (node
->type
) {
1852 case nir_cf_node_block
:
1853 visit_block(bld_base
, nir_cf_node_as_block(node
));
1856 case nir_cf_node_if
:
1857 visit_if(bld_base
, nir_cf_node_as_if(node
));
1860 case nir_cf_node_loop
:
1861 visit_loop(bld_base
, nir_cf_node_as_loop(node
));
1871 handle_shader_output_decl(struct lp_build_nir_context
*bld_base
,
1872 struct nir_shader
*nir
,
1873 struct nir_variable
*variable
)
1875 bld_base
->emit_var_decl(bld_base
, variable
);
1878 /* vector registers are stored as arrays in LLVM side,
1879 so we can use GEP on them, as to do exec mask stores
1880 we need to operate on a single components.
1886 static LLVMTypeRef
get_register_type(struct lp_build_nir_context
*bld_base
,
1889 struct lp_build_context
*int_bld
= get_int_bld(bld_base
, true, reg
->bit_size
);
1891 LLVMTypeRef type
= int_bld
->vec_type
;
1892 if (reg
->num_array_elems
)
1893 type
= LLVMArrayType(type
, reg
->num_array_elems
);
1894 if (reg
->num_components
> 1)
1895 type
= LLVMArrayType(type
, reg
->num_components
);
1901 bool lp_build_nir_llvm(
1902 struct lp_build_nir_context
*bld_base
,
1903 struct nir_shader
*nir
)
1905 struct nir_function
*func
;
1907 nir_convert_from_ssa(nir
, true);
1908 nir_lower_locals_to_regs(nir
);
1909 nir_remove_dead_derefs(nir
);
1910 nir_remove_dead_variables(nir
, nir_var_function_temp
);
1912 nir_foreach_variable(variable
, &nir
->outputs
)
1913 handle_shader_output_decl(bld_base
, nir
, variable
);
1915 bld_base
->regs
= _mesa_hash_table_create(NULL
, _mesa_hash_pointer
,
1916 _mesa_key_pointer_equal
);
1917 bld_base
->vars
= _mesa_hash_table_create(NULL
, _mesa_hash_pointer
,
1918 _mesa_key_pointer_equal
);
1920 func
= (struct nir_function
*)exec_list_get_head(&nir
->functions
);
1922 nir_foreach_register(reg
, &func
->impl
->registers
) {
1923 LLVMTypeRef type
= get_register_type(bld_base
, reg
);
1924 LLVMValueRef reg_alloc
= lp_build_alloca_undef(bld_base
->base
.gallivm
,
1926 _mesa_hash_table_insert(bld_base
->regs
, reg
, reg_alloc
);
1928 nir_index_ssa_defs(func
->impl
);
1929 bld_base
->ssa_defs
= calloc(func
->impl
->ssa_alloc
, sizeof(LLVMValueRef
));
1930 visit_cf_list(bld_base
, &func
->impl
->body
);
1932 free(bld_base
->ssa_defs
);
1933 ralloc_free(bld_base
->vars
);
1934 ralloc_free(bld_base
->regs
);
1938 /* do some basic opts to remove some things we don't want to see. */
1939 void lp_build_opt_nir(struct nir_shader
*nir
)
1944 NIR_PASS_V(nir
, nir_opt_constant_folding
);
1945 NIR_PASS_V(nir
, nir_opt_algebraic
);
1946 NIR_PASS_V(nir
, nir_lower_pack
);
1948 nir_lower_tex_options options
= { .lower_tex_without_implicit_lod
= true };
1949 NIR_PASS_V(nir
, nir_lower_tex
, &options
);
1951 nir_lower_bool_to_int32(nir
);