1 /**************************************************************************
3 * Copyright 2019 Red Hat.
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the "Software"),
8 * to deal in the Software without restriction, including without limitation
9 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
10 * and/or sell copies of the Software, and to permit persons to whom the
11 * Software is furnished to do so, subject to the following conditions:
13 * The above copyright notice and this permission notice shall be included
14 * in all copies or substantial portions of the Software.
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
17 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
24 **************************************************************************/
26 #include "lp_bld_nir.h"
27 #include "lp_bld_arit.h"
28 #include "lp_bld_bitarit.h"
29 #include "lp_bld_const.h"
30 #include "lp_bld_gather.h"
31 #include "lp_bld_logic.h"
32 #include "lp_bld_quad.h"
33 #include "lp_bld_flow.h"
34 #include "lp_bld_struct.h"
35 #include "lp_bld_debug.h"
36 #include "lp_bld_printf.h"
37 #include "nir_deref.h"
39 static void visit_cf_list(struct lp_build_nir_context
*bld_base
,
40 struct exec_list
*list
);
42 static LLVMValueRef
cast_type(struct lp_build_nir_context
*bld_base
, LLVMValueRef val
,
43 nir_alu_type alu_type
, unsigned bit_size
)
45 LLVMBuilderRef builder
= bld_base
->base
.gallivm
->builder
;
50 return LLVMBuildBitCast(builder
, val
, bld_base
->base
.vec_type
, "");
52 return LLVMBuildBitCast(builder
, val
, bld_base
->dbl_bld
.vec_type
, "");
61 return LLVMBuildBitCast(builder
, val
, bld_base
->int8_bld
.vec_type
, "");
63 return LLVMBuildBitCast(builder
, val
, bld_base
->int16_bld
.vec_type
, "");
65 return LLVMBuildBitCast(builder
, val
, bld_base
->int_bld
.vec_type
, "");
67 return LLVMBuildBitCast(builder
, val
, bld_base
->int64_bld
.vec_type
, "");
76 return LLVMBuildBitCast(builder
, val
, bld_base
->uint8_bld
.vec_type
, "");
78 return LLVMBuildBitCast(builder
, val
, bld_base
->uint16_bld
.vec_type
, "");
80 return LLVMBuildBitCast(builder
, val
, bld_base
->uint_bld
.vec_type
, "");
82 return LLVMBuildBitCast(builder
, val
, bld_base
->uint64_bld
.vec_type
, "");
89 return LLVMBuildBitCast(builder
, val
, bld_base
->uint_bld
.vec_type
, "");
97 static struct lp_build_context
*get_flt_bld(struct lp_build_nir_context
*bld_base
,
100 if (op_bit_size
== 64)
101 return &bld_base
->dbl_bld
;
103 return &bld_base
->base
;
106 static unsigned glsl_sampler_to_pipe(int sampler_dim
, bool is_array
)
108 unsigned pipe_target
= PIPE_BUFFER
;
109 switch (sampler_dim
) {
110 case GLSL_SAMPLER_DIM_1D
:
111 pipe_target
= is_array
? PIPE_TEXTURE_1D_ARRAY
: PIPE_TEXTURE_1D
;
113 case GLSL_SAMPLER_DIM_2D
:
114 pipe_target
= is_array
? PIPE_TEXTURE_2D_ARRAY
: PIPE_TEXTURE_2D
;
116 case GLSL_SAMPLER_DIM_3D
:
117 pipe_target
= PIPE_TEXTURE_3D
;
119 case GLSL_SAMPLER_DIM_CUBE
:
120 pipe_target
= is_array
? PIPE_TEXTURE_CUBE_ARRAY
: PIPE_TEXTURE_CUBE
;
122 case GLSL_SAMPLER_DIM_RECT
:
123 pipe_target
= PIPE_TEXTURE_RECT
;
125 case GLSL_SAMPLER_DIM_BUF
:
126 pipe_target
= PIPE_BUFFER
;
134 static LLVMValueRef
get_ssa_src(struct lp_build_nir_context
*bld_base
, nir_ssa_def
*ssa
)
136 return bld_base
->ssa_defs
[ssa
->index
];
139 static LLVMValueRef
get_src(struct lp_build_nir_context
*bld_base
, nir_src src
);
141 static LLVMValueRef
get_reg_src(struct lp_build_nir_context
*bld_base
, nir_reg_src src
)
143 struct hash_entry
*entry
= _mesa_hash_table_search(bld_base
->regs
, src
.reg
);
144 LLVMValueRef reg_storage
= (LLVMValueRef
)entry
->data
;
145 struct lp_build_context
*reg_bld
= get_int_bld(bld_base
, true, src
.reg
->bit_size
);
146 LLVMValueRef indir_src
= NULL
;
148 indir_src
= get_src(bld_base
, *src
.indirect
);
149 return bld_base
->load_reg(bld_base
, reg_bld
, &src
, indir_src
, reg_storage
);
152 static LLVMValueRef
get_src(struct lp_build_nir_context
*bld_base
, nir_src src
)
155 return get_ssa_src(bld_base
, src
.ssa
);
157 return get_reg_src(bld_base
, src
.reg
);
160 static void assign_ssa(struct lp_build_nir_context
*bld_base
, int idx
, LLVMValueRef ptr
)
162 bld_base
->ssa_defs
[idx
] = ptr
;
165 static void assign_ssa_dest(struct lp_build_nir_context
*bld_base
, const nir_ssa_def
*ssa
,
166 LLVMValueRef vals
[NIR_MAX_VEC_COMPONENTS
])
168 assign_ssa(bld_base
, ssa
->index
, ssa
->num_components
== 1 ? vals
[0] : lp_nir_array_build_gather_values(bld_base
->base
.gallivm
->builder
, vals
, ssa
->num_components
));
171 static void assign_reg(struct lp_build_nir_context
*bld_base
, const nir_reg_dest
*reg
,
173 LLVMValueRef vals
[NIR_MAX_VEC_COMPONENTS
])
175 struct hash_entry
*entry
= _mesa_hash_table_search(bld_base
->regs
, reg
->reg
);
176 LLVMValueRef reg_storage
= (LLVMValueRef
)entry
->data
;
177 struct lp_build_context
*reg_bld
= get_int_bld(bld_base
, true, reg
->reg
->bit_size
);
178 LLVMValueRef indir_src
= NULL
;
180 indir_src
= get_src(bld_base
, *reg
->indirect
);
181 bld_base
->store_reg(bld_base
, reg_bld
, reg
, write_mask
? write_mask
: 0xf, indir_src
, reg_storage
, vals
);
184 static void assign_dest(struct lp_build_nir_context
*bld_base
, const nir_dest
*dest
, LLVMValueRef vals
[NIR_MAX_VEC_COMPONENTS
])
187 assign_ssa_dest(bld_base
, &dest
->ssa
, vals
);
189 assign_reg(bld_base
, &dest
->reg
, 0, vals
);
192 static void assign_alu_dest(struct lp_build_nir_context
*bld_base
, const nir_alu_dest
*dest
, LLVMValueRef vals
[NIR_MAX_VEC_COMPONENTS
])
194 if (dest
->dest
.is_ssa
)
195 assign_ssa_dest(bld_base
, &dest
->dest
.ssa
, vals
);
197 assign_reg(bld_base
, &dest
->dest
.reg
, dest
->write_mask
, vals
);
200 static LLVMValueRef
int_to_bool32(struct lp_build_nir_context
*bld_base
,
201 uint32_t src_bit_size
,
205 LLVMBuilderRef builder
= bld_base
->base
.gallivm
->builder
;
206 struct lp_build_context
*int_bld
= get_int_bld(bld_base
, is_unsigned
, src_bit_size
);
207 LLVMValueRef result
= lp_build_compare(bld_base
->base
.gallivm
, int_bld
->type
, PIPE_FUNC_NOTEQUAL
, val
, int_bld
->zero
);
208 if (src_bit_size
== 64)
209 result
= LLVMBuildTrunc(builder
, result
, bld_base
->int_bld
.vec_type
, "");
213 static LLVMValueRef
flt_to_bool32(struct lp_build_nir_context
*bld_base
,
214 uint32_t src_bit_size
,
217 LLVMBuilderRef builder
= bld_base
->base
.gallivm
->builder
;
218 struct lp_build_context
*flt_bld
= get_flt_bld(bld_base
, src_bit_size
);
219 LLVMValueRef result
= lp_build_cmp(flt_bld
, PIPE_FUNC_NOTEQUAL
, val
, flt_bld
->zero
);
220 if (src_bit_size
== 64)
221 result
= LLVMBuildTrunc(builder
, result
, bld_base
->int_bld
.vec_type
, "");
225 static LLVMValueRef
fcmp32(struct lp_build_nir_context
*bld_base
,
226 enum pipe_compare_func compare
,
227 uint32_t src_bit_size
,
228 LLVMValueRef src
[NIR_MAX_VEC_COMPONENTS
])
230 LLVMBuilderRef builder
= bld_base
->base
.gallivm
->builder
;
231 struct lp_build_context
*flt_bld
= get_flt_bld(bld_base
, src_bit_size
);
234 if (compare
!= PIPE_FUNC_NOTEQUAL
)
235 result
= lp_build_cmp_ordered(flt_bld
, compare
, src
[0], src
[1]);
237 result
= lp_build_cmp(flt_bld
, compare
, src
[0], src
[1]);
238 if (src_bit_size
== 64)
239 result
= LLVMBuildTrunc(builder
, result
, bld_base
->int_bld
.vec_type
, "");
243 static LLVMValueRef
icmp32(struct lp_build_nir_context
*bld_base
,
244 enum pipe_compare_func compare
,
246 uint32_t src_bit_size
,
247 LLVMValueRef src
[NIR_MAX_VEC_COMPONENTS
])
249 LLVMBuilderRef builder
= bld_base
->base
.gallivm
->builder
;
250 struct lp_build_context
*i_bld
= get_int_bld(bld_base
, is_unsigned
, src_bit_size
);
251 LLVMValueRef result
= lp_build_cmp(i_bld
, compare
, src
[0], src
[1]);
252 if (src_bit_size
< 32)
253 result
= LLVMBuildSExt(builder
, result
, bld_base
->int_bld
.vec_type
, "");
254 else if (src_bit_size
== 64)
255 result
= LLVMBuildTrunc(builder
, result
, bld_base
->int_bld
.vec_type
, "");
259 static LLVMValueRef
get_alu_src(struct lp_build_nir_context
*bld_base
,
261 unsigned num_components
)
263 LLVMBuilderRef builder
= bld_base
->base
.gallivm
->builder
;
264 struct gallivm_state
*gallivm
= bld_base
->base
.gallivm
;
265 LLVMValueRef value
= get_src(bld_base
, src
.src
);
266 bool need_swizzle
= false;
269 unsigned src_components
= nir_src_num_components(src
.src
);
270 for (unsigned i
= 0; i
< num_components
; ++i
) {
271 assert(src
.swizzle
[i
] < src_components
);
272 if (src
.swizzle
[i
] != i
)
276 if (need_swizzle
|| num_components
!= src_components
) {
277 if (src_components
> 1 && num_components
== 1) {
278 value
= LLVMBuildExtractValue(gallivm
->builder
, value
,
280 } else if (src_components
== 1 && num_components
> 1) {
281 LLVMValueRef values
[] = {value
, value
, value
, value
, value
, value
, value
, value
, value
, value
, value
, value
, value
, value
, value
, value
};
282 value
= lp_nir_array_build_gather_values(builder
, values
, num_components
);
284 LLVMValueRef arr
= LLVMGetUndef(LLVMArrayType(LLVMTypeOf(LLVMBuildExtractValue(builder
, value
, 0, "")), num_components
));
285 for (unsigned i
= 0; i
< num_components
; i
++)
286 arr
= LLVMBuildInsertValue(builder
, arr
, LLVMBuildExtractValue(builder
, value
, src
.swizzle
[i
], ""), i
, "");
295 static LLVMValueRef
emit_b2f(struct lp_build_nir_context
*bld_base
,
299 LLVMBuilderRef builder
= bld_base
->base
.gallivm
->builder
;
300 LLVMValueRef result
= LLVMBuildAnd(builder
, cast_type(bld_base
, src0
, nir_type_int
, 32),
301 LLVMBuildBitCast(builder
, lp_build_const_vec(bld_base
->base
.gallivm
, bld_base
->base
.type
,
302 1.0), bld_base
->int_bld
.vec_type
, ""),
304 result
= LLVMBuildBitCast(builder
, result
, bld_base
->base
.vec_type
, "");
309 result
= LLVMBuildFPExt(builder
, result
, bld_base
->dbl_bld
.vec_type
, "");
312 unreachable("unsupported bit size.");
317 static LLVMValueRef
emit_b2i(struct lp_build_nir_context
*bld_base
,
321 LLVMBuilderRef builder
= bld_base
->base
.gallivm
->builder
;
322 LLVMValueRef result
= LLVMBuildAnd(builder
, cast_type(bld_base
, src0
, nir_type_int
, 32),
323 lp_build_const_int_vec(bld_base
->base
.gallivm
, bld_base
->base
.type
, 1), "");
328 return LLVMBuildZExt(builder
, result
, bld_base
->int64_bld
.vec_type
, "");
330 unreachable("unsupported bit size.");
334 static LLVMValueRef
emit_b32csel(struct lp_build_nir_context
*bld_base
,
335 unsigned src_bit_size
[NIR_MAX_VEC_COMPONENTS
],
336 LLVMValueRef src
[NIR_MAX_VEC_COMPONENTS
])
338 LLVMValueRef sel
= cast_type(bld_base
, src
[0], nir_type_int
, 32);
339 LLVMValueRef v
= lp_build_compare(bld_base
->base
.gallivm
, bld_base
->int_bld
.type
, PIPE_FUNC_NOTEQUAL
, sel
, bld_base
->int_bld
.zero
);
340 struct lp_build_context
*bld
= get_int_bld(bld_base
, false, src_bit_size
[1]);
341 return lp_build_select(bld
, v
, src
[1], src
[2]);
344 static LLVMValueRef
split_64bit(struct lp_build_nir_context
*bld_base
,
348 struct gallivm_state
*gallivm
= bld_base
->base
.gallivm
;
349 LLVMValueRef shuffles
[LP_MAX_VECTOR_WIDTH
/32];
350 LLVMValueRef shuffles2
[LP_MAX_VECTOR_WIDTH
/32];
351 int len
= bld_base
->base
.type
.length
* 2;
352 for (unsigned i
= 0; i
< bld_base
->base
.type
.length
; i
++) {
353 shuffles
[i
] = lp_build_const_int32(gallivm
, i
* 2);
354 shuffles2
[i
] = lp_build_const_int32(gallivm
, (i
* 2) + 1);
357 src
= LLVMBuildBitCast(gallivm
->builder
, src
, LLVMVectorType(LLVMInt32TypeInContext(gallivm
->context
), len
), "");
358 return LLVMBuildShuffleVector(gallivm
->builder
, src
,
359 LLVMGetUndef(LLVMTypeOf(src
)),
360 LLVMConstVector(hi
? shuffles2
: shuffles
,
361 bld_base
->base
.type
.length
),
366 merge_64bit(struct lp_build_nir_context
*bld_base
,
370 struct gallivm_state
*gallivm
= bld_base
->base
.gallivm
;
371 LLVMBuilderRef builder
= gallivm
->builder
;
373 LLVMValueRef shuffles
[2 * (LP_MAX_VECTOR_WIDTH
/32)];
374 int len
= bld_base
->base
.type
.length
* 2;
375 assert(len
<= (2 * (LP_MAX_VECTOR_WIDTH
/32)));
377 for (i
= 0; i
< bld_base
->base
.type
.length
* 2; i
+=2) {
378 shuffles
[i
] = lp_build_const_int32(gallivm
, i
/ 2);
379 shuffles
[i
+ 1] = lp_build_const_int32(gallivm
, i
/ 2 + bld_base
->base
.type
.length
);
381 return LLVMBuildShuffleVector(builder
, input
, input2
, LLVMConstVector(shuffles
, len
), "");
385 do_int_divide(struct lp_build_nir_context
*bld_base
,
386 bool is_unsigned
, unsigned src_bit_size
,
387 LLVMValueRef src
, LLVMValueRef src2
)
389 struct gallivm_state
*gallivm
= bld_base
->base
.gallivm
;
390 LLVMBuilderRef builder
= gallivm
->builder
;
391 struct lp_build_context
*int_bld
= get_int_bld(bld_base
, is_unsigned
, src_bit_size
);
392 LLVMValueRef div_mask
= lp_build_cmp(int_bld
, PIPE_FUNC_EQUAL
, src2
,
394 LLVMValueRef divisor
= LLVMBuildOr(builder
,
397 LLVMValueRef result
= lp_build_div(int_bld
, src
, divisor
);
398 /* udiv by zero is guaranteed to return 0xffffffff at least with d3d10
399 * may as well do same for idiv */
400 return LLVMBuildOr(builder
, div_mask
, result
, "");
403 static LLVMValueRef
do_alu_action(struct lp_build_nir_context
*bld_base
,
404 nir_op op
, unsigned src_bit_size
[NIR_MAX_VEC_COMPONENTS
], LLVMValueRef src
[NIR_MAX_VEC_COMPONENTS
])
406 struct gallivm_state
*gallivm
= bld_base
->base
.gallivm
;
407 LLVMBuilderRef builder
= gallivm
->builder
;
411 result
= emit_b2f(bld_base
, src
[0], 32);
414 result
= emit_b2f(bld_base
, src
[0], 64);
417 result
= emit_b2i(bld_base
, src
[0], 32);
420 result
= emit_b2i(bld_base
, src
[0], 64);
423 result
= emit_b32csel(bld_base
, src_bit_size
, src
);
425 case nir_op_bit_count
:
426 result
= lp_build_popcount(get_int_bld(bld_base
, false, src_bit_size
[0]), src
[0]);
428 case nir_op_bitfield_select
:
429 result
= lp_build_xor(&bld_base
->uint_bld
, src
[2], lp_build_and(&bld_base
->uint_bld
, src
[0], lp_build_xor(&bld_base
->uint_bld
, src
[1], src
[2])));
431 case nir_op_bitfield_reverse
:
432 result
= lp_build_bitfield_reverse(get_int_bld(bld_base
, false, src_bit_size
[0]), src
[0]);
435 result
= flt_to_bool32(bld_base
, src_bit_size
[0], src
[0]);
438 result
= LLVMBuildFPTrunc(builder
, src
[0],
439 bld_base
->base
.vec_type
, "");
442 result
= LLVMBuildFPExt(builder
, src
[0],
443 bld_base
->dbl_bld
.vec_type
, "");
446 result
= LLVMBuildFPToSI(builder
, src
[0], bld_base
->base
.int_vec_type
, "");
449 result
= LLVMBuildFPToUI(builder
,
451 bld_base
->base
.int_vec_type
, "");
454 result
= LLVMBuildFPToSI(builder
,
456 bld_base
->int64_bld
.vec_type
, "");
459 result
= LLVMBuildFPToUI(builder
,
461 bld_base
->uint64_bld
.vec_type
, "");
464 result
= lp_build_abs(get_flt_bld(bld_base
, src_bit_size
[0]), src
[0]);
467 result
= lp_build_add(get_flt_bld(bld_base
, src_bit_size
[0]),
471 result
= lp_build_ceil(get_flt_bld(bld_base
, src_bit_size
[0]), src
[0]);
474 result
= lp_build_cos(&bld_base
->base
, src
[0]);
477 result
= lp_build_ddx(&bld_base
->base
, src
[0]);
480 result
= lp_build_ddy(&bld_base
->base
, src
[0]);
483 result
= lp_build_div(get_flt_bld(bld_base
, src_bit_size
[0]),
487 result
= fcmp32(bld_base
, PIPE_FUNC_EQUAL
, src_bit_size
[0], src
);
490 result
= lp_build_exp2(&bld_base
->base
, src
[0]);
493 result
= lp_build_floor(get_flt_bld(bld_base
, src_bit_size
[0]), src
[0]);
496 result
= lp_build_fmuladd(builder
, src
[0], src
[1], src
[2]);
498 case nir_op_ffract
: {
499 struct lp_build_context
*flt_bld
= get_flt_bld(bld_base
, src_bit_size
[0]);
500 LLVMValueRef tmp
= lp_build_floor(flt_bld
, src
[0]);
501 result
= lp_build_sub(flt_bld
, src
[0], tmp
);
505 result
= fcmp32(bld_base
, PIPE_FUNC_GEQUAL
, src_bit_size
[0], src
);
507 case nir_op_find_lsb
:
508 result
= lp_build_cttz(get_int_bld(bld_base
, false, src_bit_size
[0]), src
[0]);
511 result
= lp_build_log2_safe(&bld_base
->base
, src
[0]);
514 result
= fcmp32(bld_base
, PIPE_FUNC_LESS
, src_bit_size
[0], src
);
517 result
= lp_build_min(get_flt_bld(bld_base
, src_bit_size
[0]), src
[0], src
[1]);
520 struct lp_build_context
*flt_bld
= get_flt_bld(bld_base
, src_bit_size
[0]);
521 result
= lp_build_div(flt_bld
, src
[0], src
[1]);
522 result
= lp_build_floor(flt_bld
, result
);
523 result
= lp_build_mul(flt_bld
, src
[1], result
);
524 result
= lp_build_sub(flt_bld
, src
[0], result
);
528 result
= lp_build_mul(get_flt_bld(bld_base
, src_bit_size
[0]),
532 result
= lp_build_max(get_flt_bld(bld_base
, src_bit_size
[0]), src
[0], src
[1]);
535 result
= fcmp32(bld_base
, PIPE_FUNC_NOTEQUAL
, src_bit_size
[0], src
);
538 result
= lp_build_negate(get_flt_bld(bld_base
, src_bit_size
[0]), src
[0]);
541 result
= lp_build_pow(&bld_base
->base
, src
[0], src
[1]);
544 result
= lp_build_rcp(get_flt_bld(bld_base
, src_bit_size
[0]), src
[0]);
546 case nir_op_fround_even
:
547 result
= lp_build_round(get_flt_bld(bld_base
, src_bit_size
[0]), src
[0]);
550 result
= lp_build_rsqrt(get_flt_bld(bld_base
, src_bit_size
[0]), src
[0]);
553 result
= lp_build_clamp_zero_one_nanzero(get_flt_bld(bld_base
, src_bit_size
[0]), src
[0]);
556 result
= lp_build_sgn(get_flt_bld(bld_base
, src_bit_size
[0]), src
[0]);
559 result
= lp_build_sin(&bld_base
->base
, src
[0]);
562 result
= lp_build_sqrt(get_flt_bld(bld_base
, src_bit_size
[0]), src
[0]);
565 result
= lp_build_trunc(get_flt_bld(bld_base
, src_bit_size
[0]), src
[0]);
568 result
= int_to_bool32(bld_base
, src_bit_size
[0], false, src
[0]);
571 result
= lp_build_int_to_float(&bld_base
->base
, src
[0]);
574 result
= lp_build_int_to_float(&bld_base
->dbl_bld
, src
[0]);
577 result
= LLVMBuildTrunc(builder
, src
[0], bld_base
->int8_bld
.vec_type
, "");
580 if (src_bit_size
[0] < 16)
581 result
= LLVMBuildSExt(builder
, src
[0], bld_base
->int16_bld
.vec_type
, "");
583 result
= LLVMBuildTrunc(builder
, src
[0], bld_base
->int16_bld
.vec_type
, "");
586 if (src_bit_size
[0] < 32)
587 result
= LLVMBuildSExt(builder
, src
[0], bld_base
->int_bld
.vec_type
, "");
589 result
= LLVMBuildTrunc(builder
, src
[0], bld_base
->int_bld
.vec_type
, "");
592 result
= LLVMBuildSExt(builder
, src
[0], bld_base
->int64_bld
.vec_type
, "");
595 result
= lp_build_abs(get_int_bld(bld_base
, false, src_bit_size
[0]), src
[0]);
598 result
= lp_build_add(get_int_bld(bld_base
, false, src_bit_size
[0]),
602 result
= lp_build_and(get_int_bld(bld_base
, false, src_bit_size
[0]),
606 result
= do_int_divide(bld_base
, false, src_bit_size
[0], src
[0], src
[1]);
609 result
= icmp32(bld_base
, PIPE_FUNC_EQUAL
, false, src_bit_size
[0], src
);
612 result
= icmp32(bld_base
, PIPE_FUNC_GEQUAL
, false, src_bit_size
[0], src
);
615 result
= icmp32(bld_base
, PIPE_FUNC_LESS
, false, src_bit_size
[0], src
);
618 result
= lp_build_max(get_int_bld(bld_base
, false, src_bit_size
[0]), src
[0], src
[1]);
621 result
= lp_build_min(get_int_bld(bld_base
, false, src_bit_size
[0]), src
[0], src
[1]);
625 result
= lp_build_mul(get_int_bld(bld_base
, false, src_bit_size
[0]),
628 case nir_op_imul_high
: {
629 LLVMValueRef hi_bits
;
630 lp_build_mul_32_lohi(&bld_base
->int_bld
, src
[0], src
[1], &hi_bits
);
635 result
= icmp32(bld_base
, PIPE_FUNC_NOTEQUAL
, false, src_bit_size
[0], src
);
638 result
= lp_build_negate(get_int_bld(bld_base
, false, src_bit_size
[0]), src
[0]);
641 result
= lp_build_not(get_int_bld(bld_base
, false, src_bit_size
[0]), src
[0]);
644 result
= lp_build_or(get_int_bld(bld_base
, false, src_bit_size
[0]),
648 result
= lp_build_mod(get_int_bld(bld_base
, false, src_bit_size
[0]),
652 struct lp_build_context
*uint_bld
= get_int_bld(bld_base
, true, src_bit_size
[0]);
653 struct lp_build_context
*int_bld
= get_int_bld(bld_base
, false, src_bit_size
[0]);
654 if (src_bit_size
[0] == 64)
655 src
[1] = LLVMBuildZExt(builder
, src
[1], uint_bld
->vec_type
, "");
656 if (src_bit_size
[0] < 32)
657 src
[1] = LLVMBuildTrunc(builder
, src
[1], uint_bld
->vec_type
, "");
658 src
[1] = lp_build_and(uint_bld
, src
[1], lp_build_const_int_vec(gallivm
, uint_bld
->type
, (src_bit_size
[0] - 1)));
659 result
= lp_build_shl(int_bld
, src
[0], src
[1]);
663 struct lp_build_context
*uint_bld
= get_int_bld(bld_base
, true, src_bit_size
[0]);
664 struct lp_build_context
*int_bld
= get_int_bld(bld_base
, false, src_bit_size
[0]);
665 if (src_bit_size
[0] == 64)
666 src
[1] = LLVMBuildZExt(builder
, src
[1], uint_bld
->vec_type
, "");
667 if (src_bit_size
[0] < 32)
668 src
[1] = LLVMBuildTrunc(builder
, src
[1], uint_bld
->vec_type
, "");
669 src
[1] = lp_build_and(uint_bld
, src
[1], lp_build_const_int_vec(gallivm
, uint_bld
->type
, (src_bit_size
[0] - 1)));
670 result
= lp_build_shr(int_bld
, src
[0], src
[1]);
674 result
= lp_build_sgn(get_int_bld(bld_base
, false, src_bit_size
[0]), src
[0]);
677 result
= lp_build_sub(get_int_bld(bld_base
, false, src_bit_size
[0]),
681 result
= lp_build_xor(get_int_bld(bld_base
, false, src_bit_size
[0]),
687 case nir_op_unpack_64_2x32_split_x
:
688 result
= split_64bit(bld_base
, src
[0], false);
690 case nir_op_unpack_64_2x32_split_y
:
691 result
= split_64bit(bld_base
, src
[0], true);
694 case nir_op_pack_64_2x32_split
: {
695 LLVMValueRef tmp
= merge_64bit(bld_base
, src
[0], src
[1]);
696 result
= LLVMBuildBitCast(builder
, tmp
, bld_base
->dbl_bld
.vec_type
, "");
700 result
= LLVMBuildUIToFP(builder
, src
[0], bld_base
->base
.vec_type
, "");
703 result
= LLVMBuildUIToFP(builder
, src
[0], bld_base
->dbl_bld
.vec_type
, "");
706 result
= LLVMBuildTrunc(builder
, src
[0], bld_base
->uint8_bld
.vec_type
, "");
709 if (src_bit_size
[0] < 16)
710 result
= LLVMBuildZExt(builder
, src
[0], bld_base
->uint16_bld
.vec_type
, "");
712 result
= LLVMBuildTrunc(builder
, src
[0], bld_base
->uint16_bld
.vec_type
, "");
715 if (src_bit_size
[0] < 32)
716 result
= LLVMBuildZExt(builder
, src
[0], bld_base
->uint_bld
.vec_type
, "");
718 result
= LLVMBuildTrunc(builder
, src
[0], bld_base
->uint_bld
.vec_type
, "");
721 result
= LLVMBuildZExt(builder
, src
[0], bld_base
->uint64_bld
.vec_type
, "");
724 result
= do_int_divide(bld_base
, true, src_bit_size
[0], src
[0], src
[1]);
726 case nir_op_ufind_msb
: {
727 struct lp_build_context
*uint_bld
= get_int_bld(bld_base
, true, src_bit_size
[0]);
728 result
= lp_build_ctlz(uint_bld
, src
[0]);
729 result
= lp_build_sub(uint_bld
, lp_build_const_int_vec(gallivm
, uint_bld
->type
, src_bit_size
[0] - 1), result
);
733 result
= icmp32(bld_base
, PIPE_FUNC_GEQUAL
, true, src_bit_size
[0], src
);
736 result
= icmp32(bld_base
, PIPE_FUNC_LESS
, true, src_bit_size
[0], src
);
739 result
= lp_build_max(get_int_bld(bld_base
, true, src_bit_size
[0]), src
[0], src
[1]);
742 result
= lp_build_min(get_int_bld(bld_base
, true, src_bit_size
[0]), src
[0], src
[1]);
745 result
= lp_build_mod(get_int_bld(bld_base
, true, src_bit_size
[0]), src
[0], src
[1]);
747 case nir_op_umul_high
: {
748 LLVMValueRef hi_bits
;
749 lp_build_mul_32_lohi(&bld_base
->uint_bld
, src
[0], src
[1], &hi_bits
);
754 struct lp_build_context
*uint_bld
= get_int_bld(bld_base
, true, src_bit_size
[0]);
755 if (src_bit_size
[0] == 64)
756 src
[1] = LLVMBuildZExt(builder
, src
[1], uint_bld
->vec_type
, "");
757 if (src_bit_size
[0] < 32)
758 src
[1] = LLVMBuildTrunc(builder
, src
[1], uint_bld
->vec_type
, "");
759 src
[1] = lp_build_and(uint_bld
, src
[1], lp_build_const_int_vec(gallivm
, uint_bld
->type
, (src_bit_size
[0] - 1)));
760 result
= lp_build_shr(uint_bld
, src
[0], src
[1]);
770 static void visit_alu(struct lp_build_nir_context
*bld_base
, const nir_alu_instr
*instr
)
772 struct gallivm_state
*gallivm
= bld_base
->base
.gallivm
;
773 LLVMValueRef src
[NIR_MAX_VEC_COMPONENTS
];
774 unsigned src_bit_size
[NIR_MAX_VEC_COMPONENTS
];
775 unsigned num_components
= nir_dest_num_components(instr
->dest
.dest
);
776 unsigned src_components
;
785 case nir_op_pack_half_2x16
:
788 case nir_op_unpack_half_2x16
:
791 case nir_op_cube_face_coord
:
792 case nir_op_cube_face_index
:
796 src_components
= num_components
;
799 for (unsigned i
= 0; i
< nir_op_infos
[instr
->op
].num_inputs
; i
++) {
800 src
[i
] = get_alu_src(bld_base
, instr
->src
[i
], src_components
);
801 src_bit_size
[i
] = nir_src_bit_size(instr
->src
[i
].src
);
804 LLVMValueRef result
[NIR_MAX_VEC_COMPONENTS
];
805 if (instr
->op
== nir_op_vec4
|| instr
->op
== nir_op_vec3
|| instr
->op
== nir_op_vec2
|| instr
->op
== nir_op_vec8
|| instr
->op
== nir_op_vec16
) {
806 for (unsigned i
= 0; i
< nir_op_infos
[instr
->op
].num_inputs
; i
++) {
807 result
[i
] = cast_type(bld_base
, src
[i
], nir_op_infos
[instr
->op
].input_types
[i
], src_bit_size
[i
]);
810 for (unsigned c
= 0; c
< num_components
; c
++) {
811 LLVMValueRef src_chan
[NIR_MAX_VEC_COMPONENTS
];
813 for (unsigned i
= 0; i
< nir_op_infos
[instr
->op
].num_inputs
; i
++) {
814 if (num_components
> 1) {
815 src_chan
[i
] = LLVMBuildExtractValue(gallivm
->builder
,
818 src_chan
[i
] = src
[i
];
819 src_chan
[i
] = cast_type(bld_base
, src_chan
[i
], nir_op_infos
[instr
->op
].input_types
[i
], src_bit_size
[i
]);
821 result
[c
] = do_alu_action(bld_base
, instr
->op
, src_bit_size
, src_chan
);
822 result
[c
] = cast_type(bld_base
, result
[c
], nir_op_infos
[instr
->op
].output_type
, nir_dest_bit_size(instr
->dest
.dest
));
825 assign_alu_dest(bld_base
, &instr
->dest
, result
);
828 static void visit_load_const(struct lp_build_nir_context
*bld_base
,
829 const nir_load_const_instr
*instr
)
831 LLVMValueRef result
[NIR_MAX_VEC_COMPONENTS
];
832 struct lp_build_context
*int_bld
= get_int_bld(bld_base
, true, instr
->def
.bit_size
);
833 for (unsigned i
= 0; i
< instr
->def
.num_components
; i
++)
834 result
[i
] = lp_build_const_int_vec(bld_base
->base
.gallivm
, int_bld
->type
, instr
->value
[i
].u64
);
835 assign_ssa_dest(bld_base
, &instr
->def
, result
);
839 get_deref_offset(struct lp_build_nir_context
*bld_base
, nir_deref_instr
*instr
,
840 bool vs_in
, unsigned *vertex_index_out
,
841 LLVMValueRef
*vertex_index_ref
,
842 unsigned *const_out
, LLVMValueRef
*indir_out
)
844 LLVMBuilderRef builder
= bld_base
->base
.gallivm
->builder
;
845 nir_variable
*var
= nir_deref_instr_get_variable(instr
);
847 unsigned idx_lvl
= 1;
849 nir_deref_path_init(&path
, instr
, NULL
);
851 if (vertex_index_out
!= NULL
|| vertex_index_ref
!= NULL
) {
852 if (vertex_index_ref
) {
853 *vertex_index_ref
= get_src(bld_base
, path
.path
[idx_lvl
]->arr
.index
);
854 if (vertex_index_out
)
855 *vertex_index_out
= 0;
857 *vertex_index_out
= nir_src_as_uint(path
.path
[idx_lvl
]->arr
.index
);
862 uint32_t const_offset
= 0;
863 LLVMValueRef offset
= NULL
;
865 if (var
->data
.compact
) {
866 assert(instr
->deref_type
== nir_deref_type_array
);
867 const_offset
= nir_src_as_uint(instr
->arr
.index
);
871 for (; path
.path
[idx_lvl
]; ++idx_lvl
) {
872 const struct glsl_type
*parent_type
= path
.path
[idx_lvl
- 1]->type
;
873 if (path
.path
[idx_lvl
]->deref_type
== nir_deref_type_struct
) {
874 unsigned index
= path
.path
[idx_lvl
]->strct
.index
;
876 for (unsigned i
= 0; i
< index
; i
++) {
877 const struct glsl_type
*ft
= glsl_get_struct_field(parent_type
, i
);
878 const_offset
+= glsl_count_attribute_slots(ft
, vs_in
);
880 } else if(path
.path
[idx_lvl
]->deref_type
== nir_deref_type_array
) {
881 unsigned size
= glsl_count_attribute_slots(path
.path
[idx_lvl
]->type
, vs_in
);
882 if (nir_src_is_const(path
.path
[idx_lvl
]->arr
.index
)) {
883 const_offset
+= nir_src_comp_as_int(path
.path
[idx_lvl
]->arr
.index
, 0) * size
;
885 LLVMValueRef idx_src
= get_src(bld_base
, path
.path
[idx_lvl
]->arr
.index
);
886 idx_src
= cast_type(bld_base
, idx_src
, nir_type_uint
, 32);
887 LLVMValueRef array_off
= lp_build_mul(&bld_base
->uint_bld
, lp_build_const_int_vec(bld_base
->base
.gallivm
, bld_base
->base
.type
, size
),
890 offset
= lp_build_add(&bld_base
->uint_bld
, offset
, array_off
);
895 unreachable("Uhandled deref type in get_deref_instr_offset");
899 nir_deref_path_finish(&path
);
901 if (const_offset
&& offset
)
902 offset
= LLVMBuildAdd(builder
, offset
,
903 lp_build_const_int_vec(bld_base
->base
.gallivm
, bld_base
->uint_bld
.type
, const_offset
),
905 *const_out
= const_offset
;
909 static void visit_load_var(struct lp_build_nir_context
*bld_base
,
910 nir_intrinsic_instr
*instr
,
911 LLVMValueRef result
[NIR_MAX_VEC_COMPONENTS
])
913 nir_deref_instr
*deref
= nir_instr_as_deref(instr
->src
[0].ssa
->parent_instr
);
914 nir_variable
*var
= nir_deref_instr_get_variable(deref
);
915 nir_variable_mode mode
= deref
->mode
;
916 unsigned const_index
;
917 LLVMValueRef indir_index
;
918 unsigned vertex_index
= 0;
919 unsigned nc
= nir_dest_num_components(instr
->dest
);
920 unsigned bit_size
= nir_dest_bit_size(instr
->dest
);
922 bool vs_in
= bld_base
->shader
->info
.stage
== MESA_SHADER_VERTEX
&&
923 var
->data
.mode
== nir_var_shader_in
;
924 bool gs_in
= bld_base
->shader
->info
.stage
== MESA_SHADER_GEOMETRY
&&
925 var
->data
.mode
== nir_var_shader_in
;
926 mode
= var
->data
.mode
;
928 get_deref_offset(bld_base
, deref
, vs_in
, gs_in
? &vertex_index
: NULL
, NULL
,
929 &const_index
, &indir_index
);
931 bld_base
->load_var(bld_base
, mode
, nc
, bit_size
, var
, vertex_index
, const_index
, indir_index
, result
);
935 visit_store_var(struct lp_build_nir_context
*bld_base
,
936 nir_intrinsic_instr
*instr
)
938 nir_deref_instr
*deref
= nir_instr_as_deref(instr
->src
[0].ssa
->parent_instr
);
939 nir_variable
*var
= nir_deref_instr_get_variable(deref
);
940 nir_variable_mode mode
= deref
->mode
;
941 int writemask
= instr
->const_index
[0];
942 unsigned bit_size
= nir_src_bit_size(instr
->src
[1]);
943 LLVMValueRef src
= get_src(bld_base
, instr
->src
[1]);
944 unsigned const_index
= 0;
945 LLVMValueRef indir_index
;
947 get_deref_offset(bld_base
, deref
, false, NULL
, NULL
,
948 &const_index
, &indir_index
);
949 bld_base
->store_var(bld_base
, mode
, bit_size
, instr
->num_components
, writemask
, const_index
, var
, src
);
952 static void visit_load_ubo(struct lp_build_nir_context
*bld_base
,
953 nir_intrinsic_instr
*instr
,
954 LLVMValueRef result
[NIR_MAX_VEC_COMPONENTS
])
956 struct gallivm_state
*gallivm
= bld_base
->base
.gallivm
;
957 LLVMBuilderRef builder
= gallivm
->builder
;
958 LLVMValueRef idx
= get_src(bld_base
, instr
->src
[0]);
959 LLVMValueRef offset
= get_src(bld_base
, instr
->src
[1]);
961 bool offset_is_uniform
= nir_src_is_dynamically_uniform(instr
->src
[1]);
962 idx
= LLVMBuildExtractElement(builder
, idx
, lp_build_const_int32(gallivm
, 0), "");
963 bld_base
->load_ubo(bld_base
, nir_dest_num_components(instr
->dest
), nir_dest_bit_size(instr
->dest
),
964 offset_is_uniform
, idx
, offset
, result
);
968 static void visit_load_ssbo(struct lp_build_nir_context
*bld_base
,
969 nir_intrinsic_instr
*instr
,
970 LLVMValueRef result
[NIR_MAX_VEC_COMPONENTS
])
972 LLVMValueRef idx
= get_src(bld_base
, instr
->src
[0]);
973 LLVMValueRef offset
= get_src(bld_base
, instr
->src
[1]);
974 bld_base
->load_mem(bld_base
, nir_dest_num_components(instr
->dest
), nir_dest_bit_size(instr
->dest
),
975 idx
, offset
, result
);
978 static void visit_store_ssbo(struct lp_build_nir_context
*bld_base
,
979 nir_intrinsic_instr
*instr
)
981 LLVMValueRef val
= get_src(bld_base
, instr
->src
[0]);
982 LLVMValueRef idx
= get_src(bld_base
, instr
->src
[1]);
983 LLVMValueRef offset
= get_src(bld_base
, instr
->src
[2]);
984 int writemask
= instr
->const_index
[0];
985 int nc
= nir_src_num_components(instr
->src
[0]);
986 int bitsize
= nir_src_bit_size(instr
->src
[0]);
987 bld_base
->store_mem(bld_base
, writemask
, nc
, bitsize
, idx
, offset
, val
);
990 static void visit_get_buffer_size(struct lp_build_nir_context
*bld_base
,
991 nir_intrinsic_instr
*instr
,
992 LLVMValueRef result
[NIR_MAX_VEC_COMPONENTS
])
994 LLVMValueRef idx
= get_src(bld_base
, instr
->src
[0]);
995 result
[0] = bld_base
->get_buffer_size(bld_base
, idx
);
998 static void visit_ssbo_atomic(struct lp_build_nir_context
*bld_base
,
999 nir_intrinsic_instr
*instr
,
1000 LLVMValueRef result
[NIR_MAX_VEC_COMPONENTS
])
1002 LLVMValueRef idx
= get_src(bld_base
, instr
->src
[0]);
1003 LLVMValueRef offset
= get_src(bld_base
, instr
->src
[1]);
1004 LLVMValueRef val
= get_src(bld_base
, instr
->src
[2]);
1005 LLVMValueRef val2
= NULL
;
1006 if (instr
->intrinsic
== nir_intrinsic_ssbo_atomic_comp_swap
)
1007 val2
= get_src(bld_base
, instr
->src
[3]);
1009 bld_base
->atomic_mem(bld_base
, instr
->intrinsic
, idx
, offset
, val
, val2
, &result
[0]);
1013 static void visit_load_image(struct lp_build_nir_context
*bld_base
,
1014 nir_intrinsic_instr
*instr
,
1015 LLVMValueRef result
[NIR_MAX_VEC_COMPONENTS
])
1017 struct gallivm_state
*gallivm
= bld_base
->base
.gallivm
;
1018 LLVMBuilderRef builder
= gallivm
->builder
;
1019 nir_deref_instr
*deref
= nir_instr_as_deref(instr
->src
[0].ssa
->parent_instr
);
1020 nir_variable
*var
= nir_deref_instr_get_variable(deref
);
1021 LLVMValueRef coord_val
= get_src(bld_base
, instr
->src
[1]);
1022 LLVMValueRef coords
[5];
1023 struct lp_img_params params
;
1024 const struct glsl_type
*type
= glsl_without_array(var
->type
);
1026 memset(¶ms
, 0, sizeof(params
));
1027 params
.target
= glsl_sampler_to_pipe(glsl_get_sampler_dim(type
), glsl_sampler_type_is_array(type
));
1028 for (unsigned i
= 0; i
< 4; i
++)
1029 coords
[i
] = LLVMBuildExtractValue(builder
, coord_val
, i
, "");
1030 if (params
.target
== PIPE_TEXTURE_1D_ARRAY
)
1031 coords
[2] = coords
[1];
1033 params
.coords
= coords
;
1034 params
.outdata
= result
;
1035 params
.img_op
= LP_IMG_LOAD
;
1036 params
.image_index
= var
->data
.binding
;
1037 bld_base
->image_op(bld_base
, ¶ms
);
1040 static void visit_store_image(struct lp_build_nir_context
*bld_base
,
1041 nir_intrinsic_instr
*instr
)
1043 struct gallivm_state
*gallivm
= bld_base
->base
.gallivm
;
1044 LLVMBuilderRef builder
= gallivm
->builder
;
1045 nir_deref_instr
*deref
= nir_instr_as_deref(instr
->src
[0].ssa
->parent_instr
);
1046 nir_variable
*var
= nir_deref_instr_get_variable(deref
);
1047 LLVMValueRef coord_val
= get_src(bld_base
, instr
->src
[1]);
1048 LLVMValueRef in_val
= get_src(bld_base
, instr
->src
[3]);
1049 LLVMValueRef coords
[5];
1050 struct lp_img_params params
;
1051 const struct glsl_type
*type
= glsl_without_array(var
->type
);
1053 memset(¶ms
, 0, sizeof(params
));
1054 params
.target
= glsl_sampler_to_pipe(glsl_get_sampler_dim(type
), glsl_sampler_type_is_array(type
));
1055 for (unsigned i
= 0; i
< 4; i
++)
1056 coords
[i
] = LLVMBuildExtractValue(builder
, coord_val
, i
, "");
1057 if (params
.target
== PIPE_TEXTURE_1D_ARRAY
)
1058 coords
[2] = coords
[1];
1059 params
.coords
= coords
;
1061 for (unsigned i
= 0; i
< 4; i
++) {
1062 params
.indata
[i
] = LLVMBuildExtractValue(builder
, in_val
, i
, "");
1063 params
.indata
[i
] = LLVMBuildBitCast(builder
, params
.indata
[i
], bld_base
->base
.vec_type
, "");
1065 params
.img_op
= LP_IMG_STORE
;
1066 params
.image_index
= var
->data
.binding
;
1068 if (params
.target
== PIPE_TEXTURE_1D_ARRAY
)
1069 coords
[2] = coords
[1];
1070 bld_base
->image_op(bld_base
, ¶ms
);
1073 static void visit_atomic_image(struct lp_build_nir_context
*bld_base
,
1074 nir_intrinsic_instr
*instr
,
1075 LLVMValueRef result
[NIR_MAX_VEC_COMPONENTS
])
1077 struct gallivm_state
*gallivm
= bld_base
->base
.gallivm
;
1078 LLVMBuilderRef builder
= gallivm
->builder
;
1079 nir_deref_instr
*deref
= nir_instr_as_deref(instr
->src
[0].ssa
->parent_instr
);
1080 nir_variable
*var
= nir_deref_instr_get_variable(deref
);
1081 struct lp_img_params params
;
1082 LLVMValueRef coord_val
= get_src(bld_base
, instr
->src
[1]);
1083 LLVMValueRef in_val
= get_src(bld_base
, instr
->src
[3]);
1084 LLVMValueRef coords
[5];
1085 const struct glsl_type
*type
= glsl_without_array(var
->type
);
1087 memset(¶ms
, 0, sizeof(params
));
1089 switch (instr
->intrinsic
) {
1090 case nir_intrinsic_image_deref_atomic_add
:
1091 params
.op
= LLVMAtomicRMWBinOpAdd
;
1093 case nir_intrinsic_image_deref_atomic_exchange
:
1094 params
.op
= LLVMAtomicRMWBinOpXchg
;
1096 case nir_intrinsic_image_deref_atomic_and
:
1097 params
.op
= LLVMAtomicRMWBinOpAnd
;
1099 case nir_intrinsic_image_deref_atomic_or
:
1100 params
.op
= LLVMAtomicRMWBinOpOr
;
1102 case nir_intrinsic_image_deref_atomic_xor
:
1103 params
.op
= LLVMAtomicRMWBinOpXor
;
1105 case nir_intrinsic_image_deref_atomic_umin
:
1106 params
.op
= LLVMAtomicRMWBinOpUMin
;
1108 case nir_intrinsic_image_deref_atomic_umax
:
1109 params
.op
= LLVMAtomicRMWBinOpUMax
;
1111 case nir_intrinsic_image_deref_atomic_imin
:
1112 params
.op
= LLVMAtomicRMWBinOpMin
;
1114 case nir_intrinsic_image_deref_atomic_imax
:
1115 params
.op
= LLVMAtomicRMWBinOpMax
;
1121 params
.target
= glsl_sampler_to_pipe(glsl_get_sampler_dim(type
), glsl_sampler_type_is_array(type
));
1122 for (unsigned i
= 0; i
< 4; i
++)
1123 coords
[i
] = LLVMBuildExtractValue(builder
, coord_val
, i
, "");
1124 if (params
.target
== PIPE_TEXTURE_1D_ARRAY
)
1125 coords
[2] = coords
[1];
1126 params
.coords
= coords
;
1127 if (instr
->intrinsic
== nir_intrinsic_image_deref_atomic_comp_swap
) {
1128 LLVMValueRef cas_val
= get_src(bld_base
, instr
->src
[4]);
1129 params
.indata
[0] = in_val
;
1130 params
.indata2
[0] = cas_val
;
1132 params
.indata
[0] = in_val
;
1134 params
.outdata
= result
;
1135 params
.img_op
= (instr
->intrinsic
== nir_intrinsic_image_deref_atomic_comp_swap
) ? LP_IMG_ATOMIC_CAS
: LP_IMG_ATOMIC
;
1136 params
.image_index
= var
->data
.binding
;
1138 bld_base
->image_op(bld_base
, ¶ms
);
1142 static void visit_image_size(struct lp_build_nir_context
*bld_base
,
1143 nir_intrinsic_instr
*instr
,
1144 LLVMValueRef result
[NIR_MAX_VEC_COMPONENTS
])
1146 nir_deref_instr
*deref
= nir_instr_as_deref(instr
->src
[0].ssa
->parent_instr
);
1147 nir_variable
*var
= nir_deref_instr_get_variable(deref
);
1148 struct lp_sampler_size_query_params params
= { 0 };
1149 params
.texture_unit
= var
->data
.binding
;
1150 params
.target
= glsl_sampler_to_pipe(glsl_get_sampler_dim(var
->type
), glsl_sampler_type_is_array(var
->type
));
1151 params
.sizes_out
= result
;
1153 bld_base
->image_size(bld_base
, ¶ms
);
1156 static void visit_shared_load(struct lp_build_nir_context
*bld_base
,
1157 nir_intrinsic_instr
*instr
,
1158 LLVMValueRef result
[NIR_MAX_VEC_COMPONENTS
])
1160 LLVMValueRef offset
= get_src(bld_base
, instr
->src
[0]);
1161 bld_base
->load_mem(bld_base
, nir_dest_num_components(instr
->dest
), nir_dest_bit_size(instr
->dest
),
1162 NULL
, offset
, result
);
1165 static void visit_shared_store(struct lp_build_nir_context
*bld_base
,
1166 nir_intrinsic_instr
*instr
)
1168 LLVMValueRef val
= get_src(bld_base
, instr
->src
[0]);
1169 LLVMValueRef offset
= get_src(bld_base
, instr
->src
[1]);
1170 int writemask
= instr
->const_index
[1];
1171 int nc
= nir_src_num_components(instr
->src
[0]);
1172 int bitsize
= nir_src_bit_size(instr
->src
[0]);
1173 bld_base
->store_mem(bld_base
, writemask
, nc
, bitsize
, NULL
, offset
, val
);
1176 static void visit_shared_atomic(struct lp_build_nir_context
*bld_base
,
1177 nir_intrinsic_instr
*instr
,
1178 LLVMValueRef result
[NIR_MAX_VEC_COMPONENTS
])
1180 LLVMValueRef offset
= get_src(bld_base
, instr
->src
[0]);
1181 LLVMValueRef val
= get_src(bld_base
, instr
->src
[1]);
1182 LLVMValueRef val2
= NULL
;
1183 if (instr
->intrinsic
== nir_intrinsic_shared_atomic_comp_swap
)
1184 val2
= get_src(bld_base
, instr
->src
[2]);
1186 bld_base
->atomic_mem(bld_base
, instr
->intrinsic
, NULL
, offset
, val
, val2
, &result
[0]);
1190 static void visit_barrier(struct lp_build_nir_context
*bld_base
)
1192 bld_base
->barrier(bld_base
);
1195 static void visit_discard(struct lp_build_nir_context
*bld_base
,
1196 nir_intrinsic_instr
*instr
)
1198 LLVMValueRef cond
= NULL
;
1199 if (instr
->intrinsic
== nir_intrinsic_discard_if
) {
1200 cond
= get_src(bld_base
, instr
->src
[0]);
1201 cond
= cast_type(bld_base
, cond
, nir_type_int
, 32);
1203 bld_base
->discard(bld_base
, cond
);
1206 static void visit_load_kernel_input(struct lp_build_nir_context
*bld_base
,
1207 nir_intrinsic_instr
*instr
, LLVMValueRef result
[NIR_MAX_VEC_COMPONENTS
])
1209 LLVMValueRef offset
= get_src(bld_base
, instr
->src
[0]);
1211 bool offset_is_uniform
= nir_src_is_dynamically_uniform(instr
->src
[0]);
1212 bld_base
->load_kernel_arg(bld_base
, nir_dest_num_components(instr
->dest
), nir_dest_bit_size(instr
->dest
),
1213 nir_src_bit_size(instr
->src
[0]),
1214 offset_is_uniform
, offset
, result
);
1217 static void visit_load_global(struct lp_build_nir_context
*bld_base
,
1218 nir_intrinsic_instr
*instr
, LLVMValueRef result
[NIR_MAX_VEC_COMPONENTS
])
1220 LLVMValueRef addr
= get_src(bld_base
, instr
->src
[0]);
1221 bld_base
->load_global(bld_base
, nir_dest_num_components(instr
->dest
), nir_dest_bit_size(instr
->dest
),
1222 nir_src_bit_size(instr
->src
[0]),
1226 static void visit_store_global(struct lp_build_nir_context
*bld_base
,
1227 nir_intrinsic_instr
*instr
)
1229 LLVMValueRef val
= get_src(bld_base
, instr
->src
[0]);
1230 int nc
= nir_src_num_components(instr
->src
[0]);
1231 int bitsize
= nir_src_bit_size(instr
->src
[0]);
1232 LLVMValueRef addr
= get_src(bld_base
, instr
->src
[1]);
1233 int addr_bitsize
= nir_src_bit_size(instr
->src
[1]);
1234 int writemask
= instr
->const_index
[0];
1235 bld_base
->store_global(bld_base
, writemask
, nc
, bitsize
, addr_bitsize
, addr
, val
);
1238 static void visit_global_atomic(struct lp_build_nir_context
*bld_base
,
1239 nir_intrinsic_instr
*instr
,
1240 LLVMValueRef result
[NIR_MAX_VEC_COMPONENTS
])
1242 LLVMValueRef addr
= get_src(bld_base
, instr
->src
[0]);
1243 LLVMValueRef val
= get_src(bld_base
, instr
->src
[1]);
1244 LLVMValueRef val2
= NULL
;
1245 int addr_bitsize
= nir_src_bit_size(instr
->src
[0]);
1246 if (instr
->intrinsic
== nir_intrinsic_global_atomic_comp_swap
)
1247 val2
= get_src(bld_base
, instr
->src
[2]);
1249 bld_base
->atomic_global(bld_base
, instr
->intrinsic
, addr_bitsize
, addr
, val
, val2
, &result
[0]);
1252 static void visit_intrinsic(struct lp_build_nir_context
*bld_base
,
1253 nir_intrinsic_instr
*instr
)
1255 LLVMValueRef result
[NIR_MAX_VEC_COMPONENTS
] = {0};
1256 switch (instr
->intrinsic
) {
1257 case nir_intrinsic_load_deref
:
1258 visit_load_var(bld_base
, instr
, result
);
1260 case nir_intrinsic_store_deref
:
1261 visit_store_var(bld_base
, instr
);
1263 case nir_intrinsic_load_ubo
:
1264 visit_load_ubo(bld_base
, instr
, result
);
1266 case nir_intrinsic_load_ssbo
:
1267 visit_load_ssbo(bld_base
, instr
, result
);
1269 case nir_intrinsic_store_ssbo
:
1270 visit_store_ssbo(bld_base
, instr
);
1272 case nir_intrinsic_get_buffer_size
:
1273 visit_get_buffer_size(bld_base
, instr
, result
);
1275 case nir_intrinsic_load_vertex_id
:
1276 case nir_intrinsic_load_primitive_id
:
1277 case nir_intrinsic_load_instance_id
:
1278 case nir_intrinsic_load_base_instance
:
1279 case nir_intrinsic_load_base_vertex
:
1280 case nir_intrinsic_load_work_group_id
:
1281 case nir_intrinsic_load_local_invocation_id
:
1282 case nir_intrinsic_load_num_work_groups
:
1283 case nir_intrinsic_load_invocation_id
:
1284 case nir_intrinsic_load_front_face
:
1285 case nir_intrinsic_load_draw_id
:
1286 case nir_intrinsic_load_local_group_size
:
1287 case nir_intrinsic_load_work_dim
:
1288 bld_base
->sysval_intrin(bld_base
, instr
, result
);
1290 case nir_intrinsic_discard_if
:
1291 case nir_intrinsic_discard
:
1292 visit_discard(bld_base
, instr
);
1294 case nir_intrinsic_emit_vertex
:
1295 bld_base
->emit_vertex(bld_base
, nir_intrinsic_stream_id(instr
));
1297 case nir_intrinsic_end_primitive
:
1298 bld_base
->end_primitive(bld_base
, nir_intrinsic_stream_id(instr
));
1300 case nir_intrinsic_ssbo_atomic_add
:
1301 case nir_intrinsic_ssbo_atomic_imin
:
1302 case nir_intrinsic_ssbo_atomic_imax
:
1303 case nir_intrinsic_ssbo_atomic_umin
:
1304 case nir_intrinsic_ssbo_atomic_umax
:
1305 case nir_intrinsic_ssbo_atomic_and
:
1306 case nir_intrinsic_ssbo_atomic_or
:
1307 case nir_intrinsic_ssbo_atomic_xor
:
1308 case nir_intrinsic_ssbo_atomic_exchange
:
1309 case nir_intrinsic_ssbo_atomic_comp_swap
:
1310 visit_ssbo_atomic(bld_base
, instr
, result
);
1312 case nir_intrinsic_image_deref_load
:
1313 visit_load_image(bld_base
, instr
, result
);
1315 case nir_intrinsic_image_deref_store
:
1316 visit_store_image(bld_base
, instr
);
1318 case nir_intrinsic_image_deref_atomic_add
:
1319 case nir_intrinsic_image_deref_atomic_imin
:
1320 case nir_intrinsic_image_deref_atomic_imax
:
1321 case nir_intrinsic_image_deref_atomic_umin
:
1322 case nir_intrinsic_image_deref_atomic_umax
:
1323 case nir_intrinsic_image_deref_atomic_and
:
1324 case nir_intrinsic_image_deref_atomic_or
:
1325 case nir_intrinsic_image_deref_atomic_xor
:
1326 case nir_intrinsic_image_deref_atomic_exchange
:
1327 case nir_intrinsic_image_deref_atomic_comp_swap
:
1328 visit_atomic_image(bld_base
, instr
, result
);
1330 case nir_intrinsic_image_deref_size
:
1331 visit_image_size(bld_base
, instr
, result
);
1333 case nir_intrinsic_load_shared
:
1334 visit_shared_load(bld_base
, instr
, result
);
1336 case nir_intrinsic_store_shared
:
1337 visit_shared_store(bld_base
, instr
);
1339 case nir_intrinsic_shared_atomic_add
:
1340 case nir_intrinsic_shared_atomic_imin
:
1341 case nir_intrinsic_shared_atomic_umin
:
1342 case nir_intrinsic_shared_atomic_imax
:
1343 case nir_intrinsic_shared_atomic_umax
:
1344 case nir_intrinsic_shared_atomic_and
:
1345 case nir_intrinsic_shared_atomic_or
:
1346 case nir_intrinsic_shared_atomic_xor
:
1347 case nir_intrinsic_shared_atomic_exchange
:
1348 case nir_intrinsic_shared_atomic_comp_swap
:
1349 visit_shared_atomic(bld_base
, instr
, result
);
1351 case nir_intrinsic_barrier
:
1352 visit_barrier(bld_base
);
1354 case nir_intrinsic_memory_barrier
:
1356 case nir_intrinsic_load_kernel_input
:
1357 visit_load_kernel_input(bld_base
, instr
, result
);
1359 case nir_intrinsic_load_global
:
1360 visit_load_global(bld_base
, instr
, result
);
1362 case nir_intrinsic_store_global
:
1363 visit_store_global(bld_base
, instr
);
1365 case nir_intrinsic_global_atomic_add
:
1366 case nir_intrinsic_global_atomic_imin
:
1367 case nir_intrinsic_global_atomic_umin
:
1368 case nir_intrinsic_global_atomic_imax
:
1369 case nir_intrinsic_global_atomic_umax
:
1370 case nir_intrinsic_global_atomic_and
:
1371 case nir_intrinsic_global_atomic_or
:
1372 case nir_intrinsic_global_atomic_xor
:
1373 case nir_intrinsic_global_atomic_exchange
:
1374 case nir_intrinsic_global_atomic_comp_swap
:
1375 visit_global_atomic(bld_base
, instr
, result
);
1376 case nir_intrinsic_vote_all
:
1377 case nir_intrinsic_vote_any
:
1378 case nir_intrinsic_vote_ieq
:
1379 bld_base
->vote(bld_base
, cast_type(bld_base
, get_src(bld_base
, instr
->src
[0]), nir_type_int
, 32), instr
, result
);
1386 assign_dest(bld_base
, &instr
->dest
, result
);
1390 static void visit_txs(struct lp_build_nir_context
*bld_base
, nir_tex_instr
*instr
)
1392 struct lp_sampler_size_query_params params
;
1393 LLVMValueRef sizes_out
[NIR_MAX_VEC_COMPONENTS
];
1394 LLVMValueRef explicit_lod
= NULL
;
1396 for (unsigned i
= 0; i
< instr
->num_srcs
; i
++) {
1397 switch (instr
->src
[i
].src_type
) {
1398 case nir_tex_src_lod
:
1399 explicit_lod
= cast_type(bld_base
, get_src(bld_base
, instr
->src
[i
].src
), nir_type_int
, 32);
1406 params
.target
= glsl_sampler_to_pipe(instr
->sampler_dim
, instr
->is_array
);
1407 params
.texture_unit
= instr
->texture_index
;
1408 params
.explicit_lod
= explicit_lod
;
1409 params
.is_sviewinfo
= TRUE
;
1410 params
.sizes_out
= sizes_out
;
1412 if (instr
->op
== nir_texop_query_levels
)
1413 params
.explicit_lod
= bld_base
->uint_bld
.zero
;
1414 bld_base
->tex_size(bld_base
, ¶ms
);
1415 assign_dest(bld_base
, &instr
->dest
, &sizes_out
[instr
->op
== nir_texop_query_levels
? 3 : 0]);
1418 static enum lp_sampler_lod_property
lp_build_nir_lod_property(struct lp_build_nir_context
*bld_base
,
1421 enum lp_sampler_lod_property lod_property
;
1423 if (nir_src_is_dynamically_uniform(lod_src
))
1424 lod_property
= LP_SAMPLER_LOD_SCALAR
;
1425 else if (bld_base
->shader
->info
.stage
== MESA_SHADER_FRAGMENT
) {
1426 if (gallivm_perf
& GALLIVM_PERF_NO_QUAD_LOD
)
1427 lod_property
= LP_SAMPLER_LOD_PER_ELEMENT
;
1429 lod_property
= LP_SAMPLER_LOD_PER_QUAD
;
1432 lod_property
= LP_SAMPLER_LOD_PER_ELEMENT
;
1433 return lod_property
;
1436 static void visit_tex(struct lp_build_nir_context
*bld_base
, nir_tex_instr
*instr
)
1438 struct gallivm_state
*gallivm
= bld_base
->base
.gallivm
;
1439 LLVMBuilderRef builder
= gallivm
->builder
;
1440 LLVMValueRef coords
[5];
1441 LLVMValueRef offsets
[3] = { NULL
};
1442 LLVMValueRef explicit_lod
= NULL
, projector
= NULL
;
1443 struct lp_sampler_params params
;
1444 struct lp_derivatives derivs
;
1445 unsigned sample_key
= 0;
1446 nir_deref_instr
*texture_deref_instr
= NULL
;
1447 nir_deref_instr
*sampler_deref_instr
= NULL
;
1448 LLVMValueRef texel
[NIR_MAX_VEC_COMPONENTS
];
1449 unsigned lod_src
= 0;
1450 LLVMValueRef coord_undef
= LLVMGetUndef(bld_base
->base
.int_vec_type
);
1452 memset(¶ms
, 0, sizeof(params
));
1453 enum lp_sampler_lod_property lod_property
= LP_SAMPLER_LOD_SCALAR
;
1455 if (instr
->op
== nir_texop_txs
|| instr
->op
== nir_texop_query_levels
) {
1456 visit_txs(bld_base
, instr
);
1459 if (instr
->op
== nir_texop_txf
|| instr
->op
== nir_texop_txf_ms
)
1460 sample_key
|= LP_SAMPLER_OP_FETCH
<< LP_SAMPLER_OP_TYPE_SHIFT
;
1461 else if (instr
->op
== nir_texop_tg4
)
1462 sample_key
|= LP_SAMPLER_OP_GATHER
<< LP_SAMPLER_OP_TYPE_SHIFT
;
1463 else if (instr
->op
== nir_texop_lod
)
1464 sample_key
|= LP_SAMPLER_OP_LODQ
<< LP_SAMPLER_OP_TYPE_SHIFT
;
1465 for (unsigned i
= 0; i
< instr
->num_srcs
; i
++) {
1466 switch (instr
->src
[i
].src_type
) {
1467 case nir_tex_src_coord
: {
1468 LLVMValueRef coord
= get_src(bld_base
, instr
->src
[i
].src
);
1469 if (instr
->coord_components
== 1)
1472 for (unsigned chan
= 0; chan
< instr
->coord_components
; ++chan
)
1473 coords
[chan
] = LLVMBuildExtractValue(builder
, coord
,
1476 for (unsigned chan
= instr
->coord_components
; chan
< 5; chan
++)
1477 coords
[chan
] = coord_undef
;
1481 case nir_tex_src_texture_deref
:
1482 texture_deref_instr
= nir_src_as_deref(instr
->src
[i
].src
);
1484 case nir_tex_src_sampler_deref
:
1485 sampler_deref_instr
= nir_src_as_deref(instr
->src
[i
].src
);
1487 case nir_tex_src_projector
:
1488 projector
= lp_build_rcp(&bld_base
->base
, cast_type(bld_base
, get_src(bld_base
, instr
->src
[i
].src
), nir_type_float
, 32));
1490 case nir_tex_src_comparator
:
1491 sample_key
|= LP_SAMPLER_SHADOW
;
1492 coords
[4] = get_src(bld_base
, instr
->src
[i
].src
);
1493 coords
[4] = cast_type(bld_base
, coords
[4], nir_type_float
, 32);
1495 case nir_tex_src_bias
:
1496 sample_key
|= LP_SAMPLER_LOD_BIAS
<< LP_SAMPLER_LOD_CONTROL_SHIFT
;
1498 explicit_lod
= cast_type(bld_base
, get_src(bld_base
, instr
->src
[i
].src
), nir_type_float
, 32);
1500 case nir_tex_src_lod
:
1501 sample_key
|= LP_SAMPLER_LOD_EXPLICIT
<< LP_SAMPLER_LOD_CONTROL_SHIFT
;
1503 if (instr
->op
== nir_texop_txf
)
1504 explicit_lod
= cast_type(bld_base
, get_src(bld_base
, instr
->src
[i
].src
), nir_type_int
, 32);
1506 explicit_lod
= cast_type(bld_base
, get_src(bld_base
, instr
->src
[i
].src
), nir_type_float
, 32);
1508 case nir_tex_src_ddx
: {
1509 int deriv_cnt
= instr
->coord_components
;
1510 if (instr
->is_array
)
1512 LLVMValueRef deriv_val
= get_src(bld_base
, instr
->src
[i
].src
);
1514 derivs
.ddx
[0] = deriv_val
;
1516 for (unsigned chan
= 0; chan
< deriv_cnt
; ++chan
)
1517 derivs
.ddx
[chan
] = LLVMBuildExtractValue(builder
, deriv_val
,
1519 for (unsigned chan
= 0; chan
< deriv_cnt
; ++chan
)
1520 derivs
.ddx
[chan
] = cast_type(bld_base
, derivs
.ddx
[chan
], nir_type_float
, 32);
1523 case nir_tex_src_ddy
: {
1524 int deriv_cnt
= instr
->coord_components
;
1525 if (instr
->is_array
)
1527 LLVMValueRef deriv_val
= get_src(bld_base
, instr
->src
[i
].src
);
1529 derivs
.ddy
[0] = deriv_val
;
1531 for (unsigned chan
= 0; chan
< deriv_cnt
; ++chan
)
1532 derivs
.ddy
[chan
] = LLVMBuildExtractValue(builder
, deriv_val
,
1534 for (unsigned chan
= 0; chan
< deriv_cnt
; ++chan
)
1535 derivs
.ddy
[chan
] = cast_type(bld_base
, derivs
.ddy
[chan
], nir_type_float
, 32);
1538 case nir_tex_src_offset
: {
1539 int offset_cnt
= instr
->coord_components
;
1540 if (instr
->is_array
)
1542 LLVMValueRef offset_val
= get_src(bld_base
, instr
->src
[i
].src
);
1543 sample_key
|= LP_SAMPLER_OFFSETS
;
1544 if (offset_cnt
== 1)
1545 offsets
[0] = offset_val
;
1547 for (unsigned chan
= 0; chan
< offset_cnt
; ++chan
)
1548 offsets
[chan
] = LLVMBuildExtractValue(builder
, offset_val
,
1553 case nir_tex_src_ms_index
:
1560 if (!sampler_deref_instr
)
1561 sampler_deref_instr
= texture_deref_instr
;
1564 lod_property
= lp_build_nir_lod_property(bld_base
, instr
->src
[lod_src
].src
);
1566 if (instr
->op
== nir_texop_tex
|| instr
->op
== nir_texop_tg4
|| instr
->op
== nir_texop_txb
||
1567 instr
->op
== nir_texop_txl
|| instr
->op
== nir_texop_txd
|| instr
->op
== nir_texop_lod
)
1568 for (unsigned chan
= 0; chan
< instr
->coord_components
; ++chan
)
1569 coords
[chan
] = cast_type(bld_base
, coords
[chan
], nir_type_float
, 32);
1570 else if (instr
->op
== nir_texop_txf
|| instr
->op
== nir_texop_txf_ms
)
1571 for (unsigned chan
= 0; chan
< instr
->coord_components
; ++chan
)
1572 coords
[chan
] = cast_type(bld_base
, coords
[chan
], nir_type_int
, 32);
1574 if (instr
->is_array
&& instr
->sampler_dim
== GLSL_SAMPLER_DIM_1D
) {
1575 /* move layer coord for 1d arrays. */
1576 coords
[2] = coords
[1];
1577 coords
[1] = coord_undef
;
1581 for (unsigned chan
= 0; chan
< instr
->coord_components
; ++chan
)
1582 coords
[chan
] = lp_build_mul(&bld_base
->base
, coords
[chan
], projector
);
1583 if (sample_key
& LP_SAMPLER_SHADOW
)
1584 coords
[4] = lp_build_mul(&bld_base
->base
, coords
[4], projector
);
1587 uint32_t base_index
= 0;
1588 if (!texture_deref_instr
) {
1589 int samp_src_index
= nir_tex_instr_src_index(instr
, nir_tex_src_sampler_handle
);
1590 if (samp_src_index
== -1) {
1591 base_index
= instr
->sampler_index
;
1595 if (instr
->op
== nir_texop_txd
) {
1596 sample_key
|= LP_SAMPLER_LOD_DERIVATIVES
<< LP_SAMPLER_LOD_CONTROL_SHIFT
;
1597 params
.derivs
= &derivs
;
1598 if (bld_base
->shader
->info
.stage
== MESA_SHADER_FRAGMENT
) {
1599 if (gallivm_perf
& GALLIVM_PERF_NO_QUAD_LOD
)
1600 lod_property
= LP_SAMPLER_LOD_PER_ELEMENT
;
1602 lod_property
= LP_SAMPLER_LOD_PER_QUAD
;
1604 lod_property
= LP_SAMPLER_LOD_PER_ELEMENT
;
1607 sample_key
|= lod_property
<< LP_SAMPLER_LOD_PROPERTY_SHIFT
;
1608 params
.sample_key
= sample_key
;
1609 params
.offsets
= offsets
;
1610 params
.texture_index
= base_index
;
1611 params
.sampler_index
= base_index
;
1612 params
.coords
= coords
;
1613 params
.texel
= texel
;
1614 params
.lod
= explicit_lod
;
1615 bld_base
->tex(bld_base
, ¶ms
);
1616 assign_dest(bld_base
, &instr
->dest
, texel
);
1619 static void visit_ssa_undef(struct lp_build_nir_context
*bld_base
,
1620 const nir_ssa_undef_instr
*instr
)
1622 unsigned num_components
= instr
->def
.num_components
;
1623 LLVMValueRef undef
[NIR_MAX_VEC_COMPONENTS
];
1624 struct lp_build_context
*undef_bld
= get_int_bld(bld_base
, true, instr
->def
.bit_size
);
1625 for (unsigned i
= 0; i
< num_components
; i
++)
1626 undef
[i
] = LLVMGetUndef(undef_bld
->vec_type
);
1627 assign_ssa_dest(bld_base
, &instr
->def
, undef
);
1630 static void visit_jump(struct lp_build_nir_context
*bld_base
,
1631 const nir_jump_instr
*instr
)
1633 switch (instr
->type
) {
1634 case nir_jump_break
:
1635 bld_base
->break_stmt(bld_base
);
1637 case nir_jump_continue
:
1638 bld_base
->continue_stmt(bld_base
);
1641 unreachable("Unknown jump instr\n");
1645 static void visit_deref(struct lp_build_nir_context
*bld_base
,
1646 nir_deref_instr
*instr
)
1648 if (instr
->mode
!= nir_var_mem_shared
&&
1649 instr
->mode
!= nir_var_mem_global
)
1651 LLVMValueRef result
= NULL
;
1652 switch(instr
->deref_type
) {
1653 case nir_deref_type_var
: {
1654 struct hash_entry
*entry
= _mesa_hash_table_search(bld_base
->vars
, instr
->var
);
1655 result
= entry
->data
;
1659 unreachable("Unhandled deref_instr deref type");
1662 assign_ssa(bld_base
, instr
->dest
.ssa
.index
, result
);
1665 static void visit_block(struct lp_build_nir_context
*bld_base
, nir_block
*block
)
1667 nir_foreach_instr(instr
, block
)
1669 switch (instr
->type
) {
1670 case nir_instr_type_alu
:
1671 visit_alu(bld_base
, nir_instr_as_alu(instr
));
1673 case nir_instr_type_load_const
:
1674 visit_load_const(bld_base
, nir_instr_as_load_const(instr
));
1676 case nir_instr_type_intrinsic
:
1677 visit_intrinsic(bld_base
, nir_instr_as_intrinsic(instr
));
1679 case nir_instr_type_tex
:
1680 visit_tex(bld_base
, nir_instr_as_tex(instr
));
1682 case nir_instr_type_phi
:
1685 case nir_instr_type_ssa_undef
:
1686 visit_ssa_undef(bld_base
, nir_instr_as_ssa_undef(instr
));
1688 case nir_instr_type_jump
:
1689 visit_jump(bld_base
, nir_instr_as_jump(instr
));
1691 case nir_instr_type_deref
:
1692 visit_deref(bld_base
, nir_instr_as_deref(instr
));
1695 fprintf(stderr
, "Unknown NIR instr type: ");
1696 nir_print_instr(instr
, stderr
);
1697 fprintf(stderr
, "\n");
1703 static void visit_if(struct lp_build_nir_context
*bld_base
, nir_if
*if_stmt
)
1705 LLVMValueRef cond
= get_src(bld_base
, if_stmt
->condition
);
1707 bld_base
->if_cond(bld_base
, cond
);
1708 visit_cf_list(bld_base
, &if_stmt
->then_list
);
1710 if (!exec_list_is_empty(&if_stmt
->else_list
)) {
1711 bld_base
->else_stmt(bld_base
);
1712 visit_cf_list(bld_base
, &if_stmt
->else_list
);
1714 bld_base
->endif_stmt(bld_base
);
1717 static void visit_loop(struct lp_build_nir_context
*bld_base
, nir_loop
*loop
)
1719 bld_base
->bgnloop(bld_base
);
1720 visit_cf_list(bld_base
, &loop
->body
);
1721 bld_base
->endloop(bld_base
);
1724 static void visit_cf_list(struct lp_build_nir_context
*bld_base
,
1725 struct exec_list
*list
)
1727 foreach_list_typed(nir_cf_node
, node
, node
, list
)
1729 switch (node
->type
) {
1730 case nir_cf_node_block
:
1731 visit_block(bld_base
, nir_cf_node_as_block(node
));
1734 case nir_cf_node_if
:
1735 visit_if(bld_base
, nir_cf_node_as_if(node
));
1738 case nir_cf_node_loop
:
1739 visit_loop(bld_base
, nir_cf_node_as_loop(node
));
1749 handle_shader_output_decl(struct lp_build_nir_context
*bld_base
,
1750 struct nir_shader
*nir
,
1751 struct nir_variable
*variable
)
1753 bld_base
->emit_var_decl(bld_base
, variable
);
1756 /* vector registers are stored as arrays in LLVM side,
1757 so we can use GEP on them, as to do exec mask stores
1758 we need to operate on a single components.
1764 static LLVMTypeRef
get_register_type(struct lp_build_nir_context
*bld_base
,
1767 struct lp_build_context
*int_bld
= get_int_bld(bld_base
, true, reg
->bit_size
);
1769 LLVMTypeRef type
= int_bld
->vec_type
;
1770 if (reg
->num_array_elems
)
1771 type
= LLVMArrayType(type
, reg
->num_array_elems
);
1772 if (reg
->num_components
> 1)
1773 type
= LLVMArrayType(type
, reg
->num_components
);
1779 bool lp_build_nir_llvm(
1780 struct lp_build_nir_context
*bld_base
,
1781 struct nir_shader
*nir
)
1783 struct nir_function
*func
;
1785 nir_convert_from_ssa(nir
, true);
1786 nir_lower_locals_to_regs(nir
);
1787 nir_remove_dead_derefs(nir
);
1788 nir_remove_dead_variables(nir
, nir_var_function_temp
);
1790 nir_foreach_variable(variable
, &nir
->outputs
)
1791 handle_shader_output_decl(bld_base
, nir
, variable
);
1793 bld_base
->regs
= _mesa_hash_table_create(NULL
, _mesa_hash_pointer
,
1794 _mesa_key_pointer_equal
);
1795 bld_base
->vars
= _mesa_hash_table_create(NULL
, _mesa_hash_pointer
,
1796 _mesa_key_pointer_equal
);
1798 func
= (struct nir_function
*)exec_list_get_head(&nir
->functions
);
1800 nir_foreach_register(reg
, &func
->impl
->registers
) {
1801 LLVMTypeRef type
= get_register_type(bld_base
, reg
);
1802 LLVMValueRef reg_alloc
= lp_build_alloca_undef(bld_base
->base
.gallivm
,
1804 _mesa_hash_table_insert(bld_base
->regs
, reg
, reg_alloc
);
1806 nir_index_ssa_defs(func
->impl
);
1807 bld_base
->ssa_defs
= calloc(func
->impl
->ssa_alloc
, sizeof(LLVMValueRef
));
1808 visit_cf_list(bld_base
, &func
->impl
->body
);
1810 free(bld_base
->ssa_defs
);
1811 ralloc_free(bld_base
->vars
);
1812 ralloc_free(bld_base
->regs
);
1816 /* do some basic opts to remove some things we don't want to see. */
1817 void lp_build_opt_nir(struct nir_shader
*nir
)
1822 NIR_PASS_V(nir
, nir_opt_constant_folding
);
1823 NIR_PASS_V(nir
, nir_opt_algebraic
);
1824 NIR_PASS_V(nir
, nir_lower_pack
);
1826 nir_lower_bool_to_int32(nir
);