projects / mesa.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
history | raw | HEAD
gallivm/nir: support passing image index into image code.
[mesa.git] / src / gallium / auxiliary / gallivm / lp_bld_nir.c
1 /**************************************************************************
2 *
3 * Copyright 2019 Red Hat.
4 * All Rights Reserved.
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the "Software"),
8 * to deal in the Software without restriction, including without limitation
9 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
10 * and/or sell copies of the Software, and to permit persons to whom the
11 * Software is furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included
14 * in all copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
17 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
22 * SOFTWARE.
23 *
24 **************************************************************************/
25
26 #include "lp_bld_nir.h"
27 #include "lp_bld_arit.h"
28 #include "lp_bld_bitarit.h"
29 #include "lp_bld_const.h"
30 #include "lp_bld_gather.h"
31 #include "lp_bld_logic.h"
32 #include "lp_bld_quad.h"
33 #include "lp_bld_flow.h"
34 #include "lp_bld_struct.h"
35 #include "lp_bld_debug.h"
36 #include "lp_bld_printf.h"
37 #include "nir_deref.h"
38
39 static void visit_cf_list(struct lp_build_nir_context *bld_base,
40 struct exec_list *list);
41
42 static LLVMValueRef cast_type(struct lp_build_nir_context *bld_base, LLVMValueRef val,
43 nir_alu_type alu_type, unsigned bit_size)
44 {
45 LLVMBuilderRef builder = bld_base->base.gallivm->builder;
46 switch (alu_type) {
47 case nir_type_float:
48 switch (bit_size) {
49 case 32:
50 return LLVMBuildBitCast(builder, val, bld_base->base.vec_type, "");
51 case 64:
52 return LLVMBuildBitCast(builder, val, bld_base->dbl_bld.vec_type, "");
53 default:
54 assert(0);
55 break;
56 }
57 break;
58 case nir_type_int:
59 switch (bit_size) {
60 case 8:
61 return LLVMBuildBitCast(builder, val, bld_base->int8_bld.vec_type, "");
62 case 16:
63 return LLVMBuildBitCast(builder, val, bld_base->int16_bld.vec_type, "");
64 case 32:
65 return LLVMBuildBitCast(builder, val, bld_base->int_bld.vec_type, "");
66 case 64:
67 return LLVMBuildBitCast(builder, val, bld_base->int64_bld.vec_type, "");
68 default:
69 assert(0);
70 break;
71 }
72 break;
73 case nir_type_uint:
74 switch (bit_size) {
75 case 8:
76 return LLVMBuildBitCast(builder, val, bld_base->uint8_bld.vec_type, "");
77 case 16:
78 return LLVMBuildBitCast(builder, val, bld_base->uint16_bld.vec_type, "");
79 case 32:
80 return LLVMBuildBitCast(builder, val, bld_base->uint_bld.vec_type, "");
81 case 64:
82 return LLVMBuildBitCast(builder, val, bld_base->uint64_bld.vec_type, "");
83 default:
84 assert(0);
85 break;
86 }
87 break;
88 case nir_type_uint32:
89 return LLVMBuildBitCast(builder, val, bld_base->uint_bld.vec_type, "");
90 default:
91 return val;
92 }
93 return NULL;
94 }
95
96
97 static struct lp_build_context *get_flt_bld(struct lp_build_nir_context *bld_base,
98 unsigned op_bit_size)
99 {
100 if (op_bit_size == 64)
101 return &bld_base->dbl_bld;
102 else
103 return &bld_base->base;
104 }
105
106 static unsigned glsl_sampler_to_pipe(int sampler_dim, bool is_array)
107 {
108 unsigned pipe_target = PIPE_BUFFER;
109 switch (sampler_dim) {
110 case GLSL_SAMPLER_DIM_1D:
111 pipe_target = is_array ? PIPE_TEXTURE_1D_ARRAY : PIPE_TEXTURE_1D;
112 break;
113 case GLSL_SAMPLER_DIM_2D:
114 pipe_target = is_array ? PIPE_TEXTURE_2D_ARRAY : PIPE_TEXTURE_2D;
115 break;
116 case GLSL_SAMPLER_DIM_3D:
117 pipe_target = PIPE_TEXTURE_3D;
118 break;
119 case GLSL_SAMPLER_DIM_MS:
120 pipe_target = is_array ? PIPE_TEXTURE_2D_ARRAY : PIPE_TEXTURE_2D;
121 break;
122 case GLSL_SAMPLER_DIM_CUBE:
123 pipe_target = is_array ? PIPE_TEXTURE_CUBE_ARRAY : PIPE_TEXTURE_CUBE;
124 break;
125 case GLSL_SAMPLER_DIM_RECT:
126 pipe_target = PIPE_TEXTURE_RECT;
127 break;
128 case GLSL_SAMPLER_DIM_BUF:
129 pipe_target = PIPE_BUFFER;
130 break;
131 default:
132 break;
133 }
134 return pipe_target;
135 }
136
137 static LLVMValueRef get_ssa_src(struct lp_build_nir_context *bld_base, nir_ssa_def *ssa)
138 {
139 return bld_base->ssa_defs[ssa->index];
140 }
141
142 static LLVMValueRef get_src(struct lp_build_nir_context *bld_base, nir_src src);
143
144 static LLVMValueRef get_reg_src(struct lp_build_nir_context *bld_base, nir_reg_src src)
145 {
146 struct hash_entry *entry = _mesa_hash_table_search(bld_base->regs, src.reg);
147 LLVMValueRef reg_storage = (LLVMValueRef)entry->data;
148 struct lp_build_context *reg_bld = get_int_bld(bld_base, true, src.reg->bit_size);
149 LLVMValueRef indir_src = NULL;
150 if (src.indirect)
151 indir_src = get_src(bld_base, *src.indirect);
152 return bld_base->load_reg(bld_base, reg_bld, &src, indir_src, reg_storage);
153 }
154
155 static LLVMValueRef get_src(struct lp_build_nir_context *bld_base, nir_src src)
156 {
157 if (src.is_ssa)
158 return get_ssa_src(bld_base, src.ssa);
159 else
160 return get_reg_src(bld_base, src.reg);
161 }
162
163 static void assign_ssa(struct lp_build_nir_context *bld_base, int idx, LLVMValueRef ptr)
164 {
165 bld_base->ssa_defs[idx] = ptr;
166 }
167
168 static void assign_ssa_dest(struct lp_build_nir_context *bld_base, const nir_ssa_def *ssa,
169 LLVMValueRef vals[NIR_MAX_VEC_COMPONENTS])
170 {
171 assign_ssa(bld_base, ssa->index, ssa->num_components == 1 ? vals[0] : lp_nir_array_build_gather_values(bld_base->base.gallivm->builder, vals, ssa->num_components));
172 }
173
174 static void assign_reg(struct lp_build_nir_context *bld_base, const nir_reg_dest *reg,
175 unsigned write_mask,
176 LLVMValueRef vals[NIR_MAX_VEC_COMPONENTS])
177 {
178 struct hash_entry *entry = _mesa_hash_table_search(bld_base->regs, reg->reg);
179 LLVMValueRef reg_storage = (LLVMValueRef)entry->data;
180 struct lp_build_context *reg_bld = get_int_bld(bld_base, true, reg->reg->bit_size);
181 LLVMValueRef indir_src = NULL;
182 if (reg->indirect)
183 indir_src = get_src(bld_base, *reg->indirect);
184 bld_base->store_reg(bld_base, reg_bld, reg, write_mask ? write_mask : 0xf, indir_src, reg_storage, vals);
185 }
186
187 static void assign_dest(struct lp_build_nir_context *bld_base, const nir_dest *dest, LLVMValueRef vals[NIR_MAX_VEC_COMPONENTS])
188 {
189 if (dest->is_ssa)
190 assign_ssa_dest(bld_base, &dest->ssa, vals);
191 else
192 assign_reg(bld_base, &dest->reg, 0, vals);
193 }
194
195 static void assign_alu_dest(struct lp_build_nir_context *bld_base, const nir_alu_dest *dest, LLVMValueRef vals[NIR_MAX_VEC_COMPONENTS])
196 {
197 if (dest->dest.is_ssa)
198 assign_ssa_dest(bld_base, &dest->dest.ssa, vals);
199 else
200 assign_reg(bld_base, &dest->dest.reg, dest->write_mask, vals);
201 }
202
203 static LLVMValueRef int_to_bool32(struct lp_build_nir_context *bld_base,
204 uint32_t src_bit_size,
205 bool is_unsigned,
206 LLVMValueRef val)
207 {
208 LLVMBuilderRef builder = bld_base->base.gallivm->builder;
209 struct lp_build_context *int_bld = get_int_bld(bld_base, is_unsigned, src_bit_size);
210 LLVMValueRef result = lp_build_compare(bld_base->base.gallivm, int_bld->type, PIPE_FUNC_NOTEQUAL, val, int_bld->zero);
211 if (src_bit_size == 64)
212 result = LLVMBuildTrunc(builder, result, bld_base->int_bld.vec_type, "");
213 return result;
214 }
215
216 static LLVMValueRef flt_to_bool32(struct lp_build_nir_context *bld_base,
217 uint32_t src_bit_size,
218 LLVMValueRef val)
219 {
220 LLVMBuilderRef builder = bld_base->base.gallivm->builder;
221 struct lp_build_context *flt_bld = get_flt_bld(bld_base, src_bit_size);
222 LLVMValueRef result = lp_build_cmp(flt_bld, PIPE_FUNC_NOTEQUAL, val, flt_bld->zero);
223 if (src_bit_size == 64)
224 result = LLVMBuildTrunc(builder, result, bld_base->int_bld.vec_type, "");
225 return result;
226 }
227
228 static LLVMValueRef fcmp32(struct lp_build_nir_context *bld_base,
229 enum pipe_compare_func compare,
230 uint32_t src_bit_size,
231 LLVMValueRef src[NIR_MAX_VEC_COMPONENTS])
232 {
233 LLVMBuilderRef builder = bld_base->base.gallivm->builder;
234 struct lp_build_context *flt_bld = get_flt_bld(bld_base, src_bit_size);
235 LLVMValueRef result;
236
237 if (compare != PIPE_FUNC_NOTEQUAL)
238 result = lp_build_cmp_ordered(flt_bld, compare, src[0], src[1]);
239 else
240 result = lp_build_cmp(flt_bld, compare, src[0], src[1]);
241 if (src_bit_size == 64)
242 result = LLVMBuildTrunc(builder, result, bld_base->int_bld.vec_type, "");
243 return result;
244 }
245
246 static LLVMValueRef icmp32(struct lp_build_nir_context *bld_base,
247 enum pipe_compare_func compare,
248 bool is_unsigned,
249 uint32_t src_bit_size,
250 LLVMValueRef src[NIR_MAX_VEC_COMPONENTS])
251 {
252 LLVMBuilderRef builder = bld_base->base.gallivm->builder;
253 struct lp_build_context *i_bld = get_int_bld(bld_base, is_unsigned, src_bit_size);
254 LLVMValueRef result = lp_build_cmp(i_bld, compare, src[0], src[1]);
255 if (src_bit_size < 32)
256 result = LLVMBuildSExt(builder, result, bld_base->int_bld.vec_type, "");
257 else if (src_bit_size == 64)
258 result = LLVMBuildTrunc(builder, result, bld_base->int_bld.vec_type, "");
259 return result;
260 }
261
262 static LLVMValueRef get_alu_src(struct lp_build_nir_context *bld_base,
263 nir_alu_src src,
264 unsigned num_components)
265 {
266 LLVMBuilderRef builder = bld_base->base.gallivm->builder;
267 struct gallivm_state *gallivm = bld_base->base.gallivm;
268 LLVMValueRef value = get_src(bld_base, src.src);
269 bool need_swizzle = false;
270
271 assert(value);
272 unsigned src_components = nir_src_num_components(src.src);
273 for (unsigned i = 0; i < num_components; ++i) {
274 assert(src.swizzle[i] < src_components);
275 if (src.swizzle[i] != i)
276 need_swizzle = true;
277 }
278
279 if (need_swizzle || num_components != src_components) {
280 if (src_components > 1 && num_components == 1) {
281 value = LLVMBuildExtractValue(gallivm->builder, value,
282 src.swizzle[0], "");
283 } else if (src_components == 1 && num_components > 1) {
284 LLVMValueRef values[] = {value, value, value, value, value, value, value, value, value, value, value, value, value, value, value, value};
285 value = lp_nir_array_build_gather_values(builder, values, num_components);
286 } else {
287 LLVMValueRef arr = LLVMGetUndef(LLVMArrayType(LLVMTypeOf(LLVMBuildExtractValue(builder, value, 0, "")), num_components));
288 for (unsigned i = 0; i < num_components; i++)
289 arr = LLVMBuildInsertValue(builder, arr, LLVMBuildExtractValue(builder, value, src.swizzle[i], ""), i, "");
290 value = arr;
291 }
292 }
293 assert(!src.negate);
294 assert(!src.abs);
295 return value;
296 }
297
298 static LLVMValueRef emit_b2f(struct lp_build_nir_context *bld_base,
299 LLVMValueRef src0,
300 unsigned bitsize)
301 {
302 LLVMBuilderRef builder = bld_base->base.gallivm->builder;
303 LLVMValueRef result = LLVMBuildAnd(builder, cast_type(bld_base, src0, nir_type_int, 32),
304 LLVMBuildBitCast(builder, lp_build_const_vec(bld_base->base.gallivm, bld_base->base.type,
305 1.0), bld_base->int_bld.vec_type, ""),
306 "");
307 result = LLVMBuildBitCast(builder, result, bld_base->base.vec_type, "");
308 switch (bitsize) {
309 case 32:
310 break;
311 case 64:
312 result = LLVMBuildFPExt(builder, result, bld_base->dbl_bld.vec_type, "");
313 break;
314 default:
315 unreachable("unsupported bit size.");
316 }
317 return result;
318 }
319
320 static LLVMValueRef emit_b2i(struct lp_build_nir_context *bld_base,
321 LLVMValueRef src0,
322 unsigned bitsize)
323 {
324 LLVMBuilderRef builder = bld_base->base.gallivm->builder;
325 LLVMValueRef result = LLVMBuildAnd(builder, cast_type(bld_base, src0, nir_type_int, 32),
326 lp_build_const_int_vec(bld_base->base.gallivm, bld_base->base.type, 1), "");
327 switch (bitsize) {
328 case 32:
329 return result;
330 case 64:
331 return LLVMBuildZExt(builder, result, bld_base->int64_bld.vec_type, "");
332 default:
333 unreachable("unsupported bit size.");
334 }
335 }
336
337 static LLVMValueRef emit_b32csel(struct lp_build_nir_context *bld_base,
338 unsigned src_bit_size[NIR_MAX_VEC_COMPONENTS],
339 LLVMValueRef src[NIR_MAX_VEC_COMPONENTS])
340 {
341 LLVMValueRef sel = cast_type(bld_base, src[0], nir_type_int, 32);
342 LLVMValueRef v = lp_build_compare(bld_base->base.gallivm, bld_base->int_bld.type, PIPE_FUNC_NOTEQUAL, sel, bld_base->int_bld.zero);
343 struct lp_build_context *bld = get_int_bld(bld_base, false, src_bit_size[1]);
344 return lp_build_select(bld, v, src[1], src[2]);
345 }
346
347 static LLVMValueRef split_64bit(struct lp_build_nir_context *bld_base,
348 LLVMValueRef src,
349 bool hi)
350 {
351 struct gallivm_state *gallivm = bld_base->base.gallivm;
352 LLVMValueRef shuffles[LP_MAX_VECTOR_WIDTH/32];
353 LLVMValueRef shuffles2[LP_MAX_VECTOR_WIDTH/32];
354 int len = bld_base->base.type.length * 2;
355 for (unsigned i = 0; i < bld_base->base.type.length; i++) {
356 #if UTIL_ARCH_LITTLE_ENDIAN
357 shuffles[i] = lp_build_const_int32(gallivm, i * 2);
358 shuffles2[i] = lp_build_const_int32(gallivm, (i * 2) + 1);
359 #else
360 shuffles[i] = lp_build_const_int32(gallivm, (i * 2) + 1);
361 shuffles2[i] = lp_build_const_int32(gallivm, (i * 2));
362 #endif
363 }
364
365 src = LLVMBuildBitCast(gallivm->builder, src, LLVMVectorType(LLVMInt32TypeInContext(gallivm->context), len), "");
366 return LLVMBuildShuffleVector(gallivm->builder, src,
367 LLVMGetUndef(LLVMTypeOf(src)),
368 LLVMConstVector(hi ? shuffles2 : shuffles,
369 bld_base->base.type.length),
370 "");
371 }
372
373 static LLVMValueRef
374 merge_64bit(struct lp_build_nir_context *bld_base,
375 LLVMValueRef input,
376 LLVMValueRef input2)
377 {
378 struct gallivm_state *gallivm = bld_base->base.gallivm;
379 LLVMBuilderRef builder = gallivm->builder;
380 int i;
381 LLVMValueRef shuffles[2 * (LP_MAX_VECTOR_WIDTH/32)];
382 int len = bld_base->base.type.length * 2;
383 assert(len <= (2 * (LP_MAX_VECTOR_WIDTH/32)));
384
385 for (i = 0; i < bld_base->base.type.length * 2; i+=2) {
386 #if UTIL_ARCH_LITTLE_ENDIAN
387 shuffles[i] = lp_build_const_int32(gallivm, i / 2);
388 shuffles[i + 1] = lp_build_const_int32(gallivm, i / 2 + bld_base->base.type.length);
389 #else
390 shuffles[i] = lp_build_const_int32(gallivm, i / 2 + bld_base->base.type.length);
391 shuffles[i + 1] = lp_build_const_int32(gallivm, i / 2);
392 #endif
393 }
394 return LLVMBuildShuffleVector(builder, input, input2, LLVMConstVector(shuffles, len), "");
395 }
396
397 static LLVMValueRef
398 do_int_divide(struct lp_build_nir_context *bld_base,
399 bool is_unsigned, unsigned src_bit_size,
400 LLVMValueRef src, LLVMValueRef src2)
401 {
402 struct gallivm_state *gallivm = bld_base->base.gallivm;
403 LLVMBuilderRef builder = gallivm->builder;
404 struct lp_build_context *int_bld = get_int_bld(bld_base, is_unsigned, src_bit_size);
405 struct lp_build_context *mask_bld = get_int_bld(bld_base, true, src_bit_size);
406 LLVMValueRef div_mask = lp_build_cmp(mask_bld, PIPE_FUNC_EQUAL, src2,
407 mask_bld->zero);
408
409 if (!is_unsigned) {
410 /* INT_MIN (0x80000000) / -1 (0xffffffff) causes sigfpe, seen with blender. */
411 div_mask = LLVMBuildAnd(builder, div_mask, lp_build_const_int_vec(gallivm, int_bld->type, 0x7fffffff), "");
412 }
413 LLVMValueRef divisor = LLVMBuildOr(builder,
414 div_mask,
415 src2, "");
416 LLVMValueRef result = lp_build_div(int_bld, src, divisor);
417
418 if (!is_unsigned) {
419 LLVMValueRef not_div_mask = LLVMBuildNot(builder, div_mask, "");
420 return LLVMBuildAnd(builder, not_div_mask, result, "");
421 } else
422 /* udiv by zero is guaranteed to return 0xffffffff at least with d3d10
423 * may as well do same for idiv */
424 return LLVMBuildOr(builder, div_mask, result, "");
425 }
426
427 static LLVMValueRef
428 do_int_mod(struct lp_build_nir_context *bld_base,
429 bool is_unsigned, unsigned src_bit_size,
430 LLVMValueRef src, LLVMValueRef src2)
431 {
432 struct gallivm_state *gallivm = bld_base->base.gallivm;
433 LLVMBuilderRef builder = gallivm->builder;
434 struct lp_build_context *int_bld = get_int_bld(bld_base, is_unsigned, src_bit_size);
435 LLVMValueRef div_mask = lp_build_cmp(int_bld, PIPE_FUNC_EQUAL, src2,
436 int_bld->zero);
437 LLVMValueRef divisor = LLVMBuildOr(builder,
438 div_mask,
439 src2, "");
440 LLVMValueRef result = lp_build_mod(int_bld, src, divisor);
441 return LLVMBuildOr(builder, div_mask, result, "");
442 }
443
444 static LLVMValueRef do_alu_action(struct lp_build_nir_context *bld_base,
445 nir_op op, unsigned src_bit_size[NIR_MAX_VEC_COMPONENTS], LLVMValueRef src[NIR_MAX_VEC_COMPONENTS])
446 {
447 struct gallivm_state *gallivm = bld_base->base.gallivm;
448 LLVMBuilderRef builder = gallivm->builder;
449 LLVMValueRef result;
450 switch (op) {
451 case nir_op_b2f32:
452 result = emit_b2f(bld_base, src[0], 32);
453 break;
454 case nir_op_b2f64:
455 result = emit_b2f(bld_base, src[0], 64);
456 break;
457 case nir_op_b2i32:
458 result = emit_b2i(bld_base, src[0], 32);
459 break;
460 case nir_op_b2i64:
461 result = emit_b2i(bld_base, src[0], 64);
462 break;
463 case nir_op_b32csel:
464 result = emit_b32csel(bld_base, src_bit_size, src);
465 break;
466 case nir_op_bit_count:
467 result = lp_build_popcount(get_int_bld(bld_base, false, src_bit_size[0]), src[0]);
468 break;
469 case nir_op_bitfield_select:
470 result = lp_build_xor(&bld_base->uint_bld, src[2], lp_build_and(&bld_base->uint_bld, src[0], lp_build_xor(&bld_base->uint_bld, src[1], src[2])));
471 break;
472 case nir_op_bitfield_reverse:
473 result = lp_build_bitfield_reverse(get_int_bld(bld_base, false, src_bit_size[0]), src[0]);
474 break;
475 case nir_op_f2b32:
476 result = flt_to_bool32(bld_base, src_bit_size[0], src[0]);
477 break;
478 case nir_op_f2f32:
479 result = LLVMBuildFPTrunc(builder, src[0],
480 bld_base->base.vec_type, "");
481 break;
482 case nir_op_f2f64:
483 result = LLVMBuildFPExt(builder, src[0],
484 bld_base->dbl_bld.vec_type, "");
485 break;
486 case nir_op_f2i32:
487 result = LLVMBuildFPToSI(builder, src[0], bld_base->base.int_vec_type, "");
488 break;
489 case nir_op_f2u32:
490 result = LLVMBuildFPToUI(builder,
491 src[0],
492 bld_base->base.int_vec_type, "");
493 break;
494 case nir_op_f2i64:
495 result = LLVMBuildFPToSI(builder,
496 src[0],
497 bld_base->int64_bld.vec_type, "");
498 break;
499 case nir_op_f2u64:
500 result = LLVMBuildFPToUI(builder,
501 src[0],
502 bld_base->uint64_bld.vec_type, "");
503 break;
504 case nir_op_fabs:
505 result = lp_build_abs(get_flt_bld(bld_base, src_bit_size[0]), src[0]);
506 break;
507 case nir_op_fadd:
508 result = lp_build_add(get_flt_bld(bld_base, src_bit_size[0]),
509 src[0], src[1]);
510 break;
511 case nir_op_fceil:
512 result = lp_build_ceil(get_flt_bld(bld_base, src_bit_size[0]), src[0]);
513 break;
514 case nir_op_fcos:
515 result = lp_build_cos(&bld_base->base, src[0]);
516 break;
517 case nir_op_fddx:
518 case nir_op_fddx_coarse:
519 case nir_op_fddx_fine:
520 result = lp_build_ddx(&bld_base->base, src[0]);
521 break;
522 case nir_op_fddy:
523 case nir_op_fddy_coarse:
524 case nir_op_fddy_fine:
525 result = lp_build_ddy(&bld_base->base, src[0]);
526 break;
527 case nir_op_fdiv:
528 result = lp_build_div(get_flt_bld(bld_base, src_bit_size[0]),
529 src[0], src[1]);
530 break;
531 case nir_op_feq32:
532 result = fcmp32(bld_base, PIPE_FUNC_EQUAL, src_bit_size[0], src);
533 break;
534 case nir_op_fexp2:
535 result = lp_build_exp2(&bld_base->base, src[0]);
536 break;
537 case nir_op_ffloor:
538 result = lp_build_floor(get_flt_bld(bld_base, src_bit_size[0]), src[0]);
539 break;
540 case nir_op_ffma:
541 result = lp_build_fmuladd(builder, src[0], src[1], src[2]);
542 break;
543 case nir_op_ffract: {
544 struct lp_build_context *flt_bld = get_flt_bld(bld_base, src_bit_size[0]);
545 LLVMValueRef tmp = lp_build_floor(flt_bld, src[0]);
546 result = lp_build_sub(flt_bld, src[0], tmp);
547 break;
548 }
549 case nir_op_fge32:
550 result = fcmp32(bld_base, PIPE_FUNC_GEQUAL, src_bit_size[0], src);
551 break;
552 case nir_op_find_lsb:
553 result = lp_build_cttz(get_int_bld(bld_base, false, src_bit_size[0]), src[0]);
554 break;
555 case nir_op_flog2:
556 result = lp_build_log2_safe(&bld_base->base, src[0]);
557 break;
558 case nir_op_flt32:
559 result = fcmp32(bld_base, PIPE_FUNC_LESS, src_bit_size[0], src);
560 break;
561 case nir_op_fmin:
562 result = lp_build_min(get_flt_bld(bld_base, src_bit_size[0]), src[0], src[1]);
563 break;
564 case nir_op_fmod: {
565 struct lp_build_context *flt_bld = get_flt_bld(bld_base, src_bit_size[0]);
566 result = lp_build_div(flt_bld, src[0], src[1]);
567 result = lp_build_floor(flt_bld, result);
568 result = lp_build_mul(flt_bld, src[1], result);
569 result = lp_build_sub(flt_bld, src[0], result);
570 break;
571 }
572 case nir_op_fmul:
573 result = lp_build_mul(get_flt_bld(bld_base, src_bit_size[0]),
574 src[0], src[1]);
575 break;
576 case nir_op_fmax:
577 result = lp_build_max(get_flt_bld(bld_base, src_bit_size[0]), src[0], src[1]);
578 break;
579 case nir_op_fne32:
580 result = fcmp32(bld_base, PIPE_FUNC_NOTEQUAL, src_bit_size[0], src);
581 break;
582 case nir_op_fneg:
583 result = lp_build_negate(get_flt_bld(bld_base, src_bit_size[0]), src[0]);
584 break;
585 case nir_op_fpow:
586 result = lp_build_pow(&bld_base->base, src[0], src[1]);
587 break;
588 case nir_op_frcp:
589 result = lp_build_rcp(get_flt_bld(bld_base, src_bit_size[0]), src[0]);
590 break;
591 case nir_op_fround_even:
592 result = lp_build_round(get_flt_bld(bld_base, src_bit_size[0]), src[0]);
593 break;
594 case nir_op_frsq:
595 result = lp_build_rsqrt(get_flt_bld(bld_base, src_bit_size[0]), src[0]);
596 break;
597 case nir_op_fsat:
598 result = lp_build_clamp_zero_one_nanzero(get_flt_bld(bld_base, src_bit_size[0]), src[0]);
599 break;
600 case nir_op_fsign:
601 result = lp_build_sgn(get_flt_bld(bld_base, src_bit_size[0]), src[0]);
602 break;
603 case nir_op_fsin:
604 result = lp_build_sin(&bld_base->base, src[0]);
605 break;
606 case nir_op_fsqrt:
607 result = lp_build_sqrt(get_flt_bld(bld_base, src_bit_size[0]), src[0]);
608 break;
609 case nir_op_ftrunc:
610 result = lp_build_trunc(get_flt_bld(bld_base, src_bit_size[0]), src[0]);
611 break;
612 case nir_op_i2b32:
613 result = int_to_bool32(bld_base, src_bit_size[0], false, src[0]);
614 break;
615 case nir_op_i2f32:
616 result = lp_build_int_to_float(&bld_base->base, src[0]);
617 break;
618 case nir_op_i2f64:
619 result = lp_build_int_to_float(&bld_base->dbl_bld, src[0]);
620 break;
621 case nir_op_i2i8:
622 result = LLVMBuildTrunc(builder, src[0], bld_base->int8_bld.vec_type, "");
623 break;
624 case nir_op_i2i16:
625 if (src_bit_size[0] < 16)
626 result = LLVMBuildSExt(builder, src[0], bld_base->int16_bld.vec_type, "");
627 else
628 result = LLVMBuildTrunc(builder, src[0], bld_base->int16_bld.vec_type, "");
629 break;
630 case nir_op_i2i32:
631 if (src_bit_size[0] < 32)
632 result = LLVMBuildSExt(builder, src[0], bld_base->int_bld.vec_type, "");
633 else
634 result = LLVMBuildTrunc(builder, src[0], bld_base->int_bld.vec_type, "");
635 break;
636 case nir_op_i2i64:
637 result = LLVMBuildSExt(builder, src[0], bld_base->int64_bld.vec_type, "");
638 break;
639 case nir_op_iabs:
640 result = lp_build_abs(get_int_bld(bld_base, false, src_bit_size[0]), src[0]);
641 break;
642 case nir_op_iadd:
643 result = lp_build_add(get_int_bld(bld_base, false, src_bit_size[0]),
644 src[0], src[1]);
645 break;
646 case nir_op_iand:
647 result = lp_build_and(get_int_bld(bld_base, false, src_bit_size[0]),
648 src[0], src[1]);
649 break;
650 case nir_op_idiv:
651 result = do_int_divide(bld_base, false, src_bit_size[0], src[0], src[1]);
652 break;
653 case nir_op_ieq32:
654 result = icmp32(bld_base, PIPE_FUNC_EQUAL, false, src_bit_size[0], src);
655 break;
656 case nir_op_ige32:
657 result = icmp32(bld_base, PIPE_FUNC_GEQUAL, false, src_bit_size[0], src);
658 break;
659 case nir_op_ilt32:
660 result = icmp32(bld_base, PIPE_FUNC_LESS, false, src_bit_size[0], src);
661 break;
662 case nir_op_imax:
663 result = lp_build_max(get_int_bld(bld_base, false, src_bit_size[0]), src[0], src[1]);
664 break;
665 case nir_op_imin:
666 result = lp_build_min(get_int_bld(bld_base, false, src_bit_size[0]), src[0], src[1]);
667 break;
668 case nir_op_imul:
669 case nir_op_imul24:
670 result = lp_build_mul(get_int_bld(bld_base, false, src_bit_size[0]),
671 src[0], src[1]);
672 break;
673 case nir_op_imul_high: {
674 LLVMValueRef hi_bits;
675 lp_build_mul_32_lohi(&bld_base->int_bld, src[0], src[1], &hi_bits);
676 result = hi_bits;
677 break;
678 }
679 case nir_op_ine32:
680 result = icmp32(bld_base, PIPE_FUNC_NOTEQUAL, false, src_bit_size[0], src);
681 break;
682 case nir_op_ineg:
683 result = lp_build_negate(get_int_bld(bld_base, false, src_bit_size[0]), src[0]);
684 break;
685 case nir_op_inot:
686 result = lp_build_not(get_int_bld(bld_base, false, src_bit_size[0]), src[0]);
687 break;
688 case nir_op_ior:
689 result = lp_build_or(get_int_bld(bld_base, false, src_bit_size[0]),
690 src[0], src[1]);
691 break;
692 case nir_op_irem:
693 result = do_int_mod(bld_base, false, src_bit_size[0], src[0], src[1]);
694 break;
695 case nir_op_ishl: {
696 struct lp_build_context *uint_bld = get_int_bld(bld_base, true, src_bit_size[0]);
697 struct lp_build_context *int_bld = get_int_bld(bld_base, false, src_bit_size[0]);
698 if (src_bit_size[0] == 64)
699 src[1] = LLVMBuildZExt(builder, src[1], uint_bld->vec_type, "");
700 if (src_bit_size[0] < 32)
701 src[1] = LLVMBuildTrunc(builder, src[1], uint_bld->vec_type, "");
702 src[1] = lp_build_and(uint_bld, src[1], lp_build_const_int_vec(gallivm, uint_bld->type, (src_bit_size[0] - 1)));
703 result = lp_build_shl(int_bld, src[0], src[1]);
704 break;
705 }
706 case nir_op_ishr: {
707 struct lp_build_context *uint_bld = get_int_bld(bld_base, true, src_bit_size[0]);
708 struct lp_build_context *int_bld = get_int_bld(bld_base, false, src_bit_size[0]);
709 if (src_bit_size[0] == 64)
710 src[1] = LLVMBuildZExt(builder, src[1], uint_bld->vec_type, "");
711 if (src_bit_size[0] < 32)
712 src[1] = LLVMBuildTrunc(builder, src[1], uint_bld->vec_type, "");
713 src[1] = lp_build_and(uint_bld, src[1], lp_build_const_int_vec(gallivm, uint_bld->type, (src_bit_size[0] - 1)));
714 result = lp_build_shr(int_bld, src[0], src[1]);
715 break;
716 }
717 case nir_op_isign:
718 result = lp_build_sgn(get_int_bld(bld_base, false, src_bit_size[0]), src[0]);
719 break;
720 case nir_op_isub:
721 result = lp_build_sub(get_int_bld(bld_base, false, src_bit_size[0]),
722 src[0], src[1]);
723 break;
724 case nir_op_ixor:
725 result = lp_build_xor(get_int_bld(bld_base, false, src_bit_size[0]),
726 src[0], src[1]);
727 break;
728 case nir_op_mov:
729 result = src[0];
730 break;
731 case nir_op_unpack_64_2x32_split_x:
732 result = split_64bit(bld_base, src[0], false);
733 break;
734 case nir_op_unpack_64_2x32_split_y:
735 result = split_64bit(bld_base, src[0], true);
736 break;
737
738 case nir_op_pack_64_2x32_split: {
739 LLVMValueRef tmp = merge_64bit(bld_base, src[0], src[1]);
740 result = LLVMBuildBitCast(builder, tmp, bld_base->dbl_bld.vec_type, "");
741 break;
742 }
743 case nir_op_u2f32:
744 result = LLVMBuildUIToFP(builder, src[0], bld_base->base.vec_type, "");
745 break;
746 case nir_op_u2f64:
747 result = LLVMBuildUIToFP(builder, src[0], bld_base->dbl_bld.vec_type, "");
748 break;
749 case nir_op_u2u8:
750 result = LLVMBuildTrunc(builder, src[0], bld_base->uint8_bld.vec_type, "");
751 break;
752 case nir_op_u2u16:
753 if (src_bit_size[0] < 16)
754 result = LLVMBuildZExt(builder, src[0], bld_base->uint16_bld.vec_type, "");
755 else
756 result = LLVMBuildTrunc(builder, src[0], bld_base->uint16_bld.vec_type, "");
757 break;
758 case nir_op_u2u32:
759 if (src_bit_size[0] < 32)
760 result = LLVMBuildZExt(builder, src[0], bld_base->uint_bld.vec_type, "");
761 else
762 result = LLVMBuildTrunc(builder, src[0], bld_base->uint_bld.vec_type, "");
763 break;
764 case nir_op_u2u64:
765 result = LLVMBuildZExt(builder, src[0], bld_base->uint64_bld.vec_type, "");
766 break;
767 case nir_op_udiv:
768 result = do_int_divide(bld_base, true, src_bit_size[0], src[0], src[1]);
769 break;
770 case nir_op_ufind_msb: {
771 struct lp_build_context *uint_bld = get_int_bld(bld_base, true, src_bit_size[0]);
772 result = lp_build_ctlz(uint_bld, src[0]);
773 result = lp_build_sub(uint_bld, lp_build_const_int_vec(gallivm, uint_bld->type, src_bit_size[0] - 1), result);
774 break;
775 }
776 case nir_op_uge32:
777 result = icmp32(bld_base, PIPE_FUNC_GEQUAL, true, src_bit_size[0], src);
778 break;
779 case nir_op_ult32:
780 result = icmp32(bld_base, PIPE_FUNC_LESS, true, src_bit_size[0], src);
781 break;
782 case nir_op_umax:
783 result = lp_build_max(get_int_bld(bld_base, true, src_bit_size[0]), src[0], src[1]);
784 break;
785 case nir_op_umin:
786 result = lp_build_min(get_int_bld(bld_base, true, src_bit_size[0]), src[0], src[1]);
787 break;
788 case nir_op_umod:
789 result = do_int_mod(bld_base, true, src_bit_size[0], src[0], src[1]);
790 break;
791 case nir_op_umul_high: {
792 LLVMValueRef hi_bits;
793 lp_build_mul_32_lohi(&bld_base->uint_bld, src[0], src[1], &hi_bits);
794 result = hi_bits;
795 break;
796 }
797 case nir_op_ushr: {
798 struct lp_build_context *uint_bld = get_int_bld(bld_base, true, src_bit_size[0]);
799 if (src_bit_size[0] == 64)
800 src[1] = LLVMBuildZExt(builder, src[1], uint_bld->vec_type, "");
801 if (src_bit_size[0] < 32)
802 src[1] = LLVMBuildTrunc(builder, src[1], uint_bld->vec_type, "");
803 src[1] = lp_build_and(uint_bld, src[1], lp_build_const_int_vec(gallivm, uint_bld->type, (src_bit_size[0] - 1)));
804 result = lp_build_shr(uint_bld, src[0], src[1]);
805 break;
806 }
807 default:
808 assert(0);
809 break;
810 }
811 return result;
812 }
813
814 static void visit_alu(struct lp_build_nir_context *bld_base, const nir_alu_instr *instr)
815 {
816 struct gallivm_state *gallivm = bld_base->base.gallivm;
817 LLVMValueRef src[NIR_MAX_VEC_COMPONENTS];
818 unsigned src_bit_size[NIR_MAX_VEC_COMPONENTS];
819 unsigned num_components = nir_dest_num_components(instr->dest.dest);
820 unsigned src_components;
821 switch (instr->op) {
822 case nir_op_vec2:
823 case nir_op_vec3:
824 case nir_op_vec4:
825 case nir_op_vec8:
826 case nir_op_vec16:
827 src_components = 1;
828 break;
829 case nir_op_pack_half_2x16:
830 src_components = 2;
831 break;
832 case nir_op_unpack_half_2x16:
833 src_components = 1;
834 break;
835 case nir_op_cube_face_coord:
836 case nir_op_cube_face_index:
837 src_components = 3;
838 break;
839 default:
840 src_components = num_components;
841 break;
842 }
843 for (unsigned i = 0; i < nir_op_infos[instr->op].num_inputs; i++) {
844 src[i] = get_alu_src(bld_base, instr->src[i], src_components);
845 src_bit_size[i] = nir_src_bit_size(instr->src[i].src);
846 }
847
848 LLVMValueRef result[NIR_MAX_VEC_COMPONENTS];
849 if (instr->op == nir_op_vec4 || instr->op == nir_op_vec3 || instr->op == nir_op_vec2 || instr->op == nir_op_vec8 || instr->op == nir_op_vec16) {
850 for (unsigned i = 0; i < nir_op_infos[instr->op].num_inputs; i++) {
851 result[i] = cast_type(bld_base, src[i], nir_op_infos[instr->op].input_types[i], src_bit_size[i]);
852 }
853 } else {
854 for (unsigned c = 0; c < num_components; c++) {
855 LLVMValueRef src_chan[NIR_MAX_VEC_COMPONENTS];
856
857 for (unsigned i = 0; i < nir_op_infos[instr->op].num_inputs; i++) {
858 if (num_components > 1) {
859 src_chan[i] = LLVMBuildExtractValue(gallivm->builder,
860 src[i], c, "");
861 } else
862 src_chan[i] = src[i];
863 src_chan[i] = cast_type(bld_base, src_chan[i], nir_op_infos[instr->op].input_types[i], src_bit_size[i]);
864 }
865 result[c] = do_alu_action(bld_base, instr->op, src_bit_size, src_chan);
866 result[c] = cast_type(bld_base, result[c], nir_op_infos[instr->op].output_type, nir_dest_bit_size(instr->dest.dest));
867 }
868 }
869 assign_alu_dest(bld_base, &instr->dest, result);
870 }
871
872 static void visit_load_const(struct lp_build_nir_context *bld_base,
873 const nir_load_const_instr *instr)
874 {
875 LLVMValueRef result[NIR_MAX_VEC_COMPONENTS];
876 struct lp_build_context *int_bld = get_int_bld(bld_base, true, instr->def.bit_size);
877 for (unsigned i = 0; i < instr->def.num_components; i++)
878 result[i] = lp_build_const_int_vec(bld_base->base.gallivm, int_bld->type, instr->def.bit_size == 32 ? instr->value[i].u32 : instr->value[i].u64);
879 assign_ssa_dest(bld_base, &instr->def, result);
880 }
881
882 static void
883 get_deref_offset(struct lp_build_nir_context *bld_base, nir_deref_instr *instr,
884 bool vs_in, unsigned *vertex_index_out,
885 LLVMValueRef *vertex_index_ref,
886 unsigned *const_out, LLVMValueRef *indir_out)
887 {
888 LLVMBuilderRef builder = bld_base->base.gallivm->builder;
889 nir_variable *var = nir_deref_instr_get_variable(instr);
890 nir_deref_path path;
891 unsigned idx_lvl = 1;
892
893 nir_deref_path_init(&path, instr, NULL);
894
895 if (vertex_index_out != NULL || vertex_index_ref != NULL) {
896 if (vertex_index_ref) {
897 *vertex_index_ref = get_src(bld_base, path.path[idx_lvl]->arr.index);
898 if (vertex_index_out)
899 *vertex_index_out = 0;
900 } else {
901 *vertex_index_out = nir_src_as_uint(path.path[idx_lvl]->arr.index);
902 }
903 ++idx_lvl;
904 }
905
906 uint32_t const_offset = 0;
907 LLVMValueRef offset = NULL;
908
909 if (var->data.compact) {
910 assert(instr->deref_type == nir_deref_type_array);
911 const_offset = nir_src_as_uint(instr->arr.index);
912 goto out;
913 }
914
915 for (; path.path[idx_lvl]; ++idx_lvl) {
916 const struct glsl_type *parent_type = path.path[idx_lvl - 1]->type;
917 if (path.path[idx_lvl]->deref_type == nir_deref_type_struct) {
918 unsigned index = path.path[idx_lvl]->strct.index;
919
920 for (unsigned i = 0; i < index; i++) {
921 const struct glsl_type *ft = glsl_get_struct_field(parent_type, i);
922 const_offset += glsl_count_attribute_slots(ft, vs_in);
923 }
924 } else if(path.path[idx_lvl]->deref_type == nir_deref_type_array) {
925 unsigned size = glsl_count_attribute_slots(path.path[idx_lvl]->type, vs_in);
926 if (nir_src_is_const(path.path[idx_lvl]->arr.index)) {
927 const_offset += nir_src_comp_as_int(path.path[idx_lvl]->arr.index, 0) * size;
928 } else {
929 LLVMValueRef idx_src = get_src(bld_base, path.path[idx_lvl]->arr.index);
930 idx_src = cast_type(bld_base, idx_src, nir_type_uint, 32);
931 LLVMValueRef array_off = lp_build_mul(&bld_base->uint_bld, lp_build_const_int_vec(bld_base->base.gallivm, bld_base->base.type, size),
932 idx_src);
933 if (offset)
934 offset = lp_build_add(&bld_base->uint_bld, offset, array_off);
935 else
936 offset = array_off;
937 }
938 } else
939 unreachable("Uhandled deref type in get_deref_instr_offset");
940 }
941
942 out:
943 nir_deref_path_finish(&path);
944
945 if (const_offset && offset)
946 offset = LLVMBuildAdd(builder, offset,
947 lp_build_const_int_vec(bld_base->base.gallivm, bld_base->uint_bld.type, const_offset),
948 "");
949 *const_out = const_offset;
950 *indir_out = offset;
951 }
952
953 static void visit_load_var(struct lp_build_nir_context *bld_base,
954 nir_intrinsic_instr *instr,
955 LLVMValueRef result[NIR_MAX_VEC_COMPONENTS])
956 {
957 nir_deref_instr *deref = nir_instr_as_deref(instr->src[0].ssa->parent_instr);
958 nir_variable *var = nir_deref_instr_get_variable(deref);
959 nir_variable_mode mode = deref->mode;
960 unsigned const_index;
961 LLVMValueRef indir_index;
962 LLVMValueRef indir_vertex_index = NULL;
963 unsigned vertex_index = 0;
964 unsigned nc = nir_dest_num_components(instr->dest);
965 unsigned bit_size = nir_dest_bit_size(instr->dest);
966 if (var) {
967 bool vs_in = bld_base->shader->info.stage == MESA_SHADER_VERTEX &&
968 var->data.mode == nir_var_shader_in;
969 bool gs_in = bld_base->shader->info.stage == MESA_SHADER_GEOMETRY &&
970 var->data.mode == nir_var_shader_in;
971 bool tcs_in = bld_base->shader->info.stage == MESA_SHADER_TESS_CTRL &&
972 var->data.mode == nir_var_shader_in;
973 bool tcs_out = bld_base->shader->info.stage == MESA_SHADER_TESS_CTRL &&
974 var->data.mode == nir_var_shader_out && !var->data.patch;
975 bool tes_in = bld_base->shader->info.stage == MESA_SHADER_TESS_EVAL &&
976 var->data.mode == nir_var_shader_in && !var->data.patch;
977
978 mode = var->data.mode;
979
980 get_deref_offset(bld_base, deref, vs_in, gs_in ? &vertex_index : NULL, (tcs_in || tcs_out || tes_in) ? &indir_vertex_index : NULL,
981 &const_index, &indir_index);
982 }
983 bld_base->load_var(bld_base, mode, nc, bit_size, var, vertex_index, indir_vertex_index, const_index, indir_index, result);
984 }
985
986 static void
987 visit_store_var(struct lp_build_nir_context *bld_base,
988 nir_intrinsic_instr *instr)
989 {
990 nir_deref_instr *deref = nir_instr_as_deref(instr->src[0].ssa->parent_instr);
991 nir_variable *var = nir_deref_instr_get_variable(deref);
992 nir_variable_mode mode = deref->mode;
993 int writemask = instr->const_index[0];
994 unsigned bit_size = nir_src_bit_size(instr->src[1]);
995 LLVMValueRef src = get_src(bld_base, instr->src[1]);
996 unsigned const_index = 0;
997 LLVMValueRef indir_index, indir_vertex_index = NULL;
998 if (var) {
999 bool tcs_out = bld_base->shader->info.stage == MESA_SHADER_TESS_CTRL &&
1000 var->data.mode == nir_var_shader_out && !var->data.patch;
1001 get_deref_offset(bld_base, deref, false, NULL, tcs_out ? &indir_vertex_index : NULL,
1002 &const_index, &indir_index);
1003 }
1004 bld_base->store_var(bld_base, mode, instr->num_components, bit_size, var, writemask, indir_vertex_index, const_index, indir_index, src);
1005 }
1006
1007 static void visit_load_ubo(struct lp_build_nir_context *bld_base,
1008 nir_intrinsic_instr *instr,
1009 LLVMValueRef result[NIR_MAX_VEC_COMPONENTS])
1010 {
1011 struct gallivm_state *gallivm = bld_base->base.gallivm;
1012 LLVMBuilderRef builder = gallivm->builder;
1013 LLVMValueRef idx = get_src(bld_base, instr->src[0]);
1014 LLVMValueRef offset = get_src(bld_base, instr->src[1]);
1015
1016 bool offset_is_uniform = nir_src_is_dynamically_uniform(instr->src[1]);
1017 idx = LLVMBuildExtractElement(builder, idx, lp_build_const_int32(gallivm, 0), "");
1018 bld_base->load_ubo(bld_base, nir_dest_num_components(instr->dest), nir_dest_bit_size(instr->dest),
1019 offset_is_uniform, idx, offset, result);
1020 }
1021
1022
1023 static void visit_load_ssbo(struct lp_build_nir_context *bld_base,
1024 nir_intrinsic_instr *instr,
1025 LLVMValueRef result[NIR_MAX_VEC_COMPONENTS])
1026 {
1027 LLVMValueRef idx = get_src(bld_base, instr->src[0]);
1028 LLVMValueRef offset = get_src(bld_base, instr->src[1]);
1029 bld_base->load_mem(bld_base, nir_dest_num_components(instr->dest), nir_dest_bit_size(instr->dest),
1030 idx, offset, result);
1031 }
1032
1033 static void visit_store_ssbo(struct lp_build_nir_context *bld_base,
1034 nir_intrinsic_instr *instr)
1035 {
1036 LLVMValueRef val = get_src(bld_base, instr->src[0]);
1037 LLVMValueRef idx = get_src(bld_base, instr->src[1]);
1038 LLVMValueRef offset = get_src(bld_base, instr->src[2]);
1039 int writemask = instr->const_index[0];
1040 int nc = nir_src_num_components(instr->src[0]);
1041 int bitsize = nir_src_bit_size(instr->src[0]);
1042 bld_base->store_mem(bld_base, writemask, nc, bitsize, idx, offset, val);
1043 }
1044
1045 static void visit_get_buffer_size(struct lp_build_nir_context *bld_base,
1046 nir_intrinsic_instr *instr,
1047 LLVMValueRef result[NIR_MAX_VEC_COMPONENTS])
1048 {
1049 LLVMValueRef idx = get_src(bld_base, instr->src[0]);
1050 result[0] = bld_base->get_buffer_size(bld_base, idx);
1051 }
1052
1053 static void visit_ssbo_atomic(struct lp_build_nir_context *bld_base,
1054 nir_intrinsic_instr *instr,
1055 LLVMValueRef result[NIR_MAX_VEC_COMPONENTS])
1056 {
1057 LLVMValueRef idx = get_src(bld_base, instr->src[0]);
1058 LLVMValueRef offset = get_src(bld_base, instr->src[1]);
1059 LLVMValueRef val = get_src(bld_base, instr->src[2]);
1060 LLVMValueRef val2 = NULL;
1061 if (instr->intrinsic == nir_intrinsic_ssbo_atomic_comp_swap)
1062 val2 = get_src(bld_base, instr->src[3]);
1063
1064 bld_base->atomic_mem(bld_base, instr->intrinsic, idx, offset, val, val2, &result[0]);
1065
1066 }
1067
1068 static void visit_load_image(struct lp_build_nir_context *bld_base,
1069 nir_intrinsic_instr *instr,
1070 LLVMValueRef result[NIR_MAX_VEC_COMPONENTS])
1071 {
1072 struct gallivm_state *gallivm = bld_base->base.gallivm;
1073 LLVMBuilderRef builder = gallivm->builder;
1074 nir_deref_instr *deref = nir_instr_as_deref(instr->src[0].ssa->parent_instr);
1075 nir_variable *var = nir_deref_instr_get_variable(deref);
1076 LLVMValueRef coord_val = get_src(bld_base, instr->src[1]);
1077 LLVMValueRef coords[5];
1078 struct lp_img_params params;
1079 const struct glsl_type *type = glsl_without_array(var->type);
1080 unsigned const_index;
1081 LLVMValueRef indir_index;
1082 get_deref_offset(bld_base, deref, false, NULL, NULL,
1083 &const_index, &indir_index);
1084
1085 memset(&params, 0, sizeof(params));
1086 params.target = glsl_sampler_to_pipe(glsl_get_sampler_dim(type), glsl_sampler_type_is_array(type));
1087 for (unsigned i = 0; i < 4; i++)
1088 coords[i] = LLVMBuildExtractValue(builder, coord_val, i, "");
1089 if (params.target == PIPE_TEXTURE_1D_ARRAY)
1090 coords[2] = coords[1];
1091
1092 params.coords = coords;
1093 params.outdata = result;
1094 params.img_op = LP_IMG_LOAD;
1095 if (glsl_get_sampler_dim(type) == GLSL_SAMPLER_DIM_MS)
1096 params.ms_index = get_src(bld_base, instr->src[2]);
1097 params.image_index = var->data.binding + (indir_index ? 0 : const_index);
1098 params.image_index_offset = indir_index;
1099 bld_base->image_op(bld_base, &params);
1100 }
1101
1102 static void visit_store_image(struct lp_build_nir_context *bld_base,
1103 nir_intrinsic_instr *instr)
1104 {
1105 struct gallivm_state *gallivm = bld_base->base.gallivm;
1106 LLVMBuilderRef builder = gallivm->builder;
1107 nir_deref_instr *deref = nir_instr_as_deref(instr->src[0].ssa->parent_instr);
1108 nir_variable *var = nir_deref_instr_get_variable(deref);
1109 LLVMValueRef coord_val = get_src(bld_base, instr->src[1]);
1110 LLVMValueRef in_val = get_src(bld_base, instr->src[3]);
1111 LLVMValueRef coords[5];
1112 struct lp_img_params params;
1113 const struct glsl_type *type = glsl_without_array(var->type);
1114 unsigned const_index;
1115 LLVMValueRef indir_index;
1116 get_deref_offset(bld_base, deref, false, NULL, NULL,
1117 &const_index, &indir_index);
1118
1119 memset(&params, 0, sizeof(params));
1120 params.target = glsl_sampler_to_pipe(glsl_get_sampler_dim(type), glsl_sampler_type_is_array(type));
1121 for (unsigned i = 0; i < 4; i++)
1122 coords[i] = LLVMBuildExtractValue(builder, coord_val, i, "");
1123 if (params.target == PIPE_TEXTURE_1D_ARRAY)
1124 coords[2] = coords[1];
1125 params.coords = coords;
1126
1127 for (unsigned i = 0; i < 4; i++) {
1128 params.indata[i] = LLVMBuildExtractValue(builder, in_val, i, "");
1129 params.indata[i] = LLVMBuildBitCast(builder, params.indata[i], bld_base->base.vec_type, "");
1130 }
1131 if (glsl_get_sampler_dim(type) == GLSL_SAMPLER_DIM_MS)
1132 params.ms_index = get_src(bld_base, instr->src[2]);
1133 params.img_op = LP_IMG_STORE;
1134 params.image_index = var->data.binding + (indir_index ? 0 : const_index);
1135 params.image_index_offset = indir_index;
1136
1137 if (params.target == PIPE_TEXTURE_1D_ARRAY)
1138 coords[2] = coords[1];
1139 bld_base->image_op(bld_base, &params);
1140 }
1141
1142 static void visit_atomic_image(struct lp_build_nir_context *bld_base,
1143 nir_intrinsic_instr *instr,
1144 LLVMValueRef result[NIR_MAX_VEC_COMPONENTS])
1145 {
1146 struct gallivm_state *gallivm = bld_base->base.gallivm;
1147 LLVMBuilderRef builder = gallivm->builder;
1148 nir_deref_instr *deref = nir_instr_as_deref(instr->src[0].ssa->parent_instr);
1149 nir_variable *var = nir_deref_instr_get_variable(deref);
1150 struct lp_img_params params;
1151 LLVMValueRef coord_val = get_src(bld_base, instr->src[1]);
1152 LLVMValueRef in_val = get_src(bld_base, instr->src[3]);
1153 LLVMValueRef coords[5];
1154 const struct glsl_type *type = glsl_without_array(var->type);
1155 unsigned const_index;
1156 LLVMValueRef indir_index;
1157 get_deref_offset(bld_base, deref, false, NULL, NULL,
1158 &const_index, &indir_index);
1159
1160 memset(&params, 0, sizeof(params));
1161
1162 switch (instr->intrinsic) {
1163 case nir_intrinsic_image_deref_atomic_add:
1164 params.op = LLVMAtomicRMWBinOpAdd;
1165 break;
1166 case nir_intrinsic_image_deref_atomic_exchange:
1167 params.op = LLVMAtomicRMWBinOpXchg;
1168 break;
1169 case nir_intrinsic_image_deref_atomic_and:
1170 params.op = LLVMAtomicRMWBinOpAnd;
1171 break;
1172 case nir_intrinsic_image_deref_atomic_or:
1173 params.op = LLVMAtomicRMWBinOpOr;
1174 break;
1175 case nir_intrinsic_image_deref_atomic_xor:
1176 params.op = LLVMAtomicRMWBinOpXor;
1177 break;
1178 case nir_intrinsic_image_deref_atomic_umin:
1179 params.op = LLVMAtomicRMWBinOpUMin;
1180 break;
1181 case nir_intrinsic_image_deref_atomic_umax:
1182 params.op = LLVMAtomicRMWBinOpUMax;
1183 break;
1184 case nir_intrinsic_image_deref_atomic_imin:
1185 params.op = LLVMAtomicRMWBinOpMin;
1186 break;
1187 case nir_intrinsic_image_deref_atomic_imax:
1188 params.op = LLVMAtomicRMWBinOpMax;
1189 break;
1190 default:
1191 break;
1192 }
1193
1194 params.target = glsl_sampler_to_pipe(glsl_get_sampler_dim(type), glsl_sampler_type_is_array(type));
1195 for (unsigned i = 0; i < 4; i++)
1196 coords[i] = LLVMBuildExtractValue(builder, coord_val, i, "");
1197 if (params.target == PIPE_TEXTURE_1D_ARRAY)
1198 coords[2] = coords[1];
1199 params.coords = coords;
1200 if (glsl_get_sampler_dim(type) == GLSL_SAMPLER_DIM_MS)
1201 params.ms_index = get_src(bld_base, instr->src[2]);
1202 if (instr->intrinsic == nir_intrinsic_image_deref_atomic_comp_swap) {
1203 LLVMValueRef cas_val = get_src(bld_base, instr->src[4]);
1204 params.indata[0] = in_val;
1205 params.indata2[0] = cas_val;
1206 } else
1207 params.indata[0] = in_val;
1208
1209 params.outdata = result;
1210 params.img_op = (instr->intrinsic == nir_intrinsic_image_deref_atomic_comp_swap) ? LP_IMG_ATOMIC_CAS : LP_IMG_ATOMIC;
1211 params.image_index = var->data.binding + (indir_index ? 0 : const_index);
1212 params.image_index_offset = indir_index;
1213
1214 bld_base->image_op(bld_base, &params);
1215 }
1216
1217
1218 static void visit_image_size(struct lp_build_nir_context *bld_base,
1219 nir_intrinsic_instr *instr,
1220 LLVMValueRef result[NIR_MAX_VEC_COMPONENTS])
1221 {
1222 nir_deref_instr *deref = nir_instr_as_deref(instr->src[0].ssa->parent_instr);
1223 nir_variable *var = nir_deref_instr_get_variable(deref);
1224 struct lp_sampler_size_query_params params = { 0 };
1225 unsigned const_index;
1226 LLVMValueRef indir_index;
1227 const struct glsl_type *type = glsl_without_array(var->type);
1228 get_deref_offset(bld_base, deref, false, NULL, NULL,
1229 &const_index, &indir_index);
1230 params.texture_unit = var->data.binding + (indir_index ? 0 : const_index);
1231 params.texture_unit_offset = indir_index;
1232 params.target = glsl_sampler_to_pipe(glsl_get_sampler_dim(type), glsl_sampler_type_is_array(type));
1233 params.sizes_out = result;
1234
1235 bld_base->image_size(bld_base, &params);
1236 }
1237
1238 static void visit_image_samples(struct lp_build_nir_context *bld_base,
1239 nir_intrinsic_instr *instr,
1240 LLVMValueRef result[NIR_MAX_VEC_COMPONENTS])
1241 {
1242 nir_deref_instr *deref = nir_instr_as_deref(instr->src[0].ssa->parent_instr);
1243 nir_variable *var = nir_deref_instr_get_variable(deref);
1244 struct lp_sampler_size_query_params params = { 0 };
1245 unsigned const_index;
1246 LLVMValueRef indir_index;
1247 const struct glsl_type *type = glsl_without_array(var->type);
1248 get_deref_offset(bld_base, deref, false, NULL, NULL,
1249 &const_index, &indir_index);
1250
1251 params.texture_unit = var->data.binding + (indir_index ? 0 : const_index);
1252 params.texture_unit_offset = indir_index;
1253 params.target = glsl_sampler_to_pipe(glsl_get_sampler_dim(type), glsl_sampler_type_is_array(type));
1254 params.sizes_out = result;
1255 params.samples_only = true;
1256
1257 bld_base->image_size(bld_base, &params);
1258 }
1259
1260 static void visit_shared_load(struct lp_build_nir_context *bld_base,
1261 nir_intrinsic_instr *instr,
1262 LLVMValueRef result[NIR_MAX_VEC_COMPONENTS])
1263 {
1264 LLVMValueRef offset = get_src(bld_base, instr->src[0]);
1265 bld_base->load_mem(bld_base, nir_dest_num_components(instr->dest), nir_dest_bit_size(instr->dest),
1266 NULL, offset, result);
1267 }
1268
1269 static void visit_shared_store(struct lp_build_nir_context *bld_base,
1270 nir_intrinsic_instr *instr)
1271 {
1272 LLVMValueRef val = get_src(bld_base, instr->src[0]);
1273 LLVMValueRef offset = get_src(bld_base, instr->src[1]);
1274 int writemask = instr->const_index[1];
1275 int nc = nir_src_num_components(instr->src[0]);
1276 int bitsize = nir_src_bit_size(instr->src[0]);
1277 bld_base->store_mem(bld_base, writemask, nc, bitsize, NULL, offset, val);
1278 }
1279
1280 static void visit_shared_atomic(struct lp_build_nir_context *bld_base,
1281 nir_intrinsic_instr *instr,
1282 LLVMValueRef result[NIR_MAX_VEC_COMPONENTS])
1283 {
1284 LLVMValueRef offset = get_src(bld_base, instr->src[0]);
1285 LLVMValueRef val = get_src(bld_base, instr->src[1]);
1286 LLVMValueRef val2 = NULL;
1287 if (instr->intrinsic == nir_intrinsic_shared_atomic_comp_swap)
1288 val2 = get_src(bld_base, instr->src[2]);
1289
1290 bld_base->atomic_mem(bld_base, instr->intrinsic, NULL, offset, val, val2, &result[0]);
1291
1292 }
1293
1294 static void visit_barrier(struct lp_build_nir_context *bld_base)
1295 {
1296 bld_base->barrier(bld_base);
1297 }
1298
1299 static void visit_discard(struct lp_build_nir_context *bld_base,
1300 nir_intrinsic_instr *instr)
1301 {
1302 LLVMValueRef cond = NULL;
1303 if (instr->intrinsic == nir_intrinsic_discard_if) {
1304 cond = get_src(bld_base, instr->src[0]);
1305 cond = cast_type(bld_base, cond, nir_type_int, 32);
1306 }
1307 bld_base->discard(bld_base, cond);
1308 }
1309
1310 static void visit_load_kernel_input(struct lp_build_nir_context *bld_base,
1311 nir_intrinsic_instr *instr, LLVMValueRef result[NIR_MAX_VEC_COMPONENTS])
1312 {
1313 LLVMValueRef offset = get_src(bld_base, instr->src[0]);
1314
1315 bool offset_is_uniform = nir_src_is_dynamically_uniform(instr->src[0]);
1316 bld_base->load_kernel_arg(bld_base, nir_dest_num_components(instr->dest), nir_dest_bit_size(instr->dest),
1317 nir_src_bit_size(instr->src[0]),
1318 offset_is_uniform, offset, result);
1319 }
1320
1321 static void visit_load_global(struct lp_build_nir_context *bld_base,
1322 nir_intrinsic_instr *instr, LLVMValueRef result[NIR_MAX_VEC_COMPONENTS])
1323 {
1324 LLVMValueRef addr = get_src(bld_base, instr->src[0]);
1325 bld_base->load_global(bld_base, nir_dest_num_components(instr->dest), nir_dest_bit_size(instr->dest),
1326 nir_src_bit_size(instr->src[0]),
1327 addr, result);
1328 }
1329
1330 static void visit_store_global(struct lp_build_nir_context *bld_base,
1331 nir_intrinsic_instr *instr)
1332 {
1333 LLVMValueRef val = get_src(bld_base, instr->src[0]);
1334 int nc = nir_src_num_components(instr->src[0]);
1335 int bitsize = nir_src_bit_size(instr->src[0]);
1336 LLVMValueRef addr = get_src(bld_base, instr->src[1]);
1337 int addr_bitsize = nir_src_bit_size(instr->src[1]);
1338 int writemask = instr->const_index[0];
1339 bld_base->store_global(bld_base, writemask, nc, bitsize, addr_bitsize, addr, val);
1340 }
1341
1342 static void visit_global_atomic(struct lp_build_nir_context *bld_base,
1343 nir_intrinsic_instr *instr,
1344 LLVMValueRef result[NIR_MAX_VEC_COMPONENTS])
1345 {
1346 LLVMValueRef addr = get_src(bld_base, instr->src[0]);
1347 LLVMValueRef val = get_src(bld_base, instr->src[1]);
1348 LLVMValueRef val2 = NULL;
1349 int addr_bitsize = nir_src_bit_size(instr->src[0]);
1350 if (instr->intrinsic == nir_intrinsic_global_atomic_comp_swap)
1351 val2 = get_src(bld_base, instr->src[2]);
1352
1353 bld_base->atomic_global(bld_base, instr->intrinsic, addr_bitsize, addr, val, val2, &result[0]);
1354 }
1355
1356 static void visit_interp(struct lp_build_nir_context *bld_base,
1357 nir_intrinsic_instr *instr,
1358 LLVMValueRef result[NIR_MAX_VEC_COMPONENTS])
1359 {
1360 struct gallivm_state *gallivm = bld_base->base.gallivm;
1361 LLVMBuilderRef builder = gallivm->builder;
1362 nir_deref_instr *deref = nir_instr_as_deref(instr->src[0].ssa->parent_instr);
1363 unsigned num_components = nir_dest_num_components(instr->dest);
1364 nir_variable *var = nir_deref_instr_get_variable(deref);
1365 unsigned const_index;
1366 LLVMValueRef indir_index;
1367 LLVMValueRef offsets[2] = { NULL, NULL };
1368 get_deref_offset(bld_base, deref, false, NULL, NULL,
1369 &const_index, &indir_index);
1370 bool centroid = instr->intrinsic == nir_intrinsic_interp_deref_at_centroid;
1371 bool sample = false;
1372 if (instr->intrinsic == nir_intrinsic_interp_deref_at_offset) {
1373 for (unsigned i = 0; i < 2; i++) {
1374 offsets[i] = LLVMBuildExtractValue(builder, get_src(bld_base, instr->src[1]), i, "");
1375 offsets[i] = cast_type(bld_base, offsets[i], nir_type_float, 32);
1376 }
1377 } else if (instr->intrinsic == nir_intrinsic_interp_deref_at_sample) {
1378 offsets[0] = get_src(bld_base, instr->src[1]);
1379 offsets[0] = cast_type(bld_base, offsets[0], nir_type_int, 32);
1380 sample = true;
1381 }
1382 bld_base->interp_at(bld_base, num_components, var, centroid, sample, const_index, indir_index, offsets, result);
1383 }
1384
1385 static void visit_intrinsic(struct lp_build_nir_context *bld_base,
1386 nir_intrinsic_instr *instr)
1387 {
1388 LLVMValueRef result[NIR_MAX_VEC_COMPONENTS] = {0};
1389 switch (instr->intrinsic) {
1390 case nir_intrinsic_load_deref:
1391 visit_load_var(bld_base, instr, result);
1392 break;
1393 case nir_intrinsic_store_deref:
1394 visit_store_var(bld_base, instr);
1395 break;
1396 case nir_intrinsic_load_ubo:
1397 visit_load_ubo(bld_base, instr, result);
1398 break;
1399 case nir_intrinsic_load_ssbo:
1400 visit_load_ssbo(bld_base, instr, result);
1401 break;
1402 case nir_intrinsic_store_ssbo:
1403 visit_store_ssbo(bld_base, instr);
1404 break;
1405 case nir_intrinsic_get_buffer_size:
1406 visit_get_buffer_size(bld_base, instr, result);
1407 break;
1408 case nir_intrinsic_load_vertex_id:
1409 case nir_intrinsic_load_primitive_id:
1410 case nir_intrinsic_load_instance_id:
1411 case nir_intrinsic_load_base_instance:
1412 case nir_intrinsic_load_base_vertex:
1413 case nir_intrinsic_load_work_group_id:
1414 case nir_intrinsic_load_local_invocation_id:
1415 case nir_intrinsic_load_num_work_groups:
1416 case nir_intrinsic_load_invocation_id:
1417 case nir_intrinsic_load_front_face:
1418 case nir_intrinsic_load_draw_id:
1419 case nir_intrinsic_load_local_group_size:
1420 case nir_intrinsic_load_work_dim:
1421 case nir_intrinsic_load_tess_coord:
1422 case nir_intrinsic_load_tess_level_outer:
1423 case nir_intrinsic_load_tess_level_inner:
1424 case nir_intrinsic_load_patch_vertices_in:
1425 case nir_intrinsic_load_sample_id:
1426 case nir_intrinsic_load_sample_pos:
1427 case nir_intrinsic_load_sample_mask_in:
1428 bld_base->sysval_intrin(bld_base, instr, result);
1429 break;
1430 case nir_intrinsic_load_helper_invocation:
1431 bld_base->helper_invocation(bld_base, &result[0]);
1432 break;
1433 case nir_intrinsic_discard_if:
1434 case nir_intrinsic_discard:
1435 visit_discard(bld_base, instr);
1436 break;
1437 case nir_intrinsic_emit_vertex:
1438 bld_base->emit_vertex(bld_base, nir_intrinsic_stream_id(instr));
1439 break;
1440 case nir_intrinsic_end_primitive:
1441 bld_base->end_primitive(bld_base, nir_intrinsic_stream_id(instr));
1442 break;
1443 case nir_intrinsic_ssbo_atomic_add:
1444 case nir_intrinsic_ssbo_atomic_imin:
1445 case nir_intrinsic_ssbo_atomic_imax:
1446 case nir_intrinsic_ssbo_atomic_umin:
1447 case nir_intrinsic_ssbo_atomic_umax:
1448 case nir_intrinsic_ssbo_atomic_and:
1449 case nir_intrinsic_ssbo_atomic_or:
1450 case nir_intrinsic_ssbo_atomic_xor:
1451 case nir_intrinsic_ssbo_atomic_exchange:
1452 case nir_intrinsic_ssbo_atomic_comp_swap:
1453 visit_ssbo_atomic(bld_base, instr, result);
1454 break;
1455 case nir_intrinsic_image_deref_load:
1456 visit_load_image(bld_base, instr, result);
1457 break;
1458 case nir_intrinsic_image_deref_store:
1459 visit_store_image(bld_base, instr);
1460 break;
1461 case nir_intrinsic_image_deref_atomic_add:
1462 case nir_intrinsic_image_deref_atomic_imin:
1463 case nir_intrinsic_image_deref_atomic_imax:
1464 case nir_intrinsic_image_deref_atomic_umin:
1465 case nir_intrinsic_image_deref_atomic_umax:
1466 case nir_intrinsic_image_deref_atomic_and:
1467 case nir_intrinsic_image_deref_atomic_or:
1468 case nir_intrinsic_image_deref_atomic_xor:
1469 case nir_intrinsic_image_deref_atomic_exchange:
1470 case nir_intrinsic_image_deref_atomic_comp_swap:
1471 visit_atomic_image(bld_base, instr, result);
1472 break;
1473 case nir_intrinsic_image_deref_size:
1474 visit_image_size(bld_base, instr, result);
1475 break;
1476 case nir_intrinsic_image_deref_samples:
1477 visit_image_samples(bld_base, instr, result);
1478 break;
1479 case nir_intrinsic_load_shared:
1480 visit_shared_load(bld_base, instr, result);
1481 break;
1482 case nir_intrinsic_store_shared:
1483 visit_shared_store(bld_base, instr);
1484 break;
1485 case nir_intrinsic_shared_atomic_add:
1486 case nir_intrinsic_shared_atomic_imin:
1487 case nir_intrinsic_shared_atomic_umin:
1488 case nir_intrinsic_shared_atomic_imax:
1489 case nir_intrinsic_shared_atomic_umax:
1490 case nir_intrinsic_shared_atomic_and:
1491 case nir_intrinsic_shared_atomic_or:
1492 case nir_intrinsic_shared_atomic_xor:
1493 case nir_intrinsic_shared_atomic_exchange:
1494 case nir_intrinsic_shared_atomic_comp_swap:
1495 visit_shared_atomic(bld_base, instr, result);
1496 break;
1497 case nir_intrinsic_control_barrier:
1498 visit_barrier(bld_base);
1499 break;
1500 case nir_intrinsic_group_memory_barrier:
1501 case nir_intrinsic_memory_barrier:
1502 case nir_intrinsic_memory_barrier_shared:
1503 case nir_intrinsic_memory_barrier_buffer:
1504 case nir_intrinsic_memory_barrier_image:
1505 case nir_intrinsic_memory_barrier_tcs_patch:
1506 break;
1507 case nir_intrinsic_load_kernel_input:
1508 visit_load_kernel_input(bld_base, instr, result);
1509 break;
1510 case nir_intrinsic_load_global:
1511 visit_load_global(bld_base, instr, result);
1512 break;
1513 case nir_intrinsic_store_global:
1514 visit_store_global(bld_base, instr);
1515 break;
1516 case nir_intrinsic_global_atomic_add:
1517 case nir_intrinsic_global_atomic_imin:
1518 case nir_intrinsic_global_atomic_umin:
1519 case nir_intrinsic_global_atomic_imax:
1520 case nir_intrinsic_global_atomic_umax:
1521 case nir_intrinsic_global_atomic_and:
1522 case nir_intrinsic_global_atomic_or:
1523 case nir_intrinsic_global_atomic_xor:
1524 case nir_intrinsic_global_atomic_exchange:
1525 case nir_intrinsic_global_atomic_comp_swap:
1526 visit_global_atomic(bld_base, instr, result);
1527 break;
1528 case nir_intrinsic_vote_all:
1529 case nir_intrinsic_vote_any:
1530 case nir_intrinsic_vote_ieq:
1531 bld_base->vote(bld_base, cast_type(bld_base, get_src(bld_base, instr->src[0]), nir_type_int, 32), instr, result);
1532 break;
1533 case nir_intrinsic_interp_deref_at_offset:
1534 case nir_intrinsic_interp_deref_at_centroid:
1535 case nir_intrinsic_interp_deref_at_sample:
1536 visit_interp(bld_base, instr, result);
1537 break;
1538 default:
1539 assert(0);
1540 break;
1541 }
1542 if (result[0]) {
1543 assign_dest(bld_base, &instr->dest, result);
1544 }
1545 }
1546
1547 static void visit_txs(struct lp_build_nir_context *bld_base, nir_tex_instr *instr)
1548 {
1549 struct lp_sampler_size_query_params params = { 0 };
1550 LLVMValueRef sizes_out[NIR_MAX_VEC_COMPONENTS];
1551 LLVMValueRef explicit_lod = NULL;
1552 LLVMValueRef texture_unit_offset = NULL;
1553 for (unsigned i = 0; i < instr->num_srcs; i++) {
1554 switch (instr->src[i].src_type) {
1555 case nir_tex_src_lod:
1556 explicit_lod = cast_type(bld_base, get_src(bld_base, instr->src[i].src), nir_type_int, 32);
1557 break;
1558 case nir_tex_src_texture_offset:
1559 texture_unit_offset = get_src(bld_base, instr->src[i].src);
1560 break;
1561 default:
1562 break;
1563 }
1564 }
1565
1566 params.target = glsl_sampler_to_pipe(instr->sampler_dim, instr->is_array);
1567 params.texture_unit = instr->texture_index;
1568 params.explicit_lod = explicit_lod;
1569 params.is_sviewinfo = TRUE;
1570 params.sizes_out = sizes_out;
1571 params.samples_only = (instr->op == nir_texop_texture_samples);
1572 params.texture_unit_offset = texture_unit_offset;
1573
1574 if (instr->op == nir_texop_query_levels)
1575 params.explicit_lod = bld_base->uint_bld.zero;
1576 bld_base->tex_size(bld_base, &params);
1577 assign_dest(bld_base, &instr->dest, &sizes_out[instr->op == nir_texop_query_levels ? 3 : 0]);
1578 }
1579
1580 static enum lp_sampler_lod_property lp_build_nir_lod_property(struct lp_build_nir_context *bld_base,
1581 nir_src lod_src)
1582 {
1583 enum lp_sampler_lod_property lod_property;
1584
1585 if (nir_src_is_dynamically_uniform(lod_src))
1586 lod_property = LP_SAMPLER_LOD_SCALAR;
1587 else if (bld_base->shader->info.stage == MESA_SHADER_FRAGMENT) {
1588 if (gallivm_perf & GALLIVM_PERF_NO_QUAD_LOD)
1589 lod_property = LP_SAMPLER_LOD_PER_ELEMENT;
1590 else
1591 lod_property = LP_SAMPLER_LOD_PER_QUAD;
1592 }
1593 else
1594 lod_property = LP_SAMPLER_LOD_PER_ELEMENT;
1595 return lod_property;
1596 }
1597
1598 static void visit_tex(struct lp_build_nir_context *bld_base, nir_tex_instr *instr)
1599 {
1600 struct gallivm_state *gallivm = bld_base->base.gallivm;
1601 LLVMBuilderRef builder = gallivm->builder;
1602 LLVMValueRef coords[5];
1603 LLVMValueRef offsets[3] = { NULL };
1604 LLVMValueRef explicit_lod = NULL, projector = NULL, ms_index = NULL;
1605 struct lp_sampler_params params;
1606 struct lp_derivatives derivs;
1607 unsigned sample_key = 0;
1608 nir_deref_instr *texture_deref_instr = NULL;
1609 nir_deref_instr *sampler_deref_instr = NULL;
1610 LLVMValueRef texture_unit_offset = NULL;
1611 LLVMValueRef texel[NIR_MAX_VEC_COMPONENTS];
1612 unsigned lod_src = 0;
1613 LLVMValueRef coord_undef = LLVMGetUndef(bld_base->base.int_vec_type);
1614
1615 memset(&params, 0, sizeof(params));
1616 enum lp_sampler_lod_property lod_property = LP_SAMPLER_LOD_SCALAR;
1617
1618 if (instr->op == nir_texop_txs || instr->op == nir_texop_query_levels || instr->op == nir_texop_texture_samples) {
1619 visit_txs(bld_base, instr);
1620 return;
1621 }
1622 if (instr->op == nir_texop_txf || instr->op == nir_texop_txf_ms)
1623 sample_key |= LP_SAMPLER_OP_FETCH << LP_SAMPLER_OP_TYPE_SHIFT;
1624 else if (instr->op == nir_texop_tg4) {
1625 sample_key |= LP_SAMPLER_OP_GATHER << LP_SAMPLER_OP_TYPE_SHIFT;
1626 sample_key |= (instr->component << LP_SAMPLER_GATHER_COMP_SHIFT);
1627 } else if (instr->op == nir_texop_lod)
1628 sample_key |= LP_SAMPLER_OP_LODQ << LP_SAMPLER_OP_TYPE_SHIFT;
1629 for (unsigned i = 0; i < instr->num_srcs; i++) {
1630 switch (instr->src[i].src_type) {
1631 case nir_tex_src_coord: {
1632 LLVMValueRef coord = get_src(bld_base, instr->src[i].src);
1633 if (instr->coord_components == 1)
1634 coords[0] = coord;
1635 else {
1636 for (unsigned chan = 0; chan < instr->coord_components; ++chan)
1637 coords[chan] = LLVMBuildExtractValue(builder, coord,
1638 chan, "");
1639 }
1640 for (unsigned chan = instr->coord_components; chan < 5; chan++)
1641 coords[chan] = coord_undef;
1642
1643 break;
1644 }
1645 case nir_tex_src_texture_deref:
1646 texture_deref_instr = nir_src_as_deref(instr->src[i].src);
1647 break;
1648 case nir_tex_src_sampler_deref:
1649 sampler_deref_instr = nir_src_as_deref(instr->src[i].src);
1650 break;
1651 case nir_tex_src_projector:
1652 projector = lp_build_rcp(&bld_base->base, cast_type(bld_base, get_src(bld_base, instr->src[i].src), nir_type_float, 32));
1653 break;
1654 case nir_tex_src_comparator:
1655 sample_key |= LP_SAMPLER_SHADOW;
1656 coords[4] = get_src(bld_base, instr->src[i].src);
1657 coords[4] = cast_type(bld_base, coords[4], nir_type_float, 32);
1658 break;
1659 case nir_tex_src_bias:
1660 sample_key |= LP_SAMPLER_LOD_BIAS << LP_SAMPLER_LOD_CONTROL_SHIFT;
1661 lod_src = i;
1662 explicit_lod = cast_type(bld_base, get_src(bld_base, instr->src[i].src), nir_type_float, 32);
1663 break;
1664 case nir_tex_src_lod:
1665 sample_key |= LP_SAMPLER_LOD_EXPLICIT << LP_SAMPLER_LOD_CONTROL_SHIFT;
1666 lod_src = i;
1667 if (instr->op == nir_texop_txf)
1668 explicit_lod = cast_type(bld_base, get_src(bld_base, instr->src[i].src), nir_type_int, 32);
1669 else
1670 explicit_lod = cast_type(bld_base, get_src(bld_base, instr->src[i].src), nir_type_float, 32);
1671 break;
1672 case nir_tex_src_ddx: {
1673 int deriv_cnt = instr->coord_components;
1674 if (instr->is_array)
1675 deriv_cnt--;
1676 LLVMValueRef deriv_val = get_src(bld_base, instr->src[i].src);
1677 if (deriv_cnt == 1)
1678 derivs.ddx[0] = deriv_val;
1679 else
1680 for (unsigned chan = 0; chan < deriv_cnt; ++chan)
1681 derivs.ddx[chan] = LLVMBuildExtractValue(builder, deriv_val,
1682 chan, "");
1683 for (unsigned chan = 0; chan < deriv_cnt; ++chan)
1684 derivs.ddx[chan] = cast_type(bld_base, derivs.ddx[chan], nir_type_float, 32);
1685 break;
1686 }
1687 case nir_tex_src_ddy: {
1688 int deriv_cnt = instr->coord_components;
1689 if (instr->is_array)
1690 deriv_cnt--;
1691 LLVMValueRef deriv_val = get_src(bld_base, instr->src[i].src);
1692 if (deriv_cnt == 1)
1693 derivs.ddy[0] = deriv_val;
1694 else
1695 for (unsigned chan = 0; chan < deriv_cnt; ++chan)
1696 derivs.ddy[chan] = LLVMBuildExtractValue(builder, deriv_val,
1697 chan, "");
1698 for (unsigned chan = 0; chan < deriv_cnt; ++chan)
1699 derivs.ddy[chan] = cast_type(bld_base, derivs.ddy[chan], nir_type_float, 32);
1700 break;
1701 }
1702 case nir_tex_src_offset: {
1703 int offset_cnt = instr->coord_components;
1704 if (instr->is_array)
1705 offset_cnt--;
1706 LLVMValueRef offset_val = get_src(bld_base, instr->src[i].src);
1707 sample_key |= LP_SAMPLER_OFFSETS;
1708 if (offset_cnt == 1)
1709 offsets[0] = cast_type(bld_base, offset_val, nir_type_int, 32);
1710 else {
1711 for (unsigned chan = 0; chan < offset_cnt; ++chan) {
1712 offsets[chan] = LLVMBuildExtractValue(builder, offset_val,
1713 chan, "");
1714 offsets[chan] = cast_type(bld_base, offsets[chan], nir_type_int, 32);
1715 }
1716 }
1717 break;
1718 }
1719 case nir_tex_src_ms_index:
1720 sample_key |= LP_SAMPLER_FETCH_MS;
1721 ms_index = cast_type(bld_base, get_src(bld_base, instr->src[i].src), nir_type_int, 32);
1722 break;
1723
1724 case nir_tex_src_texture_offset:
1725 texture_unit_offset = get_src(bld_base, instr->src[i].src);
1726 break;
1727 case nir_tex_src_sampler_offset:
1728 break;
1729 default:
1730 assert(0);
1731 break;
1732 }
1733 }
1734 if (!sampler_deref_instr)
1735 sampler_deref_instr = texture_deref_instr;
1736
1737 if (explicit_lod)
1738 lod_property = lp_build_nir_lod_property(bld_base, instr->src[lod_src].src);
1739
1740 if (instr->op == nir_texop_tex || instr->op == nir_texop_tg4 || instr->op == nir_texop_txb ||
1741 instr->op == nir_texop_txl || instr->op == nir_texop_txd || instr->op == nir_texop_lod)
1742 for (unsigned chan = 0; chan < instr->coord_components; ++chan)
1743 coords[chan] = cast_type(bld_base, coords[chan], nir_type_float, 32);
1744 else if (instr->op == nir_texop_txf || instr->op == nir_texop_txf_ms)
1745 for (unsigned chan = 0; chan < instr->coord_components; ++chan)
1746 coords[chan] = cast_type(bld_base, coords[chan], nir_type_int, 32);
1747
1748 if (instr->is_array && instr->sampler_dim == GLSL_SAMPLER_DIM_1D) {
1749 /* move layer coord for 1d arrays. */
1750 coords[2] = coords[1];
1751 coords[1] = coord_undef;
1752 }
1753
1754 if (projector) {
1755 for (unsigned chan = 0; chan < instr->coord_components; ++chan)
1756 coords[chan] = lp_build_mul(&bld_base->base, coords[chan], projector);
1757 if (sample_key & LP_SAMPLER_SHADOW)
1758 coords[4] = lp_build_mul(&bld_base->base, coords[4], projector);
1759 }
1760
1761 uint32_t base_index = 0;
1762 if (!texture_deref_instr) {
1763 int samp_src_index = nir_tex_instr_src_index(instr, nir_tex_src_sampler_handle);
1764 if (samp_src_index == -1) {
1765 base_index = instr->sampler_index;
1766 }
1767 }
1768
1769 if (instr->op == nir_texop_txd) {
1770 sample_key |= LP_SAMPLER_LOD_DERIVATIVES << LP_SAMPLER_LOD_CONTROL_SHIFT;
1771 params.derivs = &derivs;
1772 if (bld_base->shader->info.stage == MESA_SHADER_FRAGMENT) {
1773 if (gallivm_perf & GALLIVM_PERF_NO_QUAD_LOD)
1774 lod_property = LP_SAMPLER_LOD_PER_ELEMENT;
1775 else
1776 lod_property = LP_SAMPLER_LOD_PER_QUAD;
1777 } else
1778 lod_property = LP_SAMPLER_LOD_PER_ELEMENT;
1779 }
1780
1781 sample_key |= lod_property << LP_SAMPLER_LOD_PROPERTY_SHIFT;
1782 params.sample_key = sample_key;
1783 params.offsets = offsets;
1784 params.texture_index = base_index;
1785 params.texture_index_offset = texture_unit_offset;
1786 params.sampler_index = base_index;
1787 params.coords = coords;
1788 params.texel = texel;
1789 params.lod = explicit_lod;
1790 params.ms_index = ms_index;
1791 bld_base->tex(bld_base, &params);
1792 assign_dest(bld_base, &instr->dest, texel);
1793 }
1794
1795 static void visit_ssa_undef(struct lp_build_nir_context *bld_base,
1796 const nir_ssa_undef_instr *instr)
1797 {
1798 unsigned num_components = instr->def.num_components;
1799 LLVMValueRef undef[NIR_MAX_VEC_COMPONENTS];
1800 struct lp_build_context *undef_bld = get_int_bld(bld_base, true, instr->def.bit_size);
1801 for (unsigned i = 0; i < num_components; i++)
1802 undef[i] = LLVMGetUndef(undef_bld->vec_type);
1803 assign_ssa_dest(bld_base, &instr->def, undef);
1804 }
1805
1806 static void visit_jump(struct lp_build_nir_context *bld_base,
1807 const nir_jump_instr *instr)
1808 {
1809 switch (instr->type) {
1810 case nir_jump_break:
1811 bld_base->break_stmt(bld_base);
1812 break;
1813 case nir_jump_continue:
1814 bld_base->continue_stmt(bld_base);
1815 break;
1816 default:
1817 unreachable("Unknown jump instr\n");
1818 }
1819 }
1820
1821 static void visit_deref(struct lp_build_nir_context *bld_base,
1822 nir_deref_instr *instr)
1823 {
1824 if (instr->mode != nir_var_mem_shared &&
1825 instr->mode != nir_var_mem_global)
1826 return;
1827 LLVMValueRef result = NULL;
1828 switch(instr->deref_type) {
1829 case nir_deref_type_var: {
1830 struct hash_entry *entry = _mesa_hash_table_search(bld_base->vars, instr->var);
1831 result = entry->data;
1832 break;
1833 }
1834 default:
1835 unreachable("Unhandled deref_instr deref type");
1836 }
1837
1838 assign_ssa(bld_base, instr->dest.ssa.index, result);
1839 }
1840
1841 static void visit_block(struct lp_build_nir_context *bld_base, nir_block *block)
1842 {
1843 nir_foreach_instr(instr, block)
1844 {
1845 switch (instr->type) {
1846 case nir_instr_type_alu:
1847 visit_alu(bld_base, nir_instr_as_alu(instr));
1848 break;
1849 case nir_instr_type_load_const:
1850 visit_load_const(bld_base, nir_instr_as_load_const(instr));
1851 break;
1852 case nir_instr_type_intrinsic:
1853 visit_intrinsic(bld_base, nir_instr_as_intrinsic(instr));
1854 break;
1855 case nir_instr_type_tex:
1856 visit_tex(bld_base, nir_instr_as_tex(instr));
1857 break;
1858 case nir_instr_type_phi:
1859 assert(0);
1860 break;
1861 case nir_instr_type_ssa_undef:
1862 visit_ssa_undef(bld_base, nir_instr_as_ssa_undef(instr));
1863 break;
1864 case nir_instr_type_jump:
1865 visit_jump(bld_base, nir_instr_as_jump(instr));
1866 break;
1867 case nir_instr_type_deref:
1868 visit_deref(bld_base, nir_instr_as_deref(instr));
1869 break;
1870 default:
1871 fprintf(stderr, "Unknown NIR instr type: ");
1872 nir_print_instr(instr, stderr);
1873 fprintf(stderr, "\n");
1874 abort();
1875 }
1876 }
1877 }
1878
1879 static void visit_if(struct lp_build_nir_context *bld_base, nir_if *if_stmt)
1880 {
1881 LLVMValueRef cond = get_src(bld_base, if_stmt->condition);
1882
1883 bld_base->if_cond(bld_base, cond);
1884 visit_cf_list(bld_base, &if_stmt->then_list);
1885
1886 if (!exec_list_is_empty(&if_stmt->else_list)) {
1887 bld_base->else_stmt(bld_base);
1888 visit_cf_list(bld_base, &if_stmt->else_list);
1889 }
1890 bld_base->endif_stmt(bld_base);
1891 }
1892
1893 static void visit_loop(struct lp_build_nir_context *bld_base, nir_loop *loop)
1894 {
1895 bld_base->bgnloop(bld_base);
1896 visit_cf_list(bld_base, &loop->body);
1897 bld_base->endloop(bld_base);
1898 }
1899
1900 static void visit_cf_list(struct lp_build_nir_context *bld_base,
1901 struct exec_list *list)
1902 {
1903 foreach_list_typed(nir_cf_node, node, node, list)
1904 {
1905 switch (node->type) {
1906 case nir_cf_node_block:
1907 visit_block(bld_base, nir_cf_node_as_block(node));
1908 break;
1909
1910 case nir_cf_node_if:
1911 visit_if(bld_base, nir_cf_node_as_if(node));
1912 break;
1913
1914 case nir_cf_node_loop:
1915 visit_loop(bld_base, nir_cf_node_as_loop(node));
1916 break;
1917
1918 default:
1919 assert(0);
1920 }
1921 }
1922 }
1923
1924 static void
1925 handle_shader_output_decl(struct lp_build_nir_context *bld_base,
1926 struct nir_shader *nir,
1927 struct nir_variable *variable)
1928 {
1929 bld_base->emit_var_decl(bld_base, variable);
1930 }
1931
1932 /* vector registers are stored as arrays in LLVM side,
1933 so we can use GEP on them, as to do exec mask stores
1934 we need to operate on a single components.
1935 arrays are:
1936 0.x, 1.x, 2.x, 3.x
1937 0.y, 1.y, 2.y, 3.y
1938 ....
1939 */
1940 static LLVMTypeRef get_register_type(struct lp_build_nir_context *bld_base,
1941 nir_register *reg)
1942 {
1943 struct lp_build_context *int_bld = get_int_bld(bld_base, true, reg->bit_size);
1944
1945 LLVMTypeRef type = int_bld->vec_type;
1946 if (reg->num_array_elems)
1947 type = LLVMArrayType(type, reg->num_array_elems);
1948 if (reg->num_components > 1)
1949 type = LLVMArrayType(type, reg->num_components);
1950
1951 return type;
1952 }
1953
1954
1955 bool lp_build_nir_llvm(
1956 struct lp_build_nir_context *bld_base,
1957 struct nir_shader *nir)
1958 {
1959 struct nir_function *func;
1960
1961 nir_convert_from_ssa(nir, true);
1962 nir_lower_locals_to_regs(nir);
1963 nir_remove_dead_derefs(nir);
1964 nir_remove_dead_variables(nir, nir_var_function_temp, NULL);
1965
1966 nir_foreach_variable(variable, &nir->outputs)
1967 handle_shader_output_decl(bld_base, nir, variable);
1968
1969 bld_base->regs = _mesa_hash_table_create(NULL, _mesa_hash_pointer,
1970 _mesa_key_pointer_equal);
1971 bld_base->vars = _mesa_hash_table_create(NULL, _mesa_hash_pointer,
1972 _mesa_key_pointer_equal);
1973
1974 func = (struct nir_function *)exec_list_get_head(&nir->functions);
1975
1976 nir_foreach_register(reg, &func->impl->registers) {
1977 LLVMTypeRef type = get_register_type(bld_base, reg);
1978 LLVMValueRef reg_alloc = lp_build_alloca_undef(bld_base->base.gallivm,
1979 type, "reg");
1980 _mesa_hash_table_insert(bld_base->regs, reg, reg_alloc);
1981 }
1982 nir_index_ssa_defs(func->impl);
1983 bld_base->ssa_defs = calloc(func->impl->ssa_alloc, sizeof(LLVMValueRef));
1984 visit_cf_list(bld_base, &func->impl->body);
1985
1986 free(bld_base->ssa_defs);
1987 ralloc_free(bld_base->vars);
1988 ralloc_free(bld_base->regs);
1989 return true;
1990 }
1991
1992 /* do some basic opts to remove some things we don't want to see. */
1993 void lp_build_opt_nir(struct nir_shader *nir)
1994 {
1995 bool progress;
1996 do {
1997 progress = false;
1998 NIR_PASS_V(nir, nir_opt_constant_folding);
1999 NIR_PASS_V(nir, nir_opt_algebraic);
2000 NIR_PASS_V(nir, nir_lower_pack);
2001
2002 nir_lower_tex_options options = { .lower_tex_without_implicit_lod = true };
2003 NIR_PASS_V(nir, nir_lower_tex, &options);
2004 } while (progress);
2005 nir_lower_bool_to_int32(nir);
2006 }