projects / mesa.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
history | raw | HEAD
gallivm/nir: add texture unit indexing
[mesa.git] / src / gallium / auxiliary / gallivm / lp_bld_nir.c
1 /**************************************************************************
2 *
3 * Copyright 2019 Red Hat.
4 * All Rights Reserved.
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the "Software"),
8 * to deal in the Software without restriction, including without limitation
9 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
10 * and/or sell copies of the Software, and to permit persons to whom the
11 * Software is furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included
14 * in all copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
17 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
22 * SOFTWARE.
23 *
24 **************************************************************************/
25
26 #include "lp_bld_nir.h"
27 #include "lp_bld_arit.h"
28 #include "lp_bld_bitarit.h"
29 #include "lp_bld_const.h"
30 #include "lp_bld_gather.h"
31 #include "lp_bld_logic.h"
32 #include "lp_bld_quad.h"
33 #include "lp_bld_flow.h"
34 #include "lp_bld_struct.h"
35 #include "lp_bld_debug.h"
36 #include "lp_bld_printf.h"
37 #include "nir_deref.h"
38
39 static void visit_cf_list(struct lp_build_nir_context *bld_base,
40 struct exec_list *list);
41
42 static LLVMValueRef cast_type(struct lp_build_nir_context *bld_base, LLVMValueRef val,
43 nir_alu_type alu_type, unsigned bit_size)
44 {
45 LLVMBuilderRef builder = bld_base->base.gallivm->builder;
46 switch (alu_type) {
47 case nir_type_float:
48 switch (bit_size) {
49 case 32:
50 return LLVMBuildBitCast(builder, val, bld_base->base.vec_type, "");
51 case 64:
52 return LLVMBuildBitCast(builder, val, bld_base->dbl_bld.vec_type, "");
53 default:
54 assert(0);
55 break;
56 }
57 break;
58 case nir_type_int:
59 switch (bit_size) {
60 case 8:
61 return LLVMBuildBitCast(builder, val, bld_base->int8_bld.vec_type, "");
62 case 16:
63 return LLVMBuildBitCast(builder, val, bld_base->int16_bld.vec_type, "");
64 case 32:
65 return LLVMBuildBitCast(builder, val, bld_base->int_bld.vec_type, "");
66 case 64:
67 return LLVMBuildBitCast(builder, val, bld_base->int64_bld.vec_type, "");
68 default:
69 assert(0);
70 break;
71 }
72 break;
73 case nir_type_uint:
74 switch (bit_size) {
75 case 8:
76 return LLVMBuildBitCast(builder, val, bld_base->uint8_bld.vec_type, "");
77 case 16:
78 return LLVMBuildBitCast(builder, val, bld_base->uint16_bld.vec_type, "");
79 case 32:
80 return LLVMBuildBitCast(builder, val, bld_base->uint_bld.vec_type, "");
81 case 64:
82 return LLVMBuildBitCast(builder, val, bld_base->uint64_bld.vec_type, "");
83 default:
84 assert(0);
85 break;
86 }
87 break;
88 case nir_type_uint32:
89 return LLVMBuildBitCast(builder, val, bld_base->uint_bld.vec_type, "");
90 default:
91 return val;
92 }
93 return NULL;
94 }
95
96
97 static struct lp_build_context *get_flt_bld(struct lp_build_nir_context *bld_base,
98 unsigned op_bit_size)
99 {
100 if (op_bit_size == 64)
101 return &bld_base->dbl_bld;
102 else
103 return &bld_base->base;
104 }
105
106 static unsigned glsl_sampler_to_pipe(int sampler_dim, bool is_array)
107 {
108 unsigned pipe_target = PIPE_BUFFER;
109 switch (sampler_dim) {
110 case GLSL_SAMPLER_DIM_1D:
111 pipe_target = is_array ? PIPE_TEXTURE_1D_ARRAY : PIPE_TEXTURE_1D;
112 break;
113 case GLSL_SAMPLER_DIM_2D:
114 pipe_target = is_array ? PIPE_TEXTURE_2D_ARRAY : PIPE_TEXTURE_2D;
115 break;
116 case GLSL_SAMPLER_DIM_3D:
117 pipe_target = PIPE_TEXTURE_3D;
118 break;
119 case GLSL_SAMPLER_DIM_MS:
120 pipe_target = is_array ? PIPE_TEXTURE_2D_ARRAY : PIPE_TEXTURE_2D;
121 break;
122 case GLSL_SAMPLER_DIM_CUBE:
123 pipe_target = is_array ? PIPE_TEXTURE_CUBE_ARRAY : PIPE_TEXTURE_CUBE;
124 break;
125 case GLSL_SAMPLER_DIM_RECT:
126 pipe_target = PIPE_TEXTURE_RECT;
127 break;
128 case GLSL_SAMPLER_DIM_BUF:
129 pipe_target = PIPE_BUFFER;
130 break;
131 default:
132 break;
133 }
134 return pipe_target;
135 }
136
137 static LLVMValueRef get_ssa_src(struct lp_build_nir_context *bld_base, nir_ssa_def *ssa)
138 {
139 return bld_base->ssa_defs[ssa->index];
140 }
141
142 static LLVMValueRef get_src(struct lp_build_nir_context *bld_base, nir_src src);
143
144 static LLVMValueRef get_reg_src(struct lp_build_nir_context *bld_base, nir_reg_src src)
145 {
146 struct hash_entry *entry = _mesa_hash_table_search(bld_base->regs, src.reg);
147 LLVMValueRef reg_storage = (LLVMValueRef)entry->data;
148 struct lp_build_context *reg_bld = get_int_bld(bld_base, true, src.reg->bit_size);
149 LLVMValueRef indir_src = NULL;
150 if (src.indirect)
151 indir_src = get_src(bld_base, *src.indirect);
152 return bld_base->load_reg(bld_base, reg_bld, &src, indir_src, reg_storage);
153 }
154
155 static LLVMValueRef get_src(struct lp_build_nir_context *bld_base, nir_src src)
156 {
157 if (src.is_ssa)
158 return get_ssa_src(bld_base, src.ssa);
159 else
160 return get_reg_src(bld_base, src.reg);
161 }
162
163 static void assign_ssa(struct lp_build_nir_context *bld_base, int idx, LLVMValueRef ptr)
164 {
165 bld_base->ssa_defs[idx] = ptr;
166 }
167
168 static void assign_ssa_dest(struct lp_build_nir_context *bld_base, const nir_ssa_def *ssa,
169 LLVMValueRef vals[NIR_MAX_VEC_COMPONENTS])
170 {
171 assign_ssa(bld_base, ssa->index, ssa->num_components == 1 ? vals[0] : lp_nir_array_build_gather_values(bld_base->base.gallivm->builder, vals, ssa->num_components));
172 }
173
174 static void assign_reg(struct lp_build_nir_context *bld_base, const nir_reg_dest *reg,
175 unsigned write_mask,
176 LLVMValueRef vals[NIR_MAX_VEC_COMPONENTS])
177 {
178 struct hash_entry *entry = _mesa_hash_table_search(bld_base->regs, reg->reg);
179 LLVMValueRef reg_storage = (LLVMValueRef)entry->data;
180 struct lp_build_context *reg_bld = get_int_bld(bld_base, true, reg->reg->bit_size);
181 LLVMValueRef indir_src = NULL;
182 if (reg->indirect)
183 indir_src = get_src(bld_base, *reg->indirect);
184 bld_base->store_reg(bld_base, reg_bld, reg, write_mask ? write_mask : 0xf, indir_src, reg_storage, vals);
185 }
186
187 static void assign_dest(struct lp_build_nir_context *bld_base, const nir_dest *dest, LLVMValueRef vals[NIR_MAX_VEC_COMPONENTS])
188 {
189 if (dest->is_ssa)
190 assign_ssa_dest(bld_base, &dest->ssa, vals);
191 else
192 assign_reg(bld_base, &dest->reg, 0, vals);
193 }
194
195 static void assign_alu_dest(struct lp_build_nir_context *bld_base, const nir_alu_dest *dest, LLVMValueRef vals[NIR_MAX_VEC_COMPONENTS])
196 {
197 if (dest->dest.is_ssa)
198 assign_ssa_dest(bld_base, &dest->dest.ssa, vals);
199 else
200 assign_reg(bld_base, &dest->dest.reg, dest->write_mask, vals);
201 }
202
203 static LLVMValueRef int_to_bool32(struct lp_build_nir_context *bld_base,
204 uint32_t src_bit_size,
205 bool is_unsigned,
206 LLVMValueRef val)
207 {
208 LLVMBuilderRef builder = bld_base->base.gallivm->builder;
209 struct lp_build_context *int_bld = get_int_bld(bld_base, is_unsigned, src_bit_size);
210 LLVMValueRef result = lp_build_compare(bld_base->base.gallivm, int_bld->type, PIPE_FUNC_NOTEQUAL, val, int_bld->zero);
211 if (src_bit_size == 64)
212 result = LLVMBuildTrunc(builder, result, bld_base->int_bld.vec_type, "");
213 return result;
214 }
215
216 static LLVMValueRef flt_to_bool32(struct lp_build_nir_context *bld_base,
217 uint32_t src_bit_size,
218 LLVMValueRef val)
219 {
220 LLVMBuilderRef builder = bld_base->base.gallivm->builder;
221 struct lp_build_context *flt_bld = get_flt_bld(bld_base, src_bit_size);
222 LLVMValueRef result = lp_build_cmp(flt_bld, PIPE_FUNC_NOTEQUAL, val, flt_bld->zero);
223 if (src_bit_size == 64)
224 result = LLVMBuildTrunc(builder, result, bld_base->int_bld.vec_type, "");
225 return result;
226 }
227
228 static LLVMValueRef fcmp32(struct lp_build_nir_context *bld_base,
229 enum pipe_compare_func compare,
230 uint32_t src_bit_size,
231 LLVMValueRef src[NIR_MAX_VEC_COMPONENTS])
232 {
233 LLVMBuilderRef builder = bld_base->base.gallivm->builder;
234 struct lp_build_context *flt_bld = get_flt_bld(bld_base, src_bit_size);
235 LLVMValueRef result;
236
237 if (compare != PIPE_FUNC_NOTEQUAL)
238 result = lp_build_cmp_ordered(flt_bld, compare, src[0], src[1]);
239 else
240 result = lp_build_cmp(flt_bld, compare, src[0], src[1]);
241 if (src_bit_size == 64)
242 result = LLVMBuildTrunc(builder, result, bld_base->int_bld.vec_type, "");
243 return result;
244 }
245
246 static LLVMValueRef icmp32(struct lp_build_nir_context *bld_base,
247 enum pipe_compare_func compare,
248 bool is_unsigned,
249 uint32_t src_bit_size,
250 LLVMValueRef src[NIR_MAX_VEC_COMPONENTS])
251 {
252 LLVMBuilderRef builder = bld_base->base.gallivm->builder;
253 struct lp_build_context *i_bld = get_int_bld(bld_base, is_unsigned, src_bit_size);
254 LLVMValueRef result = lp_build_cmp(i_bld, compare, src[0], src[1]);
255 if (src_bit_size < 32)
256 result = LLVMBuildSExt(builder, result, bld_base->int_bld.vec_type, "");
257 else if (src_bit_size == 64)
258 result = LLVMBuildTrunc(builder, result, bld_base->int_bld.vec_type, "");
259 return result;
260 }
261
262 static LLVMValueRef get_alu_src(struct lp_build_nir_context *bld_base,
263 nir_alu_src src,
264 unsigned num_components)
265 {
266 LLVMBuilderRef builder = bld_base->base.gallivm->builder;
267 struct gallivm_state *gallivm = bld_base->base.gallivm;
268 LLVMValueRef value = get_src(bld_base, src.src);
269 bool need_swizzle = false;
270
271 assert(value);
272 unsigned src_components = nir_src_num_components(src.src);
273 for (unsigned i = 0; i < num_components; ++i) {
274 assert(src.swizzle[i] < src_components);
275 if (src.swizzle[i] != i)
276 need_swizzle = true;
277 }
278
279 if (need_swizzle || num_components != src_components) {
280 if (src_components > 1 && num_components == 1) {
281 value = LLVMBuildExtractValue(gallivm->builder, value,
282 src.swizzle[0], "");
283 } else if (src_components == 1 && num_components > 1) {
284 LLVMValueRef values[] = {value, value, value, value, value, value, value, value, value, value, value, value, value, value, value, value};
285 value = lp_nir_array_build_gather_values(builder, values, num_components);
286 } else {
287 LLVMValueRef arr = LLVMGetUndef(LLVMArrayType(LLVMTypeOf(LLVMBuildExtractValue(builder, value, 0, "")), num_components));
288 for (unsigned i = 0; i < num_components; i++)
289 arr = LLVMBuildInsertValue(builder, arr, LLVMBuildExtractValue(builder, value, src.swizzle[i], ""), i, "");
290 value = arr;
291 }
292 }
293 assert(!src.negate);
294 assert(!src.abs);
295 return value;
296 }
297
298 static LLVMValueRef emit_b2f(struct lp_build_nir_context *bld_base,
299 LLVMValueRef src0,
300 unsigned bitsize)
301 {
302 LLVMBuilderRef builder = bld_base->base.gallivm->builder;
303 LLVMValueRef result = LLVMBuildAnd(builder, cast_type(bld_base, src0, nir_type_int, 32),
304 LLVMBuildBitCast(builder, lp_build_const_vec(bld_base->base.gallivm, bld_base->base.type,
305 1.0), bld_base->int_bld.vec_type, ""),
306 "");
307 result = LLVMBuildBitCast(builder, result, bld_base->base.vec_type, "");
308 switch (bitsize) {
309 case 32:
310 break;
311 case 64:
312 result = LLVMBuildFPExt(builder, result, bld_base->dbl_bld.vec_type, "");
313 break;
314 default:
315 unreachable("unsupported bit size.");
316 }
317 return result;
318 }
319
320 static LLVMValueRef emit_b2i(struct lp_build_nir_context *bld_base,
321 LLVMValueRef src0,
322 unsigned bitsize)
323 {
324 LLVMBuilderRef builder = bld_base->base.gallivm->builder;
325 LLVMValueRef result = LLVMBuildAnd(builder, cast_type(bld_base, src0, nir_type_int, 32),
326 lp_build_const_int_vec(bld_base->base.gallivm, bld_base->base.type, 1), "");
327 switch (bitsize) {
328 case 32:
329 return result;
330 case 64:
331 return LLVMBuildZExt(builder, result, bld_base->int64_bld.vec_type, "");
332 default:
333 unreachable("unsupported bit size.");
334 }
335 }
336
337 static LLVMValueRef emit_b32csel(struct lp_build_nir_context *bld_base,
338 unsigned src_bit_size[NIR_MAX_VEC_COMPONENTS],
339 LLVMValueRef src[NIR_MAX_VEC_COMPONENTS])
340 {
341 LLVMValueRef sel = cast_type(bld_base, src[0], nir_type_int, 32);
342 LLVMValueRef v = lp_build_compare(bld_base->base.gallivm, bld_base->int_bld.type, PIPE_FUNC_NOTEQUAL, sel, bld_base->int_bld.zero);
343 struct lp_build_context *bld = get_int_bld(bld_base, false, src_bit_size[1]);
344 return lp_build_select(bld, v, src[1], src[2]);
345 }
346
347 static LLVMValueRef split_64bit(struct lp_build_nir_context *bld_base,
348 LLVMValueRef src,
349 bool hi)
350 {
351 struct gallivm_state *gallivm = bld_base->base.gallivm;
352 LLVMValueRef shuffles[LP_MAX_VECTOR_WIDTH/32];
353 LLVMValueRef shuffles2[LP_MAX_VECTOR_WIDTH/32];
354 int len = bld_base->base.type.length * 2;
355 for (unsigned i = 0; i < bld_base->base.type.length; i++) {
356 #if UTIL_ARCH_LITTLE_ENDIAN
357 shuffles[i] = lp_build_const_int32(gallivm, i * 2);
358 shuffles2[i] = lp_build_const_int32(gallivm, (i * 2) + 1);
359 #else
360 shuffles[i] = lp_build_const_int32(gallivm, (i * 2) + 1);
361 shuffles2[i] = lp_build_const_int32(gallivm, (i * 2));
362 #endif
363 }
364
365 src = LLVMBuildBitCast(gallivm->builder, src, LLVMVectorType(LLVMInt32TypeInContext(gallivm->context), len), "");
366 return LLVMBuildShuffleVector(gallivm->builder, src,
367 LLVMGetUndef(LLVMTypeOf(src)),
368 LLVMConstVector(hi ? shuffles2 : shuffles,
369 bld_base->base.type.length),
370 "");
371 }
372
373 static LLVMValueRef
374 merge_64bit(struct lp_build_nir_context *bld_base,
375 LLVMValueRef input,
376 LLVMValueRef input2)
377 {
378 struct gallivm_state *gallivm = bld_base->base.gallivm;
379 LLVMBuilderRef builder = gallivm->builder;
380 int i;
381 LLVMValueRef shuffles[2 * (LP_MAX_VECTOR_WIDTH/32)];
382 int len = bld_base->base.type.length * 2;
383 assert(len <= (2 * (LP_MAX_VECTOR_WIDTH/32)));
384
385 for (i = 0; i < bld_base->base.type.length * 2; i+=2) {
386 #if UTIL_ARCH_LITTLE_ENDIAN
387 shuffles[i] = lp_build_const_int32(gallivm, i / 2);
388 shuffles[i + 1] = lp_build_const_int32(gallivm, i / 2 + bld_base->base.type.length);
389 #else
390 shuffles[i] = lp_build_const_int32(gallivm, i / 2 + bld_base->base.type.length);
391 shuffles[i + 1] = lp_build_const_int32(gallivm, i / 2);
392 #endif
393 }
394 return LLVMBuildShuffleVector(builder, input, input2, LLVMConstVector(shuffles, len), "");
395 }
396
397 static LLVMValueRef
398 do_int_divide(struct lp_build_nir_context *bld_base,
399 bool is_unsigned, unsigned src_bit_size,
400 LLVMValueRef src, LLVMValueRef src2)
401 {
402 struct gallivm_state *gallivm = bld_base->base.gallivm;
403 LLVMBuilderRef builder = gallivm->builder;
404 struct lp_build_context *int_bld = get_int_bld(bld_base, is_unsigned, src_bit_size);
405 struct lp_build_context *mask_bld = get_int_bld(bld_base, true, src_bit_size);
406 LLVMValueRef div_mask = lp_build_cmp(mask_bld, PIPE_FUNC_EQUAL, src2,
407 mask_bld->zero);
408
409 if (!is_unsigned) {
410 /* INT_MIN (0x80000000) / -1 (0xffffffff) causes sigfpe, seen with blender. */
411 div_mask = LLVMBuildAnd(builder, div_mask, lp_build_const_int_vec(gallivm, int_bld->type, 0x7fffffff), "");
412 }
413 LLVMValueRef divisor = LLVMBuildOr(builder,
414 div_mask,
415 src2, "");
416 LLVMValueRef result = lp_build_div(int_bld, src, divisor);
417
418 if (!is_unsigned) {
419 LLVMValueRef not_div_mask = LLVMBuildNot(builder, div_mask, "");
420 return LLVMBuildAnd(builder, not_div_mask, result, "");
421 } else
422 /* udiv by zero is guaranteed to return 0xffffffff at least with d3d10
423 * may as well do same for idiv */
424 return LLVMBuildOr(builder, div_mask, result, "");
425 }
426
427 static LLVMValueRef
428 do_int_mod(struct lp_build_nir_context *bld_base,
429 bool is_unsigned, unsigned src_bit_size,
430 LLVMValueRef src, LLVMValueRef src2)
431 {
432 struct gallivm_state *gallivm = bld_base->base.gallivm;
433 LLVMBuilderRef builder = gallivm->builder;
434 struct lp_build_context *int_bld = get_int_bld(bld_base, is_unsigned, src_bit_size);
435 LLVMValueRef div_mask = lp_build_cmp(int_bld, PIPE_FUNC_EQUAL, src2,
436 int_bld->zero);
437 LLVMValueRef divisor = LLVMBuildOr(builder,
438 div_mask,
439 src2, "");
440 LLVMValueRef result = lp_build_mod(int_bld, src, divisor);
441 return LLVMBuildOr(builder, div_mask, result, "");
442 }
443
444 static LLVMValueRef do_alu_action(struct lp_build_nir_context *bld_base,
445 nir_op op, unsigned src_bit_size[NIR_MAX_VEC_COMPONENTS], LLVMValueRef src[NIR_MAX_VEC_COMPONENTS])
446 {
447 struct gallivm_state *gallivm = bld_base->base.gallivm;
448 LLVMBuilderRef builder = gallivm->builder;
449 LLVMValueRef result;
450 switch (op) {
451 case nir_op_b2f32:
452 result = emit_b2f(bld_base, src[0], 32);
453 break;
454 case nir_op_b2f64:
455 result = emit_b2f(bld_base, src[0], 64);
456 break;
457 case nir_op_b2i32:
458 result = emit_b2i(bld_base, src[0], 32);
459 break;
460 case nir_op_b2i64:
461 result = emit_b2i(bld_base, src[0], 64);
462 break;
463 case nir_op_b32csel:
464 result = emit_b32csel(bld_base, src_bit_size, src);
465 break;
466 case nir_op_bit_count:
467 result = lp_build_popcount(get_int_bld(bld_base, false, src_bit_size[0]), src[0]);
468 break;
469 case nir_op_bitfield_select:
470 result = lp_build_xor(&bld_base->uint_bld, src[2], lp_build_and(&bld_base->uint_bld, src[0], lp_build_xor(&bld_base->uint_bld, src[1], src[2])));
471 break;
472 case nir_op_bitfield_reverse:
473 result = lp_build_bitfield_reverse(get_int_bld(bld_base, false, src_bit_size[0]), src[0]);
474 break;
475 case nir_op_f2b32:
476 result = flt_to_bool32(bld_base, src_bit_size[0], src[0]);
477 break;
478 case nir_op_f2f32:
479 result = LLVMBuildFPTrunc(builder, src[0],
480 bld_base->base.vec_type, "");
481 break;
482 case nir_op_f2f64:
483 result = LLVMBuildFPExt(builder, src[0],
484 bld_base->dbl_bld.vec_type, "");
485 break;
486 case nir_op_f2i32:
487 result = LLVMBuildFPToSI(builder, src[0], bld_base->base.int_vec_type, "");
488 break;
489 case nir_op_f2u32:
490 result = LLVMBuildFPToUI(builder,
491 src[0],
492 bld_base->base.int_vec_type, "");
493 break;
494 case nir_op_f2i64:
495 result = LLVMBuildFPToSI(builder,
496 src[0],
497 bld_base->int64_bld.vec_type, "");
498 break;
499 case nir_op_f2u64:
500 result = LLVMBuildFPToUI(builder,
501 src[0],
502 bld_base->uint64_bld.vec_type, "");
503 break;
504 case nir_op_fabs:
505 result = lp_build_abs(get_flt_bld(bld_base, src_bit_size[0]), src[0]);
506 break;
507 case nir_op_fadd:
508 result = lp_build_add(get_flt_bld(bld_base, src_bit_size[0]),
509 src[0], src[1]);
510 break;
511 case nir_op_fceil:
512 result = lp_build_ceil(get_flt_bld(bld_base, src_bit_size[0]), src[0]);
513 break;
514 case nir_op_fcos:
515 result = lp_build_cos(&bld_base->base, src[0]);
516 break;
517 case nir_op_fddx:
518 case nir_op_fddx_coarse:
519 case nir_op_fddx_fine:
520 result = lp_build_ddx(&bld_base->base, src[0]);
521 break;
522 case nir_op_fddy:
523 case nir_op_fddy_coarse:
524 case nir_op_fddy_fine:
525 result = lp_build_ddy(&bld_base->base, src[0]);
526 break;
527 case nir_op_fdiv:
528 result = lp_build_div(get_flt_bld(bld_base, src_bit_size[0]),
529 src[0], src[1]);
530 break;
531 case nir_op_feq32:
532 result = fcmp32(bld_base, PIPE_FUNC_EQUAL, src_bit_size[0], src);
533 break;
534 case nir_op_fexp2:
535 result = lp_build_exp2(&bld_base->base, src[0]);
536 break;
537 case nir_op_ffloor:
538 result = lp_build_floor(get_flt_bld(bld_base, src_bit_size[0]), src[0]);
539 break;
540 case nir_op_ffma:
541 result = lp_build_fmuladd(builder, src[0], src[1], src[2]);
542 break;
543 case nir_op_ffract: {
544 struct lp_build_context *flt_bld = get_flt_bld(bld_base, src_bit_size[0]);
545 LLVMValueRef tmp = lp_build_floor(flt_bld, src[0]);
546 result = lp_build_sub(flt_bld, src[0], tmp);
547 break;
548 }
549 case nir_op_fge32:
550 result = fcmp32(bld_base, PIPE_FUNC_GEQUAL, src_bit_size[0], src);
551 break;
552 case nir_op_find_lsb:
553 result = lp_build_cttz(get_int_bld(bld_base, false, src_bit_size[0]), src[0]);
554 break;
555 case nir_op_flog2:
556 result = lp_build_log2_safe(&bld_base->base, src[0]);
557 break;
558 case nir_op_flt32:
559 result = fcmp32(bld_base, PIPE_FUNC_LESS, src_bit_size[0], src);
560 break;
561 case nir_op_fmin:
562 result = lp_build_min(get_flt_bld(bld_base, src_bit_size[0]), src[0], src[1]);
563 break;
564 case nir_op_fmod: {
565 struct lp_build_context *flt_bld = get_flt_bld(bld_base, src_bit_size[0]);
566 result = lp_build_div(flt_bld, src[0], src[1]);
567 result = lp_build_floor(flt_bld, result);
568 result = lp_build_mul(flt_bld, src[1], result);
569 result = lp_build_sub(flt_bld, src[0], result);
570 break;
571 }
572 case nir_op_fmul:
573 result = lp_build_mul(get_flt_bld(bld_base, src_bit_size[0]),
574 src[0], src[1]);
575 break;
576 case nir_op_fmax:
577 result = lp_build_max(get_flt_bld(bld_base, src_bit_size[0]), src[0], src[1]);
578 break;
579 case nir_op_fne32:
580 result = fcmp32(bld_base, PIPE_FUNC_NOTEQUAL, src_bit_size[0], src);
581 break;
582 case nir_op_fneg:
583 result = lp_build_negate(get_flt_bld(bld_base, src_bit_size[0]), src[0]);
584 break;
585 case nir_op_fpow:
586 result = lp_build_pow(&bld_base->base, src[0], src[1]);
587 break;
588 case nir_op_frcp:
589 result = lp_build_rcp(get_flt_bld(bld_base, src_bit_size[0]), src[0]);
590 break;
591 case nir_op_fround_even:
592 result = lp_build_round(get_flt_bld(bld_base, src_bit_size[0]), src[0]);
593 break;
594 case nir_op_frsq:
595 result = lp_build_rsqrt(get_flt_bld(bld_base, src_bit_size[0]), src[0]);
596 break;
597 case nir_op_fsat:
598 result = lp_build_clamp_zero_one_nanzero(get_flt_bld(bld_base, src_bit_size[0]), src[0]);
599 break;
600 case nir_op_fsign:
601 result = lp_build_sgn(get_flt_bld(bld_base, src_bit_size[0]), src[0]);
602 break;
603 case nir_op_fsin:
604 result = lp_build_sin(&bld_base->base, src[0]);
605 break;
606 case nir_op_fsqrt:
607 result = lp_build_sqrt(get_flt_bld(bld_base, src_bit_size[0]), src[0]);
608 break;
609 case nir_op_ftrunc:
610 result = lp_build_trunc(get_flt_bld(bld_base, src_bit_size[0]), src[0]);
611 break;
612 case nir_op_i2b32:
613 result = int_to_bool32(bld_base, src_bit_size[0], false, src[0]);
614 break;
615 case nir_op_i2f32:
616 result = lp_build_int_to_float(&bld_base->base, src[0]);
617 break;
618 case nir_op_i2f64:
619 result = lp_build_int_to_float(&bld_base->dbl_bld, src[0]);
620 break;
621 case nir_op_i2i8:
622 result = LLVMBuildTrunc(builder, src[0], bld_base->int8_bld.vec_type, "");
623 break;
624 case nir_op_i2i16:
625 if (src_bit_size[0] < 16)
626 result = LLVMBuildSExt(builder, src[0], bld_base->int16_bld.vec_type, "");
627 else
628 result = LLVMBuildTrunc(builder, src[0], bld_base->int16_bld.vec_type, "");
629 break;
630 case nir_op_i2i32:
631 if (src_bit_size[0] < 32)
632 result = LLVMBuildSExt(builder, src[0], bld_base->int_bld.vec_type, "");
633 else
634 result = LLVMBuildTrunc(builder, src[0], bld_base->int_bld.vec_type, "");
635 break;
636 case nir_op_i2i64:
637 result = LLVMBuildSExt(builder, src[0], bld_base->int64_bld.vec_type, "");
638 break;
639 case nir_op_iabs:
640 result = lp_build_abs(get_int_bld(bld_base, false, src_bit_size[0]), src[0]);
641 break;
642 case nir_op_iadd:
643 result = lp_build_add(get_int_bld(bld_base, false, src_bit_size[0]),
644 src[0], src[1]);
645 break;
646 case nir_op_iand:
647 result = lp_build_and(get_int_bld(bld_base, false, src_bit_size[0]),
648 src[0], src[1]);
649 break;
650 case nir_op_idiv:
651 result = do_int_divide(bld_base, false, src_bit_size[0], src[0], src[1]);
652 break;
653 case nir_op_ieq32:
654 result = icmp32(bld_base, PIPE_FUNC_EQUAL, false, src_bit_size[0], src);
655 break;
656 case nir_op_ige32:
657 result = icmp32(bld_base, PIPE_FUNC_GEQUAL, false, src_bit_size[0], src);
658 break;
659 case nir_op_ilt32:
660 result = icmp32(bld_base, PIPE_FUNC_LESS, false, src_bit_size[0], src);
661 break;
662 case nir_op_imax:
663 result = lp_build_max(get_int_bld(bld_base, false, src_bit_size[0]), src[0], src[1]);
664 break;
665 case nir_op_imin:
666 result = lp_build_min(get_int_bld(bld_base, false, src_bit_size[0]), src[0], src[1]);
667 break;
668 case nir_op_imul:
669 case nir_op_imul24:
670 result = lp_build_mul(get_int_bld(bld_base, false, src_bit_size[0]),
671 src[0], src[1]);
672 break;
673 case nir_op_imul_high: {
674 LLVMValueRef hi_bits;
675 lp_build_mul_32_lohi(&bld_base->int_bld, src[0], src[1], &hi_bits);
676 result = hi_bits;
677 break;
678 }
679 case nir_op_ine32:
680 result = icmp32(bld_base, PIPE_FUNC_NOTEQUAL, false, src_bit_size[0], src);
681 break;
682 case nir_op_ineg:
683 result = lp_build_negate(get_int_bld(bld_base, false, src_bit_size[0]), src[0]);
684 break;
685 case nir_op_inot:
686 result = lp_build_not(get_int_bld(bld_base, false, src_bit_size[0]), src[0]);
687 break;
688 case nir_op_ior:
689 result = lp_build_or(get_int_bld(bld_base, false, src_bit_size[0]),
690 src[0], src[1]);
691 break;
692 case nir_op_irem:
693 result = do_int_mod(bld_base, false, src_bit_size[0], src[0], src[1]);
694 break;
695 case nir_op_ishl: {
696 struct lp_build_context *uint_bld = get_int_bld(bld_base, true, src_bit_size[0]);
697 struct lp_build_context *int_bld = get_int_bld(bld_base, false, src_bit_size[0]);
698 if (src_bit_size[0] == 64)
699 src[1] = LLVMBuildZExt(builder, src[1], uint_bld->vec_type, "");
700 if (src_bit_size[0] < 32)
701 src[1] = LLVMBuildTrunc(builder, src[1], uint_bld->vec_type, "");
702 src[1] = lp_build_and(uint_bld, src[1], lp_build_const_int_vec(gallivm, uint_bld->type, (src_bit_size[0] - 1)));
703 result = lp_build_shl(int_bld, src[0], src[1]);
704 break;
705 }
706 case nir_op_ishr: {
707 struct lp_build_context *uint_bld = get_int_bld(bld_base, true, src_bit_size[0]);
708 struct lp_build_context *int_bld = get_int_bld(bld_base, false, src_bit_size[0]);
709 if (src_bit_size[0] == 64)
710 src[1] = LLVMBuildZExt(builder, src[1], uint_bld->vec_type, "");
711 if (src_bit_size[0] < 32)
712 src[1] = LLVMBuildTrunc(builder, src[1], uint_bld->vec_type, "");
713 src[1] = lp_build_and(uint_bld, src[1], lp_build_const_int_vec(gallivm, uint_bld->type, (src_bit_size[0] - 1)));
714 result = lp_build_shr(int_bld, src[0], src[1]);
715 break;
716 }
717 case nir_op_isign:
718 result = lp_build_sgn(get_int_bld(bld_base, false, src_bit_size[0]), src[0]);
719 break;
720 case nir_op_isub:
721 result = lp_build_sub(get_int_bld(bld_base, false, src_bit_size[0]),
722 src[0], src[1]);
723 break;
724 case nir_op_ixor:
725 result = lp_build_xor(get_int_bld(bld_base, false, src_bit_size[0]),
726 src[0], src[1]);
727 break;
728 case nir_op_mov:
729 result = src[0];
730 break;
731 case nir_op_unpack_64_2x32_split_x:
732 result = split_64bit(bld_base, src[0], false);
733 break;
734 case nir_op_unpack_64_2x32_split_y:
735 result = split_64bit(bld_base, src[0], true);
736 break;
737
738 case nir_op_pack_64_2x32_split: {
739 LLVMValueRef tmp = merge_64bit(bld_base, src[0], src[1]);
740 result = LLVMBuildBitCast(builder, tmp, bld_base->dbl_bld.vec_type, "");
741 break;
742 }
743 case nir_op_u2f32:
744 result = LLVMBuildUIToFP(builder, src[0], bld_base->base.vec_type, "");
745 break;
746 case nir_op_u2f64:
747 result = LLVMBuildUIToFP(builder, src[0], bld_base->dbl_bld.vec_type, "");
748 break;
749 case nir_op_u2u8:
750 result = LLVMBuildTrunc(builder, src[0], bld_base->uint8_bld.vec_type, "");
751 break;
752 case nir_op_u2u16:
753 if (src_bit_size[0] < 16)
754 result = LLVMBuildZExt(builder, src[0], bld_base->uint16_bld.vec_type, "");
755 else
756 result = LLVMBuildTrunc(builder, src[0], bld_base->uint16_bld.vec_type, "");
757 break;
758 case nir_op_u2u32:
759 if (src_bit_size[0] < 32)
760 result = LLVMBuildZExt(builder, src[0], bld_base->uint_bld.vec_type, "");
761 else
762 result = LLVMBuildTrunc(builder, src[0], bld_base->uint_bld.vec_type, "");
763 break;
764 case nir_op_u2u64:
765 result = LLVMBuildZExt(builder, src[0], bld_base->uint64_bld.vec_type, "");
766 break;
767 case nir_op_udiv:
768 result = do_int_divide(bld_base, true, src_bit_size[0], src[0], src[1]);
769 break;
770 case nir_op_ufind_msb: {
771 struct lp_build_context *uint_bld = get_int_bld(bld_base, true, src_bit_size[0]);
772 result = lp_build_ctlz(uint_bld, src[0]);
773 result = lp_build_sub(uint_bld, lp_build_const_int_vec(gallivm, uint_bld->type, src_bit_size[0] - 1), result);
774 break;
775 }
776 case nir_op_uge32:
777 result = icmp32(bld_base, PIPE_FUNC_GEQUAL, true, src_bit_size[0], src);
778 break;
779 case nir_op_ult32:
780 result = icmp32(bld_base, PIPE_FUNC_LESS, true, src_bit_size[0], src);
781 break;
782 case nir_op_umax:
783 result = lp_build_max(get_int_bld(bld_base, true, src_bit_size[0]), src[0], src[1]);
784 break;
785 case nir_op_umin:
786 result = lp_build_min(get_int_bld(bld_base, true, src_bit_size[0]), src[0], src[1]);
787 break;
788 case nir_op_umod:
789 result = do_int_mod(bld_base, true, src_bit_size[0], src[0], src[1]);
790 break;
791 case nir_op_umul_high: {
792 LLVMValueRef hi_bits;
793 lp_build_mul_32_lohi(&bld_base->uint_bld, src[0], src[1], &hi_bits);
794 result = hi_bits;
795 break;
796 }
797 case nir_op_ushr: {
798 struct lp_build_context *uint_bld = get_int_bld(bld_base, true, src_bit_size[0]);
799 if (src_bit_size[0] == 64)
800 src[1] = LLVMBuildZExt(builder, src[1], uint_bld->vec_type, "");
801 if (src_bit_size[0] < 32)
802 src[1] = LLVMBuildTrunc(builder, src[1], uint_bld->vec_type, "");
803 src[1] = lp_build_and(uint_bld, src[1], lp_build_const_int_vec(gallivm, uint_bld->type, (src_bit_size[0] - 1)));
804 result = lp_build_shr(uint_bld, src[0], src[1]);
805 break;
806 }
807 default:
808 assert(0);
809 break;
810 }
811 return result;
812 }
813
814 static void visit_alu(struct lp_build_nir_context *bld_base, const nir_alu_instr *instr)
815 {
816 struct gallivm_state *gallivm = bld_base->base.gallivm;
817 LLVMValueRef src[NIR_MAX_VEC_COMPONENTS];
818 unsigned src_bit_size[NIR_MAX_VEC_COMPONENTS];
819 unsigned num_components = nir_dest_num_components(instr->dest.dest);
820 unsigned src_components;
821 switch (instr->op) {
822 case nir_op_vec2:
823 case nir_op_vec3:
824 case nir_op_vec4:
825 case nir_op_vec8:
826 case nir_op_vec16:
827 src_components = 1;
828 break;
829 case nir_op_pack_half_2x16:
830 src_components = 2;
831 break;
832 case nir_op_unpack_half_2x16:
833 src_components = 1;
834 break;
835 case nir_op_cube_face_coord:
836 case nir_op_cube_face_index:
837 src_components = 3;
838 break;
839 default:
840 src_components = num_components;
841 break;
842 }
843 for (unsigned i = 0; i < nir_op_infos[instr->op].num_inputs; i++) {
844 src[i] = get_alu_src(bld_base, instr->src[i], src_components);
845 src_bit_size[i] = nir_src_bit_size(instr->src[i].src);
846 }
847
848 LLVMValueRef result[NIR_MAX_VEC_COMPONENTS];
849 if (instr->op == nir_op_vec4 || instr->op == nir_op_vec3 || instr->op == nir_op_vec2 || instr->op == nir_op_vec8 || instr->op == nir_op_vec16) {
850 for (unsigned i = 0; i < nir_op_infos[instr->op].num_inputs; i++) {
851 result[i] = cast_type(bld_base, src[i], nir_op_infos[instr->op].input_types[i], src_bit_size[i]);
852 }
853 } else {
854 for (unsigned c = 0; c < num_components; c++) {
855 LLVMValueRef src_chan[NIR_MAX_VEC_COMPONENTS];
856
857 for (unsigned i = 0; i < nir_op_infos[instr->op].num_inputs; i++) {
858 if (num_components > 1) {
859 src_chan[i] = LLVMBuildExtractValue(gallivm->builder,
860 src[i], c, "");
861 } else
862 src_chan[i] = src[i];
863 src_chan[i] = cast_type(bld_base, src_chan[i], nir_op_infos[instr->op].input_types[i], src_bit_size[i]);
864 }
865 result[c] = do_alu_action(bld_base, instr->op, src_bit_size, src_chan);
866 result[c] = cast_type(bld_base, result[c], nir_op_infos[instr->op].output_type, nir_dest_bit_size(instr->dest.dest));
867 }
868 }
869 assign_alu_dest(bld_base, &instr->dest, result);
870 }
871
872 static void visit_load_const(struct lp_build_nir_context *bld_base,
873 const nir_load_const_instr *instr)
874 {
875 LLVMValueRef result[NIR_MAX_VEC_COMPONENTS];
876 struct lp_build_context *int_bld = get_int_bld(bld_base, true, instr->def.bit_size);
877 for (unsigned i = 0; i < instr->def.num_components; i++)
878 result[i] = lp_build_const_int_vec(bld_base->base.gallivm, int_bld->type, instr->def.bit_size == 32 ? instr->value[i].u32 : instr->value[i].u64);
879 assign_ssa_dest(bld_base, &instr->def, result);
880 }
881
882 static void
883 get_deref_offset(struct lp_build_nir_context *bld_base, nir_deref_instr *instr,
884 bool vs_in, unsigned *vertex_index_out,
885 LLVMValueRef *vertex_index_ref,
886 unsigned *const_out, LLVMValueRef *indir_out)
887 {
888 LLVMBuilderRef builder = bld_base->base.gallivm->builder;
889 nir_variable *var = nir_deref_instr_get_variable(instr);
890 nir_deref_path path;
891 unsigned idx_lvl = 1;
892
893 nir_deref_path_init(&path, instr, NULL);
894
895 if (vertex_index_out != NULL || vertex_index_ref != NULL) {
896 if (vertex_index_ref) {
897 *vertex_index_ref = get_src(bld_base, path.path[idx_lvl]->arr.index);
898 if (vertex_index_out)
899 *vertex_index_out = 0;
900 } else {
901 *vertex_index_out = nir_src_as_uint(path.path[idx_lvl]->arr.index);
902 }
903 ++idx_lvl;
904 }
905
906 uint32_t const_offset = 0;
907 LLVMValueRef offset = NULL;
908
909 if (var->data.compact) {
910 assert(instr->deref_type == nir_deref_type_array);
911 const_offset = nir_src_as_uint(instr->arr.index);
912 goto out;
913 }
914
915 for (; path.path[idx_lvl]; ++idx_lvl) {
916 const struct glsl_type *parent_type = path.path[idx_lvl - 1]->type;
917 if (path.path[idx_lvl]->deref_type == nir_deref_type_struct) {
918 unsigned index = path.path[idx_lvl]->strct.index;
919
920 for (unsigned i = 0; i < index; i++) {
921 const struct glsl_type *ft = glsl_get_struct_field(parent_type, i);
922 const_offset += glsl_count_attribute_slots(ft, vs_in);
923 }
924 } else if(path.path[idx_lvl]->deref_type == nir_deref_type_array) {
925 unsigned size = glsl_count_attribute_slots(path.path[idx_lvl]->type, vs_in);
926 if (nir_src_is_const(path.path[idx_lvl]->arr.index)) {
927 const_offset += nir_src_comp_as_int(path.path[idx_lvl]->arr.index, 0) * size;
928 } else {
929 LLVMValueRef idx_src = get_src(bld_base, path.path[idx_lvl]->arr.index);
930 idx_src = cast_type(bld_base, idx_src, nir_type_uint, 32);
931 LLVMValueRef array_off = lp_build_mul(&bld_base->uint_bld, lp_build_const_int_vec(bld_base->base.gallivm, bld_base->base.type, size),
932 idx_src);
933 if (offset)
934 offset = lp_build_add(&bld_base->uint_bld, offset, array_off);
935 else
936 offset = array_off;
937 }
938 } else
939 unreachable("Uhandled deref type in get_deref_instr_offset");
940 }
941
942 out:
943 nir_deref_path_finish(&path);
944
945 if (const_offset && offset)
946 offset = LLVMBuildAdd(builder, offset,
947 lp_build_const_int_vec(bld_base->base.gallivm, bld_base->uint_bld.type, const_offset),
948 "");
949 *const_out = const_offset;
950 *indir_out = offset;
951 }
952
953 static void visit_load_var(struct lp_build_nir_context *bld_base,
954 nir_intrinsic_instr *instr,
955 LLVMValueRef result[NIR_MAX_VEC_COMPONENTS])
956 {
957 nir_deref_instr *deref = nir_instr_as_deref(instr->src[0].ssa->parent_instr);
958 nir_variable *var = nir_deref_instr_get_variable(deref);
959 nir_variable_mode mode = deref->mode;
960 unsigned const_index;
961 LLVMValueRef indir_index;
962 LLVMValueRef indir_vertex_index = NULL;
963 unsigned vertex_index = 0;
964 unsigned nc = nir_dest_num_components(instr->dest);
965 unsigned bit_size = nir_dest_bit_size(instr->dest);
966 if (var) {
967 bool vs_in = bld_base->shader->info.stage == MESA_SHADER_VERTEX &&
968 var->data.mode == nir_var_shader_in;
969 bool gs_in = bld_base->shader->info.stage == MESA_SHADER_GEOMETRY &&
970 var->data.mode == nir_var_shader_in;
971 bool tcs_in = bld_base->shader->info.stage == MESA_SHADER_TESS_CTRL &&
972 var->data.mode == nir_var_shader_in;
973 bool tcs_out = bld_base->shader->info.stage == MESA_SHADER_TESS_CTRL &&
974 var->data.mode == nir_var_shader_out && !var->data.patch;
975 bool tes_in = bld_base->shader->info.stage == MESA_SHADER_TESS_EVAL &&
976 var->data.mode == nir_var_shader_in && !var->data.patch;
977
978 mode = var->data.mode;
979
980 get_deref_offset(bld_base, deref, vs_in, gs_in ? &vertex_index : NULL, (tcs_in || tcs_out || tes_in) ? &indir_vertex_index : NULL,
981 &const_index, &indir_index);
982 }
983 bld_base->load_var(bld_base, mode, nc, bit_size, var, vertex_index, indir_vertex_index, const_index, indir_index, result);
984 }
985
986 static void
987 visit_store_var(struct lp_build_nir_context *bld_base,
988 nir_intrinsic_instr *instr)
989 {
990 nir_deref_instr *deref = nir_instr_as_deref(instr->src[0].ssa->parent_instr);
991 nir_variable *var = nir_deref_instr_get_variable(deref);
992 nir_variable_mode mode = deref->mode;
993 int writemask = instr->const_index[0];
994 unsigned bit_size = nir_src_bit_size(instr->src[1]);
995 LLVMValueRef src = get_src(bld_base, instr->src[1]);
996 unsigned const_index = 0;
997 LLVMValueRef indir_index, indir_vertex_index = NULL;
998 if (var) {
999 bool tcs_out = bld_base->shader->info.stage == MESA_SHADER_TESS_CTRL &&
1000 var->data.mode == nir_var_shader_out && !var->data.patch;
1001 get_deref_offset(bld_base, deref, false, NULL, tcs_out ? &indir_vertex_index : NULL,
1002 &const_index, &indir_index);
1003 }
1004 bld_base->store_var(bld_base, mode, instr->num_components, bit_size, var, writemask, indir_vertex_index, const_index, indir_index, src);
1005 }
1006
1007 static void visit_load_ubo(struct lp_build_nir_context *bld_base,
1008 nir_intrinsic_instr *instr,
1009 LLVMValueRef result[NIR_MAX_VEC_COMPONENTS])
1010 {
1011 struct gallivm_state *gallivm = bld_base->base.gallivm;
1012 LLVMBuilderRef builder = gallivm->builder;
1013 LLVMValueRef idx = get_src(bld_base, instr->src[0]);
1014 LLVMValueRef offset = get_src(bld_base, instr->src[1]);
1015
1016 bool offset_is_uniform = nir_src_is_dynamically_uniform(instr->src[1]);
1017 idx = LLVMBuildExtractElement(builder, idx, lp_build_const_int32(gallivm, 0), "");
1018 bld_base->load_ubo(bld_base, nir_dest_num_components(instr->dest), nir_dest_bit_size(instr->dest),
1019 offset_is_uniform, idx, offset, result);
1020 }
1021
1022
1023 static void visit_load_ssbo(struct lp_build_nir_context *bld_base,
1024 nir_intrinsic_instr *instr,
1025 LLVMValueRef result[NIR_MAX_VEC_COMPONENTS])
1026 {
1027 LLVMValueRef idx = get_src(bld_base, instr->src[0]);
1028 LLVMValueRef offset = get_src(bld_base, instr->src[1]);
1029 bld_base->load_mem(bld_base, nir_dest_num_components(instr->dest), nir_dest_bit_size(instr->dest),
1030 idx, offset, result);
1031 }
1032
1033 static void visit_store_ssbo(struct lp_build_nir_context *bld_base,
1034 nir_intrinsic_instr *instr)
1035 {
1036 LLVMValueRef val = get_src(bld_base, instr->src[0]);
1037 LLVMValueRef idx = get_src(bld_base, instr->src[1]);
1038 LLVMValueRef offset = get_src(bld_base, instr->src[2]);
1039 int writemask = instr->const_index[0];
1040 int nc = nir_src_num_components(instr->src[0]);
1041 int bitsize = nir_src_bit_size(instr->src[0]);
1042 bld_base->store_mem(bld_base, writemask, nc, bitsize, idx, offset, val);
1043 }
1044
1045 static void visit_get_buffer_size(struct lp_build_nir_context *bld_base,
1046 nir_intrinsic_instr *instr,
1047 LLVMValueRef result[NIR_MAX_VEC_COMPONENTS])
1048 {
1049 LLVMValueRef idx = get_src(bld_base, instr->src[0]);
1050 result[0] = bld_base->get_buffer_size(bld_base, idx);
1051 }
1052
1053 static void visit_ssbo_atomic(struct lp_build_nir_context *bld_base,
1054 nir_intrinsic_instr *instr,
1055 LLVMValueRef result[NIR_MAX_VEC_COMPONENTS])
1056 {
1057 LLVMValueRef idx = get_src(bld_base, instr->src[0]);
1058 LLVMValueRef offset = get_src(bld_base, instr->src[1]);
1059 LLVMValueRef val = get_src(bld_base, instr->src[2]);
1060 LLVMValueRef val2 = NULL;
1061 if (instr->intrinsic == nir_intrinsic_ssbo_atomic_comp_swap)
1062 val2 = get_src(bld_base, instr->src[3]);
1063
1064 bld_base->atomic_mem(bld_base, instr->intrinsic, idx, offset, val, val2, &result[0]);
1065
1066 }
1067
1068 static void visit_load_image(struct lp_build_nir_context *bld_base,
1069 nir_intrinsic_instr *instr,
1070 LLVMValueRef result[NIR_MAX_VEC_COMPONENTS])
1071 {
1072 struct gallivm_state *gallivm = bld_base->base.gallivm;
1073 LLVMBuilderRef builder = gallivm->builder;
1074 nir_deref_instr *deref = nir_instr_as_deref(instr->src[0].ssa->parent_instr);
1075 nir_variable *var = nir_deref_instr_get_variable(deref);
1076 LLVMValueRef coord_val = get_src(bld_base, instr->src[1]);
1077 LLVMValueRef coords[5];
1078 struct lp_img_params params;
1079 const struct glsl_type *type = glsl_without_array(var->type);
1080
1081 memset(&params, 0, sizeof(params));
1082 params.target = glsl_sampler_to_pipe(glsl_get_sampler_dim(type), glsl_sampler_type_is_array(type));
1083 for (unsigned i = 0; i < 4; i++)
1084 coords[i] = LLVMBuildExtractValue(builder, coord_val, i, "");
1085 if (params.target == PIPE_TEXTURE_1D_ARRAY)
1086 coords[2] = coords[1];
1087
1088 params.coords = coords;
1089 params.outdata = result;
1090 params.img_op = LP_IMG_LOAD;
1091 if (glsl_get_sampler_dim(type) == GLSL_SAMPLER_DIM_MS)
1092 params.ms_index = get_src(bld_base, instr->src[2]);
1093 params.image_index = var->data.binding;
1094 bld_base->image_op(bld_base, &params);
1095 }
1096
1097 static void visit_store_image(struct lp_build_nir_context *bld_base,
1098 nir_intrinsic_instr *instr)
1099 {
1100 struct gallivm_state *gallivm = bld_base->base.gallivm;
1101 LLVMBuilderRef builder = gallivm->builder;
1102 nir_deref_instr *deref = nir_instr_as_deref(instr->src[0].ssa->parent_instr);
1103 nir_variable *var = nir_deref_instr_get_variable(deref);
1104 LLVMValueRef coord_val = get_src(bld_base, instr->src[1]);
1105 LLVMValueRef in_val = get_src(bld_base, instr->src[3]);
1106 LLVMValueRef coords[5];
1107 struct lp_img_params params;
1108 const struct glsl_type *type = glsl_without_array(var->type);
1109
1110 memset(&params, 0, sizeof(params));
1111 params.target = glsl_sampler_to_pipe(glsl_get_sampler_dim(type), glsl_sampler_type_is_array(type));
1112 for (unsigned i = 0; i < 4; i++)
1113 coords[i] = LLVMBuildExtractValue(builder, coord_val, i, "");
1114 if (params.target == PIPE_TEXTURE_1D_ARRAY)
1115 coords[2] = coords[1];
1116 params.coords = coords;
1117
1118 for (unsigned i = 0; i < 4; i++) {
1119 params.indata[i] = LLVMBuildExtractValue(builder, in_val, i, "");
1120 params.indata[i] = LLVMBuildBitCast(builder, params.indata[i], bld_base->base.vec_type, "");
1121 }
1122 if (glsl_get_sampler_dim(type) == GLSL_SAMPLER_DIM_MS)
1123 params.ms_index = get_src(bld_base, instr->src[2]);
1124 params.img_op = LP_IMG_STORE;
1125 params.image_index = var->data.binding;
1126
1127 if (params.target == PIPE_TEXTURE_1D_ARRAY)
1128 coords[2] = coords[1];
1129 bld_base->image_op(bld_base, &params);
1130 }
1131
1132 static void visit_atomic_image(struct lp_build_nir_context *bld_base,
1133 nir_intrinsic_instr *instr,
1134 LLVMValueRef result[NIR_MAX_VEC_COMPONENTS])
1135 {
1136 struct gallivm_state *gallivm = bld_base->base.gallivm;
1137 LLVMBuilderRef builder = gallivm->builder;
1138 nir_deref_instr *deref = nir_instr_as_deref(instr->src[0].ssa->parent_instr);
1139 nir_variable *var = nir_deref_instr_get_variable(deref);
1140 struct lp_img_params params;
1141 LLVMValueRef coord_val = get_src(bld_base, instr->src[1]);
1142 LLVMValueRef in_val = get_src(bld_base, instr->src[3]);
1143 LLVMValueRef coords[5];
1144 const struct glsl_type *type = glsl_without_array(var->type);
1145
1146 memset(&params, 0, sizeof(params));
1147
1148 switch (instr->intrinsic) {
1149 case nir_intrinsic_image_deref_atomic_add:
1150 params.op = LLVMAtomicRMWBinOpAdd;
1151 break;
1152 case nir_intrinsic_image_deref_atomic_exchange:
1153 params.op = LLVMAtomicRMWBinOpXchg;
1154 break;
1155 case nir_intrinsic_image_deref_atomic_and:
1156 params.op = LLVMAtomicRMWBinOpAnd;
1157 break;
1158 case nir_intrinsic_image_deref_atomic_or:
1159 params.op = LLVMAtomicRMWBinOpOr;
1160 break;
1161 case nir_intrinsic_image_deref_atomic_xor:
1162 params.op = LLVMAtomicRMWBinOpXor;
1163 break;
1164 case nir_intrinsic_image_deref_atomic_umin:
1165 params.op = LLVMAtomicRMWBinOpUMin;
1166 break;
1167 case nir_intrinsic_image_deref_atomic_umax:
1168 params.op = LLVMAtomicRMWBinOpUMax;
1169 break;
1170 case nir_intrinsic_image_deref_atomic_imin:
1171 params.op = LLVMAtomicRMWBinOpMin;
1172 break;
1173 case nir_intrinsic_image_deref_atomic_imax:
1174 params.op = LLVMAtomicRMWBinOpMax;
1175 break;
1176 default:
1177 break;
1178 }
1179
1180 params.target = glsl_sampler_to_pipe(glsl_get_sampler_dim(type), glsl_sampler_type_is_array(type));
1181 for (unsigned i = 0; i < 4; i++)
1182 coords[i] = LLVMBuildExtractValue(builder, coord_val, i, "");
1183 if (params.target == PIPE_TEXTURE_1D_ARRAY)
1184 coords[2] = coords[1];
1185 params.coords = coords;
1186 if (glsl_get_sampler_dim(type) == GLSL_SAMPLER_DIM_MS)
1187 params.ms_index = get_src(bld_base, instr->src[2]);
1188 if (instr->intrinsic == nir_intrinsic_image_deref_atomic_comp_swap) {
1189 LLVMValueRef cas_val = get_src(bld_base, instr->src[4]);
1190 params.indata[0] = in_val;
1191 params.indata2[0] = cas_val;
1192 } else
1193 params.indata[0] = in_val;
1194
1195 params.outdata = result;
1196 params.img_op = (instr->intrinsic == nir_intrinsic_image_deref_atomic_comp_swap) ? LP_IMG_ATOMIC_CAS : LP_IMG_ATOMIC;
1197 params.image_index = var->data.binding;
1198
1199 bld_base->image_op(bld_base, &params);
1200 }
1201
1202
1203 static void visit_image_size(struct lp_build_nir_context *bld_base,
1204 nir_intrinsic_instr *instr,
1205 LLVMValueRef result[NIR_MAX_VEC_COMPONENTS])
1206 {
1207 nir_deref_instr *deref = nir_instr_as_deref(instr->src[0].ssa->parent_instr);
1208 nir_variable *var = nir_deref_instr_get_variable(deref);
1209 struct lp_sampler_size_query_params params = { 0 };
1210 params.texture_unit = var->data.binding;
1211 params.target = glsl_sampler_to_pipe(glsl_get_sampler_dim(var->type), glsl_sampler_type_is_array(var->type));
1212 params.sizes_out = result;
1213
1214 bld_base->image_size(bld_base, &params);
1215 }
1216
1217 static void visit_image_samples(struct lp_build_nir_context *bld_base,
1218 nir_intrinsic_instr *instr,
1219 LLVMValueRef result[NIR_MAX_VEC_COMPONENTS])
1220 {
1221 nir_deref_instr *deref = nir_instr_as_deref(instr->src[0].ssa->parent_instr);
1222 nir_variable *var = nir_deref_instr_get_variable(deref);
1223 struct lp_sampler_size_query_params params = { 0 };
1224 params.texture_unit = var->data.binding;
1225 params.target = glsl_sampler_to_pipe(glsl_get_sampler_dim(var->type), glsl_sampler_type_is_array(var->type));
1226 params.sizes_out = result;
1227 params.samples_only = true;
1228
1229 bld_base->image_size(bld_base, &params);
1230 }
1231
1232 static void visit_shared_load(struct lp_build_nir_context *bld_base,
1233 nir_intrinsic_instr *instr,
1234 LLVMValueRef result[NIR_MAX_VEC_COMPONENTS])
1235 {
1236 LLVMValueRef offset = get_src(bld_base, instr->src[0]);
1237 bld_base->load_mem(bld_base, nir_dest_num_components(instr->dest), nir_dest_bit_size(instr->dest),
1238 NULL, offset, result);
1239 }
1240
1241 static void visit_shared_store(struct lp_build_nir_context *bld_base,
1242 nir_intrinsic_instr *instr)
1243 {
1244 LLVMValueRef val = get_src(bld_base, instr->src[0]);
1245 LLVMValueRef offset = get_src(bld_base, instr->src[1]);
1246 int writemask = instr->const_index[1];
1247 int nc = nir_src_num_components(instr->src[0]);
1248 int bitsize = nir_src_bit_size(instr->src[0]);
1249 bld_base->store_mem(bld_base, writemask, nc, bitsize, NULL, offset, val);
1250 }
1251
1252 static void visit_shared_atomic(struct lp_build_nir_context *bld_base,
1253 nir_intrinsic_instr *instr,
1254 LLVMValueRef result[NIR_MAX_VEC_COMPONENTS])
1255 {
1256 LLVMValueRef offset = get_src(bld_base, instr->src[0]);
1257 LLVMValueRef val = get_src(bld_base, instr->src[1]);
1258 LLVMValueRef val2 = NULL;
1259 if (instr->intrinsic == nir_intrinsic_shared_atomic_comp_swap)
1260 val2 = get_src(bld_base, instr->src[2]);
1261
1262 bld_base->atomic_mem(bld_base, instr->intrinsic, NULL, offset, val, val2, &result[0]);
1263
1264 }
1265
1266 static void visit_barrier(struct lp_build_nir_context *bld_base)
1267 {
1268 bld_base->barrier(bld_base);
1269 }
1270
1271 static void visit_discard(struct lp_build_nir_context *bld_base,
1272 nir_intrinsic_instr *instr)
1273 {
1274 LLVMValueRef cond = NULL;
1275 if (instr->intrinsic == nir_intrinsic_discard_if) {
1276 cond = get_src(bld_base, instr->src[0]);
1277 cond = cast_type(bld_base, cond, nir_type_int, 32);
1278 }
1279 bld_base->discard(bld_base, cond);
1280 }
1281
1282 static void visit_load_kernel_input(struct lp_build_nir_context *bld_base,
1283 nir_intrinsic_instr *instr, LLVMValueRef result[NIR_MAX_VEC_COMPONENTS])
1284 {
1285 LLVMValueRef offset = get_src(bld_base, instr->src[0]);
1286
1287 bool offset_is_uniform = nir_src_is_dynamically_uniform(instr->src[0]);
1288 bld_base->load_kernel_arg(bld_base, nir_dest_num_components(instr->dest), nir_dest_bit_size(instr->dest),
1289 nir_src_bit_size(instr->src[0]),
1290 offset_is_uniform, offset, result);
1291 }
1292
1293 static void visit_load_global(struct lp_build_nir_context *bld_base,
1294 nir_intrinsic_instr *instr, LLVMValueRef result[NIR_MAX_VEC_COMPONENTS])
1295 {
1296 LLVMValueRef addr = get_src(bld_base, instr->src[0]);
1297 bld_base->load_global(bld_base, nir_dest_num_components(instr->dest), nir_dest_bit_size(instr->dest),
1298 nir_src_bit_size(instr->src[0]),
1299 addr, result);
1300 }
1301
1302 static void visit_store_global(struct lp_build_nir_context *bld_base,
1303 nir_intrinsic_instr *instr)
1304 {
1305 LLVMValueRef val = get_src(bld_base, instr->src[0]);
1306 int nc = nir_src_num_components(instr->src[0]);
1307 int bitsize = nir_src_bit_size(instr->src[0]);
1308 LLVMValueRef addr = get_src(bld_base, instr->src[1]);
1309 int addr_bitsize = nir_src_bit_size(instr->src[1]);
1310 int writemask = instr->const_index[0];
1311 bld_base->store_global(bld_base, writemask, nc, bitsize, addr_bitsize, addr, val);
1312 }
1313
1314 static void visit_global_atomic(struct lp_build_nir_context *bld_base,
1315 nir_intrinsic_instr *instr,
1316 LLVMValueRef result[NIR_MAX_VEC_COMPONENTS])
1317 {
1318 LLVMValueRef addr = get_src(bld_base, instr->src[0]);
1319 LLVMValueRef val = get_src(bld_base, instr->src[1]);
1320 LLVMValueRef val2 = NULL;
1321 int addr_bitsize = nir_src_bit_size(instr->src[0]);
1322 if (instr->intrinsic == nir_intrinsic_global_atomic_comp_swap)
1323 val2 = get_src(bld_base, instr->src[2]);
1324
1325 bld_base->atomic_global(bld_base, instr->intrinsic, addr_bitsize, addr, val, val2, &result[0]);
1326 }
1327
1328 static void visit_interp(struct lp_build_nir_context *bld_base,
1329 nir_intrinsic_instr *instr,
1330 LLVMValueRef result[NIR_MAX_VEC_COMPONENTS])
1331 {
1332 struct gallivm_state *gallivm = bld_base->base.gallivm;
1333 LLVMBuilderRef builder = gallivm->builder;
1334 nir_deref_instr *deref = nir_instr_as_deref(instr->src[0].ssa->parent_instr);
1335 unsigned num_components = nir_dest_num_components(instr->dest);
1336 nir_variable *var = nir_deref_instr_get_variable(deref);
1337 unsigned const_index;
1338 LLVMValueRef indir_index;
1339 LLVMValueRef offsets[2] = { NULL, NULL };
1340 get_deref_offset(bld_base, deref, false, NULL, NULL,
1341 &const_index, &indir_index);
1342 bool centroid = instr->intrinsic == nir_intrinsic_interp_deref_at_centroid;
1343 bool sample = false;
1344 if (instr->intrinsic == nir_intrinsic_interp_deref_at_offset) {
1345 for (unsigned i = 0; i < 2; i++) {
1346 offsets[i] = LLVMBuildExtractValue(builder, get_src(bld_base, instr->src[1]), i, "");
1347 offsets[i] = cast_type(bld_base, offsets[i], nir_type_float, 32);
1348 }
1349 } else if (instr->intrinsic == nir_intrinsic_interp_deref_at_sample) {
1350 offsets[0] = get_src(bld_base, instr->src[1]);
1351 offsets[0] = cast_type(bld_base, offsets[0], nir_type_int, 32);
1352 sample = true;
1353 }
1354 bld_base->interp_at(bld_base, num_components, var, centroid, sample, const_index, indir_index, offsets, result);
1355 }
1356
1357 static void visit_intrinsic(struct lp_build_nir_context *bld_base,
1358 nir_intrinsic_instr *instr)
1359 {
1360 LLVMValueRef result[NIR_MAX_VEC_COMPONENTS] = {0};
1361 switch (instr->intrinsic) {
1362 case nir_intrinsic_load_deref:
1363 visit_load_var(bld_base, instr, result);
1364 break;
1365 case nir_intrinsic_store_deref:
1366 visit_store_var(bld_base, instr);
1367 break;
1368 case nir_intrinsic_load_ubo:
1369 visit_load_ubo(bld_base, instr, result);
1370 break;
1371 case nir_intrinsic_load_ssbo:
1372 visit_load_ssbo(bld_base, instr, result);
1373 break;
1374 case nir_intrinsic_store_ssbo:
1375 visit_store_ssbo(bld_base, instr);
1376 break;
1377 case nir_intrinsic_get_buffer_size:
1378 visit_get_buffer_size(bld_base, instr, result);
1379 break;
1380 case nir_intrinsic_load_vertex_id:
1381 case nir_intrinsic_load_primitive_id:
1382 case nir_intrinsic_load_instance_id:
1383 case nir_intrinsic_load_base_instance:
1384 case nir_intrinsic_load_base_vertex:
1385 case nir_intrinsic_load_work_group_id:
1386 case nir_intrinsic_load_local_invocation_id:
1387 case nir_intrinsic_load_num_work_groups:
1388 case nir_intrinsic_load_invocation_id:
1389 case nir_intrinsic_load_front_face:
1390 case nir_intrinsic_load_draw_id:
1391 case nir_intrinsic_load_local_group_size:
1392 case nir_intrinsic_load_work_dim:
1393 case nir_intrinsic_load_tess_coord:
1394 case nir_intrinsic_load_tess_level_outer:
1395 case nir_intrinsic_load_tess_level_inner:
1396 case nir_intrinsic_load_patch_vertices_in:
1397 case nir_intrinsic_load_sample_id:
1398 case nir_intrinsic_load_sample_pos:
1399 case nir_intrinsic_load_sample_mask_in:
1400 bld_base->sysval_intrin(bld_base, instr, result);
1401 break;
1402 case nir_intrinsic_load_helper_invocation:
1403 bld_base->helper_invocation(bld_base, &result[0]);
1404 break;
1405 case nir_intrinsic_discard_if:
1406 case nir_intrinsic_discard:
1407 visit_discard(bld_base, instr);
1408 break;
1409 case nir_intrinsic_emit_vertex:
1410 bld_base->emit_vertex(bld_base, nir_intrinsic_stream_id(instr));
1411 break;
1412 case nir_intrinsic_end_primitive:
1413 bld_base->end_primitive(bld_base, nir_intrinsic_stream_id(instr));
1414 break;
1415 case nir_intrinsic_ssbo_atomic_add:
1416 case nir_intrinsic_ssbo_atomic_imin:
1417 case nir_intrinsic_ssbo_atomic_imax:
1418 case nir_intrinsic_ssbo_atomic_umin:
1419 case nir_intrinsic_ssbo_atomic_umax:
1420 case nir_intrinsic_ssbo_atomic_and:
1421 case nir_intrinsic_ssbo_atomic_or:
1422 case nir_intrinsic_ssbo_atomic_xor:
1423 case nir_intrinsic_ssbo_atomic_exchange:
1424 case nir_intrinsic_ssbo_atomic_comp_swap:
1425 visit_ssbo_atomic(bld_base, instr, result);
1426 break;
1427 case nir_intrinsic_image_deref_load:
1428 visit_load_image(bld_base, instr, result);
1429 break;
1430 case nir_intrinsic_image_deref_store:
1431 visit_store_image(bld_base, instr);
1432 break;
1433 case nir_intrinsic_image_deref_atomic_add:
1434 case nir_intrinsic_image_deref_atomic_imin:
1435 case nir_intrinsic_image_deref_atomic_imax:
1436 case nir_intrinsic_image_deref_atomic_umin:
1437 case nir_intrinsic_image_deref_atomic_umax:
1438 case nir_intrinsic_image_deref_atomic_and:
1439 case nir_intrinsic_image_deref_atomic_or:
1440 case nir_intrinsic_image_deref_atomic_xor:
1441 case nir_intrinsic_image_deref_atomic_exchange:
1442 case nir_intrinsic_image_deref_atomic_comp_swap:
1443 visit_atomic_image(bld_base, instr, result);
1444 break;
1445 case nir_intrinsic_image_deref_size:
1446 visit_image_size(bld_base, instr, result);
1447 break;
1448 case nir_intrinsic_image_deref_samples:
1449 visit_image_samples(bld_base, instr, result);
1450 break;
1451 case nir_intrinsic_load_shared:
1452 visit_shared_load(bld_base, instr, result);
1453 break;
1454 case nir_intrinsic_store_shared:
1455 visit_shared_store(bld_base, instr);
1456 break;
1457 case nir_intrinsic_shared_atomic_add:
1458 case nir_intrinsic_shared_atomic_imin:
1459 case nir_intrinsic_shared_atomic_umin:
1460 case nir_intrinsic_shared_atomic_imax:
1461 case nir_intrinsic_shared_atomic_umax:
1462 case nir_intrinsic_shared_atomic_and:
1463 case nir_intrinsic_shared_atomic_or:
1464 case nir_intrinsic_shared_atomic_xor:
1465 case nir_intrinsic_shared_atomic_exchange:
1466 case nir_intrinsic_shared_atomic_comp_swap:
1467 visit_shared_atomic(bld_base, instr, result);
1468 break;
1469 case nir_intrinsic_control_barrier:
1470 visit_barrier(bld_base);
1471 break;
1472 case nir_intrinsic_group_memory_barrier:
1473 case nir_intrinsic_memory_barrier:
1474 case nir_intrinsic_memory_barrier_shared:
1475 case nir_intrinsic_memory_barrier_buffer:
1476 case nir_intrinsic_memory_barrier_image:
1477 case nir_intrinsic_memory_barrier_tcs_patch:
1478 break;
1479 case nir_intrinsic_load_kernel_input:
1480 visit_load_kernel_input(bld_base, instr, result);
1481 break;
1482 case nir_intrinsic_load_global:
1483 visit_load_global(bld_base, instr, result);
1484 break;
1485 case nir_intrinsic_store_global:
1486 visit_store_global(bld_base, instr);
1487 break;
1488 case nir_intrinsic_global_atomic_add:
1489 case nir_intrinsic_global_atomic_imin:
1490 case nir_intrinsic_global_atomic_umin:
1491 case nir_intrinsic_global_atomic_imax:
1492 case nir_intrinsic_global_atomic_umax:
1493 case nir_intrinsic_global_atomic_and:
1494 case nir_intrinsic_global_atomic_or:
1495 case nir_intrinsic_global_atomic_xor:
1496 case nir_intrinsic_global_atomic_exchange:
1497 case nir_intrinsic_global_atomic_comp_swap:
1498 visit_global_atomic(bld_base, instr, result);
1499 break;
1500 case nir_intrinsic_vote_all:
1501 case nir_intrinsic_vote_any:
1502 case nir_intrinsic_vote_ieq:
1503 bld_base->vote(bld_base, cast_type(bld_base, get_src(bld_base, instr->src[0]), nir_type_int, 32), instr, result);
1504 break;
1505 case nir_intrinsic_interp_deref_at_offset:
1506 case nir_intrinsic_interp_deref_at_centroid:
1507 case nir_intrinsic_interp_deref_at_sample:
1508 visit_interp(bld_base, instr, result);
1509 break;
1510 default:
1511 assert(0);
1512 break;
1513 }
1514 if (result[0]) {
1515 assign_dest(bld_base, &instr->dest, result);
1516 }
1517 }
1518
1519 static void visit_txs(struct lp_build_nir_context *bld_base, nir_tex_instr *instr)
1520 {
1521 struct lp_sampler_size_query_params params = { 0 };
1522 LLVMValueRef sizes_out[NIR_MAX_VEC_COMPONENTS];
1523 LLVMValueRef explicit_lod = NULL;
1524 LLVMValueRef texture_unit_offset = NULL;
1525 for (unsigned i = 0; i < instr->num_srcs; i++) {
1526 switch (instr->src[i].src_type) {
1527 case nir_tex_src_lod:
1528 explicit_lod = cast_type(bld_base, get_src(bld_base, instr->src[i].src), nir_type_int, 32);
1529 break;
1530 case nir_tex_src_texture_offset:
1531 texture_unit_offset = get_src(bld_base, instr->src[i].src);
1532 break;
1533 default:
1534 break;
1535 }
1536 }
1537
1538 params.target = glsl_sampler_to_pipe(instr->sampler_dim, instr->is_array);
1539 params.texture_unit = instr->texture_index;
1540 params.explicit_lod = explicit_lod;
1541 params.is_sviewinfo = TRUE;
1542 params.sizes_out = sizes_out;
1543 params.samples_only = (instr->op == nir_texop_texture_samples);
1544 params.texture_unit_offset = texture_unit_offset;
1545
1546 if (instr->op == nir_texop_query_levels)
1547 params.explicit_lod = bld_base->uint_bld.zero;
1548 bld_base->tex_size(bld_base, &params);
1549 assign_dest(bld_base, &instr->dest, &sizes_out[instr->op == nir_texop_query_levels ? 3 : 0]);
1550 }
1551
1552 static enum lp_sampler_lod_property lp_build_nir_lod_property(struct lp_build_nir_context *bld_base,
1553 nir_src lod_src)
1554 {
1555 enum lp_sampler_lod_property lod_property;
1556
1557 if (nir_src_is_dynamically_uniform(lod_src))
1558 lod_property = LP_SAMPLER_LOD_SCALAR;
1559 else if (bld_base->shader->info.stage == MESA_SHADER_FRAGMENT) {
1560 if (gallivm_perf & GALLIVM_PERF_NO_QUAD_LOD)
1561 lod_property = LP_SAMPLER_LOD_PER_ELEMENT;
1562 else
1563 lod_property = LP_SAMPLER_LOD_PER_QUAD;
1564 }
1565 else
1566 lod_property = LP_SAMPLER_LOD_PER_ELEMENT;
1567 return lod_property;
1568 }
1569
1570 static void visit_tex(struct lp_build_nir_context *bld_base, nir_tex_instr *instr)
1571 {
1572 struct gallivm_state *gallivm = bld_base->base.gallivm;
1573 LLVMBuilderRef builder = gallivm->builder;
1574 LLVMValueRef coords[5];
1575 LLVMValueRef offsets[3] = { NULL };
1576 LLVMValueRef explicit_lod = NULL, projector = NULL, ms_index = NULL;
1577 struct lp_sampler_params params;
1578 struct lp_derivatives derivs;
1579 unsigned sample_key = 0;
1580 nir_deref_instr *texture_deref_instr = NULL;
1581 nir_deref_instr *sampler_deref_instr = NULL;
1582 LLVMValueRef texture_unit_offset = NULL;
1583 LLVMValueRef texel[NIR_MAX_VEC_COMPONENTS];
1584 unsigned lod_src = 0;
1585 LLVMValueRef coord_undef = LLVMGetUndef(bld_base->base.int_vec_type);
1586
1587 memset(&params, 0, sizeof(params));
1588 enum lp_sampler_lod_property lod_property = LP_SAMPLER_LOD_SCALAR;
1589
1590 if (instr->op == nir_texop_txs || instr->op == nir_texop_query_levels || instr->op == nir_texop_texture_samples) {
1591 visit_txs(bld_base, instr);
1592 return;
1593 }
1594 if (instr->op == nir_texop_txf || instr->op == nir_texop_txf_ms)
1595 sample_key |= LP_SAMPLER_OP_FETCH << LP_SAMPLER_OP_TYPE_SHIFT;
1596 else if (instr->op == nir_texop_tg4) {
1597 sample_key |= LP_SAMPLER_OP_GATHER << LP_SAMPLER_OP_TYPE_SHIFT;
1598 sample_key |= (instr->component << LP_SAMPLER_GATHER_COMP_SHIFT);
1599 } else if (instr->op == nir_texop_lod)
1600 sample_key |= LP_SAMPLER_OP_LODQ << LP_SAMPLER_OP_TYPE_SHIFT;
1601 for (unsigned i = 0; i < instr->num_srcs; i++) {
1602 switch (instr->src[i].src_type) {
1603 case nir_tex_src_coord: {
1604 LLVMValueRef coord = get_src(bld_base, instr->src[i].src);
1605 if (instr->coord_components == 1)
1606 coords[0] = coord;
1607 else {
1608 for (unsigned chan = 0; chan < instr->coord_components; ++chan)
1609 coords[chan] = LLVMBuildExtractValue(builder, coord,
1610 chan, "");
1611 }
1612 for (unsigned chan = instr->coord_components; chan < 5; chan++)
1613 coords[chan] = coord_undef;
1614
1615 break;
1616 }
1617 case nir_tex_src_texture_deref:
1618 texture_deref_instr = nir_src_as_deref(instr->src[i].src);
1619 break;
1620 case nir_tex_src_sampler_deref:
1621 sampler_deref_instr = nir_src_as_deref(instr->src[i].src);
1622 break;
1623 case nir_tex_src_projector:
1624 projector = lp_build_rcp(&bld_base->base, cast_type(bld_base, get_src(bld_base, instr->src[i].src), nir_type_float, 32));
1625 break;
1626 case nir_tex_src_comparator:
1627 sample_key |= LP_SAMPLER_SHADOW;
1628 coords[4] = get_src(bld_base, instr->src[i].src);
1629 coords[4] = cast_type(bld_base, coords[4], nir_type_float, 32);
1630 break;
1631 case nir_tex_src_bias:
1632 sample_key |= LP_SAMPLER_LOD_BIAS << LP_SAMPLER_LOD_CONTROL_SHIFT;
1633 lod_src = i;
1634 explicit_lod = cast_type(bld_base, get_src(bld_base, instr->src[i].src), nir_type_float, 32);
1635 break;
1636 case nir_tex_src_lod:
1637 sample_key |= LP_SAMPLER_LOD_EXPLICIT << LP_SAMPLER_LOD_CONTROL_SHIFT;
1638 lod_src = i;
1639 if (instr->op == nir_texop_txf)
1640 explicit_lod = cast_type(bld_base, get_src(bld_base, instr->src[i].src), nir_type_int, 32);
1641 else
1642 explicit_lod = cast_type(bld_base, get_src(bld_base, instr->src[i].src), nir_type_float, 32);
1643 break;
1644 case nir_tex_src_ddx: {
1645 int deriv_cnt = instr->coord_components;
1646 if (instr->is_array)
1647 deriv_cnt--;
1648 LLVMValueRef deriv_val = get_src(bld_base, instr->src[i].src);
1649 if (deriv_cnt == 1)
1650 derivs.ddx[0] = deriv_val;
1651 else
1652 for (unsigned chan = 0; chan < deriv_cnt; ++chan)
1653 derivs.ddx[chan] = LLVMBuildExtractValue(builder, deriv_val,
1654 chan, "");
1655 for (unsigned chan = 0; chan < deriv_cnt; ++chan)
1656 derivs.ddx[chan] = cast_type(bld_base, derivs.ddx[chan], nir_type_float, 32);
1657 break;
1658 }
1659 case nir_tex_src_ddy: {
1660 int deriv_cnt = instr->coord_components;
1661 if (instr->is_array)
1662 deriv_cnt--;
1663 LLVMValueRef deriv_val = get_src(bld_base, instr->src[i].src);
1664 if (deriv_cnt == 1)
1665 derivs.ddy[0] = deriv_val;
1666 else
1667 for (unsigned chan = 0; chan < deriv_cnt; ++chan)
1668 derivs.ddy[chan] = LLVMBuildExtractValue(builder, deriv_val,
1669 chan, "");
1670 for (unsigned chan = 0; chan < deriv_cnt; ++chan)
1671 derivs.ddy[chan] = cast_type(bld_base, derivs.ddy[chan], nir_type_float, 32);
1672 break;
1673 }
1674 case nir_tex_src_offset: {
1675 int offset_cnt = instr->coord_components;
1676 if (instr->is_array)
1677 offset_cnt--;
1678 LLVMValueRef offset_val = get_src(bld_base, instr->src[i].src);
1679 sample_key |= LP_SAMPLER_OFFSETS;
1680 if (offset_cnt == 1)
1681 offsets[0] = cast_type(bld_base, offset_val, nir_type_int, 32);
1682 else {
1683 for (unsigned chan = 0; chan < offset_cnt; ++chan) {
1684 offsets[chan] = LLVMBuildExtractValue(builder, offset_val,
1685 chan, "");
1686 offsets[chan] = cast_type(bld_base, offsets[chan], nir_type_int, 32);
1687 }
1688 }
1689 break;
1690 }
1691 case nir_tex_src_ms_index:
1692 sample_key |= LP_SAMPLER_FETCH_MS;
1693 ms_index = cast_type(bld_base, get_src(bld_base, instr->src[i].src), nir_type_int, 32);
1694 break;
1695
1696 case nir_tex_src_texture_offset:
1697 texture_unit_offset = get_src(bld_base, instr->src[i].src);
1698 break;
1699 case nir_tex_src_sampler_offset:
1700 break;
1701 default:
1702 assert(0);
1703 break;
1704 }
1705 }
1706 if (!sampler_deref_instr)
1707 sampler_deref_instr = texture_deref_instr;
1708
1709 if (explicit_lod)
1710 lod_property = lp_build_nir_lod_property(bld_base, instr->src[lod_src].src);
1711
1712 if (instr->op == nir_texop_tex || instr->op == nir_texop_tg4 || instr->op == nir_texop_txb ||
1713 instr->op == nir_texop_txl || instr->op == nir_texop_txd || instr->op == nir_texop_lod)
1714 for (unsigned chan = 0; chan < instr->coord_components; ++chan)
1715 coords[chan] = cast_type(bld_base, coords[chan], nir_type_float, 32);
1716 else if (instr->op == nir_texop_txf || instr->op == nir_texop_txf_ms)
1717 for (unsigned chan = 0; chan < instr->coord_components; ++chan)
1718 coords[chan] = cast_type(bld_base, coords[chan], nir_type_int, 32);
1719
1720 if (instr->is_array && instr->sampler_dim == GLSL_SAMPLER_DIM_1D) {
1721 /* move layer coord for 1d arrays. */
1722 coords[2] = coords[1];
1723 coords[1] = coord_undef;
1724 }
1725
1726 if (projector) {
1727 for (unsigned chan = 0; chan < instr->coord_components; ++chan)
1728 coords[chan] = lp_build_mul(&bld_base->base, coords[chan], projector);
1729 if (sample_key & LP_SAMPLER_SHADOW)
1730 coords[4] = lp_build_mul(&bld_base->base, coords[4], projector);
1731 }
1732
1733 uint32_t base_index = 0;
1734 if (!texture_deref_instr) {
1735 int samp_src_index = nir_tex_instr_src_index(instr, nir_tex_src_sampler_handle);
1736 if (samp_src_index == -1) {
1737 base_index = instr->sampler_index;
1738 }
1739 }
1740
1741 if (instr->op == nir_texop_txd) {
1742 sample_key |= LP_SAMPLER_LOD_DERIVATIVES << LP_SAMPLER_LOD_CONTROL_SHIFT;
1743 params.derivs = &derivs;
1744 if (bld_base->shader->info.stage == MESA_SHADER_FRAGMENT) {
1745 if (gallivm_perf & GALLIVM_PERF_NO_QUAD_LOD)
1746 lod_property = LP_SAMPLER_LOD_PER_ELEMENT;
1747 else
1748 lod_property = LP_SAMPLER_LOD_PER_QUAD;
1749 } else
1750 lod_property = LP_SAMPLER_LOD_PER_ELEMENT;
1751 }
1752
1753 sample_key |= lod_property << LP_SAMPLER_LOD_PROPERTY_SHIFT;
1754 params.sample_key = sample_key;
1755 params.offsets = offsets;
1756 params.texture_index = base_index;
1757 params.texture_index_offset = texture_unit_offset;
1758 params.sampler_index = base_index;
1759 params.coords = coords;
1760 params.texel = texel;
1761 params.lod = explicit_lod;
1762 params.ms_index = ms_index;
1763 bld_base->tex(bld_base, &params);
1764 assign_dest(bld_base, &instr->dest, texel);
1765 }
1766
1767 static void visit_ssa_undef(struct lp_build_nir_context *bld_base,
1768 const nir_ssa_undef_instr *instr)
1769 {
1770 unsigned num_components = instr->def.num_components;
1771 LLVMValueRef undef[NIR_MAX_VEC_COMPONENTS];
1772 struct lp_build_context *undef_bld = get_int_bld(bld_base, true, instr->def.bit_size);
1773 for (unsigned i = 0; i < num_components; i++)
1774 undef[i] = LLVMGetUndef(undef_bld->vec_type);
1775 assign_ssa_dest(bld_base, &instr->def, undef);
1776 }
1777
1778 static void visit_jump(struct lp_build_nir_context *bld_base,
1779 const nir_jump_instr *instr)
1780 {
1781 switch (instr->type) {
1782 case nir_jump_break:
1783 bld_base->break_stmt(bld_base);
1784 break;
1785 case nir_jump_continue:
1786 bld_base->continue_stmt(bld_base);
1787 break;
1788 default:
1789 unreachable("Unknown jump instr\n");
1790 }
1791 }
1792
1793 static void visit_deref(struct lp_build_nir_context *bld_base,
1794 nir_deref_instr *instr)
1795 {
1796 if (instr->mode != nir_var_mem_shared &&
1797 instr->mode != nir_var_mem_global)
1798 return;
1799 LLVMValueRef result = NULL;
1800 switch(instr->deref_type) {
1801 case nir_deref_type_var: {
1802 struct hash_entry *entry = _mesa_hash_table_search(bld_base->vars, instr->var);
1803 result = entry->data;
1804 break;
1805 }
1806 default:
1807 unreachable("Unhandled deref_instr deref type");
1808 }
1809
1810 assign_ssa(bld_base, instr->dest.ssa.index, result);
1811 }
1812
1813 static void visit_block(struct lp_build_nir_context *bld_base, nir_block *block)
1814 {
1815 nir_foreach_instr(instr, block)
1816 {
1817 switch (instr->type) {
1818 case nir_instr_type_alu:
1819 visit_alu(bld_base, nir_instr_as_alu(instr));
1820 break;
1821 case nir_instr_type_load_const:
1822 visit_load_const(bld_base, nir_instr_as_load_const(instr));
1823 break;
1824 case nir_instr_type_intrinsic:
1825 visit_intrinsic(bld_base, nir_instr_as_intrinsic(instr));
1826 break;
1827 case nir_instr_type_tex:
1828 visit_tex(bld_base, nir_instr_as_tex(instr));
1829 break;
1830 case nir_instr_type_phi:
1831 assert(0);
1832 break;
1833 case nir_instr_type_ssa_undef:
1834 visit_ssa_undef(bld_base, nir_instr_as_ssa_undef(instr));
1835 break;
1836 case nir_instr_type_jump:
1837 visit_jump(bld_base, nir_instr_as_jump(instr));
1838 break;
1839 case nir_instr_type_deref:
1840 visit_deref(bld_base, nir_instr_as_deref(instr));
1841 break;
1842 default:
1843 fprintf(stderr, "Unknown NIR instr type: ");
1844 nir_print_instr(instr, stderr);
1845 fprintf(stderr, "\n");
1846 abort();
1847 }
1848 }
1849 }
1850
1851 static void visit_if(struct lp_build_nir_context *bld_base, nir_if *if_stmt)
1852 {
1853 LLVMValueRef cond = get_src(bld_base, if_stmt->condition);
1854
1855 bld_base->if_cond(bld_base, cond);
1856 visit_cf_list(bld_base, &if_stmt->then_list);
1857
1858 if (!exec_list_is_empty(&if_stmt->else_list)) {
1859 bld_base->else_stmt(bld_base);
1860 visit_cf_list(bld_base, &if_stmt->else_list);
1861 }
1862 bld_base->endif_stmt(bld_base);
1863 }
1864
1865 static void visit_loop(struct lp_build_nir_context *bld_base, nir_loop *loop)
1866 {
1867 bld_base->bgnloop(bld_base);
1868 visit_cf_list(bld_base, &loop->body);
1869 bld_base->endloop(bld_base);
1870 }
1871
1872 static void visit_cf_list(struct lp_build_nir_context *bld_base,
1873 struct exec_list *list)
1874 {
1875 foreach_list_typed(nir_cf_node, node, node, list)
1876 {
1877 switch (node->type) {
1878 case nir_cf_node_block:
1879 visit_block(bld_base, nir_cf_node_as_block(node));
1880 break;
1881
1882 case nir_cf_node_if:
1883 visit_if(bld_base, nir_cf_node_as_if(node));
1884 break;
1885
1886 case nir_cf_node_loop:
1887 visit_loop(bld_base, nir_cf_node_as_loop(node));
1888 break;
1889
1890 default:
1891 assert(0);
1892 }
1893 }
1894 }
1895
1896 static void
1897 handle_shader_output_decl(struct lp_build_nir_context *bld_base,
1898 struct nir_shader *nir,
1899 struct nir_variable *variable)
1900 {
1901 bld_base->emit_var_decl(bld_base, variable);
1902 }
1903
1904 /* vector registers are stored as arrays in LLVM side,
1905 so we can use GEP on them, as to do exec mask stores
1906 we need to operate on a single components.
1907 arrays are:
1908 0.x, 1.x, 2.x, 3.x
1909 0.y, 1.y, 2.y, 3.y
1910 ....
1911 */
1912 static LLVMTypeRef get_register_type(struct lp_build_nir_context *bld_base,
1913 nir_register *reg)
1914 {
1915 struct lp_build_context *int_bld = get_int_bld(bld_base, true, reg->bit_size);
1916
1917 LLVMTypeRef type = int_bld->vec_type;
1918 if (reg->num_array_elems)
1919 type = LLVMArrayType(type, reg->num_array_elems);
1920 if (reg->num_components > 1)
1921 type = LLVMArrayType(type, reg->num_components);
1922
1923 return type;
1924 }
1925
1926
1927 bool lp_build_nir_llvm(
1928 struct lp_build_nir_context *bld_base,
1929 struct nir_shader *nir)
1930 {
1931 struct nir_function *func;
1932
1933 nir_convert_from_ssa(nir, true);
1934 nir_lower_locals_to_regs(nir);
1935 nir_remove_dead_derefs(nir);
1936 nir_remove_dead_variables(nir, nir_var_function_temp, NULL);
1937
1938 nir_foreach_variable(variable, &nir->outputs)
1939 handle_shader_output_decl(bld_base, nir, variable);
1940
1941 bld_base->regs = _mesa_hash_table_create(NULL, _mesa_hash_pointer,
1942 _mesa_key_pointer_equal);
1943 bld_base->vars = _mesa_hash_table_create(NULL, _mesa_hash_pointer,
1944 _mesa_key_pointer_equal);
1945
1946 func = (struct nir_function *)exec_list_get_head(&nir->functions);
1947
1948 nir_foreach_register(reg, &func->impl->registers) {
1949 LLVMTypeRef type = get_register_type(bld_base, reg);
1950 LLVMValueRef reg_alloc = lp_build_alloca_undef(bld_base->base.gallivm,
1951 type, "reg");
1952 _mesa_hash_table_insert(bld_base->regs, reg, reg_alloc);
1953 }
1954 nir_index_ssa_defs(func->impl);
1955 bld_base->ssa_defs = calloc(func->impl->ssa_alloc, sizeof(LLVMValueRef));
1956 visit_cf_list(bld_base, &func->impl->body);
1957
1958 free(bld_base->ssa_defs);
1959 ralloc_free(bld_base->vars);
1960 ralloc_free(bld_base->regs);
1961 return true;
1962 }
1963
1964 /* do some basic opts to remove some things we don't want to see. */
1965 void lp_build_opt_nir(struct nir_shader *nir)
1966 {
1967 bool progress;
1968 do {
1969 progress = false;
1970 NIR_PASS_V(nir, nir_opt_constant_folding);
1971 NIR_PASS_V(nir, nir_opt_algebraic);
1972 NIR_PASS_V(nir, nir_lower_pack);
1973
1974 nir_lower_tex_options options = { .lower_tex_without_implicit_lod = true };
1975 NIR_PASS_V(nir, nir_lower_tex, &options);
1976 } while (progress);
1977 nir_lower_bool_to_int32(nir);
1978 }