projects / mesa.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
history | raw | HEAD
Added few more stubs so that control reaches to DestroyDevice().
[mesa.git] / src / gallium / auxiliary / gallivm / lp_bld_nir.c
1 /**************************************************************************
2 *
3 * Copyright 2019 Red Hat.
4 * All Rights Reserved.
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the "Software"),
8 * to deal in the Software without restriction, including without limitation
9 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
10 * and/or sell copies of the Software, and to permit persons to whom the
11 * Software is furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included
14 * in all copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
17 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
22 * SOFTWARE.
23 *
24 **************************************************************************/
25
26 #include "lp_bld_nir.h"
27 #include "lp_bld_arit.h"
28 #include "lp_bld_bitarit.h"
29 #include "lp_bld_const.h"
30 #include "lp_bld_gather.h"
31 #include "lp_bld_logic.h"
32 #include "lp_bld_quad.h"
33 #include "lp_bld_flow.h"
34 #include "lp_bld_struct.h"
35 #include "lp_bld_debug.h"
36 #include "lp_bld_printf.h"
37 #include "nir_deref.h"
38
39 static void visit_cf_list(struct lp_build_nir_context *bld_base,
40 struct exec_list *list);
41
42 static LLVMValueRef cast_type(struct lp_build_nir_context *bld_base, LLVMValueRef val,
43 nir_alu_type alu_type, unsigned bit_size)
44 {
45 LLVMBuilderRef builder = bld_base->base.gallivm->builder;
46 switch (alu_type) {
47 case nir_type_float:
48 switch (bit_size) {
49 case 16:
50 return LLVMBuildBitCast(builder, val, LLVMVectorType(LLVMHalfTypeInContext(bld_base->base.gallivm->context), bld_base->base.type.length), "");
51 case 32:
52 return LLVMBuildBitCast(builder, val, bld_base->base.vec_type, "");
53 case 64:
54 return LLVMBuildBitCast(builder, val, bld_base->dbl_bld.vec_type, "");
55 default:
56 assert(0);
57 break;
58 }
59 break;
60 case nir_type_int:
61 switch (bit_size) {
62 case 8:
63 return LLVMBuildBitCast(builder, val, bld_base->int8_bld.vec_type, "");
64 case 16:
65 return LLVMBuildBitCast(builder, val, bld_base->int16_bld.vec_type, "");
66 case 32:
67 return LLVMBuildBitCast(builder, val, bld_base->int_bld.vec_type, "");
68 case 64:
69 return LLVMBuildBitCast(builder, val, bld_base->int64_bld.vec_type, "");
70 default:
71 assert(0);
72 break;
73 }
74 break;
75 case nir_type_uint:
76 switch (bit_size) {
77 case 8:
78 return LLVMBuildBitCast(builder, val, bld_base->uint8_bld.vec_type, "");
79 case 16:
80 return LLVMBuildBitCast(builder, val, bld_base->uint16_bld.vec_type, "");
81 case 32:
82 return LLVMBuildBitCast(builder, val, bld_base->uint_bld.vec_type, "");
83 case 64:
84 return LLVMBuildBitCast(builder, val, bld_base->uint64_bld.vec_type, "");
85 default:
86 assert(0);
87 break;
88 }
89 break;
90 case nir_type_uint32:
91 return LLVMBuildBitCast(builder, val, bld_base->uint_bld.vec_type, "");
92 default:
93 return val;
94 }
95 return NULL;
96 }
97
98
99 static struct lp_build_context *get_flt_bld(struct lp_build_nir_context *bld_base,
100 unsigned op_bit_size)
101 {
102 if (op_bit_size == 64)
103 return &bld_base->dbl_bld;
104 else
105 return &bld_base->base;
106 }
107
108 static unsigned glsl_sampler_to_pipe(int sampler_dim, bool is_array)
109 {
110 unsigned pipe_target = PIPE_BUFFER;
111 switch (sampler_dim) {
112 case GLSL_SAMPLER_DIM_1D:
113 pipe_target = is_array ? PIPE_TEXTURE_1D_ARRAY : PIPE_TEXTURE_1D;
114 break;
115 case GLSL_SAMPLER_DIM_2D:
116 case GLSL_SAMPLER_DIM_SUBPASS:
117 pipe_target = is_array ? PIPE_TEXTURE_2D_ARRAY : PIPE_TEXTURE_2D;
118 break;
119 case GLSL_SAMPLER_DIM_3D:
120 pipe_target = PIPE_TEXTURE_3D;
121 break;
122 case GLSL_SAMPLER_DIM_MS:
123 case GLSL_SAMPLER_DIM_SUBPASS_MS:
124 pipe_target = is_array ? PIPE_TEXTURE_2D_ARRAY : PIPE_TEXTURE_2D;
125 break;
126 case GLSL_SAMPLER_DIM_CUBE:
127 pipe_target = is_array ? PIPE_TEXTURE_CUBE_ARRAY : PIPE_TEXTURE_CUBE;
128 break;
129 case GLSL_SAMPLER_DIM_RECT:
130 pipe_target = PIPE_TEXTURE_RECT;
131 break;
132 case GLSL_SAMPLER_DIM_BUF:
133 pipe_target = PIPE_BUFFER;
134 break;
135 default:
136 break;
137 }
138 return pipe_target;
139 }
140
141 static LLVMValueRef get_ssa_src(struct lp_build_nir_context *bld_base, nir_ssa_def *ssa)
142 {
143 return bld_base->ssa_defs[ssa->index];
144 }
145
146 static LLVMValueRef get_src(struct lp_build_nir_context *bld_base, nir_src src);
147
148 static LLVMValueRef get_reg_src(struct lp_build_nir_context *bld_base, nir_reg_src src)
149 {
150 struct hash_entry *entry = _mesa_hash_table_search(bld_base->regs, src.reg);
151 LLVMValueRef reg_storage = (LLVMValueRef)entry->data;
152 struct lp_build_context *reg_bld = get_int_bld(bld_base, true, src.reg->bit_size);
153 LLVMValueRef indir_src = NULL;
154 if (src.indirect)
155 indir_src = get_src(bld_base, *src.indirect);
156 return bld_base->load_reg(bld_base, reg_bld, &src, indir_src, reg_storage);
157 }
158
159 static LLVMValueRef get_src(struct lp_build_nir_context *bld_base, nir_src src)
160 {
161 if (src.is_ssa)
162 return get_ssa_src(bld_base, src.ssa);
163 else
164 return get_reg_src(bld_base, src.reg);
165 }
166
167 static void assign_ssa(struct lp_build_nir_context *bld_base, int idx, LLVMValueRef ptr)
168 {
169 bld_base->ssa_defs[idx] = ptr;
170 }
171
172 static void assign_ssa_dest(struct lp_build_nir_context *bld_base, const nir_ssa_def *ssa,
173 LLVMValueRef vals[NIR_MAX_VEC_COMPONENTS])
174 {
175 assign_ssa(bld_base, ssa->index, ssa->num_components == 1 ? vals[0] : lp_nir_array_build_gather_values(bld_base->base.gallivm->builder, vals, ssa->num_components));
176 }
177
178 static void assign_reg(struct lp_build_nir_context *bld_base, const nir_reg_dest *reg,
179 unsigned write_mask,
180 LLVMValueRef vals[NIR_MAX_VEC_COMPONENTS])
181 {
182 struct hash_entry *entry = _mesa_hash_table_search(bld_base->regs, reg->reg);
183 LLVMValueRef reg_storage = (LLVMValueRef)entry->data;
184 struct lp_build_context *reg_bld = get_int_bld(bld_base, true, reg->reg->bit_size);
185 LLVMValueRef indir_src = NULL;
186 if (reg->indirect)
187 indir_src = get_src(bld_base, *reg->indirect);
188 bld_base->store_reg(bld_base, reg_bld, reg, write_mask ? write_mask : 0xf, indir_src, reg_storage, vals);
189 }
190
191 static void assign_dest(struct lp_build_nir_context *bld_base, const nir_dest *dest, LLVMValueRef vals[NIR_MAX_VEC_COMPONENTS])
192 {
193 if (dest->is_ssa)
194 assign_ssa_dest(bld_base, &dest->ssa, vals);
195 else
196 assign_reg(bld_base, &dest->reg, 0, vals);
197 }
198
199 static void assign_alu_dest(struct lp_build_nir_context *bld_base, const nir_alu_dest *dest, LLVMValueRef vals[NIR_MAX_VEC_COMPONENTS])
200 {
201 if (dest->dest.is_ssa)
202 assign_ssa_dest(bld_base, &dest->dest.ssa, vals);
203 else
204 assign_reg(bld_base, &dest->dest.reg, dest->write_mask, vals);
205 }
206
207 static LLVMValueRef int_to_bool32(struct lp_build_nir_context *bld_base,
208 uint32_t src_bit_size,
209 bool is_unsigned,
210 LLVMValueRef val)
211 {
212 LLVMBuilderRef builder = bld_base->base.gallivm->builder;
213 struct lp_build_context *int_bld = get_int_bld(bld_base, is_unsigned, src_bit_size);
214 LLVMValueRef result = lp_build_compare(bld_base->base.gallivm, int_bld->type, PIPE_FUNC_NOTEQUAL, val, int_bld->zero);
215 if (src_bit_size == 64)
216 result = LLVMBuildTrunc(builder, result, bld_base->int_bld.vec_type, "");
217 return result;
218 }
219
220 static LLVMValueRef flt_to_bool32(struct lp_build_nir_context *bld_base,
221 uint32_t src_bit_size,
222 LLVMValueRef val)
223 {
224 LLVMBuilderRef builder = bld_base->base.gallivm->builder;
225 struct lp_build_context *flt_bld = get_flt_bld(bld_base, src_bit_size);
226 LLVMValueRef result = lp_build_cmp(flt_bld, PIPE_FUNC_NOTEQUAL, val, flt_bld->zero);
227 if (src_bit_size == 64)
228 result = LLVMBuildTrunc(builder, result, bld_base->int_bld.vec_type, "");
229 return result;
230 }
231
232 static LLVMValueRef fcmp32(struct lp_build_nir_context *bld_base,
233 enum pipe_compare_func compare,
234 uint32_t src_bit_size,
235 LLVMValueRef src[NIR_MAX_VEC_COMPONENTS])
236 {
237 LLVMBuilderRef builder = bld_base->base.gallivm->builder;
238 struct lp_build_context *flt_bld = get_flt_bld(bld_base, src_bit_size);
239 LLVMValueRef result;
240
241 if (compare != PIPE_FUNC_NOTEQUAL)
242 result = lp_build_cmp_ordered(flt_bld, compare, src[0], src[1]);
243 else
244 result = lp_build_cmp(flt_bld, compare, src[0], src[1]);
245 if (src_bit_size == 64)
246 result = LLVMBuildTrunc(builder, result, bld_base->int_bld.vec_type, "");
247 return result;
248 }
249
250 static LLVMValueRef icmp32(struct lp_build_nir_context *bld_base,
251 enum pipe_compare_func compare,
252 bool is_unsigned,
253 uint32_t src_bit_size,
254 LLVMValueRef src[NIR_MAX_VEC_COMPONENTS])
255 {
256 LLVMBuilderRef builder = bld_base->base.gallivm->builder;
257 struct lp_build_context *i_bld = get_int_bld(bld_base, is_unsigned, src_bit_size);
258 LLVMValueRef result = lp_build_cmp(i_bld, compare, src[0], src[1]);
259 if (src_bit_size < 32)
260 result = LLVMBuildSExt(builder, result, bld_base->int_bld.vec_type, "");
261 else if (src_bit_size == 64)
262 result = LLVMBuildTrunc(builder, result, bld_base->int_bld.vec_type, "");
263 return result;
264 }
265
266 static LLVMValueRef get_alu_src(struct lp_build_nir_context *bld_base,
267 nir_alu_src src,
268 unsigned num_components)
269 {
270 LLVMBuilderRef builder = bld_base->base.gallivm->builder;
271 struct gallivm_state *gallivm = bld_base->base.gallivm;
272 LLVMValueRef value = get_src(bld_base, src.src);
273 bool need_swizzle = false;
274
275 assert(value);
276 unsigned src_components = nir_src_num_components(src.src);
277 for (unsigned i = 0; i < num_components; ++i) {
278 assert(src.swizzle[i] < src_components);
279 if (src.swizzle[i] != i)
280 need_swizzle = true;
281 }
282
283 if (need_swizzle || num_components != src_components) {
284 if (src_components > 1 && num_components == 1) {
285 value = LLVMBuildExtractValue(gallivm->builder, value,
286 src.swizzle[0], "");
287 } else if (src_components == 1 && num_components > 1) {
288 LLVMValueRef values[] = {value, value, value, value, value, value, value, value, value, value, value, value, value, value, value, value};
289 value = lp_nir_array_build_gather_values(builder, values, num_components);
290 } else {
291 LLVMValueRef arr = LLVMGetUndef(LLVMArrayType(LLVMTypeOf(LLVMBuildExtractValue(builder, value, 0, "")), num_components));
292 for (unsigned i = 0; i < num_components; i++)
293 arr = LLVMBuildInsertValue(builder, arr, LLVMBuildExtractValue(builder, value, src.swizzle[i], ""), i, "");
294 value = arr;
295 }
296 }
297 assert(!src.negate);
298 assert(!src.abs);
299 return value;
300 }
301
302 static LLVMValueRef emit_b2f(struct lp_build_nir_context *bld_base,
303 LLVMValueRef src0,
304 unsigned bitsize)
305 {
306 LLVMBuilderRef builder = bld_base->base.gallivm->builder;
307 LLVMValueRef result = LLVMBuildAnd(builder, cast_type(bld_base, src0, nir_type_int, 32),
308 LLVMBuildBitCast(builder, lp_build_const_vec(bld_base->base.gallivm, bld_base->base.type,
309 1.0), bld_base->int_bld.vec_type, ""),
310 "");
311 result = LLVMBuildBitCast(builder, result, bld_base->base.vec_type, "");
312 switch (bitsize) {
313 case 32:
314 break;
315 case 64:
316 result = LLVMBuildFPExt(builder, result, bld_base->dbl_bld.vec_type, "");
317 break;
318 default:
319 unreachable("unsupported bit size.");
320 }
321 return result;
322 }
323
324 static LLVMValueRef emit_b2i(struct lp_build_nir_context *bld_base,
325 LLVMValueRef src0,
326 unsigned bitsize)
327 {
328 LLVMBuilderRef builder = bld_base->base.gallivm->builder;
329 LLVMValueRef result = LLVMBuildAnd(builder, cast_type(bld_base, src0, nir_type_int, 32),
330 lp_build_const_int_vec(bld_base->base.gallivm, bld_base->base.type, 1), "");
331 switch (bitsize) {
332 case 32:
333 return result;
334 case 64:
335 return LLVMBuildZExt(builder, result, bld_base->int64_bld.vec_type, "");
336 default:
337 unreachable("unsupported bit size.");
338 }
339 }
340
341 static LLVMValueRef emit_b32csel(struct lp_build_nir_context *bld_base,
342 unsigned src_bit_size[NIR_MAX_VEC_COMPONENTS],
343 LLVMValueRef src[NIR_MAX_VEC_COMPONENTS])
344 {
345 LLVMValueRef sel = cast_type(bld_base, src[0], nir_type_int, 32);
346 LLVMValueRef v = lp_build_compare(bld_base->base.gallivm, bld_base->int_bld.type, PIPE_FUNC_NOTEQUAL, sel, bld_base->int_bld.zero);
347 struct lp_build_context *bld = get_int_bld(bld_base, false, src_bit_size[1]);
348 return lp_build_select(bld, v, src[1], src[2]);
349 }
350
351 static LLVMValueRef split_64bit(struct lp_build_nir_context *bld_base,
352 LLVMValueRef src,
353 bool hi)
354 {
355 struct gallivm_state *gallivm = bld_base->base.gallivm;
356 LLVMValueRef shuffles[LP_MAX_VECTOR_WIDTH/32];
357 LLVMValueRef shuffles2[LP_MAX_VECTOR_WIDTH/32];
358 int len = bld_base->base.type.length * 2;
359 for (unsigned i = 0; i < bld_base->base.type.length; i++) {
360 #if UTIL_ARCH_LITTLE_ENDIAN
361 shuffles[i] = lp_build_const_int32(gallivm, i * 2);
362 shuffles2[i] = lp_build_const_int32(gallivm, (i * 2) + 1);
363 #else
364 shuffles[i] = lp_build_const_int32(gallivm, (i * 2) + 1);
365 shuffles2[i] = lp_build_const_int32(gallivm, (i * 2));
366 #endif
367 }
368
369 src = LLVMBuildBitCast(gallivm->builder, src, LLVMVectorType(LLVMInt32TypeInContext(gallivm->context), len), "");
370 return LLVMBuildShuffleVector(gallivm->builder, src,
371 LLVMGetUndef(LLVMTypeOf(src)),
372 LLVMConstVector(hi ? shuffles2 : shuffles,
373 bld_base->base.type.length),
374 "");
375 }
376
377 static LLVMValueRef
378 merge_64bit(struct lp_build_nir_context *bld_base,
379 LLVMValueRef input,
380 LLVMValueRef input2)
381 {
382 struct gallivm_state *gallivm = bld_base->base.gallivm;
383 LLVMBuilderRef builder = gallivm->builder;
384 int i;
385 LLVMValueRef shuffles[2 * (LP_MAX_VECTOR_WIDTH/32)];
386 int len = bld_base->base.type.length * 2;
387 assert(len <= (2 * (LP_MAX_VECTOR_WIDTH/32)));
388
389 for (i = 0; i < bld_base->base.type.length * 2; i+=2) {
390 #if UTIL_ARCH_LITTLE_ENDIAN
391 shuffles[i] = lp_build_const_int32(gallivm, i / 2);
392 shuffles[i + 1] = lp_build_const_int32(gallivm, i / 2 + bld_base->base.type.length);
393 #else
394 shuffles[i] = lp_build_const_int32(gallivm, i / 2 + bld_base->base.type.length);
395 shuffles[i + 1] = lp_build_const_int32(gallivm, i / 2);
396 #endif
397 }
398 return LLVMBuildShuffleVector(builder, input, input2, LLVMConstVector(shuffles, len), "");
399 }
400
401 static LLVMValueRef
402 do_int_divide(struct lp_build_nir_context *bld_base,
403 bool is_unsigned, unsigned src_bit_size,
404 LLVMValueRef src, LLVMValueRef src2)
405 {
406 struct gallivm_state *gallivm = bld_base->base.gallivm;
407 LLVMBuilderRef builder = gallivm->builder;
408 struct lp_build_context *int_bld = get_int_bld(bld_base, is_unsigned, src_bit_size);
409 struct lp_build_context *mask_bld = get_int_bld(bld_base, true, src_bit_size);
410 LLVMValueRef div_mask = lp_build_cmp(mask_bld, PIPE_FUNC_EQUAL, src2,
411 mask_bld->zero);
412
413 if (!is_unsigned) {
414 /* INT_MIN (0x80000000) / -1 (0xffffffff) causes sigfpe, seen with blender. */
415 div_mask = LLVMBuildAnd(builder, div_mask, lp_build_const_int_vec(gallivm, int_bld->type, 0x7fffffff), "");
416 }
417 LLVMValueRef divisor = LLVMBuildOr(builder,
418 div_mask,
419 src2, "");
420 LLVMValueRef result = lp_build_div(int_bld, src, divisor);
421
422 if (!is_unsigned) {
423 LLVMValueRef not_div_mask = LLVMBuildNot(builder, div_mask, "");
424 return LLVMBuildAnd(builder, not_div_mask, result, "");
425 } else
426 /* udiv by zero is guaranteed to return 0xffffffff at least with d3d10
427 * may as well do same for idiv */
428 return LLVMBuildOr(builder, div_mask, result, "");
429 }
430
431 static LLVMValueRef
432 do_int_mod(struct lp_build_nir_context *bld_base,
433 bool is_unsigned, unsigned src_bit_size,
434 LLVMValueRef src, LLVMValueRef src2)
435 {
436 struct gallivm_state *gallivm = bld_base->base.gallivm;
437 LLVMBuilderRef builder = gallivm->builder;
438 struct lp_build_context *int_bld = get_int_bld(bld_base, is_unsigned, src_bit_size);
439 LLVMValueRef div_mask = lp_build_cmp(int_bld, PIPE_FUNC_EQUAL, src2,
440 int_bld->zero);
441 LLVMValueRef divisor = LLVMBuildOr(builder,
442 div_mask,
443 src2, "");
444 LLVMValueRef result = lp_build_mod(int_bld, src, divisor);
445 return LLVMBuildOr(builder, div_mask, result, "");
446 }
447
448 static LLVMValueRef
449 do_quantize_to_f16(struct lp_build_nir_context *bld_base,
450 LLVMValueRef src)
451 {
452 struct gallivm_state *gallivm = bld_base->base.gallivm;
453 LLVMBuilderRef builder = gallivm->builder;
454 LLVMValueRef result;
455 result = LLVMBuildFPTrunc(builder, src, LLVMVectorType(LLVMHalfTypeInContext(gallivm->context), bld_base->base.type.length), "");
456 result = LLVMBuildFPExt(builder, result, bld_base->base.vec_type, "");
457 return result;
458 }
459
460 static LLVMValueRef do_alu_action(struct lp_build_nir_context *bld_base,
461 nir_op op, unsigned src_bit_size[NIR_MAX_VEC_COMPONENTS], LLVMValueRef src[NIR_MAX_VEC_COMPONENTS])
462 {
463 struct gallivm_state *gallivm = bld_base->base.gallivm;
464 LLVMBuilderRef builder = gallivm->builder;
465 LLVMValueRef result;
466 switch (op) {
467 case nir_op_b2f32:
468 result = emit_b2f(bld_base, src[0], 32);
469 break;
470 case nir_op_b2f64:
471 result = emit_b2f(bld_base, src[0], 64);
472 break;
473 case nir_op_b2i32:
474 result = emit_b2i(bld_base, src[0], 32);
475 break;
476 case nir_op_b2i64:
477 result = emit_b2i(bld_base, src[0], 64);
478 break;
479 case nir_op_b32csel:
480 result = emit_b32csel(bld_base, src_bit_size, src);
481 break;
482 case nir_op_bit_count:
483 result = lp_build_popcount(get_int_bld(bld_base, false, src_bit_size[0]), src[0]);
484 break;
485 case nir_op_bitfield_select:
486 result = lp_build_xor(&bld_base->uint_bld, src[2], lp_build_and(&bld_base->uint_bld, src[0], lp_build_xor(&bld_base->uint_bld, src[1], src[2])));
487 break;
488 case nir_op_bitfield_reverse:
489 result = lp_build_bitfield_reverse(get_int_bld(bld_base, false, src_bit_size[0]), src[0]);
490 break;
491 case nir_op_f2b32:
492 result = flt_to_bool32(bld_base, src_bit_size[0], src[0]);
493 break;
494 case nir_op_f2f16:
495 result = LLVMBuildFPTrunc(builder, src[0],
496 LLVMVectorType(LLVMHalfTypeInContext(gallivm->context), bld_base->base.type.length), "");
497 break;
498 case nir_op_f2f32:
499 if (src_bit_size[0] < 32)
500 result = LLVMBuildFPExt(builder, src[0],
501 bld_base->base.vec_type, "");
502 else
503 result = LLVMBuildFPTrunc(builder, src[0],
504 bld_base->base.vec_type, "");
505 break;
506 case nir_op_f2f64:
507 result = LLVMBuildFPExt(builder, src[0],
508 bld_base->dbl_bld.vec_type, "");
509 break;
510 case nir_op_f2i32:
511 result = LLVMBuildFPToSI(builder, src[0], bld_base->base.int_vec_type, "");
512 break;
513 case nir_op_f2u32:
514 result = LLVMBuildFPToUI(builder,
515 src[0],
516 bld_base->base.int_vec_type, "");
517 break;
518 case nir_op_f2i64:
519 result = LLVMBuildFPToSI(builder,
520 src[0],
521 bld_base->int64_bld.vec_type, "");
522 break;
523 case nir_op_f2u64:
524 result = LLVMBuildFPToUI(builder,
525 src[0],
526 bld_base->uint64_bld.vec_type, "");
527 break;
528 case nir_op_fabs:
529 result = lp_build_abs(get_flt_bld(bld_base, src_bit_size[0]), src[0]);
530 break;
531 case nir_op_fadd:
532 result = lp_build_add(get_flt_bld(bld_base, src_bit_size[0]),
533 src[0], src[1]);
534 break;
535 case nir_op_fceil:
536 result = lp_build_ceil(get_flt_bld(bld_base, src_bit_size[0]), src[0]);
537 break;
538 case nir_op_fcos:
539 result = lp_build_cos(&bld_base->base, src[0]);
540 break;
541 case nir_op_fddx:
542 case nir_op_fddx_coarse:
543 case nir_op_fddx_fine:
544 result = lp_build_ddx(&bld_base->base, src[0]);
545 break;
546 case nir_op_fddy:
547 case nir_op_fddy_coarse:
548 case nir_op_fddy_fine:
549 result = lp_build_ddy(&bld_base->base, src[0]);
550 break;
551 case nir_op_fdiv:
552 result = lp_build_div(get_flt_bld(bld_base, src_bit_size[0]),
553 src[0], src[1]);
554 break;
555 case nir_op_feq32:
556 result = fcmp32(bld_base, PIPE_FUNC_EQUAL, src_bit_size[0], src);
557 break;
558 case nir_op_fexp2:
559 result = lp_build_exp2(&bld_base->base, src[0]);
560 break;
561 case nir_op_ffloor:
562 result = lp_build_floor(get_flt_bld(bld_base, src_bit_size[0]), src[0]);
563 break;
564 case nir_op_ffma:
565 result = lp_build_fmuladd(builder, src[0], src[1], src[2]);
566 break;
567 case nir_op_ffract: {
568 struct lp_build_context *flt_bld = get_flt_bld(bld_base, src_bit_size[0]);
569 LLVMValueRef tmp = lp_build_floor(flt_bld, src[0]);
570 result = lp_build_sub(flt_bld, src[0], tmp);
571 break;
572 }
573 case nir_op_fge32:
574 result = fcmp32(bld_base, PIPE_FUNC_GEQUAL, src_bit_size[0], src);
575 break;
576 case nir_op_find_lsb:
577 result = lp_build_cttz(get_int_bld(bld_base, false, src_bit_size[0]), src[0]);
578 break;
579 case nir_op_flog2:
580 result = lp_build_log2_safe(&bld_base->base, src[0]);
581 break;
582 case nir_op_flt32:
583 result = fcmp32(bld_base, PIPE_FUNC_LESS, src_bit_size[0], src);
584 break;
585 case nir_op_fmin:
586 result = lp_build_min(get_flt_bld(bld_base, src_bit_size[0]), src[0], src[1]);
587 break;
588 case nir_op_fmod: {
589 struct lp_build_context *flt_bld = get_flt_bld(bld_base, src_bit_size[0]);
590 result = lp_build_div(flt_bld, src[0], src[1]);
591 result = lp_build_floor(flt_bld, result);
592 result = lp_build_mul(flt_bld, src[1], result);
593 result = lp_build_sub(flt_bld, src[0], result);
594 break;
595 }
596 case nir_op_fmul:
597 result = lp_build_mul(get_flt_bld(bld_base, src_bit_size[0]),
598 src[0], src[1]);
599 break;
600 case nir_op_fmax:
601 result = lp_build_max(get_flt_bld(bld_base, src_bit_size[0]), src[0], src[1]);
602 break;
603 case nir_op_fneu32:
604 result = fcmp32(bld_base, PIPE_FUNC_NOTEQUAL, src_bit_size[0], src);
605 break;
606 case nir_op_fneg:
607 result = lp_build_negate(get_flt_bld(bld_base, src_bit_size[0]), src[0]);
608 break;
609 case nir_op_fpow:
610 result = lp_build_pow(&bld_base->base, src[0], src[1]);
611 break;
612 case nir_op_fquantize2f16:
613 result = do_quantize_to_f16(bld_base, src[0]);
614 break;
615 case nir_op_frcp:
616 result = lp_build_rcp(get_flt_bld(bld_base, src_bit_size[0]), src[0]);
617 break;
618 case nir_op_fround_even:
619 result = lp_build_round(get_flt_bld(bld_base, src_bit_size[0]), src[0]);
620 break;
621 case nir_op_frsq:
622 result = lp_build_rsqrt(get_flt_bld(bld_base, src_bit_size[0]), src[0]);
623 break;
624 case nir_op_fsat:
625 result = lp_build_clamp_zero_one_nanzero(get_flt_bld(bld_base, src_bit_size[0]), src[0]);
626 break;
627 case nir_op_fsign:
628 result = lp_build_sgn(get_flt_bld(bld_base, src_bit_size[0]), src[0]);
629 break;
630 case nir_op_fsin:
631 result = lp_build_sin(&bld_base->base, src[0]);
632 break;
633 case nir_op_fsqrt:
634 result = lp_build_sqrt(get_flt_bld(bld_base, src_bit_size[0]), src[0]);
635 break;
636 case nir_op_ftrunc:
637 result = lp_build_trunc(get_flt_bld(bld_base, src_bit_size[0]), src[0]);
638 break;
639 case nir_op_i2b32:
640 result = int_to_bool32(bld_base, src_bit_size[0], false, src[0]);
641 break;
642 case nir_op_i2f32:
643 result = lp_build_int_to_float(&bld_base->base, src[0]);
644 break;
645 case nir_op_i2f64:
646 result = lp_build_int_to_float(&bld_base->dbl_bld, src[0]);
647 break;
648 case nir_op_i2i8:
649 result = LLVMBuildTrunc(builder, src[0], bld_base->int8_bld.vec_type, "");
650 break;
651 case nir_op_i2i16:
652 if (src_bit_size[0] < 16)
653 result = LLVMBuildSExt(builder, src[0], bld_base->int16_bld.vec_type, "");
654 else
655 result = LLVMBuildTrunc(builder, src[0], bld_base->int16_bld.vec_type, "");
656 break;
657 case nir_op_i2i32:
658 if (src_bit_size[0] < 32)
659 result = LLVMBuildSExt(builder, src[0], bld_base->int_bld.vec_type, "");
660 else
661 result = LLVMBuildTrunc(builder, src[0], bld_base->int_bld.vec_type, "");
662 break;
663 case nir_op_i2i64:
664 result = LLVMBuildSExt(builder, src[0], bld_base->int64_bld.vec_type, "");
665 break;
666 case nir_op_iabs:
667 result = lp_build_abs(get_int_bld(bld_base, false, src_bit_size[0]), src[0]);
668 break;
669 case nir_op_iadd:
670 result = lp_build_add(get_int_bld(bld_base, false, src_bit_size[0]),
671 src[0], src[1]);
672 break;
673 case nir_op_iand:
674 result = lp_build_and(get_int_bld(bld_base, false, src_bit_size[0]),
675 src[0], src[1]);
676 break;
677 case nir_op_idiv:
678 result = do_int_divide(bld_base, false, src_bit_size[0], src[0], src[1]);
679 break;
680 case nir_op_ieq32:
681 result = icmp32(bld_base, PIPE_FUNC_EQUAL, false, src_bit_size[0], src);
682 break;
683 case nir_op_ige32:
684 result = icmp32(bld_base, PIPE_FUNC_GEQUAL, false, src_bit_size[0], src);
685 break;
686 case nir_op_ilt32:
687 result = icmp32(bld_base, PIPE_FUNC_LESS, false, src_bit_size[0], src);
688 break;
689 case nir_op_imax:
690 result = lp_build_max(get_int_bld(bld_base, false, src_bit_size[0]), src[0], src[1]);
691 break;
692 case nir_op_imin:
693 result = lp_build_min(get_int_bld(bld_base, false, src_bit_size[0]), src[0], src[1]);
694 break;
695 case nir_op_imul:
696 case nir_op_imul24:
697 result = lp_build_mul(get_int_bld(bld_base, false, src_bit_size[0]),
698 src[0], src[1]);
699 break;
700 case nir_op_imul_high: {
701 LLVMValueRef hi_bits;
702 lp_build_mul_32_lohi(&bld_base->int_bld, src[0], src[1], &hi_bits);
703 result = hi_bits;
704 break;
705 }
706 case nir_op_ine32:
707 result = icmp32(bld_base, PIPE_FUNC_NOTEQUAL, false, src_bit_size[0], src);
708 break;
709 case nir_op_ineg:
710 result = lp_build_negate(get_int_bld(bld_base, false, src_bit_size[0]), src[0]);
711 break;
712 case nir_op_inot:
713 result = lp_build_not(get_int_bld(bld_base, false, src_bit_size[0]), src[0]);
714 break;
715 case nir_op_ior:
716 result = lp_build_or(get_int_bld(bld_base, false, src_bit_size[0]),
717 src[0], src[1]);
718 break;
719 case nir_op_imod:
720 case nir_op_irem:
721 result = do_int_mod(bld_base, false, src_bit_size[0], src[0], src[1]);
722 break;
723 case nir_op_ishl: {
724 struct lp_build_context *uint_bld = get_int_bld(bld_base, true, src_bit_size[0]);
725 struct lp_build_context *int_bld = get_int_bld(bld_base, false, src_bit_size[0]);
726 if (src_bit_size[0] == 64)
727 src[1] = LLVMBuildZExt(builder, src[1], uint_bld->vec_type, "");
728 if (src_bit_size[0] < 32)
729 src[1] = LLVMBuildTrunc(builder, src[1], uint_bld->vec_type, "");
730 src[1] = lp_build_and(uint_bld, src[1], lp_build_const_int_vec(gallivm, uint_bld->type, (src_bit_size[0] - 1)));
731 result = lp_build_shl(int_bld, src[0], src[1]);
732 break;
733 }
734 case nir_op_ishr: {
735 struct lp_build_context *uint_bld = get_int_bld(bld_base, true, src_bit_size[0]);
736 struct lp_build_context *int_bld = get_int_bld(bld_base, false, src_bit_size[0]);
737 if (src_bit_size[0] == 64)
738 src[1] = LLVMBuildZExt(builder, src[1], uint_bld->vec_type, "");
739 if (src_bit_size[0] < 32)
740 src[1] = LLVMBuildTrunc(builder, src[1], uint_bld->vec_type, "");
741 src[1] = lp_build_and(uint_bld, src[1], lp_build_const_int_vec(gallivm, uint_bld->type, (src_bit_size[0] - 1)));
742 result = lp_build_shr(int_bld, src[0], src[1]);
743 break;
744 }
745 case nir_op_isign:
746 result = lp_build_sgn(get_int_bld(bld_base, false, src_bit_size[0]), src[0]);
747 break;
748 case nir_op_isub:
749 result = lp_build_sub(get_int_bld(bld_base, false, src_bit_size[0]),
750 src[0], src[1]);
751 break;
752 case nir_op_ixor:
753 result = lp_build_xor(get_int_bld(bld_base, false, src_bit_size[0]),
754 src[0], src[1]);
755 break;
756 case nir_op_mov:
757 result = src[0];
758 break;
759 case nir_op_unpack_64_2x32_split_x:
760 result = split_64bit(bld_base, src[0], false);
761 break;
762 case nir_op_unpack_64_2x32_split_y:
763 result = split_64bit(bld_base, src[0], true);
764 break;
765
766 case nir_op_pack_64_2x32_split: {
767 LLVMValueRef tmp = merge_64bit(bld_base, src[0], src[1]);
768 result = LLVMBuildBitCast(builder, tmp, bld_base->dbl_bld.vec_type, "");
769 break;
770 }
771 case nir_op_u2f32:
772 result = LLVMBuildUIToFP(builder, src[0], bld_base->base.vec_type, "");
773 break;
774 case nir_op_u2f64:
775 result = LLVMBuildUIToFP(builder, src[0], bld_base->dbl_bld.vec_type, "");
776 break;
777 case nir_op_u2u8:
778 result = LLVMBuildTrunc(builder, src[0], bld_base->uint8_bld.vec_type, "");
779 break;
780 case nir_op_u2u16:
781 if (src_bit_size[0] < 16)
782 result = LLVMBuildZExt(builder, src[0], bld_base->uint16_bld.vec_type, "");
783 else
784 result = LLVMBuildTrunc(builder, src[0], bld_base->uint16_bld.vec_type, "");
785 break;
786 case nir_op_u2u32:
787 if (src_bit_size[0] < 32)
788 result = LLVMBuildZExt(builder, src[0], bld_base->uint_bld.vec_type, "");
789 else
790 result = LLVMBuildTrunc(builder, src[0], bld_base->uint_bld.vec_type, "");
791 break;
792 case nir_op_u2u64:
793 result = LLVMBuildZExt(builder, src[0], bld_base->uint64_bld.vec_type, "");
794 break;
795 case nir_op_udiv:
796 result = do_int_divide(bld_base, true, src_bit_size[0], src[0], src[1]);
797 break;
798 case nir_op_ufind_msb: {
799 struct lp_build_context *uint_bld = get_int_bld(bld_base, true, src_bit_size[0]);
800 result = lp_build_ctlz(uint_bld, src[0]);
801 result = lp_build_sub(uint_bld, lp_build_const_int_vec(gallivm, uint_bld->type, src_bit_size[0] - 1), result);
802 break;
803 }
804 case nir_op_uge32:
805 result = icmp32(bld_base, PIPE_FUNC_GEQUAL, true, src_bit_size[0], src);
806 break;
807 case nir_op_ult32:
808 result = icmp32(bld_base, PIPE_FUNC_LESS, true, src_bit_size[0], src);
809 break;
810 case nir_op_umax:
811 result = lp_build_max(get_int_bld(bld_base, true, src_bit_size[0]), src[0], src[1]);
812 break;
813 case nir_op_umin:
814 result = lp_build_min(get_int_bld(bld_base, true, src_bit_size[0]), src[0], src[1]);
815 break;
816 case nir_op_umod:
817 result = do_int_mod(bld_base, true, src_bit_size[0], src[0], src[1]);
818 break;
819 case nir_op_umul_high: {
820 LLVMValueRef hi_bits;
821 lp_build_mul_32_lohi(&bld_base->uint_bld, src[0], src[1], &hi_bits);
822 result = hi_bits;
823 break;
824 }
825 case nir_op_ushr: {
826 struct lp_build_context *uint_bld = get_int_bld(bld_base, true, src_bit_size[0]);
827 if (src_bit_size[0] == 64)
828 src[1] = LLVMBuildZExt(builder, src[1], uint_bld->vec_type, "");
829 if (src_bit_size[0] < 32)
830 src[1] = LLVMBuildTrunc(builder, src[1], uint_bld->vec_type, "");
831 src[1] = lp_build_and(uint_bld, src[1], lp_build_const_int_vec(gallivm, uint_bld->type, (src_bit_size[0] - 1)));
832 result = lp_build_shr(uint_bld, src[0], src[1]);
833 break;
834 }
835 default:
836 assert(0);
837 break;
838 }
839 return result;
840 }
841
842 static void visit_alu(struct lp_build_nir_context *bld_base, const nir_alu_instr *instr)
843 {
844 struct gallivm_state *gallivm = bld_base->base.gallivm;
845 LLVMValueRef src[NIR_MAX_VEC_COMPONENTS];
846 unsigned src_bit_size[NIR_MAX_VEC_COMPONENTS];
847 unsigned num_components = nir_dest_num_components(instr->dest.dest);
848 unsigned src_components;
849 switch (instr->op) {
850 case nir_op_vec2:
851 case nir_op_vec3:
852 case nir_op_vec4:
853 case nir_op_vec8:
854 case nir_op_vec16:
855 src_components = 1;
856 break;
857 case nir_op_pack_half_2x16:
858 src_components = 2;
859 break;
860 case nir_op_unpack_half_2x16:
861 src_components = 1;
862 break;
863 case nir_op_cube_face_coord:
864 case nir_op_cube_face_index:
865 src_components = 3;
866 break;
867 default:
868 src_components = num_components;
869 break;
870 }
871 for (unsigned i = 0; i < nir_op_infos[instr->op].num_inputs; i++) {
872 src[i] = get_alu_src(bld_base, instr->src[i], src_components);
873 src_bit_size[i] = nir_src_bit_size(instr->src[i].src);
874 }
875
876 LLVMValueRef result[NIR_MAX_VEC_COMPONENTS];
877 if (instr->op == nir_op_vec4 || instr->op == nir_op_vec3 || instr->op == nir_op_vec2 || instr->op == nir_op_vec8 || instr->op == nir_op_vec16) {
878 for (unsigned i = 0; i < nir_op_infos[instr->op].num_inputs; i++) {
879 result[i] = cast_type(bld_base, src[i], nir_op_infos[instr->op].input_types[i], src_bit_size[i]);
880 }
881 } else {
882 for (unsigned c = 0; c < num_components; c++) {
883 LLVMValueRef src_chan[NIR_MAX_VEC_COMPONENTS];
884
885 for (unsigned i = 0; i < nir_op_infos[instr->op].num_inputs; i++) {
886 if (num_components > 1) {
887 src_chan[i] = LLVMBuildExtractValue(gallivm->builder,
888 src[i], c, "");
889 } else
890 src_chan[i] = src[i];
891 src_chan[i] = cast_type(bld_base, src_chan[i], nir_op_infos[instr->op].input_types[i], src_bit_size[i]);
892 }
893 result[c] = do_alu_action(bld_base, instr->op, src_bit_size, src_chan);
894 result[c] = cast_type(bld_base, result[c], nir_op_infos[instr->op].output_type, nir_dest_bit_size(instr->dest.dest));
895 }
896 }
897 assign_alu_dest(bld_base, &instr->dest, result);
898 }
899
900 static void visit_load_const(struct lp_build_nir_context *bld_base,
901 const nir_load_const_instr *instr)
902 {
903 LLVMValueRef result[NIR_MAX_VEC_COMPONENTS];
904 struct lp_build_context *int_bld = get_int_bld(bld_base, true, instr->def.bit_size);
905 for (unsigned i = 0; i < instr->def.num_components; i++)
906 result[i] = lp_build_const_int_vec(bld_base->base.gallivm, int_bld->type, instr->def.bit_size == 32 ? instr->value[i].u32 : instr->value[i].u64);
907 assign_ssa_dest(bld_base, &instr->def, result);
908 }
909
910 static void
911 get_deref_offset(struct lp_build_nir_context *bld_base, nir_deref_instr *instr,
912 bool vs_in, unsigned *vertex_index_out,
913 LLVMValueRef *vertex_index_ref,
914 unsigned *const_out, LLVMValueRef *indir_out)
915 {
916 LLVMBuilderRef builder = bld_base->base.gallivm->builder;
917 nir_variable *var = nir_deref_instr_get_variable(instr);
918 nir_deref_path path;
919 unsigned idx_lvl = 1;
920
921 nir_deref_path_init(&path, instr, NULL);
922
923 if (vertex_index_out != NULL || vertex_index_ref != NULL) {
924 if (vertex_index_ref) {
925 *vertex_index_ref = get_src(bld_base, path.path[idx_lvl]->arr.index);
926 if (vertex_index_out)
927 *vertex_index_out = 0;
928 } else {
929 *vertex_index_out = nir_src_as_uint(path.path[idx_lvl]->arr.index);
930 }
931 ++idx_lvl;
932 }
933
934 uint32_t const_offset = 0;
935 LLVMValueRef offset = NULL;
936
937 if (var->data.compact && nir_src_is_const(instr->arr.index)) {
938 assert(instr->deref_type == nir_deref_type_array);
939 const_offset = nir_src_as_uint(instr->arr.index);
940 goto out;
941 }
942
943 for (; path.path[idx_lvl]; ++idx_lvl) {
944 const struct glsl_type *parent_type = path.path[idx_lvl - 1]->type;
945 if (path.path[idx_lvl]->deref_type == nir_deref_type_struct) {
946 unsigned index = path.path[idx_lvl]->strct.index;
947
948 for (unsigned i = 0; i < index; i++) {
949 const struct glsl_type *ft = glsl_get_struct_field(parent_type, i);
950 const_offset += glsl_count_attribute_slots(ft, vs_in);
951 }
952 } else if(path.path[idx_lvl]->deref_type == nir_deref_type_array) {
953 unsigned size = glsl_count_attribute_slots(path.path[idx_lvl]->type, vs_in);
954 if (nir_src_is_const(path.path[idx_lvl]->arr.index)) {
955 const_offset += nir_src_comp_as_int(path.path[idx_lvl]->arr.index, 0) * size;
956 } else {
957 LLVMValueRef idx_src = get_src(bld_base, path.path[idx_lvl]->arr.index);
958 idx_src = cast_type(bld_base, idx_src, nir_type_uint, 32);
959 LLVMValueRef array_off = lp_build_mul(&bld_base->uint_bld, lp_build_const_int_vec(bld_base->base.gallivm, bld_base->base.type, size),
960 idx_src);
961 if (offset)
962 offset = lp_build_add(&bld_base->uint_bld, offset, array_off);
963 else
964 offset = array_off;
965 }
966 } else
967 unreachable("Uhandled deref type in get_deref_instr_offset");
968 }
969
970 out:
971 nir_deref_path_finish(&path);
972
973 if (const_offset && offset)
974 offset = LLVMBuildAdd(builder, offset,
975 lp_build_const_int_vec(bld_base->base.gallivm, bld_base->uint_bld.type, const_offset),
976 "");
977 *const_out = const_offset;
978 *indir_out = offset;
979 }
980
981 static void visit_load_var(struct lp_build_nir_context *bld_base,
982 nir_intrinsic_instr *instr,
983 LLVMValueRef result[NIR_MAX_VEC_COMPONENTS])
984 {
985 nir_deref_instr *deref = nir_instr_as_deref(instr->src[0].ssa->parent_instr);
986 nir_variable *var = nir_deref_instr_get_variable(deref);
987 nir_variable_mode mode = deref->mode;
988 unsigned const_index;
989 LLVMValueRef indir_index;
990 LLVMValueRef indir_vertex_index = NULL;
991 unsigned vertex_index = 0;
992 unsigned nc = nir_dest_num_components(instr->dest);
993 unsigned bit_size = nir_dest_bit_size(instr->dest);
994 if (var) {
995 bool vs_in = bld_base->shader->info.stage == MESA_SHADER_VERTEX &&
996 var->data.mode == nir_var_shader_in;
997 bool gs_in = bld_base->shader->info.stage == MESA_SHADER_GEOMETRY &&
998 var->data.mode == nir_var_shader_in;
999 bool tcs_in = bld_base->shader->info.stage == MESA_SHADER_TESS_CTRL &&
1000 var->data.mode == nir_var_shader_in;
1001 bool tcs_out = bld_base->shader->info.stage == MESA_SHADER_TESS_CTRL &&
1002 var->data.mode == nir_var_shader_out && !var->data.patch;
1003 bool tes_in = bld_base->shader->info.stage == MESA_SHADER_TESS_EVAL &&
1004 var->data.mode == nir_var_shader_in && !var->data.patch;
1005
1006 mode = var->data.mode;
1007
1008 get_deref_offset(bld_base, deref, vs_in, gs_in ? &vertex_index : NULL, (tcs_in || tcs_out || tes_in) ? &indir_vertex_index : NULL,
1009 &const_index, &indir_index);
1010 }
1011 bld_base->load_var(bld_base, mode, nc, bit_size, var, vertex_index, indir_vertex_index, const_index, indir_index, result);
1012 }
1013
1014 static void
1015 visit_store_var(struct lp_build_nir_context *bld_base,
1016 nir_intrinsic_instr *instr)
1017 {
1018 nir_deref_instr *deref = nir_instr_as_deref(instr->src[0].ssa->parent_instr);
1019 nir_variable *var = nir_deref_instr_get_variable(deref);
1020 nir_variable_mode mode = deref->mode;
1021 int writemask = instr->const_index[0];
1022 unsigned bit_size = nir_src_bit_size(instr->src[1]);
1023 LLVMValueRef src = get_src(bld_base, instr->src[1]);
1024 unsigned const_index = 0;
1025 LLVMValueRef indir_index, indir_vertex_index = NULL;
1026 if (var) {
1027 bool tcs_out = bld_base->shader->info.stage == MESA_SHADER_TESS_CTRL &&
1028 var->data.mode == nir_var_shader_out && !var->data.patch;
1029 get_deref_offset(bld_base, deref, false, NULL, tcs_out ? &indir_vertex_index : NULL,
1030 &const_index, &indir_index);
1031 }
1032 bld_base->store_var(bld_base, mode, instr->num_components, bit_size, var, writemask, indir_vertex_index, const_index, indir_index, src);
1033 }
1034
1035 static void visit_load_ubo(struct lp_build_nir_context *bld_base,
1036 nir_intrinsic_instr *instr,
1037 LLVMValueRef result[NIR_MAX_VEC_COMPONENTS])
1038 {
1039 struct gallivm_state *gallivm = bld_base->base.gallivm;
1040 LLVMBuilderRef builder = gallivm->builder;
1041 LLVMValueRef idx = get_src(bld_base, instr->src[0]);
1042 LLVMValueRef offset = get_src(bld_base, instr->src[1]);
1043
1044 bool offset_is_uniform = nir_src_is_dynamically_uniform(instr->src[1]);
1045 idx = LLVMBuildExtractElement(builder, idx, lp_build_const_int32(gallivm, 0), "");
1046 bld_base->load_ubo(bld_base, nir_dest_num_components(instr->dest), nir_dest_bit_size(instr->dest),
1047 offset_is_uniform, idx, offset, result);
1048 }
1049
1050 static void visit_load_push_constant(struct lp_build_nir_context *bld_base,
1051 nir_intrinsic_instr *instr,
1052 LLVMValueRef result[4])
1053 {
1054 struct gallivm_state *gallivm = bld_base->base.gallivm;
1055 LLVMValueRef offset = get_src(bld_base, instr->src[0]);
1056 LLVMValueRef idx = lp_build_const_int32(gallivm, 0);
1057 bool offset_is_uniform = nir_src_is_dynamically_uniform(instr->src[0]);
1058
1059 bld_base->load_ubo(bld_base, nir_dest_num_components(instr->dest), nir_dest_bit_size(instr->dest),
1060 offset_is_uniform, idx, offset, result);
1061 }
1062
1063
1064 static void visit_load_ssbo(struct lp_build_nir_context *bld_base,
1065 nir_intrinsic_instr *instr,
1066 LLVMValueRef result[NIR_MAX_VEC_COMPONENTS])
1067 {
1068 LLVMValueRef idx = get_src(bld_base, instr->src[0]);
1069 LLVMValueRef offset = get_src(bld_base, instr->src[1]);
1070 bld_base->load_mem(bld_base, nir_dest_num_components(instr->dest), nir_dest_bit_size(instr->dest),
1071 idx, offset, result);
1072 }
1073
1074 static void visit_store_ssbo(struct lp_build_nir_context *bld_base,
1075 nir_intrinsic_instr *instr)
1076 {
1077 LLVMValueRef val = get_src(bld_base, instr->src[0]);
1078 LLVMValueRef idx = get_src(bld_base, instr->src[1]);
1079 LLVMValueRef offset = get_src(bld_base, instr->src[2]);
1080 int writemask = instr->const_index[0];
1081 int nc = nir_src_num_components(instr->src[0]);
1082 int bitsize = nir_src_bit_size(instr->src[0]);
1083 bld_base->store_mem(bld_base, writemask, nc, bitsize, idx, offset, val);
1084 }
1085
1086 static void visit_get_buffer_size(struct lp_build_nir_context *bld_base,
1087 nir_intrinsic_instr *instr,
1088 LLVMValueRef result[NIR_MAX_VEC_COMPONENTS])
1089 {
1090 LLVMValueRef idx = get_src(bld_base, instr->src[0]);
1091 result[0] = bld_base->get_buffer_size(bld_base, idx);
1092 }
1093
1094 static void visit_ssbo_atomic(struct lp_build_nir_context *bld_base,
1095 nir_intrinsic_instr *instr,
1096 LLVMValueRef result[NIR_MAX_VEC_COMPONENTS])
1097 {
1098 LLVMValueRef idx = get_src(bld_base, instr->src[0]);
1099 LLVMValueRef offset = get_src(bld_base, instr->src[1]);
1100 LLVMValueRef val = get_src(bld_base, instr->src[2]);
1101 LLVMValueRef val2 = NULL;
1102 if (instr->intrinsic == nir_intrinsic_ssbo_atomic_comp_swap)
1103 val2 = get_src(bld_base, instr->src[3]);
1104
1105 bld_base->atomic_mem(bld_base, instr->intrinsic, idx, offset, val, val2, &result[0]);
1106
1107 }
1108
1109 static void visit_load_image(struct lp_build_nir_context *bld_base,
1110 nir_intrinsic_instr *instr,
1111 LLVMValueRef result[NIR_MAX_VEC_COMPONENTS])
1112 {
1113 struct gallivm_state *gallivm = bld_base->base.gallivm;
1114 LLVMBuilderRef builder = gallivm->builder;
1115 nir_deref_instr *deref = nir_instr_as_deref(instr->src[0].ssa->parent_instr);
1116 nir_variable *var = nir_deref_instr_get_variable(deref);
1117 LLVMValueRef coord_val = get_src(bld_base, instr->src[1]);
1118 LLVMValueRef coords[5];
1119 struct lp_img_params params;
1120 const struct glsl_type *type = glsl_without_array(var->type);
1121 unsigned const_index;
1122 LLVMValueRef indir_index;
1123 get_deref_offset(bld_base, deref, false, NULL, NULL,
1124 &const_index, &indir_index);
1125
1126 memset(&params, 0, sizeof(params));
1127 params.target = glsl_sampler_to_pipe(glsl_get_sampler_dim(type), glsl_sampler_type_is_array(type));
1128 for (unsigned i = 0; i < 4; i++)
1129 coords[i] = LLVMBuildExtractValue(builder, coord_val, i, "");
1130 if (params.target == PIPE_TEXTURE_1D_ARRAY)
1131 coords[2] = coords[1];
1132
1133 params.coords = coords;
1134 params.outdata = result;
1135 params.img_op = LP_IMG_LOAD;
1136 if (glsl_get_sampler_dim(type) == GLSL_SAMPLER_DIM_MS || glsl_get_sampler_dim(type) == GLSL_SAMPLER_DIM_SUBPASS_MS)
1137 params.ms_index = cast_type(bld_base, get_src(bld_base, instr->src[2]), nir_type_uint, 32);
1138 params.image_index = var->data.binding + (indir_index ? 0 : const_index);
1139 params.image_index_offset = indir_index;
1140 bld_base->image_op(bld_base, &params);
1141 }
1142
1143 static void visit_store_image(struct lp_build_nir_context *bld_base,
1144 nir_intrinsic_instr *instr)
1145 {
1146 struct gallivm_state *gallivm = bld_base->base.gallivm;
1147 LLVMBuilderRef builder = gallivm->builder;
1148 nir_deref_instr *deref = nir_instr_as_deref(instr->src[0].ssa->parent_instr);
1149 nir_variable *var = nir_deref_instr_get_variable(deref);
1150 LLVMValueRef coord_val = get_src(bld_base, instr->src[1]);
1151 LLVMValueRef in_val = get_src(bld_base, instr->src[3]);
1152 LLVMValueRef coords[5];
1153 struct lp_img_params params;
1154 const struct glsl_type *type = glsl_without_array(var->type);
1155 unsigned const_index;
1156 LLVMValueRef indir_index;
1157 get_deref_offset(bld_base, deref, false, NULL, NULL,
1158 &const_index, &indir_index);
1159
1160 memset(&params, 0, sizeof(params));
1161 params.target = glsl_sampler_to_pipe(glsl_get_sampler_dim(type), glsl_sampler_type_is_array(type));
1162 for (unsigned i = 0; i < 4; i++)
1163 coords[i] = LLVMBuildExtractValue(builder, coord_val, i, "");
1164 if (params.target == PIPE_TEXTURE_1D_ARRAY)
1165 coords[2] = coords[1];
1166 params.coords = coords;
1167
1168 for (unsigned i = 0; i < 4; i++) {
1169 params.indata[i] = LLVMBuildExtractValue(builder, in_val, i, "");
1170 params.indata[i] = LLVMBuildBitCast(builder, params.indata[i], bld_base->base.vec_type, "");
1171 }
1172 if (glsl_get_sampler_dim(type) == GLSL_SAMPLER_DIM_MS)
1173 params.ms_index = get_src(bld_base, instr->src[2]);
1174 params.img_op = LP_IMG_STORE;
1175 params.image_index = var->data.binding + (indir_index ? 0 : const_index);
1176 params.image_index_offset = indir_index;
1177
1178 if (params.target == PIPE_TEXTURE_1D_ARRAY)
1179 coords[2] = coords[1];
1180 bld_base->image_op(bld_base, &params);
1181 }
1182
1183 static void visit_atomic_image(struct lp_build_nir_context *bld_base,
1184 nir_intrinsic_instr *instr,
1185 LLVMValueRef result[NIR_MAX_VEC_COMPONENTS])
1186 {
1187 struct gallivm_state *gallivm = bld_base->base.gallivm;
1188 LLVMBuilderRef builder = gallivm->builder;
1189 nir_deref_instr *deref = nir_instr_as_deref(instr->src[0].ssa->parent_instr);
1190 nir_variable *var = nir_deref_instr_get_variable(deref);
1191 struct lp_img_params params;
1192 LLVMValueRef coord_val = get_src(bld_base, instr->src[1]);
1193 LLVMValueRef in_val = get_src(bld_base, instr->src[3]);
1194 LLVMValueRef coords[5];
1195 const struct glsl_type *type = glsl_without_array(var->type);
1196 unsigned const_index;
1197 LLVMValueRef indir_index;
1198 get_deref_offset(bld_base, deref, false, NULL, NULL,
1199 &const_index, &indir_index);
1200
1201 memset(&params, 0, sizeof(params));
1202
1203 switch (instr->intrinsic) {
1204 case nir_intrinsic_image_deref_atomic_add:
1205 params.op = LLVMAtomicRMWBinOpAdd;
1206 break;
1207 case nir_intrinsic_image_deref_atomic_exchange:
1208 params.op = LLVMAtomicRMWBinOpXchg;
1209 break;
1210 case nir_intrinsic_image_deref_atomic_and:
1211 params.op = LLVMAtomicRMWBinOpAnd;
1212 break;
1213 case nir_intrinsic_image_deref_atomic_or:
1214 params.op = LLVMAtomicRMWBinOpOr;
1215 break;
1216 case nir_intrinsic_image_deref_atomic_xor:
1217 params.op = LLVMAtomicRMWBinOpXor;
1218 break;
1219 case nir_intrinsic_image_deref_atomic_umin:
1220 params.op = LLVMAtomicRMWBinOpUMin;
1221 break;
1222 case nir_intrinsic_image_deref_atomic_umax:
1223 params.op = LLVMAtomicRMWBinOpUMax;
1224 break;
1225 case nir_intrinsic_image_deref_atomic_imin:
1226 params.op = LLVMAtomicRMWBinOpMin;
1227 break;
1228 case nir_intrinsic_image_deref_atomic_imax:
1229 params.op = LLVMAtomicRMWBinOpMax;
1230 break;
1231 default:
1232 break;
1233 }
1234
1235 params.target = glsl_sampler_to_pipe(glsl_get_sampler_dim(type), glsl_sampler_type_is_array(type));
1236 for (unsigned i = 0; i < 4; i++)
1237 coords[i] = LLVMBuildExtractValue(builder, coord_val, i, "");
1238 if (params.target == PIPE_TEXTURE_1D_ARRAY)
1239 coords[2] = coords[1];
1240 params.coords = coords;
1241 if (glsl_get_sampler_dim(type) == GLSL_SAMPLER_DIM_MS)
1242 params.ms_index = get_src(bld_base, instr->src[2]);
1243 if (instr->intrinsic == nir_intrinsic_image_deref_atomic_comp_swap) {
1244 LLVMValueRef cas_val = get_src(bld_base, instr->src[4]);
1245 params.indata[0] = in_val;
1246 params.indata2[0] = cas_val;
1247 } else
1248 params.indata[0] = in_val;
1249
1250 params.outdata = result;
1251 params.img_op = (instr->intrinsic == nir_intrinsic_image_deref_atomic_comp_swap) ? LP_IMG_ATOMIC_CAS : LP_IMG_ATOMIC;
1252 params.image_index = var->data.binding + (indir_index ? 0 : const_index);
1253 params.image_index_offset = indir_index;
1254
1255 bld_base->image_op(bld_base, &params);
1256 }
1257
1258
1259 static void visit_image_size(struct lp_build_nir_context *bld_base,
1260 nir_intrinsic_instr *instr,
1261 LLVMValueRef result[NIR_MAX_VEC_COMPONENTS])
1262 {
1263 nir_deref_instr *deref = nir_instr_as_deref(instr->src[0].ssa->parent_instr);
1264 nir_variable *var = nir_deref_instr_get_variable(deref);
1265 struct lp_sampler_size_query_params params = { 0 };
1266 unsigned const_index;
1267 LLVMValueRef indir_index;
1268 const struct glsl_type *type = glsl_without_array(var->type);
1269 get_deref_offset(bld_base, deref, false, NULL, NULL,
1270 &const_index, &indir_index);
1271 params.texture_unit = var->data.binding + (indir_index ? 0 : const_index);
1272 params.texture_unit_offset = indir_index;
1273 params.target = glsl_sampler_to_pipe(glsl_get_sampler_dim(type), glsl_sampler_type_is_array(type));
1274 params.sizes_out = result;
1275
1276 bld_base->image_size(bld_base, &params);
1277 }
1278
1279 static void visit_image_samples(struct lp_build_nir_context *bld_base,
1280 nir_intrinsic_instr *instr,
1281 LLVMValueRef result[NIR_MAX_VEC_COMPONENTS])
1282 {
1283 nir_deref_instr *deref = nir_instr_as_deref(instr->src[0].ssa->parent_instr);
1284 nir_variable *var = nir_deref_instr_get_variable(deref);
1285 struct lp_sampler_size_query_params params = { 0 };
1286 unsigned const_index;
1287 LLVMValueRef indir_index;
1288 const struct glsl_type *type = glsl_without_array(var->type);
1289 get_deref_offset(bld_base, deref, false, NULL, NULL,
1290 &const_index, &indir_index);
1291
1292 params.texture_unit = var->data.binding + (indir_index ? 0 : const_index);
1293 params.texture_unit_offset = indir_index;
1294 params.target = glsl_sampler_to_pipe(glsl_get_sampler_dim(type), glsl_sampler_type_is_array(type));
1295 params.sizes_out = result;
1296 params.samples_only = true;
1297
1298 bld_base->image_size(bld_base, &params);
1299 }
1300
1301 static void visit_shared_load(struct lp_build_nir_context *bld_base,
1302 nir_intrinsic_instr *instr,
1303 LLVMValueRef result[NIR_MAX_VEC_COMPONENTS])
1304 {
1305 LLVMValueRef offset = get_src(bld_base, instr->src[0]);
1306 bld_base->load_mem(bld_base, nir_dest_num_components(instr->dest), nir_dest_bit_size(instr->dest),
1307 NULL, offset, result);
1308 }
1309
1310 static void visit_shared_store(struct lp_build_nir_context *bld_base,
1311 nir_intrinsic_instr *instr)
1312 {
1313 LLVMValueRef val = get_src(bld_base, instr->src[0]);
1314 LLVMValueRef offset = get_src(bld_base, instr->src[1]);
1315 int writemask = instr->const_index[1];
1316 int nc = nir_src_num_components(instr->src[0]);
1317 int bitsize = nir_src_bit_size(instr->src[0]);
1318 bld_base->store_mem(bld_base, writemask, nc, bitsize, NULL, offset, val);
1319 }
1320
1321 static void visit_shared_atomic(struct lp_build_nir_context *bld_base,
1322 nir_intrinsic_instr *instr,
1323 LLVMValueRef result[NIR_MAX_VEC_COMPONENTS])
1324 {
1325 LLVMValueRef offset = get_src(bld_base, instr->src[0]);
1326 LLVMValueRef val = get_src(bld_base, instr->src[1]);
1327 LLVMValueRef val2 = NULL;
1328 if (instr->intrinsic == nir_intrinsic_shared_atomic_comp_swap)
1329 val2 = get_src(bld_base, instr->src[2]);
1330
1331 bld_base->atomic_mem(bld_base, instr->intrinsic, NULL, offset, val, val2, &result[0]);
1332
1333 }
1334
1335 static void visit_barrier(struct lp_build_nir_context *bld_base)
1336 {
1337 bld_base->barrier(bld_base);
1338 }
1339
1340 static void visit_discard(struct lp_build_nir_context *bld_base,
1341 nir_intrinsic_instr *instr)
1342 {
1343 LLVMValueRef cond = NULL;
1344 if (instr->intrinsic == nir_intrinsic_discard_if) {
1345 cond = get_src(bld_base, instr->src[0]);
1346 cond = cast_type(bld_base, cond, nir_type_int, 32);
1347 }
1348 bld_base->discard(bld_base, cond);
1349 }
1350
1351 static void visit_load_kernel_input(struct lp_build_nir_context *bld_base,
1352 nir_intrinsic_instr *instr, LLVMValueRef result[NIR_MAX_VEC_COMPONENTS])
1353 {
1354 LLVMValueRef offset = get_src(bld_base, instr->src[0]);
1355
1356 bool offset_is_uniform = nir_src_is_dynamically_uniform(instr->src[0]);
1357 bld_base->load_kernel_arg(bld_base, nir_dest_num_components(instr->dest), nir_dest_bit_size(instr->dest),
1358 nir_src_bit_size(instr->src[0]),
1359 offset_is_uniform, offset, result);
1360 }
1361
1362 static void visit_load_global(struct lp_build_nir_context *bld_base,
1363 nir_intrinsic_instr *instr, LLVMValueRef result[NIR_MAX_VEC_COMPONENTS])
1364 {
1365 LLVMValueRef addr = get_src(bld_base, instr->src[0]);
1366 bld_base->load_global(bld_base, nir_dest_num_components(instr->dest), nir_dest_bit_size(instr->dest),
1367 nir_src_bit_size(instr->src[0]),
1368 addr, result);
1369 }
1370
1371 static void visit_store_global(struct lp_build_nir_context *bld_base,
1372 nir_intrinsic_instr *instr)
1373 {
1374 LLVMValueRef val = get_src(bld_base, instr->src[0]);
1375 int nc = nir_src_num_components(instr->src[0]);
1376 int bitsize = nir_src_bit_size(instr->src[0]);
1377 LLVMValueRef addr = get_src(bld_base, instr->src[1]);
1378 int addr_bitsize = nir_src_bit_size(instr->src[1]);
1379 int writemask = instr->const_index[0];
1380 bld_base->store_global(bld_base, writemask, nc, bitsize, addr_bitsize, addr, val);
1381 }
1382
1383 static void visit_global_atomic(struct lp_build_nir_context *bld_base,
1384 nir_intrinsic_instr *instr,
1385 LLVMValueRef result[NIR_MAX_VEC_COMPONENTS])
1386 {
1387 LLVMValueRef addr = get_src(bld_base, instr->src[0]);
1388 LLVMValueRef val = get_src(bld_base, instr->src[1]);
1389 LLVMValueRef val2 = NULL;
1390 int addr_bitsize = nir_src_bit_size(instr->src[0]);
1391 if (instr->intrinsic == nir_intrinsic_global_atomic_comp_swap)
1392 val2 = get_src(bld_base, instr->src[2]);
1393
1394 bld_base->atomic_global(bld_base, instr->intrinsic, addr_bitsize, addr, val, val2, &result[0]);
1395 }
1396
1397 static void visit_interp(struct lp_build_nir_context *bld_base,
1398 nir_intrinsic_instr *instr,
1399 LLVMValueRef result[NIR_MAX_VEC_COMPONENTS])
1400 {
1401 struct gallivm_state *gallivm = bld_base->base.gallivm;
1402 LLVMBuilderRef builder = gallivm->builder;
1403 nir_deref_instr *deref = nir_instr_as_deref(instr->src[0].ssa->parent_instr);
1404 unsigned num_components = nir_dest_num_components(instr->dest);
1405 nir_variable *var = nir_deref_instr_get_variable(deref);
1406 unsigned const_index;
1407 LLVMValueRef indir_index;
1408 LLVMValueRef offsets[2] = { NULL, NULL };
1409 get_deref_offset(bld_base, deref, false, NULL, NULL,
1410 &const_index, &indir_index);
1411 bool centroid = instr->intrinsic == nir_intrinsic_interp_deref_at_centroid;
1412 bool sample = false;
1413 if (instr->intrinsic == nir_intrinsic_interp_deref_at_offset) {
1414 for (unsigned i = 0; i < 2; i++) {
1415 offsets[i] = LLVMBuildExtractValue(builder, get_src(bld_base, instr->src[1]), i, "");
1416 offsets[i] = cast_type(bld_base, offsets[i], nir_type_float, 32);
1417 }
1418 } else if (instr->intrinsic == nir_intrinsic_interp_deref_at_sample) {
1419 offsets[0] = get_src(bld_base, instr->src[1]);
1420 offsets[0] = cast_type(bld_base, offsets[0], nir_type_int, 32);
1421 sample = true;
1422 }
1423 bld_base->interp_at(bld_base, num_components, var, centroid, sample, const_index, indir_index, offsets, result);
1424 }
1425
1426 static void visit_intrinsic(struct lp_build_nir_context *bld_base,
1427 nir_intrinsic_instr *instr)
1428 {
1429 LLVMValueRef result[NIR_MAX_VEC_COMPONENTS] = {0};
1430 switch (instr->intrinsic) {
1431 case nir_intrinsic_load_deref:
1432 visit_load_var(bld_base, instr, result);
1433 break;
1434 case nir_intrinsic_store_deref:
1435 visit_store_var(bld_base, instr);
1436 break;
1437 case nir_intrinsic_load_ubo:
1438 visit_load_ubo(bld_base, instr, result);
1439 break;
1440 case nir_intrinsic_load_push_constant:
1441 visit_load_push_constant(bld_base, instr, result);
1442 break;
1443 case nir_intrinsic_load_ssbo:
1444 visit_load_ssbo(bld_base, instr, result);
1445 break;
1446 case nir_intrinsic_store_ssbo:
1447 visit_store_ssbo(bld_base, instr);
1448 break;
1449 case nir_intrinsic_get_buffer_size:
1450 visit_get_buffer_size(bld_base, instr, result);
1451 break;
1452 case nir_intrinsic_load_vertex_id:
1453 case nir_intrinsic_load_primitive_id:
1454 case nir_intrinsic_load_instance_id:
1455 case nir_intrinsic_load_base_instance:
1456 case nir_intrinsic_load_base_vertex:
1457 case nir_intrinsic_load_work_group_id:
1458 case nir_intrinsic_load_local_invocation_id:
1459 case nir_intrinsic_load_num_work_groups:
1460 case nir_intrinsic_load_invocation_id:
1461 case nir_intrinsic_load_front_face:
1462 case nir_intrinsic_load_draw_id:
1463 case nir_intrinsic_load_local_group_size:
1464 case nir_intrinsic_load_work_dim:
1465 case nir_intrinsic_load_tess_coord:
1466 case nir_intrinsic_load_tess_level_outer:
1467 case nir_intrinsic_load_tess_level_inner:
1468 case nir_intrinsic_load_patch_vertices_in:
1469 case nir_intrinsic_load_sample_id:
1470 case nir_intrinsic_load_sample_pos:
1471 case nir_intrinsic_load_sample_mask_in:
1472 bld_base->sysval_intrin(bld_base, instr, result);
1473 break;
1474 case nir_intrinsic_load_helper_invocation:
1475 bld_base->helper_invocation(bld_base, &result[0]);
1476 break;
1477 case nir_intrinsic_discard_if:
1478 case nir_intrinsic_discard:
1479 visit_discard(bld_base, instr);
1480 break;
1481 case nir_intrinsic_emit_vertex:
1482 bld_base->emit_vertex(bld_base, nir_intrinsic_stream_id(instr));
1483 break;
1484 case nir_intrinsic_end_primitive:
1485 bld_base->end_primitive(bld_base, nir_intrinsic_stream_id(instr));
1486 break;
1487 case nir_intrinsic_ssbo_atomic_add:
1488 case nir_intrinsic_ssbo_atomic_imin:
1489 case nir_intrinsic_ssbo_atomic_imax:
1490 case nir_intrinsic_ssbo_atomic_umin:
1491 case nir_intrinsic_ssbo_atomic_umax:
1492 case nir_intrinsic_ssbo_atomic_and:
1493 case nir_intrinsic_ssbo_atomic_or:
1494 case nir_intrinsic_ssbo_atomic_xor:
1495 case nir_intrinsic_ssbo_atomic_exchange:
1496 case nir_intrinsic_ssbo_atomic_comp_swap:
1497 visit_ssbo_atomic(bld_base, instr, result);
1498 break;
1499 case nir_intrinsic_image_deref_load:
1500 visit_load_image(bld_base, instr, result);
1501 break;
1502 case nir_intrinsic_image_deref_store:
1503 visit_store_image(bld_base, instr);
1504 break;
1505 case nir_intrinsic_image_deref_atomic_add:
1506 case nir_intrinsic_image_deref_atomic_imin:
1507 case nir_intrinsic_image_deref_atomic_imax:
1508 case nir_intrinsic_image_deref_atomic_umin:
1509 case nir_intrinsic_image_deref_atomic_umax:
1510 case nir_intrinsic_image_deref_atomic_and:
1511 case nir_intrinsic_image_deref_atomic_or:
1512 case nir_intrinsic_image_deref_atomic_xor:
1513 case nir_intrinsic_image_deref_atomic_exchange:
1514 case nir_intrinsic_image_deref_atomic_comp_swap:
1515 visit_atomic_image(bld_base, instr, result);
1516 break;
1517 case nir_intrinsic_image_deref_size:
1518 visit_image_size(bld_base, instr, result);
1519 break;
1520 case nir_intrinsic_image_deref_samples:
1521 visit_image_samples(bld_base, instr, result);
1522 break;
1523 case nir_intrinsic_load_shared:
1524 visit_shared_load(bld_base, instr, result);
1525 break;
1526 case nir_intrinsic_store_shared:
1527 visit_shared_store(bld_base, instr);
1528 break;
1529 case nir_intrinsic_shared_atomic_add:
1530 case nir_intrinsic_shared_atomic_imin:
1531 case nir_intrinsic_shared_atomic_umin:
1532 case nir_intrinsic_shared_atomic_imax:
1533 case nir_intrinsic_shared_atomic_umax:
1534 case nir_intrinsic_shared_atomic_and:
1535 case nir_intrinsic_shared_atomic_or:
1536 case nir_intrinsic_shared_atomic_xor:
1537 case nir_intrinsic_shared_atomic_exchange:
1538 case nir_intrinsic_shared_atomic_comp_swap:
1539 visit_shared_atomic(bld_base, instr, result);
1540 break;
1541 case nir_intrinsic_control_barrier:
1542 visit_barrier(bld_base);
1543 break;
1544 case nir_intrinsic_group_memory_barrier:
1545 case nir_intrinsic_memory_barrier:
1546 case nir_intrinsic_memory_barrier_shared:
1547 case nir_intrinsic_memory_barrier_buffer:
1548 case nir_intrinsic_memory_barrier_image:
1549 case nir_intrinsic_memory_barrier_tcs_patch:
1550 break;
1551 case nir_intrinsic_load_kernel_input:
1552 visit_load_kernel_input(bld_base, instr, result);
1553 break;
1554 case nir_intrinsic_load_global:
1555 case nir_intrinsic_load_global_constant:
1556 visit_load_global(bld_base, instr, result);
1557 break;
1558 case nir_intrinsic_store_global:
1559 visit_store_global(bld_base, instr);
1560 break;
1561 case nir_intrinsic_global_atomic_add:
1562 case nir_intrinsic_global_atomic_imin:
1563 case nir_intrinsic_global_atomic_umin:
1564 case nir_intrinsic_global_atomic_imax:
1565 case nir_intrinsic_global_atomic_umax:
1566 case nir_intrinsic_global_atomic_and:
1567 case nir_intrinsic_global_atomic_or:
1568 case nir_intrinsic_global_atomic_xor:
1569 case nir_intrinsic_global_atomic_exchange:
1570 case nir_intrinsic_global_atomic_comp_swap:
1571 visit_global_atomic(bld_base, instr, result);
1572 break;
1573 case nir_intrinsic_vote_all:
1574 case nir_intrinsic_vote_any:
1575 case nir_intrinsic_vote_ieq:
1576 bld_base->vote(bld_base, cast_type(bld_base, get_src(bld_base, instr->src[0]), nir_type_int, 32), instr, result);
1577 break;
1578 case nir_intrinsic_interp_deref_at_offset:
1579 case nir_intrinsic_interp_deref_at_centroid:
1580 case nir_intrinsic_interp_deref_at_sample:
1581 visit_interp(bld_base, instr, result);
1582 break;
1583 default:
1584 fprintf(stderr, "Unsupported intrinsic: ");
1585 nir_print_instr(&instr->instr, stderr);
1586 fprintf(stderr, "\n");
1587 assert(0);
1588 break;
1589 }
1590 if (result[0]) {
1591 assign_dest(bld_base, &instr->dest, result);
1592 }
1593 }
1594
1595 static void visit_txs(struct lp_build_nir_context *bld_base, nir_tex_instr *instr)
1596 {
1597 struct lp_sampler_size_query_params params = { 0 };
1598 LLVMValueRef sizes_out[NIR_MAX_VEC_COMPONENTS];
1599 LLVMValueRef explicit_lod = NULL;
1600 LLVMValueRef texture_unit_offset = NULL;
1601 for (unsigned i = 0; i < instr->num_srcs; i++) {
1602 switch (instr->src[i].src_type) {
1603 case nir_tex_src_lod:
1604 explicit_lod = cast_type(bld_base, get_src(bld_base, instr->src[i].src), nir_type_int, 32);
1605 break;
1606 case nir_tex_src_texture_offset:
1607 texture_unit_offset = get_src(bld_base, instr->src[i].src);
1608 break;
1609 default:
1610 break;
1611 }
1612 }
1613
1614 params.target = glsl_sampler_to_pipe(instr->sampler_dim, instr->is_array);
1615 params.texture_unit = instr->texture_index;
1616 params.explicit_lod = explicit_lod;
1617 params.is_sviewinfo = TRUE;
1618 params.sizes_out = sizes_out;
1619 params.samples_only = (instr->op == nir_texop_texture_samples);
1620 params.texture_unit_offset = texture_unit_offset;
1621
1622 if (instr->op == nir_texop_query_levels)
1623 params.explicit_lod = bld_base->uint_bld.zero;
1624 bld_base->tex_size(bld_base, &params);
1625 assign_dest(bld_base, &instr->dest, &sizes_out[instr->op == nir_texop_query_levels ? 3 : 0]);
1626 }
1627
1628 static enum lp_sampler_lod_property lp_build_nir_lod_property(struct lp_build_nir_context *bld_base,
1629 nir_src lod_src)
1630 {
1631 enum lp_sampler_lod_property lod_property;
1632
1633 if (nir_src_is_dynamically_uniform(lod_src))
1634 lod_property = LP_SAMPLER_LOD_SCALAR;
1635 else if (bld_base->shader->info.stage == MESA_SHADER_FRAGMENT) {
1636 if (gallivm_perf & GALLIVM_PERF_NO_QUAD_LOD)
1637 lod_property = LP_SAMPLER_LOD_PER_ELEMENT;
1638 else
1639 lod_property = LP_SAMPLER_LOD_PER_QUAD;
1640 }
1641 else
1642 lod_property = LP_SAMPLER_LOD_PER_ELEMENT;
1643 return lod_property;
1644 }
1645
1646 static void visit_tex(struct lp_build_nir_context *bld_base, nir_tex_instr *instr)
1647 {
1648 struct gallivm_state *gallivm = bld_base->base.gallivm;
1649 LLVMBuilderRef builder = gallivm->builder;
1650 LLVMValueRef coords[5];
1651 LLVMValueRef offsets[3] = { NULL };
1652 LLVMValueRef explicit_lod = NULL, projector = NULL, ms_index = NULL;
1653 struct lp_sampler_params params;
1654 struct lp_derivatives derivs;
1655 unsigned sample_key = 0;
1656 nir_deref_instr *texture_deref_instr = NULL;
1657 nir_deref_instr *sampler_deref_instr = NULL;
1658 LLVMValueRef texture_unit_offset = NULL;
1659 LLVMValueRef texel[NIR_MAX_VEC_COMPONENTS];
1660 unsigned lod_src = 0;
1661 LLVMValueRef coord_undef = LLVMGetUndef(bld_base->base.int_vec_type);
1662
1663 memset(&params, 0, sizeof(params));
1664 enum lp_sampler_lod_property lod_property = LP_SAMPLER_LOD_SCALAR;
1665
1666 if (instr->op == nir_texop_txs || instr->op == nir_texop_query_levels || instr->op == nir_texop_texture_samples) {
1667 visit_txs(bld_base, instr);
1668 return;
1669 }
1670 if (instr->op == nir_texop_txf || instr->op == nir_texop_txf_ms)
1671 sample_key |= LP_SAMPLER_OP_FETCH << LP_SAMPLER_OP_TYPE_SHIFT;
1672 else if (instr->op == nir_texop_tg4) {
1673 sample_key |= LP_SAMPLER_OP_GATHER << LP_SAMPLER_OP_TYPE_SHIFT;
1674 sample_key |= (instr->component << LP_SAMPLER_GATHER_COMP_SHIFT);
1675 } else if (instr->op == nir_texop_lod)
1676 sample_key |= LP_SAMPLER_OP_LODQ << LP_SAMPLER_OP_TYPE_SHIFT;
1677 for (unsigned i = 0; i < instr->num_srcs; i++) {
1678 switch (instr->src[i].src_type) {
1679 case nir_tex_src_coord: {
1680 LLVMValueRef coord = get_src(bld_base, instr->src[i].src);
1681 if (instr->coord_components == 1)
1682 coords[0] = coord;
1683 else {
1684 for (unsigned chan = 0; chan < instr->coord_components; ++chan)
1685 coords[chan] = LLVMBuildExtractValue(builder, coord,
1686 chan, "");
1687 }
1688 for (unsigned chan = instr->coord_components; chan < 5; chan++)
1689 coords[chan] = coord_undef;
1690
1691 break;
1692 }
1693 case nir_tex_src_texture_deref:
1694 texture_deref_instr = nir_src_as_deref(instr->src[i].src);
1695 break;
1696 case nir_tex_src_sampler_deref:
1697 sampler_deref_instr = nir_src_as_deref(instr->src[i].src);
1698 break;
1699 case nir_tex_src_projector:
1700 projector = lp_build_rcp(&bld_base->base, cast_type(bld_base, get_src(bld_base, instr->src[i].src), nir_type_float, 32));
1701 break;
1702 case nir_tex_src_comparator:
1703 sample_key |= LP_SAMPLER_SHADOW;
1704 coords[4] = get_src(bld_base, instr->src[i].src);
1705 coords[4] = cast_type(bld_base, coords[4], nir_type_float, 32);
1706 break;
1707 case nir_tex_src_bias:
1708 sample_key |= LP_SAMPLER_LOD_BIAS << LP_SAMPLER_LOD_CONTROL_SHIFT;
1709 lod_src = i;
1710 explicit_lod = cast_type(bld_base, get_src(bld_base, instr->src[i].src), nir_type_float, 32);
1711 break;
1712 case nir_tex_src_lod:
1713 sample_key |= LP_SAMPLER_LOD_EXPLICIT << LP_SAMPLER_LOD_CONTROL_SHIFT;
1714 lod_src = i;
1715 if (instr->op == nir_texop_txf)
1716 explicit_lod = cast_type(bld_base, get_src(bld_base, instr->src[i].src), nir_type_int, 32);
1717 else
1718 explicit_lod = cast_type(bld_base, get_src(bld_base, instr->src[i].src), nir_type_float, 32);
1719 break;
1720 case nir_tex_src_ddx: {
1721 int deriv_cnt = instr->coord_components;
1722 if (instr->is_array)
1723 deriv_cnt--;
1724 LLVMValueRef deriv_val = get_src(bld_base, instr->src[i].src);
1725 if (deriv_cnt == 1)
1726 derivs.ddx[0] = deriv_val;
1727 else
1728 for (unsigned chan = 0; chan < deriv_cnt; ++chan)
1729 derivs.ddx[chan] = LLVMBuildExtractValue(builder, deriv_val,
1730 chan, "");
1731 for (unsigned chan = 0; chan < deriv_cnt; ++chan)
1732 derivs.ddx[chan] = cast_type(bld_base, derivs.ddx[chan], nir_type_float, 32);
1733 break;
1734 }
1735 case nir_tex_src_ddy: {
1736 int deriv_cnt = instr->coord_components;
1737 if (instr->is_array)
1738 deriv_cnt--;
1739 LLVMValueRef deriv_val = get_src(bld_base, instr->src[i].src);
1740 if (deriv_cnt == 1)
1741 derivs.ddy[0] = deriv_val;
1742 else
1743 for (unsigned chan = 0; chan < deriv_cnt; ++chan)
1744 derivs.ddy[chan] = LLVMBuildExtractValue(builder, deriv_val,
1745 chan, "");
1746 for (unsigned chan = 0; chan < deriv_cnt; ++chan)
1747 derivs.ddy[chan] = cast_type(bld_base, derivs.ddy[chan], nir_type_float, 32);
1748 break;
1749 }
1750 case nir_tex_src_offset: {
1751 int offset_cnt = instr->coord_components;
1752 if (instr->is_array)
1753 offset_cnt--;
1754 LLVMValueRef offset_val = get_src(bld_base, instr->src[i].src);
1755 sample_key |= LP_SAMPLER_OFFSETS;
1756 if (offset_cnt == 1)
1757 offsets[0] = cast_type(bld_base, offset_val, nir_type_int, 32);
1758 else {
1759 for (unsigned chan = 0; chan < offset_cnt; ++chan) {
1760 offsets[chan] = LLVMBuildExtractValue(builder, offset_val,
1761 chan, "");
1762 offsets[chan] = cast_type(bld_base, offsets[chan], nir_type_int, 32);
1763 }
1764 }
1765 break;
1766 }
1767 case nir_tex_src_ms_index:
1768 sample_key |= LP_SAMPLER_FETCH_MS;
1769 ms_index = cast_type(bld_base, get_src(bld_base, instr->src[i].src), nir_type_int, 32);
1770 break;
1771
1772 case nir_tex_src_texture_offset:
1773 texture_unit_offset = get_src(bld_base, instr->src[i].src);
1774 break;
1775 case nir_tex_src_sampler_offset:
1776 break;
1777 default:
1778 assert(0);
1779 break;
1780 }
1781 }
1782 if (!sampler_deref_instr)
1783 sampler_deref_instr = texture_deref_instr;
1784
1785 if (explicit_lod)
1786 lod_property = lp_build_nir_lod_property(bld_base, instr->src[lod_src].src);
1787
1788 if (instr->op == nir_texop_tex || instr->op == nir_texop_tg4 || instr->op == nir_texop_txb ||
1789 instr->op == nir_texop_txl || instr->op == nir_texop_txd || instr->op == nir_texop_lod)
1790 for (unsigned chan = 0; chan < instr->coord_components; ++chan)
1791 coords[chan] = cast_type(bld_base, coords[chan], nir_type_float, 32);
1792 else if (instr->op == nir_texop_txf || instr->op == nir_texop_txf_ms)
1793 for (unsigned chan = 0; chan < instr->coord_components; ++chan)
1794 coords[chan] = cast_type(bld_base, coords[chan], nir_type_int, 32);
1795
1796 if (instr->is_array && instr->sampler_dim == GLSL_SAMPLER_DIM_1D) {
1797 /* move layer coord for 1d arrays. */
1798 coords[2] = coords[1];
1799 coords[1] = coord_undef;
1800 }
1801
1802 if (projector) {
1803 for (unsigned chan = 0; chan < instr->coord_components; ++chan)
1804 coords[chan] = lp_build_mul(&bld_base->base, coords[chan], projector);
1805 if (sample_key & LP_SAMPLER_SHADOW)
1806 coords[4] = lp_build_mul(&bld_base->base, coords[4], projector);
1807 }
1808
1809 uint32_t samp_base_index = 0, tex_base_index = 0;
1810 if (!sampler_deref_instr) {
1811 int samp_src_index = nir_tex_instr_src_index(instr, nir_tex_src_sampler_handle);
1812 if (samp_src_index == -1) {
1813 samp_base_index = instr->sampler_index;
1814 }
1815 }
1816 if (!texture_deref_instr) {
1817 int tex_src_index = nir_tex_instr_src_index(instr, nir_tex_src_texture_handle);
1818 if (tex_src_index == -1) {
1819 tex_base_index = instr->texture_index;
1820 }
1821 }
1822
1823 if (instr->op == nir_texop_txd) {
1824 sample_key |= LP_SAMPLER_LOD_DERIVATIVES << LP_SAMPLER_LOD_CONTROL_SHIFT;
1825 params.derivs = &derivs;
1826 if (bld_base->shader->info.stage == MESA_SHADER_FRAGMENT) {
1827 if (gallivm_perf & GALLIVM_PERF_NO_QUAD_LOD)
1828 lod_property = LP_SAMPLER_LOD_PER_ELEMENT;
1829 else
1830 lod_property = LP_SAMPLER_LOD_PER_QUAD;
1831 } else
1832 lod_property = LP_SAMPLER_LOD_PER_ELEMENT;
1833 }
1834
1835 sample_key |= lod_property << LP_SAMPLER_LOD_PROPERTY_SHIFT;
1836 params.sample_key = sample_key;
1837 params.offsets = offsets;
1838 params.texture_index = tex_base_index;
1839 params.texture_index_offset = texture_unit_offset;
1840 params.sampler_index = samp_base_index;
1841 params.coords = coords;
1842 params.texel = texel;
1843 params.lod = explicit_lod;
1844 params.ms_index = ms_index;
1845 bld_base->tex(bld_base, &params);
1846 assign_dest(bld_base, &instr->dest, texel);
1847 }
1848
1849 static void visit_ssa_undef(struct lp_build_nir_context *bld_base,
1850 const nir_ssa_undef_instr *instr)
1851 {
1852 unsigned num_components = instr->def.num_components;
1853 LLVMValueRef undef[NIR_MAX_VEC_COMPONENTS];
1854 struct lp_build_context *undef_bld = get_int_bld(bld_base, true, instr->def.bit_size);
1855 for (unsigned i = 0; i < num_components; i++)
1856 undef[i] = LLVMGetUndef(undef_bld->vec_type);
1857 assign_ssa_dest(bld_base, &instr->def, undef);
1858 }
1859
1860 static void visit_jump(struct lp_build_nir_context *bld_base,
1861 const nir_jump_instr *instr)
1862 {
1863 switch (instr->type) {
1864 case nir_jump_break:
1865 bld_base->break_stmt(bld_base);
1866 break;
1867 case nir_jump_continue:
1868 bld_base->continue_stmt(bld_base);
1869 break;
1870 default:
1871 unreachable("Unknown jump instr\n");
1872 }
1873 }
1874
1875 static void visit_deref(struct lp_build_nir_context *bld_base,
1876 nir_deref_instr *instr)
1877 {
1878 if (instr->mode != nir_var_mem_shared &&
1879 instr->mode != nir_var_mem_global)
1880 return;
1881 LLVMValueRef result = NULL;
1882 switch(instr->deref_type) {
1883 case nir_deref_type_var: {
1884 struct hash_entry *entry = _mesa_hash_table_search(bld_base->vars, instr->var);
1885 result = entry->data;
1886 break;
1887 }
1888 default:
1889 unreachable("Unhandled deref_instr deref type");
1890 }
1891
1892 assign_ssa(bld_base, instr->dest.ssa.index, result);
1893 }
1894
1895 static void visit_block(struct lp_build_nir_context *bld_base, nir_block *block)
1896 {
1897 nir_foreach_instr(instr, block)
1898 {
1899 switch (instr->type) {
1900 case nir_instr_type_alu:
1901 visit_alu(bld_base, nir_instr_as_alu(instr));
1902 break;
1903 case nir_instr_type_load_const:
1904 visit_load_const(bld_base, nir_instr_as_load_const(instr));
1905 break;
1906 case nir_instr_type_intrinsic:
1907 visit_intrinsic(bld_base, nir_instr_as_intrinsic(instr));
1908 break;
1909 case nir_instr_type_tex:
1910 visit_tex(bld_base, nir_instr_as_tex(instr));
1911 break;
1912 case nir_instr_type_phi:
1913 assert(0);
1914 break;
1915 case nir_instr_type_ssa_undef:
1916 visit_ssa_undef(bld_base, nir_instr_as_ssa_undef(instr));
1917 break;
1918 case nir_instr_type_jump:
1919 visit_jump(bld_base, nir_instr_as_jump(instr));
1920 break;
1921 case nir_instr_type_deref:
1922 visit_deref(bld_base, nir_instr_as_deref(instr));
1923 break;
1924 default:
1925 fprintf(stderr, "Unknown NIR instr type: ");
1926 nir_print_instr(instr, stderr);
1927 fprintf(stderr, "\n");
1928 abort();
1929 }
1930 }
1931 }
1932
1933 static void visit_if(struct lp_build_nir_context *bld_base, nir_if *if_stmt)
1934 {
1935 LLVMValueRef cond = get_src(bld_base, if_stmt->condition);
1936
1937 bld_base->if_cond(bld_base, cond);
1938 visit_cf_list(bld_base, &if_stmt->then_list);
1939
1940 if (!exec_list_is_empty(&if_stmt->else_list)) {
1941 bld_base->else_stmt(bld_base);
1942 visit_cf_list(bld_base, &if_stmt->else_list);
1943 }
1944 bld_base->endif_stmt(bld_base);
1945 }
1946
1947 static void visit_loop(struct lp_build_nir_context *bld_base, nir_loop *loop)
1948 {
1949 bld_base->bgnloop(bld_base);
1950 visit_cf_list(bld_base, &loop->body);
1951 bld_base->endloop(bld_base);
1952 }
1953
1954 static void visit_cf_list(struct lp_build_nir_context *bld_base,
1955 struct exec_list *list)
1956 {
1957 foreach_list_typed(nir_cf_node, node, node, list)
1958 {
1959 switch (node->type) {
1960 case nir_cf_node_block:
1961 visit_block(bld_base, nir_cf_node_as_block(node));
1962 break;
1963
1964 case nir_cf_node_if:
1965 visit_if(bld_base, nir_cf_node_as_if(node));
1966 break;
1967
1968 case nir_cf_node_loop:
1969 visit_loop(bld_base, nir_cf_node_as_loop(node));
1970 break;
1971
1972 default:
1973 assert(0);
1974 }
1975 }
1976 }
1977
1978 static void
1979 handle_shader_output_decl(struct lp_build_nir_context *bld_base,
1980 struct nir_shader *nir,
1981 struct nir_variable *variable)
1982 {
1983 bld_base->emit_var_decl(bld_base, variable);
1984 }
1985
1986 /* vector registers are stored as arrays in LLVM side,
1987 so we can use GEP on them, as to do exec mask stores
1988 we need to operate on a single components.
1989 arrays are:
1990 0.x, 1.x, 2.x, 3.x
1991 0.y, 1.y, 2.y, 3.y
1992 ....
1993 */
1994 static LLVMTypeRef get_register_type(struct lp_build_nir_context *bld_base,
1995 nir_register *reg)
1996 {
1997 struct lp_build_context *int_bld = get_int_bld(bld_base, true, reg->bit_size);
1998
1999 LLVMTypeRef type = int_bld->vec_type;
2000 if (reg->num_array_elems)
2001 type = LLVMArrayType(type, reg->num_array_elems);
2002 if (reg->num_components > 1)
2003 type = LLVMArrayType(type, reg->num_components);
2004
2005 return type;
2006 }
2007
2008
2009 bool lp_build_nir_llvm(
2010 struct lp_build_nir_context *bld_base,
2011 struct nir_shader *nir)
2012 {
2013 struct nir_function *func;
2014
2015 nir_convert_from_ssa(nir, true);
2016 nir_lower_locals_to_regs(nir);
2017 nir_remove_dead_derefs(nir);
2018 nir_remove_dead_variables(nir, nir_var_function_temp, NULL);
2019
2020 nir_foreach_shader_out_variable(variable, nir)
2021 handle_shader_output_decl(bld_base, nir, variable);
2022
2023 bld_base->regs = _mesa_hash_table_create(NULL, _mesa_hash_pointer,
2024 _mesa_key_pointer_equal);
2025 bld_base->vars = _mesa_hash_table_create(NULL, _mesa_hash_pointer,
2026 _mesa_key_pointer_equal);
2027
2028 func = (struct nir_function *)exec_list_get_head(&nir->functions);
2029
2030 nir_foreach_register(reg, &func->impl->registers) {
2031 LLVMTypeRef type = get_register_type(bld_base, reg);
2032 LLVMValueRef reg_alloc = lp_build_alloca_undef(bld_base->base.gallivm,
2033 type, "reg");
2034 _mesa_hash_table_insert(bld_base->regs, reg, reg_alloc);
2035 }
2036 nir_index_ssa_defs(func->impl);
2037 bld_base->ssa_defs = calloc(func->impl->ssa_alloc, sizeof(LLVMValueRef));
2038 visit_cf_list(bld_base, &func->impl->body);
2039
2040 free(bld_base->ssa_defs);
2041 ralloc_free(bld_base->vars);
2042 ralloc_free(bld_base->regs);
2043 return true;
2044 }
2045
2046 /* do some basic opts to remove some things we don't want to see. */
2047 void lp_build_opt_nir(struct nir_shader *nir)
2048 {
2049 bool progress;
2050
2051 static const struct nir_lower_tex_options lower_tex_options = {
2052 .lower_tg4_offsets = true,
2053 };
2054 NIR_PASS_V(nir, nir_lower_tex, &lower_tex_options);
2055 NIR_PASS_V(nir, nir_lower_frexp);
2056
2057 do {
2058 progress = false;
2059 NIR_PASS_V(nir, nir_opt_constant_folding);
2060 NIR_PASS_V(nir, nir_opt_algebraic);
2061 NIR_PASS_V(nir, nir_lower_pack);
2062
2063 nir_lower_tex_options options = { .lower_tex_without_implicit_lod = true };
2064 NIR_PASS_V(nir, nir_lower_tex, &options);
2065 } while (progress);
2066 nir_lower_bool_to_int32(nir);
2067 }