projects / mesa.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
history | raw | HEAD
gallivm: fix 5 warnings
[mesa.git] / src / gallium / auxiliary / gallivm / lp_bld_nir_soa.c
1 /**************************************************************************
2 *
3 * Copyright 2019 Red Hat.
4 * All Rights Reserved.
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the "Software"),
8 * to deal in the Software without restriction, including without limitation
9 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
10 * and/or sell copies of the Software, and to permit persons to whom the
11 * Software is furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included
14 * in all copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
17 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
22 * SOFTWARE.
23 *
24 **************************************************************************/
25
26 #include "lp_bld_nir.h"
27 #include "lp_bld_init.h"
28 #include "lp_bld_flow.h"
29 #include "lp_bld_logic.h"
30 #include "lp_bld_gather.h"
31 #include "lp_bld_const.h"
32 #include "lp_bld_struct.h"
33 #include "lp_bld_arit.h"
34 #include "lp_bld_bitarit.h"
35 #include "lp_bld_coro.h"
36 #include "lp_bld_printf.h"
37 #include "util/u_math.h"
38 /*
39 * combine the execution mask if there is one with the current mask.
40 */
41 static LLVMValueRef
42 mask_vec(struct lp_build_nir_context *bld_base)
43 {
44 struct lp_build_nir_soa_context * bld = (struct lp_build_nir_soa_context *)bld_base;
45 LLVMBuilderRef builder = bld->bld_base.base.gallivm->builder;
46 struct lp_exec_mask *exec_mask = &bld->exec_mask;
47 LLVMValueRef bld_mask = bld->mask ? lp_build_mask_value(bld->mask) : NULL;
48 if (!exec_mask->has_mask) {
49 return bld_mask;
50 }
51 if (!bld_mask)
52 return exec_mask->exec_mask;
53 return LLVMBuildAnd(builder, lp_build_mask_value(bld->mask),
54 exec_mask->exec_mask, "");
55 }
56
57 static LLVMValueRef
58 emit_fetch_64bit(
59 struct lp_build_nir_context * bld_base,
60 LLVMValueRef input,
61 LLVMValueRef input2)
62 {
63 struct gallivm_state *gallivm = bld_base->base.gallivm;
64 LLVMBuilderRef builder = gallivm->builder;
65 LLVMValueRef res;
66 int i;
67 LLVMValueRef shuffles[2 * (LP_MAX_VECTOR_WIDTH/32)];
68 int len = bld_base->base.type.length * 2;
69 assert(len <= (2 * (LP_MAX_VECTOR_WIDTH/32)));
70
71 for (i = 0; i < bld_base->base.type.length * 2; i+=2) {
72 shuffles[i] = lp_build_const_int32(gallivm, i / 2);
73 shuffles[i + 1] = lp_build_const_int32(gallivm, i / 2 + bld_base->base.type.length);
74 }
75 res = LLVMBuildShuffleVector(builder, input, input2, LLVMConstVector(shuffles, len), "");
76
77 return LLVMBuildBitCast(builder, res, bld_base->dbl_bld.vec_type, "");
78 }
79
80 static void
81 emit_store_64bit_chan(struct lp_build_nir_context *bld_base,
82 LLVMValueRef chan_ptr,
83 LLVMValueRef chan_ptr2,
84 LLVMValueRef value)
85 {
86 struct lp_build_nir_soa_context *bld = (struct lp_build_nir_soa_context *)bld_base;
87 struct gallivm_state *gallivm = bld_base->base.gallivm;
88 LLVMBuilderRef builder = gallivm->builder;
89 struct lp_build_context *float_bld = &bld_base->base;
90 unsigned i;
91 LLVMValueRef temp, temp2;
92 LLVMValueRef shuffles[LP_MAX_VECTOR_WIDTH/32];
93 LLVMValueRef shuffles2[LP_MAX_VECTOR_WIDTH/32];
94 int len = bld_base->base.type.length * 2;
95
96 value = LLVMBuildBitCast(gallivm->builder, value, LLVMVectorType(LLVMFloatTypeInContext(gallivm->context), len), "");
97 for (i = 0; i < bld_base->base.type.length; i++) {
98 shuffles[i] = lp_build_const_int32(gallivm, i * 2);
99 shuffles2[i] = lp_build_const_int32(gallivm, (i * 2) + 1);
100 }
101
102 temp = LLVMBuildShuffleVector(builder, value,
103 LLVMGetUndef(LLVMTypeOf(value)),
104 LLVMConstVector(shuffles,
105 bld_base->base.type.length),
106 "");
107 temp2 = LLVMBuildShuffleVector(builder, value,
108 LLVMGetUndef(LLVMTypeOf(value)),
109 LLVMConstVector(shuffles2,
110 bld_base->base.type.length),
111 "");
112
113 lp_exec_mask_store(&bld->exec_mask, float_bld, temp, chan_ptr);
114 lp_exec_mask_store(&bld->exec_mask, float_bld, temp2, chan_ptr2);
115 }
116
117 static LLVMValueRef
118 get_soa_array_offsets(struct lp_build_context *uint_bld,
119 LLVMValueRef indirect_index,
120 int num_components,
121 unsigned chan_index,
122 bool need_perelement_offset)
123 {
124 struct gallivm_state *gallivm = uint_bld->gallivm;
125 LLVMValueRef chan_vec =
126 lp_build_const_int_vec(uint_bld->gallivm, uint_bld->type, chan_index);
127 LLVMValueRef length_vec =
128 lp_build_const_int_vec(gallivm, uint_bld->type, uint_bld->type.length);
129 LLVMValueRef index_vec;
130
131 /* index_vec = (indirect_index * 4 + chan_index) * length + offsets */
132 index_vec = lp_build_mul(uint_bld, indirect_index, lp_build_const_int_vec(uint_bld->gallivm, uint_bld->type, num_components));
133 index_vec = lp_build_add(uint_bld, index_vec, chan_vec);
134 index_vec = lp_build_mul(uint_bld, index_vec, length_vec);
135
136 if (need_perelement_offset) {
137 LLVMValueRef pixel_offsets;
138 unsigned i;
139 /* build pixel offset vector: {0, 1, 2, 3, ...} */
140 pixel_offsets = uint_bld->undef;
141 for (i = 0; i < uint_bld->type.length; i++) {
142 LLVMValueRef ii = lp_build_const_int32(gallivm, i);
143 pixel_offsets = LLVMBuildInsertElement(gallivm->builder, pixel_offsets,
144 ii, ii, "");
145 }
146 index_vec = lp_build_add(uint_bld, index_vec, pixel_offsets);
147 }
148 return index_vec;
149 }
150
151 static LLVMValueRef
152 build_gather(struct lp_build_nir_context *bld_base,
153 struct lp_build_context *bld,
154 LLVMValueRef base_ptr,
155 LLVMValueRef indexes,
156 LLVMValueRef overflow_mask,
157 LLVMValueRef indexes2)
158 {
159 struct gallivm_state *gallivm = bld_base->base.gallivm;
160 LLVMBuilderRef builder = gallivm->builder;
161 struct lp_build_context *uint_bld = &bld_base->uint_bld;
162 LLVMValueRef res;
163 unsigned i;
164
165 if (indexes2)
166 res = LLVMGetUndef(LLVMVectorType(LLVMFloatTypeInContext(gallivm->context), bld_base->base.type.length * 2));
167 else
168 res = bld->undef;
169 /*
170 * overflow_mask is a vector telling us which channels
171 * in the vector overflowed. We use the overflow behavior for
172 * constant buffers which is defined as:
173 * Out of bounds access to constant buffer returns 0 in all
174 * components. Out of bounds behavior is always with respect
175 * to the size of the buffer bound at that slot.
176 */
177
178 if (overflow_mask) {
179 /*
180 * We avoid per-element control flow here (also due to llvm going crazy,
181 * though I suspect it's better anyway since overflow is likely rare).
182 * Note that since we still fetch from buffers even if num_elements was
183 * zero (in this case we'll fetch from index zero) the jit func callers
184 * MUST provide valid fake constant buffers of size 4x32 (the values do
185 * not matter), otherwise we'd still need (not per element though)
186 * control flow.
187 */
188 indexes = lp_build_select(uint_bld, overflow_mask, uint_bld->zero, indexes);
189 if (indexes2)
190 indexes2 = lp_build_select(uint_bld, overflow_mask, uint_bld->zero, indexes2);
191 }
192
193 /*
194 * Loop over elements of index_vec, load scalar value, insert it into 'res'.
195 */
196 for (i = 0; i < bld->type.length * (indexes2 ? 2 : 1); i++) {
197 LLVMValueRef si, di;
198 LLVMValueRef index;
199 LLVMValueRef scalar_ptr, scalar;
200
201 di = lp_build_const_int32(gallivm, i);
202 if (indexes2)
203 si = lp_build_const_int32(gallivm, i >> 1);
204 else
205 si = di;
206
207 if (indexes2 && (i & 1)) {
208 index = LLVMBuildExtractElement(builder,
209 indexes2, si, "");
210 } else {
211 index = LLVMBuildExtractElement(builder,
212 indexes, si, "");
213 }
214 scalar_ptr = LLVMBuildGEP(builder, base_ptr,
215 &index, 1, "gather_ptr");
216 scalar = LLVMBuildLoad(builder, scalar_ptr, "");
217
218 res = LLVMBuildInsertElement(builder, res, scalar, di, "");
219 }
220
221 if (overflow_mask) {
222 if (indexes2) {
223 res = LLVMBuildBitCast(builder, res, bld_base->dbl_bld.vec_type, "");
224 overflow_mask = LLVMBuildSExt(builder, overflow_mask,
225 bld_base->dbl_bld.int_vec_type, "");
226 res = lp_build_select(&bld_base->dbl_bld, overflow_mask,
227 bld_base->dbl_bld.zero, res);
228 } else
229 res = lp_build_select(bld, overflow_mask, bld->zero, res);
230 }
231
232 return res;
233 }
234
235 /**
236 * Scatter/store vector.
237 */
238 static void
239 emit_mask_scatter(struct lp_build_nir_soa_context *bld,
240 LLVMValueRef base_ptr,
241 LLVMValueRef indexes,
242 LLVMValueRef values,
243 struct lp_exec_mask *mask)
244 {
245 struct gallivm_state *gallivm = bld->bld_base.base.gallivm;
246 LLVMBuilderRef builder = gallivm->builder;
247 unsigned i;
248 LLVMValueRef pred = mask->has_mask ? mask->exec_mask : NULL;
249
250 /*
251 * Loop over elements of index_vec, store scalar value.
252 */
253 for (i = 0; i < bld->bld_base.base.type.length; i++) {
254 LLVMValueRef ii = lp_build_const_int32(gallivm, i);
255 LLVMValueRef index = LLVMBuildExtractElement(builder, indexes, ii, "");
256 LLVMValueRef scalar_ptr = LLVMBuildGEP(builder, base_ptr, &index, 1, "scatter_ptr");
257 LLVMValueRef val = LLVMBuildExtractElement(builder, values, ii, "scatter_val");
258 LLVMValueRef scalar_pred = pred ?
259 LLVMBuildExtractElement(builder, pred, ii, "scatter_pred") : NULL;
260
261 if (0)
262 lp_build_printf(gallivm, "scatter %d: val %f at %d %p\n",
263 ii, val, index, scalar_ptr);
264
265 if (scalar_pred) {
266 LLVMValueRef real_val, dst_val;
267 dst_val = LLVMBuildLoad(builder, scalar_ptr, "");
268 real_val = lp_build_select(&bld->uint_elem_bld, scalar_pred, val, dst_val);
269 LLVMBuildStore(builder, real_val, scalar_ptr);
270 }
271 else {
272 LLVMBuildStore(builder, val, scalar_ptr);
273 }
274 }
275 }
276
277 static void emit_load_var(struct lp_build_nir_context *bld_base,
278 nir_variable_mode deref_mode,
279 unsigned num_components,
280 unsigned bit_size,
281 nir_variable *var,
282 unsigned vertex_index,
283 unsigned const_index,
284 LLVMValueRef indir_index,
285 LLVMValueRef result[NIR_MAX_VEC_COMPONENTS])
286 {
287 struct lp_build_nir_soa_context *bld = (struct lp_build_nir_soa_context *)bld_base;
288 struct gallivm_state *gallivm = bld_base->base.gallivm;
289 int dmul = bit_size == 64 ? 2 : 1;
290 switch (deref_mode) {
291 case nir_var_shader_in: {
292 for (unsigned i = 0; i < num_components; i++) {
293 int idx = (i * dmul) + var->data.location_frac;
294 if (bld->gs_iface) {
295 LLVMValueRef vertex_index_val = lp_build_const_int32(gallivm, vertex_index);
296 LLVMValueRef attrib_index_val = lp_build_const_int32(gallivm, const_index + var->data.driver_location);
297 LLVMValueRef swizzle_index_val = lp_build_const_int32(gallivm, idx);
298 LLVMValueRef result2;
299 result[i] = bld->gs_iface->fetch_input(bld->gs_iface, &bld_base->base,
300 false, vertex_index_val, 0, attrib_index_val, swizzle_index_val);
301 if (bit_size == 64) {
302 LLVMValueRef swizzle_index_val = lp_build_const_int32(gallivm, idx + 1);
303 result2 = bld->gs_iface->fetch_input(bld->gs_iface, &bld_base->base,
304 false, vertex_index_val, 0, attrib_index_val, swizzle_index_val);
305 result[i] = emit_fetch_64bit(bld_base, result[i], result2);
306 }
307 } else {
308 if (indir_index) {
309 LLVMValueRef attrib_index_val = lp_build_add(&bld_base->uint_bld, indir_index, lp_build_const_int_vec(gallivm, bld_base->uint_bld.type, var->data.driver_location));
310 LLVMValueRef index_vec = get_soa_array_offsets(&bld_base->uint_bld,
311 attrib_index_val, 4, idx,
312 TRUE);
313 LLVMValueRef index_vec2 = NULL;
314 LLVMTypeRef fptr_type;
315 LLVMValueRef inputs_array;
316 fptr_type = LLVMPointerType(LLVMFloatTypeInContext(gallivm->context), 0);
317 inputs_array = LLVMBuildBitCast(gallivm->builder, bld->inputs_array, fptr_type, "");
318
319 if (bit_size == 64)
320 index_vec2 = get_soa_array_offsets(&bld_base->uint_bld,
321 indir_index, 4, idx + 1, TRUE);
322
323 /* Gather values from the input register array */
324 result[i] = build_gather(bld_base, &bld_base->base, inputs_array, index_vec, NULL, index_vec2);
325 } else {
326 if (bld->indirects & nir_var_shader_in) {
327 LLVMValueRef lindex = lp_build_const_int32(gallivm,
328 var->data.driver_location * 4 + idx);
329 LLVMValueRef input_ptr = lp_build_pointer_get(gallivm->builder,
330 bld->inputs_array, lindex);
331 if (bit_size == 64) {
332 LLVMValueRef lindex2 = lp_build_const_int32(gallivm,
333 var->data.driver_location * 4 + (idx + 1));
334 LLVMValueRef input_ptr2 = lp_build_pointer_get(gallivm->builder,
335 bld->inputs_array, lindex2);
336 result[i] = emit_fetch_64bit(bld_base, input_ptr, input_ptr2);
337 } else {
338 result[i] = input_ptr;
339 }
340 } else {
341 if (bit_size == 64) {
342 LLVMValueRef tmp[2];
343 tmp[0] = bld->inputs[var->data.driver_location + const_index][idx];
344 tmp[1] = bld->inputs[var->data.driver_location + const_index][idx + 1];
345 result[i] = emit_fetch_64bit(bld_base, tmp[0], tmp[1]);
346 } else {
347 result[i] = bld->inputs[var->data.driver_location + const_index][idx];
348 }
349 }
350 }
351 }
352 }
353 }
354 default:
355 break;
356 }
357 }
358
359 static void emit_store_chan(struct lp_build_nir_context *bld_base,
360 nir_variable_mode deref_mode,
361 unsigned bit_size,
362 unsigned location, unsigned comp,
363 unsigned chan,
364 LLVMValueRef dst)
365 {
366 struct lp_build_nir_soa_context *bld = (struct lp_build_nir_soa_context *)bld_base;
367 LLVMBuilderRef builder = bld->bld_base.base.gallivm->builder;
368 struct lp_build_context *float_bld = &bld_base->base;
369
370 if (bit_size == 64) {
371 chan *= 2;
372 chan += comp;
373 if (chan >= 4) {
374 chan -= 4;
375 location++;
376 }
377 emit_store_64bit_chan(bld_base, bld->outputs[location][chan],
378 bld->outputs[location][chan + 1], dst);
379 } else {
380 dst = LLVMBuildBitCast(builder, dst, float_bld->vec_type, "");
381 lp_exec_mask_store(&bld->exec_mask, float_bld, dst,
382 bld->outputs[location][chan + comp]);
383 }
384 }
385
386 static void emit_store_var(struct lp_build_nir_context *bld_base,
387 nir_variable_mode deref_mode,
388 unsigned bit_size,
389 unsigned num_components,
390 unsigned writemask,
391 unsigned const_index,
392 nir_variable *var, LLVMValueRef dst)
393 {
394 struct lp_build_nir_soa_context *bld = (struct lp_build_nir_soa_context *)bld_base;
395 LLVMBuilderRef builder = bld->bld_base.base.gallivm->builder;
396 switch (deref_mode) {
397 case nir_var_shader_out: {
398 unsigned location = var->data.driver_location + const_index;
399 unsigned comp = var->data.location_frac;
400 if (bld_base->shader->info.stage == MESA_SHADER_FRAGMENT) {
401 if (var->data.location == FRAG_RESULT_STENCIL)
402 comp = 1;
403 else if (var->data.location == FRAG_RESULT_DEPTH)
404 comp = 2;
405 }
406 for (unsigned chan = 0; chan < num_components; chan++) {
407 if (writemask & (1u << chan)) {
408 LLVMValueRef chan_val = (num_components == 1) ? dst : LLVMBuildExtractValue(builder, dst, chan, "");
409 emit_store_chan(bld_base, deref_mode, bit_size, location, comp, chan, chan_val);
410 }
411 }
412 break;
413 }
414 default:
415 break;
416 }
417 }
418
419 static LLVMValueRef emit_load_reg(struct lp_build_nir_context *bld_base,
420 struct lp_build_context *reg_bld,
421 const nir_reg_src *reg,
422 LLVMValueRef indir_src,
423 LLVMValueRef reg_storage)
424 {
425 struct gallivm_state *gallivm = bld_base->base.gallivm;
426 LLVMBuilderRef builder = gallivm->builder;
427 int nc = reg->reg->num_components;
428 LLVMValueRef vals[NIR_MAX_VEC_COMPONENTS] = { NULL };
429 struct lp_build_context *uint_bld = &bld_base->uint_bld;
430 if (reg->reg->num_array_elems) {
431 LLVMValueRef indirect_val = lp_build_const_int_vec(gallivm, uint_bld->type, reg->base_offset);
432 if (reg->indirect) {
433 LLVMValueRef max_index = lp_build_const_int_vec(gallivm, uint_bld->type, reg->reg->num_array_elems - 1);
434 indirect_val = LLVMBuildAdd(builder, indirect_val, indir_src, "");
435 indirect_val = lp_build_min(uint_bld, indirect_val, max_index);
436 }
437 reg_storage = LLVMBuildBitCast(builder, reg_storage, LLVMPointerType(reg_bld->elem_type, 0), "");
438 for (unsigned i = 0; i < nc; i++) {
439 LLVMValueRef indirect_offset = get_soa_array_offsets(uint_bld, indirect_val, nc, i, TRUE);
440 vals[i] = build_gather(bld_base, reg_bld, reg_storage, indirect_offset, NULL, NULL);
441 }
442 } else {
443 for (unsigned i = 0; i < nc; i++) {
444 LLVMValueRef this_storage = nc == 1 ? reg_storage : lp_build_array_get_ptr(gallivm, reg_storage,
445 lp_build_const_int32(gallivm, i));
446 vals[i] = LLVMBuildLoad(builder, this_storage, "");
447 }
448 }
449 return nc == 1 ? vals[0] : lp_nir_array_build_gather_values(builder, vals, nc);
450 }
451
452 static void emit_store_reg(struct lp_build_nir_context *bld_base,
453 struct lp_build_context *reg_bld,
454 const nir_reg_dest *reg,
455 unsigned writemask,
456 LLVMValueRef indir_src,
457 LLVMValueRef reg_storage,
458 LLVMValueRef dst[NIR_MAX_VEC_COMPONENTS])
459 {
460 struct lp_build_nir_soa_context *bld = (struct lp_build_nir_soa_context *)bld_base;
461 struct gallivm_state *gallivm = bld_base->base.gallivm;
462 LLVMBuilderRef builder = gallivm->builder;
463 struct lp_build_context *uint_bld = &bld_base->uint_bld;
464 int nc = reg->reg->num_components;
465 if (reg->reg->num_array_elems > 0) {
466 LLVMValueRef indirect_val = lp_build_const_int_vec(gallivm, uint_bld->type, reg->base_offset);
467 if (reg->indirect) {
468 LLVMValueRef max_index = lp_build_const_int_vec(gallivm, uint_bld->type, reg->reg->num_array_elems - 1);
469 indirect_val = LLVMBuildAdd(builder, indirect_val, indir_src, "");
470 indirect_val = lp_build_min(uint_bld, indirect_val, max_index);
471 }
472 reg_storage = LLVMBuildBitCast(builder, reg_storage, LLVMPointerType(reg_bld->elem_type, 0), "");
473 for (unsigned i = 0; i < nc; i++) {
474 if (!(writemask & (1 << i)))
475 continue;
476 LLVMValueRef indirect_offset = get_soa_array_offsets(uint_bld, indirect_val, nc, i, TRUE);
477 dst[i] = LLVMBuildBitCast(builder, dst[i], reg_bld->vec_type, "");
478 emit_mask_scatter(bld, reg_storage, indirect_offset, dst[i], &bld->exec_mask);
479 }
480 return;
481 }
482
483 for (unsigned i = 0; i < nc; i++) {
484 LLVMValueRef this_storage = nc == 1 ? reg_storage : lp_build_array_get_ptr(gallivm, reg_storage,
485 lp_build_const_int32(gallivm, i));
486 dst[i] = LLVMBuildBitCast(builder, dst[i], reg_bld->vec_type, "");
487 lp_exec_mask_store(&bld->exec_mask, reg_bld, dst[i], this_storage);
488 }
489 }
490
491 static void emit_load_kernel_arg(struct lp_build_nir_context *bld_base,
492 unsigned nc,
493 unsigned bit_size,
494 unsigned offset_bit_size,
495 bool offset_is_uniform,
496 LLVMValueRef offset,
497 LLVMValueRef result[NIR_MAX_VEC_COMPONENTS])
498 {
499 struct lp_build_nir_soa_context *bld = (struct lp_build_nir_soa_context *)bld_base;
500 struct gallivm_state *gallivm = bld_base->base.gallivm;
501 LLVMBuilderRef builder = gallivm->builder;
502 struct lp_build_context *bld_broad = get_int_bld(bld_base, true, bit_size);
503 LLVMValueRef kernel_args_ptr = bld->kernel_args_ptr;
504 unsigned size_shift = 0;
505 struct lp_build_context *bld_offset = get_int_bld(bld_base, true, offset_bit_size);
506 if (bit_size == 16)
507 size_shift = 1;
508 else if (bit_size == 32)
509 size_shift = 2;
510 else if (bit_size == 64)
511 size_shift = 3;
512 if (size_shift)
513 offset = lp_build_shr(bld_offset, offset, lp_build_const_int_vec(gallivm, bld_offset->type, size_shift));
514
515 LLVMTypeRef ptr_type = LLVMPointerType(bld_broad->elem_type, 0);
516 kernel_args_ptr = LLVMBuildBitCast(builder, kernel_args_ptr, ptr_type, "");
517
518 if (offset_is_uniform) {
519 offset = LLVMBuildExtractElement(builder, offset, lp_build_const_int32(gallivm, 0), "");
520
521 for (unsigned c = 0; c < nc; c++) {
522 LLVMValueRef this_offset = LLVMBuildAdd(builder, offset, offset_bit_size == 64 ? lp_build_const_int64(gallivm, c) : lp_build_const_int32(gallivm, c), "");
523
524 LLVMValueRef scalar = lp_build_pointer_get(builder, kernel_args_ptr, this_offset);
525 result[c] = lp_build_broadcast_scalar(bld_broad, scalar);
526 }
527 }
528 }
529
530 static LLVMValueRef global_addr_to_ptr(struct gallivm_state *gallivm, LLVMValueRef addr_ptr, unsigned bit_size)
531 {
532 LLVMBuilderRef builder = gallivm->builder;
533 switch (bit_size) {
534 case 8:
535 addr_ptr = LLVMBuildIntToPtr(builder, addr_ptr, LLVMPointerType(LLVMInt8TypeInContext(gallivm->context), 0), "");
536 break;
537 case 16:
538 addr_ptr = LLVMBuildIntToPtr(builder, addr_ptr, LLVMPointerType(LLVMInt16TypeInContext(gallivm->context), 0), "");
539 break;
540 case 32:
541 default:
542 addr_ptr = LLVMBuildIntToPtr(builder, addr_ptr, LLVMPointerType(LLVMInt32TypeInContext(gallivm->context), 0), "");
543 break;
544 case 64:
545 addr_ptr = LLVMBuildIntToPtr(builder, addr_ptr, LLVMPointerType(LLVMInt64TypeInContext(gallivm->context), 0), "");
546 break;
547 }
548 return addr_ptr;
549 }
550
551 static void emit_load_global(struct lp_build_nir_context *bld_base,
552 unsigned nc,
553 unsigned bit_size,
554 unsigned addr_bit_size,
555 LLVMValueRef addr,
556 LLVMValueRef outval[NIR_MAX_VEC_COMPONENTS])
557 {
558 struct gallivm_state *gallivm = bld_base->base.gallivm;
559 LLVMBuilderRef builder = gallivm->builder;
560 struct lp_build_context *uint_bld = &bld_base->uint_bld;
561 struct lp_build_context *res_bld;
562
563 res_bld = get_int_bld(bld_base, true, bit_size);
564
565 for (unsigned c = 0; c < nc; c++) {
566 LLVMValueRef result = lp_build_alloca(gallivm, res_bld->vec_type, "");
567
568 struct lp_build_loop_state loop_state;
569 lp_build_loop_begin(&loop_state, gallivm, lp_build_const_int32(gallivm, 0));
570
571 LLVMValueRef addr_ptr = LLVMBuildExtractElement(gallivm->builder, addr,
572 loop_state.counter, "");
573 addr_ptr = global_addr_to_ptr(gallivm, addr_ptr, bit_size);
574
575 LLVMValueRef value_ptr = lp_build_pointer_get(builder, addr_ptr, lp_build_const_int32(gallivm, c));
576
577 LLVMValueRef temp_res;
578 temp_res = LLVMBuildLoad(builder, result, "");
579 temp_res = LLVMBuildInsertElement(builder, temp_res, value_ptr, loop_state.counter, "");
580 LLVMBuildStore(builder, temp_res, result);
581 lp_build_loop_end_cond(&loop_state, lp_build_const_int32(gallivm, uint_bld->type.length),
582 NULL, LLVMIntUGE);
583 outval[c] = LLVMBuildLoad(builder, result, "");
584 }
585 }
586
587 static void emit_store_global(struct lp_build_nir_context *bld_base,
588 unsigned writemask,
589 unsigned nc, unsigned bit_size,
590 unsigned addr_bit_size,
591 LLVMValueRef addr,
592 LLVMValueRef dst)
593 {
594 struct gallivm_state *gallivm = bld_base->base.gallivm;
595 LLVMBuilderRef builder = gallivm->builder;
596 struct lp_build_context *uint_bld = &bld_base->uint_bld;
597
598 for (unsigned c = 0; c < nc; c++) {
599 if (!(writemask & (1u << c)))
600 continue;
601 LLVMValueRef val = (nc == 1) ? dst : LLVMBuildExtractValue(builder, dst, c, "");
602
603 LLVMValueRef exec_mask = mask_vec(bld_base);
604 struct lp_build_loop_state loop_state;
605 lp_build_loop_begin(&loop_state, gallivm, lp_build_const_int32(gallivm, 0));
606 LLVMValueRef value_ptr = LLVMBuildExtractElement(gallivm->builder, val,
607 loop_state.counter, "");
608
609 LLVMValueRef addr_ptr = LLVMBuildExtractElement(gallivm->builder, addr,
610 loop_state.counter, "");
611 addr_ptr = global_addr_to_ptr(gallivm, addr_ptr, bit_size);
612 switch (bit_size) {
613 case 32:
614 value_ptr = LLVMBuildBitCast(builder, value_ptr, LLVMInt32TypeInContext(gallivm->context), "");
615 break;
616 case 64:
617 value_ptr = LLVMBuildBitCast(builder, value_ptr, LLVMInt64TypeInContext(gallivm->context), "");
618 break;
619 default:
620 break;
621 }
622 struct lp_build_if_state ifthen;
623
624 LLVMValueRef cond = LLVMBuildICmp(gallivm->builder, LLVMIntNE, exec_mask, uint_bld->zero, "");
625 cond = LLVMBuildExtractElement(gallivm->builder, cond, loop_state.counter, "");
626 lp_build_if(&ifthen, gallivm, cond);
627 lp_build_pointer_set(builder, addr_ptr, lp_build_const_int32(gallivm, c), value_ptr);
628 lp_build_endif(&ifthen);
629 lp_build_loop_end_cond(&loop_state, lp_build_const_int32(gallivm, uint_bld->type.length),
630 NULL, LLVMIntUGE);
631 }
632 }
633
634 static void emit_atomic_global(struct lp_build_nir_context *bld_base,
635 nir_intrinsic_op nir_op,
636 unsigned addr_bit_size,
637 LLVMValueRef addr,
638 LLVMValueRef val, LLVMValueRef val2,
639 LLVMValueRef *result)
640 {
641 struct gallivm_state *gallivm = bld_base->base.gallivm;
642 LLVMBuilderRef builder = gallivm->builder;
643 struct lp_build_context *uint_bld = &bld_base->uint_bld;
644
645 LLVMValueRef atom_res = lp_build_alloca(gallivm,
646 uint_bld->vec_type, "");
647 LLVMValueRef exec_mask = mask_vec(bld_base);
648 struct lp_build_loop_state loop_state;
649 lp_build_loop_begin(&loop_state, gallivm, lp_build_const_int32(gallivm, 0));
650
651 LLVMValueRef value_ptr = LLVMBuildExtractElement(gallivm->builder, val,
652 loop_state.counter, "");
653
654 LLVMValueRef addr_ptr = LLVMBuildExtractElement(gallivm->builder, addr,
655 loop_state.counter, "");
656 addr_ptr = global_addr_to_ptr(gallivm, addr_ptr, 32);
657 struct lp_build_if_state ifthen;
658 LLVMValueRef cond, temp_res;
659 LLVMValueRef scalar;
660 cond = LLVMBuildICmp(gallivm->builder, LLVMIntNE, exec_mask, uint_bld->zero, "");
661 cond = LLVMBuildExtractElement(gallivm->builder, cond, loop_state.counter, "");
662 lp_build_if(&ifthen, gallivm, cond);
663
664 if (nir_op == nir_intrinsic_global_atomic_comp_swap) {
665 LLVMValueRef cas_src_ptr = LLVMBuildExtractElement(gallivm->builder, val2,
666 loop_state.counter, "");
667 cas_src_ptr = LLVMBuildBitCast(gallivm->builder, cas_src_ptr, uint_bld->elem_type, "");
668 scalar = LLVMBuildAtomicCmpXchg(builder, addr_ptr, value_ptr,
669 cas_src_ptr,
670 LLVMAtomicOrderingSequentiallyConsistent,
671 LLVMAtomicOrderingSequentiallyConsistent,
672 false);
673 scalar = LLVMBuildExtractValue(gallivm->builder, scalar, 0, "");
674 } else {
675 LLVMAtomicRMWBinOp op;
676 switch (nir_op) {
677 case nir_intrinsic_global_atomic_add:
678 op = LLVMAtomicRMWBinOpAdd;
679 break;
680 case nir_intrinsic_global_atomic_exchange:
681 op = LLVMAtomicRMWBinOpXchg;
682 break;
683 case nir_intrinsic_global_atomic_and:
684 op = LLVMAtomicRMWBinOpAnd;
685 break;
686 case nir_intrinsic_global_atomic_or:
687 op = LLVMAtomicRMWBinOpOr;
688 break;
689 case nir_intrinsic_global_atomic_xor:
690 op = LLVMAtomicRMWBinOpXor;
691 break;
692 case nir_intrinsic_global_atomic_umin:
693 op = LLVMAtomicRMWBinOpUMin;
694 break;
695 case nir_intrinsic_global_atomic_umax:
696 op = LLVMAtomicRMWBinOpUMax;
697 break;
698 case nir_intrinsic_global_atomic_imin:
699 op = LLVMAtomicRMWBinOpMin;
700 break;
701 case nir_intrinsic_global_atomic_imax:
702 op = LLVMAtomicRMWBinOpMax;
703 break;
704 default:
705 unreachable("unknown atomic op");
706 }
707
708 scalar = LLVMBuildAtomicRMW(builder, op,
709 addr_ptr, value_ptr,
710 LLVMAtomicOrderingSequentiallyConsistent,
711 false);
712 }
713 temp_res = LLVMBuildLoad(builder, atom_res, "");
714 temp_res = LLVMBuildInsertElement(builder, temp_res, scalar, loop_state.counter, "");
715 LLVMBuildStore(builder, temp_res, atom_res);
716 lp_build_else(&ifthen);
717 temp_res = LLVMBuildLoad(builder, atom_res, "");
718 temp_res = LLVMBuildInsertElement(builder, temp_res, lp_build_const_int32(gallivm, 0), loop_state.counter, "");
719 LLVMBuildStore(builder, temp_res, atom_res);
720 lp_build_endif(&ifthen);
721 lp_build_loop_end_cond(&loop_state, lp_build_const_int32(gallivm, uint_bld->type.length),
722 NULL, LLVMIntUGE);
723 *result = LLVMBuildLoad(builder, atom_res, "");
724 }
725
726 static void emit_load_ubo(struct lp_build_nir_context *bld_base,
727 unsigned nc,
728 unsigned bit_size,
729 bool offset_is_uniform,
730 LLVMValueRef index,
731 LLVMValueRef offset,
732 LLVMValueRef result[NIR_MAX_VEC_COMPONENTS])
733 {
734 struct lp_build_nir_soa_context *bld = (struct lp_build_nir_soa_context *)bld_base;
735 struct gallivm_state *gallivm = bld_base->base.gallivm;
736 LLVMBuilderRef builder = gallivm->builder;
737 struct lp_build_context *uint_bld = &bld_base->uint_bld;
738 struct lp_build_context *bld_broad = bit_size == 64 ? &bld_base->dbl_bld : &bld_base->base;
739 LLVMValueRef consts_ptr = lp_build_array_get(gallivm, bld->consts_ptr, index);
740 unsigned size_shift = 0;
741 if (bit_size == 32)
742 size_shift = 2;
743 else if (bit_size == 64)
744 size_shift = 3;
745 if (size_shift)
746 offset = lp_build_shr(uint_bld, offset, lp_build_const_int_vec(gallivm, uint_bld->type, size_shift));
747 if (bit_size == 64) {
748 LLVMTypeRef dptr_type = LLVMPointerType(bld_base->dbl_bld.elem_type, 0);
749 consts_ptr = LLVMBuildBitCast(builder, consts_ptr, dptr_type, "");
750 }
751
752 if (offset_is_uniform) {
753 offset = LLVMBuildExtractElement(builder, offset, lp_build_const_int32(gallivm, 0), "");
754
755 for (unsigned c = 0; c < nc; c++) {
756 LLVMValueRef this_offset = LLVMBuildAdd(builder, offset, lp_build_const_int32(gallivm, c), "");
757
758 LLVMValueRef scalar = lp_build_pointer_get(builder, consts_ptr, this_offset);
759 result[c] = lp_build_broadcast_scalar(bld_broad, scalar);
760 }
761 } else {
762 LLVMValueRef overflow_mask;
763 LLVMValueRef num_consts = lp_build_array_get(gallivm, bld->const_sizes_ptr, index);
764
765 num_consts = LLVMBuildShl(gallivm->builder, num_consts, lp_build_const_int32(gallivm, 4), "");
766 num_consts = lp_build_broadcast_scalar(uint_bld, num_consts);
767 for (unsigned c = 0; c < nc; c++) {
768 LLVMValueRef this_offset = lp_build_add(uint_bld, offset, lp_build_const_int_vec(gallivm, uint_bld->type, c));
769 overflow_mask = lp_build_compare(gallivm, uint_bld->type, PIPE_FUNC_GEQUAL,
770 this_offset, num_consts);
771
772 result[c] = build_gather(bld_base, bld_broad, consts_ptr, this_offset, overflow_mask, NULL);
773 }
774 }
775 }
776
777
778 static void emit_load_mem(struct lp_build_nir_context *bld_base,
779 unsigned nc,
780 unsigned bit_size,
781 LLVMValueRef index,
782 LLVMValueRef offset,
783 LLVMValueRef outval[NIR_MAX_VEC_COMPONENTS])
784 {
785 struct gallivm_state *gallivm = bld_base->base.gallivm;
786 struct lp_build_nir_soa_context *bld = (struct lp_build_nir_soa_context *)bld_base;
787 LLVMBuilderRef builder = bld->bld_base.base.gallivm->builder;
788 LLVMValueRef ssbo_ptr = NULL;
789 struct lp_build_context *uint_bld = &bld_base->uint_bld;
790 struct lp_build_context *uint64_bld = &bld_base->uint64_bld;
791 LLVMValueRef ssbo_limit = NULL;
792
793 if (index) {
794 LLVMValueRef ssbo_size_ptr = lp_build_array_get(gallivm, bld->ssbo_sizes_ptr, LLVMBuildExtractElement(builder, index, lp_build_const_int32(gallivm, 0), ""));
795 ssbo_limit = LLVMBuildAShr(gallivm->builder, ssbo_size_ptr, lp_build_const_int32(gallivm, bit_size == 64 ? 3 : 2), "");
796 ssbo_limit = lp_build_broadcast_scalar(uint_bld, ssbo_limit);
797
798 ssbo_ptr = lp_build_array_get(gallivm, bld->ssbo_ptr, LLVMBuildExtractElement(builder, index, lp_build_const_int32(gallivm, 0), ""));
799 } else
800 ssbo_ptr = bld->shared_ptr;
801
802 offset = LLVMBuildAShr(gallivm->builder, offset, lp_build_const_int_vec(gallivm, uint_bld->type, bit_size == 64 ? 3 : 2), "");
803 for (unsigned c = 0; c < nc; c++) {
804 LLVMValueRef loop_index = lp_build_add(uint_bld, offset, lp_build_const_int_vec(gallivm, uint_bld->type, c));
805 LLVMValueRef exec_mask = mask_vec(bld_base);
806
807 if (ssbo_limit) {
808 LLVMValueRef ssbo_oob_cmp = lp_build_cmp(uint_bld, PIPE_FUNC_LESS, loop_index, ssbo_limit);
809 exec_mask = LLVMBuildAnd(builder, exec_mask, ssbo_oob_cmp, "");
810 }
811
812 LLVMValueRef result = lp_build_alloca(gallivm, bit_size == 64 ? uint64_bld->vec_type : uint_bld->vec_type, "");
813 struct lp_build_loop_state loop_state;
814 lp_build_loop_begin(&loop_state, gallivm, lp_build_const_int32(gallivm, 0));
815
816 struct lp_build_if_state ifthen;
817 LLVMValueRef cond, temp_res;
818
819 loop_index = LLVMBuildExtractElement(gallivm->builder, loop_index,
820 loop_state.counter, "");
821
822 cond = LLVMBuildICmp(gallivm->builder, LLVMIntNE, exec_mask, uint_bld->zero, "");
823 cond = LLVMBuildExtractElement(gallivm->builder, cond, loop_state.counter, "");
824
825 lp_build_if(&ifthen, gallivm, cond);
826 LLVMValueRef scalar;
827 if (bit_size == 64) {
828 LLVMValueRef ssbo_ptr2 = LLVMBuildBitCast(builder, ssbo_ptr, LLVMPointerType(uint64_bld->elem_type, 0), "");
829 scalar = lp_build_pointer_get(builder, ssbo_ptr2, loop_index);
830 } else
831 scalar = lp_build_pointer_get(builder, ssbo_ptr, loop_index);
832
833 temp_res = LLVMBuildLoad(builder, result, "");
834 temp_res = LLVMBuildInsertElement(builder, temp_res, scalar, loop_state.counter, "");
835 LLVMBuildStore(builder, temp_res, result);
836 lp_build_else(&ifthen);
837 temp_res = LLVMBuildLoad(builder, result, "");
838 LLVMValueRef zero;
839 if (bit_size == 64)
840 zero = LLVMConstInt(LLVMInt64TypeInContext(gallivm->context), 0, 0);
841 else
842 zero = lp_build_const_int32(gallivm, 0);
843 temp_res = LLVMBuildInsertElement(builder, temp_res, zero, loop_state.counter, "");
844 LLVMBuildStore(builder, temp_res, result);
845 lp_build_endif(&ifthen);
846 lp_build_loop_end_cond(&loop_state, lp_build_const_int32(gallivm, uint_bld->type.length),
847 NULL, LLVMIntUGE);
848 outval[c] = LLVMBuildLoad(gallivm->builder, result, "");
849 }
850 }
851
852 static void emit_store_mem(struct lp_build_nir_context *bld_base,
853 unsigned writemask,
854 unsigned nc,
855 unsigned bit_size,
856 LLVMValueRef index,
857 LLVMValueRef offset,
858 LLVMValueRef dst)
859 {
860 struct gallivm_state *gallivm = bld_base->base.gallivm;
861 struct lp_build_nir_soa_context *bld = (struct lp_build_nir_soa_context *)bld_base;
862 LLVMBuilderRef builder = bld->bld_base.base.gallivm->builder;
863 LLVMValueRef ssbo_ptr;
864 struct lp_build_context *uint_bld = &bld_base->uint_bld;
865 LLVMValueRef ssbo_limit = NULL;
866
867 if (index) {
868 LLVMValueRef ssbo_size_ptr = lp_build_array_get(gallivm, bld->ssbo_sizes_ptr, LLVMBuildExtractElement(builder, index, lp_build_const_int32(gallivm, 0), ""));
869 ssbo_limit = LLVMBuildAShr(gallivm->builder, ssbo_size_ptr, lp_build_const_int32(gallivm, bit_size == 64 ? 3 : 2), "");
870 ssbo_limit = lp_build_broadcast_scalar(uint_bld, ssbo_limit);
871 ssbo_ptr = lp_build_array_get(gallivm, bld->ssbo_ptr, LLVMBuildExtractElement(builder, index, lp_build_const_int32(gallivm, 0), ""));
872 } else
873 ssbo_ptr = bld->shared_ptr;
874
875 offset = lp_build_shr_imm(uint_bld, offset, bit_size == 64 ? 3 : 2);
876 for (unsigned c = 0; c < nc; c++) {
877 if (!(writemask & (1u << c)))
878 continue;
879 LLVMValueRef loop_index = lp_build_add(uint_bld, offset, lp_build_const_int_vec(gallivm, uint_bld->type, c));
880 LLVMValueRef val = (nc == 1) ? dst : LLVMBuildExtractValue(builder, dst, c, "");
881
882 LLVMValueRef exec_mask = mask_vec(bld_base);
883 if (ssbo_limit) {
884 LLVMValueRef ssbo_oob_cmp = lp_build_cmp(uint_bld, PIPE_FUNC_LESS, loop_index, ssbo_limit);
885 exec_mask = LLVMBuildAnd(builder, exec_mask, ssbo_oob_cmp, "");
886 }
887
888 struct lp_build_loop_state loop_state;
889 lp_build_loop_begin(&loop_state, gallivm, lp_build_const_int32(gallivm, 0));
890 LLVMValueRef value_ptr = LLVMBuildExtractElement(gallivm->builder, val,
891 loop_state.counter, "");
892 if (bit_size == 64)
893 value_ptr = LLVMBuildBitCast(gallivm->builder, value_ptr, bld_base->uint64_bld.elem_type, "");
894 else
895 value_ptr = LLVMBuildBitCast(gallivm->builder, value_ptr, uint_bld->elem_type, "");
896 struct lp_build_if_state ifthen;
897 LLVMValueRef cond;
898
899 loop_index = LLVMBuildExtractElement(gallivm->builder, loop_index,
900 loop_state.counter, "");
901 cond = LLVMBuildICmp(gallivm->builder, LLVMIntNE, exec_mask, uint_bld->zero, "");
902 cond = LLVMBuildExtractElement(gallivm->builder, cond, loop_state.counter, "");
903 lp_build_if(&ifthen, gallivm, cond);
904 if (bit_size == 64) {
905 LLVMValueRef ssbo_ptr2 = LLVMBuildBitCast(builder, ssbo_ptr, LLVMPointerType(bld_base->uint64_bld.elem_type, 0), "");
906 lp_build_pointer_set(builder, ssbo_ptr2, loop_index, value_ptr);
907 } else
908 lp_build_pointer_set(builder, ssbo_ptr, loop_index, value_ptr);
909 lp_build_endif(&ifthen);
910 lp_build_loop_end_cond(&loop_state, lp_build_const_int32(gallivm, uint_bld->type.length),
911 NULL, LLVMIntUGE);
912 }
913 }
914
915 static void emit_atomic_mem(struct lp_build_nir_context *bld_base,
916 nir_intrinsic_op nir_op,
917 LLVMValueRef index, LLVMValueRef offset,
918 LLVMValueRef val, LLVMValueRef val2,
919 LLVMValueRef *result)
920 {
921 struct gallivm_state *gallivm = bld_base->base.gallivm;
922 struct lp_build_nir_soa_context *bld = (struct lp_build_nir_soa_context *)bld_base;
923 LLVMBuilderRef builder = bld->bld_base.base.gallivm->builder;
924 LLVMValueRef ssbo_ptr;
925 struct lp_build_context *uint_bld = &bld_base->uint_bld;
926 LLVMValueRef ssbo_limit = NULL;
927
928 if (index) {
929 LLVMValueRef ssbo_size_ptr = lp_build_array_get(gallivm, bld->ssbo_sizes_ptr, LLVMBuildExtractElement(builder, index, lp_build_const_int32(gallivm, 0), ""));
930 ssbo_limit = LLVMBuildAShr(gallivm->builder, ssbo_size_ptr, lp_build_const_int32(gallivm, 2), "");
931 ssbo_limit = lp_build_broadcast_scalar(uint_bld, ssbo_limit);
932 ssbo_ptr = lp_build_array_get(gallivm, bld->ssbo_ptr, LLVMBuildExtractElement(builder, index, lp_build_const_int32(gallivm, 0), ""));
933 } else
934 ssbo_ptr = bld->shared_ptr;
935
936 offset = lp_build_shr_imm(uint_bld, offset, 2);
937 LLVMValueRef atom_res = lp_build_alloca(gallivm,
938 uint_bld->vec_type, "");
939
940 LLVMValueRef exec_mask = mask_vec(bld_base);
941 if (ssbo_limit) {
942 LLVMValueRef ssbo_oob_cmp = lp_build_cmp(uint_bld, PIPE_FUNC_LESS, offset, ssbo_limit);
943 exec_mask = LLVMBuildAnd(builder, exec_mask, ssbo_oob_cmp, "");
944 }
945
946 struct lp_build_loop_state loop_state;
947 lp_build_loop_begin(&loop_state, gallivm, lp_build_const_int32(gallivm, 0));
948
949 LLVMValueRef value_ptr = LLVMBuildExtractElement(gallivm->builder, val,
950 loop_state.counter, "");
951 value_ptr = LLVMBuildBitCast(gallivm->builder, value_ptr, uint_bld->elem_type, "");
952
953 offset = LLVMBuildExtractElement(gallivm->builder, offset,
954 loop_state.counter, "");
955
956 LLVMValueRef scalar_ptr = LLVMBuildGEP(builder, ssbo_ptr,
957 &offset, 1, "");
958
959 struct lp_build_if_state ifthen;
960 LLVMValueRef cond, temp_res;
961 LLVMValueRef scalar;
962 cond = LLVMBuildICmp(gallivm->builder, LLVMIntNE, exec_mask, uint_bld->zero, "");
963 cond = LLVMBuildExtractElement(gallivm->builder, cond, loop_state.counter, "");
964 lp_build_if(&ifthen, gallivm, cond);
965
966 if (nir_op == nir_intrinsic_ssbo_atomic_comp_swap || nir_op == nir_intrinsic_shared_atomic_comp_swap) {
967 LLVMValueRef cas_src_ptr = LLVMBuildExtractElement(gallivm->builder, val2,
968 loop_state.counter, "");
969 cas_src_ptr = LLVMBuildBitCast(gallivm->builder, cas_src_ptr, uint_bld->elem_type, "");
970 scalar = LLVMBuildAtomicCmpXchg(builder, scalar_ptr, value_ptr,
971 cas_src_ptr,
972 LLVMAtomicOrderingSequentiallyConsistent,
973 LLVMAtomicOrderingSequentiallyConsistent,
974 false);
975 scalar = LLVMBuildExtractValue(gallivm->builder, scalar, 0, "");
976 } else {
977 LLVMAtomicRMWBinOp op;
978
979 switch (nir_op) {
980 case nir_intrinsic_shared_atomic_add:
981 case nir_intrinsic_ssbo_atomic_add:
982 op = LLVMAtomicRMWBinOpAdd;
983 break;
984 case nir_intrinsic_shared_atomic_exchange:
985 case nir_intrinsic_ssbo_atomic_exchange:
986 op = LLVMAtomicRMWBinOpXchg;
987 break;
988 case nir_intrinsic_shared_atomic_and:
989 case nir_intrinsic_ssbo_atomic_and:
990 op = LLVMAtomicRMWBinOpAnd;
991 break;
992 case nir_intrinsic_shared_atomic_or:
993 case nir_intrinsic_ssbo_atomic_or:
994 op = LLVMAtomicRMWBinOpOr;
995 break;
996 case nir_intrinsic_shared_atomic_xor:
997 case nir_intrinsic_ssbo_atomic_xor:
998 op = LLVMAtomicRMWBinOpXor;
999 break;
1000 case nir_intrinsic_shared_atomic_umin:
1001 case nir_intrinsic_ssbo_atomic_umin:
1002 op = LLVMAtomicRMWBinOpUMin;
1003 break;
1004 case nir_intrinsic_shared_atomic_umax:
1005 case nir_intrinsic_ssbo_atomic_umax:
1006 op = LLVMAtomicRMWBinOpUMax;
1007 break;
1008 case nir_intrinsic_ssbo_atomic_imin:
1009 case nir_intrinsic_shared_atomic_imin:
1010 op = LLVMAtomicRMWBinOpMin;
1011 break;
1012 case nir_intrinsic_ssbo_atomic_imax:
1013 case nir_intrinsic_shared_atomic_imax:
1014 op = LLVMAtomicRMWBinOpMax;
1015 break;
1016 default:
1017 unreachable("unknown atomic op");
1018 }
1019 scalar = LLVMBuildAtomicRMW(builder, op,
1020 scalar_ptr, value_ptr,
1021 LLVMAtomicOrderingSequentiallyConsistent,
1022 false);
1023 }
1024 temp_res = LLVMBuildLoad(builder, atom_res, "");
1025 temp_res = LLVMBuildInsertElement(builder, temp_res, scalar, loop_state.counter, "");
1026 LLVMBuildStore(builder, temp_res, atom_res);
1027 lp_build_else(&ifthen);
1028 temp_res = LLVMBuildLoad(builder, atom_res, "");
1029 temp_res = LLVMBuildInsertElement(builder, temp_res, lp_build_const_int32(gallivm, 0), loop_state.counter, "");
1030 LLVMBuildStore(builder, temp_res, atom_res);
1031 lp_build_endif(&ifthen);
1032
1033 lp_build_loop_end_cond(&loop_state, lp_build_const_int32(gallivm, uint_bld->type.length),
1034 NULL, LLVMIntUGE);
1035 *result = LLVMBuildLoad(builder, atom_res, "");
1036 }
1037
1038 static void emit_barrier(struct lp_build_nir_context *bld_base)
1039 {
1040 struct lp_build_nir_soa_context *bld = (struct lp_build_nir_soa_context *)bld_base;
1041 struct gallivm_state * gallivm = bld_base->base.gallivm;
1042
1043 LLVMBasicBlockRef resume = lp_build_insert_new_block(gallivm, "resume");
1044
1045 lp_build_coro_suspend_switch(gallivm, bld->coro, resume, false);
1046 LLVMPositionBuilderAtEnd(gallivm->builder, resume);
1047 }
1048
1049 static LLVMValueRef emit_get_buffer_size(struct lp_build_nir_context *bld_base,
1050 LLVMValueRef index)
1051 {
1052 struct gallivm_state *gallivm = bld_base->base.gallivm;
1053 struct lp_build_nir_soa_context *bld = (struct lp_build_nir_soa_context *)bld_base;
1054 LLVMBuilderRef builder = bld->bld_base.base.gallivm->builder;
1055 struct lp_build_context *bld_broad = &bld_base->uint_bld;
1056 LLVMValueRef size_ptr = lp_build_array_get(bld_base->base.gallivm, bld->ssbo_sizes_ptr,
1057 LLVMBuildExtractElement(builder, index, lp_build_const_int32(gallivm, 0), ""));
1058 return lp_build_broadcast_scalar(bld_broad, size_ptr);
1059 }
1060
1061 static void emit_image_op(struct lp_build_nir_context *bld_base,
1062 struct lp_img_params *params)
1063 {
1064 struct lp_build_nir_soa_context *bld = (struct lp_build_nir_soa_context *)bld_base;
1065 params->type = bld_base->base.type;
1066 params->context_ptr = bld->context_ptr;
1067 params->thread_data_ptr = bld->thread_data_ptr;
1068 params->exec_mask = mask_vec(bld_base);
1069 bld->image->emit_op(bld->image,
1070 bld->bld_base.base.gallivm,
1071 params);
1072
1073 }
1074
1075 static void emit_image_size(struct lp_build_nir_context *bld_base,
1076 struct lp_sampler_size_query_params *params)
1077 {
1078 struct lp_build_nir_soa_context *bld = (struct lp_build_nir_soa_context *)bld_base;
1079
1080 params->int_type = bld_base->int_bld.type;
1081 params->context_ptr = bld->context_ptr;
1082
1083 bld->image->emit_size_query(bld->image,
1084 bld->bld_base.base.gallivm,
1085 params);
1086
1087 }
1088
1089 static void init_var_slots(struct lp_build_nir_context *bld_base,
1090 nir_variable *var, unsigned sc)
1091 {
1092 struct lp_build_nir_soa_context *bld = (struct lp_build_nir_soa_context *)bld_base;
1093 unsigned slots = glsl_count_attribute_slots(var->type, false) * 4;
1094
1095 for (unsigned comp = sc; comp < slots + sc; comp++) {
1096 unsigned this_loc = var->data.driver_location + (comp / 4);
1097 unsigned this_chan = comp % 4;
1098
1099 if (!bld->outputs[this_loc][this_chan])
1100 bld->outputs[this_loc][this_chan] = lp_build_alloca(bld_base->base.gallivm,
1101 bld_base->base.vec_type, "output");
1102 }
1103 }
1104
1105 static void emit_var_decl(struct lp_build_nir_context *bld_base,
1106 nir_variable *var)
1107 {
1108 unsigned sc = var->data.location_frac;
1109 switch (var->data.mode) {
1110 case nir_var_shader_out: {
1111 if (bld_base->shader->info.stage == MESA_SHADER_FRAGMENT) {
1112 if (var->data.location == FRAG_RESULT_STENCIL)
1113 sc = 1;
1114 else if (var->data.location == FRAG_RESULT_DEPTH)
1115 sc = 2;
1116 }
1117 init_var_slots(bld_base, var, sc);
1118 break;
1119 }
1120 default:
1121 break;
1122 }
1123 }
1124
1125 static void emit_tex(struct lp_build_nir_context *bld_base,
1126 struct lp_sampler_params *params)
1127 {
1128 struct lp_build_nir_soa_context *bld = (struct lp_build_nir_soa_context *)bld_base;
1129
1130 params->type = bld_base->base.type;
1131 params->context_ptr = bld->context_ptr;
1132 params->thread_data_ptr = bld->thread_data_ptr;
1133
1134 bld->sampler->emit_tex_sample(bld->sampler,
1135 bld->bld_base.base.gallivm,
1136 params);
1137 }
1138
1139 static void emit_tex_size(struct lp_build_nir_context *bld_base,
1140 struct lp_sampler_size_query_params *params)
1141 {
1142 struct lp_build_nir_soa_context *bld = (struct lp_build_nir_soa_context *)bld_base;
1143
1144 params->int_type = bld_base->int_bld.type;
1145 params->context_ptr = bld->context_ptr;
1146
1147 bld->sampler->emit_size_query(bld->sampler,
1148 bld->bld_base.base.gallivm,
1149 params);
1150 }
1151
1152 static void emit_sysval_intrin(struct lp_build_nir_context *bld_base,
1153 nir_intrinsic_instr *instr,
1154 LLVMValueRef result[NIR_MAX_VEC_COMPONENTS])
1155 {
1156 struct lp_build_nir_soa_context *bld = (struct lp_build_nir_soa_context *)bld_base;
1157 struct gallivm_state *gallivm = bld_base->base.gallivm;
1158 switch (instr->intrinsic) {
1159 case nir_intrinsic_load_instance_id:
1160 result[0] = lp_build_broadcast_scalar(&bld_base->uint_bld, bld->system_values.instance_id);
1161 break;
1162 case nir_intrinsic_load_base_instance:
1163 result[0] = lp_build_broadcast_scalar(&bld_base->uint_bld, bld->system_values.base_instance);
1164 break;
1165 case nir_intrinsic_load_base_vertex:
1166 result[0] = bld->system_values.basevertex;
1167 break;
1168 case nir_intrinsic_load_vertex_id:
1169 result[0] = bld->system_values.vertex_id;
1170 break;
1171 case nir_intrinsic_load_primitive_id:
1172 result[0] = bld->system_values.prim_id;
1173 break;
1174 case nir_intrinsic_load_work_group_id:
1175 for (unsigned i = 0; i < 3; i++)
1176 result[i] = lp_build_broadcast_scalar(&bld_base->uint_bld, LLVMBuildExtractElement(gallivm->builder, bld->system_values.block_id, lp_build_const_int32(gallivm, i), ""));
1177 break;
1178 case nir_intrinsic_load_local_invocation_id:
1179 for (unsigned i = 0; i < 3; i++)
1180 result[i] = LLVMBuildExtractValue(gallivm->builder, bld->system_values.thread_id, i, "");
1181 break;
1182 case nir_intrinsic_load_num_work_groups:
1183 for (unsigned i = 0; i < 3; i++)
1184 result[i] = lp_build_broadcast_scalar(&bld_base->uint_bld, LLVMBuildExtractElement(gallivm->builder, bld->system_values.grid_size, lp_build_const_int32(gallivm, i), ""));
1185 break;
1186 case nir_intrinsic_load_invocation_id:
1187 result[0] = lp_build_broadcast_scalar(&bld_base->uint_bld, bld->system_values.invocation_id);
1188 break;
1189 case nir_intrinsic_load_front_face:
1190 result[0] = lp_build_broadcast_scalar(&bld_base->uint_bld, bld->system_values.front_facing);
1191 break;
1192 case nir_intrinsic_load_draw_id:
1193 result[0] = lp_build_broadcast_scalar(&bld_base->uint_bld, bld->system_values.draw_id);
1194 break;
1195 default:
1196 break;
1197 case nir_intrinsic_load_local_group_size:
1198 for (unsigned i = 0; i < 3; i++)
1199 result[i] = lp_build_broadcast_scalar(&bld_base->uint_bld, LLVMBuildExtractElement(gallivm->builder, bld->system_values.block_size, lp_build_const_int32(gallivm, i), ""));
1200 break;
1201 case nir_intrinsic_load_work_dim:
1202 result[0] = lp_build_broadcast_scalar(&bld_base->uint_bld, bld->system_values.work_dim);
1203 break;
1204 }
1205 }
1206
1207 static void bgnloop(struct lp_build_nir_context *bld_base)
1208 {
1209 struct lp_build_nir_soa_context *bld = (struct lp_build_nir_soa_context *)bld_base;
1210 lp_exec_bgnloop(&bld->exec_mask, true);
1211 }
1212
1213 static void endloop(struct lp_build_nir_context *bld_base)
1214 {
1215 struct lp_build_nir_soa_context *bld = (struct lp_build_nir_soa_context *)bld_base;
1216 lp_exec_endloop(bld_base->base.gallivm, &bld->exec_mask);
1217 }
1218
1219 static void if_cond(struct lp_build_nir_context *bld_base, LLVMValueRef cond)
1220 {
1221 LLVMBuilderRef builder = bld_base->base.gallivm->builder;
1222 struct lp_build_nir_soa_context *bld = (struct lp_build_nir_soa_context *)bld_base;
1223 lp_exec_mask_cond_push(&bld->exec_mask, LLVMBuildBitCast(builder, cond, bld_base->base.int_vec_type, ""));
1224 }
1225
1226 static void else_stmt(struct lp_build_nir_context *bld_base)
1227 {
1228 struct lp_build_nir_soa_context *bld = (struct lp_build_nir_soa_context *)bld_base;
1229 lp_exec_mask_cond_invert(&bld->exec_mask);
1230 }
1231
1232 static void endif_stmt(struct lp_build_nir_context *bld_base)
1233 {
1234 struct lp_build_nir_soa_context *bld = (struct lp_build_nir_soa_context *)bld_base;
1235 lp_exec_mask_cond_pop(&bld->exec_mask);
1236 }
1237
1238 static void break_stmt(struct lp_build_nir_context *bld_base)
1239 {
1240 struct lp_build_nir_soa_context *bld = (struct lp_build_nir_soa_context *)bld_base;
1241
1242 lp_exec_break(&bld->exec_mask, NULL, false);
1243 }
1244
1245 static void continue_stmt(struct lp_build_nir_context *bld_base)
1246 {
1247 struct lp_build_nir_soa_context *bld = (struct lp_build_nir_soa_context *)bld_base;
1248 lp_exec_continue(&bld->exec_mask);
1249 }
1250
1251 static void discard(struct lp_build_nir_context *bld_base, LLVMValueRef cond)
1252 {
1253 struct lp_build_nir_soa_context *bld = (struct lp_build_nir_soa_context *)bld_base;
1254 LLVMBuilderRef builder = bld->bld_base.base.gallivm->builder;
1255 LLVMValueRef mask;
1256
1257 if (!cond) {
1258 if (bld->exec_mask.has_mask) {
1259 mask = LLVMBuildNot(builder, bld->exec_mask.exec_mask, "kilp");
1260 } else {
1261 mask = LLVMConstNull(bld->bld_base.base.int_vec_type);
1262 }
1263 } else {
1264 mask = LLVMBuildNot(builder, cond, "");
1265 if (bld->exec_mask.has_mask) {
1266 LLVMValueRef invmask;
1267 invmask = LLVMBuildNot(builder, bld->exec_mask.exec_mask, "kilp");
1268 mask = LLVMBuildOr(builder, mask, invmask, "");
1269 }
1270 }
1271 lp_build_mask_update(bld->mask, mask);
1272 }
1273
1274 static void
1275 increment_vec_ptr_by_mask(struct lp_build_nir_context * bld_base,
1276 LLVMValueRef ptr,
1277 LLVMValueRef mask)
1278 {
1279 LLVMBuilderRef builder = bld_base->base.gallivm->builder;
1280 LLVMValueRef current_vec = LLVMBuildLoad(builder, ptr, "");
1281
1282 current_vec = LLVMBuildSub(builder, current_vec, mask, "");
1283
1284 LLVMBuildStore(builder, current_vec, ptr);
1285 }
1286
1287 static void
1288 clear_uint_vec_ptr_from_mask(struct lp_build_nir_context * bld_base,
1289 LLVMValueRef ptr,
1290 LLVMValueRef mask)
1291 {
1292 LLVMBuilderRef builder = bld_base->base.gallivm->builder;
1293 LLVMValueRef current_vec = LLVMBuildLoad(builder, ptr, "");
1294
1295 current_vec = lp_build_select(&bld_base->uint_bld,
1296 mask,
1297 bld_base->uint_bld.zero,
1298 current_vec);
1299
1300 LLVMBuildStore(builder, current_vec, ptr);
1301 }
1302
1303 static LLVMValueRef
1304 clamp_mask_to_max_output_vertices(struct lp_build_nir_soa_context * bld,
1305 LLVMValueRef current_mask_vec,
1306 LLVMValueRef total_emitted_vertices_vec)
1307 {
1308 LLVMBuilderRef builder = bld->bld_base.base.gallivm->builder;
1309 struct lp_build_context *int_bld = &bld->bld_base.int_bld;
1310 LLVMValueRef max_mask = lp_build_cmp(int_bld, PIPE_FUNC_LESS,
1311 total_emitted_vertices_vec,
1312 bld->max_output_vertices_vec);
1313
1314 return LLVMBuildAnd(builder, current_mask_vec, max_mask, "");
1315 }
1316
1317 static void emit_vertex(struct lp_build_nir_context *bld_base, uint32_t stream_id)
1318 {
1319 struct lp_build_nir_soa_context *bld = (struct lp_build_nir_soa_context *)bld_base;
1320 LLVMBuilderRef builder = bld->bld_base.base.gallivm->builder;
1321
1322 assert(bld->gs_iface->emit_vertex);
1323 LLVMValueRef total_emitted_vertices_vec =
1324 LLVMBuildLoad(builder, bld->total_emitted_vertices_vec_ptr[stream_id], "");
1325 LLVMValueRef mask = mask_vec(bld_base);
1326 mask = clamp_mask_to_max_output_vertices(bld, mask,
1327 total_emitted_vertices_vec);
1328 bld->gs_iface->emit_vertex(bld->gs_iface, &bld->bld_base.base,
1329 bld->outputs,
1330 total_emitted_vertices_vec,
1331 lp_build_const_int_vec(bld->bld_base.base.gallivm, bld->bld_base.base.type, stream_id));
1332
1333 increment_vec_ptr_by_mask(bld_base, bld->emitted_vertices_vec_ptr[stream_id],
1334 mask);
1335 increment_vec_ptr_by_mask(bld_base, bld->total_emitted_vertices_vec_ptr[stream_id],
1336 mask);
1337 }
1338
1339 static void
1340 end_primitive_masked(struct lp_build_nir_context * bld_base,
1341 LLVMValueRef mask, uint32_t stream_id)
1342 {
1343 struct lp_build_nir_soa_context *bld = (struct lp_build_nir_soa_context *)bld_base;
1344 LLVMBuilderRef builder = bld->bld_base.base.gallivm->builder;
1345
1346 struct lp_build_context *uint_bld = &bld_base->uint_bld;
1347 LLVMValueRef emitted_vertices_vec =
1348 LLVMBuildLoad(builder, bld->emitted_vertices_vec_ptr[stream_id], "");
1349 LLVMValueRef emitted_prims_vec =
1350 LLVMBuildLoad(builder, bld->emitted_prims_vec_ptr[stream_id], "");
1351 LLVMValueRef total_emitted_vertices_vec =
1352 LLVMBuildLoad(builder, bld->total_emitted_vertices_vec_ptr[stream_id], "");
1353
1354 LLVMValueRef emitted_mask = lp_build_cmp(uint_bld,
1355 PIPE_FUNC_NOTEQUAL,
1356 emitted_vertices_vec,
1357 uint_bld->zero);
1358 mask = LLVMBuildAnd(builder, mask, emitted_mask, "");
1359 if (stream_id == 0)
1360 bld->gs_iface->end_primitive(bld->gs_iface, &bld->bld_base.base,
1361 total_emitted_vertices_vec,
1362 emitted_vertices_vec, emitted_prims_vec, mask_vec(bld_base));
1363 increment_vec_ptr_by_mask(bld_base, bld->emitted_prims_vec_ptr[stream_id],
1364 mask);
1365 clear_uint_vec_ptr_from_mask(bld_base, bld->emitted_vertices_vec_ptr[stream_id],
1366 mask);
1367 }
1368
1369 static void end_primitive(struct lp_build_nir_context *bld_base, uint32_t stream_id)
1370 {
1371 ASSERTED struct lp_build_nir_soa_context *bld = (struct lp_build_nir_soa_context *)bld_base;
1372
1373 assert(bld->gs_iface->end_primitive);
1374
1375 LLVMValueRef mask = mask_vec(bld_base);
1376 end_primitive_masked(bld_base, mask, stream_id);
1377 }
1378
1379 static void
1380 emit_prologue(struct lp_build_nir_soa_context *bld)
1381 {
1382 struct gallivm_state * gallivm = bld->bld_base.base.gallivm;
1383 if (bld->indirects & nir_var_shader_in && !bld->gs_iface) {
1384 uint32_t num_inputs = util_bitcount64(bld->bld_base.shader->info.inputs_read);
1385 unsigned index, chan;
1386 LLVMTypeRef vec_type = bld->bld_base.base.vec_type;
1387 LLVMValueRef array_size = lp_build_const_int32(gallivm, num_inputs * 4);
1388 bld->inputs_array = lp_build_array_alloca(gallivm,
1389 vec_type, array_size,
1390 "input_array");
1391
1392 for (index = 0; index < num_inputs; ++index) {
1393 for (chan = 0; chan < TGSI_NUM_CHANNELS; ++chan) {
1394 LLVMValueRef lindex =
1395 lp_build_const_int32(gallivm, index * 4 + chan);
1396 LLVMValueRef input_ptr =
1397 LLVMBuildGEP(gallivm->builder, bld->inputs_array,
1398 &lindex, 1, "");
1399 LLVMValueRef value = bld->inputs[index][chan];
1400 if (value)
1401 LLVMBuildStore(gallivm->builder, value, input_ptr);
1402 }
1403 }
1404 }
1405 }
1406
1407 static void emit_vote(struct lp_build_nir_context *bld_base, LLVMValueRef src, nir_intrinsic_instr *instr, LLVMValueRef result[4])
1408 {
1409 struct gallivm_state * gallivm = bld_base->base.gallivm;
1410 LLVMBuilderRef builder = gallivm->builder;
1411
1412 LLVMValueRef exec_mask = mask_vec(bld_base);
1413 struct lp_build_loop_state loop_state;
1414
1415 LLVMValueRef outer_cond = LLVMBuildICmp(builder, LLVMIntNE, exec_mask, bld_base->uint_bld.zero, "");
1416
1417 LLVMValueRef res_store = lp_build_alloca(gallivm, bld_base->int_bld.elem_type, "");
1418 LLVMValueRef init_val = NULL;
1419 if (instr->intrinsic == nir_intrinsic_vote_ieq) {
1420 /* for equal we unfortunately have to loop and find the first valid one. */
1421 lp_build_loop_begin(&loop_state, gallivm, lp_build_const_int32(gallivm, 0));
1422 LLVMValueRef if_cond = LLVMBuildExtractElement(gallivm->builder, outer_cond, loop_state.counter, "");
1423
1424 struct lp_build_if_state ifthen;
1425 lp_build_if(&ifthen, gallivm, if_cond);
1426 LLVMValueRef value_ptr = LLVMBuildExtractElement(gallivm->builder, src,
1427 loop_state.counter, "");
1428 LLVMBuildStore(builder, value_ptr, res_store);
1429 lp_build_endif(&ifthen);
1430 lp_build_loop_end_cond(&loop_state, lp_build_const_int32(gallivm, bld_base->uint_bld.type.length),
1431 NULL, LLVMIntUGE);
1432 lp_build_print_value(gallivm, "init_val is ", LLVMBuildLoad(builder, res_store, ""));
1433 init_val = LLVMBuildLoad(builder, res_store, "");
1434 } else {
1435 LLVMBuildStore(builder, lp_build_const_int32(gallivm, instr->intrinsic == nir_intrinsic_vote_any ? 0 : -1), res_store);
1436 }
1437
1438 LLVMValueRef res;
1439 lp_build_loop_begin(&loop_state, gallivm, lp_build_const_int32(gallivm, 0));
1440 LLVMValueRef value_ptr = LLVMBuildExtractElement(gallivm->builder, src,
1441 loop_state.counter, "");
1442 struct lp_build_if_state ifthen;
1443 LLVMValueRef if_cond;
1444 if_cond = LLVMBuildExtractElement(gallivm->builder, outer_cond, loop_state.counter, "");
1445
1446 lp_build_if(&ifthen, gallivm, if_cond);
1447 res = LLVMBuildLoad(builder, res_store, "");
1448
1449 if (instr->intrinsic == nir_intrinsic_vote_ieq) {
1450 LLVMValueRef tmp = LLVMBuildICmp(builder, LLVMIntEQ, init_val, value_ptr, "");
1451 tmp = LLVMBuildSExt(builder, tmp, bld_base->uint_bld.elem_type, "");
1452 res = LLVMBuildOr(builder, res, tmp, "");
1453 } else if (instr->intrinsic == nir_intrinsic_vote_any)
1454 res = LLVMBuildOr(builder, res, value_ptr, "");
1455 else
1456 res = LLVMBuildAnd(builder, res, value_ptr, "");
1457 LLVMBuildStore(builder, res, res_store);
1458 lp_build_endif(&ifthen);
1459 lp_build_loop_end_cond(&loop_state, lp_build_const_int32(gallivm, bld_base->uint_bld.type.length),
1460 NULL, LLVMIntUGE);
1461 result[0] = lp_build_broadcast_scalar(&bld_base->uint_bld, LLVMBuildLoad(builder, res_store, ""));
1462 }
1463
1464 void lp_build_nir_soa(struct gallivm_state *gallivm,
1465 struct nir_shader *shader,
1466 const struct lp_build_tgsi_params *params,
1467 LLVMValueRef (*outputs)[4])
1468 {
1469 struct lp_build_nir_soa_context bld;
1470 struct lp_type type = params->type;
1471 struct lp_type res_type;
1472
1473 assert(type.length <= LP_MAX_VECTOR_LENGTH);
1474 memset(&res_type, 0, sizeof res_type);
1475 res_type.width = type.width;
1476 res_type.length = type.length;
1477 res_type.sign = 1;
1478
1479 /* Setup build context */
1480 memset(&bld, 0, sizeof bld);
1481 lp_build_context_init(&bld.bld_base.base, gallivm, type);
1482 lp_build_context_init(&bld.bld_base.uint_bld, gallivm, lp_uint_type(type));
1483 lp_build_context_init(&bld.bld_base.int_bld, gallivm, lp_int_type(type));
1484 lp_build_context_init(&bld.elem_bld, gallivm, lp_elem_type(type));
1485 lp_build_context_init(&bld.uint_elem_bld, gallivm, lp_elem_type(lp_uint_type(type)));
1486 {
1487 struct lp_type dbl_type;
1488 dbl_type = type;
1489 dbl_type.width *= 2;
1490 lp_build_context_init(&bld.bld_base.dbl_bld, gallivm, dbl_type);
1491 }
1492 {
1493 struct lp_type uint64_type;
1494 uint64_type = lp_uint_type(type);
1495 uint64_type.width *= 2;
1496 lp_build_context_init(&bld.bld_base.uint64_bld, gallivm, uint64_type);
1497 }
1498 {
1499 struct lp_type int64_type;
1500 int64_type = lp_int_type(type);
1501 int64_type.width *= 2;
1502 lp_build_context_init(&bld.bld_base.int64_bld, gallivm, int64_type);
1503 }
1504 {
1505 struct lp_type uint16_type;
1506 uint16_type = lp_uint_type(type);
1507 uint16_type.width /= 2;
1508 lp_build_context_init(&bld.bld_base.uint16_bld, gallivm, uint16_type);
1509 }
1510 {
1511 struct lp_type int16_type;
1512 int16_type = lp_int_type(type);
1513 int16_type.width /= 2;
1514 lp_build_context_init(&bld.bld_base.int16_bld, gallivm, int16_type);
1515 }
1516 {
1517 struct lp_type uint8_type;
1518 uint8_type = lp_uint_type(type);
1519 uint8_type.width /= 4;
1520 lp_build_context_init(&bld.bld_base.uint8_bld, gallivm, uint8_type);
1521 }
1522 {
1523 struct lp_type int8_type;
1524 int8_type = lp_int_type(type);
1525 int8_type.width /= 4;
1526 lp_build_context_init(&bld.bld_base.int8_bld, gallivm, int8_type);
1527 }
1528 bld.bld_base.load_var = emit_load_var;
1529 bld.bld_base.store_var = emit_store_var;
1530 bld.bld_base.load_reg = emit_load_reg;
1531 bld.bld_base.store_reg = emit_store_reg;
1532 bld.bld_base.emit_var_decl = emit_var_decl;
1533 bld.bld_base.load_ubo = emit_load_ubo;
1534 bld.bld_base.load_kernel_arg = emit_load_kernel_arg;
1535 bld.bld_base.load_global = emit_load_global;
1536 bld.bld_base.store_global = emit_store_global;
1537 bld.bld_base.atomic_global = emit_atomic_global;
1538 bld.bld_base.tex = emit_tex;
1539 bld.bld_base.tex_size = emit_tex_size;
1540 bld.bld_base.bgnloop = bgnloop;
1541 bld.bld_base.endloop = endloop;
1542 bld.bld_base.if_cond = if_cond;
1543 bld.bld_base.else_stmt = else_stmt;
1544 bld.bld_base.endif_stmt = endif_stmt;
1545 bld.bld_base.break_stmt = break_stmt;
1546 bld.bld_base.continue_stmt = continue_stmt;
1547 bld.bld_base.sysval_intrin = emit_sysval_intrin;
1548 bld.bld_base.discard = discard;
1549 bld.bld_base.emit_vertex = emit_vertex;
1550 bld.bld_base.end_primitive = end_primitive;
1551 bld.bld_base.load_mem = emit_load_mem;
1552 bld.bld_base.store_mem = emit_store_mem;
1553 bld.bld_base.get_buffer_size = emit_get_buffer_size;
1554 bld.bld_base.atomic_mem = emit_atomic_mem;
1555 bld.bld_base.barrier = emit_barrier;
1556 bld.bld_base.image_op = emit_image_op;
1557 bld.bld_base.image_size = emit_image_size;
1558 bld.bld_base.vote = emit_vote;
1559
1560 bld.mask = params->mask;
1561 bld.inputs = params->inputs;
1562 bld.outputs = outputs;
1563 bld.consts_ptr = params->consts_ptr;
1564 bld.const_sizes_ptr = params->const_sizes_ptr;
1565 bld.ssbo_ptr = params->ssbo_ptr;
1566 bld.ssbo_sizes_ptr = params->ssbo_sizes_ptr;
1567 bld.sampler = params->sampler;
1568 // bld.bld_base.info = params->info;
1569
1570 bld.context_ptr = params->context_ptr;
1571 bld.thread_data_ptr = params->thread_data_ptr;
1572 bld.image = params->image;
1573 bld.shared_ptr = params->shared_ptr;
1574 bld.coro = params->coro;
1575 bld.kernel_args_ptr = params->kernel_args;
1576 bld.indirects = 0;
1577 if (params->info->indirect_files & (1 << TGSI_FILE_INPUT))
1578 bld.indirects |= nir_var_shader_in;
1579
1580 bld.gs_iface = params->gs_iface;
1581 if (bld.gs_iface) {
1582 struct lp_build_context *uint_bld = &bld.bld_base.uint_bld;
1583
1584 bld.max_output_vertices_vec = lp_build_const_int_vec(gallivm, bld.bld_base.int_bld.type,
1585 shader->info.gs.vertices_out);
1586 for (int i = 0; i < PIPE_MAX_VERTEX_STREAMS; i++) {
1587 bld.emitted_prims_vec_ptr[i] =
1588 lp_build_alloca(gallivm, uint_bld->vec_type, "emitted_prims_ptr");
1589 bld.emitted_vertices_vec_ptr[i] =
1590 lp_build_alloca(gallivm, uint_bld->vec_type, "emitted_vertices_ptr");
1591 bld.total_emitted_vertices_vec_ptr[i] =
1592 lp_build_alloca(gallivm, uint_bld->vec_type, "total_emitted_vertices_ptr");
1593 }
1594 }
1595 lp_exec_mask_init(&bld.exec_mask, &bld.bld_base.int_bld);
1596
1597 bld.system_values = *params->system_values;
1598
1599 bld.bld_base.shader = shader;
1600
1601 emit_prologue(&bld);
1602 lp_build_nir_llvm(&bld.bld_base, shader);
1603
1604 if (bld.gs_iface) {
1605 LLVMBuilderRef builder = bld.bld_base.base.gallivm->builder;
1606 LLVMValueRef total_emitted_vertices_vec;
1607 LLVMValueRef emitted_prims_vec;
1608
1609 end_primitive_masked(&bld.bld_base, lp_build_mask_value(bld.mask), 0);
1610 for (int i = 0; i < PIPE_MAX_VERTEX_STREAMS; i++) {
1611 total_emitted_vertices_vec =
1612 LLVMBuildLoad(builder, bld.total_emitted_vertices_vec_ptr[i], "");
1613
1614 emitted_prims_vec =
1615 LLVMBuildLoad(builder, bld.emitted_prims_vec_ptr[i], "");
1616 bld.gs_iface->gs_epilogue(bld.gs_iface,
1617 total_emitted_vertices_vec,
1618 emitted_prims_vec, i);
1619 }
1620 }
1621 lp_exec_mask_fini(&bld.exec_mask);
1622 }