1 /**************************************************************************
3 * Copyright 2019 Red Hat.
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the "Software"),
8 * to deal in the Software without restriction, including without limitation
9 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
10 * and/or sell copies of the Software, and to permit persons to whom the
11 * Software is furnished to do so, subject to the following conditions:
13 * The above copyright notice and this permission notice shall be included
14 * in all copies or substantial portions of the Software.
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
17 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
24 **************************************************************************/
26 #include "lp_bld_nir.h"
27 #include "lp_bld_init.h"
28 #include "lp_bld_flow.h"
29 #include "lp_bld_logic.h"
30 #include "lp_bld_gather.h"
31 #include "lp_bld_const.h"
32 #include "lp_bld_struct.h"
33 #include "lp_bld_arit.h"
34 #include "lp_bld_bitarit.h"
35 #include "lp_bld_coro.h"
36 #include "lp_bld_printf.h"
37 #include "util/u_math.h"
39 * combine the execution mask if there is one with the current mask.
42 mask_vec(struct lp_build_nir_context
*bld_base
)
44 struct lp_build_nir_soa_context
* bld
= (struct lp_build_nir_soa_context
*)bld_base
;
45 LLVMBuilderRef builder
= bld
->bld_base
.base
.gallivm
->builder
;
46 struct lp_exec_mask
*exec_mask
= &bld
->exec_mask
;
47 LLVMValueRef bld_mask
= bld
->mask
? lp_build_mask_value(bld
->mask
) : NULL
;
48 if (!exec_mask
->has_mask
) {
52 return exec_mask
->exec_mask
;
53 return LLVMBuildAnd(builder
, lp_build_mask_value(bld
->mask
),
54 exec_mask
->exec_mask
, "");
59 struct lp_build_nir_context
* bld_base
,
63 struct gallivm_state
*gallivm
= bld_base
->base
.gallivm
;
64 LLVMBuilderRef builder
= gallivm
->builder
;
67 LLVMValueRef shuffles
[2 * (LP_MAX_VECTOR_WIDTH
/32)];
68 int len
= bld_base
->base
.type
.length
* 2;
69 assert(len
<= (2 * (LP_MAX_VECTOR_WIDTH
/32)));
71 for (i
= 0; i
< bld_base
->base
.type
.length
* 2; i
+=2) {
72 #if UTIL_ARCH_LITTLE_ENDIAN
73 shuffles
[i
] = lp_build_const_int32(gallivm
, i
/ 2);
74 shuffles
[i
+ 1] = lp_build_const_int32(gallivm
, i
/ 2 + bld_base
->base
.type
.length
);
76 shuffles
[i
] = lp_build_const_int32(gallivm
, i
/ 2 + bld_base
->base
.type
.length
);
77 shuffles
[i
+ 1] = lp_build_const_int32(gallivm
, i
/ 2);
80 res
= LLVMBuildShuffleVector(builder
, input
, input2
, LLVMConstVector(shuffles
, len
), "");
82 return LLVMBuildBitCast(builder
, res
, bld_base
->dbl_bld
.vec_type
, "");
86 emit_store_64bit_split(struct lp_build_nir_context
*bld_base
,
88 LLVMValueRef split_values
[2])
90 struct gallivm_state
*gallivm
= bld_base
->base
.gallivm
;
91 LLVMBuilderRef builder
= gallivm
->builder
;
93 LLVMValueRef shuffles
[LP_MAX_VECTOR_WIDTH
/32];
94 LLVMValueRef shuffles2
[LP_MAX_VECTOR_WIDTH
/32];
95 int len
= bld_base
->base
.type
.length
* 2;
97 value
= LLVMBuildBitCast(gallivm
->builder
, value
, LLVMVectorType(LLVMFloatTypeInContext(gallivm
->context
), len
), "");
98 for (i
= 0; i
< bld_base
->base
.type
.length
; i
++) {
99 #if UTIL_ARCH_LITTLE_ENDIAN
100 shuffles
[i
] = lp_build_const_int32(gallivm
, i
* 2);
101 shuffles2
[i
] = lp_build_const_int32(gallivm
, (i
* 2) + 1);
103 shuffles
[i
] = lp_build_const_int32(gallivm
, i
* 2 + 1);
104 shuffles2
[i
] = lp_build_const_int32(gallivm
, i
* 2);
108 split_values
[0] = LLVMBuildShuffleVector(builder
, value
,
109 LLVMGetUndef(LLVMTypeOf(value
)),
110 LLVMConstVector(shuffles
,
111 bld_base
->base
.type
.length
),
113 split_values
[1] = LLVMBuildShuffleVector(builder
, value
,
114 LLVMGetUndef(LLVMTypeOf(value
)),
115 LLVMConstVector(shuffles2
,
116 bld_base
->base
.type
.length
),
121 emit_store_64bit_chan(struct lp_build_nir_context
*bld_base
,
122 LLVMValueRef chan_ptr
,
123 LLVMValueRef chan_ptr2
,
126 struct lp_build_nir_soa_context
*bld
= (struct lp_build_nir_soa_context
*)bld_base
;
127 struct lp_build_context
*float_bld
= &bld_base
->base
;
128 LLVMValueRef split_vals
[2];
130 emit_store_64bit_split(bld_base
, value
, split_vals
);
132 lp_exec_mask_store(&bld
->exec_mask
, float_bld
, split_vals
[0], chan_ptr
);
133 lp_exec_mask_store(&bld
->exec_mask
, float_bld
, split_vals
[1], chan_ptr2
);
137 get_soa_array_offsets(struct lp_build_context
*uint_bld
,
138 LLVMValueRef indirect_index
,
141 bool need_perelement_offset
)
143 struct gallivm_state
*gallivm
= uint_bld
->gallivm
;
144 LLVMValueRef chan_vec
=
145 lp_build_const_int_vec(uint_bld
->gallivm
, uint_bld
->type
, chan_index
);
146 LLVMValueRef length_vec
=
147 lp_build_const_int_vec(gallivm
, uint_bld
->type
, uint_bld
->type
.length
);
148 LLVMValueRef index_vec
;
150 /* index_vec = (indirect_index * 4 + chan_index) * length + offsets */
151 index_vec
= lp_build_mul(uint_bld
, indirect_index
, lp_build_const_int_vec(uint_bld
->gallivm
, uint_bld
->type
, num_components
));
152 index_vec
= lp_build_add(uint_bld
, index_vec
, chan_vec
);
153 index_vec
= lp_build_mul(uint_bld
, index_vec
, length_vec
);
155 if (need_perelement_offset
) {
156 LLVMValueRef pixel_offsets
;
158 /* build pixel offset vector: {0, 1, 2, 3, ...} */
159 pixel_offsets
= uint_bld
->undef
;
160 for (i
= 0; i
< uint_bld
->type
.length
; i
++) {
161 LLVMValueRef ii
= lp_build_const_int32(gallivm
, i
);
162 pixel_offsets
= LLVMBuildInsertElement(gallivm
->builder
, pixel_offsets
,
165 index_vec
= lp_build_add(uint_bld
, index_vec
, pixel_offsets
);
171 build_gather(struct lp_build_nir_context
*bld_base
,
172 struct lp_build_context
*bld
,
173 LLVMValueRef base_ptr
,
174 LLVMValueRef indexes
,
175 LLVMValueRef overflow_mask
,
176 LLVMValueRef indexes2
)
178 struct gallivm_state
*gallivm
= bld_base
->base
.gallivm
;
179 LLVMBuilderRef builder
= gallivm
->builder
;
180 struct lp_build_context
*uint_bld
= &bld_base
->uint_bld
;
185 res
= LLVMGetUndef(LLVMVectorType(LLVMFloatTypeInContext(gallivm
->context
), bld_base
->base
.type
.length
* 2));
189 * overflow_mask is a vector telling us which channels
190 * in the vector overflowed. We use the overflow behavior for
191 * constant buffers which is defined as:
192 * Out of bounds access to constant buffer returns 0 in all
193 * components. Out of bounds behavior is always with respect
194 * to the size of the buffer bound at that slot.
199 * We avoid per-element control flow here (also due to llvm going crazy,
200 * though I suspect it's better anyway since overflow is likely rare).
201 * Note that since we still fetch from buffers even if num_elements was
202 * zero (in this case we'll fetch from index zero) the jit func callers
203 * MUST provide valid fake constant buffers of size 4x32 (the values do
204 * not matter), otherwise we'd still need (not per element though)
207 indexes
= lp_build_select(uint_bld
, overflow_mask
, uint_bld
->zero
, indexes
);
209 indexes2
= lp_build_select(uint_bld
, overflow_mask
, uint_bld
->zero
, indexes2
);
213 * Loop over elements of index_vec, load scalar value, insert it into 'res'.
215 for (i
= 0; i
< bld
->type
.length
* (indexes2
? 2 : 1); i
++) {
218 LLVMValueRef scalar_ptr
, scalar
;
220 di
= lp_build_const_int32(gallivm
, i
);
222 si
= lp_build_const_int32(gallivm
, i
>> 1);
226 if (indexes2
&& (i
& 1)) {
227 index
= LLVMBuildExtractElement(builder
,
230 index
= LLVMBuildExtractElement(builder
,
233 scalar_ptr
= LLVMBuildGEP(builder
, base_ptr
,
234 &index
, 1, "gather_ptr");
235 scalar
= LLVMBuildLoad(builder
, scalar_ptr
, "");
237 res
= LLVMBuildInsertElement(builder
, res
, scalar
, di
, "");
242 res
= LLVMBuildBitCast(builder
, res
, bld_base
->dbl_bld
.vec_type
, "");
243 overflow_mask
= LLVMBuildSExt(builder
, overflow_mask
,
244 bld_base
->dbl_bld
.int_vec_type
, "");
245 res
= lp_build_select(&bld_base
->dbl_bld
, overflow_mask
,
246 bld_base
->dbl_bld
.zero
, res
);
248 res
= lp_build_select(bld
, overflow_mask
, bld
->zero
, res
);
255 * Scatter/store vector.
258 emit_mask_scatter(struct lp_build_nir_soa_context
*bld
,
259 LLVMValueRef base_ptr
,
260 LLVMValueRef indexes
,
262 struct lp_exec_mask
*mask
)
264 struct gallivm_state
*gallivm
= bld
->bld_base
.base
.gallivm
;
265 LLVMBuilderRef builder
= gallivm
->builder
;
267 LLVMValueRef pred
= mask
->has_mask
? mask
->exec_mask
: NULL
;
270 * Loop over elements of index_vec, store scalar value.
272 for (i
= 0; i
< bld
->bld_base
.base
.type
.length
; i
++) {
273 LLVMValueRef ii
= lp_build_const_int32(gallivm
, i
);
274 LLVMValueRef index
= LLVMBuildExtractElement(builder
, indexes
, ii
, "");
275 LLVMValueRef scalar_ptr
= LLVMBuildGEP(builder
, base_ptr
, &index
, 1, "scatter_ptr");
276 LLVMValueRef val
= LLVMBuildExtractElement(builder
, values
, ii
, "scatter_val");
277 LLVMValueRef scalar_pred
= pred
?
278 LLVMBuildExtractElement(builder
, pred
, ii
, "scatter_pred") : NULL
;
281 lp_build_printf(gallivm
, "scatter %d: val %f at %d %p\n",
282 ii
, val
, index
, scalar_ptr
);
285 LLVMValueRef real_val
, dst_val
;
286 dst_val
= LLVMBuildLoad(builder
, scalar_ptr
, "");
287 real_val
= lp_build_select(&bld
->uint_elem_bld
, scalar_pred
, val
, dst_val
);
288 LLVMBuildStore(builder
, real_val
, scalar_ptr
);
291 LLVMBuildStore(builder
, val
, scalar_ptr
);
296 static void emit_load_var(struct lp_build_nir_context
*bld_base
,
297 nir_variable_mode deref_mode
,
298 unsigned num_components
,
301 unsigned vertex_index
,
302 LLVMValueRef indir_vertex_index
,
303 unsigned const_index
,
304 LLVMValueRef indir_index
,
305 LLVMValueRef result
[NIR_MAX_VEC_COMPONENTS
])
307 struct lp_build_nir_soa_context
*bld
= (struct lp_build_nir_soa_context
*)bld_base
;
308 struct gallivm_state
*gallivm
= bld_base
->base
.gallivm
;
309 int dmul
= bit_size
== 64 ? 2 : 1;
310 unsigned location
= var
->data
.driver_location
;
311 unsigned location_frac
= var
->data
.location_frac
;
313 if (!var
->data
.compact
&& !indir_index
)
314 location
+= const_index
;
315 else if (var
->data
.compact
) {
316 location
+= const_index
/ 4;
317 location_frac
+= const_index
% 4;
320 switch (deref_mode
) {
321 case nir_var_shader_in
:
322 for (unsigned i
= 0; i
< num_components
; i
++) {
323 int idx
= (i
* dmul
) + location_frac
;
326 LLVMValueRef vertex_index_val
= lp_build_const_int32(gallivm
, vertex_index
);
327 LLVMValueRef attrib_index_val
= lp_build_const_int32(gallivm
, location
);
328 LLVMValueRef swizzle_index_val
= lp_build_const_int32(gallivm
, idx
);
329 LLVMValueRef result2
;
331 result
[i
] = bld
->gs_iface
->fetch_input(bld
->gs_iface
, &bld_base
->base
,
332 false, vertex_index_val
, 0, attrib_index_val
, swizzle_index_val
);
333 if (bit_size
== 64) {
334 LLVMValueRef swizzle_index_val
= lp_build_const_int32(gallivm
, idx
+ 1);
335 result2
= bld
->gs_iface
->fetch_input(bld
->gs_iface
, &bld_base
->base
,
336 false, vertex_index_val
, 0, attrib_index_val
, swizzle_index_val
);
337 result
[i
] = emit_fetch_64bit(bld_base
, result
[i
], result2
);
339 } else if (bld
->tes_iface
) {
340 LLVMValueRef vertex_index_val
= lp_build_const_int32(gallivm
, vertex_index
);
341 LLVMValueRef attrib_index_val
;
342 LLVMValueRef swizzle_index_val
= lp_build_const_int32(gallivm
, idx
);
343 LLVMValueRef result2
;
346 if (var
->data
.compact
) {
347 swizzle_index_val
= lp_build_add(&bld_base
->uint_bld
, indir_index
, lp_build_const_int_vec(gallivm
, bld_base
->uint_bld
.type
, idx
));
348 attrib_index_val
= lp_build_const_int32(gallivm
, location
);
350 attrib_index_val
= lp_build_add(&bld_base
->uint_bld
, indir_index
, lp_build_const_int_vec(gallivm
, bld_base
->uint_bld
.type
, var
->data
.driver_location
));
352 attrib_index_val
= lp_build_const_int32(gallivm
, location
);
354 if (var
->data
.patch
) {
355 result
[i
] = bld
->tes_iface
->fetch_patch_input(bld
->tes_iface
, &bld_base
->base
,
356 indir_index
? true : false, attrib_index_val
, swizzle_index_val
);
357 if (bit_size
== 64) {
358 LLVMValueRef swizzle_index_val
= lp_build_const_int32(gallivm
, idx
+ 1);
359 result2
= bld
->tes_iface
->fetch_patch_input(bld
->tes_iface
, &bld_base
->base
,
360 indir_index
? true : false, attrib_index_val
, swizzle_index_val
);
361 result
[i
] = emit_fetch_64bit(bld_base
, result
[i
], result2
);
365 result
[i
] = bld
->tes_iface
->fetch_vertex_input(bld
->tes_iface
, &bld_base
->base
,
366 indir_vertex_index
? true : false,
367 indir_vertex_index
? indir_vertex_index
: vertex_index_val
,
368 (indir_index
&& !var
->data
.compact
) ? true : false, attrib_index_val
,
369 (indir_index
&& var
->data
.compact
) ? true : false, swizzle_index_val
);
370 if (bit_size
== 64) {
371 LLVMValueRef swizzle_index_val
= lp_build_const_int32(gallivm
, idx
+ 1);
372 result2
= bld
->tes_iface
->fetch_vertex_input(bld
->tes_iface
, &bld_base
->base
,
373 indir_vertex_index
? true : false,
374 indir_vertex_index
? indir_vertex_index
: vertex_index_val
,
375 indir_index
? true : false, attrib_index_val
, false, swizzle_index_val
);
376 result
[i
] = emit_fetch_64bit(bld_base
, result
[i
], result2
);
379 } else if (bld
->tcs_iface
) {
380 LLVMValueRef vertex_index_val
= lp_build_const_int32(gallivm
, vertex_index
);
381 LLVMValueRef attrib_index_val
;
382 LLVMValueRef swizzle_index_val
= lp_build_const_int32(gallivm
, idx
);
385 if (var
->data
.compact
) {
386 swizzle_index_val
= lp_build_add(&bld_base
->uint_bld
, indir_index
, lp_build_const_int_vec(gallivm
, bld_base
->uint_bld
.type
, idx
));
387 attrib_index_val
= lp_build_const_int32(gallivm
, location
);
389 attrib_index_val
= lp_build_add(&bld_base
->uint_bld
, indir_index
, lp_build_const_int_vec(gallivm
, bld_base
->uint_bld
.type
, var
->data
.driver_location
));
391 attrib_index_val
= lp_build_const_int32(gallivm
, location
);
392 result
[i
] = bld
->tcs_iface
->emit_fetch_input(bld
->tcs_iface
, &bld_base
->base
,
393 indir_vertex_index
? true : false, indir_vertex_index
? indir_vertex_index
: vertex_index_val
,
394 (indir_index
&& !var
->data
.compact
) ? true : false, attrib_index_val
,
395 (indir_index
&& var
->data
.compact
) ? true : false, swizzle_index_val
);
396 if (bit_size
== 64) {
397 LLVMValueRef swizzle_index_val
= lp_build_const_int32(gallivm
, idx
+ 1);
398 LLVMValueRef result2
= bld
->tcs_iface
->emit_fetch_input(bld
->tcs_iface
, &bld_base
->base
,
399 indir_vertex_index
? true : false, indir_vertex_index
? indir_vertex_index
: vertex_index_val
,
400 indir_index
? true : false, attrib_index_val
,
401 false, swizzle_index_val
);
402 result
[i
] = emit_fetch_64bit(bld_base
, result
[i
], result2
);
406 LLVMValueRef attrib_index_val
= lp_build_add(&bld_base
->uint_bld
, indir_index
, lp_build_const_int_vec(gallivm
, bld_base
->uint_bld
.type
, var
->data
.driver_location
));
407 LLVMValueRef index_vec
= get_soa_array_offsets(&bld_base
->uint_bld
,
408 attrib_index_val
, 4, idx
,
410 LLVMValueRef index_vec2
= NULL
;
411 LLVMTypeRef fptr_type
;
412 LLVMValueRef inputs_array
;
413 fptr_type
= LLVMPointerType(LLVMFloatTypeInContext(gallivm
->context
), 0);
414 inputs_array
= LLVMBuildBitCast(gallivm
->builder
, bld
->inputs_array
, fptr_type
, "");
417 index_vec2
= get_soa_array_offsets(&bld_base
->uint_bld
,
418 indir_index
, 4, idx
+ 1, TRUE
);
420 /* Gather values from the input register array */
421 result
[i
] = build_gather(bld_base
, &bld_base
->base
, inputs_array
, index_vec
, NULL
, index_vec2
);
423 if (bld
->indirects
& nir_var_shader_in
) {
424 LLVMValueRef lindex
= lp_build_const_int32(gallivm
,
426 LLVMValueRef input_ptr
= lp_build_pointer_get(gallivm
->builder
,
427 bld
->inputs_array
, lindex
);
428 if (bit_size
== 64) {
429 LLVMValueRef lindex2
= lp_build_const_int32(gallivm
,
430 location
* 4 + (idx
+ 1));
431 LLVMValueRef input_ptr2
= lp_build_pointer_get(gallivm
->builder
,
432 bld
->inputs_array
, lindex2
);
433 result
[i
] = emit_fetch_64bit(bld_base
, input_ptr
, input_ptr2
);
435 result
[i
] = input_ptr
;
438 if (bit_size
== 64) {
440 tmp
[0] = bld
->inputs
[location
][idx
];
441 tmp
[1] = bld
->inputs
[location
][idx
+ 1];
442 result
[i
] = emit_fetch_64bit(bld_base
, tmp
[0], tmp
[1]);
444 result
[i
] = bld
->inputs
[location
][idx
];
451 case nir_var_shader_out
:
452 if (bld
->fs_iface
&& bld
->fs_iface
->fb_fetch
) {
453 bld
->fs_iface
->fb_fetch(bld
->fs_iface
, &bld_base
->base
, var
->data
.driver_location
, result
);
456 for (unsigned i
= 0; i
< num_components
; i
++) {
457 int idx
= (i
* dmul
) + location_frac
;
458 if (bld
->tcs_iface
) {
459 LLVMValueRef vertex_index_val
= lp_build_const_int32(gallivm
, vertex_index
);
460 LLVMValueRef attrib_index_val
;
461 LLVMValueRef swizzle_index_val
= lp_build_const_int32(gallivm
, idx
);
464 attrib_index_val
= lp_build_add(&bld_base
->uint_bld
, indir_index
, lp_build_const_int_vec(gallivm
, bld_base
->uint_bld
.type
, var
->data
.driver_location
));
466 attrib_index_val
= lp_build_const_int32(gallivm
, location
);
468 result
[i
] = bld
->tcs_iface
->emit_fetch_output(bld
->tcs_iface
, &bld_base
->base
,
469 indir_vertex_index
? true : false, indir_vertex_index
? indir_vertex_index
: vertex_index_val
,
470 (indir_index
&& !var
->data
.compact
) ? true : false, attrib_index_val
,
471 (indir_index
&& var
->data
.compact
) ? true : false, swizzle_index_val
, 0);
472 if (bit_size
== 64) {
473 LLVMValueRef swizzle_index_val
= lp_build_const_int32(gallivm
, idx
+ 1);
474 LLVMValueRef result2
= bld
->tcs_iface
->emit_fetch_output(bld
->tcs_iface
, &bld_base
->base
,
475 indir_vertex_index
? true : false, indir_vertex_index
? indir_vertex_index
: vertex_index_val
,
476 indir_index
? true : false, attrib_index_val
,
477 false, swizzle_index_val
, 0);
478 result
[i
] = emit_fetch_64bit(bld_base
, result
[i
], result2
);
488 static void emit_store_chan(struct lp_build_nir_context
*bld_base
,
489 nir_variable_mode deref_mode
,
491 unsigned location
, unsigned comp
,
495 struct lp_build_nir_soa_context
*bld
= (struct lp_build_nir_soa_context
*)bld_base
;
496 LLVMBuilderRef builder
= bld
->bld_base
.base
.gallivm
->builder
;
497 struct lp_build_context
*float_bld
= &bld_base
->base
;
499 if (bit_size
== 64) {
506 emit_store_64bit_chan(bld_base
, bld
->outputs
[location
][chan
],
507 bld
->outputs
[location
][chan
+ 1], dst
);
509 dst
= LLVMBuildBitCast(builder
, dst
, float_bld
->vec_type
, "");
510 lp_exec_mask_store(&bld
->exec_mask
, float_bld
, dst
,
511 bld
->outputs
[location
][chan
+ comp
]);
515 static void emit_store_tcs_chan(struct lp_build_nir_context
*bld_base
,
519 unsigned const_index
,
520 LLVMValueRef indir_vertex_index
,
521 LLVMValueRef indir_index
,
524 LLVMValueRef chan_val
)
526 struct gallivm_state
*gallivm
= bld_base
->base
.gallivm
;
527 struct lp_build_nir_soa_context
*bld
= (struct lp_build_nir_soa_context
*)bld_base
;
528 LLVMBuilderRef builder
= bld
->bld_base
.base
.gallivm
->builder
;
529 unsigned swizzle
= chan
;
530 if (bit_size
== 64) {
539 LLVMValueRef attrib_index_val
;
540 LLVMValueRef swizzle_index_val
= lp_build_const_int32(gallivm
, swizzle
);
544 swizzle_index_val
= lp_build_add(&bld_base
->uint_bld
, indir_index
, lp_build_const_int_vec(gallivm
, bld_base
->uint_bld
.type
, swizzle
));
545 attrib_index_val
= lp_build_const_int32(gallivm
, const_index
+ location
);
547 attrib_index_val
= lp_build_add(&bld_base
->uint_bld
, indir_index
, lp_build_const_int_vec(gallivm
, bld_base
->uint_bld
.type
, location
));
549 attrib_index_val
= lp_build_const_int32(gallivm
, const_index
+ location
);
550 if (bit_size
== 64) {
551 LLVMValueRef split_vals
[2];
552 LLVMValueRef swizzle_index_val2
= lp_build_const_int32(gallivm
, swizzle
+ 1);
553 emit_store_64bit_split(bld_base
, chan_val
, split_vals
);
554 bld
->tcs_iface
->emit_store_output(bld
->tcs_iface
, &bld_base
->base
, 0,
555 indir_vertex_index
? true : false,
557 indir_index
? true : false,
559 false, swizzle_index_val
,
560 split_vals
[0], mask_vec(bld_base
));
561 bld
->tcs_iface
->emit_store_output(bld
->tcs_iface
, &bld_base
->base
, 0,
562 indir_vertex_index
? true : false,
564 indir_index
? true : false,
566 false, swizzle_index_val2
,
567 split_vals
[1], mask_vec(bld_base
));
569 chan_val
= LLVMBuildBitCast(builder
, chan_val
, bld_base
->base
.vec_type
, "");
570 bld
->tcs_iface
->emit_store_output(bld
->tcs_iface
, &bld_base
->base
, 0,
571 indir_vertex_index
? true : false,
573 indir_index
&& !is_compact
? true : false,
575 indir_index
&& is_compact
? true : false,
577 chan_val
, mask_vec(bld_base
));
581 static void emit_store_var(struct lp_build_nir_context
*bld_base
,
582 nir_variable_mode deref_mode
,
583 unsigned num_components
,
587 LLVMValueRef indir_vertex_index
,
588 unsigned const_index
,
589 LLVMValueRef indir_index
,
592 struct lp_build_nir_soa_context
*bld
= (struct lp_build_nir_soa_context
*)bld_base
;
593 LLVMBuilderRef builder
= bld
->bld_base
.base
.gallivm
->builder
;
594 switch (deref_mode
) {
595 case nir_var_shader_out
: {
596 unsigned location
= var
->data
.driver_location
;
597 unsigned comp
= var
->data
.location_frac
;
598 if (bld_base
->shader
->info
.stage
== MESA_SHADER_FRAGMENT
) {
599 if (var
->data
.location
== FRAG_RESULT_STENCIL
)
601 else if (var
->data
.location
== FRAG_RESULT_DEPTH
)
605 if (var
->data
.compact
) {
606 location
+= const_index
/ 4;
607 comp
+= const_index
% 4;
611 for (unsigned chan
= 0; chan
< num_components
; chan
++) {
612 if (writemask
& (1u << chan
)) {
613 LLVMValueRef chan_val
= (num_components
== 1) ? dst
: LLVMBuildExtractValue(builder
, dst
, chan
, "");
614 if (bld
->tcs_iface
) {
615 emit_store_tcs_chan(bld_base
, var
->data
.compact
, bit_size
, location
, const_index
, indir_vertex_index
, indir_index
, comp
, chan
, chan_val
);
617 emit_store_chan(bld_base
, deref_mode
, bit_size
, location
+ const_index
, comp
, chan
, chan_val
);
627 static LLVMValueRef
emit_load_reg(struct lp_build_nir_context
*bld_base
,
628 struct lp_build_context
*reg_bld
,
629 const nir_reg_src
*reg
,
630 LLVMValueRef indir_src
,
631 LLVMValueRef reg_storage
)
633 struct gallivm_state
*gallivm
= bld_base
->base
.gallivm
;
634 LLVMBuilderRef builder
= gallivm
->builder
;
635 int nc
= reg
->reg
->num_components
;
636 LLVMValueRef vals
[NIR_MAX_VEC_COMPONENTS
] = { NULL
};
637 struct lp_build_context
*uint_bld
= &bld_base
->uint_bld
;
638 if (reg
->reg
->num_array_elems
) {
639 LLVMValueRef indirect_val
= lp_build_const_int_vec(gallivm
, uint_bld
->type
, reg
->base_offset
);
641 LLVMValueRef max_index
= lp_build_const_int_vec(gallivm
, uint_bld
->type
, reg
->reg
->num_array_elems
- 1);
642 indirect_val
= LLVMBuildAdd(builder
, indirect_val
, indir_src
, "");
643 indirect_val
= lp_build_min(uint_bld
, indirect_val
, max_index
);
645 reg_storage
= LLVMBuildBitCast(builder
, reg_storage
, LLVMPointerType(reg_bld
->elem_type
, 0), "");
646 for (unsigned i
= 0; i
< nc
; i
++) {
647 LLVMValueRef indirect_offset
= get_soa_array_offsets(uint_bld
, indirect_val
, nc
, i
, TRUE
);
648 vals
[i
] = build_gather(bld_base
, reg_bld
, reg_storage
, indirect_offset
, NULL
, NULL
);
651 for (unsigned i
= 0; i
< nc
; i
++) {
652 LLVMValueRef this_storage
= nc
== 1 ? reg_storage
: lp_build_array_get_ptr(gallivm
, reg_storage
,
653 lp_build_const_int32(gallivm
, i
));
654 vals
[i
] = LLVMBuildLoad(builder
, this_storage
, "");
657 return nc
== 1 ? vals
[0] : lp_nir_array_build_gather_values(builder
, vals
, nc
);
660 static void emit_store_reg(struct lp_build_nir_context
*bld_base
,
661 struct lp_build_context
*reg_bld
,
662 const nir_reg_dest
*reg
,
664 LLVMValueRef indir_src
,
665 LLVMValueRef reg_storage
,
666 LLVMValueRef dst
[NIR_MAX_VEC_COMPONENTS
])
668 struct lp_build_nir_soa_context
*bld
= (struct lp_build_nir_soa_context
*)bld_base
;
669 struct gallivm_state
*gallivm
= bld_base
->base
.gallivm
;
670 LLVMBuilderRef builder
= gallivm
->builder
;
671 struct lp_build_context
*uint_bld
= &bld_base
->uint_bld
;
672 int nc
= reg
->reg
->num_components
;
673 if (reg
->reg
->num_array_elems
> 0) {
674 LLVMValueRef indirect_val
= lp_build_const_int_vec(gallivm
, uint_bld
->type
, reg
->base_offset
);
676 LLVMValueRef max_index
= lp_build_const_int_vec(gallivm
, uint_bld
->type
, reg
->reg
->num_array_elems
- 1);
677 indirect_val
= LLVMBuildAdd(builder
, indirect_val
, indir_src
, "");
678 indirect_val
= lp_build_min(uint_bld
, indirect_val
, max_index
);
680 reg_storage
= LLVMBuildBitCast(builder
, reg_storage
, LLVMPointerType(reg_bld
->elem_type
, 0), "");
681 for (unsigned i
= 0; i
< nc
; i
++) {
682 if (!(writemask
& (1 << i
)))
684 LLVMValueRef indirect_offset
= get_soa_array_offsets(uint_bld
, indirect_val
, nc
, i
, TRUE
);
685 dst
[i
] = LLVMBuildBitCast(builder
, dst
[i
], reg_bld
->vec_type
, "");
686 emit_mask_scatter(bld
, reg_storage
, indirect_offset
, dst
[i
], &bld
->exec_mask
);
691 for (unsigned i
= 0; i
< nc
; i
++) {
692 LLVMValueRef this_storage
= nc
== 1 ? reg_storage
: lp_build_array_get_ptr(gallivm
, reg_storage
,
693 lp_build_const_int32(gallivm
, i
));
694 dst
[i
] = LLVMBuildBitCast(builder
, dst
[i
], reg_bld
->vec_type
, "");
695 lp_exec_mask_store(&bld
->exec_mask
, reg_bld
, dst
[i
], this_storage
);
699 static void emit_load_kernel_arg(struct lp_build_nir_context
*bld_base
,
702 unsigned offset_bit_size
,
703 bool offset_is_uniform
,
705 LLVMValueRef result
[NIR_MAX_VEC_COMPONENTS
])
707 struct lp_build_nir_soa_context
*bld
= (struct lp_build_nir_soa_context
*)bld_base
;
708 struct gallivm_state
*gallivm
= bld_base
->base
.gallivm
;
709 LLVMBuilderRef builder
= gallivm
->builder
;
710 struct lp_build_context
*bld_broad
= get_int_bld(bld_base
, true, bit_size
);
711 LLVMValueRef kernel_args_ptr
= bld
->kernel_args_ptr
;
712 unsigned size_shift
= 0;
713 struct lp_build_context
*bld_offset
= get_int_bld(bld_base
, true, offset_bit_size
);
716 else if (bit_size
== 32)
718 else if (bit_size
== 64)
721 offset
= lp_build_shr(bld_offset
, offset
, lp_build_const_int_vec(gallivm
, bld_offset
->type
, size_shift
));
723 LLVMTypeRef ptr_type
= LLVMPointerType(bld_broad
->elem_type
, 0);
724 kernel_args_ptr
= LLVMBuildBitCast(builder
, kernel_args_ptr
, ptr_type
, "");
726 if (offset_is_uniform
) {
727 offset
= LLVMBuildExtractElement(builder
, offset
, lp_build_const_int32(gallivm
, 0), "");
729 for (unsigned c
= 0; c
< nc
; c
++) {
730 LLVMValueRef this_offset
= LLVMBuildAdd(builder
, offset
, offset_bit_size
== 64 ? lp_build_const_int64(gallivm
, c
) : lp_build_const_int32(gallivm
, c
), "");
732 LLVMValueRef scalar
= lp_build_pointer_get(builder
, kernel_args_ptr
, this_offset
);
733 result
[c
] = lp_build_broadcast_scalar(bld_broad
, scalar
);
738 static LLVMValueRef
global_addr_to_ptr(struct gallivm_state
*gallivm
, LLVMValueRef addr_ptr
, unsigned bit_size
)
740 LLVMBuilderRef builder
= gallivm
->builder
;
743 addr_ptr
= LLVMBuildIntToPtr(builder
, addr_ptr
, LLVMPointerType(LLVMInt8TypeInContext(gallivm
->context
), 0), "");
746 addr_ptr
= LLVMBuildIntToPtr(builder
, addr_ptr
, LLVMPointerType(LLVMInt16TypeInContext(gallivm
->context
), 0), "");
750 addr_ptr
= LLVMBuildIntToPtr(builder
, addr_ptr
, LLVMPointerType(LLVMInt32TypeInContext(gallivm
->context
), 0), "");
753 addr_ptr
= LLVMBuildIntToPtr(builder
, addr_ptr
, LLVMPointerType(LLVMInt64TypeInContext(gallivm
->context
), 0), "");
759 static void emit_load_global(struct lp_build_nir_context
*bld_base
,
762 unsigned addr_bit_size
,
764 LLVMValueRef outval
[NIR_MAX_VEC_COMPONENTS
])
766 struct gallivm_state
*gallivm
= bld_base
->base
.gallivm
;
767 LLVMBuilderRef builder
= gallivm
->builder
;
768 struct lp_build_context
*uint_bld
= &bld_base
->uint_bld
;
769 struct lp_build_context
*res_bld
;
771 res_bld
= get_int_bld(bld_base
, true, bit_size
);
773 for (unsigned c
= 0; c
< nc
; c
++) {
774 LLVMValueRef result
= lp_build_alloca(gallivm
, res_bld
->vec_type
, "");
776 struct lp_build_loop_state loop_state
;
777 lp_build_loop_begin(&loop_state
, gallivm
, lp_build_const_int32(gallivm
, 0));
779 LLVMValueRef addr_ptr
= LLVMBuildExtractElement(gallivm
->builder
, addr
,
780 loop_state
.counter
, "");
781 addr_ptr
= global_addr_to_ptr(gallivm
, addr_ptr
, bit_size
);
783 LLVMValueRef value_ptr
= lp_build_pointer_get(builder
, addr_ptr
, lp_build_const_int32(gallivm
, c
));
785 LLVMValueRef temp_res
;
786 temp_res
= LLVMBuildLoad(builder
, result
, "");
787 temp_res
= LLVMBuildInsertElement(builder
, temp_res
, value_ptr
, loop_state
.counter
, "");
788 LLVMBuildStore(builder
, temp_res
, result
);
789 lp_build_loop_end_cond(&loop_state
, lp_build_const_int32(gallivm
, uint_bld
->type
.length
),
791 outval
[c
] = LLVMBuildLoad(builder
, result
, "");
795 static void emit_store_global(struct lp_build_nir_context
*bld_base
,
797 unsigned nc
, unsigned bit_size
,
798 unsigned addr_bit_size
,
802 struct gallivm_state
*gallivm
= bld_base
->base
.gallivm
;
803 LLVMBuilderRef builder
= gallivm
->builder
;
804 struct lp_build_context
*uint_bld
= &bld_base
->uint_bld
;
806 for (unsigned c
= 0; c
< nc
; c
++) {
807 if (!(writemask
& (1u << c
)))
809 LLVMValueRef val
= (nc
== 1) ? dst
: LLVMBuildExtractValue(builder
, dst
, c
, "");
811 LLVMValueRef exec_mask
= mask_vec(bld_base
);
812 struct lp_build_loop_state loop_state
;
813 lp_build_loop_begin(&loop_state
, gallivm
, lp_build_const_int32(gallivm
, 0));
814 LLVMValueRef value_ptr
= LLVMBuildExtractElement(gallivm
->builder
, val
,
815 loop_state
.counter
, "");
817 LLVMValueRef addr_ptr
= LLVMBuildExtractElement(gallivm
->builder
, addr
,
818 loop_state
.counter
, "");
819 addr_ptr
= global_addr_to_ptr(gallivm
, addr_ptr
, bit_size
);
822 value_ptr
= LLVMBuildBitCast(builder
, value_ptr
, LLVMInt32TypeInContext(gallivm
->context
), "");
825 value_ptr
= LLVMBuildBitCast(builder
, value_ptr
, LLVMInt64TypeInContext(gallivm
->context
), "");
830 struct lp_build_if_state ifthen
;
832 LLVMValueRef cond
= LLVMBuildICmp(gallivm
->builder
, LLVMIntNE
, exec_mask
, uint_bld
->zero
, "");
833 cond
= LLVMBuildExtractElement(gallivm
->builder
, cond
, loop_state
.counter
, "");
834 lp_build_if(&ifthen
, gallivm
, cond
);
835 lp_build_pointer_set(builder
, addr_ptr
, lp_build_const_int32(gallivm
, c
), value_ptr
);
836 lp_build_endif(&ifthen
);
837 lp_build_loop_end_cond(&loop_state
, lp_build_const_int32(gallivm
, uint_bld
->type
.length
),
842 static void emit_atomic_global(struct lp_build_nir_context
*bld_base
,
843 nir_intrinsic_op nir_op
,
844 unsigned addr_bit_size
,
846 LLVMValueRef val
, LLVMValueRef val2
,
847 LLVMValueRef
*result
)
849 struct gallivm_state
*gallivm
= bld_base
->base
.gallivm
;
850 LLVMBuilderRef builder
= gallivm
->builder
;
851 struct lp_build_context
*uint_bld
= &bld_base
->uint_bld
;
853 LLVMValueRef atom_res
= lp_build_alloca(gallivm
,
854 uint_bld
->vec_type
, "");
855 LLVMValueRef exec_mask
= mask_vec(bld_base
);
856 struct lp_build_loop_state loop_state
;
857 lp_build_loop_begin(&loop_state
, gallivm
, lp_build_const_int32(gallivm
, 0));
859 LLVMValueRef value_ptr
= LLVMBuildExtractElement(gallivm
->builder
, val
,
860 loop_state
.counter
, "");
862 LLVMValueRef addr_ptr
= LLVMBuildExtractElement(gallivm
->builder
, addr
,
863 loop_state
.counter
, "");
864 addr_ptr
= global_addr_to_ptr(gallivm
, addr_ptr
, 32);
865 struct lp_build_if_state ifthen
;
866 LLVMValueRef cond
, temp_res
;
868 cond
= LLVMBuildICmp(gallivm
->builder
, LLVMIntNE
, exec_mask
, uint_bld
->zero
, "");
869 cond
= LLVMBuildExtractElement(gallivm
->builder
, cond
, loop_state
.counter
, "");
870 lp_build_if(&ifthen
, gallivm
, cond
);
872 if (nir_op
== nir_intrinsic_global_atomic_comp_swap
) {
873 LLVMValueRef cas_src_ptr
= LLVMBuildExtractElement(gallivm
->builder
, val2
,
874 loop_state
.counter
, "");
875 cas_src_ptr
= LLVMBuildBitCast(gallivm
->builder
, cas_src_ptr
, uint_bld
->elem_type
, "");
876 scalar
= LLVMBuildAtomicCmpXchg(builder
, addr_ptr
, value_ptr
,
878 LLVMAtomicOrderingSequentiallyConsistent
,
879 LLVMAtomicOrderingSequentiallyConsistent
,
881 scalar
= LLVMBuildExtractValue(gallivm
->builder
, scalar
, 0, "");
883 LLVMAtomicRMWBinOp op
;
885 case nir_intrinsic_global_atomic_add
:
886 op
= LLVMAtomicRMWBinOpAdd
;
888 case nir_intrinsic_global_atomic_exchange
:
889 op
= LLVMAtomicRMWBinOpXchg
;
891 case nir_intrinsic_global_atomic_and
:
892 op
= LLVMAtomicRMWBinOpAnd
;
894 case nir_intrinsic_global_atomic_or
:
895 op
= LLVMAtomicRMWBinOpOr
;
897 case nir_intrinsic_global_atomic_xor
:
898 op
= LLVMAtomicRMWBinOpXor
;
900 case nir_intrinsic_global_atomic_umin
:
901 op
= LLVMAtomicRMWBinOpUMin
;
903 case nir_intrinsic_global_atomic_umax
:
904 op
= LLVMAtomicRMWBinOpUMax
;
906 case nir_intrinsic_global_atomic_imin
:
907 op
= LLVMAtomicRMWBinOpMin
;
909 case nir_intrinsic_global_atomic_imax
:
910 op
= LLVMAtomicRMWBinOpMax
;
913 unreachable("unknown atomic op");
916 scalar
= LLVMBuildAtomicRMW(builder
, op
,
918 LLVMAtomicOrderingSequentiallyConsistent
,
921 temp_res
= LLVMBuildLoad(builder
, atom_res
, "");
922 temp_res
= LLVMBuildInsertElement(builder
, temp_res
, scalar
, loop_state
.counter
, "");
923 LLVMBuildStore(builder
, temp_res
, atom_res
);
924 lp_build_else(&ifthen
);
925 temp_res
= LLVMBuildLoad(builder
, atom_res
, "");
926 temp_res
= LLVMBuildInsertElement(builder
, temp_res
, lp_build_const_int32(gallivm
, 0), loop_state
.counter
, "");
927 LLVMBuildStore(builder
, temp_res
, atom_res
);
928 lp_build_endif(&ifthen
);
929 lp_build_loop_end_cond(&loop_state
, lp_build_const_int32(gallivm
, uint_bld
->type
.length
),
931 *result
= LLVMBuildLoad(builder
, atom_res
, "");
934 static void emit_load_ubo(struct lp_build_nir_context
*bld_base
,
937 bool offset_is_uniform
,
940 LLVMValueRef result
[NIR_MAX_VEC_COMPONENTS
])
942 struct lp_build_nir_soa_context
*bld
= (struct lp_build_nir_soa_context
*)bld_base
;
943 struct gallivm_state
*gallivm
= bld_base
->base
.gallivm
;
944 LLVMBuilderRef builder
= gallivm
->builder
;
945 struct lp_build_context
*uint_bld
= &bld_base
->uint_bld
;
946 struct lp_build_context
*bld_broad
= bit_size
== 64 ? &bld_base
->dbl_bld
: &bld_base
->base
;
947 LLVMValueRef consts_ptr
= lp_build_array_get(gallivm
, bld
->consts_ptr
, index
);
948 unsigned size_shift
= 0;
951 else if (bit_size
== 64)
954 offset
= lp_build_shr(uint_bld
, offset
, lp_build_const_int_vec(gallivm
, uint_bld
->type
, size_shift
));
955 if (bit_size
== 64) {
956 LLVMTypeRef dptr_type
= LLVMPointerType(bld_base
->dbl_bld
.elem_type
, 0);
957 consts_ptr
= LLVMBuildBitCast(builder
, consts_ptr
, dptr_type
, "");
960 if (offset_is_uniform
) {
961 offset
= LLVMBuildExtractElement(builder
, offset
, lp_build_const_int32(gallivm
, 0), "");
963 for (unsigned c
= 0; c
< nc
; c
++) {
964 LLVMValueRef this_offset
= LLVMBuildAdd(builder
, offset
, lp_build_const_int32(gallivm
, c
), "");
966 LLVMValueRef scalar
= lp_build_pointer_get(builder
, consts_ptr
, this_offset
);
967 result
[c
] = lp_build_broadcast_scalar(bld_broad
, scalar
);
970 LLVMValueRef overflow_mask
;
971 LLVMValueRef num_consts
= lp_build_array_get(gallivm
, bld
->const_sizes_ptr
, index
);
973 num_consts
= lp_build_broadcast_scalar(uint_bld
, num_consts
);
974 for (unsigned c
= 0; c
< nc
; c
++) {
975 LLVMValueRef this_offset
= lp_build_add(uint_bld
, offset
, lp_build_const_int_vec(gallivm
, uint_bld
->type
, c
));
976 overflow_mask
= lp_build_compare(gallivm
, uint_bld
->type
, PIPE_FUNC_GEQUAL
,
977 this_offset
, num_consts
);
978 result
[c
] = build_gather(bld_base
, bld_broad
, consts_ptr
, this_offset
, overflow_mask
, NULL
);
984 static void emit_load_mem(struct lp_build_nir_context
*bld_base
,
989 LLVMValueRef outval
[NIR_MAX_VEC_COMPONENTS
])
991 struct gallivm_state
*gallivm
= bld_base
->base
.gallivm
;
992 struct lp_build_nir_soa_context
*bld
= (struct lp_build_nir_soa_context
*)bld_base
;
993 LLVMBuilderRef builder
= bld
->bld_base
.base
.gallivm
->builder
;
994 LLVMValueRef ssbo_ptr
= NULL
;
995 struct lp_build_context
*uint_bld
= &bld_base
->uint_bld
;
996 struct lp_build_context
*uint64_bld
= &bld_base
->uint64_bld
;
997 LLVMValueRef ssbo_limit
= NULL
;
1000 LLVMValueRef ssbo_size_ptr
= lp_build_array_get(gallivm
, bld
->ssbo_sizes_ptr
, LLVMBuildExtractElement(builder
, index
, lp_build_const_int32(gallivm
, 0), ""));
1001 ssbo_limit
= LLVMBuildAShr(gallivm
->builder
, ssbo_size_ptr
, lp_build_const_int32(gallivm
, bit_size
== 64 ? 3 : 2), "");
1002 ssbo_limit
= lp_build_broadcast_scalar(uint_bld
, ssbo_limit
);
1004 ssbo_ptr
= lp_build_array_get(gallivm
, bld
->ssbo_ptr
, LLVMBuildExtractElement(builder
, index
, lp_build_const_int32(gallivm
, 0), ""));
1006 ssbo_ptr
= bld
->shared_ptr
;
1008 offset
= LLVMBuildAShr(gallivm
->builder
, offset
, lp_build_const_int_vec(gallivm
, uint_bld
->type
, bit_size
== 64 ? 3 : 2), "");
1009 for (unsigned c
= 0; c
< nc
; c
++) {
1010 LLVMValueRef loop_index
= lp_build_add(uint_bld
, offset
, lp_build_const_int_vec(gallivm
, uint_bld
->type
, c
));
1011 LLVMValueRef exec_mask
= mask_vec(bld_base
);
1014 LLVMValueRef ssbo_oob_cmp
= lp_build_cmp(uint_bld
, PIPE_FUNC_LESS
, loop_index
, ssbo_limit
);
1015 exec_mask
= LLVMBuildAnd(builder
, exec_mask
, ssbo_oob_cmp
, "");
1018 LLVMValueRef result
= lp_build_alloca(gallivm
, bit_size
== 64 ? uint64_bld
->vec_type
: uint_bld
->vec_type
, "");
1019 struct lp_build_loop_state loop_state
;
1020 lp_build_loop_begin(&loop_state
, gallivm
, lp_build_const_int32(gallivm
, 0));
1022 struct lp_build_if_state ifthen
;
1023 LLVMValueRef cond
, temp_res
;
1025 loop_index
= LLVMBuildExtractElement(gallivm
->builder
, loop_index
,
1026 loop_state
.counter
, "");
1028 cond
= LLVMBuildICmp(gallivm
->builder
, LLVMIntNE
, exec_mask
, uint_bld
->zero
, "");
1029 cond
= LLVMBuildExtractElement(gallivm
->builder
, cond
, loop_state
.counter
, "");
1031 lp_build_if(&ifthen
, gallivm
, cond
);
1032 LLVMValueRef scalar
;
1033 if (bit_size
== 64) {
1034 LLVMValueRef ssbo_ptr2
= LLVMBuildBitCast(builder
, ssbo_ptr
, LLVMPointerType(uint64_bld
->elem_type
, 0), "");
1035 scalar
= lp_build_pointer_get(builder
, ssbo_ptr2
, loop_index
);
1037 scalar
= lp_build_pointer_get(builder
, ssbo_ptr
, loop_index
);
1039 temp_res
= LLVMBuildLoad(builder
, result
, "");
1040 temp_res
= LLVMBuildInsertElement(builder
, temp_res
, scalar
, loop_state
.counter
, "");
1041 LLVMBuildStore(builder
, temp_res
, result
);
1042 lp_build_else(&ifthen
);
1043 temp_res
= LLVMBuildLoad(builder
, result
, "");
1046 zero
= LLVMConstInt(LLVMInt64TypeInContext(gallivm
->context
), 0, 0);
1048 zero
= lp_build_const_int32(gallivm
, 0);
1049 temp_res
= LLVMBuildInsertElement(builder
, temp_res
, zero
, loop_state
.counter
, "");
1050 LLVMBuildStore(builder
, temp_res
, result
);
1051 lp_build_endif(&ifthen
);
1052 lp_build_loop_end_cond(&loop_state
, lp_build_const_int32(gallivm
, uint_bld
->type
.length
),
1054 outval
[c
] = LLVMBuildLoad(gallivm
->builder
, result
, "");
1058 static void emit_store_mem(struct lp_build_nir_context
*bld_base
,
1063 LLVMValueRef offset
,
1066 struct gallivm_state
*gallivm
= bld_base
->base
.gallivm
;
1067 struct lp_build_nir_soa_context
*bld
= (struct lp_build_nir_soa_context
*)bld_base
;
1068 LLVMBuilderRef builder
= bld
->bld_base
.base
.gallivm
->builder
;
1069 LLVMValueRef ssbo_ptr
;
1070 struct lp_build_context
*uint_bld
= &bld_base
->uint_bld
;
1071 LLVMValueRef ssbo_limit
= NULL
;
1074 LLVMValueRef ssbo_size_ptr
= lp_build_array_get(gallivm
, bld
->ssbo_sizes_ptr
, LLVMBuildExtractElement(builder
, index
, lp_build_const_int32(gallivm
, 0), ""));
1075 ssbo_limit
= LLVMBuildAShr(gallivm
->builder
, ssbo_size_ptr
, lp_build_const_int32(gallivm
, bit_size
== 64 ? 3 : 2), "");
1076 ssbo_limit
= lp_build_broadcast_scalar(uint_bld
, ssbo_limit
);
1077 ssbo_ptr
= lp_build_array_get(gallivm
, bld
->ssbo_ptr
, LLVMBuildExtractElement(builder
, index
, lp_build_const_int32(gallivm
, 0), ""));
1079 ssbo_ptr
= bld
->shared_ptr
;
1081 offset
= lp_build_shr_imm(uint_bld
, offset
, bit_size
== 64 ? 3 : 2);
1082 for (unsigned c
= 0; c
< nc
; c
++) {
1083 if (!(writemask
& (1u << c
)))
1085 LLVMValueRef loop_index
= lp_build_add(uint_bld
, offset
, lp_build_const_int_vec(gallivm
, uint_bld
->type
, c
));
1086 LLVMValueRef val
= (nc
== 1) ? dst
: LLVMBuildExtractValue(builder
, dst
, c
, "");
1088 LLVMValueRef exec_mask
= mask_vec(bld_base
);
1090 LLVMValueRef ssbo_oob_cmp
= lp_build_cmp(uint_bld
, PIPE_FUNC_LESS
, loop_index
, ssbo_limit
);
1091 exec_mask
= LLVMBuildAnd(builder
, exec_mask
, ssbo_oob_cmp
, "");
1094 struct lp_build_loop_state loop_state
;
1095 lp_build_loop_begin(&loop_state
, gallivm
, lp_build_const_int32(gallivm
, 0));
1096 LLVMValueRef value_ptr
= LLVMBuildExtractElement(gallivm
->builder
, val
,
1097 loop_state
.counter
, "");
1099 value_ptr
= LLVMBuildBitCast(gallivm
->builder
, value_ptr
, bld_base
->uint64_bld
.elem_type
, "");
1101 value_ptr
= LLVMBuildBitCast(gallivm
->builder
, value_ptr
, uint_bld
->elem_type
, "");
1102 struct lp_build_if_state ifthen
;
1105 loop_index
= LLVMBuildExtractElement(gallivm
->builder
, loop_index
,
1106 loop_state
.counter
, "");
1107 cond
= LLVMBuildICmp(gallivm
->builder
, LLVMIntNE
, exec_mask
, uint_bld
->zero
, "");
1108 cond
= LLVMBuildExtractElement(gallivm
->builder
, cond
, loop_state
.counter
, "");
1109 lp_build_if(&ifthen
, gallivm
, cond
);
1110 if (bit_size
== 64) {
1111 LLVMValueRef ssbo_ptr2
= LLVMBuildBitCast(builder
, ssbo_ptr
, LLVMPointerType(bld_base
->uint64_bld
.elem_type
, 0), "");
1112 lp_build_pointer_set(builder
, ssbo_ptr2
, loop_index
, value_ptr
);
1114 lp_build_pointer_set(builder
, ssbo_ptr
, loop_index
, value_ptr
);
1115 lp_build_endif(&ifthen
);
1116 lp_build_loop_end_cond(&loop_state
, lp_build_const_int32(gallivm
, uint_bld
->type
.length
),
1121 static void emit_atomic_mem(struct lp_build_nir_context
*bld_base
,
1122 nir_intrinsic_op nir_op
,
1123 LLVMValueRef index
, LLVMValueRef offset
,
1124 LLVMValueRef val
, LLVMValueRef val2
,
1125 LLVMValueRef
*result
)
1127 struct gallivm_state
*gallivm
= bld_base
->base
.gallivm
;
1128 struct lp_build_nir_soa_context
*bld
= (struct lp_build_nir_soa_context
*)bld_base
;
1129 LLVMBuilderRef builder
= bld
->bld_base
.base
.gallivm
->builder
;
1130 LLVMValueRef ssbo_ptr
;
1131 struct lp_build_context
*uint_bld
= &bld_base
->uint_bld
;
1132 LLVMValueRef ssbo_limit
= NULL
;
1135 LLVMValueRef ssbo_size_ptr
= lp_build_array_get(gallivm
, bld
->ssbo_sizes_ptr
, LLVMBuildExtractElement(builder
, index
, lp_build_const_int32(gallivm
, 0), ""));
1136 ssbo_limit
= LLVMBuildAShr(gallivm
->builder
, ssbo_size_ptr
, lp_build_const_int32(gallivm
, 2), "");
1137 ssbo_limit
= lp_build_broadcast_scalar(uint_bld
, ssbo_limit
);
1138 ssbo_ptr
= lp_build_array_get(gallivm
, bld
->ssbo_ptr
, LLVMBuildExtractElement(builder
, index
, lp_build_const_int32(gallivm
, 0), ""));
1140 ssbo_ptr
= bld
->shared_ptr
;
1142 offset
= lp_build_shr_imm(uint_bld
, offset
, 2);
1143 LLVMValueRef atom_res
= lp_build_alloca(gallivm
,
1144 uint_bld
->vec_type
, "");
1146 LLVMValueRef exec_mask
= mask_vec(bld_base
);
1148 LLVMValueRef ssbo_oob_cmp
= lp_build_cmp(uint_bld
, PIPE_FUNC_LESS
, offset
, ssbo_limit
);
1149 exec_mask
= LLVMBuildAnd(builder
, exec_mask
, ssbo_oob_cmp
, "");
1152 struct lp_build_loop_state loop_state
;
1153 lp_build_loop_begin(&loop_state
, gallivm
, lp_build_const_int32(gallivm
, 0));
1155 LLVMValueRef value_ptr
= LLVMBuildExtractElement(gallivm
->builder
, val
,
1156 loop_state
.counter
, "");
1157 value_ptr
= LLVMBuildBitCast(gallivm
->builder
, value_ptr
, uint_bld
->elem_type
, "");
1159 offset
= LLVMBuildExtractElement(gallivm
->builder
, offset
,
1160 loop_state
.counter
, "");
1162 LLVMValueRef scalar_ptr
= LLVMBuildGEP(builder
, ssbo_ptr
,
1165 struct lp_build_if_state ifthen
;
1166 LLVMValueRef cond
, temp_res
;
1167 LLVMValueRef scalar
;
1168 cond
= LLVMBuildICmp(gallivm
->builder
, LLVMIntNE
, exec_mask
, uint_bld
->zero
, "");
1169 cond
= LLVMBuildExtractElement(gallivm
->builder
, cond
, loop_state
.counter
, "");
1170 lp_build_if(&ifthen
, gallivm
, cond
);
1172 if (nir_op
== nir_intrinsic_ssbo_atomic_comp_swap
|| nir_op
== nir_intrinsic_shared_atomic_comp_swap
) {
1173 LLVMValueRef cas_src_ptr
= LLVMBuildExtractElement(gallivm
->builder
, val2
,
1174 loop_state
.counter
, "");
1175 cas_src_ptr
= LLVMBuildBitCast(gallivm
->builder
, cas_src_ptr
, uint_bld
->elem_type
, "");
1176 scalar
= LLVMBuildAtomicCmpXchg(builder
, scalar_ptr
, value_ptr
,
1178 LLVMAtomicOrderingSequentiallyConsistent
,
1179 LLVMAtomicOrderingSequentiallyConsistent
,
1181 scalar
= LLVMBuildExtractValue(gallivm
->builder
, scalar
, 0, "");
1183 LLVMAtomicRMWBinOp op
;
1186 case nir_intrinsic_shared_atomic_add
:
1187 case nir_intrinsic_ssbo_atomic_add
:
1188 op
= LLVMAtomicRMWBinOpAdd
;
1190 case nir_intrinsic_shared_atomic_exchange
:
1191 case nir_intrinsic_ssbo_atomic_exchange
:
1192 op
= LLVMAtomicRMWBinOpXchg
;
1194 case nir_intrinsic_shared_atomic_and
:
1195 case nir_intrinsic_ssbo_atomic_and
:
1196 op
= LLVMAtomicRMWBinOpAnd
;
1198 case nir_intrinsic_shared_atomic_or
:
1199 case nir_intrinsic_ssbo_atomic_or
:
1200 op
= LLVMAtomicRMWBinOpOr
;
1202 case nir_intrinsic_shared_atomic_xor
:
1203 case nir_intrinsic_ssbo_atomic_xor
:
1204 op
= LLVMAtomicRMWBinOpXor
;
1206 case nir_intrinsic_shared_atomic_umin
:
1207 case nir_intrinsic_ssbo_atomic_umin
:
1208 op
= LLVMAtomicRMWBinOpUMin
;
1210 case nir_intrinsic_shared_atomic_umax
:
1211 case nir_intrinsic_ssbo_atomic_umax
:
1212 op
= LLVMAtomicRMWBinOpUMax
;
1214 case nir_intrinsic_ssbo_atomic_imin
:
1215 case nir_intrinsic_shared_atomic_imin
:
1216 op
= LLVMAtomicRMWBinOpMin
;
1218 case nir_intrinsic_ssbo_atomic_imax
:
1219 case nir_intrinsic_shared_atomic_imax
:
1220 op
= LLVMAtomicRMWBinOpMax
;
1223 unreachable("unknown atomic op");
1225 scalar
= LLVMBuildAtomicRMW(builder
, op
,
1226 scalar_ptr
, value_ptr
,
1227 LLVMAtomicOrderingSequentiallyConsistent
,
1230 temp_res
= LLVMBuildLoad(builder
, atom_res
, "");
1231 temp_res
= LLVMBuildInsertElement(builder
, temp_res
, scalar
, loop_state
.counter
, "");
1232 LLVMBuildStore(builder
, temp_res
, atom_res
);
1233 lp_build_else(&ifthen
);
1234 temp_res
= LLVMBuildLoad(builder
, atom_res
, "");
1235 temp_res
= LLVMBuildInsertElement(builder
, temp_res
, lp_build_const_int32(gallivm
, 0), loop_state
.counter
, "");
1236 LLVMBuildStore(builder
, temp_res
, atom_res
);
1237 lp_build_endif(&ifthen
);
1239 lp_build_loop_end_cond(&loop_state
, lp_build_const_int32(gallivm
, uint_bld
->type
.length
),
1241 *result
= LLVMBuildLoad(builder
, atom_res
, "");
1244 static void emit_barrier(struct lp_build_nir_context
*bld_base
)
1246 struct lp_build_nir_soa_context
*bld
= (struct lp_build_nir_soa_context
*)bld_base
;
1247 struct gallivm_state
* gallivm
= bld_base
->base
.gallivm
;
1249 LLVMBasicBlockRef resume
= lp_build_insert_new_block(gallivm
, "resume");
1251 lp_build_coro_suspend_switch(gallivm
, bld
->coro
, resume
, false);
1252 LLVMPositionBuilderAtEnd(gallivm
->builder
, resume
);
1255 static LLVMValueRef
emit_get_buffer_size(struct lp_build_nir_context
*bld_base
,
1258 struct gallivm_state
*gallivm
= bld_base
->base
.gallivm
;
1259 struct lp_build_nir_soa_context
*bld
= (struct lp_build_nir_soa_context
*)bld_base
;
1260 LLVMBuilderRef builder
= bld
->bld_base
.base
.gallivm
->builder
;
1261 struct lp_build_context
*bld_broad
= &bld_base
->uint_bld
;
1262 LLVMValueRef size_ptr
= lp_build_array_get(bld_base
->base
.gallivm
, bld
->ssbo_sizes_ptr
,
1263 LLVMBuildExtractElement(builder
, index
, lp_build_const_int32(gallivm
, 0), ""));
1264 return lp_build_broadcast_scalar(bld_broad
, size_ptr
);
1267 static void emit_image_op(struct lp_build_nir_context
*bld_base
,
1268 struct lp_img_params
*params
)
1270 struct lp_build_nir_soa_context
*bld
= (struct lp_build_nir_soa_context
*)bld_base
;
1271 struct gallivm_state
*gallivm
= bld_base
->base
.gallivm
;
1273 params
->type
= bld_base
->base
.type
;
1274 params
->context_ptr
= bld
->context_ptr
;
1275 params
->thread_data_ptr
= bld
->thread_data_ptr
;
1276 params
->exec_mask
= mask_vec(bld_base
);
1278 if (params
->image_index_offset
)
1279 params
->image_index_offset
= LLVMBuildExtractElement(gallivm
->builder
, params
->image_index_offset
,
1280 lp_build_const_int32(gallivm
, 0), "");
1282 bld
->image
->emit_op(bld
->image
,
1283 bld
->bld_base
.base
.gallivm
,
1288 static void emit_image_size(struct lp_build_nir_context
*bld_base
,
1289 struct lp_sampler_size_query_params
*params
)
1291 struct lp_build_nir_soa_context
*bld
= (struct lp_build_nir_soa_context
*)bld_base
;
1292 struct gallivm_state
*gallivm
= bld_base
->base
.gallivm
;
1294 params
->int_type
= bld_base
->int_bld
.type
;
1295 params
->context_ptr
= bld
->context_ptr
;
1297 if (params
->texture_unit_offset
)
1298 params
->texture_unit_offset
= LLVMBuildExtractElement(gallivm
->builder
, params
->texture_unit_offset
,
1299 lp_build_const_int32(gallivm
, 0), "");
1300 bld
->image
->emit_size_query(bld
->image
,
1301 bld
->bld_base
.base
.gallivm
,
1306 static void init_var_slots(struct lp_build_nir_context
*bld_base
,
1307 nir_variable
*var
, unsigned sc
)
1309 struct lp_build_nir_soa_context
*bld
= (struct lp_build_nir_soa_context
*)bld_base
;
1310 unsigned slots
= glsl_count_attribute_slots(var
->type
, false) * 4;
1314 for (unsigned comp
= sc
; comp
< slots
+ sc
; comp
++) {
1315 unsigned this_loc
= var
->data
.driver_location
+ (comp
/ 4);
1316 unsigned this_chan
= comp
% 4;
1318 if (!bld
->outputs
[this_loc
][this_chan
])
1319 bld
->outputs
[this_loc
][this_chan
] = lp_build_alloca(bld_base
->base
.gallivm
,
1320 bld_base
->base
.vec_type
, "output");
1324 static void emit_var_decl(struct lp_build_nir_context
*bld_base
,
1327 unsigned sc
= var
->data
.location_frac
;
1328 switch (var
->data
.mode
) {
1329 case nir_var_shader_out
: {
1330 if (bld_base
->shader
->info
.stage
== MESA_SHADER_FRAGMENT
) {
1331 if (var
->data
.location
== FRAG_RESULT_STENCIL
)
1333 else if (var
->data
.location
== FRAG_RESULT_DEPTH
)
1336 init_var_slots(bld_base
, var
, sc
);
1344 static void emit_tex(struct lp_build_nir_context
*bld_base
,
1345 struct lp_sampler_params
*params
)
1347 struct lp_build_nir_soa_context
*bld
= (struct lp_build_nir_soa_context
*)bld_base
;
1348 struct gallivm_state
*gallivm
= bld_base
->base
.gallivm
;
1350 params
->type
= bld_base
->base
.type
;
1351 params
->context_ptr
= bld
->context_ptr
;
1352 params
->thread_data_ptr
= bld
->thread_data_ptr
;
1354 if (params
->texture_index_offset
&& bld_base
->shader
->info
.stage
!= MESA_SHADER_FRAGMENT
) {
1355 /* this is horrible but this can be dynamic */
1356 LLVMValueRef coords
[5];
1357 LLVMValueRef
*orig_texel_ptr
;
1358 struct lp_build_context
*uint_bld
= &bld_base
->uint_bld
;
1359 LLVMValueRef result
[4] = { LLVMGetUndef(bld_base
->base
.vec_type
),
1360 LLVMGetUndef(bld_base
->base
.vec_type
),
1361 LLVMGetUndef(bld_base
->base
.vec_type
),
1362 LLVMGetUndef(bld_base
->base
.vec_type
) };
1363 LLVMValueRef texel
[4], orig_offset
;
1365 orig_texel_ptr
= params
->texel
;
1367 for (i
= 0; i
< 5; i
++) {
1368 coords
[i
] = params
->coords
[i
];
1370 orig_offset
= params
->texture_index_offset
;
1372 for (unsigned v
= 0; v
< uint_bld
->type
.length
; v
++) {
1373 LLVMValueRef idx
= lp_build_const_int32(gallivm
, v
);
1374 LLVMValueRef new_coords
[5];
1375 for (i
= 0; i
< 5; i
++) {
1376 new_coords
[i
] = LLVMBuildExtractElement(gallivm
->builder
,
1377 coords
[i
], idx
, "");
1379 params
->coords
= new_coords
;
1380 params
->texture_index_offset
= LLVMBuildExtractElement(gallivm
->builder
,
1383 params
->type
= lp_elem_type(bld_base
->base
.type
);
1385 params
->texel
= texel
;
1386 bld
->sampler
->emit_tex_sample(bld
->sampler
,
1390 for (i
= 0; i
< 4; i
++) {
1391 result
[i
] = LLVMBuildInsertElement(gallivm
->builder
, result
[i
], texel
[i
], idx
, "");
1394 for (i
= 0; i
< 4; i
++) {
1395 orig_texel_ptr
[i
] = result
[i
];
1400 if (params
->texture_index_offset
)
1401 params
->texture_index_offset
= LLVMBuildExtractElement(bld_base
->base
.gallivm
->builder
,
1402 params
->texture_index_offset
,
1403 lp_build_const_int32(bld_base
->base
.gallivm
, 0), "");
1405 params
->type
= bld_base
->base
.type
;
1406 bld
->sampler
->emit_tex_sample(bld
->sampler
,
1407 bld
->bld_base
.base
.gallivm
,
1411 static void emit_tex_size(struct lp_build_nir_context
*bld_base
,
1412 struct lp_sampler_size_query_params
*params
)
1414 struct lp_build_nir_soa_context
*bld
= (struct lp_build_nir_soa_context
*)bld_base
;
1416 params
->int_type
= bld_base
->int_bld
.type
;
1417 params
->context_ptr
= bld
->context_ptr
;
1419 if (params
->texture_unit_offset
)
1420 params
->texture_unit_offset
= LLVMBuildExtractElement(bld_base
->base
.gallivm
->builder
,
1421 params
->texture_unit_offset
,
1422 lp_build_const_int32(bld_base
->base
.gallivm
, 0), "");
1423 bld
->sampler
->emit_size_query(bld
->sampler
,
1424 bld
->bld_base
.base
.gallivm
,
1428 static void emit_sysval_intrin(struct lp_build_nir_context
*bld_base
,
1429 nir_intrinsic_instr
*instr
,
1430 LLVMValueRef result
[NIR_MAX_VEC_COMPONENTS
])
1432 struct lp_build_nir_soa_context
*bld
= (struct lp_build_nir_soa_context
*)bld_base
;
1433 struct gallivm_state
*gallivm
= bld_base
->base
.gallivm
;
1434 switch (instr
->intrinsic
) {
1435 case nir_intrinsic_load_instance_id
:
1436 result
[0] = lp_build_broadcast_scalar(&bld_base
->uint_bld
, bld
->system_values
.instance_id
);
1438 case nir_intrinsic_load_base_instance
:
1439 result
[0] = lp_build_broadcast_scalar(&bld_base
->uint_bld
, bld
->system_values
.base_instance
);
1441 case nir_intrinsic_load_base_vertex
:
1442 result
[0] = bld
->system_values
.basevertex
;
1444 case nir_intrinsic_load_vertex_id
:
1445 result
[0] = bld
->system_values
.vertex_id
;
1447 case nir_intrinsic_load_primitive_id
:
1448 result
[0] = bld
->system_values
.prim_id
;
1450 case nir_intrinsic_load_work_group_id
:
1451 for (unsigned i
= 0; i
< 3; i
++)
1452 result
[i
] = lp_build_broadcast_scalar(&bld_base
->uint_bld
, LLVMBuildExtractElement(gallivm
->builder
, bld
->system_values
.block_id
, lp_build_const_int32(gallivm
, i
), ""));
1454 case nir_intrinsic_load_local_invocation_id
:
1455 for (unsigned i
= 0; i
< 3; i
++)
1456 result
[i
] = LLVMBuildExtractValue(gallivm
->builder
, bld
->system_values
.thread_id
, i
, "");
1458 case nir_intrinsic_load_num_work_groups
:
1459 for (unsigned i
= 0; i
< 3; i
++)
1460 result
[i
] = lp_build_broadcast_scalar(&bld_base
->uint_bld
, LLVMBuildExtractElement(gallivm
->builder
, bld
->system_values
.grid_size
, lp_build_const_int32(gallivm
, i
), ""));
1462 case nir_intrinsic_load_invocation_id
:
1463 if (bld_base
->shader
->info
.stage
== MESA_SHADER_TESS_CTRL
)
1464 result
[0] = bld
->system_values
.invocation_id
;
1466 result
[0] = lp_build_broadcast_scalar(&bld_base
->uint_bld
, bld
->system_values
.invocation_id
);
1468 case nir_intrinsic_load_front_face
:
1469 result
[0] = lp_build_broadcast_scalar(&bld_base
->uint_bld
, bld
->system_values
.front_facing
);
1471 case nir_intrinsic_load_draw_id
:
1472 result
[0] = lp_build_broadcast_scalar(&bld_base
->uint_bld
, bld
->system_values
.draw_id
);
1476 case nir_intrinsic_load_local_group_size
:
1477 for (unsigned i
= 0; i
< 3; i
++)
1478 result
[i
] = lp_build_broadcast_scalar(&bld_base
->uint_bld
, LLVMBuildExtractElement(gallivm
->builder
, bld
->system_values
.block_size
, lp_build_const_int32(gallivm
, i
), ""));
1480 case nir_intrinsic_load_work_dim
:
1481 result
[0] = lp_build_broadcast_scalar(&bld_base
->uint_bld
, bld
->system_values
.work_dim
);
1483 case nir_intrinsic_load_tess_coord
:
1484 for (unsigned i
= 0; i
< 3; i
++) {
1485 result
[i
] = LLVMBuildExtractValue(gallivm
->builder
, bld
->system_values
.tess_coord
, i
, "");
1488 case nir_intrinsic_load_tess_level_outer
:
1489 for (unsigned i
= 0; i
< 4; i
++)
1490 result
[i
] = lp_build_broadcast_scalar(&bld_base
->base
, LLVMBuildExtractValue(gallivm
->builder
, bld
->system_values
.tess_outer
, i
, ""));
1492 case nir_intrinsic_load_tess_level_inner
:
1493 for (unsigned i
= 0; i
< 2; i
++)
1494 result
[i
] = lp_build_broadcast_scalar(&bld_base
->base
, LLVMBuildExtractValue(gallivm
->builder
, bld
->system_values
.tess_inner
, i
, ""));
1496 case nir_intrinsic_load_patch_vertices_in
:
1497 result
[0] = bld
->system_values
.vertices_in
;
1499 case nir_intrinsic_load_sample_id
:
1500 result
[0] = lp_build_broadcast_scalar(&bld_base
->uint_bld
, bld
->system_values
.sample_id
);
1502 case nir_intrinsic_load_sample_pos
:
1503 for (unsigned i
= 0; i
< 2; i
++) {
1504 LLVMValueRef idx
= LLVMBuildMul(gallivm
->builder
, bld
->system_values
.sample_id
, lp_build_const_int32(gallivm
, 2), "");
1505 idx
= LLVMBuildAdd(gallivm
->builder
, idx
, lp_build_const_int32(gallivm
, i
), "");
1506 LLVMValueRef val
= lp_build_array_get(gallivm
, bld
->system_values
.sample_pos
, idx
);
1507 result
[i
] = lp_build_broadcast_scalar(&bld_base
->base
, val
);
1510 case nir_intrinsic_load_sample_mask_in
:
1511 result
[0] = bld
->system_values
.sample_mask_in
;
1516 static void emit_helper_invocation(struct lp_build_nir_context
*bld_base
,
1519 struct gallivm_state
*gallivm
= bld_base
->base
.gallivm
;
1520 struct lp_build_context
*uint_bld
= &bld_base
->uint_bld
;
1521 *dst
= lp_build_cmp(uint_bld
, PIPE_FUNC_NOTEQUAL
, mask_vec(bld_base
), lp_build_const_int_vec(gallivm
, uint_bld
->type
, -1));
1524 static void bgnloop(struct lp_build_nir_context
*bld_base
)
1526 struct lp_build_nir_soa_context
*bld
= (struct lp_build_nir_soa_context
*)bld_base
;
1527 lp_exec_bgnloop(&bld
->exec_mask
, true);
1530 static void endloop(struct lp_build_nir_context
*bld_base
)
1532 struct lp_build_nir_soa_context
*bld
= (struct lp_build_nir_soa_context
*)bld_base
;
1533 lp_exec_endloop(bld_base
->base
.gallivm
, &bld
->exec_mask
);
1536 static void if_cond(struct lp_build_nir_context
*bld_base
, LLVMValueRef cond
)
1538 LLVMBuilderRef builder
= bld_base
->base
.gallivm
->builder
;
1539 struct lp_build_nir_soa_context
*bld
= (struct lp_build_nir_soa_context
*)bld_base
;
1540 lp_exec_mask_cond_push(&bld
->exec_mask
, LLVMBuildBitCast(builder
, cond
, bld_base
->base
.int_vec_type
, ""));
1543 static void else_stmt(struct lp_build_nir_context
*bld_base
)
1545 struct lp_build_nir_soa_context
*bld
= (struct lp_build_nir_soa_context
*)bld_base
;
1546 lp_exec_mask_cond_invert(&bld
->exec_mask
);
1549 static void endif_stmt(struct lp_build_nir_context
*bld_base
)
1551 struct lp_build_nir_soa_context
*bld
= (struct lp_build_nir_soa_context
*)bld_base
;
1552 lp_exec_mask_cond_pop(&bld
->exec_mask
);
1555 static void break_stmt(struct lp_build_nir_context
*bld_base
)
1557 struct lp_build_nir_soa_context
*bld
= (struct lp_build_nir_soa_context
*)bld_base
;
1559 lp_exec_break(&bld
->exec_mask
, NULL
, false);
1562 static void continue_stmt(struct lp_build_nir_context
*bld_base
)
1564 struct lp_build_nir_soa_context
*bld
= (struct lp_build_nir_soa_context
*)bld_base
;
1565 lp_exec_continue(&bld
->exec_mask
);
1568 static void discard(struct lp_build_nir_context
*bld_base
, LLVMValueRef cond
)
1570 struct lp_build_nir_soa_context
*bld
= (struct lp_build_nir_soa_context
*)bld_base
;
1571 LLVMBuilderRef builder
= bld
->bld_base
.base
.gallivm
->builder
;
1575 if (bld
->exec_mask
.has_mask
) {
1576 mask
= LLVMBuildNot(builder
, bld
->exec_mask
.exec_mask
, "kilp");
1578 mask
= LLVMConstNull(bld
->bld_base
.base
.int_vec_type
);
1581 mask
= LLVMBuildNot(builder
, cond
, "");
1582 if (bld
->exec_mask
.has_mask
) {
1583 LLVMValueRef invmask
;
1584 invmask
= LLVMBuildNot(builder
, bld
->exec_mask
.exec_mask
, "kilp");
1585 mask
= LLVMBuildOr(builder
, mask
, invmask
, "");
1588 lp_build_mask_update(bld
->mask
, mask
);
1592 increment_vec_ptr_by_mask(struct lp_build_nir_context
* bld_base
,
1596 LLVMBuilderRef builder
= bld_base
->base
.gallivm
->builder
;
1597 LLVMValueRef current_vec
= LLVMBuildLoad(builder
, ptr
, "");
1599 current_vec
= LLVMBuildSub(builder
, current_vec
, mask
, "");
1601 LLVMBuildStore(builder
, current_vec
, ptr
);
1605 clear_uint_vec_ptr_from_mask(struct lp_build_nir_context
* bld_base
,
1609 LLVMBuilderRef builder
= bld_base
->base
.gallivm
->builder
;
1610 LLVMValueRef current_vec
= LLVMBuildLoad(builder
, ptr
, "");
1612 current_vec
= lp_build_select(&bld_base
->uint_bld
,
1614 bld_base
->uint_bld
.zero
,
1617 LLVMBuildStore(builder
, current_vec
, ptr
);
1621 clamp_mask_to_max_output_vertices(struct lp_build_nir_soa_context
* bld
,
1622 LLVMValueRef current_mask_vec
,
1623 LLVMValueRef total_emitted_vertices_vec
)
1625 LLVMBuilderRef builder
= bld
->bld_base
.base
.gallivm
->builder
;
1626 struct lp_build_context
*int_bld
= &bld
->bld_base
.int_bld
;
1627 LLVMValueRef max_mask
= lp_build_cmp(int_bld
, PIPE_FUNC_LESS
,
1628 total_emitted_vertices_vec
,
1629 bld
->max_output_vertices_vec
);
1631 return LLVMBuildAnd(builder
, current_mask_vec
, max_mask
, "");
1634 static void emit_vertex(struct lp_build_nir_context
*bld_base
, uint32_t stream_id
)
1636 struct lp_build_nir_soa_context
*bld
= (struct lp_build_nir_soa_context
*)bld_base
;
1637 LLVMBuilderRef builder
= bld
->bld_base
.base
.gallivm
->builder
;
1639 if (stream_id
>= bld
->gs_vertex_streams
)
1641 assert(bld
->gs_iface
->emit_vertex
);
1642 LLVMValueRef total_emitted_vertices_vec
=
1643 LLVMBuildLoad(builder
, bld
->total_emitted_vertices_vec_ptr
[stream_id
], "");
1644 LLVMValueRef mask
= mask_vec(bld_base
);
1645 mask
= clamp_mask_to_max_output_vertices(bld
, mask
,
1646 total_emitted_vertices_vec
);
1647 bld
->gs_iface
->emit_vertex(bld
->gs_iface
, &bld
->bld_base
.base
,
1649 total_emitted_vertices_vec
,
1651 lp_build_const_int_vec(bld
->bld_base
.base
.gallivm
, bld
->bld_base
.base
.type
, stream_id
));
1653 increment_vec_ptr_by_mask(bld_base
, bld
->emitted_vertices_vec_ptr
[stream_id
],
1655 increment_vec_ptr_by_mask(bld_base
, bld
->total_emitted_vertices_vec_ptr
[stream_id
],
1660 end_primitive_masked(struct lp_build_nir_context
* bld_base
,
1661 LLVMValueRef mask
, uint32_t stream_id
)
1663 struct lp_build_nir_soa_context
*bld
= (struct lp_build_nir_soa_context
*)bld_base
;
1664 LLVMBuilderRef builder
= bld
->bld_base
.base
.gallivm
->builder
;
1666 if (stream_id
>= bld
->gs_vertex_streams
)
1668 struct lp_build_context
*uint_bld
= &bld_base
->uint_bld
;
1669 LLVMValueRef emitted_vertices_vec
=
1670 LLVMBuildLoad(builder
, bld
->emitted_vertices_vec_ptr
[stream_id
], "");
1671 LLVMValueRef emitted_prims_vec
=
1672 LLVMBuildLoad(builder
, bld
->emitted_prims_vec_ptr
[stream_id
], "");
1673 LLVMValueRef total_emitted_vertices_vec
=
1674 LLVMBuildLoad(builder
, bld
->total_emitted_vertices_vec_ptr
[stream_id
], "");
1676 LLVMValueRef emitted_mask
= lp_build_cmp(uint_bld
,
1678 emitted_vertices_vec
,
1680 mask
= LLVMBuildAnd(builder
, mask
, emitted_mask
, "");
1681 bld
->gs_iface
->end_primitive(bld
->gs_iface
, &bld
->bld_base
.base
,
1682 total_emitted_vertices_vec
,
1683 emitted_vertices_vec
, emitted_prims_vec
, mask
, stream_id
);
1684 increment_vec_ptr_by_mask(bld_base
, bld
->emitted_prims_vec_ptr
[stream_id
],
1686 clear_uint_vec_ptr_from_mask(bld_base
, bld
->emitted_vertices_vec_ptr
[stream_id
],
1690 static void end_primitive(struct lp_build_nir_context
*bld_base
, uint32_t stream_id
)
1692 ASSERTED
struct lp_build_nir_soa_context
*bld
= (struct lp_build_nir_soa_context
*)bld_base
;
1694 assert(bld
->gs_iface
->end_primitive
);
1696 LLVMValueRef mask
= mask_vec(bld_base
);
1697 end_primitive_masked(bld_base
, mask
, stream_id
);
1701 emit_prologue(struct lp_build_nir_soa_context
*bld
)
1703 struct gallivm_state
* gallivm
= bld
->bld_base
.base
.gallivm
;
1704 if (bld
->indirects
& nir_var_shader_in
&& !bld
->gs_iface
&& !bld
->tcs_iface
&& !bld
->tes_iface
) {
1705 uint32_t num_inputs
= util_bitcount64(bld
->bld_base
.shader
->info
.inputs_read
);
1706 unsigned index
, chan
;
1707 LLVMTypeRef vec_type
= bld
->bld_base
.base
.vec_type
;
1708 LLVMValueRef array_size
= lp_build_const_int32(gallivm
, num_inputs
* 4);
1709 bld
->inputs_array
= lp_build_array_alloca(gallivm
,
1710 vec_type
, array_size
,
1713 for (index
= 0; index
< num_inputs
; ++index
) {
1714 for (chan
= 0; chan
< TGSI_NUM_CHANNELS
; ++chan
) {
1715 LLVMValueRef lindex
=
1716 lp_build_const_int32(gallivm
, index
* 4 + chan
);
1717 LLVMValueRef input_ptr
=
1718 LLVMBuildGEP(gallivm
->builder
, bld
->inputs_array
,
1720 LLVMValueRef value
= bld
->inputs
[index
][chan
];
1722 LLVMBuildStore(gallivm
->builder
, value
, input_ptr
);
1728 static void emit_vote(struct lp_build_nir_context
*bld_base
, LLVMValueRef src
, nir_intrinsic_instr
*instr
, LLVMValueRef result
[4])
1730 struct gallivm_state
* gallivm
= bld_base
->base
.gallivm
;
1731 LLVMBuilderRef builder
= gallivm
->builder
;
1733 LLVMValueRef exec_mask
= mask_vec(bld_base
);
1734 struct lp_build_loop_state loop_state
;
1736 LLVMValueRef outer_cond
= LLVMBuildICmp(builder
, LLVMIntNE
, exec_mask
, bld_base
->uint_bld
.zero
, "");
1738 LLVMValueRef res_store
= lp_build_alloca(gallivm
, bld_base
->int_bld
.elem_type
, "");
1739 LLVMValueRef init_val
= NULL
;
1740 if (instr
->intrinsic
== nir_intrinsic_vote_ieq
) {
1741 /* for equal we unfortunately have to loop and find the first valid one. */
1742 lp_build_loop_begin(&loop_state
, gallivm
, lp_build_const_int32(gallivm
, 0));
1743 LLVMValueRef if_cond
= LLVMBuildExtractElement(gallivm
->builder
, outer_cond
, loop_state
.counter
, "");
1745 struct lp_build_if_state ifthen
;
1746 lp_build_if(&ifthen
, gallivm
, if_cond
);
1747 LLVMValueRef value_ptr
= LLVMBuildExtractElement(gallivm
->builder
, src
,
1748 loop_state
.counter
, "");
1749 LLVMBuildStore(builder
, value_ptr
, res_store
);
1750 lp_build_endif(&ifthen
);
1751 lp_build_loop_end_cond(&loop_state
, lp_build_const_int32(gallivm
, bld_base
->uint_bld
.type
.length
),
1753 init_val
= LLVMBuildLoad(builder
, res_store
, "");
1755 LLVMBuildStore(builder
, lp_build_const_int32(gallivm
, instr
->intrinsic
== nir_intrinsic_vote_any
? 0 : -1), res_store
);
1759 lp_build_loop_begin(&loop_state
, gallivm
, lp_build_const_int32(gallivm
, 0));
1760 LLVMValueRef value_ptr
= LLVMBuildExtractElement(gallivm
->builder
, src
,
1761 loop_state
.counter
, "");
1762 struct lp_build_if_state ifthen
;
1763 LLVMValueRef if_cond
;
1764 if_cond
= LLVMBuildExtractElement(gallivm
->builder
, outer_cond
, loop_state
.counter
, "");
1766 lp_build_if(&ifthen
, gallivm
, if_cond
);
1767 res
= LLVMBuildLoad(builder
, res_store
, "");
1769 if (instr
->intrinsic
== nir_intrinsic_vote_ieq
) {
1770 LLVMValueRef tmp
= LLVMBuildICmp(builder
, LLVMIntEQ
, init_val
, value_ptr
, "");
1771 tmp
= LLVMBuildSExt(builder
, tmp
, bld_base
->uint_bld
.elem_type
, "");
1772 res
= LLVMBuildOr(builder
, res
, tmp
, "");
1773 } else if (instr
->intrinsic
== nir_intrinsic_vote_any
)
1774 res
= LLVMBuildOr(builder
, res
, value_ptr
, "");
1776 res
= LLVMBuildAnd(builder
, res
, value_ptr
, "");
1777 LLVMBuildStore(builder
, res
, res_store
);
1778 lp_build_endif(&ifthen
);
1779 lp_build_loop_end_cond(&loop_state
, lp_build_const_int32(gallivm
, bld_base
->uint_bld
.type
.length
),
1781 result
[0] = lp_build_broadcast_scalar(&bld_base
->uint_bld
, LLVMBuildLoad(builder
, res_store
, ""));
1785 emit_interp_at(struct lp_build_nir_context
*bld_base
,
1786 unsigned num_components
,
1790 unsigned const_index
,
1791 LLVMValueRef indir_index
,
1792 LLVMValueRef offsets
[2],
1793 LLVMValueRef dst
[4])
1795 struct lp_build_nir_soa_context
*bld
= (struct lp_build_nir_soa_context
*)bld_base
;
1797 for (unsigned i
= 0; i
< num_components
; i
++) {
1798 dst
[i
] = bld
->fs_iface
->interp_fn(bld
->fs_iface
, &bld_base
->base
,
1799 const_index
+ var
->data
.driver_location
, i
+ var
->data
.location_frac
,
1800 centroid
, sample
, indir_index
, offsets
);
1804 void lp_build_nir_soa(struct gallivm_state
*gallivm
,
1805 struct nir_shader
*shader
,
1806 const struct lp_build_tgsi_params
*params
,
1807 LLVMValueRef (*outputs
)[4])
1809 struct lp_build_nir_soa_context bld
;
1810 struct lp_type type
= params
->type
;
1811 struct lp_type res_type
;
1813 assert(type
.length
<= LP_MAX_VECTOR_LENGTH
);
1814 memset(&res_type
, 0, sizeof res_type
);
1815 res_type
.width
= type
.width
;
1816 res_type
.length
= type
.length
;
1819 /* Setup build context */
1820 memset(&bld
, 0, sizeof bld
);
1821 lp_build_context_init(&bld
.bld_base
.base
, gallivm
, type
);
1822 lp_build_context_init(&bld
.bld_base
.uint_bld
, gallivm
, lp_uint_type(type
));
1823 lp_build_context_init(&bld
.bld_base
.int_bld
, gallivm
, lp_int_type(type
));
1824 lp_build_context_init(&bld
.elem_bld
, gallivm
, lp_elem_type(type
));
1825 lp_build_context_init(&bld
.uint_elem_bld
, gallivm
, lp_elem_type(lp_uint_type(type
)));
1827 struct lp_type dbl_type
;
1829 dbl_type
.width
*= 2;
1830 lp_build_context_init(&bld
.bld_base
.dbl_bld
, gallivm
, dbl_type
);
1833 struct lp_type uint64_type
;
1834 uint64_type
= lp_uint_type(type
);
1835 uint64_type
.width
*= 2;
1836 lp_build_context_init(&bld
.bld_base
.uint64_bld
, gallivm
, uint64_type
);
1839 struct lp_type int64_type
;
1840 int64_type
= lp_int_type(type
);
1841 int64_type
.width
*= 2;
1842 lp_build_context_init(&bld
.bld_base
.int64_bld
, gallivm
, int64_type
);
1845 struct lp_type uint16_type
;
1846 uint16_type
= lp_uint_type(type
);
1847 uint16_type
.width
/= 2;
1848 lp_build_context_init(&bld
.bld_base
.uint16_bld
, gallivm
, uint16_type
);
1851 struct lp_type int16_type
;
1852 int16_type
= lp_int_type(type
);
1853 int16_type
.width
/= 2;
1854 lp_build_context_init(&bld
.bld_base
.int16_bld
, gallivm
, int16_type
);
1857 struct lp_type uint8_type
;
1858 uint8_type
= lp_uint_type(type
);
1859 uint8_type
.width
/= 4;
1860 lp_build_context_init(&bld
.bld_base
.uint8_bld
, gallivm
, uint8_type
);
1863 struct lp_type int8_type
;
1864 int8_type
= lp_int_type(type
);
1865 int8_type
.width
/= 4;
1866 lp_build_context_init(&bld
.bld_base
.int8_bld
, gallivm
, int8_type
);
1868 bld
.bld_base
.load_var
= emit_load_var
;
1869 bld
.bld_base
.store_var
= emit_store_var
;
1870 bld
.bld_base
.load_reg
= emit_load_reg
;
1871 bld
.bld_base
.store_reg
= emit_store_reg
;
1872 bld
.bld_base
.emit_var_decl
= emit_var_decl
;
1873 bld
.bld_base
.load_ubo
= emit_load_ubo
;
1874 bld
.bld_base
.load_kernel_arg
= emit_load_kernel_arg
;
1875 bld
.bld_base
.load_global
= emit_load_global
;
1876 bld
.bld_base
.store_global
= emit_store_global
;
1877 bld
.bld_base
.atomic_global
= emit_atomic_global
;
1878 bld
.bld_base
.tex
= emit_tex
;
1879 bld
.bld_base
.tex_size
= emit_tex_size
;
1880 bld
.bld_base
.bgnloop
= bgnloop
;
1881 bld
.bld_base
.endloop
= endloop
;
1882 bld
.bld_base
.if_cond
= if_cond
;
1883 bld
.bld_base
.else_stmt
= else_stmt
;
1884 bld
.bld_base
.endif_stmt
= endif_stmt
;
1885 bld
.bld_base
.break_stmt
= break_stmt
;
1886 bld
.bld_base
.continue_stmt
= continue_stmt
;
1887 bld
.bld_base
.sysval_intrin
= emit_sysval_intrin
;
1888 bld
.bld_base
.discard
= discard
;
1889 bld
.bld_base
.emit_vertex
= emit_vertex
;
1890 bld
.bld_base
.end_primitive
= end_primitive
;
1891 bld
.bld_base
.load_mem
= emit_load_mem
;
1892 bld
.bld_base
.store_mem
= emit_store_mem
;
1893 bld
.bld_base
.get_buffer_size
= emit_get_buffer_size
;
1894 bld
.bld_base
.atomic_mem
= emit_atomic_mem
;
1895 bld
.bld_base
.barrier
= emit_barrier
;
1896 bld
.bld_base
.image_op
= emit_image_op
;
1897 bld
.bld_base
.image_size
= emit_image_size
;
1898 bld
.bld_base
.vote
= emit_vote
;
1899 bld
.bld_base
.helper_invocation
= emit_helper_invocation
;
1900 bld
.bld_base
.interp_at
= emit_interp_at
;
1902 bld
.mask
= params
->mask
;
1903 bld
.inputs
= params
->inputs
;
1904 bld
.outputs
= outputs
;
1905 bld
.consts_ptr
= params
->consts_ptr
;
1906 bld
.const_sizes_ptr
= params
->const_sizes_ptr
;
1907 bld
.ssbo_ptr
= params
->ssbo_ptr
;
1908 bld
.ssbo_sizes_ptr
= params
->ssbo_sizes_ptr
;
1909 bld
.sampler
= params
->sampler
;
1910 // bld.bld_base.info = params->info;
1912 bld
.context_ptr
= params
->context_ptr
;
1913 bld
.thread_data_ptr
= params
->thread_data_ptr
;
1914 bld
.image
= params
->image
;
1915 bld
.shared_ptr
= params
->shared_ptr
;
1916 bld
.coro
= params
->coro
;
1917 bld
.kernel_args_ptr
= params
->kernel_args
;
1919 if (params
->info
->indirect_files
& (1 << TGSI_FILE_INPUT
))
1920 bld
.indirects
|= nir_var_shader_in
;
1922 bld
.gs_iface
= params
->gs_iface
;
1923 bld
.tcs_iface
= params
->tcs_iface
;
1924 bld
.tes_iface
= params
->tes_iface
;
1925 bld
.fs_iface
= params
->fs_iface
;
1927 struct lp_build_context
*uint_bld
= &bld
.bld_base
.uint_bld
;
1929 bld
.gs_vertex_streams
= params
->gs_vertex_streams
;
1930 bld
.max_output_vertices_vec
= lp_build_const_int_vec(gallivm
, bld
.bld_base
.int_bld
.type
,
1931 shader
->info
.gs
.vertices_out
);
1932 for (int i
= 0; i
< params
->gs_vertex_streams
; i
++) {
1933 bld
.emitted_prims_vec_ptr
[i
] =
1934 lp_build_alloca(gallivm
, uint_bld
->vec_type
, "emitted_prims_ptr");
1935 bld
.emitted_vertices_vec_ptr
[i
] =
1936 lp_build_alloca(gallivm
, uint_bld
->vec_type
, "emitted_vertices_ptr");
1937 bld
.total_emitted_vertices_vec_ptr
[i
] =
1938 lp_build_alloca(gallivm
, uint_bld
->vec_type
, "total_emitted_vertices_ptr");
1941 lp_exec_mask_init(&bld
.exec_mask
, &bld
.bld_base
.int_bld
);
1943 bld
.system_values
= *params
->system_values
;
1945 bld
.bld_base
.shader
= shader
;
1947 emit_prologue(&bld
);
1948 lp_build_nir_llvm(&bld
.bld_base
, shader
);
1951 LLVMBuilderRef builder
= bld
.bld_base
.base
.gallivm
->builder
;
1952 LLVMValueRef total_emitted_vertices_vec
;
1953 LLVMValueRef emitted_prims_vec
;
1955 for (int i
= 0; i
< params
->gs_vertex_streams
; i
++) {
1956 end_primitive_masked(&bld
.bld_base
, lp_build_mask_value(bld
.mask
), i
);
1958 total_emitted_vertices_vec
=
1959 LLVMBuildLoad(builder
, bld
.total_emitted_vertices_vec_ptr
[i
], "");
1962 LLVMBuildLoad(builder
, bld
.emitted_prims_vec_ptr
[i
], "");
1963 bld
.gs_iface
->gs_epilogue(bld
.gs_iface
,
1964 total_emitted_vertices_vec
,
1965 emitted_prims_vec
, i
);
1968 lp_exec_mask_fini(&bld
.exec_mask
);