projects / mesa.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
history | raw | HEAD
ac/nir: use ac_emit_llvm_intrinsic throughout
[mesa.git] / src / amd / common / ac_nir_to_llvm.c
1 /*
2 * Copyright © 2016 Bas Nieuwenhuizen
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 */
23
24 #include "ac_nir_to_llvm.h"
25 #include "ac_llvm_util.h"
26 #include "ac_binary.h"
27 #include "sid.h"
28 #include "nir/nir.h"
29 #include "../vulkan/radv_descriptor_set.h"
30 #include "util/bitscan.h"
31 #include <llvm-c/Transforms/Scalar.h>
32
33 enum radeon_llvm_calling_convention {
34 RADEON_LLVM_AMDGPU_VS = 87,
35 RADEON_LLVM_AMDGPU_GS = 88,
36 RADEON_LLVM_AMDGPU_PS = 89,
37 RADEON_LLVM_AMDGPU_CS = 90,
38 };
39
40 #define CONST_ADDR_SPACE 2
41 #define LOCAL_ADDR_SPACE 3
42
43 #define RADEON_LLVM_MAX_INPUTS (VARYING_SLOT_VAR31 + 1)
44 #define RADEON_LLVM_MAX_OUTPUTS (VARYING_SLOT_VAR31 + 1)
45
46 enum desc_type {
47 DESC_IMAGE,
48 DESC_FMASK,
49 DESC_SAMPLER,
50 DESC_BUFFER,
51 };
52
53 struct nir_to_llvm_context {
54 struct ac_llvm_context ac;
55 const struct ac_nir_compiler_options *options;
56 struct ac_shader_variant_info *shader_info;
57
58 LLVMContextRef context;
59 LLVMModuleRef module;
60 LLVMBuilderRef builder;
61 LLVMValueRef main_function;
62
63 struct hash_table *defs;
64 struct hash_table *phis;
65
66 LLVMValueRef descriptor_sets[AC_UD_MAX_SETS];
67 LLVMValueRef push_constants;
68 LLVMValueRef num_work_groups;
69 LLVMValueRef workgroup_ids;
70 LLVMValueRef local_invocation_ids;
71 LLVMValueRef tg_size;
72
73 LLVMValueRef vertex_buffers;
74 LLVMValueRef base_vertex;
75 LLVMValueRef start_instance;
76 LLVMValueRef vertex_id;
77 LLVMValueRef rel_auto_id;
78 LLVMValueRef vs_prim_id;
79 LLVMValueRef instance_id;
80
81 LLVMValueRef prim_mask;
82 LLVMValueRef sample_positions;
83 LLVMValueRef persp_sample, persp_center, persp_centroid;
84 LLVMValueRef linear_sample, linear_center, linear_centroid;
85 LLVMValueRef front_face;
86 LLVMValueRef ancillary;
87 LLVMValueRef frag_pos[4];
88
89 LLVMBasicBlockRef continue_block;
90 LLVMBasicBlockRef break_block;
91
92 LLVMTypeRef i1;
93 LLVMTypeRef i8;
94 LLVMTypeRef i16;
95 LLVMTypeRef i32;
96 LLVMTypeRef i64;
97 LLVMTypeRef v2i32;
98 LLVMTypeRef v3i32;
99 LLVMTypeRef v4i32;
100 LLVMTypeRef v8i32;
101 LLVMTypeRef f32;
102 LLVMTypeRef f16;
103 LLVMTypeRef v2f32;
104 LLVMTypeRef v4f32;
105 LLVMTypeRef v16i8;
106 LLVMTypeRef voidt;
107
108 LLVMValueRef i32zero;
109 LLVMValueRef i32one;
110 LLVMValueRef f32zero;
111 LLVMValueRef f32one;
112 LLVMValueRef v4f32empty;
113
114 unsigned range_md_kind;
115 unsigned uniform_md_kind;
116 unsigned fpmath_md_kind;
117 unsigned invariant_load_md_kind;
118 LLVMValueRef empty_md;
119 LLVMValueRef fpmath_md_2p5_ulp;
120 gl_shader_stage stage;
121
122 LLVMValueRef lds;
123 LLVMValueRef inputs[RADEON_LLVM_MAX_INPUTS * 4];
124 LLVMValueRef outputs[RADEON_LLVM_MAX_OUTPUTS * 4];
125
126 LLVMValueRef shared_memory;
127 uint64_t input_mask;
128 uint64_t output_mask;
129 int num_locals;
130 LLVMValueRef *locals;
131 bool has_ddxy;
132 unsigned num_clips;
133 unsigned num_culls;
134
135 bool has_ds_bpermute;
136 };
137
138 struct ac_tex_info {
139 LLVMValueRef args[12];
140 int arg_count;
141 LLVMTypeRef dst_type;
142 bool has_offset;
143 };
144
145 static LLVMValueRef get_sampler_desc(struct nir_to_llvm_context *ctx,
146 nir_deref_var *deref,
147 enum desc_type desc_type);
148 static unsigned radeon_llvm_reg_index_soa(unsigned index, unsigned chan)
149 {
150 return (index * 4) + chan;
151 }
152
153 static unsigned llvm_get_type_size(LLVMTypeRef type)
154 {
155 LLVMTypeKind kind = LLVMGetTypeKind(type);
156
157 switch (kind) {
158 case LLVMIntegerTypeKind:
159 return LLVMGetIntTypeWidth(type) / 8;
160 case LLVMFloatTypeKind:
161 return 4;
162 case LLVMPointerTypeKind:
163 return 8;
164 case LLVMVectorTypeKind:
165 return LLVMGetVectorSize(type) *
166 llvm_get_type_size(LLVMGetElementType(type));
167 default:
168 assert(0);
169 return 0;
170 }
171 }
172
173 static void set_llvm_calling_convention(LLVMValueRef func,
174 gl_shader_stage stage)
175 {
176 enum radeon_llvm_calling_convention calling_conv;
177
178 switch (stage) {
179 case MESA_SHADER_VERTEX:
180 case MESA_SHADER_TESS_CTRL:
181 case MESA_SHADER_TESS_EVAL:
182 calling_conv = RADEON_LLVM_AMDGPU_VS;
183 break;
184 case MESA_SHADER_GEOMETRY:
185 calling_conv = RADEON_LLVM_AMDGPU_GS;
186 break;
187 case MESA_SHADER_FRAGMENT:
188 calling_conv = RADEON_LLVM_AMDGPU_PS;
189 break;
190 case MESA_SHADER_COMPUTE:
191 calling_conv = RADEON_LLVM_AMDGPU_CS;
192 break;
193 default:
194 unreachable("Unhandle shader type");
195 }
196
197 LLVMSetFunctionCallConv(func, calling_conv);
198 }
199
200 static LLVMValueRef
201 create_llvm_function(LLVMContextRef ctx, LLVMModuleRef module,
202 LLVMBuilderRef builder, LLVMTypeRef *return_types,
203 unsigned num_return_elems, LLVMTypeRef *param_types,
204 unsigned param_count, unsigned array_params_mask,
205 unsigned sgpr_params, bool unsafe_math)
206 {
207 LLVMTypeRef main_function_type, ret_type;
208 LLVMBasicBlockRef main_function_body;
209
210 if (num_return_elems)
211 ret_type = LLVMStructTypeInContext(ctx, return_types,
212 num_return_elems, true);
213 else
214 ret_type = LLVMVoidTypeInContext(ctx);
215
216 /* Setup the function */
217 main_function_type =
218 LLVMFunctionType(ret_type, param_types, param_count, 0);
219 LLVMValueRef main_function =
220 LLVMAddFunction(module, "main", main_function_type);
221 main_function_body =
222 LLVMAppendBasicBlockInContext(ctx, main_function, "main_body");
223 LLVMPositionBuilderAtEnd(builder, main_function_body);
224
225 LLVMSetFunctionCallConv(main_function, RADEON_LLVM_AMDGPU_CS);
226 for (unsigned i = 0; i < sgpr_params; ++i) {
227 if (array_params_mask & (1 << i)) {
228 LLVMValueRef P = LLVMGetParam(main_function, i);
229 ac_add_function_attr(main_function, i + 1, AC_FUNC_ATTR_BYVAL);
230 ac_add_attr_dereferenceable(P, UINT64_MAX);
231 }
232 else {
233 ac_add_function_attr(main_function, i + 1, AC_FUNC_ATTR_INREG);
234 }
235 }
236
237 if (unsafe_math) {
238 /* These were copied from some LLVM test. */
239 LLVMAddTargetDependentFunctionAttr(main_function,
240 "less-precise-fpmad",
241 "true");
242 LLVMAddTargetDependentFunctionAttr(main_function,
243 "no-infs-fp-math",
244 "true");
245 LLVMAddTargetDependentFunctionAttr(main_function,
246 "no-nans-fp-math",
247 "true");
248 LLVMAddTargetDependentFunctionAttr(main_function,
249 "unsafe-fp-math",
250 "true");
251 }
252 return main_function;
253 }
254
255 static LLVMTypeRef const_array(LLVMTypeRef elem_type, int num_elements)
256 {
257 return LLVMPointerType(LLVMArrayType(elem_type, num_elements),
258 CONST_ADDR_SPACE);
259 }
260
261 static LLVMValueRef get_shared_memory_ptr(struct nir_to_llvm_context *ctx,
262 int idx,
263 LLVMTypeRef type)
264 {
265 LLVMValueRef offset;
266 LLVMValueRef ptr;
267 int addr_space;
268
269 offset = LLVMConstInt(ctx->i32, idx, false);
270
271 ptr = ctx->shared_memory;
272 ptr = LLVMBuildGEP(ctx->builder, ptr, &offset, 1, "");
273 addr_space = LLVMGetPointerAddressSpace(LLVMTypeOf(ptr));
274 ptr = LLVMBuildBitCast(ctx->builder, ptr, LLVMPointerType(type, addr_space), "");
275 return ptr;
276 }
277
278 static LLVMValueRef to_integer(struct nir_to_llvm_context *ctx, LLVMValueRef v)
279 {
280 LLVMTypeRef type = LLVMTypeOf(v);
281 if (type == ctx->f32) {
282 return LLVMBuildBitCast(ctx->builder, v, ctx->i32, "");
283 } else if (LLVMGetTypeKind(type) == LLVMVectorTypeKind) {
284 LLVMTypeRef elem_type = LLVMGetElementType(type);
285 if (elem_type == ctx->f32) {
286 LLVMTypeRef nt = LLVMVectorType(ctx->i32, LLVMGetVectorSize(type));
287 return LLVMBuildBitCast(ctx->builder, v, nt, "");
288 }
289 }
290 return v;
291 }
292
293 static LLVMValueRef to_float(struct nir_to_llvm_context *ctx, LLVMValueRef v)
294 {
295 LLVMTypeRef type = LLVMTypeOf(v);
296 if (type == ctx->i32) {
297 return LLVMBuildBitCast(ctx->builder, v, ctx->f32, "");
298 } else if (LLVMGetTypeKind(type) == LLVMVectorTypeKind) {
299 LLVMTypeRef elem_type = LLVMGetElementType(type);
300 if (elem_type == ctx->i32) {
301 LLVMTypeRef nt = LLVMVectorType(ctx->f32, LLVMGetVectorSize(type));
302 return LLVMBuildBitCast(ctx->builder, v, nt, "");
303 }
304 }
305 return v;
306 }
307
308 static LLVMValueRef unpack_param(struct nir_to_llvm_context *ctx,
309 LLVMValueRef param, unsigned rshift,
310 unsigned bitwidth)
311 {
312 LLVMValueRef value = param;
313 if (rshift)
314 value = LLVMBuildLShr(ctx->builder, value,
315 LLVMConstInt(ctx->i32, rshift, false), "");
316
317 if (rshift + bitwidth < 32) {
318 unsigned mask = (1 << bitwidth) - 1;
319 value = LLVMBuildAnd(ctx->builder, value,
320 LLVMConstInt(ctx->i32, mask, false), "");
321 }
322 return value;
323 }
324
325 static LLVMValueRef build_gep0(struct nir_to_llvm_context *ctx,
326 LLVMValueRef base_ptr, LLVMValueRef index)
327 {
328 LLVMValueRef indices[2] = {
329 ctx->i32zero,
330 index,
331 };
332 return LLVMBuildGEP(ctx->builder, base_ptr,
333 indices, 2, "");
334 }
335
336 static LLVMValueRef build_indexed_load(struct nir_to_llvm_context *ctx,
337 LLVMValueRef base_ptr, LLVMValueRef index,
338 bool uniform)
339 {
340 LLVMValueRef pointer;
341 pointer = build_gep0(ctx, base_ptr, index);
342 if (uniform)
343 LLVMSetMetadata(pointer, ctx->uniform_md_kind, ctx->empty_md);
344 return LLVMBuildLoad(ctx->builder, pointer, "");
345 }
346
347 static LLVMValueRef build_indexed_load_const(struct nir_to_llvm_context *ctx,
348 LLVMValueRef base_ptr, LLVMValueRef index)
349 {
350 LLVMValueRef result = build_indexed_load(ctx, base_ptr, index, true);
351 LLVMSetMetadata(result, ctx->invariant_load_md_kind, ctx->empty_md);
352 return result;
353 }
354
355 static void set_userdata_location(struct ac_userdata_info *ud_info, uint8_t sgpr_idx, uint8_t num_sgprs)
356 {
357 ud_info->sgpr_idx = sgpr_idx;
358 ud_info->num_sgprs = num_sgprs;
359 ud_info->indirect = false;
360 ud_info->indirect_offset = 0;
361 }
362
363 static void set_userdata_location_shader(struct nir_to_llvm_context *ctx,
364 int idx, uint8_t sgpr_idx, uint8_t num_sgprs)
365 {
366 set_userdata_location(&ctx->shader_info->user_sgprs_locs.shader_data[idx], sgpr_idx, num_sgprs);
367 }
368
369 #if 0
370 static void set_userdata_location_indirect(struct ac_userdata_info *ud_info, uint8_t sgpr_idx, uint8_t num_sgprs,
371 uint32_t indirect_offset)
372 {
373 ud_info->sgpr_idx = sgpr_idx;
374 ud_info->num_sgprs = num_sgprs;
375 ud_info->indirect = true;
376 ud_info->indirect_offset = indirect_offset;
377 }
378 #endif
379
380 static void create_function(struct nir_to_llvm_context *ctx)
381 {
382 LLVMTypeRef arg_types[23];
383 unsigned arg_idx = 0;
384 unsigned array_params_mask = 0;
385 unsigned sgpr_count = 0, user_sgpr_count;
386 unsigned i;
387 unsigned num_sets = ctx->options->layout ? ctx->options->layout->num_sets : 0;
388 unsigned user_sgpr_idx;
389 bool need_push_constants;
390
391 need_push_constants = true;
392 if (!ctx->options->layout)
393 need_push_constants = false;
394 else if (!ctx->options->layout->push_constant_size &&
395 !ctx->options->layout->dynamic_offset_count)
396 need_push_constants = false;
397
398 /* 1 for each descriptor set */
399 for (unsigned i = 0; i < num_sets; ++i) {
400 if (ctx->options->layout->set[i].layout->shader_stages & (1 << ctx->stage)) {
401 array_params_mask |= (1 << arg_idx);
402 arg_types[arg_idx++] = const_array(ctx->i8, 1024 * 1024);
403 }
404 }
405
406 if (need_push_constants) {
407 /* 1 for push constants and dynamic descriptors */
408 array_params_mask |= (1 << arg_idx);
409 arg_types[arg_idx++] = const_array(ctx->i8, 1024 * 1024);
410 }
411
412 switch (ctx->stage) {
413 case MESA_SHADER_COMPUTE:
414 arg_types[arg_idx++] = LLVMVectorType(ctx->i32, 3); /* grid size */
415 user_sgpr_count = arg_idx;
416 arg_types[arg_idx++] = LLVMVectorType(ctx->i32, 3);
417 arg_types[arg_idx++] = ctx->i32;
418 sgpr_count = arg_idx;
419
420 arg_types[arg_idx++] = LLVMVectorType(ctx->i32, 3);
421 break;
422 case MESA_SHADER_VERTEX:
423 arg_types[arg_idx++] = const_array(ctx->v16i8, 16); /* vertex buffers */
424 arg_types[arg_idx++] = ctx->i32; // base vertex
425 arg_types[arg_idx++] = ctx->i32; // start instance
426 user_sgpr_count = sgpr_count = arg_idx;
427 arg_types[arg_idx++] = ctx->i32; // vertex id
428 arg_types[arg_idx++] = ctx->i32; // rel auto id
429 arg_types[arg_idx++] = ctx->i32; // vs prim id
430 arg_types[arg_idx++] = ctx->i32; // instance id
431 break;
432 case MESA_SHADER_FRAGMENT:
433 arg_types[arg_idx++] = const_array(ctx->f32, 32); /* sample positions */
434 user_sgpr_count = arg_idx;
435 arg_types[arg_idx++] = ctx->i32; /* prim mask */
436 sgpr_count = arg_idx;
437 arg_types[arg_idx++] = ctx->v2i32; /* persp sample */
438 arg_types[arg_idx++] = ctx->v2i32; /* persp center */
439 arg_types[arg_idx++] = ctx->v2i32; /* persp centroid */
440 arg_types[arg_idx++] = ctx->v3i32; /* persp pull model */
441 arg_types[arg_idx++] = ctx->v2i32; /* linear sample */
442 arg_types[arg_idx++] = ctx->v2i32; /* linear center */
443 arg_types[arg_idx++] = ctx->v2i32; /* linear centroid */
444 arg_types[arg_idx++] = ctx->f32; /* line stipple tex */
445 arg_types[arg_idx++] = ctx->f32; /* pos x float */
446 arg_types[arg_idx++] = ctx->f32; /* pos y float */
447 arg_types[arg_idx++] = ctx->f32; /* pos z float */
448 arg_types[arg_idx++] = ctx->f32; /* pos w float */
449 arg_types[arg_idx++] = ctx->i32; /* front face */
450 arg_types[arg_idx++] = ctx->i32; /* ancillary */
451 arg_types[arg_idx++] = ctx->f32; /* sample coverage */
452 arg_types[arg_idx++] = ctx->i32; /* fixed pt */
453 break;
454 default:
455 unreachable("Shader stage not implemented");
456 }
457
458 ctx->main_function = create_llvm_function(
459 ctx->context, ctx->module, ctx->builder, NULL, 0, arg_types,
460 arg_idx, array_params_mask, sgpr_count, ctx->options->unsafe_math);
461 set_llvm_calling_convention(ctx->main_function, ctx->stage);
462
463
464 ctx->shader_info->num_input_sgprs = 0;
465 ctx->shader_info->num_input_vgprs = 0;
466
467 for (i = 0; i < user_sgpr_count; i++)
468 ctx->shader_info->num_user_sgprs += llvm_get_type_size(arg_types[i]) / 4;
469
470 ctx->shader_info->num_input_sgprs = ctx->shader_info->num_user_sgprs;
471 for (; i < sgpr_count; i++)
472 ctx->shader_info->num_input_sgprs += llvm_get_type_size(arg_types[i]) / 4;
473
474 if (ctx->stage != MESA_SHADER_FRAGMENT)
475 for (; i < arg_idx; ++i)
476 ctx->shader_info->num_input_vgprs += llvm_get_type_size(arg_types[i]) / 4;
477
478 arg_idx = 0;
479 user_sgpr_idx = 0;
480 for (unsigned i = 0; i < num_sets; ++i) {
481 if (ctx->options->layout->set[i].layout->shader_stages & (1 << ctx->stage)) {
482 set_userdata_location(&ctx->shader_info->user_sgprs_locs.descriptor_sets[i], user_sgpr_idx, 2);
483 user_sgpr_idx += 2;
484 ctx->descriptor_sets[i] =
485 LLVMGetParam(ctx->main_function, arg_idx++);
486 } else
487 ctx->descriptor_sets[i] = NULL;
488 }
489
490 if (need_push_constants) {
491 ctx->push_constants = LLVMGetParam(ctx->main_function, arg_idx++);
492 set_userdata_location_shader(ctx, AC_UD_PUSH_CONSTANTS, user_sgpr_idx, 2);
493 user_sgpr_idx += 2;
494 }
495
496 switch (ctx->stage) {
497 case MESA_SHADER_COMPUTE:
498 set_userdata_location_shader(ctx, AC_UD_CS_GRID_SIZE, user_sgpr_idx, 3);
499 user_sgpr_idx += 3;
500 ctx->num_work_groups =
501 LLVMGetParam(ctx->main_function, arg_idx++);
502 ctx->workgroup_ids =
503 LLVMGetParam(ctx->main_function, arg_idx++);
504 ctx->tg_size =
505 LLVMGetParam(ctx->main_function, arg_idx++);
506 ctx->local_invocation_ids =
507 LLVMGetParam(ctx->main_function, arg_idx++);
508 break;
509 case MESA_SHADER_VERTEX:
510 set_userdata_location_shader(ctx, AC_UD_VS_VERTEX_BUFFERS, user_sgpr_idx, 2);
511 user_sgpr_idx += 2;
512 ctx->vertex_buffers = LLVMGetParam(ctx->main_function, arg_idx++);
513 set_userdata_location_shader(ctx, AC_UD_VS_BASE_VERTEX_START_INSTANCE, user_sgpr_idx, 2);
514 user_sgpr_idx += 2;
515 ctx->base_vertex = LLVMGetParam(ctx->main_function, arg_idx++);
516 ctx->start_instance = LLVMGetParam(ctx->main_function, arg_idx++);
517 ctx->vertex_id = LLVMGetParam(ctx->main_function, arg_idx++);
518 ctx->rel_auto_id = LLVMGetParam(ctx->main_function, arg_idx++);
519 ctx->vs_prim_id = LLVMGetParam(ctx->main_function, arg_idx++);
520 ctx->instance_id = LLVMGetParam(ctx->main_function, arg_idx++);
521 break;
522 case MESA_SHADER_FRAGMENT:
523 set_userdata_location_shader(ctx, AC_UD_PS_SAMPLE_POS, user_sgpr_idx, 2);
524 user_sgpr_idx += 2;
525 ctx->sample_positions = LLVMGetParam(ctx->main_function, arg_idx++);
526 ctx->prim_mask = LLVMGetParam(ctx->main_function, arg_idx++);
527 ctx->persp_sample = LLVMGetParam(ctx->main_function, arg_idx++);
528 ctx->persp_center = LLVMGetParam(ctx->main_function, arg_idx++);
529 ctx->persp_centroid = LLVMGetParam(ctx->main_function, arg_idx++);
530 arg_idx++;
531 ctx->linear_sample = LLVMGetParam(ctx->main_function, arg_idx++);
532 ctx->linear_center = LLVMGetParam(ctx->main_function, arg_idx++);
533 ctx->linear_centroid = LLVMGetParam(ctx->main_function, arg_idx++);
534 arg_idx++; /* line stipple */
535 ctx->frag_pos[0] = LLVMGetParam(ctx->main_function, arg_idx++);
536 ctx->frag_pos[1] = LLVMGetParam(ctx->main_function, arg_idx++);
537 ctx->frag_pos[2] = LLVMGetParam(ctx->main_function, arg_idx++);
538 ctx->frag_pos[3] = LLVMGetParam(ctx->main_function, arg_idx++);
539 ctx->front_face = LLVMGetParam(ctx->main_function, arg_idx++);
540 ctx->ancillary = LLVMGetParam(ctx->main_function, arg_idx++);
541 break;
542 default:
543 unreachable("Shader stage not implemented");
544 }
545 }
546
547 static void setup_types(struct nir_to_llvm_context *ctx)
548 {
549 LLVMValueRef args[4];
550
551 ctx->voidt = LLVMVoidTypeInContext(ctx->context);
552 ctx->i1 = LLVMIntTypeInContext(ctx->context, 1);
553 ctx->i8 = LLVMIntTypeInContext(ctx->context, 8);
554 ctx->i16 = LLVMIntTypeInContext(ctx->context, 16);
555 ctx->i32 = LLVMIntTypeInContext(ctx->context, 32);
556 ctx->i64 = LLVMIntTypeInContext(ctx->context, 64);
557 ctx->v2i32 = LLVMVectorType(ctx->i32, 2);
558 ctx->v3i32 = LLVMVectorType(ctx->i32, 3);
559 ctx->v4i32 = LLVMVectorType(ctx->i32, 4);
560 ctx->v8i32 = LLVMVectorType(ctx->i32, 8);
561 ctx->f32 = LLVMFloatTypeInContext(ctx->context);
562 ctx->f16 = LLVMHalfTypeInContext(ctx->context);
563 ctx->v2f32 = LLVMVectorType(ctx->f32, 2);
564 ctx->v4f32 = LLVMVectorType(ctx->f32, 4);
565 ctx->v16i8 = LLVMVectorType(ctx->i8, 16);
566
567 ctx->i32zero = LLVMConstInt(ctx->i32, 0, false);
568 ctx->i32one = LLVMConstInt(ctx->i32, 1, false);
569 ctx->f32zero = LLVMConstReal(ctx->f32, 0.0);
570 ctx->f32one = LLVMConstReal(ctx->f32, 1.0);
571
572 args[0] = ctx->f32zero;
573 args[1] = ctx->f32zero;
574 args[2] = ctx->f32zero;
575 args[3] = ctx->f32one;
576 ctx->v4f32empty = LLVMConstVector(args, 4);
577
578 ctx->range_md_kind = LLVMGetMDKindIDInContext(ctx->context,
579 "range", 5);
580 ctx->invariant_load_md_kind = LLVMGetMDKindIDInContext(ctx->context,
581 "invariant.load", 14);
582 ctx->uniform_md_kind =
583 LLVMGetMDKindIDInContext(ctx->context, "amdgpu.uniform", 14);
584 ctx->empty_md = LLVMMDNodeInContext(ctx->context, NULL, 0);
585
586 ctx->fpmath_md_kind = LLVMGetMDKindIDInContext(ctx->context, "fpmath", 6);
587
588 args[0] = LLVMConstReal(ctx->f32, 2.5);
589 ctx->fpmath_md_2p5_ulp = LLVMMDNodeInContext(ctx->context, args, 1);
590 }
591
592 static int get_llvm_num_components(LLVMValueRef value)
593 {
594 LLVMTypeRef type = LLVMTypeOf(value);
595 unsigned num_components = LLVMGetTypeKind(type) == LLVMVectorTypeKind
596 ? LLVMGetVectorSize(type)
597 : 1;
598 return num_components;
599 }
600
601 static LLVMValueRef llvm_extract_elem(struct nir_to_llvm_context *ctx,
602 LLVMValueRef value,
603 int index)
604 {
605 int count = get_llvm_num_components(value);
606
607 assert(index < count);
608 if (count == 1)
609 return value;
610
611 return LLVMBuildExtractElement(ctx->builder, value,
612 LLVMConstInt(ctx->i32, index, false), "");
613 }
614
615 static LLVMValueRef trim_vector(struct nir_to_llvm_context *ctx,
616 LLVMValueRef value, unsigned count)
617 {
618 unsigned num_components = get_llvm_num_components(value);
619 if (count == num_components)
620 return value;
621
622 LLVMValueRef masks[] = {
623 LLVMConstInt(ctx->i32, 0, false), LLVMConstInt(ctx->i32, 1, false),
624 LLVMConstInt(ctx->i32, 2, false), LLVMConstInt(ctx->i32, 3, false)};
625
626 if (count == 1)
627 return LLVMBuildExtractElement(ctx->builder, value, masks[0],
628 "");
629
630 LLVMValueRef swizzle = LLVMConstVector(masks, count);
631 return LLVMBuildShuffleVector(ctx->builder, value, value, swizzle, "");
632 }
633
634 static LLVMValueRef
635 build_gather_values_extended(struct nir_to_llvm_context *ctx,
636 LLVMValueRef *values,
637 unsigned value_count,
638 unsigned value_stride,
639 bool load)
640 {
641 LLVMBuilderRef builder = ctx->builder;
642 LLVMValueRef vec;
643 unsigned i;
644
645
646 if (value_count == 1) {
647 if (load)
648 return LLVMBuildLoad(builder, values[0], "");
649 return values[0];
650 } else if (!value_count)
651 unreachable("value_count is 0");
652
653 for (i = 0; i < value_count; i++) {
654 LLVMValueRef value = values[i * value_stride];
655 if (load)
656 value = LLVMBuildLoad(builder, value, "");
657
658 if (!i)
659 vec = LLVMGetUndef( LLVMVectorType(LLVMTypeOf(value), value_count));
660 LLVMValueRef index = LLVMConstInt(ctx->i32, i, false);
661 vec = LLVMBuildInsertElement(builder, vec, value, index, "");
662 }
663 return vec;
664 }
665
666
667 static void
668 build_store_values_extended(struct nir_to_llvm_context *ctx,
669 LLVMValueRef *values,
670 unsigned value_count,
671 unsigned value_stride,
672 LLVMValueRef vec)
673 {
674 LLVMBuilderRef builder = ctx->builder;
675 unsigned i;
676
677 if (value_count == 1) {
678 LLVMBuildStore(builder, vec, values[0]);
679 return;
680 }
681
682 for (i = 0; i < value_count; i++) {
683 LLVMValueRef ptr = values[i * value_stride];
684 LLVMValueRef index = LLVMConstInt(ctx->i32, i, false);
685 LLVMValueRef value = LLVMBuildExtractElement(builder, vec, index, "");
686 LLVMBuildStore(builder, value, ptr);
687 }
688 }
689
690 static LLVMValueRef
691 build_gather_values(struct nir_to_llvm_context *ctx,
692 LLVMValueRef *values,
693 unsigned value_count)
694 {
695 return build_gather_values_extended(ctx, values, value_count, 1, false);
696 }
697
698 static LLVMTypeRef get_def_type(struct nir_to_llvm_context *ctx,
699 nir_ssa_def *def)
700 {
701 LLVMTypeRef type = LLVMIntTypeInContext(ctx->context, def->bit_size);
702 if (def->num_components > 1) {
703 type = LLVMVectorType(type, def->num_components);
704 }
705 return type;
706 }
707
708 static LLVMValueRef get_src(struct nir_to_llvm_context *ctx, nir_src src)
709 {
710 assert(src.is_ssa);
711 struct hash_entry *entry = _mesa_hash_table_search(ctx->defs, src.ssa);
712 return (LLVMValueRef)entry->data;
713 }
714
715
716 static LLVMBasicBlockRef get_block(struct nir_to_llvm_context *ctx,
717 struct nir_block *b)
718 {
719 struct hash_entry *entry = _mesa_hash_table_search(ctx->defs, b);
720 return (LLVMBasicBlockRef)entry->data;
721 }
722
723 static LLVMValueRef get_alu_src(struct nir_to_llvm_context *ctx,
724 nir_alu_src src,
725 unsigned num_components)
726 {
727 LLVMValueRef value = get_src(ctx, src.src);
728 bool need_swizzle = false;
729
730 assert(value);
731 LLVMTypeRef type = LLVMTypeOf(value);
732 unsigned src_components = LLVMGetTypeKind(type) == LLVMVectorTypeKind
733 ? LLVMGetVectorSize(type)
734 : 1;
735
736 for (unsigned i = 0; i < num_components; ++i) {
737 assert(src.swizzle[i] < src_components);
738 if (src.swizzle[i] != i)
739 need_swizzle = true;
740 }
741
742 if (need_swizzle || num_components != src_components) {
743 LLVMValueRef masks[] = {
744 LLVMConstInt(ctx->i32, src.swizzle[0], false),
745 LLVMConstInt(ctx->i32, src.swizzle[1], false),
746 LLVMConstInt(ctx->i32, src.swizzle[2], false),
747 LLVMConstInt(ctx->i32, src.swizzle[3], false)};
748
749 if (src_components > 1 && num_components == 1) {
750 value = LLVMBuildExtractElement(ctx->builder, value,
751 masks[0], "");
752 } else if (src_components == 1 && num_components > 1) {
753 LLVMValueRef values[] = {value, value, value, value};
754 value = build_gather_values(ctx, values, num_components);
755 } else {
756 LLVMValueRef swizzle = LLVMConstVector(masks, num_components);
757 value = LLVMBuildShuffleVector(ctx->builder, value, value,
758 swizzle, "");
759 }
760 }
761 assert(!src.negate);
762 assert(!src.abs);
763 return value;
764 }
765
766 static LLVMValueRef emit_int_cmp(struct nir_to_llvm_context *ctx,
767 LLVMIntPredicate pred, LLVMValueRef src0,
768 LLVMValueRef src1)
769 {
770 LLVMValueRef result = LLVMBuildICmp(ctx->builder, pred, src0, src1, "");
771 return LLVMBuildSelect(ctx->builder, result,
772 LLVMConstInt(ctx->i32, 0xFFFFFFFF, false),
773 LLVMConstInt(ctx->i32, 0, false), "");
774 }
775
776 static LLVMValueRef emit_float_cmp(struct nir_to_llvm_context *ctx,
777 LLVMRealPredicate pred, LLVMValueRef src0,
778 LLVMValueRef src1)
779 {
780 LLVMValueRef result;
781 src0 = to_float(ctx, src0);
782 src1 = to_float(ctx, src1);
783 result = LLVMBuildFCmp(ctx->builder, pred, src0, src1, "");
784 return LLVMBuildSelect(ctx->builder, result,
785 LLVMConstInt(ctx->i32, 0xFFFFFFFF, false),
786 LLVMConstInt(ctx->i32, 0, false), "");
787 }
788
789 static LLVMValueRef emit_intrin_1f_param(struct nir_to_llvm_context *ctx,
790 const char *intrin,
791 LLVMValueRef src0)
792 {
793 LLVMValueRef params[] = {
794 to_float(ctx, src0),
795 };
796 return ac_emit_llvm_intrinsic(&ctx->ac, intrin, ctx->f32, params, 1, AC_FUNC_ATTR_READNONE);
797 }
798
799 static LLVMValueRef emit_intrin_2f_param(struct nir_to_llvm_context *ctx,
800 const char *intrin,
801 LLVMValueRef src0, LLVMValueRef src1)
802 {
803 LLVMValueRef params[] = {
804 to_float(ctx, src0),
805 to_float(ctx, src1),
806 };
807 return ac_emit_llvm_intrinsic(&ctx->ac, intrin, ctx->f32, params, 2, AC_FUNC_ATTR_READNONE);
808 }
809
810 static LLVMValueRef emit_intrin_3f_param(struct nir_to_llvm_context *ctx,
811 const char *intrin,
812 LLVMValueRef src0, LLVMValueRef src1, LLVMValueRef src2)
813 {
814 LLVMValueRef params[] = {
815 to_float(ctx, src0),
816 to_float(ctx, src1),
817 to_float(ctx, src2),
818 };
819 return ac_emit_llvm_intrinsic(&ctx->ac, intrin, ctx->f32, params, 3, AC_FUNC_ATTR_READNONE);
820 }
821
822 static LLVMValueRef emit_bcsel(struct nir_to_llvm_context *ctx,
823 LLVMValueRef src0, LLVMValueRef src1, LLVMValueRef src2)
824 {
825 LLVMValueRef v = LLVMBuildICmp(ctx->builder, LLVMIntNE, src0,
826 ctx->i32zero, "");
827 return LLVMBuildSelect(ctx->builder, v, src1, src2, "");
828 }
829
830 static LLVMValueRef emit_find_lsb(struct nir_to_llvm_context *ctx,
831 LLVMValueRef src0)
832 {
833 LLVMValueRef params[2] = {
834 src0,
835
836 /* The value of 1 means that ffs(x=0) = undef, so LLVM won't
837 * add special code to check for x=0. The reason is that
838 * the LLVM behavior for x=0 is different from what we
839 * need here.
840 *
841 * The hardware already implements the correct behavior.
842 */
843 LLVMConstInt(ctx->i32, 1, false),
844 };
845 return ac_emit_llvm_intrinsic(&ctx->ac, "llvm.cttz.i32", ctx->i32, params, 2, AC_FUNC_ATTR_READNONE);
846 }
847
848 static LLVMValueRef emit_ifind_msb(struct nir_to_llvm_context *ctx,
849 LLVMValueRef src0)
850 {
851 LLVMValueRef msb = ac_emit_llvm_intrinsic(&ctx->ac, "llvm.AMDGPU.flbit.i32",
852 ctx->i32, &src0, 1,
853 AC_FUNC_ATTR_READNONE);
854
855 /* The HW returns the last bit index from MSB, but NIR wants
856 * the index from LSB. Invert it by doing "31 - msb". */
857 msb = LLVMBuildSub(ctx->builder, LLVMConstInt(ctx->i32, 31, false),
858 msb, "");
859
860 LLVMValueRef all_ones = LLVMConstInt(ctx->i32, -1, true);
861 LLVMValueRef cond = LLVMBuildOr(ctx->builder,
862 LLVMBuildICmp(ctx->builder, LLVMIntEQ,
863 src0, ctx->i32zero, ""),
864 LLVMBuildICmp(ctx->builder, LLVMIntEQ,
865 src0, all_ones, ""), "");
866
867 return LLVMBuildSelect(ctx->builder, cond, all_ones, msb, "");
868 }
869
870 static LLVMValueRef emit_ufind_msb(struct nir_to_llvm_context *ctx,
871 LLVMValueRef src0)
872 {
873 LLVMValueRef args[2] = {
874 src0,
875 ctx->i32one,
876 };
877 LLVMValueRef msb = ac_emit_llvm_intrinsic(&ctx->ac, "llvm.ctlz.i32",
878 ctx->i32, args, ARRAY_SIZE(args),
879 AC_FUNC_ATTR_READNONE);
880
881 /* The HW returns the last bit index from MSB, but NIR wants
882 * the index from LSB. Invert it by doing "31 - msb". */
883 msb = LLVMBuildSub(ctx->builder, LLVMConstInt(ctx->i32, 31, false),
884 msb, "");
885
886 return LLVMBuildSelect(ctx->builder,
887 LLVMBuildICmp(ctx->builder, LLVMIntEQ, src0,
888 ctx->i32zero, ""),
889 LLVMConstInt(ctx->i32, -1, true), msb, "");
890 }
891
892 static LLVMValueRef emit_minmax_int(struct nir_to_llvm_context *ctx,
893 LLVMIntPredicate pred,
894 LLVMValueRef src0, LLVMValueRef src1)
895 {
896 return LLVMBuildSelect(ctx->builder,
897 LLVMBuildICmp(ctx->builder, pred, src0, src1, ""),
898 src0,
899 src1, "");
900
901 }
902 static LLVMValueRef emit_iabs(struct nir_to_llvm_context *ctx,
903 LLVMValueRef src0)
904 {
905 return emit_minmax_int(ctx, LLVMIntSGT, src0,
906 LLVMBuildNeg(ctx->builder, src0, ""));
907 }
908
909 static LLVMValueRef emit_fsign(struct nir_to_llvm_context *ctx,
910 LLVMValueRef src0)
911 {
912 LLVMValueRef cmp, val;
913
914 cmp = LLVMBuildFCmp(ctx->builder, LLVMRealOGT, src0, ctx->f32zero, "");
915 val = LLVMBuildSelect(ctx->builder, cmp, ctx->f32one, src0, "");
916 cmp = LLVMBuildFCmp(ctx->builder, LLVMRealOGE, val, ctx->f32zero, "");
917 val = LLVMBuildSelect(ctx->builder, cmp, val, LLVMConstReal(ctx->f32, -1.0), "");
918 return val;
919 }
920
921 static LLVMValueRef emit_isign(struct nir_to_llvm_context *ctx,
922 LLVMValueRef src0)
923 {
924 LLVMValueRef cmp, val;
925
926 cmp = LLVMBuildICmp(ctx->builder, LLVMIntSGT, src0, ctx->i32zero, "");
927 val = LLVMBuildSelect(ctx->builder, cmp, ctx->i32one, src0, "");
928 cmp = LLVMBuildICmp(ctx->builder, LLVMIntSGE, val, ctx->i32zero, "");
929 val = LLVMBuildSelect(ctx->builder, cmp, val, LLVMConstInt(ctx->i32, -1, true), "");
930 return val;
931 }
932
933 static LLVMValueRef emit_ffract(struct nir_to_llvm_context *ctx,
934 LLVMValueRef src0)
935 {
936 const char *intr = "llvm.floor.f32";
937 LLVMValueRef fsrc0 = to_float(ctx, src0);
938 LLVMValueRef params[] = {
939 fsrc0,
940 };
941 LLVMValueRef floor = ac_emit_llvm_intrinsic(&ctx->ac, intr,
942 ctx->f32, params, 1,
943 AC_FUNC_ATTR_READNONE);
944 return LLVMBuildFSub(ctx->builder, fsrc0, floor, "");
945 }
946
947 static LLVMValueRef emit_uint_carry(struct nir_to_llvm_context *ctx,
948 const char *intrin,
949 LLVMValueRef src0, LLVMValueRef src1)
950 {
951 LLVMTypeRef ret_type;
952 LLVMTypeRef types[] = { ctx->i32, ctx->i1 };
953 LLVMValueRef res;
954 LLVMValueRef params[] = { src0, src1 };
955 ret_type = LLVMStructTypeInContext(ctx->context, types,
956 2, true);
957
958 res = ac_emit_llvm_intrinsic(&ctx->ac, intrin, ret_type,
959 params, 2, AC_FUNC_ATTR_READNONE);
960
961 res = LLVMBuildExtractValue(ctx->builder, res, 1, "");
962 res = LLVMBuildZExt(ctx->builder, res, ctx->i32, "");
963 return res;
964 }
965
966 static LLVMValueRef emit_b2f(struct nir_to_llvm_context *ctx,
967 LLVMValueRef src0)
968 {
969 return LLVMBuildAnd(ctx->builder, src0, LLVMBuildBitCast(ctx->builder, LLVMConstReal(ctx->f32, 1.0), ctx->i32, ""), "");
970 }
971
972 static LLVMValueRef emit_umul_high(struct nir_to_llvm_context *ctx,
973 LLVMValueRef src0, LLVMValueRef src1)
974 {
975 LLVMValueRef dst64, result;
976 src0 = LLVMBuildZExt(ctx->builder, src0, ctx->i64, "");
977 src1 = LLVMBuildZExt(ctx->builder, src1, ctx->i64, "");
978
979 dst64 = LLVMBuildMul(ctx->builder, src0, src1, "");
980 dst64 = LLVMBuildLShr(ctx->builder, dst64, LLVMConstInt(ctx->i64, 32, false), "");
981 result = LLVMBuildTrunc(ctx->builder, dst64, ctx->i32, "");
982 return result;
983 }
984
985 static LLVMValueRef emit_imul_high(struct nir_to_llvm_context *ctx,
986 LLVMValueRef src0, LLVMValueRef src1)
987 {
988 LLVMValueRef dst64, result;
989 src0 = LLVMBuildSExt(ctx->builder, src0, ctx->i64, "");
990 src1 = LLVMBuildSExt(ctx->builder, src1, ctx->i64, "");
991
992 dst64 = LLVMBuildMul(ctx->builder, src0, src1, "");
993 dst64 = LLVMBuildAShr(ctx->builder, dst64, LLVMConstInt(ctx->i64, 32, false), "");
994 result = LLVMBuildTrunc(ctx->builder, dst64, ctx->i32, "");
995 return result;
996 }
997
998 static LLVMValueRef emit_bitfield_extract(struct nir_to_llvm_context *ctx,
999 const char *intrin,
1000 LLVMValueRef srcs[3])
1001 {
1002 LLVMValueRef result;
1003 LLVMValueRef icond = LLVMBuildICmp(ctx->builder, LLVMIntEQ, srcs[2], LLVMConstInt(ctx->i32, 32, false), "");
1004 result = ac_emit_llvm_intrinsic(&ctx->ac, intrin, ctx->i32, srcs, 3, AC_FUNC_ATTR_READNONE);
1005
1006 result = LLVMBuildSelect(ctx->builder, icond, srcs[0], result, "");
1007 return result;
1008 }
1009
1010 static LLVMValueRef emit_bitfield_insert(struct nir_to_llvm_context *ctx,
1011 LLVMValueRef src0, LLVMValueRef src1,
1012 LLVMValueRef src2, LLVMValueRef src3)
1013 {
1014 LLVMValueRef bfi_args[3], result;
1015
1016 bfi_args[0] = LLVMBuildShl(ctx->builder,
1017 LLVMBuildSub(ctx->builder,
1018 LLVMBuildShl(ctx->builder,
1019 ctx->i32one,
1020 src3, ""),
1021 ctx->i32one, ""),
1022 src2, "");
1023 bfi_args[1] = LLVMBuildShl(ctx->builder, src1, src2, "");
1024 bfi_args[2] = src0;
1025
1026 LLVMValueRef icond = LLVMBuildICmp(ctx->builder, LLVMIntEQ, src3, LLVMConstInt(ctx->i32, 32, false), "");
1027
1028 /* Calculate:
1029 * (arg0 & arg1) | (~arg0 & arg2) = arg2 ^ (arg0 & (arg1 ^ arg2)
1030 * Use the right-hand side, which the LLVM backend can convert to V_BFI.
1031 */
1032 result = LLVMBuildXor(ctx->builder, bfi_args[2],
1033 LLVMBuildAnd(ctx->builder, bfi_args[0],
1034 LLVMBuildXor(ctx->builder, bfi_args[1], bfi_args[2], ""), ""), "");
1035
1036 result = LLVMBuildSelect(ctx->builder, icond, src1, result, "");
1037 return result;
1038 }
1039
1040 static LLVMValueRef emit_pack_half_2x16(struct nir_to_llvm_context *ctx,
1041 LLVMValueRef src0)
1042 {
1043 LLVMValueRef const16 = LLVMConstInt(ctx->i32, 16, false);
1044 int i;
1045 LLVMValueRef comp[2];
1046
1047 src0 = to_float(ctx, src0);
1048 comp[0] = LLVMBuildExtractElement(ctx->builder, src0, ctx->i32zero, "");
1049 comp[1] = LLVMBuildExtractElement(ctx->builder, src0, ctx->i32one, "");
1050 for (i = 0; i < 2; i++) {
1051 comp[i] = LLVMBuildFPTrunc(ctx->builder, comp[i], ctx->f16, "");
1052 comp[i] = LLVMBuildBitCast(ctx->builder, comp[i], ctx->i16, "");
1053 comp[i] = LLVMBuildZExt(ctx->builder, comp[i], ctx->i32, "");
1054 }
1055
1056 comp[1] = LLVMBuildShl(ctx->builder, comp[1], const16, "");
1057 comp[0] = LLVMBuildOr(ctx->builder, comp[0], comp[1], "");
1058
1059 return comp[0];
1060 }
1061
1062 static LLVMValueRef emit_unpack_half_2x16(struct nir_to_llvm_context *ctx,
1063 LLVMValueRef src0)
1064 {
1065 LLVMValueRef const16 = LLVMConstInt(ctx->i32, 16, false);
1066 LLVMValueRef temps[2], result, val;
1067 int i;
1068
1069 for (i = 0; i < 2; i++) {
1070 val = i == 1 ? LLVMBuildLShr(ctx->builder, src0, const16, "") : src0;
1071 val = LLVMBuildTrunc(ctx->builder, val, ctx->i16, "");
1072 val = LLVMBuildBitCast(ctx->builder, val, ctx->f16, "");
1073 temps[i] = LLVMBuildFPExt(ctx->builder, val, ctx->f32, "");
1074 }
1075
1076 result = LLVMBuildInsertElement(ctx->builder, LLVMGetUndef(ctx->v2f32), temps[0],
1077 ctx->i32zero, "");
1078 result = LLVMBuildInsertElement(ctx->builder, result, temps[1],
1079 ctx->i32one, "");
1080 return result;
1081 }
1082
1083 /**
1084 * Set range metadata on an instruction. This can only be used on load and
1085 * call instructions. If you know an instruction can only produce the values
1086 * 0, 1, 2, you would do set_range_metadata(value, 0, 3);
1087 * \p lo is the minimum value inclusive.
1088 * \p hi is the maximum value exclusive.
1089 */
1090 static void set_range_metadata(struct nir_to_llvm_context *ctx,
1091 LLVMValueRef value, unsigned lo, unsigned hi)
1092 {
1093 LLVMValueRef range_md, md_args[2];
1094 LLVMTypeRef type = LLVMTypeOf(value);
1095 LLVMContextRef context = LLVMGetTypeContext(type);
1096
1097 md_args[0] = LLVMConstInt(type, lo, false);
1098 md_args[1] = LLVMConstInt(type, hi, false);
1099 range_md = LLVMMDNodeInContext(context, md_args, 2);
1100 LLVMSetMetadata(value, ctx->range_md_kind, range_md);
1101 }
1102
1103 static LLVMValueRef get_thread_id(struct nir_to_llvm_context *ctx)
1104 {
1105 LLVMValueRef tid;
1106 LLVMValueRef tid_args[2];
1107 tid_args[0] = LLVMConstInt(ctx->i32, 0xffffffff, false);
1108 tid_args[1] = ctx->i32zero;
1109 tid_args[1] = ac_emit_llvm_intrinsic(&ctx->ac,
1110 "llvm.amdgcn.mbcnt.lo", ctx->i32,
1111 tid_args, 2, AC_FUNC_ATTR_READNONE);
1112
1113 tid = ac_emit_llvm_intrinsic(&ctx->ac,
1114 "llvm.amdgcn.mbcnt.hi", ctx->i32,
1115 tid_args, 2, AC_FUNC_ATTR_READNONE);
1116 set_range_metadata(ctx, tid, 0, 64);
1117 return tid;
1118 }
1119
1120 /*
1121 * SI implements derivatives using the local data store (LDS)
1122 * All writes to the LDS happen in all executing threads at
1123 * the same time. TID is the Thread ID for the current
1124 * thread and is a value between 0 and 63, representing
1125 * the thread's position in the wavefront.
1126 *
1127 * For the pixel shader threads are grouped into quads of four pixels.
1128 * The TIDs of the pixels of a quad are:
1129 *
1130 * +------+------+
1131 * |4n + 0|4n + 1|
1132 * +------+------+
1133 * |4n + 2|4n + 3|
1134 * +------+------+
1135 *
1136 * So, masking the TID with 0xfffffffc yields the TID of the top left pixel
1137 * of the quad, masking with 0xfffffffd yields the TID of the top pixel of
1138 * the current pixel's column, and masking with 0xfffffffe yields the TID
1139 * of the left pixel of the current pixel's row.
1140 *
1141 * Adding 1 yields the TID of the pixel to the right of the left pixel, and
1142 * adding 2 yields the TID of the pixel below the top pixel.
1143 */
1144 /* masks for thread ID. */
1145 #define TID_MASK_TOP_LEFT 0xfffffffc
1146 #define TID_MASK_TOP 0xfffffffd
1147 #define TID_MASK_LEFT 0xfffffffe
1148 static LLVMValueRef emit_ddxy(struct nir_to_llvm_context *ctx,
1149 nir_op op,
1150 LLVMValueRef src0)
1151 {
1152 LLVMValueRef tl, trbl, result;
1153 LLVMValueRef tl_tid, trbl_tid;
1154 LLVMValueRef args[2];
1155 LLVMValueRef thread_id;
1156 unsigned mask;
1157 int idx;
1158 ctx->has_ddxy = true;
1159
1160 if (!ctx->lds && !ctx->has_ds_bpermute)
1161 ctx->lds = LLVMAddGlobalInAddressSpace(ctx->module,
1162 LLVMArrayType(ctx->i32, 64),
1163 "ddxy_lds", LOCAL_ADDR_SPACE);
1164
1165 thread_id = get_thread_id(ctx);
1166 if (op == nir_op_fddx_fine || op == nir_op_fddx)
1167 mask = TID_MASK_LEFT;
1168 else if (op == nir_op_fddy_fine || op == nir_op_fddy)
1169 mask = TID_MASK_TOP;
1170 else
1171 mask = TID_MASK_TOP_LEFT;
1172
1173 tl_tid = LLVMBuildAnd(ctx->builder, thread_id,
1174 LLVMConstInt(ctx->i32, mask, false), "");
1175 /* for DDX we want to next X pixel, DDY next Y pixel. */
1176 if (op == nir_op_fddx_fine ||
1177 op == nir_op_fddx_coarse ||
1178 op == nir_op_fddx)
1179 idx = 1;
1180 else
1181 idx = 2;
1182
1183 trbl_tid = LLVMBuildAdd(ctx->builder, tl_tid,
1184 LLVMConstInt(ctx->i32, idx, false), "");
1185
1186 if (ctx->has_ds_bpermute) {
1187 args[0] = LLVMBuildMul(ctx->builder, tl_tid,
1188 LLVMConstInt(ctx->i32, 4, false), "");
1189 args[1] = src0;
1190 tl = ac_emit_llvm_intrinsic(&ctx->ac, "llvm.amdgcn.ds.bpermute",
1191 ctx->i32, args, 2,
1192 AC_FUNC_ATTR_READNONE);
1193
1194 args[0] = LLVMBuildMul(ctx->builder, trbl_tid,
1195 LLVMConstInt(ctx->i32, 4, false), "");
1196 trbl = ac_emit_llvm_intrinsic(&ctx->ac, "llvm.amdgcn.ds.bpermute",
1197 ctx->i32, args, 2,
1198 AC_FUNC_ATTR_READNONE);
1199 } else {
1200 LLVMValueRef store_ptr, load_ptr0, load_ptr1;
1201
1202 store_ptr = build_gep0(ctx, ctx->lds, thread_id);
1203 load_ptr0 = build_gep0(ctx, ctx->lds, tl_tid);
1204 load_ptr1 = build_gep0(ctx, ctx->lds, trbl_tid);
1205
1206 LLVMBuildStore(ctx->builder, src0, store_ptr);
1207 tl = LLVMBuildLoad(ctx->builder, load_ptr0, "");
1208 trbl = LLVMBuildLoad(ctx->builder, load_ptr1, "");
1209 }
1210 tl = LLVMBuildBitCast(ctx->builder, tl, ctx->f32, "");
1211 trbl = LLVMBuildBitCast(ctx->builder, trbl, ctx->f32, "");
1212 result = LLVMBuildFSub(ctx->builder, trbl, tl, "");
1213 return result;
1214 }
1215
1216 /*
1217 * this takes an I,J coordinate pair,
1218 * and works out the X and Y derivatives.
1219 * it returns DDX(I), DDX(J), DDY(I), DDY(J).
1220 */
1221 static LLVMValueRef emit_ddxy_interp(
1222 struct nir_to_llvm_context *ctx,
1223 LLVMValueRef interp_ij)
1224 {
1225 LLVMValueRef result[4], a;
1226 unsigned i;
1227
1228 for (i = 0; i < 2; i++) {
1229 a = LLVMBuildExtractElement(ctx->builder, interp_ij,
1230 LLVMConstInt(ctx->i32, i, false), "");
1231 result[i] = emit_ddxy(ctx, nir_op_fddx, a);
1232 result[2+i] = emit_ddxy(ctx, nir_op_fddy, a);
1233 }
1234 return build_gather_values(ctx, result, 4);
1235 }
1236
1237 static LLVMValueRef emit_fdiv(struct nir_to_llvm_context *ctx,
1238 LLVMValueRef num,
1239 LLVMValueRef den)
1240 {
1241 LLVMValueRef ret = LLVMBuildFDiv(ctx->builder, num, den, "");
1242
1243 if (!LLVMIsConstant(ret))
1244 LLVMSetMetadata(ret, ctx->fpmath_md_kind, ctx->fpmath_md_2p5_ulp);
1245 return ret;
1246 }
1247
1248 static void visit_alu(struct nir_to_llvm_context *ctx, nir_alu_instr *instr)
1249 {
1250 LLVMValueRef src[4], result = NULL;
1251 unsigned num_components = instr->dest.dest.ssa.num_components;
1252 unsigned src_components;
1253
1254 assert(nir_op_infos[instr->op].num_inputs <= ARRAY_SIZE(src));
1255 switch (instr->op) {
1256 case nir_op_vec2:
1257 case nir_op_vec3:
1258 case nir_op_vec4:
1259 src_components = 1;
1260 break;
1261 case nir_op_pack_half_2x16:
1262 src_components = 2;
1263 break;
1264 case nir_op_unpack_half_2x16:
1265 src_components = 1;
1266 break;
1267 default:
1268 src_components = num_components;
1269 break;
1270 }
1271 for (unsigned i = 0; i < nir_op_infos[instr->op].num_inputs; i++)
1272 src[i] = get_alu_src(ctx, instr->src[i], src_components);
1273
1274 switch (instr->op) {
1275 case nir_op_fmov:
1276 case nir_op_imov:
1277 result = src[0];
1278 break;
1279 case nir_op_fneg:
1280 src[0] = to_float(ctx, src[0]);
1281 result = LLVMBuildFNeg(ctx->builder, src[0], "");
1282 break;
1283 case nir_op_ineg:
1284 result = LLVMBuildNeg(ctx->builder, src[0], "");
1285 break;
1286 case nir_op_inot:
1287 result = LLVMBuildNot(ctx->builder, src[0], "");
1288 break;
1289 case nir_op_iadd:
1290 result = LLVMBuildAdd(ctx->builder, src[0], src[1], "");
1291 break;
1292 case nir_op_fadd:
1293 src[0] = to_float(ctx, src[0]);
1294 src[1] = to_float(ctx, src[1]);
1295 result = LLVMBuildFAdd(ctx->builder, src[0], src[1], "");
1296 break;
1297 case nir_op_fsub:
1298 src[0] = to_float(ctx, src[0]);
1299 src[1] = to_float(ctx, src[1]);
1300 result = LLVMBuildFSub(ctx->builder, src[0], src[1], "");
1301 break;
1302 case nir_op_isub:
1303 result = LLVMBuildSub(ctx->builder, src[0], src[1], "");
1304 break;
1305 case nir_op_imul:
1306 result = LLVMBuildMul(ctx->builder, src[0], src[1], "");
1307 break;
1308 case nir_op_imod:
1309 result = LLVMBuildSRem(ctx->builder, src[0], src[1], "");
1310 break;
1311 case nir_op_umod:
1312 result = LLVMBuildURem(ctx->builder, src[0], src[1], "");
1313 break;
1314 case nir_op_fmod:
1315 src[0] = to_float(ctx, src[0]);
1316 src[1] = to_float(ctx, src[1]);
1317 result = emit_fdiv(ctx, src[0], src[1]);
1318 result = emit_intrin_1f_param(ctx, "llvm.floor.f32", result);
1319 result = LLVMBuildFMul(ctx->builder, src[1] , result, "");
1320 result = LLVMBuildFSub(ctx->builder, src[0], result, "");
1321 break;
1322 case nir_op_frem:
1323 src[0] = to_float(ctx, src[0]);
1324 src[1] = to_float(ctx, src[1]);
1325 result = LLVMBuildFRem(ctx->builder, src[0], src[1], "");
1326 break;
1327 case nir_op_idiv:
1328 result = LLVMBuildSDiv(ctx->builder, src[0], src[1], "");
1329 break;
1330 case nir_op_udiv:
1331 result = LLVMBuildUDiv(ctx->builder, src[0], src[1], "");
1332 break;
1333 case nir_op_fmul:
1334 src[0] = to_float(ctx, src[0]);
1335 src[1] = to_float(ctx, src[1]);
1336 result = LLVMBuildFMul(ctx->builder, src[0], src[1], "");
1337 break;
1338 case nir_op_fdiv:
1339 src[0] = to_float(ctx, src[0]);
1340 src[1] = to_float(ctx, src[1]);
1341 result = emit_fdiv(ctx, src[0], src[1]);
1342 break;
1343 case nir_op_frcp:
1344 src[0] = to_float(ctx, src[0]);
1345 result = emit_fdiv(ctx, ctx->f32one, src[0]);
1346 break;
1347 case nir_op_iand:
1348 result = LLVMBuildAnd(ctx->builder, src[0], src[1], "");
1349 break;
1350 case nir_op_ior:
1351 result = LLVMBuildOr(ctx->builder, src[0], src[1], "");
1352 break;
1353 case nir_op_ixor:
1354 result = LLVMBuildXor(ctx->builder, src[0], src[1], "");
1355 break;
1356 case nir_op_ishl:
1357 result = LLVMBuildShl(ctx->builder, src[0], src[1], "");
1358 break;
1359 case nir_op_ishr:
1360 result = LLVMBuildAShr(ctx->builder, src[0], src[1], "");
1361 break;
1362 case nir_op_ushr:
1363 result = LLVMBuildLShr(ctx->builder, src[0], src[1], "");
1364 break;
1365 case nir_op_ilt:
1366 result = emit_int_cmp(ctx, LLVMIntSLT, src[0], src[1]);
1367 break;
1368 case nir_op_ine:
1369 result = emit_int_cmp(ctx, LLVMIntNE, src[0], src[1]);
1370 break;
1371 case nir_op_ieq:
1372 result = emit_int_cmp(ctx, LLVMIntEQ, src[0], src[1]);
1373 break;
1374 case nir_op_ige:
1375 result = emit_int_cmp(ctx, LLVMIntSGE, src[0], src[1]);
1376 break;
1377 case nir_op_ult:
1378 result = emit_int_cmp(ctx, LLVMIntULT, src[0], src[1]);
1379 break;
1380 case nir_op_uge:
1381 result = emit_int_cmp(ctx, LLVMIntUGE, src[0], src[1]);
1382 break;
1383 case nir_op_feq:
1384 result = emit_float_cmp(ctx, LLVMRealUEQ, src[0], src[1]);
1385 break;
1386 case nir_op_fne:
1387 result = emit_float_cmp(ctx, LLVMRealUNE, src[0], src[1]);
1388 break;
1389 case nir_op_flt:
1390 result = emit_float_cmp(ctx, LLVMRealULT, src[0], src[1]);
1391 break;
1392 case nir_op_fge:
1393 result = emit_float_cmp(ctx, LLVMRealUGE, src[0], src[1]);
1394 break;
1395 case nir_op_fabs:
1396 result = emit_intrin_1f_param(ctx, "llvm.fabs.f32", src[0]);
1397 break;
1398 case nir_op_iabs:
1399 result = emit_iabs(ctx, src[0]);
1400 break;
1401 case nir_op_imax:
1402 result = emit_minmax_int(ctx, LLVMIntSGT, src[0], src[1]);
1403 break;
1404 case nir_op_imin:
1405 result = emit_minmax_int(ctx, LLVMIntSLT, src[0], src[1]);
1406 break;
1407 case nir_op_umax:
1408 result = emit_minmax_int(ctx, LLVMIntUGT, src[0], src[1]);
1409 break;
1410 case nir_op_umin:
1411 result = emit_minmax_int(ctx, LLVMIntULT, src[0], src[1]);
1412 break;
1413 case nir_op_isign:
1414 result = emit_isign(ctx, src[0]);
1415 break;
1416 case nir_op_fsign:
1417 src[0] = to_float(ctx, src[0]);
1418 result = emit_fsign(ctx, src[0]);
1419 break;
1420 case nir_op_ffloor:
1421 result = emit_intrin_1f_param(ctx, "llvm.floor.f32", src[0]);
1422 break;
1423 case nir_op_ftrunc:
1424 result = emit_intrin_1f_param(ctx, "llvm.trunc.f32", src[0]);
1425 break;
1426 case nir_op_fceil:
1427 result = emit_intrin_1f_param(ctx, "llvm.ceil.f32", src[0]);
1428 break;
1429 case nir_op_fround_even:
1430 result = emit_intrin_1f_param(ctx, "llvm.rint.f32", src[0]);
1431 break;
1432 case nir_op_ffract:
1433 result = emit_ffract(ctx, src[0]);
1434 break;
1435 case nir_op_fsin:
1436 result = emit_intrin_1f_param(ctx, "llvm.sin.f32", src[0]);
1437 break;
1438 case nir_op_fcos:
1439 result = emit_intrin_1f_param(ctx, "llvm.cos.f32", src[0]);
1440 break;
1441 case nir_op_fsqrt:
1442 result = emit_intrin_1f_param(ctx, "llvm.sqrt.f32", src[0]);
1443 break;
1444 case nir_op_fexp2:
1445 result = emit_intrin_1f_param(ctx, "llvm.exp2.f32", src[0]);
1446 break;
1447 case nir_op_flog2:
1448 result = emit_intrin_1f_param(ctx, "llvm.log2.f32", src[0]);
1449 break;
1450 case nir_op_frsq:
1451 result = emit_intrin_1f_param(ctx, "llvm.sqrt.f32", src[0]);
1452 result = emit_fdiv(ctx, ctx->f32one, result);
1453 break;
1454 case nir_op_fpow:
1455 result = emit_intrin_2f_param(ctx, "llvm.pow.f32", src[0], src[1]);
1456 break;
1457 case nir_op_fmax:
1458 result = emit_intrin_2f_param(ctx, "llvm.maxnum.f32", src[0], src[1]);
1459 break;
1460 case nir_op_fmin:
1461 result = emit_intrin_2f_param(ctx, "llvm.minnum.f32", src[0], src[1]);
1462 break;
1463 case nir_op_ffma:
1464 result = emit_intrin_3f_param(ctx, "llvm.fma.f32", src[0], src[1], src[2]);
1465 break;
1466 case nir_op_ibitfield_extract:
1467 result = emit_bitfield_extract(ctx, "llvm.AMDGPU.bfe.i32", src);
1468 break;
1469 case nir_op_ubitfield_extract:
1470 result = emit_bitfield_extract(ctx, "llvm.AMDGPU.bfe.u32", src);
1471 break;
1472 case nir_op_bitfield_insert:
1473 result = emit_bitfield_insert(ctx, src[0], src[1], src[2], src[3]);
1474 break;
1475 case nir_op_bitfield_reverse:
1476 result = ac_emit_llvm_intrinsic(&ctx->ac, "llvm.bitreverse.i32", ctx->i32, src, 1, AC_FUNC_ATTR_READNONE);
1477 break;
1478 case nir_op_bit_count:
1479 result = ac_emit_llvm_intrinsic(&ctx->ac, "llvm.ctpop.i32", ctx->i32, src, 1, AC_FUNC_ATTR_READNONE);
1480 break;
1481 case nir_op_vec2:
1482 case nir_op_vec3:
1483 case nir_op_vec4:
1484 for (unsigned i = 0; i < nir_op_infos[instr->op].num_inputs; i++)
1485 src[i] = to_integer(ctx, src[i]);
1486 result = build_gather_values(ctx, src, num_components);
1487 break;
1488 case nir_op_f2i:
1489 src[0] = to_float(ctx, src[0]);
1490 result = LLVMBuildFPToSI(ctx->builder, src[0], ctx->i32, "");
1491 break;
1492 case nir_op_f2u:
1493 src[0] = to_float(ctx, src[0]);
1494 result = LLVMBuildFPToUI(ctx->builder, src[0], ctx->i32, "");
1495 break;
1496 case nir_op_i2f:
1497 result = LLVMBuildSIToFP(ctx->builder, src[0], ctx->f32, "");
1498 break;
1499 case nir_op_u2f:
1500 result = LLVMBuildUIToFP(ctx->builder, src[0], ctx->f32, "");
1501 break;
1502 case nir_op_bcsel:
1503 result = emit_bcsel(ctx, src[0], src[1], src[2]);
1504 break;
1505 case nir_op_find_lsb:
1506 result = emit_find_lsb(ctx, src[0]);
1507 break;
1508 case nir_op_ufind_msb:
1509 result = emit_ufind_msb(ctx, src[0]);
1510 break;
1511 case nir_op_ifind_msb:
1512 result = emit_ifind_msb(ctx, src[0]);
1513 break;
1514 case nir_op_uadd_carry:
1515 result = emit_uint_carry(ctx, "llvm.uadd.with.overflow.i32", src[0], src[1]);
1516 break;
1517 case nir_op_usub_borrow:
1518 result = emit_uint_carry(ctx, "llvm.usub.with.overflow.i32", src[0], src[1]);
1519 break;
1520 case nir_op_b2f:
1521 result = emit_b2f(ctx, src[0]);
1522 break;
1523 case nir_op_fquantize2f16:
1524 src[0] = to_float(ctx, src[0]);
1525 result = LLVMBuildFPTrunc(ctx->builder, src[0], ctx->f16, "");
1526 /* need to convert back up to f32 */
1527 result = LLVMBuildFPExt(ctx->builder, result, ctx->f32, "");
1528 break;
1529 case nir_op_umul_high:
1530 result = emit_umul_high(ctx, src[0], src[1]);
1531 break;
1532 case nir_op_imul_high:
1533 result = emit_imul_high(ctx, src[0], src[1]);
1534 break;
1535 case nir_op_pack_half_2x16:
1536 result = emit_pack_half_2x16(ctx, src[0]);
1537 break;
1538 case nir_op_unpack_half_2x16:
1539 result = emit_unpack_half_2x16(ctx, src[0]);
1540 break;
1541 case nir_op_fddx:
1542 case nir_op_fddy:
1543 case nir_op_fddx_fine:
1544 case nir_op_fddy_fine:
1545 case nir_op_fddx_coarse:
1546 case nir_op_fddy_coarse:
1547 result = emit_ddxy(ctx, instr->op, src[0]);
1548 break;
1549 default:
1550 fprintf(stderr, "Unknown NIR alu instr: ");
1551 nir_print_instr(&instr->instr, stderr);
1552 fprintf(stderr, "\n");
1553 abort();
1554 }
1555
1556 if (result) {
1557 assert(instr->dest.dest.is_ssa);
1558 result = to_integer(ctx, result);
1559 _mesa_hash_table_insert(ctx->defs, &instr->dest.dest.ssa,
1560 result);
1561 }
1562 }
1563
1564 static void visit_load_const(struct nir_to_llvm_context *ctx,
1565 nir_load_const_instr *instr)
1566 {
1567 LLVMValueRef values[4], value = NULL;
1568 LLVMTypeRef element_type =
1569 LLVMIntTypeInContext(ctx->context, instr->def.bit_size);
1570
1571 for (unsigned i = 0; i < instr->def.num_components; ++i) {
1572 switch (instr->def.bit_size) {
1573 case 32:
1574 values[i] = LLVMConstInt(element_type,
1575 instr->value.u32[i], false);
1576 break;
1577 case 64:
1578 values[i] = LLVMConstInt(element_type,
1579 instr->value.u64[i], false);
1580 break;
1581 default:
1582 fprintf(stderr,
1583 "unsupported nir load_const bit_size: %d\n",
1584 instr->def.bit_size);
1585 abort();
1586 }
1587 }
1588 if (instr->def.num_components > 1) {
1589 value = LLVMConstVector(values, instr->def.num_components);
1590 } else
1591 value = values[0];
1592
1593 _mesa_hash_table_insert(ctx->defs, &instr->def, value);
1594 }
1595
1596 static LLVMValueRef cast_ptr(struct nir_to_llvm_context *ctx, LLVMValueRef ptr,
1597 LLVMTypeRef type)
1598 {
1599 int addr_space = LLVMGetPointerAddressSpace(LLVMTypeOf(ptr));
1600 return LLVMBuildBitCast(ctx->builder, ptr,
1601 LLVMPointerType(type, addr_space), "");
1602 }
1603
1604 static LLVMValueRef
1605 get_buffer_size(struct nir_to_llvm_context *ctx, LLVMValueRef descriptor, bool in_elements)
1606 {
1607 LLVMValueRef size =
1608 LLVMBuildExtractElement(ctx->builder, descriptor,
1609 LLVMConstInt(ctx->i32, 2, false), "");
1610
1611 /* VI only */
1612 if (ctx->options->chip_class >= VI && in_elements) {
1613 /* On VI, the descriptor contains the size in bytes,
1614 * but TXQ must return the size in elements.
1615 * The stride is always non-zero for resources using TXQ.
1616 */
1617 LLVMValueRef stride =
1618 LLVMBuildExtractElement(ctx->builder, descriptor,
1619 LLVMConstInt(ctx->i32, 1, false), "");
1620 stride = LLVMBuildLShr(ctx->builder, stride,
1621 LLVMConstInt(ctx->i32, 16, false), "");
1622 stride = LLVMBuildAnd(ctx->builder, stride,
1623 LLVMConstInt(ctx->i32, 0x3fff, false), "");
1624
1625 size = LLVMBuildUDiv(ctx->builder, size, stride, "");
1626 }
1627 return size;
1628 }
1629
1630 /**
1631 * Given the i32 or vNi32 \p type, generate the textual name (e.g. for use with
1632 * intrinsic names).
1633 */
1634 static void build_int_type_name(
1635 LLVMTypeRef type,
1636 char *buf, unsigned bufsize)
1637 {
1638 assert(bufsize >= 6);
1639
1640 if (LLVMGetTypeKind(type) == LLVMVectorTypeKind)
1641 snprintf(buf, bufsize, "v%ui32",
1642 LLVMGetVectorSize(type));
1643 else
1644 strcpy(buf, "i32");
1645 }
1646
1647 static LLVMValueRef radv_lower_gather4_integer(struct nir_to_llvm_context *ctx,
1648 struct ac_tex_info *tinfo,
1649 nir_tex_instr *instr,
1650 const char *intr_name,
1651 unsigned coord_vgpr_index)
1652 {
1653 LLVMValueRef coord = tinfo->args[0];
1654 LLVMValueRef half_texel[2];
1655 int c;
1656
1657 //TODO Rect
1658 {
1659 LLVMValueRef txq_args[10];
1660 int txq_arg_count = 0;
1661 LLVMValueRef size;
1662 bool da = instr->is_array || instr->sampler_dim == GLSL_SAMPLER_DIM_CUBE;
1663 txq_args[txq_arg_count++] = LLVMConstInt(ctx->i32, 0, false);
1664 txq_args[txq_arg_count++] = tinfo->args[1];
1665 txq_args[txq_arg_count++] = LLVMConstInt(ctx->i32, 0xf, 0); /* dmask */
1666 txq_args[txq_arg_count++] = LLVMConstInt(ctx->i32, 0, 0); /* unorm */
1667 txq_args[txq_arg_count++] = LLVMConstInt(ctx->i32, 0, 0); /* r128 */
1668 txq_args[txq_arg_count++] = LLVMConstInt(ctx->i32, da ? 1 : 0, 0);
1669 txq_args[txq_arg_count++] = LLVMConstInt(ctx->i32, 0, 0); /* glc */
1670 txq_args[txq_arg_count++] = LLVMConstInt(ctx->i32, 0, 0); /* slc */
1671 txq_args[txq_arg_count++] = LLVMConstInt(ctx->i32, 0, 0); /* tfe */
1672 txq_args[txq_arg_count++] = LLVMConstInt(ctx->i32, 0, 0); /* lwe */
1673 size = ac_emit_llvm_intrinsic(&ctx->ac, "llvm.SI.getresinfo.i32", ctx->v4i32,
1674 txq_args, txq_arg_count,
1675 AC_FUNC_ATTR_READNONE);
1676
1677 for (c = 0; c < 2; c++) {
1678 half_texel[c] = LLVMBuildExtractElement(ctx->builder, size,
1679 LLVMConstInt(ctx->i32, c, false), "");
1680 half_texel[c] = LLVMBuildUIToFP(ctx->builder, half_texel[c], ctx->f32, "");
1681 half_texel[c] = emit_fdiv(ctx, ctx->f32one, half_texel[c]);
1682 half_texel[c] = LLVMBuildFMul(ctx->builder, half_texel[c],
1683 LLVMConstReal(ctx->f32, -0.5), "");
1684 }
1685 }
1686
1687 for (c = 0; c < 2; c++) {
1688 LLVMValueRef tmp;
1689 LLVMValueRef index = LLVMConstInt(ctx->i32, coord_vgpr_index + c, 0);
1690 tmp = LLVMBuildExtractElement(ctx->builder, coord, index, "");
1691 tmp = LLVMBuildBitCast(ctx->builder, tmp, ctx->f32, "");
1692 tmp = LLVMBuildFAdd(ctx->builder, tmp, half_texel[c], "");
1693 tmp = LLVMBuildBitCast(ctx->builder, tmp, ctx->i32, "");
1694 coord = LLVMBuildInsertElement(ctx->builder, coord, tmp, index, "");
1695 }
1696
1697 tinfo->args[0] = coord;
1698 return ac_emit_llvm_intrinsic(&ctx->ac, intr_name, tinfo->dst_type, tinfo->args, tinfo->arg_count,
1699 AC_FUNC_ATTR_READNONE | AC_FUNC_ATTR_NOUNWIND);
1700
1701 }
1702
1703 static LLVMValueRef build_tex_intrinsic(struct nir_to_llvm_context *ctx,
1704 nir_tex_instr *instr,
1705 struct ac_tex_info *tinfo)
1706 {
1707 const char *name = "llvm.SI.image.sample";
1708 const char *infix = "";
1709 char intr_name[127];
1710 char type[64];
1711 bool is_shadow = instr->is_shadow;
1712 bool has_offset = tinfo->has_offset;
1713 switch (instr->op) {
1714 case nir_texop_txf:
1715 case nir_texop_txf_ms:
1716 case nir_texop_samples_identical:
1717 name = instr->sampler_dim == GLSL_SAMPLER_DIM_MS ? "llvm.SI.image.load" :
1718 instr->sampler_dim == GLSL_SAMPLER_DIM_BUF ? "llvm.SI.vs.load.input" :
1719 "llvm.SI.image.load.mip";
1720 is_shadow = false;
1721 has_offset = false;
1722 break;
1723 case nir_texop_txb:
1724 infix = ".b";
1725 break;
1726 case nir_texop_txl:
1727 infix = ".l";
1728 break;
1729 case nir_texop_txs:
1730 name = "llvm.SI.getresinfo";
1731 break;
1732 case nir_texop_query_levels:
1733 name = "llvm.SI.getresinfo";
1734 break;
1735 case nir_texop_tex:
1736 if (ctx->stage != MESA_SHADER_FRAGMENT)
1737 infix = ".lz";
1738 break;
1739 case nir_texop_txd:
1740 infix = ".d";
1741 break;
1742 case nir_texop_tg4:
1743 name = "llvm.SI.gather4";
1744 infix = ".lz";
1745 break;
1746 case nir_texop_lod:
1747 name = "llvm.SI.getlod";
1748 is_shadow = false;
1749 has_offset = false;
1750 break;
1751 default:
1752 break;
1753 }
1754
1755 build_int_type_name(LLVMTypeOf(tinfo->args[0]), type, sizeof(type));
1756 sprintf(intr_name, "%s%s%s%s.%s", name, is_shadow ? ".c" : "", infix,
1757 has_offset ? ".o" : "", type);
1758
1759 if (instr->op == nir_texop_tg4) {
1760 enum glsl_base_type stype = glsl_get_sampler_result_type(instr->texture->var->type);
1761 if (stype == GLSL_TYPE_UINT || stype == GLSL_TYPE_INT) {
1762 return radv_lower_gather4_integer(ctx, tinfo, instr, intr_name,
1763 (int)has_offset + (int)is_shadow);
1764 }
1765 }
1766 return ac_emit_llvm_intrinsic(&ctx->ac, intr_name, tinfo->dst_type, tinfo->args, tinfo->arg_count,
1767 AC_FUNC_ATTR_READNONE | AC_FUNC_ATTR_NOUNWIND);
1768
1769 }
1770
1771 static LLVMValueRef visit_vulkan_resource_index(struct nir_to_llvm_context *ctx,
1772 nir_intrinsic_instr *instr)
1773 {
1774 LLVMValueRef index = get_src(ctx, instr->src[0]);
1775 unsigned desc_set = nir_intrinsic_desc_set(instr);
1776 unsigned binding = nir_intrinsic_binding(instr);
1777 LLVMValueRef desc_ptr = ctx->descriptor_sets[desc_set];
1778 struct radv_descriptor_set_layout *layout = ctx->options->layout->set[desc_set].layout;
1779 unsigned base_offset = layout->binding[binding].offset;
1780 LLVMValueRef offset, stride;
1781
1782 if (layout->binding[binding].type == VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER_DYNAMIC ||
1783 layout->binding[binding].type == VK_DESCRIPTOR_TYPE_STORAGE_BUFFER_DYNAMIC) {
1784 desc_ptr = ctx->push_constants;
1785 base_offset = ctx->options->layout->push_constant_size;
1786 base_offset += 16 * layout->binding[binding].dynamic_offset_offset;
1787 stride = LLVMConstInt(ctx->i32, 16, false);
1788 } else
1789 stride = LLVMConstInt(ctx->i32, layout->binding[binding].size, false);
1790
1791 offset = LLVMConstInt(ctx->i32, base_offset, false);
1792 index = LLVMBuildMul(ctx->builder, index, stride, "");
1793 offset = LLVMBuildAdd(ctx->builder, offset, index, "");
1794
1795 desc_ptr = build_gep0(ctx, desc_ptr, offset);
1796 desc_ptr = cast_ptr(ctx, desc_ptr, ctx->v4i32);
1797 LLVMSetMetadata(desc_ptr, ctx->uniform_md_kind, ctx->empty_md);
1798
1799 return LLVMBuildLoad(ctx->builder, desc_ptr, "");
1800 }
1801
1802 static LLVMValueRef visit_load_push_constant(struct nir_to_llvm_context *ctx,
1803 nir_intrinsic_instr *instr)
1804 {
1805 LLVMValueRef ptr;
1806
1807 ptr = build_gep0(ctx, ctx->push_constants, get_src(ctx, instr->src[0]));
1808 ptr = cast_ptr(ctx, ptr, get_def_type(ctx, &instr->dest.ssa));
1809
1810 return LLVMBuildLoad(ctx->builder, ptr, "");
1811 }
1812
1813 static LLVMValueRef visit_get_buffer_size(struct nir_to_llvm_context *ctx,
1814 nir_intrinsic_instr *instr)
1815 {
1816 LLVMValueRef desc = get_src(ctx, instr->src[0]);
1817
1818 return get_buffer_size(ctx, desc, false);
1819 }
1820 static void visit_store_ssbo(struct nir_to_llvm_context *ctx,
1821 nir_intrinsic_instr *instr)
1822 {
1823 const char *store_name;
1824 LLVMTypeRef data_type = ctx->f32;
1825 unsigned writemask = nir_intrinsic_write_mask(instr);
1826 LLVMValueRef base_data, base_offset;
1827 LLVMValueRef params[6];
1828
1829 if (ctx->stage == MESA_SHADER_FRAGMENT)
1830 ctx->shader_info->fs.writes_memory = true;
1831
1832 params[1] = get_src(ctx, instr->src[1]);
1833 params[2] = LLVMConstInt(ctx->i32, 0, false); /* vindex */
1834 params[4] = LLVMConstInt(ctx->i1, 0, false); /* glc */
1835 params[5] = LLVMConstInt(ctx->i1, 0, false); /* slc */
1836
1837 if (instr->num_components > 1)
1838 data_type = LLVMVectorType(ctx->f32, instr->num_components);
1839
1840 base_data = to_float(ctx, get_src(ctx, instr->src[0]));
1841 base_data = trim_vector(ctx, base_data, instr->num_components);
1842 base_data = LLVMBuildBitCast(ctx->builder, base_data,
1843 data_type, "");
1844 base_offset = get_src(ctx, instr->src[2]); /* voffset */
1845 while (writemask) {
1846 int start, count;
1847 LLVMValueRef data;
1848 LLVMValueRef offset;
1849 LLVMValueRef tmp;
1850 u_bit_scan_consecutive_range(&writemask, &start, &count);
1851
1852 /* Due to an LLVM limitation, split 3-element writes
1853 * into a 2-element and a 1-element write. */
1854 if (count == 3) {
1855 writemask |= 1 << (start + 2);
1856 count = 2;
1857 }
1858
1859 if (count == 4) {
1860 store_name = "llvm.amdgcn.buffer.store.v4f32";
1861 data = base_data;
1862 } else if (count == 2) {
1863 tmp = LLVMBuildExtractElement(ctx->builder,
1864 base_data, LLVMConstInt(ctx->i32, start, false), "");
1865 data = LLVMBuildInsertElement(ctx->builder, LLVMGetUndef(ctx->v2f32), tmp,
1866 ctx->i32zero, "");
1867
1868 tmp = LLVMBuildExtractElement(ctx->builder,
1869 base_data, LLVMConstInt(ctx->i32, start + 1, false), "");
1870 data = LLVMBuildInsertElement(ctx->builder, data, tmp,
1871 ctx->i32one, "");
1872 store_name = "llvm.amdgcn.buffer.store.v2f32";
1873
1874 } else {
1875 assert(count == 1);
1876 if (get_llvm_num_components(base_data) > 1)
1877 data = LLVMBuildExtractElement(ctx->builder, base_data,
1878 LLVMConstInt(ctx->i32, start, false), "");
1879 else
1880 data = base_data;
1881 store_name = "llvm.amdgcn.buffer.store.f32";
1882 }
1883
1884 offset = base_offset;
1885 if (start != 0) {
1886 offset = LLVMBuildAdd(ctx->builder, offset, LLVMConstInt(ctx->i32, start * 4, false), "");
1887 }
1888 params[0] = data;
1889 params[3] = offset;
1890 ac_emit_llvm_intrinsic(&ctx->ac, store_name,
1891 LLVMVoidTypeInContext(ctx->context), params, 6, 0);
1892 }
1893 }
1894
1895 static LLVMValueRef visit_atomic_ssbo(struct nir_to_llvm_context *ctx,
1896 nir_intrinsic_instr *instr)
1897 {
1898 const char *name;
1899 LLVMValueRef params[6];
1900 int arg_count = 0;
1901 if (ctx->stage == MESA_SHADER_FRAGMENT)
1902 ctx->shader_info->fs.writes_memory = true;
1903
1904 if (instr->intrinsic == nir_intrinsic_ssbo_atomic_comp_swap) {
1905 params[arg_count++] = llvm_extract_elem(ctx, get_src(ctx, instr->src[3]), 0);
1906 }
1907 params[arg_count++] = llvm_extract_elem(ctx, get_src(ctx, instr->src[2]), 0);
1908 params[arg_count++] = get_src(ctx, instr->src[0]);
1909 params[arg_count++] = LLVMConstInt(ctx->i32, 0, false); /* vindex */
1910 params[arg_count++] = get_src(ctx, instr->src[1]); /* voffset */
1911 params[arg_count++] = LLVMConstInt(ctx->i1, 0, false); /* slc */
1912
1913 switch (instr->intrinsic) {
1914 case nir_intrinsic_ssbo_atomic_add:
1915 name = "llvm.amdgcn.buffer.atomic.add";
1916 break;
1917 case nir_intrinsic_ssbo_atomic_imin:
1918 name = "llvm.amdgcn.buffer.atomic.smin";
1919 break;
1920 case nir_intrinsic_ssbo_atomic_umin:
1921 name = "llvm.amdgcn.buffer.atomic.umin";
1922 break;
1923 case nir_intrinsic_ssbo_atomic_imax:
1924 name = "llvm.amdgcn.buffer.atomic.smax";
1925 break;
1926 case nir_intrinsic_ssbo_atomic_umax:
1927 name = "llvm.amdgcn.buffer.atomic.umax";
1928 break;
1929 case nir_intrinsic_ssbo_atomic_and:
1930 name = "llvm.amdgcn.buffer.atomic.and";
1931 break;
1932 case nir_intrinsic_ssbo_atomic_or:
1933 name = "llvm.amdgcn.buffer.atomic.or";
1934 break;
1935 case nir_intrinsic_ssbo_atomic_xor:
1936 name = "llvm.amdgcn.buffer.atomic.xor";
1937 break;
1938 case nir_intrinsic_ssbo_atomic_exchange:
1939 name = "llvm.amdgcn.buffer.atomic.swap";
1940 break;
1941 case nir_intrinsic_ssbo_atomic_comp_swap:
1942 name = "llvm.amdgcn.buffer.atomic.cmpswap";
1943 break;
1944 default:
1945 abort();
1946 }
1947
1948 return ac_emit_llvm_intrinsic(&ctx->ac, name, ctx->i32, params, arg_count, 0);
1949 }
1950
1951 static LLVMValueRef visit_load_buffer(struct nir_to_llvm_context *ctx,
1952 nir_intrinsic_instr *instr)
1953 {
1954 const char *load_name;
1955 LLVMTypeRef data_type = ctx->f32;
1956 if (instr->num_components == 3)
1957 data_type = LLVMVectorType(ctx->f32, 4);
1958 else if (instr->num_components > 1)
1959 data_type = LLVMVectorType(ctx->f32, instr->num_components);
1960
1961 if (instr->num_components == 4 || instr->num_components == 3)
1962 load_name = "llvm.amdgcn.buffer.load.v4f32";
1963 else if (instr->num_components == 2)
1964 load_name = "llvm.amdgcn.buffer.load.v2f32";
1965 else if (instr->num_components == 1)
1966 load_name = "llvm.amdgcn.buffer.load.f32";
1967 else
1968 abort();
1969
1970 LLVMValueRef params[] = {
1971 get_src(ctx, instr->src[0]),
1972 LLVMConstInt(ctx->i32, 0, false),
1973 get_src(ctx, instr->src[1]),
1974 LLVMConstInt(ctx->i1, 0, false),
1975 LLVMConstInt(ctx->i1, 0, false),
1976 };
1977
1978 LLVMValueRef ret =
1979 ac_emit_llvm_intrinsic(&ctx->ac, load_name, data_type, params, 5, 0);
1980
1981 if (instr->num_components == 3)
1982 ret = trim_vector(ctx, ret, 3);
1983
1984 return LLVMBuildBitCast(ctx->builder, ret,
1985 get_def_type(ctx, &instr->dest.ssa), "");
1986 }
1987
1988 static LLVMValueRef visit_load_ubo_buffer(struct nir_to_llvm_context *ctx,
1989 nir_intrinsic_instr *instr)
1990 {
1991 LLVMValueRef results[4], ret;
1992 LLVMValueRef rsrc = get_src(ctx, instr->src[0]);
1993 LLVMValueRef offset = get_src(ctx, instr->src[1]);
1994
1995 rsrc = LLVMBuildBitCast(ctx->builder, rsrc, LLVMVectorType(ctx->i8, 16), "");
1996
1997 for (unsigned i = 0; i < instr->num_components; ++i) {
1998 LLVMValueRef params[] = {
1999 rsrc,
2000 LLVMBuildAdd(ctx->builder, LLVMConstInt(ctx->i32, 4 * i, 0),
2001 offset, "")
2002 };
2003 results[i] = ac_emit_llvm_intrinsic(&ctx->ac, "llvm.SI.load.const", ctx->f32,
2004 params, 2, AC_FUNC_ATTR_READNONE);
2005 }
2006
2007
2008 ret = build_gather_values(ctx, results, instr->num_components);
2009 return LLVMBuildBitCast(ctx->builder, ret,
2010 get_def_type(ctx, &instr->dest.ssa), "");
2011 }
2012
2013 static void
2014 radv_get_deref_offset(struct nir_to_llvm_context *ctx, nir_deref *tail,
2015 bool vs_in, unsigned *const_out, LLVMValueRef *indir_out)
2016 {
2017 unsigned const_offset = 0;
2018 LLVMValueRef offset = NULL;
2019
2020
2021 while (tail->child != NULL) {
2022 const struct glsl_type *parent_type = tail->type;
2023 tail = tail->child;
2024
2025 if (tail->deref_type == nir_deref_type_array) {
2026 nir_deref_array *deref_array = nir_deref_as_array(tail);
2027 LLVMValueRef index, stride, local_offset;
2028 unsigned size = glsl_count_attribute_slots(tail->type, vs_in);
2029
2030 const_offset += size * deref_array->base_offset;
2031 if (deref_array->deref_array_type == nir_deref_array_type_direct)
2032 continue;
2033
2034 assert(deref_array->deref_array_type == nir_deref_array_type_indirect);
2035 index = get_src(ctx, deref_array->indirect);
2036 stride = LLVMConstInt(ctx->i32, size, 0);
2037 local_offset = LLVMBuildMul(ctx->builder, stride, index, "");
2038
2039 if (offset)
2040 offset = LLVMBuildAdd(ctx->builder, offset, local_offset, "");
2041 else
2042 offset = local_offset;
2043 } else if (tail->deref_type == nir_deref_type_struct) {
2044 nir_deref_struct *deref_struct = nir_deref_as_struct(tail);
2045
2046 for (unsigned i = 0; i < deref_struct->index; i++) {
2047 const struct glsl_type *ft = glsl_get_struct_field(parent_type, i);
2048 const_offset += glsl_count_attribute_slots(ft, vs_in);
2049 }
2050 } else
2051 unreachable("unsupported deref type");
2052
2053 }
2054
2055 if (const_offset && offset)
2056 offset = LLVMBuildAdd(ctx->builder, offset,
2057 LLVMConstInt(ctx->i32, const_offset, 0),
2058 "");
2059
2060 *const_out = const_offset;
2061 *indir_out = offset;
2062 }
2063
2064 static LLVMValueRef visit_load_var(struct nir_to_llvm_context *ctx,
2065 nir_intrinsic_instr *instr)
2066 {
2067 LLVMValueRef values[4];
2068 int idx = instr->variables[0]->var->data.driver_location;
2069 int ve = instr->dest.ssa.num_components;
2070 LLVMValueRef indir_index;
2071 unsigned const_index;
2072 switch (instr->variables[0]->var->data.mode) {
2073 case nir_var_shader_in:
2074 radv_get_deref_offset(ctx, &instr->variables[0]->deref,
2075 ctx->stage == MESA_SHADER_VERTEX,
2076 &const_index, &indir_index);
2077 for (unsigned chan = 0; chan < ve; chan++) {
2078 if (indir_index) {
2079 unsigned count = glsl_count_attribute_slots(
2080 instr->variables[0]->var->type,
2081 ctx->stage == MESA_SHADER_VERTEX);
2082 LLVMValueRef tmp_vec = build_gather_values_extended(
2083 ctx, ctx->inputs + idx + chan, count,
2084 4, false);
2085
2086 values[chan] = LLVMBuildExtractElement(ctx->builder,
2087 tmp_vec,
2088 indir_index, "");
2089 } else
2090 values[chan] = ctx->inputs[idx + chan + const_index * 4];
2091 }
2092 return to_integer(ctx, build_gather_values(ctx, values, ve));
2093 break;
2094 case nir_var_local:
2095 radv_get_deref_offset(ctx, &instr->variables[0]->deref, false,
2096 &const_index, &indir_index);
2097 for (unsigned chan = 0; chan < ve; chan++) {
2098 if (indir_index) {
2099 unsigned count = glsl_count_attribute_slots(
2100 instr->variables[0]->var->type, false);
2101 LLVMValueRef tmp_vec = build_gather_values_extended(
2102 ctx, ctx->locals + idx + chan, count,
2103 4, true);
2104
2105 values[chan] = LLVMBuildExtractElement(ctx->builder,
2106 tmp_vec,
2107 indir_index, "");
2108 } else {
2109 values[chan] = LLVMBuildLoad(ctx->builder, ctx->locals[idx + chan + const_index * 4], "");
2110 }
2111 }
2112 return to_integer(ctx, build_gather_values(ctx, values, ve));
2113 case nir_var_shader_out:
2114 radv_get_deref_offset(ctx, &instr->variables[0]->deref, false,
2115 &const_index, &indir_index);
2116 for (unsigned chan = 0; chan < ve; chan++) {
2117 if (indir_index) {
2118 unsigned count = glsl_count_attribute_slots(
2119 instr->variables[0]->var->type, false);
2120 LLVMValueRef tmp_vec = build_gather_values_extended(
2121 ctx, ctx->outputs + idx + chan, count,
2122 4, true);
2123
2124 values[chan] = LLVMBuildExtractElement(ctx->builder,
2125 tmp_vec,
2126 indir_index, "");
2127 } else {
2128 values[chan] = LLVMBuildLoad(ctx->builder,
2129 ctx->outputs[idx + chan + const_index * 4],
2130 "");
2131 }
2132 }
2133 return to_integer(ctx, build_gather_values(ctx, values, ve));
2134 case nir_var_shared: {
2135 radv_get_deref_offset(ctx, &instr->variables[0]->deref, false,
2136 &const_index, &indir_index);
2137 LLVMValueRef ptr = get_shared_memory_ptr(ctx, idx, ctx->i32);
2138 LLVMValueRef derived_ptr;
2139
2140 for (unsigned chan = 0; chan < ve; chan++) {
2141 LLVMValueRef index = LLVMConstInt(ctx->i32, chan, false);
2142 if (indir_index)
2143 index = LLVMBuildAdd(ctx->builder, index, indir_index, "");
2144 derived_ptr = LLVMBuildGEP(ctx->builder, ptr, &index, 1, "");
2145 values[chan] = LLVMBuildLoad(ctx->builder, derived_ptr, "");
2146 }
2147 return to_integer(ctx, build_gather_values(ctx, values, ve));
2148 }
2149 default:
2150 break;
2151 }
2152 return NULL;
2153 }
2154
2155 static void
2156 visit_store_var(struct nir_to_llvm_context *ctx,
2157 nir_intrinsic_instr *instr)
2158 {
2159 LLVMValueRef temp_ptr, value;
2160 int idx = instr->variables[0]->var->data.driver_location;
2161 LLVMValueRef src = to_float(ctx, get_src(ctx, instr->src[0]));
2162 int writemask = instr->const_index[0];
2163 LLVMValueRef indir_index;
2164 unsigned const_index;
2165 switch (instr->variables[0]->var->data.mode) {
2166 case nir_var_shader_out:
2167 radv_get_deref_offset(ctx, &instr->variables[0]->deref, false,
2168 &const_index, &indir_index);
2169 for (unsigned chan = 0; chan < 4; chan++) {
2170 int stride = 4;
2171 if (!(writemask & (1 << chan)))
2172 continue;
2173 if (get_llvm_num_components(src) == 1)
2174 value = src;
2175 else
2176 value = LLVMBuildExtractElement(ctx->builder, src,
2177 LLVMConstInt(ctx->i32,
2178 chan, false),
2179 "");
2180
2181 if (instr->variables[0]->var->data.location == VARYING_SLOT_CLIP_DIST0 ||
2182 instr->variables[0]->var->data.location == VARYING_SLOT_CULL_DIST0)
2183 stride = 1;
2184 if (indir_index) {
2185 unsigned count = glsl_count_attribute_slots(
2186 instr->variables[0]->var->type, false);
2187 LLVMValueRef tmp_vec = build_gather_values_extended(
2188 ctx, ctx->outputs + idx + chan, count,
2189 stride, true);
2190
2191 if (get_llvm_num_components(tmp_vec) > 1) {
2192 tmp_vec = LLVMBuildInsertElement(ctx->builder, tmp_vec,
2193 value, indir_index, "");
2194 } else
2195 tmp_vec = value;
2196 build_store_values_extended(ctx, ctx->outputs + idx + chan,
2197 count, stride, tmp_vec);
2198
2199 } else {
2200 temp_ptr = ctx->outputs[idx + chan + const_index * stride];
2201
2202 LLVMBuildStore(ctx->builder, value, temp_ptr);
2203 }
2204 }
2205 break;
2206 case nir_var_local:
2207 radv_get_deref_offset(ctx, &instr->variables[0]->deref, false,
2208 &const_index, &indir_index);
2209 for (unsigned chan = 0; chan < 4; chan++) {
2210 if (!(writemask & (1 << chan)))
2211 continue;
2212
2213 if (get_llvm_num_components(src) == 1)
2214 value = src;
2215 else
2216 value = LLVMBuildExtractElement(ctx->builder, src,
2217 LLVMConstInt(ctx->i32, chan, false), "");
2218 if (indir_index) {
2219 unsigned count = glsl_count_attribute_slots(
2220 instr->variables[0]->var->type, false);
2221 LLVMValueRef tmp_vec = build_gather_values_extended(
2222 ctx, ctx->locals + idx + chan, count,
2223 4, true);
2224
2225 tmp_vec = LLVMBuildInsertElement(ctx->builder, tmp_vec,
2226 value, indir_index, "");
2227 build_store_values_extended(ctx, ctx->locals + idx + chan,
2228 count, 4, tmp_vec);
2229 } else {
2230 temp_ptr = ctx->locals[idx + chan + const_index * 4];
2231
2232 LLVMBuildStore(ctx->builder, value, temp_ptr);
2233 }
2234 }
2235 break;
2236 case nir_var_shared: {
2237 LLVMValueRef ptr;
2238 radv_get_deref_offset(ctx, &instr->variables[0]->deref, false,
2239 &const_index, &indir_index);
2240
2241 ptr = get_shared_memory_ptr(ctx, idx, ctx->i32);
2242 LLVMValueRef derived_ptr;
2243
2244 for (unsigned chan = 0; chan < 4; chan++) {
2245 if (!(writemask & (1 << chan)))
2246 continue;
2247
2248 LLVMValueRef index = LLVMConstInt(ctx->i32, chan, false);
2249
2250 if (get_llvm_num_components(src) == 1)
2251 value = src;
2252 else
2253 value = LLVMBuildExtractElement(ctx->builder, src,
2254 LLVMConstInt(ctx->i32,
2255 chan, false),
2256 "");
2257
2258 if (indir_index)
2259 index = LLVMBuildAdd(ctx->builder, index, indir_index, "");
2260
2261 derived_ptr = LLVMBuildGEP(ctx->builder, ptr, &index, 1, "");
2262 LLVMBuildStore(ctx->builder,
2263 to_integer(ctx, value), derived_ptr);
2264 }
2265 break;
2266 }
2267 default:
2268 break;
2269 }
2270 }
2271
2272 static int image_type_to_components_count(enum glsl_sampler_dim dim, bool array)
2273 {
2274 switch (dim) {
2275 case GLSL_SAMPLER_DIM_BUF:
2276 return 1;
2277 case GLSL_SAMPLER_DIM_1D:
2278 return array ? 2 : 1;
2279 case GLSL_SAMPLER_DIM_2D:
2280 return array ? 3 : 2;
2281 case GLSL_SAMPLER_DIM_MS:
2282 return array ? 4 : 3;
2283 case GLSL_SAMPLER_DIM_3D:
2284 case GLSL_SAMPLER_DIM_CUBE:
2285 return 3;
2286 case GLSL_SAMPLER_DIM_RECT:
2287 case GLSL_SAMPLER_DIM_SUBPASS:
2288 return 2;
2289 case GLSL_SAMPLER_DIM_SUBPASS_MS:
2290 return 3;
2291 default:
2292 break;
2293 }
2294 return 0;
2295 }
2296
2297 static LLVMValueRef get_image_coords(struct nir_to_llvm_context *ctx,
2298 nir_intrinsic_instr *instr, bool add_frag_pos)
2299 {
2300 const struct glsl_type *type = instr->variables[0]->var->type;
2301 if(instr->variables[0]->deref.child)
2302 type = instr->variables[0]->deref.child->type;
2303
2304 LLVMValueRef src0 = get_src(ctx, instr->src[0]);
2305 LLVMValueRef coords[4];
2306 LLVMValueRef masks[] = {
2307 LLVMConstInt(ctx->i32, 0, false), LLVMConstInt(ctx->i32, 1, false),
2308 LLVMConstInt(ctx->i32, 2, false), LLVMConstInt(ctx->i32, 3, false),
2309 };
2310 LLVMValueRef res;
2311 int count;
2312 enum glsl_sampler_dim dim = glsl_get_sampler_dim(type);
2313 bool is_ms = (dim == GLSL_SAMPLER_DIM_MS ||
2314 dim == GLSL_SAMPLER_DIM_SUBPASS_MS);
2315
2316 count = image_type_to_components_count(dim,
2317 glsl_sampler_type_is_array(type));
2318
2319 if (count == 1) {
2320 if (instr->src[0].ssa->num_components)
2321 res = LLVMBuildExtractElement(ctx->builder, src0, masks[0], "");
2322 else
2323 res = src0;
2324 } else {
2325 int chan;
2326 if (is_ms)
2327 count--;
2328 for (chan = 0; chan < count; ++chan) {
2329 coords[chan] = LLVMBuildExtractElement(ctx->builder, src0, masks[chan], "");
2330 }
2331
2332 if (add_frag_pos) {
2333 for (chan = 0; chan < count; ++chan)
2334 coords[chan] = LLVMBuildAdd(ctx->builder, coords[chan], LLVMBuildFPToUI(ctx->builder, ctx->frag_pos[chan], ctx->i32, ""), "");
2335 }
2336 if (is_ms) {
2337 coords[count] = llvm_extract_elem(ctx, get_src(ctx, instr->src[1]), 0);
2338 count++;
2339 }
2340
2341 if (count == 3) {
2342 coords[3] = LLVMGetUndef(ctx->i32);
2343 count = 4;
2344 }
2345 res = build_gather_values(ctx, coords, count);
2346 }
2347 return res;
2348 }
2349
2350 static void build_type_name_for_intr(
2351 LLVMTypeRef type,
2352 char *buf, unsigned bufsize)
2353 {
2354 LLVMTypeRef elem_type = type;
2355
2356 assert(bufsize >= 8);
2357
2358 if (LLVMGetTypeKind(type) == LLVMVectorTypeKind) {
2359 int ret = snprintf(buf, bufsize, "v%u",
2360 LLVMGetVectorSize(type));
2361 if (ret < 0) {
2362 char *type_name = LLVMPrintTypeToString(type);
2363 fprintf(stderr, "Error building type name for: %s\n",
2364 type_name);
2365 return;
2366 }
2367 elem_type = LLVMGetElementType(type);
2368 buf += ret;
2369 bufsize -= ret;
2370 }
2371 switch (LLVMGetTypeKind(elem_type)) {
2372 default: break;
2373 case LLVMIntegerTypeKind:
2374 snprintf(buf, bufsize, "i%d", LLVMGetIntTypeWidth(elem_type));
2375 break;
2376 case LLVMFloatTypeKind:
2377 snprintf(buf, bufsize, "f32");
2378 break;
2379 case LLVMDoubleTypeKind:
2380 snprintf(buf, bufsize, "f64");
2381 break;
2382 }
2383 }
2384
2385 static void get_image_intr_name(const char *base_name,
2386 LLVMTypeRef data_type,
2387 LLVMTypeRef coords_type,
2388 LLVMTypeRef rsrc_type,
2389 char *out_name, unsigned out_len)
2390 {
2391 char coords_type_name[8];
2392
2393 build_type_name_for_intr(coords_type, coords_type_name,
2394 sizeof(coords_type_name));
2395
2396 if (HAVE_LLVM <= 0x0309) {
2397 snprintf(out_name, out_len, "%s.%s", base_name, coords_type_name);
2398 } else {
2399 char data_type_name[8];
2400 char rsrc_type_name[8];
2401
2402 build_type_name_for_intr(data_type, data_type_name,
2403 sizeof(data_type_name));
2404 build_type_name_for_intr(rsrc_type, rsrc_type_name,
2405 sizeof(rsrc_type_name));
2406 snprintf(out_name, out_len, "%s.%s.%s.%s", base_name,
2407 data_type_name, coords_type_name, rsrc_type_name);
2408 }
2409 }
2410
2411 static LLVMValueRef visit_image_load(struct nir_to_llvm_context *ctx,
2412 nir_intrinsic_instr *instr)
2413 {
2414 LLVMValueRef params[7];
2415 LLVMValueRef res;
2416 char intrinsic_name[64];
2417 const nir_variable *var = instr->variables[0]->var;
2418 const struct glsl_type *type = var->type;
2419 if(instr->variables[0]->deref.child)
2420 type = instr->variables[0]->deref.child->type;
2421
2422 type = glsl_without_array(type);
2423 if (glsl_get_sampler_dim(type) == GLSL_SAMPLER_DIM_BUF) {
2424 params[0] = get_sampler_desc(ctx, instr->variables[0], DESC_BUFFER);
2425 params[1] = LLVMBuildExtractElement(ctx->builder, get_src(ctx, instr->src[0]),
2426 LLVMConstInt(ctx->i32, 0, false), ""); /* vindex */
2427 params[2] = LLVMConstInt(ctx->i32, 0, false); /* voffset */
2428 params[3] = LLVMConstInt(ctx->i1, 0, false); /* glc */
2429 params[4] = LLVMConstInt(ctx->i1, 0, false); /* slc */
2430 res = ac_emit_llvm_intrinsic(&ctx->ac, "llvm.amdgcn.buffer.load.format.v4f32", ctx->v4f32,
2431 params, 5, 0);
2432
2433 res = trim_vector(ctx, res, instr->dest.ssa.num_components);
2434 res = to_integer(ctx, res);
2435 } else {
2436 bool is_da = glsl_sampler_type_is_array(type) ||
2437 glsl_get_sampler_dim(type) == GLSL_SAMPLER_DIM_CUBE;
2438 bool add_frag_pos = glsl_get_sampler_dim(type) == GLSL_SAMPLER_DIM_SUBPASS;
2439 LLVMValueRef da = is_da ? ctx->i32one : ctx->i32zero;
2440 LLVMValueRef glc = LLVMConstInt(ctx->i1, 0, false);
2441 LLVMValueRef slc = LLVMConstInt(ctx->i1, 0, false);
2442
2443 params[0] = get_image_coords(ctx, instr, add_frag_pos);
2444 params[1] = get_sampler_desc(ctx, instr->variables[0], DESC_IMAGE);
2445 params[2] = LLVMConstInt(ctx->i32, 15, false); /* dmask */
2446 if (HAVE_LLVM <= 0x0309) {
2447 params[3] = LLVMConstInt(ctx->i1, 0, false); /* r128 */
2448 params[4] = da;
2449 params[5] = glc;
2450 params[6] = slc;
2451 } else {
2452 LLVMValueRef lwe = LLVMConstInt(ctx->i1, 0, false);
2453 params[3] = glc;
2454 params[4] = slc;
2455 params[5] = lwe;
2456 params[6] = da;
2457 }
2458
2459 get_image_intr_name("llvm.amdgcn.image.load",
2460 ctx->v4f32, /* vdata */
2461 LLVMTypeOf(params[0]), /* coords */
2462 LLVMTypeOf(params[1]), /* rsrc */
2463 intrinsic_name, sizeof(intrinsic_name));
2464
2465 res = ac_emit_llvm_intrinsic(&ctx->ac, intrinsic_name, ctx->v4f32,
2466 params, 7, AC_FUNC_ATTR_READONLY);
2467 }
2468 return to_integer(ctx, res);
2469 }
2470
2471 static void visit_image_store(struct nir_to_llvm_context *ctx,
2472 nir_intrinsic_instr *instr)
2473 {
2474 LLVMValueRef params[8];
2475 char intrinsic_name[64];
2476 const nir_variable *var = instr->variables[0]->var;
2477 LLVMValueRef i1false = LLVMConstInt(ctx->i1, 0, 0);
2478 LLVMValueRef i1true = LLVMConstInt(ctx->i1, 1, 0);
2479 const struct glsl_type *type = glsl_without_array(var->type);
2480
2481 if (ctx->stage == MESA_SHADER_FRAGMENT)
2482 ctx->shader_info->fs.writes_memory = true;
2483
2484 if (glsl_get_sampler_dim(type) == GLSL_SAMPLER_DIM_BUF) {
2485 params[0] = to_float(ctx, get_src(ctx, instr->src[2])); /* data */
2486 params[1] = get_sampler_desc(ctx, instr->variables[0], DESC_BUFFER);
2487 params[2] = LLVMBuildExtractElement(ctx->builder, get_src(ctx, instr->src[0]),
2488 LLVMConstInt(ctx->i32, 0, false), ""); /* vindex */
2489 params[3] = LLVMConstInt(ctx->i32, 0, false); /* voffset */
2490 params[4] = i1false; /* glc */
2491 params[5] = i1false; /* slc */
2492 ac_emit_llvm_intrinsic(&ctx->ac, "llvm.amdgcn.buffer.store.format.v4f32", ctx->voidt,
2493 params, 6, 0);
2494 } else {
2495 bool is_da = glsl_sampler_type_is_array(type) ||
2496 glsl_get_sampler_dim(type) == GLSL_SAMPLER_DIM_CUBE;
2497 LLVMValueRef da = is_da ? i1true : i1false;
2498 LLVMValueRef glc = i1false;
2499 LLVMValueRef slc = i1false;
2500
2501 params[0] = to_float(ctx, get_src(ctx, instr->src[2]));
2502 params[1] = get_image_coords(ctx, instr, false); /* coords */
2503 params[2] = get_sampler_desc(ctx, instr->variables[0], DESC_IMAGE);
2504 params[3] = LLVMConstInt(ctx->i32, 15, false); /* dmask */
2505 if (HAVE_LLVM <= 0x0309) {
2506 params[4] = i1false; /* r128 */
2507 params[5] = da;
2508 params[6] = glc;
2509 params[7] = slc;
2510 } else {
2511 LLVMValueRef lwe = i1false;
2512 params[4] = glc;
2513 params[5] = slc;
2514 params[6] = lwe;
2515 params[7] = da;
2516 }
2517
2518 get_image_intr_name("llvm.amdgcn.image.store",
2519 LLVMTypeOf(params[0]), /* vdata */
2520 LLVMTypeOf(params[1]), /* coords */
2521 LLVMTypeOf(params[2]), /* rsrc */
2522 intrinsic_name, sizeof(intrinsic_name));
2523
2524 ac_emit_llvm_intrinsic(&ctx->ac, intrinsic_name, ctx->voidt,
2525 params, 8, 0);
2526 }
2527
2528 }
2529
2530 static LLVMValueRef visit_image_atomic(struct nir_to_llvm_context *ctx,
2531 nir_intrinsic_instr *instr)
2532 {
2533 LLVMValueRef params[6];
2534 int param_count = 0;
2535 const nir_variable *var = instr->variables[0]->var;
2536 LLVMValueRef i1false = LLVMConstInt(ctx->i1, 0, 0);
2537 LLVMValueRef i1true = LLVMConstInt(ctx->i1, 1, 0);
2538 const char *base_name = "llvm.amdgcn.image.atomic";
2539 const char *atomic_name;
2540 LLVMValueRef coords;
2541 char intrinsic_name[32], coords_type[8];
2542 const struct glsl_type *type = glsl_without_array(var->type);
2543
2544 if (ctx->stage == MESA_SHADER_FRAGMENT)
2545 ctx->shader_info->fs.writes_memory = true;
2546
2547 params[param_count++] = get_src(ctx, instr->src[2]);
2548 if (instr->intrinsic == nir_intrinsic_image_atomic_comp_swap)
2549 params[param_count++] = get_src(ctx, instr->src[3]);
2550
2551 if (glsl_get_sampler_dim(type) == GLSL_SAMPLER_DIM_BUF) {
2552 params[param_count++] = get_sampler_desc(ctx, instr->variables[0], DESC_BUFFER);
2553 coords = params[param_count++] = LLVMBuildExtractElement(ctx->builder, get_src(ctx, instr->src[0]),
2554 LLVMConstInt(ctx->i32, 0, false), ""); /* vindex */
2555 params[param_count++] = ctx->i32zero; /* voffset */
2556 params[param_count++] = i1false; /* glc */
2557 params[param_count++] = i1false; /* slc */
2558 } else {
2559 bool da = glsl_sampler_type_is_array(type) ||
2560 glsl_get_sampler_dim(type) == GLSL_SAMPLER_DIM_CUBE;
2561
2562 coords = params[param_count++] = get_image_coords(ctx, instr, false);
2563 params[param_count++] = get_sampler_desc(ctx, instr->variables[0], DESC_IMAGE);
2564 params[param_count++] = i1false; /* r128 */
2565 params[param_count++] = da ? i1true : i1false; /* da */
2566 params[param_count++] = i1false; /* slc */
2567 }
2568
2569 switch (instr->intrinsic) {
2570 case nir_intrinsic_image_atomic_add:
2571 atomic_name = "add";
2572 break;
2573 case nir_intrinsic_image_atomic_min:
2574 atomic_name = "smin";
2575 break;
2576 case nir_intrinsic_image_atomic_max:
2577 atomic_name = "smax";
2578 break;
2579 case nir_intrinsic_image_atomic_and:
2580 atomic_name = "and";
2581 break;
2582 case nir_intrinsic_image_atomic_or:
2583 atomic_name = "or";
2584 break;
2585 case nir_intrinsic_image_atomic_xor:
2586 atomic_name = "xor";
2587 break;
2588 case nir_intrinsic_image_atomic_exchange:
2589 atomic_name = "swap";
2590 break;
2591 case nir_intrinsic_image_atomic_comp_swap:
2592 atomic_name = "cmpswap";
2593 break;
2594 default:
2595 abort();
2596 }
2597 build_int_type_name(LLVMTypeOf(coords),
2598 coords_type, sizeof(coords_type));
2599
2600 snprintf(intrinsic_name, sizeof(intrinsic_name),
2601 "%s.%s.%s", base_name, atomic_name, coords_type);
2602 return ac_emit_llvm_intrinsic(&ctx->ac, intrinsic_name, ctx->i32, params, param_count, 0);
2603 }
2604
2605 static LLVMValueRef visit_image_size(struct nir_to_llvm_context *ctx,
2606 nir_intrinsic_instr *instr)
2607 {
2608 LLVMValueRef res;
2609 LLVMValueRef params[10];
2610 const nir_variable *var = instr->variables[0]->var;
2611 const struct glsl_type *type = instr->variables[0]->var->type;
2612 bool da = glsl_sampler_type_is_array(var->type) ||
2613 glsl_get_sampler_dim(var->type) == GLSL_SAMPLER_DIM_CUBE;
2614 if(instr->variables[0]->deref.child)
2615 type = instr->variables[0]->deref.child->type;
2616
2617 if (glsl_get_sampler_dim(type) == GLSL_SAMPLER_DIM_BUF)
2618 return get_buffer_size(ctx, get_sampler_desc(ctx, instr->variables[0], DESC_BUFFER), true);
2619 params[0] = ctx->i32zero;
2620 params[1] = get_sampler_desc(ctx, instr->variables[0], DESC_IMAGE);
2621 params[2] = LLVMConstInt(ctx->i32, 15, false);
2622 params[3] = ctx->i32zero;
2623 params[4] = ctx->i32zero;
2624 params[5] = da ? ctx->i32one : ctx->i32zero;
2625 params[6] = ctx->i32zero;
2626 params[7] = ctx->i32zero;
2627 params[8] = ctx->i32zero;
2628 params[9] = ctx->i32zero;
2629
2630 res = ac_emit_llvm_intrinsic(&ctx->ac, "llvm.SI.getresinfo.i32", ctx->v4i32,
2631 params, 10, AC_FUNC_ATTR_READNONE);
2632
2633 if (glsl_get_sampler_dim(type) == GLSL_SAMPLER_DIM_CUBE &&
2634 glsl_sampler_type_is_array(type)) {
2635 LLVMValueRef two = LLVMConstInt(ctx->i32, 2, false);
2636 LLVMValueRef six = LLVMConstInt(ctx->i32, 6, false);
2637 LLVMValueRef z = LLVMBuildExtractElement(ctx->builder, res, two, "");
2638 z = LLVMBuildSDiv(ctx->builder, z, six, "");
2639 res = LLVMBuildInsertElement(ctx->builder, res, z, two, "");
2640 }
2641 return res;
2642 }
2643
2644 static void emit_waitcnt(struct nir_to_llvm_context *ctx)
2645 {
2646 LLVMValueRef args[1] = {
2647 LLVMConstInt(ctx->i32, 0xf70, false),
2648 };
2649 ac_emit_llvm_intrinsic(&ctx->ac, "llvm.amdgcn.s.waitcnt",
2650 ctx->voidt, args, 1, 0);
2651 }
2652
2653 static void emit_barrier(struct nir_to_llvm_context *ctx)
2654 {
2655 // TODO tess
2656 ac_emit_llvm_intrinsic(&ctx->ac, "llvm.amdgcn.s.barrier",
2657 ctx->voidt, NULL, 0, 0);
2658 }
2659
2660 static void emit_discard_if(struct nir_to_llvm_context *ctx,
2661 nir_intrinsic_instr *instr)
2662 {
2663 LLVMValueRef cond;
2664 ctx->shader_info->fs.can_discard = true;
2665
2666 cond = LLVMBuildICmp(ctx->builder, LLVMIntNE,
2667 get_src(ctx, instr->src[0]),
2668 ctx->i32zero, "");
2669
2670 cond = LLVMBuildSelect(ctx->builder, cond,
2671 LLVMConstReal(ctx->f32, -1.0f),
2672 ctx->f32zero, "");
2673 ac_emit_llvm_intrinsic(&ctx->ac, "llvm.AMDGPU.kill",
2674 LLVMVoidTypeInContext(ctx->context),
2675 &cond, 1, 0);
2676 }
2677
2678 static LLVMValueRef
2679 visit_load_local_invocation_index(struct nir_to_llvm_context *ctx)
2680 {
2681 LLVMValueRef result;
2682 LLVMValueRef thread_id = get_thread_id(ctx);
2683 result = LLVMBuildAnd(ctx->builder, ctx->tg_size,
2684 LLVMConstInt(ctx->i32, 0xfc0, false), "");
2685
2686 return LLVMBuildAdd(ctx->builder, result, thread_id, "");
2687 }
2688
2689 static LLVMValueRef visit_var_atomic(struct nir_to_llvm_context *ctx,
2690 nir_intrinsic_instr *instr)
2691 {
2692 LLVMValueRef ptr, result;
2693 int idx = instr->variables[0]->var->data.driver_location;
2694 LLVMValueRef src = get_src(ctx, instr->src[0]);
2695 ptr = get_shared_memory_ptr(ctx, idx, ctx->i32);
2696
2697 if (instr->intrinsic == nir_intrinsic_var_atomic_comp_swap) {
2698 LLVMValueRef src1 = get_src(ctx, instr->src[1]);
2699 result = LLVMBuildAtomicCmpXchg(ctx->builder,
2700 ptr, src, src1,
2701 LLVMAtomicOrderingSequentiallyConsistent,
2702 LLVMAtomicOrderingSequentiallyConsistent,
2703 false);
2704 } else {
2705 LLVMAtomicRMWBinOp op;
2706 switch (instr->intrinsic) {
2707 case nir_intrinsic_var_atomic_add:
2708 op = LLVMAtomicRMWBinOpAdd;
2709 break;
2710 case nir_intrinsic_var_atomic_umin:
2711 op = LLVMAtomicRMWBinOpUMin;
2712 break;
2713 case nir_intrinsic_var_atomic_umax:
2714 op = LLVMAtomicRMWBinOpUMax;
2715 break;
2716 case nir_intrinsic_var_atomic_imin:
2717 op = LLVMAtomicRMWBinOpMin;
2718 break;
2719 case nir_intrinsic_var_atomic_imax:
2720 op = LLVMAtomicRMWBinOpMax;
2721 break;
2722 case nir_intrinsic_var_atomic_and:
2723 op = LLVMAtomicRMWBinOpAnd;
2724 break;
2725 case nir_intrinsic_var_atomic_or:
2726 op = LLVMAtomicRMWBinOpOr;
2727 break;
2728 case nir_intrinsic_var_atomic_xor:
2729 op = LLVMAtomicRMWBinOpXor;
2730 break;
2731 case nir_intrinsic_var_atomic_exchange:
2732 op = LLVMAtomicRMWBinOpXchg;
2733 break;
2734 default:
2735 return NULL;
2736 }
2737
2738 result = LLVMBuildAtomicRMW(ctx->builder, op, ptr, to_integer(ctx, src),
2739 LLVMAtomicOrderingSequentiallyConsistent,
2740 false);
2741 }
2742 return result;
2743 }
2744
2745 #define INTERP_CENTER 0
2746 #define INTERP_CENTROID 1
2747 #define INTERP_SAMPLE 2
2748
2749 static LLVMValueRef lookup_interp_param(struct nir_to_llvm_context *ctx,
2750 enum glsl_interp_mode interp, unsigned location)
2751 {
2752 switch (interp) {
2753 case INTERP_MODE_FLAT:
2754 default:
2755 return NULL;
2756 case INTERP_MODE_SMOOTH:
2757 case INTERP_MODE_NONE:
2758 if (location == INTERP_CENTER)
2759 return ctx->persp_center;
2760 else if (location == INTERP_CENTROID)
2761 return ctx->persp_centroid;
2762 else if (location == INTERP_SAMPLE)
2763 return ctx->persp_sample;
2764 break;
2765 case INTERP_MODE_NOPERSPECTIVE:
2766 if (location == INTERP_CENTER)
2767 return ctx->linear_center;
2768 else if (location == INTERP_CENTROID)
2769 return ctx->linear_centroid;
2770 else if (location == INTERP_SAMPLE)
2771 return ctx->linear_sample;
2772 break;
2773 }
2774 return NULL;
2775 }
2776
2777 static LLVMValueRef load_sample_position(struct nir_to_llvm_context *ctx,
2778 LLVMValueRef sample_id)
2779 {
2780 /* offset = sample_id * 8 (8 = 2 floats containing samplepos.xy) */
2781 LLVMValueRef offset0 = LLVMBuildMul(ctx->builder, sample_id, LLVMConstInt(ctx->i32, 8, false), "");
2782 LLVMValueRef offset1 = LLVMBuildAdd(ctx->builder, offset0, LLVMConstInt(ctx->i32, 4, false), "");
2783 LLVMValueRef result[2];
2784
2785 result[0] = build_indexed_load_const(ctx, ctx->sample_positions, offset0);
2786 result[1] = build_indexed_load_const(ctx, ctx->sample_positions, offset1);
2787
2788 return build_gather_values(ctx, result, 2);
2789 }
2790
2791 static LLVMValueRef load_sample_pos(struct nir_to_llvm_context *ctx)
2792 {
2793 LLVMValueRef values[2];
2794
2795 values[0] = emit_ffract(ctx, ctx->frag_pos[0]);
2796 values[1] = emit_ffract(ctx, ctx->frag_pos[1]);
2797 return build_gather_values(ctx, values, 2);
2798 }
2799
2800 static LLVMValueRef visit_interp(struct nir_to_llvm_context *ctx,
2801 nir_intrinsic_instr *instr)
2802 {
2803 LLVMValueRef result[2];
2804 LLVMValueRef interp_param, attr_number;
2805 unsigned location;
2806 unsigned chan;
2807 LLVMValueRef src_c0, src_c1;
2808 const char *intr_name;
2809 LLVMValueRef src0;
2810 int input_index = instr->variables[0]->var->data.location - VARYING_SLOT_VAR0;
2811 switch (instr->intrinsic) {
2812 case nir_intrinsic_interp_var_at_centroid:
2813 location = INTERP_CENTROID;
2814 break;
2815 case nir_intrinsic_interp_var_at_sample:
2816 case nir_intrinsic_interp_var_at_offset:
2817 location = INTERP_SAMPLE;
2818 src0 = get_src(ctx, instr->src[0]);
2819 break;
2820 default:
2821 break;
2822 }
2823
2824 if (instr->intrinsic == nir_intrinsic_interp_var_at_offset) {
2825 src_c0 = to_float(ctx, LLVMBuildExtractElement(ctx->builder, src0, ctx->i32zero, ""));
2826 src_c1 = to_float(ctx, LLVMBuildExtractElement(ctx->builder, src0, ctx->i32one, ""));
2827 } else if (instr->intrinsic == nir_intrinsic_interp_var_at_sample) {
2828 LLVMValueRef sample_position;
2829 LLVMValueRef halfval = LLVMConstReal(ctx->f32, 0.5f);
2830
2831 /* fetch sample ID */
2832 sample_position = load_sample_position(ctx, src0);
2833
2834 src_c0 = LLVMBuildExtractElement(ctx->builder, sample_position, ctx->i32zero, "");
2835 src_c0 = LLVMBuildFSub(ctx->builder, src_c0, halfval, "");
2836 src_c1 = LLVMBuildExtractElement(ctx->builder, sample_position, ctx->i32one, "");
2837 src_c1 = LLVMBuildFSub(ctx->builder, src_c1, halfval, "");
2838 }
2839 interp_param = lookup_interp_param(ctx, instr->variables[0]->var->data.interpolation, location);
2840 attr_number = LLVMConstInt(ctx->i32, input_index, false);
2841
2842 if (location == INTERP_SAMPLE) {
2843 LLVMValueRef ij_out[2];
2844 LLVMValueRef ddxy_out = emit_ddxy_interp(ctx, interp_param);
2845
2846 /*
2847 * take the I then J parameters, and the DDX/Y for it, and
2848 * calculate the IJ inputs for the interpolator.
2849 * temp1 = ddx * offset/sample.x + I;
2850 * interp_param.I = ddy * offset/sample.y + temp1;
2851 * temp1 = ddx * offset/sample.x + J;
2852 * interp_param.J = ddy * offset/sample.y + temp1;
2853 */
2854 for (unsigned i = 0; i < 2; i++) {
2855 LLVMValueRef ix_ll = LLVMConstInt(ctx->i32, i, false);
2856 LLVMValueRef iy_ll = LLVMConstInt(ctx->i32, i + 2, false);
2857 LLVMValueRef ddx_el = LLVMBuildExtractElement(ctx->builder,
2858 ddxy_out, ix_ll, "");
2859 LLVMValueRef ddy_el = LLVMBuildExtractElement(ctx->builder,
2860 ddxy_out, iy_ll, "");
2861 LLVMValueRef interp_el = LLVMBuildExtractElement(ctx->builder,
2862 interp_param, ix_ll, "");
2863 LLVMValueRef temp1, temp2;
2864
2865 interp_el = LLVMBuildBitCast(ctx->builder, interp_el,
2866 ctx->f32, "");
2867
2868 temp1 = LLVMBuildFMul(ctx->builder, ddx_el, src_c0, "");
2869 temp1 = LLVMBuildFAdd(ctx->builder, temp1, interp_el, "");
2870
2871 temp2 = LLVMBuildFMul(ctx->builder, ddy_el, src_c1, "");
2872 temp2 = LLVMBuildFAdd(ctx->builder, temp2, temp1, "");
2873
2874 ij_out[i] = LLVMBuildBitCast(ctx->builder,
2875 temp2, ctx->i32, "");
2876 }
2877 interp_param = build_gather_values(ctx, ij_out, 2);
2878
2879 }
2880 intr_name = interp_param ? "llvm.SI.fs.interp" : "llvm.SI.fs.constant";
2881 for (chan = 0; chan < 2; chan++) {
2882 LLVMValueRef args[4];
2883 LLVMValueRef llvm_chan = LLVMConstInt(ctx->i32, chan, false);
2884
2885 args[0] = llvm_chan;
2886 args[1] = attr_number;
2887 args[2] = ctx->prim_mask;
2888 args[3] = interp_param;
2889 result[chan] = ac_emit_llvm_intrinsic(&ctx->ac, intr_name,
2890 ctx->f32, args, args[3] ? 4 : 3,
2891 AC_FUNC_ATTR_READNONE);
2892 }
2893 return build_gather_values(ctx, result, 2);
2894 }
2895
2896 static void visit_intrinsic(struct nir_to_llvm_context *ctx,
2897 nir_intrinsic_instr *instr)
2898 {
2899 LLVMValueRef result = NULL;
2900
2901 switch (instr->intrinsic) {
2902 case nir_intrinsic_load_work_group_id: {
2903 result = ctx->workgroup_ids;
2904 break;
2905 }
2906 case nir_intrinsic_load_base_vertex: {
2907 result = ctx->base_vertex;
2908 break;
2909 }
2910 case nir_intrinsic_load_vertex_id_zero_base: {
2911 result = ctx->vertex_id;
2912 break;
2913 }
2914 case nir_intrinsic_load_local_invocation_id: {
2915 result = ctx->local_invocation_ids;
2916 break;
2917 }
2918 case nir_intrinsic_load_base_instance:
2919 result = ctx->start_instance;
2920 break;
2921 case nir_intrinsic_load_sample_id:
2922 ctx->shader_info->fs.force_persample = true;
2923 result = unpack_param(ctx, ctx->ancillary, 8, 4);
2924 break;
2925 case nir_intrinsic_load_sample_pos:
2926 ctx->shader_info->fs.force_persample = true;
2927 result = load_sample_pos(ctx);
2928 break;
2929 case nir_intrinsic_load_front_face:
2930 result = ctx->front_face;
2931 break;
2932 case nir_intrinsic_load_instance_id:
2933 result = ctx->instance_id;
2934 ctx->shader_info->vs.vgpr_comp_cnt = MAX2(3,
2935 ctx->shader_info->vs.vgpr_comp_cnt);
2936 break;
2937 case nir_intrinsic_load_num_work_groups:
2938 result = ctx->num_work_groups;
2939 break;
2940 case nir_intrinsic_load_local_invocation_index:
2941 result = visit_load_local_invocation_index(ctx);
2942 break;
2943 case nir_intrinsic_load_push_constant:
2944 result = visit_load_push_constant(ctx, instr);
2945 break;
2946 case nir_intrinsic_vulkan_resource_index:
2947 result = visit_vulkan_resource_index(ctx, instr);
2948 break;
2949 case nir_intrinsic_store_ssbo:
2950 visit_store_ssbo(ctx, instr);
2951 break;
2952 case nir_intrinsic_load_ssbo:
2953 result = visit_load_buffer(ctx, instr);
2954 break;
2955 case nir_intrinsic_ssbo_atomic_add:
2956 case nir_intrinsic_ssbo_atomic_imin:
2957 case nir_intrinsic_ssbo_atomic_umin:
2958 case nir_intrinsic_ssbo_atomic_imax:
2959 case nir_intrinsic_ssbo_atomic_umax:
2960 case nir_intrinsic_ssbo_atomic_and:
2961 case nir_intrinsic_ssbo_atomic_or:
2962 case nir_intrinsic_ssbo_atomic_xor:
2963 case nir_intrinsic_ssbo_atomic_exchange:
2964 case nir_intrinsic_ssbo_atomic_comp_swap:
2965 result = visit_atomic_ssbo(ctx, instr);
2966 break;
2967 case nir_intrinsic_load_ubo:
2968 result = visit_load_ubo_buffer(ctx, instr);
2969 break;
2970 case nir_intrinsic_get_buffer_size:
2971 result = visit_get_buffer_size(ctx, instr);
2972 break;
2973 case nir_intrinsic_load_var:
2974 result = visit_load_var(ctx, instr);
2975 break;
2976 case nir_intrinsic_store_var:
2977 visit_store_var(ctx, instr);
2978 break;
2979 case nir_intrinsic_image_load:
2980 result = visit_image_load(ctx, instr);
2981 break;
2982 case nir_intrinsic_image_store:
2983 visit_image_store(ctx, instr);
2984 break;
2985 case nir_intrinsic_image_atomic_add:
2986 case nir_intrinsic_image_atomic_min:
2987 case nir_intrinsic_image_atomic_max:
2988 case nir_intrinsic_image_atomic_and:
2989 case nir_intrinsic_image_atomic_or:
2990 case nir_intrinsic_image_atomic_xor:
2991 case nir_intrinsic_image_atomic_exchange:
2992 case nir_intrinsic_image_atomic_comp_swap:
2993 result = visit_image_atomic(ctx, instr);
2994 break;
2995 case nir_intrinsic_image_size:
2996 result = visit_image_size(ctx, instr);
2997 break;
2998 case nir_intrinsic_discard:
2999 ctx->shader_info->fs.can_discard = true;
3000 ac_emit_llvm_intrinsic(&ctx->ac, "llvm.AMDGPU.kilp",
3001 LLVMVoidTypeInContext(ctx->context),
3002 NULL, 0, 0);
3003 break;
3004 case nir_intrinsic_discard_if:
3005 emit_discard_if(ctx, instr);
3006 break;
3007 case nir_intrinsic_memory_barrier:
3008 emit_waitcnt(ctx);
3009 break;
3010 case nir_intrinsic_barrier:
3011 emit_barrier(ctx);
3012 break;
3013 case nir_intrinsic_var_atomic_add:
3014 case nir_intrinsic_var_atomic_imin:
3015 case nir_intrinsic_var_atomic_umin:
3016 case nir_intrinsic_var_atomic_imax:
3017 case nir_intrinsic_var_atomic_umax:
3018 case nir_intrinsic_var_atomic_and:
3019 case nir_intrinsic_var_atomic_or:
3020 case nir_intrinsic_var_atomic_xor:
3021 case nir_intrinsic_var_atomic_exchange:
3022 case nir_intrinsic_var_atomic_comp_swap:
3023 result = visit_var_atomic(ctx, instr);
3024 break;
3025 case nir_intrinsic_interp_var_at_centroid:
3026 case nir_intrinsic_interp_var_at_sample:
3027 case nir_intrinsic_interp_var_at_offset:
3028 result = visit_interp(ctx, instr);
3029 break;
3030 default:
3031 fprintf(stderr, "Unknown intrinsic: ");
3032 nir_print_instr(&instr->instr, stderr);
3033 fprintf(stderr, "\n");
3034 break;
3035 }
3036 if (result) {
3037 _mesa_hash_table_insert(ctx->defs, &instr->dest.ssa, result);
3038 }
3039 }
3040
3041 static LLVMValueRef get_sampler_desc(struct nir_to_llvm_context *ctx,
3042 nir_deref_var *deref,
3043 enum desc_type desc_type)
3044 {
3045 unsigned desc_set = deref->var->data.descriptor_set;
3046 LLVMValueRef list = ctx->descriptor_sets[desc_set];
3047 struct radv_descriptor_set_layout *layout = ctx->options->layout->set[desc_set].layout;
3048 struct radv_descriptor_set_binding_layout *binding = layout->binding + deref->var->data.binding;
3049 unsigned offset = binding->offset;
3050 unsigned stride = binding->size;
3051 unsigned type_size;
3052 LLVMBuilderRef builder = ctx->builder;
3053 LLVMTypeRef type;
3054 LLVMValueRef index = NULL;
3055
3056 assert(deref->var->data.binding < layout->binding_count);
3057
3058 switch (desc_type) {
3059 case DESC_IMAGE:
3060 type = ctx->v8i32;
3061 type_size = 32;
3062 break;
3063 case DESC_FMASK:
3064 type = ctx->v8i32;
3065 offset += 32;
3066 type_size = 32;
3067 break;
3068 case DESC_SAMPLER:
3069 type = ctx->v4i32;
3070 if (binding->type == VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER)
3071 offset += 64;
3072
3073 type_size = 16;
3074 break;
3075 case DESC_BUFFER:
3076 type = ctx->v4i32;
3077 type_size = 16;
3078 break;
3079 default:
3080 unreachable("invalid desc_type\n");
3081 }
3082
3083 if (deref->deref.child) {
3084 nir_deref_array *child = (nir_deref_array*)deref->deref.child;
3085
3086 assert(child->deref_array_type != nir_deref_array_type_wildcard);
3087 offset += child->base_offset * stride;
3088 if (child->deref_array_type == nir_deref_array_type_indirect) {
3089 index = get_src(ctx, child->indirect);
3090 }
3091 }
3092
3093 assert(stride % type_size == 0);
3094
3095 if (!index)
3096 index = ctx->i32zero;
3097
3098 index = LLVMBuildMul(builder, index, LLVMConstInt(ctx->i32, stride / type_size, 0), "");
3099
3100 list = build_gep0(ctx, list, LLVMConstInt(ctx->i32, offset, 0));
3101 list = LLVMBuildPointerCast(builder, list, const_array(type, 0), "");
3102
3103 return build_indexed_load_const(ctx, list, index);
3104 }
3105
3106 static void set_tex_fetch_args(struct nir_to_llvm_context *ctx,
3107 struct ac_tex_info *tinfo,
3108 nir_tex_instr *instr,
3109 nir_texop op,
3110 LLVMValueRef res_ptr, LLVMValueRef samp_ptr,
3111 LLVMValueRef *param, unsigned count,
3112 unsigned dmask)
3113 {
3114 int num_args;
3115 unsigned is_rect = 0;
3116 bool da = instr->is_array || instr->sampler_dim == GLSL_SAMPLER_DIM_CUBE;
3117
3118 if (op == nir_texop_lod)
3119 da = false;
3120 /* Pad to power of two vector */
3121 while (count < util_next_power_of_two(count))
3122 param[count++] = LLVMGetUndef(ctx->i32);
3123
3124 if (count > 1)
3125 tinfo->args[0] = build_gather_values(ctx, param, count);
3126 else
3127 tinfo->args[0] = param[0];
3128
3129 tinfo->args[1] = res_ptr;
3130 num_args = 2;
3131
3132 if (op == nir_texop_txf ||
3133 op == nir_texop_txf_ms ||
3134 op == nir_texop_query_levels ||
3135 op == nir_texop_texture_samples ||
3136 op == nir_texop_txs)
3137 tinfo->dst_type = ctx->v4i32;
3138 else {
3139 tinfo->dst_type = ctx->v4f32;
3140 tinfo->args[num_args++] = samp_ptr;
3141 }
3142
3143 if (instr->sampler_dim == GLSL_SAMPLER_DIM_BUF && op == nir_texop_txf) {
3144 tinfo->args[0] = res_ptr;
3145 tinfo->args[1] = LLVMConstInt(ctx->i32, 0, false);
3146 tinfo->args[2] = param[0];
3147 tinfo->arg_count = 3;
3148 return;
3149 }
3150
3151 tinfo->args[num_args++] = LLVMConstInt(ctx->i32, dmask, 0);
3152 tinfo->args[num_args++] = LLVMConstInt(ctx->i32, is_rect, 0); /* unorm */
3153 tinfo->args[num_args++] = LLVMConstInt(ctx->i32, 0, 0); /* r128 */
3154 tinfo->args[num_args++] = LLVMConstInt(ctx->i32, da ? 1 : 0, 0);
3155 tinfo->args[num_args++] = LLVMConstInt(ctx->i32, 0, 0); /* glc */
3156 tinfo->args[num_args++] = LLVMConstInt(ctx->i32, 0, 0); /* slc */
3157 tinfo->args[num_args++] = LLVMConstInt(ctx->i32, 0, 0); /* tfe */
3158 tinfo->args[num_args++] = LLVMConstInt(ctx->i32, 0, 0); /* lwe */
3159
3160 tinfo->arg_count = num_args;
3161 }
3162
3163 /* Disable anisotropic filtering if BASE_LEVEL == LAST_LEVEL.
3164 *
3165 * SI-CI:
3166 * If BASE_LEVEL == LAST_LEVEL, the shader must disable anisotropic
3167 * filtering manually. The driver sets img7 to a mask clearing
3168 * MAX_ANISO_RATIO if BASE_LEVEL == LAST_LEVEL. The shader must do:
3169 * s_and_b32 samp0, samp0, img7
3170 *
3171 * VI:
3172 * The ANISO_OVERRIDE sampler field enables this fix in TA.
3173 */
3174 static LLVMValueRef sici_fix_sampler_aniso(struct nir_to_llvm_context *ctx,
3175 LLVMValueRef res, LLVMValueRef samp)
3176 {
3177 LLVMBuilderRef builder = ctx->builder;
3178 LLVMValueRef img7, samp0;
3179
3180 if (ctx->options->chip_class >= VI)
3181 return samp;
3182
3183 img7 = LLVMBuildExtractElement(builder, res,
3184 LLVMConstInt(ctx->i32, 7, 0), "");
3185 samp0 = LLVMBuildExtractElement(builder, samp,
3186 LLVMConstInt(ctx->i32, 0, 0), "");
3187 samp0 = LLVMBuildAnd(builder, samp0, img7, "");
3188 return LLVMBuildInsertElement(builder, samp, samp0,
3189 LLVMConstInt(ctx->i32, 0, 0), "");
3190 }
3191
3192 static void tex_fetch_ptrs(struct nir_to_llvm_context *ctx,
3193 nir_tex_instr *instr,
3194 LLVMValueRef *res_ptr, LLVMValueRef *samp_ptr,
3195 LLVMValueRef *fmask_ptr)
3196 {
3197 if (instr->sampler_dim == GLSL_SAMPLER_DIM_BUF)
3198 *res_ptr = get_sampler_desc(ctx, instr->texture, DESC_BUFFER);
3199 else
3200 *res_ptr = get_sampler_desc(ctx, instr->texture, DESC_IMAGE);
3201 if (samp_ptr) {
3202 if (instr->sampler)
3203 *samp_ptr = get_sampler_desc(ctx, instr->sampler, DESC_SAMPLER);
3204 else
3205 *samp_ptr = get_sampler_desc(ctx, instr->texture, DESC_SAMPLER);
3206 if (instr->sampler_dim < GLSL_SAMPLER_DIM_RECT)
3207 *samp_ptr = sici_fix_sampler_aniso(ctx, *res_ptr, *samp_ptr);
3208 }
3209 if (fmask_ptr && !instr->sampler && (instr->op == nir_texop_txf_ms ||
3210 instr->op == nir_texop_samples_identical))
3211 *fmask_ptr = get_sampler_desc(ctx, instr->texture, DESC_FMASK);
3212 }
3213
3214 static void visit_tex(struct nir_to_llvm_context *ctx, nir_tex_instr *instr)
3215 {
3216 LLVMValueRef result = NULL;
3217 struct ac_tex_info tinfo = { 0 };
3218 unsigned dmask = 0xf;
3219 LLVMValueRef address[16];
3220 LLVMValueRef coords[5];
3221 LLVMValueRef coord = NULL, lod = NULL, comparator = NULL;
3222 LLVMValueRef bias = NULL, offsets = NULL;
3223 LLVMValueRef res_ptr, samp_ptr, fmask_ptr = NULL, sample_index = NULL;
3224 LLVMValueRef ddx = NULL, ddy = NULL;
3225 LLVMValueRef derivs[6];
3226 unsigned chan, count = 0;
3227 unsigned const_src = 0, num_deriv_comp = 0;
3228
3229 tex_fetch_ptrs(ctx, instr, &res_ptr, &samp_ptr, &fmask_ptr);
3230
3231 for (unsigned i = 0; i < instr->num_srcs; i++) {
3232 switch (instr->src[i].src_type) {
3233 case nir_tex_src_coord:
3234 coord = get_src(ctx, instr->src[i].src);
3235 break;
3236 case nir_tex_src_projector:
3237 break;
3238 case nir_tex_src_comparator:
3239 comparator = get_src(ctx, instr->src[i].src);
3240 break;
3241 case nir_tex_src_offset:
3242 offsets = get_src(ctx, instr->src[i].src);
3243 const_src = i;
3244 break;
3245 case nir_tex_src_bias:
3246 bias = get_src(ctx, instr->src[i].src);
3247 break;
3248 case nir_tex_src_lod:
3249 lod = get_src(ctx, instr->src[i].src);
3250 break;
3251 case nir_tex_src_ms_index:
3252 sample_index = get_src(ctx, instr->src[i].src);
3253 break;
3254 case nir_tex_src_ms_mcs:
3255 break;
3256 case nir_tex_src_ddx:
3257 ddx = get_src(ctx, instr->src[i].src);
3258 num_deriv_comp = instr->src[i].src.ssa->num_components;
3259 break;
3260 case nir_tex_src_ddy:
3261 ddy = get_src(ctx, instr->src[i].src);
3262 break;
3263 case nir_tex_src_texture_offset:
3264 case nir_tex_src_sampler_offset:
3265 case nir_tex_src_plane:
3266 default:
3267 break;
3268 }
3269 }
3270
3271 if (instr->op == nir_texop_texture_samples) {
3272 LLVMValueRef res, samples, is_msaa;
3273 res = LLVMBuildBitCast(ctx->builder, res_ptr, ctx->v8i32, "");
3274 samples = LLVMBuildExtractElement(ctx->builder, res,
3275 LLVMConstInt(ctx->i32, 3, false), "");
3276 is_msaa = LLVMBuildLShr(ctx->builder, samples,
3277 LLVMConstInt(ctx->i32, 28, false), "");
3278 is_msaa = LLVMBuildAnd(ctx->builder, is_msaa,
3279 LLVMConstInt(ctx->i32, 0xe, false), "");
3280 is_msaa = LLVMBuildICmp(ctx->builder, LLVMIntEQ, is_msaa,
3281 LLVMConstInt(ctx->i32, 0xe, false), "");
3282
3283 samples = LLVMBuildLShr(ctx->builder, samples,
3284 LLVMConstInt(ctx->i32, 16, false), "");
3285 samples = LLVMBuildAnd(ctx->builder, samples,
3286 LLVMConstInt(ctx->i32, 0xf, false), "");
3287 samples = LLVMBuildShl(ctx->builder, ctx->i32one,
3288 samples, "");
3289 samples = LLVMBuildSelect(ctx->builder, is_msaa, samples,
3290 ctx->i32one, "");
3291 result = samples;
3292 goto write_result;
3293 }
3294
3295 if (coord)
3296 for (chan = 0; chan < instr->coord_components; chan++)
3297 coords[chan] = llvm_extract_elem(ctx, coord, chan);
3298
3299 if (offsets && instr->op != nir_texop_txf) {
3300 LLVMValueRef offset[3], pack;
3301 for (chan = 0; chan < 3; ++chan)
3302 offset[chan] = ctx->i32zero;
3303
3304 tinfo.has_offset = true;
3305 for (chan = 0; chan < get_llvm_num_components(offsets); chan++) {
3306 offset[chan] = llvm_extract_elem(ctx, offsets, chan);
3307 offset[chan] = LLVMBuildAnd(ctx->builder, offset[chan],
3308 LLVMConstInt(ctx->i32, 0x3f, false), "");
3309 if (chan)
3310 offset[chan] = LLVMBuildShl(ctx->builder, offset[chan],
3311 LLVMConstInt(ctx->i32, chan * 8, false), "");
3312 }
3313 pack = LLVMBuildOr(ctx->builder, offset[0], offset[1], "");
3314 pack = LLVMBuildOr(ctx->builder, pack, offset[2], "");
3315 address[count++] = pack;
3316
3317 }
3318 /* pack LOD bias value */
3319 if (instr->op == nir_texop_txb && bias) {
3320 address[count++] = bias;
3321 }
3322
3323 /* Pack depth comparison value */
3324 if (instr->is_shadow && comparator) {
3325 address[count++] = llvm_extract_elem(ctx, comparator, 0);
3326 }
3327
3328 /* pack derivatives */
3329 if (ddx || ddy) {
3330 switch (instr->sampler_dim) {
3331 case GLSL_SAMPLER_DIM_3D:
3332 case GLSL_SAMPLER_DIM_CUBE:
3333 num_deriv_comp = 3;
3334 break;
3335 case GLSL_SAMPLER_DIM_2D:
3336 default:
3337 num_deriv_comp = 2;
3338 break;
3339 case GLSL_SAMPLER_DIM_1D:
3340 num_deriv_comp = 1;
3341 break;
3342 }
3343
3344 for (unsigned i = 0; i < num_deriv_comp; i++) {
3345 derivs[i * 2] = to_float(ctx, llvm_extract_elem(ctx, ddx, i));
3346 derivs[i * 2 + 1] = to_float(ctx, llvm_extract_elem(ctx, ddy, i));
3347 }
3348 }
3349
3350 if (instr->sampler_dim == GLSL_SAMPLER_DIM_CUBE && coord) {
3351 for (chan = 0; chan < instr->coord_components; chan++)
3352 coords[chan] = to_float(ctx, coords[chan]);
3353 if (instr->coord_components == 3)
3354 coords[3] = LLVMGetUndef(ctx->f32);
3355 ac_prepare_cube_coords(&ctx->ac,
3356 instr->op == nir_texop_txd, instr->is_array,
3357 coords, derivs);
3358 if (num_deriv_comp)
3359 num_deriv_comp--;
3360 }
3361
3362 if (ddx || ddy) {
3363 for (unsigned i = 0; i < num_deriv_comp * 2; i++)
3364 address[count++] = derivs[i];
3365 }
3366
3367 /* Pack texture coordinates */
3368 if (coord) {
3369 address[count++] = coords[0];
3370 if (instr->coord_components > 1)
3371 address[count++] = coords[1];
3372 if (instr->coord_components > 2) {
3373 /* This seems like a bit of a hack - but it passes Vulkan CTS with it */
3374 if (instr->sampler_dim != GLSL_SAMPLER_DIM_3D && instr->op != nir_texop_txf) {
3375 coords[2] = to_float(ctx, coords[2]);
3376 coords[2] = ac_emit_llvm_intrinsic(&ctx->ac, "llvm.rint.f32", ctx->f32, &coords[2],
3377 1, 0);
3378 coords[2] = to_integer(ctx, coords[2]);
3379 }
3380 address[count++] = coords[2];
3381 }
3382 }
3383
3384 /* Pack LOD */
3385 if ((instr->op == nir_texop_txl || instr->op == nir_texop_txf) && lod) {
3386 address[count++] = lod;
3387 } else if (instr->op == nir_texop_txf_ms && sample_index) {
3388 address[count++] = sample_index;
3389 } else if(instr->op == nir_texop_txs) {
3390 count = 0;
3391 if (lod)
3392 address[count++] = lod;
3393 else
3394 address[count++] = ctx->i32zero;
3395 }
3396
3397 for (chan = 0; chan < count; chan++) {
3398 address[chan] = LLVMBuildBitCast(ctx->builder,
3399 address[chan], ctx->i32, "");
3400 }
3401
3402 if (instr->op == nir_texop_samples_identical) {
3403 LLVMValueRef txf_address[4];
3404 struct ac_tex_info txf_info = { 0 };
3405 unsigned txf_count = count;
3406 memcpy(txf_address, address, sizeof(txf_address));
3407
3408 if (!instr->is_array)
3409 txf_address[2] = ctx->i32zero;
3410 txf_address[3] = ctx->i32zero;
3411
3412 set_tex_fetch_args(ctx, &txf_info, instr, nir_texop_txf,
3413 fmask_ptr, NULL,
3414 txf_address, txf_count, 0xf);
3415
3416 result = build_tex_intrinsic(ctx, instr, &txf_info);
3417
3418 result = LLVMBuildExtractElement(ctx->builder, result, ctx->i32zero, "");
3419 result = emit_int_cmp(ctx, LLVMIntEQ, result, ctx->i32zero);
3420 goto write_result;
3421 }
3422
3423 /* Adjust the sample index according to FMASK.
3424 *
3425 * For uncompressed MSAA surfaces, FMASK should return 0x76543210,
3426 * which is the identity mapping. Each nibble says which physical sample
3427 * should be fetched to get that sample.
3428 *
3429 * For example, 0x11111100 means there are only 2 samples stored and
3430 * the second sample covers 3/4 of the pixel. When reading samples 0
3431 * and 1, return physical sample 0 (determined by the first two 0s
3432 * in FMASK), otherwise return physical sample 1.
3433 *
3434 * The sample index should be adjusted as follows:
3435 * sample_index = (fmask >> (sample_index * 4)) & 0xF;
3436 */
3437 if (instr->sampler_dim == GLSL_SAMPLER_DIM_MS) {
3438 LLVMValueRef txf_address[4];
3439 struct ac_tex_info txf_info = { 0 };
3440 unsigned txf_count = count;
3441 memcpy(txf_address, address, sizeof(txf_address));
3442
3443 if (!instr->is_array)
3444 txf_address[2] = ctx->i32zero;
3445 txf_address[3] = ctx->i32zero;
3446
3447 set_tex_fetch_args(ctx, &txf_info, instr, nir_texop_txf,
3448 fmask_ptr, NULL,
3449 txf_address, txf_count, 0xf);
3450
3451 result = build_tex_intrinsic(ctx, instr, &txf_info);
3452 LLVMValueRef four = LLVMConstInt(ctx->i32, 4, false);
3453 LLVMValueRef F = LLVMConstInt(ctx->i32, 0xf, false);
3454
3455 LLVMValueRef fmask = LLVMBuildExtractElement(ctx->builder,
3456 result,
3457 ctx->i32zero, "");
3458
3459 unsigned sample_chan = instr->is_array ? 3 : 2;
3460
3461 LLVMValueRef sample_index4 =
3462 LLVMBuildMul(ctx->builder, address[sample_chan], four, "");
3463 LLVMValueRef shifted_fmask =
3464 LLVMBuildLShr(ctx->builder, fmask, sample_index4, "");
3465 LLVMValueRef final_sample =
3466 LLVMBuildAnd(ctx->builder, shifted_fmask, F, "");
3467
3468 /* Don't rewrite the sample index if WORD1.DATA_FORMAT of the FMASK
3469 * resource descriptor is 0 (invalid),
3470 */
3471 LLVMValueRef fmask_desc =
3472 LLVMBuildBitCast(ctx->builder, fmask_ptr,
3473 ctx->v8i32, "");
3474
3475 LLVMValueRef fmask_word1 =
3476 LLVMBuildExtractElement(ctx->builder, fmask_desc,
3477 ctx->i32one, "");
3478
3479 LLVMValueRef word1_is_nonzero =
3480 LLVMBuildICmp(ctx->builder, LLVMIntNE,
3481 fmask_word1, ctx->i32zero, "");
3482
3483 /* Replace the MSAA sample index. */
3484 address[sample_chan] =
3485 LLVMBuildSelect(ctx->builder, word1_is_nonzero,
3486 final_sample, address[sample_chan], "");
3487 }
3488
3489 if (offsets && instr->op == nir_texop_txf) {
3490 nir_const_value *const_offset =
3491 nir_src_as_const_value(instr->src[const_src].src);
3492 int num_offsets = instr->src[const_src].src.ssa->num_components;
3493 assert(const_offset);
3494 num_offsets = MIN2(num_offsets, instr->coord_components);
3495 if (num_offsets > 2)
3496 address[2] = LLVMBuildAdd(ctx->builder,
3497 address[2], LLVMConstInt(ctx->i32, const_offset->i32[2], false), "");
3498 if (num_offsets > 1)
3499 address[1] = LLVMBuildAdd(ctx->builder,
3500 address[1], LLVMConstInt(ctx->i32, const_offset->i32[1], false), "");
3501 address[0] = LLVMBuildAdd(ctx->builder,
3502 address[0], LLVMConstInt(ctx->i32, const_offset->i32[0], false), "");
3503
3504 }
3505
3506 /* TODO TG4 support */
3507 if (instr->op == nir_texop_tg4) {
3508 if (instr->is_shadow)
3509 dmask = 1;
3510 else
3511 dmask = 1 << instr->component;
3512 }
3513 set_tex_fetch_args(ctx, &tinfo, instr, instr->op,
3514 res_ptr, samp_ptr, address, count, dmask);
3515
3516 result = build_tex_intrinsic(ctx, instr, &tinfo);
3517
3518 if (instr->op == nir_texop_query_levels)
3519 result = LLVMBuildExtractElement(ctx->builder, result, LLVMConstInt(ctx->i32, 3, false), "");
3520 else if (instr->is_shadow && instr->op != nir_texop_txs && instr->op != nir_texop_lod && instr->op != nir_texop_tg4)
3521 result = LLVMBuildExtractElement(ctx->builder, result, ctx->i32zero, "");
3522 else if (instr->op == nir_texop_txs &&
3523 instr->sampler_dim == GLSL_SAMPLER_DIM_CUBE &&
3524 instr->is_array) {
3525 LLVMValueRef two = LLVMConstInt(ctx->i32, 2, false);
3526 LLVMValueRef six = LLVMConstInt(ctx->i32, 6, false);
3527 LLVMValueRef z = LLVMBuildExtractElement(ctx->builder, result, two, "");
3528 z = LLVMBuildSDiv(ctx->builder, z, six, "");
3529 result = LLVMBuildInsertElement(ctx->builder, result, z, two, "");
3530 } else if (instr->dest.ssa.num_components != 4)
3531 result = trim_vector(ctx, result, instr->dest.ssa.num_components);
3532
3533 write_result:
3534 if (result) {
3535 assert(instr->dest.is_ssa);
3536 result = to_integer(ctx, result);
3537 _mesa_hash_table_insert(ctx->defs, &instr->dest.ssa, result);
3538 }
3539 }
3540
3541
3542 static void visit_phi(struct nir_to_llvm_context *ctx, nir_phi_instr *instr)
3543 {
3544 LLVMTypeRef type = get_def_type(ctx, &instr->dest.ssa);
3545 LLVMValueRef result = LLVMBuildPhi(ctx->builder, type, "");
3546
3547 _mesa_hash_table_insert(ctx->defs, &instr->dest.ssa, result);
3548 _mesa_hash_table_insert(ctx->phis, instr, result);
3549 }
3550
3551 static void visit_post_phi(struct nir_to_llvm_context *ctx,
3552 nir_phi_instr *instr,
3553 LLVMValueRef llvm_phi)
3554 {
3555 nir_foreach_phi_src(src, instr) {
3556 LLVMBasicBlockRef block = get_block(ctx, src->pred);
3557 LLVMValueRef llvm_src = get_src(ctx, src->src);
3558
3559 LLVMAddIncoming(llvm_phi, &llvm_src, &block, 1);
3560 }
3561 }
3562
3563 static void phi_post_pass(struct nir_to_llvm_context *ctx)
3564 {
3565 struct hash_entry *entry;
3566 hash_table_foreach(ctx->phis, entry) {
3567 visit_post_phi(ctx, (nir_phi_instr*)entry->key,
3568 (LLVMValueRef)entry->data);
3569 }
3570 }
3571
3572
3573 static void visit_ssa_undef(struct nir_to_llvm_context *ctx,
3574 nir_ssa_undef_instr *instr)
3575 {
3576 unsigned num_components = instr->def.num_components;
3577 LLVMValueRef undef;
3578
3579 if (num_components == 1)
3580 undef = LLVMGetUndef(ctx->i32);
3581 else {
3582 undef = LLVMGetUndef(LLVMVectorType(ctx->i32, num_components));
3583 }
3584 _mesa_hash_table_insert(ctx->defs, &instr->def, undef);
3585 }
3586
3587 static void visit_jump(struct nir_to_llvm_context *ctx,
3588 nir_jump_instr *instr)
3589 {
3590 switch (instr->type) {
3591 case nir_jump_break:
3592 LLVMBuildBr(ctx->builder, ctx->break_block);
3593 LLVMClearInsertionPosition(ctx->builder);
3594 break;
3595 case nir_jump_continue:
3596 LLVMBuildBr(ctx->builder, ctx->continue_block);
3597 LLVMClearInsertionPosition(ctx->builder);
3598 break;
3599 default:
3600 fprintf(stderr, "Unknown NIR jump instr: ");
3601 nir_print_instr(&instr->instr, stderr);
3602 fprintf(stderr, "\n");
3603 abort();
3604 }
3605 }
3606
3607 static void visit_cf_list(struct nir_to_llvm_context *ctx,
3608 struct exec_list *list);
3609
3610 static void visit_block(struct nir_to_llvm_context *ctx, nir_block *block)
3611 {
3612 LLVMBasicBlockRef llvm_block = LLVMGetInsertBlock(ctx->builder);
3613 nir_foreach_instr(instr, block)
3614 {
3615 switch (instr->type) {
3616 case nir_instr_type_alu:
3617 visit_alu(ctx, nir_instr_as_alu(instr));
3618 break;
3619 case nir_instr_type_load_const:
3620 visit_load_const(ctx, nir_instr_as_load_const(instr));
3621 break;
3622 case nir_instr_type_intrinsic:
3623 visit_intrinsic(ctx, nir_instr_as_intrinsic(instr));
3624 break;
3625 case nir_instr_type_tex:
3626 visit_tex(ctx, nir_instr_as_tex(instr));
3627 break;
3628 case nir_instr_type_phi:
3629 visit_phi(ctx, nir_instr_as_phi(instr));
3630 break;
3631 case nir_instr_type_ssa_undef:
3632 visit_ssa_undef(ctx, nir_instr_as_ssa_undef(instr));
3633 break;
3634 case nir_instr_type_jump:
3635 visit_jump(ctx, nir_instr_as_jump(instr));
3636 break;
3637 default:
3638 fprintf(stderr, "Unknown NIR instr type: ");
3639 nir_print_instr(instr, stderr);
3640 fprintf(stderr, "\n");
3641 abort();
3642 }
3643 }
3644
3645 _mesa_hash_table_insert(ctx->defs, block, llvm_block);
3646 }
3647
3648 static void visit_if(struct nir_to_llvm_context *ctx, nir_if *if_stmt)
3649 {
3650 LLVMValueRef value = get_src(ctx, if_stmt->condition);
3651
3652 LLVMBasicBlockRef merge_block =
3653 LLVMAppendBasicBlockInContext(ctx->context, ctx->main_function, "");
3654 LLVMBasicBlockRef if_block =
3655 LLVMAppendBasicBlockInContext(ctx->context, ctx->main_function, "");
3656 LLVMBasicBlockRef else_block = merge_block;
3657 if (!exec_list_is_empty(&if_stmt->else_list))
3658 else_block = LLVMAppendBasicBlockInContext(
3659 ctx->context, ctx->main_function, "");
3660
3661 LLVMValueRef cond = LLVMBuildICmp(ctx->builder, LLVMIntNE, value,
3662 LLVMConstInt(ctx->i32, 0, false), "");
3663 LLVMBuildCondBr(ctx->builder, cond, if_block, else_block);
3664
3665 LLVMPositionBuilderAtEnd(ctx->builder, if_block);
3666 visit_cf_list(ctx, &if_stmt->then_list);
3667 if (LLVMGetInsertBlock(ctx->builder))
3668 LLVMBuildBr(ctx->builder, merge_block);
3669
3670 if (!exec_list_is_empty(&if_stmt->else_list)) {
3671 LLVMPositionBuilderAtEnd(ctx->builder, else_block);
3672 visit_cf_list(ctx, &if_stmt->else_list);
3673 if (LLVMGetInsertBlock(ctx->builder))
3674 LLVMBuildBr(ctx->builder, merge_block);
3675 }
3676
3677 LLVMPositionBuilderAtEnd(ctx->builder, merge_block);
3678 }
3679
3680 static void visit_loop(struct nir_to_llvm_context *ctx, nir_loop *loop)
3681 {
3682 LLVMBasicBlockRef continue_parent = ctx->continue_block;
3683 LLVMBasicBlockRef break_parent = ctx->break_block;
3684
3685 ctx->continue_block =
3686 LLVMAppendBasicBlockInContext(ctx->context, ctx->main_function, "");
3687 ctx->break_block =
3688 LLVMAppendBasicBlockInContext(ctx->context, ctx->main_function, "");
3689
3690 LLVMBuildBr(ctx->builder, ctx->continue_block);
3691 LLVMPositionBuilderAtEnd(ctx->builder, ctx->continue_block);
3692 visit_cf_list(ctx, &loop->body);
3693
3694 if (LLVMGetInsertBlock(ctx->builder))
3695 LLVMBuildBr(ctx->builder, ctx->continue_block);
3696 LLVMPositionBuilderAtEnd(ctx->builder, ctx->break_block);
3697
3698 ctx->continue_block = continue_parent;
3699 ctx->break_block = break_parent;
3700 }
3701
3702 static void visit_cf_list(struct nir_to_llvm_context *ctx,
3703 struct exec_list *list)
3704 {
3705 foreach_list_typed(nir_cf_node, node, node, list)
3706 {
3707 switch (node->type) {
3708 case nir_cf_node_block:
3709 visit_block(ctx, nir_cf_node_as_block(node));
3710 break;
3711
3712 case nir_cf_node_if:
3713 visit_if(ctx, nir_cf_node_as_if(node));
3714 break;
3715
3716 case nir_cf_node_loop:
3717 visit_loop(ctx, nir_cf_node_as_loop(node));
3718 break;
3719
3720 default:
3721 assert(0);
3722 }
3723 }
3724 }
3725
3726 static void
3727 handle_vs_input_decl(struct nir_to_llvm_context *ctx,
3728 struct nir_variable *variable)
3729 {
3730 LLVMValueRef t_list_ptr = ctx->vertex_buffers;
3731 LLVMValueRef t_offset;
3732 LLVMValueRef t_list;
3733 LLVMValueRef args[3];
3734 LLVMValueRef input;
3735 LLVMValueRef buffer_index;
3736 int index = variable->data.location - VERT_ATTRIB_GENERIC0;
3737 int idx = variable->data.location;
3738 unsigned attrib_count = glsl_count_attribute_slots(variable->type, true);
3739
3740 variable->data.driver_location = idx * 4;
3741
3742 if (ctx->options->key.vs.instance_rate_inputs & (1u << index)) {
3743 buffer_index = LLVMBuildAdd(ctx->builder, ctx->instance_id,
3744 ctx->start_instance, "");
3745 ctx->shader_info->vs.vgpr_comp_cnt = MAX2(3,
3746 ctx->shader_info->vs.vgpr_comp_cnt);
3747 } else
3748 buffer_index = LLVMBuildAdd(ctx->builder, ctx->vertex_id,
3749 ctx->base_vertex, "");
3750
3751 for (unsigned i = 0; i < attrib_count; ++i, ++idx) {
3752 t_offset = LLVMConstInt(ctx->i32, index + i, false);
3753
3754 t_list = build_indexed_load_const(ctx, t_list_ptr, t_offset);
3755 args[0] = t_list;
3756 args[1] = LLVMConstInt(ctx->i32, 0, false);
3757 args[2] = buffer_index;
3758 input = ac_emit_llvm_intrinsic(&ctx->ac,
3759 "llvm.SI.vs.load.input", ctx->v4f32, args, 3,
3760 AC_FUNC_ATTR_READNONE | AC_FUNC_ATTR_NOUNWIND);
3761
3762 for (unsigned chan = 0; chan < 4; chan++) {
3763 LLVMValueRef llvm_chan = LLVMConstInt(ctx->i32, chan, false);
3764 ctx->inputs[radeon_llvm_reg_index_soa(idx, chan)] =
3765 to_integer(ctx, LLVMBuildExtractElement(ctx->builder,
3766 input, llvm_chan, ""));
3767 }
3768 }
3769 }
3770
3771
3772 static void interp_fs_input(struct nir_to_llvm_context *ctx,
3773 unsigned attr,
3774 LLVMValueRef interp_param,
3775 LLVMValueRef prim_mask,
3776 LLVMValueRef result[4])
3777 {
3778 const char *intr_name;
3779 LLVMValueRef attr_number;
3780 unsigned chan;
3781
3782 attr_number = LLVMConstInt(ctx->i32, attr, false);
3783
3784 /* fs.constant returns the param from the middle vertex, so it's not
3785 * really useful for flat shading. It's meant to be used for custom
3786 * interpolation (but the intrinsic can't fetch from the other two
3787 * vertices).
3788 *
3789 * Luckily, it doesn't matter, because we rely on the FLAT_SHADE state
3790 * to do the right thing. The only reason we use fs.constant is that
3791 * fs.interp cannot be used on integers, because they can be equal
3792 * to NaN.
3793 */
3794 intr_name = interp_param ? "llvm.SI.fs.interp" : "llvm.SI.fs.constant";
3795
3796 for (chan = 0; chan < 4; chan++) {
3797 LLVMValueRef args[4];
3798 LLVMValueRef llvm_chan = LLVMConstInt(ctx->i32, chan, false);
3799
3800 args[0] = llvm_chan;
3801 args[1] = attr_number;
3802 args[2] = prim_mask;
3803 args[3] = interp_param;
3804 result[chan] = ac_emit_llvm_intrinsic(&ctx->ac, intr_name,
3805 ctx->f32, args, args[3] ? 4 : 3,
3806 AC_FUNC_ATTR_READNONE | AC_FUNC_ATTR_NOUNWIND);
3807 }
3808 }
3809
3810 static void
3811 handle_fs_input_decl(struct nir_to_llvm_context *ctx,
3812 struct nir_variable *variable)
3813 {
3814 int idx = variable->data.location;
3815 unsigned attrib_count = glsl_count_attribute_slots(variable->type, false);
3816 LLVMValueRef interp;
3817
3818 variable->data.driver_location = idx * 4;
3819 ctx->input_mask |= ((1ull << attrib_count) - 1) << variable->data.location;
3820
3821 if (glsl_get_base_type(glsl_without_array(variable->type)) == GLSL_TYPE_FLOAT) {
3822 unsigned interp_type;
3823 if (variable->data.sample) {
3824 interp_type = INTERP_SAMPLE;
3825 ctx->shader_info->fs.force_persample = true;
3826 } else if (variable->data.centroid)
3827 interp_type = INTERP_CENTROID;
3828 else
3829 interp_type = INTERP_CENTER;
3830
3831 interp = lookup_interp_param(ctx, variable->data.interpolation, interp_type);
3832 } else
3833 interp = NULL;
3834
3835 for (unsigned i = 0; i < attrib_count; ++i)
3836 ctx->inputs[radeon_llvm_reg_index_soa(idx + i, 0)] = interp;
3837
3838 }
3839
3840 static void
3841 handle_shader_input_decl(struct nir_to_llvm_context *ctx,
3842 struct nir_variable *variable)
3843 {
3844 switch (ctx->stage) {
3845 case MESA_SHADER_VERTEX:
3846 handle_vs_input_decl(ctx, variable);
3847 break;
3848 case MESA_SHADER_FRAGMENT:
3849 handle_fs_input_decl(ctx, variable);
3850 break;
3851 default:
3852 break;
3853 }
3854
3855 }
3856
3857 static void
3858 handle_fs_inputs_pre(struct nir_to_llvm_context *ctx,
3859 struct nir_shader *nir)
3860 {
3861 unsigned index = 0;
3862 for (unsigned i = 0; i < RADEON_LLVM_MAX_INPUTS; ++i) {
3863 LLVMValueRef interp_param;
3864 LLVMValueRef *inputs = ctx->inputs +radeon_llvm_reg_index_soa(i, 0);
3865
3866 if (!(ctx->input_mask & (1ull << i)))
3867 continue;
3868
3869 if (i >= VARYING_SLOT_VAR0 || i == VARYING_SLOT_PNTC) {
3870 interp_param = *inputs;
3871 interp_fs_input(ctx, index, interp_param, ctx->prim_mask,
3872 inputs);
3873
3874 if (!interp_param)
3875 ctx->shader_info->fs.flat_shaded_mask |= 1u << index;
3876 ++index;
3877 } else if (i == VARYING_SLOT_POS) {
3878 for(int i = 0; i < 3; ++i)
3879 inputs[i] = ctx->frag_pos[i];
3880
3881 inputs[3] = emit_fdiv(ctx, ctx->f32one, ctx->frag_pos[3]);
3882 }
3883 }
3884 ctx->shader_info->fs.num_interp = index;
3885 if (ctx->input_mask & (1 << VARYING_SLOT_PNTC))
3886 ctx->shader_info->fs.has_pcoord = true;
3887 ctx->shader_info->fs.input_mask = ctx->input_mask >> VARYING_SLOT_VAR0;
3888 }
3889
3890 static LLVMValueRef
3891 ac_build_alloca(struct nir_to_llvm_context *ctx,
3892 LLVMTypeRef type,
3893 const char *name)
3894 {
3895 LLVMBuilderRef builder = ctx->builder;
3896 LLVMBasicBlockRef current_block = LLVMGetInsertBlock(builder);
3897 LLVMValueRef function = LLVMGetBasicBlockParent(current_block);
3898 LLVMBasicBlockRef first_block = LLVMGetEntryBasicBlock(function);
3899 LLVMValueRef first_instr = LLVMGetFirstInstruction(first_block);
3900 LLVMBuilderRef first_builder = LLVMCreateBuilderInContext(ctx->context);
3901 LLVMValueRef res;
3902
3903 if (first_instr) {
3904 LLVMPositionBuilderBefore(first_builder, first_instr);
3905 } else {
3906 LLVMPositionBuilderAtEnd(first_builder, first_block);
3907 }
3908
3909 res = LLVMBuildAlloca(first_builder, type, name);
3910 LLVMBuildStore(builder, LLVMConstNull(type), res);
3911
3912 LLVMDisposeBuilder(first_builder);
3913
3914 return res;
3915 }
3916
3917 static LLVMValueRef si_build_alloca_undef(struct nir_to_llvm_context *ctx,
3918 LLVMTypeRef type,
3919 const char *name)
3920 {
3921 LLVMValueRef ptr = ac_build_alloca(ctx, type, name);
3922 LLVMBuildStore(ctx->builder, LLVMGetUndef(type), ptr);
3923 return ptr;
3924 }
3925
3926 static void
3927 handle_shader_output_decl(struct nir_to_llvm_context *ctx,
3928 struct nir_variable *variable)
3929 {
3930 int idx = variable->data.location + variable->data.index;
3931 unsigned attrib_count = glsl_count_attribute_slots(variable->type, false);
3932
3933 variable->data.driver_location = idx * 4;
3934
3935 if (ctx->stage == MESA_SHADER_VERTEX) {
3936
3937 if (idx == VARYING_SLOT_CLIP_DIST0 ||
3938 idx == VARYING_SLOT_CULL_DIST0) {
3939 int length = glsl_get_length(variable->type);
3940 if (idx == VARYING_SLOT_CLIP_DIST0) {
3941 ctx->shader_info->vs.clip_dist_mask = (1 << length) - 1;
3942 ctx->num_clips = length;
3943 } else if (idx == VARYING_SLOT_CULL_DIST0) {
3944 ctx->shader_info->vs.cull_dist_mask = (1 << length) - 1;
3945 ctx->num_culls = length;
3946 }
3947 if (length > 4)
3948 attrib_count = 2;
3949 else
3950 attrib_count = 1;
3951 }
3952 }
3953
3954 for (unsigned i = 0; i < attrib_count; ++i) {
3955 for (unsigned chan = 0; chan < 4; chan++) {
3956 ctx->outputs[radeon_llvm_reg_index_soa(idx + i, chan)] =
3957 si_build_alloca_undef(ctx, ctx->f32, "");
3958 }
3959 }
3960 ctx->output_mask |= ((1ull << attrib_count) - 1) << idx;
3961 }
3962
3963 static void
3964 setup_locals(struct nir_to_llvm_context *ctx,
3965 struct nir_function *func)
3966 {
3967 int i, j;
3968 ctx->num_locals = 0;
3969 nir_foreach_variable(variable, &func->impl->locals) {
3970 unsigned attrib_count = glsl_count_attribute_slots(variable->type, false);
3971 variable->data.driver_location = ctx->num_locals * 4;
3972 ctx->num_locals += attrib_count;
3973 }
3974 ctx->locals = malloc(4 * ctx->num_locals * sizeof(LLVMValueRef));
3975 if (!ctx->locals)
3976 return;
3977
3978 for (i = 0; i < ctx->num_locals; i++) {
3979 for (j = 0; j < 4; j++) {
3980 ctx->locals[i * 4 + j] =
3981 si_build_alloca_undef(ctx, ctx->f32, "temp");
3982 }
3983 }
3984 }
3985
3986 static LLVMValueRef
3987 emit_float_saturate(struct nir_to_llvm_context *ctx, LLVMValueRef v, float lo, float hi)
3988 {
3989 v = to_float(ctx, v);
3990 v = emit_intrin_2f_param(ctx, "llvm.maxnum.f32", v, LLVMConstReal(ctx->f32, lo));
3991 return emit_intrin_2f_param(ctx, "llvm.minnum.f32", v, LLVMConstReal(ctx->f32, hi));
3992 }
3993
3994
3995 static LLVMValueRef emit_pack_int16(struct nir_to_llvm_context *ctx,
3996 LLVMValueRef src0, LLVMValueRef src1)
3997 {
3998 LLVMValueRef const16 = LLVMConstInt(ctx->i32, 16, false);
3999 LLVMValueRef comp[2];
4000
4001 comp[0] = LLVMBuildAnd(ctx->builder, src0, LLVMConstInt(ctx-> i32, 65535, 0), "");
4002 comp[1] = LLVMBuildAnd(ctx->builder, src1, LLVMConstInt(ctx-> i32, 65535, 0), "");
4003 comp[1] = LLVMBuildShl(ctx->builder, comp[1], const16, "");
4004 return LLVMBuildOr(ctx->builder, comp[0], comp[1], "");
4005 }
4006
4007 /* Initialize arguments for the shader export intrinsic */
4008 static void
4009 si_llvm_init_export_args(struct nir_to_llvm_context *ctx,
4010 LLVMValueRef *values,
4011 unsigned target,
4012 LLVMValueRef *args)
4013 {
4014 /* Default is 0xf. Adjusted below depending on the format. */
4015 args[0] = LLVMConstInt(ctx->i32, target != V_008DFC_SQ_EXP_NULL ? 0xf : 0, false);
4016 /* Specify whether the EXEC mask represents the valid mask */
4017 args[1] = LLVMConstInt(ctx->i32, 0, false);
4018
4019 /* Specify whether this is the last export */
4020 args[2] = LLVMConstInt(ctx->i32, 0, false);
4021 /* Specify the target we are exporting */
4022 args[3] = LLVMConstInt(ctx->i32, target, false);
4023
4024 args[4] = LLVMConstInt(ctx->i32, 0, false); /* COMPR flag */
4025 args[5] = LLVMGetUndef(ctx->f32);
4026 args[6] = LLVMGetUndef(ctx->f32);
4027 args[7] = LLVMGetUndef(ctx->f32);
4028 args[8] = LLVMGetUndef(ctx->f32);
4029
4030 if (!values)
4031 return;
4032
4033 if (ctx->stage == MESA_SHADER_FRAGMENT && target >= V_008DFC_SQ_EXP_MRT) {
4034 LLVMValueRef val[4];
4035 unsigned index = target - V_008DFC_SQ_EXP_MRT;
4036 unsigned col_format = (ctx->options->key.fs.col_format >> (4 * index)) & 0xf;
4037 bool is_int8 = (ctx->options->key.fs.is_int8 >> index) & 1;
4038
4039 switch(col_format) {
4040 case V_028714_SPI_SHADER_ZERO:
4041 args[0] = LLVMConstInt(ctx->i32, 0x0, 0);
4042 args[3] = LLVMConstInt(ctx->i32, V_008DFC_SQ_EXP_NULL, 0);
4043 break;
4044
4045 case V_028714_SPI_SHADER_32_R:
4046 args[0] = LLVMConstInt(ctx->i32, 0x1, 0);
4047 args[5] = values[0];
4048 break;
4049
4050 case V_028714_SPI_SHADER_32_GR:
4051 args[0] = LLVMConstInt(ctx->i32, 0x3, 0);
4052 args[5] = values[0];
4053 args[6] = values[1];
4054 break;
4055
4056 case V_028714_SPI_SHADER_32_AR:
4057 args[0] = LLVMConstInt(ctx->i32, 0x9, 0);
4058 args[5] = values[0];
4059 args[8] = values[3];
4060 break;
4061
4062 case V_028714_SPI_SHADER_FP16_ABGR:
4063 args[4] = ctx->i32one;
4064
4065 for (unsigned chan = 0; chan < 2; chan++) {
4066 LLVMValueRef pack_args[2] = {
4067 values[2 * chan],
4068 values[2 * chan + 1]
4069 };
4070 LLVMValueRef packed;
4071
4072 packed = ac_emit_llvm_intrinsic(&ctx->ac, "llvm.SI.packf16",
4073 ctx->i32, pack_args, 2,
4074 AC_FUNC_ATTR_READNONE);
4075 args[chan + 5] = packed;
4076 }
4077 break;
4078
4079 case V_028714_SPI_SHADER_UNORM16_ABGR:
4080 for (unsigned chan = 0; chan < 4; chan++) {
4081 val[chan] = emit_float_saturate(ctx, values[chan], 0, 1);
4082 val[chan] = LLVMBuildFMul(ctx->builder, val[chan],
4083 LLVMConstReal(ctx->f32, 65535), "");
4084 val[chan] = LLVMBuildFAdd(ctx->builder, val[chan],
4085 LLVMConstReal(ctx->f32, 0.5), "");
4086 val[chan] = LLVMBuildFPToUI(ctx->builder, val[chan],
4087 ctx->i32, "");
4088 }
4089
4090 args[4] = ctx->i32one;
4091 args[5] = emit_pack_int16(ctx, val[0], val[1]);
4092 args[6] = emit_pack_int16(ctx, val[2], val[3]);
4093 break;
4094
4095 case V_028714_SPI_SHADER_SNORM16_ABGR:
4096 for (unsigned chan = 0; chan < 4; chan++) {
4097 val[chan] = emit_float_saturate(ctx, values[chan], -1, 1);
4098 val[chan] = LLVMBuildFMul(ctx->builder, val[chan],
4099 LLVMConstReal(ctx->f32, 32767), "");
4100
4101 /* If positive, add 0.5, else add -0.5. */
4102 val[chan] = LLVMBuildFAdd(ctx->builder, val[chan],
4103 LLVMBuildSelect(ctx->builder,
4104 LLVMBuildFCmp(ctx->builder, LLVMRealOGE,
4105 val[chan], ctx->f32zero, ""),
4106 LLVMConstReal(ctx->f32, 0.5),
4107 LLVMConstReal(ctx->f32, -0.5), ""), "");
4108 val[chan] = LLVMBuildFPToSI(ctx->builder, val[chan], ctx->i32, "");
4109 }
4110
4111 args[4] = ctx->i32one;
4112 args[5] = emit_pack_int16(ctx, val[0], val[1]);
4113 args[6] = emit_pack_int16(ctx, val[2], val[3]);
4114 break;
4115
4116 case V_028714_SPI_SHADER_UINT16_ABGR: {
4117 LLVMValueRef max = LLVMConstInt(ctx->i32, is_int8 ? 255 : 65535, 0);
4118
4119 for (unsigned chan = 0; chan < 4; chan++) {
4120 val[chan] = to_integer(ctx, values[chan]);
4121 val[chan] = emit_minmax_int(ctx, LLVMIntULT, val[chan], max);
4122 }
4123
4124 args[4] = ctx->i32one;
4125 args[5] = emit_pack_int16(ctx, val[0], val[1]);
4126 args[6] = emit_pack_int16(ctx, val[2], val[3]);
4127 break;
4128 }
4129
4130 case V_028714_SPI_SHADER_SINT16_ABGR: {
4131 LLVMValueRef max = LLVMConstInt(ctx->i32, is_int8 ? 127 : 32767, 0);
4132 LLVMValueRef min = LLVMConstInt(ctx->i32, is_int8 ? -128 : -32768, 0);
4133
4134 /* Clamp. */
4135 for (unsigned chan = 0; chan < 4; chan++) {
4136 val[chan] = to_integer(ctx, values[chan]);
4137 val[chan] = emit_minmax_int(ctx, LLVMIntSLT, val[chan], max);
4138 val[chan] = emit_minmax_int(ctx, LLVMIntSGT, val[chan], min);
4139 }
4140
4141 args[4] = ctx->i32one;
4142 args[5] = emit_pack_int16(ctx, val[0], val[1]);
4143 args[6] = emit_pack_int16(ctx, val[2], val[3]);
4144 break;
4145 }
4146
4147 default:
4148 case V_028714_SPI_SHADER_32_ABGR:
4149 memcpy(&args[5], values, sizeof(values[0]) * 4);
4150 break;
4151 }
4152 } else
4153 memcpy(&args[5], values, sizeof(values[0]) * 4);
4154
4155 for (unsigned i = 5; i < 9; ++i)
4156 args[i] = to_float(ctx, args[i]);
4157 }
4158
4159 static void
4160 handle_vs_outputs_post(struct nir_to_llvm_context *ctx)
4161 {
4162 uint32_t param_count = 0;
4163 unsigned target;
4164 unsigned pos_idx, num_pos_exports = 0;
4165 LLVMValueRef args[9];
4166 LLVMValueRef pos_args[4][9] = { { 0 } };
4167 LLVMValueRef psize_value = 0;
4168 int i;
4169 const uint64_t clip_mask = ctx->output_mask & ((1ull << VARYING_SLOT_CLIP_DIST0) |
4170 (1ull << VARYING_SLOT_CLIP_DIST1) |
4171 (1ull << VARYING_SLOT_CULL_DIST0) |
4172 (1ull << VARYING_SLOT_CULL_DIST1));
4173
4174 if (clip_mask) {
4175 LLVMValueRef slots[8];
4176 unsigned j;
4177
4178 if (ctx->shader_info->vs.cull_dist_mask)
4179 ctx->shader_info->vs.cull_dist_mask <<= ctx->num_clips;
4180
4181 i = VARYING_SLOT_CLIP_DIST0;
4182 for (j = 0; j < ctx->num_clips; j++)
4183 slots[j] = to_float(ctx, LLVMBuildLoad(ctx->builder,
4184 ctx->outputs[radeon_llvm_reg_index_soa(i, j)], ""));
4185 i = VARYING_SLOT_CULL_DIST0;
4186 for (j = 0; j < ctx->num_culls; j++)
4187 slots[ctx->num_clips + j] = to_float(ctx, LLVMBuildLoad(ctx->builder,
4188 ctx->outputs[radeon_llvm_reg_index_soa(i, j)], ""));
4189
4190 for (i = ctx->num_clips + ctx->num_culls; i < 8; i++)
4191 slots[i] = LLVMGetUndef(ctx->f32);
4192
4193 if (ctx->num_clips + ctx->num_culls > 4) {
4194 target = V_008DFC_SQ_EXP_POS + 3;
4195 si_llvm_init_export_args(ctx, &slots[4], target, args);
4196 memcpy(pos_args[target - V_008DFC_SQ_EXP_POS],
4197 args, sizeof(args));
4198 }
4199
4200 target = V_008DFC_SQ_EXP_POS + 2;
4201 si_llvm_init_export_args(ctx, &slots[0], target, args);
4202 memcpy(pos_args[target - V_008DFC_SQ_EXP_POS],
4203 args, sizeof(args));
4204
4205 }
4206
4207 for (unsigned i = 0; i < RADEON_LLVM_MAX_OUTPUTS; ++i) {
4208 LLVMValueRef values[4];
4209 if (!(ctx->output_mask & (1ull << i)))
4210 continue;
4211
4212 for (unsigned j = 0; j < 4; j++)
4213 values[j] = to_float(ctx, LLVMBuildLoad(ctx->builder,
4214 ctx->outputs[radeon_llvm_reg_index_soa(i, j)], ""));
4215
4216 if (i == VARYING_SLOT_POS) {
4217 target = V_008DFC_SQ_EXP_POS;
4218 } else if (i == VARYING_SLOT_CLIP_DIST0 ||
4219 i == VARYING_SLOT_CLIP_DIST1 ||
4220 i == VARYING_SLOT_CULL_DIST0 ||
4221 i == VARYING_SLOT_CULL_DIST1) {
4222 continue;
4223 } else if (i == VARYING_SLOT_PSIZ) {
4224 ctx->shader_info->vs.writes_pointsize = true;
4225 psize_value = values[0];
4226 continue;
4227 } else if (i >= VARYING_SLOT_VAR0) {
4228 ctx->shader_info->vs.export_mask |= 1u << (i - VARYING_SLOT_VAR0);
4229 target = V_008DFC_SQ_EXP_PARAM + param_count;
4230 param_count++;
4231 }
4232
4233 si_llvm_init_export_args(ctx, values, target, args);
4234
4235 if (target >= V_008DFC_SQ_EXP_POS &&
4236 target <= (V_008DFC_SQ_EXP_POS + 3)) {
4237 memcpy(pos_args[target - V_008DFC_SQ_EXP_POS],
4238 args, sizeof(args));
4239 } else {
4240 ac_emit_llvm_intrinsic(&ctx->ac,
4241 "llvm.SI.export",
4242 LLVMVoidTypeInContext(ctx->context),
4243 args, 9, 0);
4244 }
4245 }
4246
4247 /* We need to add the position output manually if it's missing. */
4248 if (!pos_args[0][0]) {
4249 pos_args[0][0] = LLVMConstInt(ctx->i32, 0xf, false);
4250 pos_args[0][1] = ctx->i32zero; /* EXEC mask */
4251 pos_args[0][2] = ctx->i32zero; /* last export? */
4252 pos_args[0][3] = LLVMConstInt(ctx->i32, V_008DFC_SQ_EXP_POS, false);
4253 pos_args[0][4] = ctx->i32zero; /* COMPR flag */
4254 pos_args[0][5] = ctx->f32zero; /* X */
4255 pos_args[0][6] = ctx->f32zero; /* Y */
4256 pos_args[0][7] = ctx->f32zero; /* Z */
4257 pos_args[0][8] = ctx->f32one; /* W */
4258 }
4259
4260 if (ctx->shader_info->vs.writes_pointsize == true) {
4261 pos_args[1][0] = LLVMConstInt(ctx->i32, (ctx->shader_info->vs.writes_pointsize == true), false); /* writemask */
4262 pos_args[1][1] = ctx->i32zero; /* EXEC mask */
4263 pos_args[1][2] = ctx->i32zero; /* last export? */
4264 pos_args[1][3] = LLVMConstInt(ctx->i32, V_008DFC_SQ_EXP_POS + 1, false);
4265 pos_args[1][4] = ctx->i32zero; /* COMPR flag */
4266 pos_args[1][5] = ctx->f32zero; /* X */
4267 pos_args[1][6] = ctx->f32zero; /* Y */
4268 pos_args[1][7] = ctx->f32zero; /* Z */
4269 pos_args[1][8] = ctx->f32zero; /* W */
4270
4271 if (ctx->shader_info->vs.writes_pointsize == true)
4272 pos_args[1][5] = psize_value;
4273 }
4274 for (i = 0; i < 4; i++) {
4275 if (pos_args[i][0])
4276 num_pos_exports++;
4277 }
4278
4279 pos_idx = 0;
4280 for (i = 0; i < 4; i++) {
4281 if (!pos_args[i][0])
4282 continue;
4283
4284 /* Specify the target we are exporting */
4285 pos_args[i][3] = LLVMConstInt(ctx->i32, V_008DFC_SQ_EXP_POS + pos_idx++, false);
4286 if (pos_idx == num_pos_exports)
4287 pos_args[i][2] = ctx->i32one;
4288 ac_emit_llvm_intrinsic(&ctx->ac,
4289 "llvm.SI.export",
4290 LLVMVoidTypeInContext(ctx->context),
4291 pos_args[i], 9, 0);
4292 }
4293
4294 ctx->shader_info->vs.pos_exports = num_pos_exports;
4295 ctx->shader_info->vs.param_exports = param_count;
4296 }
4297
4298 static void
4299 si_export_mrt_color(struct nir_to_llvm_context *ctx,
4300 LLVMValueRef *color, unsigned param, bool is_last)
4301 {
4302 LLVMValueRef args[9];
4303 /* Export */
4304 si_llvm_init_export_args(ctx, color, param,
4305 args);
4306
4307 if (is_last) {
4308 args[1] = ctx->i32one; /* whether the EXEC mask is valid */
4309 args[2] = ctx->i32one; /* DONE bit */
4310 } else if (args[0] == ctx->i32zero)
4311 return; /* unnecessary NULL export */
4312
4313 ac_emit_llvm_intrinsic(&ctx->ac, "llvm.SI.export",
4314 ctx->voidt, args, 9, 0);
4315 }
4316
4317 static void
4318 si_export_mrt_z(struct nir_to_llvm_context *ctx,
4319 LLVMValueRef depth, LLVMValueRef stencil,
4320 LLVMValueRef samplemask)
4321 {
4322 LLVMValueRef args[9];
4323 unsigned mask = 0;
4324 args[1] = ctx->i32one; /* whether the EXEC mask is valid */
4325 args[2] = ctx->i32one; /* DONE bit */
4326 /* Specify the target we are exporting */
4327 args[3] = LLVMConstInt(ctx->i32, V_008DFC_SQ_EXP_MRTZ, false);
4328
4329 args[4] = ctx->i32zero; /* COMP flag */
4330 args[5] = LLVMGetUndef(ctx->f32); /* R, depth */
4331 args[6] = LLVMGetUndef(ctx->f32); /* G, stencil test val[0:7], stencil op val[8:15] */
4332 args[7] = LLVMGetUndef(ctx->f32); /* B, sample mask */
4333 args[8] = LLVMGetUndef(ctx->f32); /* A, alpha to mask */
4334
4335 if (depth) {
4336 args[5] = depth;
4337 mask |= 0x1;
4338 }
4339
4340 if (stencil) {
4341 args[6] = stencil;
4342 mask |= 0x2;
4343 }
4344
4345 if (samplemask) {
4346 args[7] = samplemask;
4347 mask |= 0x04;
4348 }
4349
4350 /* SI (except OLAND) has a bug that it only looks
4351 * at the X writemask component. */
4352 if (ctx->options->chip_class == SI &&
4353 ctx->options->family != CHIP_OLAND)
4354 mask |= 0x01;
4355
4356 args[0] = LLVMConstInt(ctx->i32, mask, false);
4357 ac_emit_llvm_intrinsic(&ctx->ac, "llvm.SI.export",
4358 ctx->voidt, args, 9, 0);
4359 }
4360
4361 static void
4362 handle_fs_outputs_post(struct nir_to_llvm_context *ctx)
4363 {
4364 unsigned index = 0;
4365 LLVMValueRef depth = NULL, stencil = NULL, samplemask = NULL;
4366
4367 for (unsigned i = 0; i < RADEON_LLVM_MAX_OUTPUTS; ++i) {
4368 LLVMValueRef values[4];
4369
4370 if (!(ctx->output_mask & (1ull << i)))
4371 continue;
4372
4373 if (i == FRAG_RESULT_DEPTH) {
4374 ctx->shader_info->fs.writes_z = true;
4375 depth = to_float(ctx, LLVMBuildLoad(ctx->builder,
4376 ctx->outputs[radeon_llvm_reg_index_soa(i, 0)], ""));
4377 } else if (i == FRAG_RESULT_STENCIL) {
4378 ctx->shader_info->fs.writes_stencil = true;
4379 stencil = to_float(ctx, LLVMBuildLoad(ctx->builder,
4380 ctx->outputs[radeon_llvm_reg_index_soa(i, 0)], ""));
4381 } else {
4382 bool last = false;
4383 for (unsigned j = 0; j < 4; j++)
4384 values[j] = to_float(ctx, LLVMBuildLoad(ctx->builder,
4385 ctx->outputs[radeon_llvm_reg_index_soa(i, j)], ""));
4386
4387 if (!ctx->shader_info->fs.writes_z && !ctx->shader_info->fs.writes_stencil)
4388 last = ctx->output_mask <= ((1ull << (i + 1)) - 1);
4389
4390 si_export_mrt_color(ctx, values, V_008DFC_SQ_EXP_MRT + index, last);
4391 index++;
4392 }
4393 }
4394
4395 if (depth || stencil)
4396 si_export_mrt_z(ctx, depth, stencil, samplemask);
4397 else if (!index)
4398 si_export_mrt_color(ctx, NULL, V_008DFC_SQ_EXP_NULL, true);
4399
4400 ctx->shader_info->fs.output_mask = index ? ((1ull << index) - 1) : 0;
4401 }
4402
4403 static void
4404 handle_shader_outputs_post(struct nir_to_llvm_context *ctx)
4405 {
4406 switch (ctx->stage) {
4407 case MESA_SHADER_VERTEX:
4408 handle_vs_outputs_post(ctx);
4409 break;
4410 case MESA_SHADER_FRAGMENT:
4411 handle_fs_outputs_post(ctx);
4412 break;
4413 default:
4414 break;
4415 }
4416 }
4417
4418 static void
4419 handle_shared_compute_var(struct nir_to_llvm_context *ctx,
4420 struct nir_variable *variable, uint32_t *offset, int idx)
4421 {
4422 unsigned size = glsl_count_attribute_slots(variable->type, false);
4423 variable->data.driver_location = *offset;
4424 *offset += size;
4425 }
4426
4427 static void ac_llvm_finalize_module(struct nir_to_llvm_context * ctx)
4428 {
4429 LLVMPassManagerRef passmgr;
4430 /* Create the pass manager */
4431 passmgr = LLVMCreateFunctionPassManagerForModule(
4432 ctx->module);
4433
4434 /* This pass should eliminate all the load and store instructions */
4435 LLVMAddPromoteMemoryToRegisterPass(passmgr);
4436
4437 /* Add some optimization passes */
4438 LLVMAddScalarReplAggregatesPass(passmgr);
4439 LLVMAddLICMPass(passmgr);
4440 LLVMAddAggressiveDCEPass(passmgr);
4441 LLVMAddCFGSimplificationPass(passmgr);
4442 LLVMAddInstructionCombiningPass(passmgr);
4443
4444 /* Run the pass */
4445 LLVMInitializeFunctionPassManager(passmgr);
4446 LLVMRunFunctionPassManager(passmgr, ctx->main_function);
4447 LLVMFinalizeFunctionPassManager(passmgr);
4448
4449 LLVMDisposeBuilder(ctx->builder);
4450 LLVMDisposePassManager(passmgr);
4451 }
4452
4453 static
4454 LLVMModuleRef ac_translate_nir_to_llvm(LLVMTargetMachineRef tm,
4455 struct nir_shader *nir,
4456 struct ac_shader_variant_info *shader_info,
4457 const struct ac_nir_compiler_options *options)
4458 {
4459 struct nir_to_llvm_context ctx = {0};
4460 struct nir_function *func;
4461 unsigned i;
4462 ctx.options = options;
4463 ctx.shader_info = shader_info;
4464 ctx.context = LLVMContextCreate();
4465 ctx.module = LLVMModuleCreateWithNameInContext("shader", ctx.context);
4466
4467 ac_llvm_context_init(&ctx.ac, ctx.context);
4468 ctx.ac.module = ctx.module;
4469
4470 ctx.has_ds_bpermute = ctx.options->chip_class >= VI;
4471
4472 memset(shader_info, 0, sizeof(*shader_info));
4473
4474 LLVMSetTarget(ctx.module, "amdgcn--");
4475 setup_types(&ctx);
4476
4477 ctx.builder = LLVMCreateBuilderInContext(ctx.context);
4478 ctx.ac.builder = ctx.builder;
4479 ctx.stage = nir->stage;
4480
4481 for (i = 0; i < AC_UD_MAX_SETS; i++)
4482 shader_info->user_sgprs_locs.descriptor_sets[i].sgpr_idx = -1;
4483 for (i = 0; i < AC_UD_MAX_UD; i++)
4484 shader_info->user_sgprs_locs.shader_data[i].sgpr_idx = -1;
4485
4486 create_function(&ctx);
4487
4488 if (nir->stage == MESA_SHADER_COMPUTE) {
4489 int num_shared = 0;
4490 nir_foreach_variable(variable, &nir->shared)
4491 num_shared++;
4492 if (num_shared) {
4493 int idx = 0;
4494 uint32_t shared_size = 0;
4495 LLVMValueRef var;
4496 LLVMTypeRef i8p = LLVMPointerType(ctx.i8, LOCAL_ADDR_SPACE);
4497 nir_foreach_variable(variable, &nir->shared) {
4498 handle_shared_compute_var(&ctx, variable, &shared_size, idx);
4499 idx++;
4500 }
4501
4502 shared_size *= 4;
4503 var = LLVMAddGlobalInAddressSpace(ctx.module,
4504 LLVMArrayType(ctx.i8, shared_size),
4505 "compute_lds",
4506 LOCAL_ADDR_SPACE);
4507 LLVMSetAlignment(var, 4);
4508 ctx.shared_memory = LLVMBuildBitCast(ctx.builder, var, i8p, "");
4509 }
4510 }
4511
4512 nir_foreach_variable(variable, &nir->inputs)
4513 handle_shader_input_decl(&ctx, variable);
4514
4515 if (nir->stage == MESA_SHADER_FRAGMENT)
4516 handle_fs_inputs_pre(&ctx, nir);
4517
4518 nir_foreach_variable(variable, &nir->outputs)
4519 handle_shader_output_decl(&ctx, variable);
4520
4521 ctx.defs = _mesa_hash_table_create(NULL, _mesa_hash_pointer,
4522 _mesa_key_pointer_equal);
4523 ctx.phis = _mesa_hash_table_create(NULL, _mesa_hash_pointer,
4524 _mesa_key_pointer_equal);
4525
4526 func = (struct nir_function *)exec_list_get_head(&nir->functions);
4527
4528 setup_locals(&ctx, func);
4529
4530 visit_cf_list(&ctx, &func->impl->body);
4531 phi_post_pass(&ctx);
4532
4533 handle_shader_outputs_post(&ctx);
4534 LLVMBuildRetVoid(ctx.builder);
4535
4536 ac_llvm_finalize_module(&ctx);
4537 free(ctx.locals);
4538 ralloc_free(ctx.defs);
4539 ralloc_free(ctx.phis);
4540
4541 return ctx.module;
4542 }
4543
4544 static void ac_diagnostic_handler(LLVMDiagnosticInfoRef di, void *context)
4545 {
4546 unsigned *retval = (unsigned *)context;
4547 LLVMDiagnosticSeverity severity = LLVMGetDiagInfoSeverity(di);
4548 char *description = LLVMGetDiagInfoDescription(di);
4549
4550 if (severity == LLVMDSError) {
4551 *retval = 1;
4552 fprintf(stderr, "LLVM triggered Diagnostic Handler: %s\n",
4553 description);
4554 }
4555
4556 LLVMDisposeMessage(description);
4557 }
4558
4559 static unsigned ac_llvm_compile(LLVMModuleRef M,
4560 struct ac_shader_binary *binary,
4561 LLVMTargetMachineRef tm)
4562 {
4563 unsigned retval = 0;
4564 char *err;
4565 LLVMContextRef llvm_ctx;
4566 LLVMMemoryBufferRef out_buffer;
4567 unsigned buffer_size;
4568 const char *buffer_data;
4569 LLVMBool mem_err;
4570
4571 /* Setup Diagnostic Handler*/
4572 llvm_ctx = LLVMGetModuleContext(M);
4573
4574 LLVMContextSetDiagnosticHandler(llvm_ctx, ac_diagnostic_handler,
4575 &retval);
4576
4577 /* Compile IR*/
4578 mem_err = LLVMTargetMachineEmitToMemoryBuffer(tm, M, LLVMObjectFile,
4579 &err, &out_buffer);
4580
4581 /* Process Errors/Warnings */
4582 if (mem_err) {
4583 fprintf(stderr, "%s: %s", __FUNCTION__, err);
4584 free(err);
4585 retval = 1;
4586 goto out;
4587 }
4588
4589 /* Extract Shader Code*/
4590 buffer_size = LLVMGetBufferSize(out_buffer);
4591 buffer_data = LLVMGetBufferStart(out_buffer);
4592
4593 ac_elf_read(buffer_data, buffer_size, binary);
4594
4595 /* Clean up */
4596 LLVMDisposeMemoryBuffer(out_buffer);
4597
4598 out:
4599 return retval;
4600 }
4601
4602 void ac_compile_nir_shader(LLVMTargetMachineRef tm,
4603 struct ac_shader_binary *binary,
4604 struct ac_shader_config *config,
4605 struct ac_shader_variant_info *shader_info,
4606 struct nir_shader *nir,
4607 const struct ac_nir_compiler_options *options,
4608 bool dump_shader)
4609 {
4610
4611 LLVMModuleRef llvm_module = ac_translate_nir_to_llvm(tm, nir, shader_info,
4612 options);
4613 if (dump_shader)
4614 LLVMDumpModule(llvm_module);
4615
4616 memset(binary, 0, sizeof(*binary));
4617 int v = ac_llvm_compile(llvm_module, binary, tm);
4618 if (v) {
4619 fprintf(stderr, "compile failed\n");
4620 }
4621
4622 if (dump_shader)
4623 fprintf(stderr, "disasm:\n%s\n", binary->disasm_string);
4624
4625 ac_shader_binary_read_config(binary, config, 0);
4626
4627 LLVMContextRef ctx = LLVMGetModuleContext(llvm_module);
4628 LLVMDisposeModule(llvm_module);
4629 LLVMContextDispose(ctx);
4630
4631 if (nir->stage == MESA_SHADER_FRAGMENT) {
4632 shader_info->num_input_vgprs = 0;
4633 if (G_0286CC_PERSP_SAMPLE_ENA(config->spi_ps_input_addr))
4634 shader_info->num_input_vgprs += 2;
4635 if (G_0286CC_PERSP_CENTER_ENA(config->spi_ps_input_addr))
4636 shader_info->num_input_vgprs += 2;
4637 if (G_0286CC_PERSP_CENTROID_ENA(config->spi_ps_input_addr))
4638 shader_info->num_input_vgprs += 2;
4639 if (G_0286CC_PERSP_PULL_MODEL_ENA(config->spi_ps_input_addr))
4640 shader_info->num_input_vgprs += 3;
4641 if (G_0286CC_LINEAR_SAMPLE_ENA(config->spi_ps_input_addr))
4642 shader_info->num_input_vgprs += 2;
4643 if (G_0286CC_LINEAR_CENTER_ENA(config->spi_ps_input_addr))
4644 shader_info->num_input_vgprs += 2;
4645 if (G_0286CC_LINEAR_CENTROID_ENA(config->spi_ps_input_addr))
4646 shader_info->num_input_vgprs += 2;
4647 if (G_0286CC_LINE_STIPPLE_TEX_ENA(config->spi_ps_input_addr))
4648 shader_info->num_input_vgprs += 1;
4649 if (G_0286CC_POS_X_FLOAT_ENA(config->spi_ps_input_addr))
4650 shader_info->num_input_vgprs += 1;
4651 if (G_0286CC_POS_Y_FLOAT_ENA(config->spi_ps_input_addr))
4652 shader_info->num_input_vgprs += 1;
4653 if (G_0286CC_POS_Z_FLOAT_ENA(config->spi_ps_input_addr))
4654 shader_info->num_input_vgprs += 1;
4655 if (G_0286CC_POS_W_FLOAT_ENA(config->spi_ps_input_addr))
4656 shader_info->num_input_vgprs += 1;
4657 if (G_0286CC_FRONT_FACE_ENA(config->spi_ps_input_addr))
4658 shader_info->num_input_vgprs += 1;
4659 if (G_0286CC_ANCILLARY_ENA(config->spi_ps_input_addr))
4660 shader_info->num_input_vgprs += 1;
4661 if (G_0286CC_SAMPLE_COVERAGE_ENA(config->spi_ps_input_addr))
4662 shader_info->num_input_vgprs += 1;
4663 if (G_0286CC_POS_FIXED_PT_ENA(config->spi_ps_input_addr))
4664 shader_info->num_input_vgprs += 1;
4665 }
4666 config->num_vgprs = MAX2(config->num_vgprs, shader_info->num_input_vgprs);
4667
4668 /* +3 for scratch wave offset and VCC */
4669 config->num_sgprs = MAX2(config->num_sgprs,
4670 shader_info->num_input_sgprs + 3);
4671 if (nir->stage == MESA_SHADER_COMPUTE) {
4672 for (int i = 0; i < 3; ++i)
4673 shader_info->cs.block_size[i] = nir->info->cs.local_size[i];
4674 }
4675
4676 if (nir->stage == MESA_SHADER_FRAGMENT)
4677 shader_info->fs.early_fragment_test = nir->info->fs.early_fragment_tests;
4678 }