projects / mesa.git / blob
? search:


443f0c9bf2a359b6891324a43a52bc2962b5c0d2
[mesa.git] / src / gallium / drivers / radeonsi / si_shader_tgsi_alu.c
1 /*
2 * Copyright 2016 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * on the rights to use, copy, modify, merge, publish, distribute, sub
8 * license, and/or sell copies of the Software, and to permit persons to whom
9 * the Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHOR(S) AND/OR THEIR SUPPLIERS BE LIABLE FOR ANY CLAIM,
19 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
20 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
21 * USE OR OTHER DEALINGS IN THE SOFTWARE.
22 */
23
24 #include "si_shader_internal.h"
25 #include "gallivm/lp_bld_const.h"
26 #include "gallivm/lp_bld_intr.h"
27 #include "gallivm/lp_bld_gather.h"
28 #include "tgsi/tgsi_parse.h"
29 #include "amd/common/ac_llvm_build.h"
30
31 static void kill_if_fetch_args(struct lp_build_tgsi_context *bld_base,
32 struct lp_build_emit_data *emit_data)
33 {
34 const struct tgsi_full_instruction *inst = emit_data->inst;
35 struct si_shader_context *ctx = si_shader_context(bld_base);
36 struct gallivm_state *gallivm = bld_base->base.gallivm;
37 LLVMBuilderRef builder = ctx->ac.builder;
38 unsigned i;
39 LLVMValueRef conds[TGSI_NUM_CHANNELS];
40
41 for (i = 0; i < TGSI_NUM_CHANNELS; i++) {
42 LLVMValueRef value = lp_build_emit_fetch(bld_base, inst, 0, i);
43 conds[i] = LLVMBuildFCmp(builder, LLVMRealOLT, value,
44 bld_base->base.zero, "");
45 }
46
47 /* Or the conditions together */
48 for (i = TGSI_NUM_CHANNELS - 1; i > 0; i--) {
49 conds[i - 1] = LLVMBuildOr(builder, conds[i], conds[i - 1], "");
50 }
51
52 emit_data->dst_type = LLVMVoidTypeInContext(gallivm->context);
53 emit_data->arg_count = 1;
54 emit_data->args[0] = LLVMBuildSelect(builder, conds[0],
55 lp_build_const_float(gallivm, -1.0f),
56 bld_base->base.zero, "");
57 }
58
59 static void kil_emit(const struct lp_build_tgsi_action *action,
60 struct lp_build_tgsi_context *bld_base,
61 struct lp_build_emit_data *emit_data)
62 {
63 struct si_shader_context *ctx = si_shader_context(bld_base);
64 LLVMBuilderRef builder = ctx->ac.builder;
65
66 if (ctx->postponed_kill) {
67 if (emit_data->inst->Instruction.Opcode == TGSI_OPCODE_KILL_IF) {
68 LLVMValueRef val;
69
70 /* Take the minimum kill value. This is the same as OR
71 * between 2 kill values. If the value is negative,
72 * the pixel will be killed.
73 */
74 val = LLVMBuildLoad(builder, ctx->postponed_kill, "");
75 val = lp_build_emit_llvm_binary(bld_base, TGSI_OPCODE_MIN,
76 val, emit_data->args[0]);
77 LLVMBuildStore(builder, val, ctx->postponed_kill);
78 } else {
79 LLVMBuildStore(builder,
80 LLVMConstReal(ctx->f32, -1),
81 ctx->postponed_kill);
82 }
83 return;
84 }
85
86 if (emit_data->inst->Instruction.Opcode == TGSI_OPCODE_KILL_IF)
87 ac_build_kill(&ctx->ac, emit_data->args[0]);
88 else
89 ac_build_kill(&ctx->ac, NULL);
90 }
91
92 static void emit_icmp(const struct lp_build_tgsi_action *action,
93 struct lp_build_tgsi_context *bld_base,
94 struct lp_build_emit_data *emit_data)
95 {
96 unsigned pred;
97 struct si_shader_context *ctx = si_shader_context(bld_base);
98 LLVMContextRef context = bld_base->base.gallivm->context;
99
100 switch (emit_data->inst->Instruction.Opcode) {
101 case TGSI_OPCODE_USEQ:
102 case TGSI_OPCODE_U64SEQ: pred = LLVMIntEQ; break;
103 case TGSI_OPCODE_USNE:
104 case TGSI_OPCODE_U64SNE: pred = LLVMIntNE; break;
105 case TGSI_OPCODE_USGE:
106 case TGSI_OPCODE_U64SGE: pred = LLVMIntUGE; break;
107 case TGSI_OPCODE_USLT:
108 case TGSI_OPCODE_U64SLT: pred = LLVMIntULT; break;
109 case TGSI_OPCODE_ISGE:
110 case TGSI_OPCODE_I64SGE: pred = LLVMIntSGE; break;
111 case TGSI_OPCODE_ISLT:
112 case TGSI_OPCODE_I64SLT: pred = LLVMIntSLT; break;
113 default:
114 assert(!"unknown instruction");
115 pred = 0;
116 break;
117 }
118
119 LLVMValueRef v = LLVMBuildICmp(ctx->ac.builder, pred,
120 emit_data->args[0], emit_data->args[1],"");
121
122 v = LLVMBuildSExtOrBitCast(ctx->ac.builder, v,
123 LLVMInt32TypeInContext(context), "");
124
125 emit_data->output[emit_data->chan] = v;
126 }
127
128 static void emit_ucmp(const struct lp_build_tgsi_action *action,
129 struct lp_build_tgsi_context *bld_base,
130 struct lp_build_emit_data *emit_data)
131 {
132 struct si_shader_context *ctx = si_shader_context(bld_base);
133 LLVMValueRef arg0 = ac_to_integer(&ctx->ac, emit_data->args[0]);
134
135 LLVMValueRef v = LLVMBuildICmp(ctx->ac.builder, LLVMIntNE, arg0,
136 ctx->i32_0, "");
137
138 emit_data->output[emit_data->chan] =
139 LLVMBuildSelect(ctx->ac.builder, v, emit_data->args[1], emit_data->args[2], "");
140 }
141
142 static void emit_cmp(const struct lp_build_tgsi_action *action,
143 struct lp_build_tgsi_context *bld_base,
144 struct lp_build_emit_data *emit_data)
145 {
146 struct si_shader_context *ctx = si_shader_context(bld_base);
147 LLVMValueRef cond, *args = emit_data->args;
148
149 cond = LLVMBuildFCmp(ctx->ac.builder, LLVMRealOLT, args[0],
150 bld_base->base.zero, "");
151
152 emit_data->output[emit_data->chan] =
153 LLVMBuildSelect(ctx->ac.builder, cond, args[1], args[2], "");
154 }
155
156 static void emit_set_cond(const struct lp_build_tgsi_action *action,
157 struct lp_build_tgsi_context *bld_base,
158 struct lp_build_emit_data *emit_data)
159 {
160 struct si_shader_context *ctx = si_shader_context(bld_base);
161 LLVMRealPredicate pred;
162 LLVMValueRef cond;
163
164 /* Use ordered for everything but NE (which is usual for
165 * float comparisons)
166 */
167 switch (emit_data->inst->Instruction.Opcode) {
168 case TGSI_OPCODE_SGE: pred = LLVMRealOGE; break;
169 case TGSI_OPCODE_SEQ: pred = LLVMRealOEQ; break;
170 case TGSI_OPCODE_SLE: pred = LLVMRealOLE; break;
171 case TGSI_OPCODE_SLT: pred = LLVMRealOLT; break;
172 case TGSI_OPCODE_SNE: pred = LLVMRealUNE; break;
173 case TGSI_OPCODE_SGT: pred = LLVMRealOGT; break;
174 default: assert(!"unknown instruction"); pred = 0; break;
175 }
176
177 cond = LLVMBuildFCmp(ctx->ac.builder,
178 pred, emit_data->args[0], emit_data->args[1], "");
179
180 emit_data->output[emit_data->chan] = LLVMBuildSelect(ctx->ac.builder,
181 cond, bld_base->base.one, bld_base->base.zero, "");
182 }
183
184 static void emit_fcmp(const struct lp_build_tgsi_action *action,
185 struct lp_build_tgsi_context *bld_base,
186 struct lp_build_emit_data *emit_data)
187 {
188 struct si_shader_context *ctx = si_shader_context(bld_base);
189 LLVMContextRef context = bld_base->base.gallivm->context;
190 LLVMRealPredicate pred;
191
192 /* Use ordered for everything but NE (which is usual for
193 * float comparisons)
194 */
195 switch (emit_data->inst->Instruction.Opcode) {
196 case TGSI_OPCODE_FSEQ: pred = LLVMRealOEQ; break;
197 case TGSI_OPCODE_FSGE: pred = LLVMRealOGE; break;
198 case TGSI_OPCODE_FSLT: pred = LLVMRealOLT; break;
199 case TGSI_OPCODE_FSNE: pred = LLVMRealUNE; break;
200 default: assert(!"unknown instruction"); pred = 0; break;
201 }
202
203 LLVMValueRef v = LLVMBuildFCmp(ctx->ac.builder, pred,
204 emit_data->args[0], emit_data->args[1],"");
205
206 v = LLVMBuildSExtOrBitCast(ctx->ac.builder, v,
207 LLVMInt32TypeInContext(context), "");
208
209 emit_data->output[emit_data->chan] = v;
210 }
211
212 static void emit_dcmp(const struct lp_build_tgsi_action *action,
213 struct lp_build_tgsi_context *bld_base,
214 struct lp_build_emit_data *emit_data)
215 {
216 struct si_shader_context *ctx = si_shader_context(bld_base);
217 LLVMContextRef context = bld_base->base.gallivm->context;
218 LLVMRealPredicate pred;
219
220 /* Use ordered for everything but NE (which is usual for
221 * float comparisons)
222 */
223 switch (emit_data->inst->Instruction.Opcode) {
224 case TGSI_OPCODE_DSEQ: pred = LLVMRealOEQ; break;
225 case TGSI_OPCODE_DSGE: pred = LLVMRealOGE; break;
226 case TGSI_OPCODE_DSLT: pred = LLVMRealOLT; break;
227 case TGSI_OPCODE_DSNE: pred = LLVMRealUNE; break;
228 default: assert(!"unknown instruction"); pred = 0; break;
229 }
230
231 LLVMValueRef v = LLVMBuildFCmp(ctx->ac.builder, pred,
232 emit_data->args[0], emit_data->args[1],"");
233
234 v = LLVMBuildSExtOrBitCast(ctx->ac.builder, v,
235 LLVMInt32TypeInContext(context), "");
236
237 emit_data->output[emit_data->chan] = v;
238 }
239
240 static void emit_not(const struct lp_build_tgsi_action *action,
241 struct lp_build_tgsi_context *bld_base,
242 struct lp_build_emit_data *emit_data)
243 {
244 struct si_shader_context *ctx = si_shader_context(bld_base);
245 LLVMValueRef v = ac_to_integer(&ctx->ac, emit_data->args[0]);
246 emit_data->output[emit_data->chan] = LLVMBuildNot(ctx->ac.builder, v, "");
247 }
248
249 static void emit_arl(const struct lp_build_tgsi_action *action,
250 struct lp_build_tgsi_context *bld_base,
251 struct lp_build_emit_data *emit_data)
252 {
253 struct si_shader_context *ctx = si_shader_context(bld_base);
254 LLVMValueRef floor_index = lp_build_emit_llvm_unary(bld_base, TGSI_OPCODE_FLR, emit_data->args[0]);
255 emit_data->output[emit_data->chan] = LLVMBuildFPToSI(ctx->ac.builder,
256 floor_index, ctx->i32, "");
257 }
258
259 static void emit_and(const struct lp_build_tgsi_action *action,
260 struct lp_build_tgsi_context *bld_base,
261 struct lp_build_emit_data *emit_data)
262 {
263 struct si_shader_context *ctx = si_shader_context(bld_base);
264 emit_data->output[emit_data->chan] = LLVMBuildAnd(ctx->ac.builder,
265 emit_data->args[0], emit_data->args[1], "");
266 }
267
268 static void emit_or(const struct lp_build_tgsi_action *action,
269 struct lp_build_tgsi_context *bld_base,
270 struct lp_build_emit_data *emit_data)
271 {
272 struct si_shader_context *ctx = si_shader_context(bld_base);
273 emit_data->output[emit_data->chan] = LLVMBuildOr(ctx->ac.builder,
274 emit_data->args[0], emit_data->args[1], "");
275 }
276
277 static void emit_uadd(const struct lp_build_tgsi_action *action,
278 struct lp_build_tgsi_context *bld_base,
279 struct lp_build_emit_data *emit_data)
280 {
281 struct si_shader_context *ctx = si_shader_context(bld_base);
282 emit_data->output[emit_data->chan] = LLVMBuildAdd(ctx->ac.builder,
283 emit_data->args[0], emit_data->args[1], "");
284 }
285
286 static void emit_udiv(const struct lp_build_tgsi_action *action,
287 struct lp_build_tgsi_context *bld_base,
288 struct lp_build_emit_data *emit_data)
289 {
290 struct si_shader_context *ctx = si_shader_context(bld_base);
291 emit_data->output[emit_data->chan] = LLVMBuildUDiv(ctx->ac.builder,
292 emit_data->args[0], emit_data->args[1], "");
293 }
294
295 static void emit_idiv(const struct lp_build_tgsi_action *action,
296 struct lp_build_tgsi_context *bld_base,
297 struct lp_build_emit_data *emit_data)
298 {
299 struct si_shader_context *ctx = si_shader_context(bld_base);
300 emit_data->output[emit_data->chan] = LLVMBuildSDiv(ctx->ac.builder,
301 emit_data->args[0], emit_data->args[1], "");
302 }
303
304 static void emit_mod(const struct lp_build_tgsi_action *action,
305 struct lp_build_tgsi_context *bld_base,
306 struct lp_build_emit_data *emit_data)
307 {
308 struct si_shader_context *ctx = si_shader_context(bld_base);
309 emit_data->output[emit_data->chan] = LLVMBuildSRem(ctx->ac.builder,
310 emit_data->args[0], emit_data->args[1], "");
311 }
312
313 static void emit_umod(const struct lp_build_tgsi_action *action,
314 struct lp_build_tgsi_context *bld_base,
315 struct lp_build_emit_data *emit_data)
316 {
317 struct si_shader_context *ctx = si_shader_context(bld_base);
318 emit_data->output[emit_data->chan] = LLVMBuildURem(ctx->ac.builder,
319 emit_data->args[0], emit_data->args[1], "");
320 }
321
322 static void emit_shl(const struct lp_build_tgsi_action *action,
323 struct lp_build_tgsi_context *bld_base,
324 struct lp_build_emit_data *emit_data)
325 {
326 struct si_shader_context *ctx = si_shader_context(bld_base);
327 emit_data->output[emit_data->chan] = LLVMBuildShl(ctx->ac.builder,
328 emit_data->args[0], emit_data->args[1], "");
329 }
330
331 static void emit_ushr(const struct lp_build_tgsi_action *action,
332 struct lp_build_tgsi_context *bld_base,
333 struct lp_build_emit_data *emit_data)
334 {
335 struct si_shader_context *ctx = si_shader_context(bld_base);
336 emit_data->output[emit_data->chan] = LLVMBuildLShr(ctx->ac.builder,
337 emit_data->args[0], emit_data->args[1], "");
338 }
339 static void emit_ishr(const struct lp_build_tgsi_action *action,
340 struct lp_build_tgsi_context *bld_base,
341 struct lp_build_emit_data *emit_data)
342 {
343 struct si_shader_context *ctx = si_shader_context(bld_base);
344 emit_data->output[emit_data->chan] = LLVMBuildAShr(ctx->ac.builder,
345 emit_data->args[0], emit_data->args[1], "");
346 }
347
348 static void emit_xor(const struct lp_build_tgsi_action *action,
349 struct lp_build_tgsi_context *bld_base,
350 struct lp_build_emit_data *emit_data)
351 {
352 struct si_shader_context *ctx = si_shader_context(bld_base);
353 emit_data->output[emit_data->chan] = LLVMBuildXor(ctx->ac.builder,
354 emit_data->args[0], emit_data->args[1], "");
355 }
356
357 static void emit_ssg(const struct lp_build_tgsi_action *action,
358 struct lp_build_tgsi_context *bld_base,
359 struct lp_build_emit_data *emit_data)
360 {
361 struct si_shader_context *ctx = si_shader_context(bld_base);
362 LLVMBuilderRef builder = ctx->ac.builder;
363
364 LLVMValueRef cmp, val;
365
366 if (emit_data->inst->Instruction.Opcode == TGSI_OPCODE_I64SSG) {
367 cmp = LLVMBuildICmp(builder, LLVMIntSGT, emit_data->args[0], bld_base->int64_bld.zero, "");
368 val = LLVMBuildSelect(builder, cmp, bld_base->int64_bld.one, emit_data->args[0], "");
369 cmp = LLVMBuildICmp(builder, LLVMIntSGE, val, bld_base->int64_bld.zero, "");
370 val = LLVMBuildSelect(builder, cmp, val, LLVMConstInt(ctx->i64, -1, true), "");
371 } else if (emit_data->inst->Instruction.Opcode == TGSI_OPCODE_ISSG) {
372 cmp = LLVMBuildICmp(builder, LLVMIntSGT, emit_data->args[0], ctx->i32_0, "");
373 val = LLVMBuildSelect(builder, cmp, ctx->i32_1, emit_data->args[0], "");
374 cmp = LLVMBuildICmp(builder, LLVMIntSGE, val, ctx->i32_0, "");
375 val = LLVMBuildSelect(builder, cmp, val, LLVMConstInt(ctx->i32, -1, true), "");
376 } else if (emit_data->inst->Instruction.Opcode == TGSI_OPCODE_DSSG) {
377 cmp = LLVMBuildFCmp(builder, LLVMRealOGT, emit_data->args[0], bld_base->dbl_bld.zero, "");
378 val = LLVMBuildSelect(builder, cmp, bld_base->dbl_bld.one, emit_data->args[0], "");
379 cmp = LLVMBuildFCmp(builder, LLVMRealOGE, val, bld_base->dbl_bld.zero, "");
380 val = LLVMBuildSelect(builder, cmp, val, LLVMConstReal(bld_base->dbl_bld.elem_type, -1), "");
381 } else { // float SSG
382 cmp = LLVMBuildFCmp(builder, LLVMRealOGT, emit_data->args[0], bld_base->base.zero, "");
383 val = LLVMBuildSelect(builder, cmp, bld_base->base.one, emit_data->args[0], "");
384 cmp = LLVMBuildFCmp(builder, LLVMRealOGE, val, bld_base->base.zero, "");
385 val = LLVMBuildSelect(builder, cmp, val, LLVMConstReal(ctx->f32, -1), "");
386 }
387
388 emit_data->output[emit_data->chan] = val;
389 }
390
391 static void emit_ineg(const struct lp_build_tgsi_action *action,
392 struct lp_build_tgsi_context *bld_base,
393 struct lp_build_emit_data *emit_data)
394 {
395 struct si_shader_context *ctx = si_shader_context(bld_base);
396 emit_data->output[emit_data->chan] = LLVMBuildNeg(ctx->ac.builder,
397 emit_data->args[0], "");
398 }
399
400 static void emit_dneg(const struct lp_build_tgsi_action *action,
401 struct lp_build_tgsi_context *bld_base,
402 struct lp_build_emit_data *emit_data)
403 {
404 struct si_shader_context *ctx = si_shader_context(bld_base);
405 emit_data->output[emit_data->chan] = LLVMBuildFNeg(ctx->ac.builder,
406 emit_data->args[0], "");
407 }
408
409 static void emit_frac(const struct lp_build_tgsi_action *action,
410 struct lp_build_tgsi_context *bld_base,
411 struct lp_build_emit_data *emit_data)
412 {
413 struct si_shader_context *ctx = si_shader_context(bld_base);
414 char *intr;
415
416 if (emit_data->info->opcode == TGSI_OPCODE_FRC)
417 intr = "llvm.floor.f32";
418 else if (emit_data->info->opcode == TGSI_OPCODE_DFRAC)
419 intr = "llvm.floor.f64";
420 else {
421 assert(0);
422 return;
423 }
424
425 LLVMValueRef floor = lp_build_intrinsic(ctx->ac.builder, intr, emit_data->dst_type,
426 &emit_data->args[0], 1,
427 LP_FUNC_ATTR_READNONE);
428 emit_data->output[emit_data->chan] = LLVMBuildFSub(ctx->ac.builder,
429 emit_data->args[0], floor, "");
430 }
431
432 static void emit_f2i(const struct lp_build_tgsi_action *action,
433 struct lp_build_tgsi_context *bld_base,
434 struct lp_build_emit_data *emit_data)
435 {
436 struct si_shader_context *ctx = si_shader_context(bld_base);
437 emit_data->output[emit_data->chan] = LLVMBuildFPToSI(ctx->ac.builder,
438 emit_data->args[0], bld_base->int_bld.elem_type, "");
439 }
440
441 static void emit_f2u(const struct lp_build_tgsi_action *action,
442 struct lp_build_tgsi_context *bld_base,
443 struct lp_build_emit_data *emit_data)
444 {
445 struct si_shader_context *ctx = si_shader_context(bld_base);
446 emit_data->output[emit_data->chan] = LLVMBuildFPToUI(ctx->ac.builder,
447 emit_data->args[0], bld_base->uint_bld.elem_type, "");
448 }
449
450 static void emit_i2f(const struct lp_build_tgsi_action *action,
451 struct lp_build_tgsi_context *bld_base,
452 struct lp_build_emit_data *emit_data)
453 {
454 struct si_shader_context *ctx = si_shader_context(bld_base);
455 emit_data->output[emit_data->chan] = LLVMBuildSIToFP(ctx->ac.builder,
456 emit_data->args[0], bld_base->base.elem_type, "");
457 }
458
459 static void emit_u2f(const struct lp_build_tgsi_action *action,
460 struct lp_build_tgsi_context *bld_base,
461 struct lp_build_emit_data *emit_data)
462 {
463 struct si_shader_context *ctx = si_shader_context(bld_base);
464 emit_data->output[emit_data->chan] = LLVMBuildUIToFP(ctx->ac.builder,
465 emit_data->args[0], bld_base->base.elem_type, "");
466 }
467
468 static void
469 build_tgsi_intrinsic_nomem(const struct lp_build_tgsi_action *action,
470 struct lp_build_tgsi_context *bld_base,
471 struct lp_build_emit_data *emit_data)
472 {
473 struct si_shader_context *ctx = si_shader_context(bld_base);
474 emit_data->output[emit_data->chan] =
475 lp_build_intrinsic(ctx->ac.builder, action->intr_name,
476 emit_data->dst_type, emit_data->args,
477 emit_data->arg_count, LP_FUNC_ATTR_READNONE);
478 }
479
480 static void emit_bfi(const struct lp_build_tgsi_action *action,
481 struct lp_build_tgsi_context *bld_base,
482 struct lp_build_emit_data *emit_data)
483 {
484 struct si_shader_context *ctx = si_shader_context(bld_base);
485 struct gallivm_state *gallivm = bld_base->base.gallivm;
486 LLVMBuilderRef builder = ctx->ac.builder;
487 LLVMValueRef bfi_args[3];
488 LLVMValueRef bfi_sm5;
489 LLVMValueRef cond;
490
491 // Calculate the bitmask: (((1 << src3) - 1) << src2
492 bfi_args[0] = LLVMBuildShl(builder,
493 LLVMBuildSub(builder,
494 LLVMBuildShl(builder,
495 ctx->i32_1,
496 emit_data->args[3], ""),
497 ctx->i32_1, ""),
498 emit_data->args[2], "");
499
500 bfi_args[1] = LLVMBuildShl(builder, emit_data->args[1],
501 emit_data->args[2], "");
502
503 bfi_args[2] = emit_data->args[0];
504
505 /* Calculate:
506 * (arg0 & arg1) | (~arg0 & arg2) = arg2 ^ (arg0 & (arg1 ^ arg2)
507 * Use the right-hand side, which the LLVM backend can convert to V_BFI.
508 */
509 bfi_sm5 =
510 LLVMBuildXor(builder, bfi_args[2],
511 LLVMBuildAnd(builder, bfi_args[0],
512 LLVMBuildXor(builder, bfi_args[1], bfi_args[2],
513 ""), ""), "");
514
515 /* Since shifts of >= 32 bits are undefined in LLVM IR, the backend
516 * uses the convenient V_BFI lowering for the above, which follows SM5
517 * and disagrees with GLSL semantics when bits (src3) is 32.
518 */
519 cond = LLVMBuildICmp(builder, LLVMIntUGE, emit_data->args[3],
520 lp_build_const_int32(gallivm, 32), "");
521 emit_data->output[emit_data->chan] =
522 LLVMBuildSelect(builder, cond, emit_data->args[1], bfi_sm5, "");
523 }
524
525 static void emit_bfe(const struct lp_build_tgsi_action *action,
526 struct lp_build_tgsi_context *bld_base,
527 struct lp_build_emit_data *emit_data)
528 {
529 struct si_shader_context *ctx = si_shader_context(bld_base);
530 LLVMValueRef bfe_sm5;
531 LLVMValueRef cond;
532
533 bfe_sm5 = ac_build_bfe(&ctx->ac, emit_data->args[0],
534 emit_data->args[1], emit_data->args[2],
535 emit_data->info->opcode == TGSI_OPCODE_IBFE);
536
537 /* Correct for GLSL semantics. */
538 cond = LLVMBuildICmp(ctx->ac.builder, LLVMIntUGE, emit_data->args[2],
539 LLVMConstInt(ctx->i32, 32, 0), "");
540 emit_data->output[emit_data->chan] =
541 LLVMBuildSelect(ctx->ac.builder, cond, emit_data->args[0], bfe_sm5, "");
542 }
543
544 /* this is ffs in C */
545 static void emit_lsb(const struct lp_build_tgsi_action *action,
546 struct lp_build_tgsi_context *bld_base,
547 struct lp_build_emit_data *emit_data)
548 {
549 struct si_shader_context *ctx = si_shader_context(bld_base);
550 struct gallivm_state *gallivm = bld_base->base.gallivm;
551 LLVMValueRef args[2] = {
552 emit_data->args[0],
553
554 /* The value of 1 means that ffs(x=0) = undef, so LLVM won't
555 * add special code to check for x=0. The reason is that
556 * the LLVM behavior for x=0 is different from what we
557 * need here. However, LLVM also assumes that ffs(x) is
558 * in [0, 31], but GLSL expects that ffs(0) = -1, so
559 * a conditional assignment to handle 0 is still required.
560 */
561 LLVMConstInt(LLVMInt1TypeInContext(gallivm->context), 1, 0)
562 };
563
564 LLVMValueRef lsb =
565 lp_build_intrinsic(ctx->ac.builder, "llvm.cttz.i32",
566 emit_data->dst_type, args, ARRAY_SIZE(args),
567 LP_FUNC_ATTR_READNONE);
568
569 /* TODO: We need an intrinsic to skip this conditional. */
570 /* Check for zero: */
571 emit_data->output[emit_data->chan] =
572 LLVMBuildSelect(ctx->ac.builder,
573 LLVMBuildICmp(ctx->ac.builder, LLVMIntEQ, args[0],
574 ctx->i32_0, ""),
575 lp_build_const_int32(gallivm, -1), lsb, "");
576 }
577
578 /* Find the last bit set. */
579 static void emit_umsb(const struct lp_build_tgsi_action *action,
580 struct lp_build_tgsi_context *bld_base,
581 struct lp_build_emit_data *emit_data)
582 {
583 struct si_shader_context *ctx = si_shader_context(bld_base);
584
585 emit_data->output[emit_data->chan] =
586 ac_build_umsb(&ctx->ac, emit_data->args[0], emit_data->dst_type);
587 }
588
589 /* Find the last bit opposite of the sign bit. */
590 static void emit_imsb(const struct lp_build_tgsi_action *action,
591 struct lp_build_tgsi_context *bld_base,
592 struct lp_build_emit_data *emit_data)
593 {
594 struct si_shader_context *ctx = si_shader_context(bld_base);
595 emit_data->output[emit_data->chan] =
596 ac_build_imsb(&ctx->ac, emit_data->args[0],
597 emit_data->dst_type);
598 }
599
600 static void emit_iabs(const struct lp_build_tgsi_action *action,
601 struct lp_build_tgsi_context *bld_base,
602 struct lp_build_emit_data *emit_data)
603 {
604 struct si_shader_context *ctx = si_shader_context(bld_base);
605
606 emit_data->output[emit_data->chan] =
607 lp_build_emit_llvm_binary(bld_base, TGSI_OPCODE_IMAX,
608 emit_data->args[0],
609 LLVMBuildNeg(ctx->ac.builder,
610 emit_data->args[0], ""));
611 }
612
613 static void emit_minmax_int(const struct lp_build_tgsi_action *action,
614 struct lp_build_tgsi_context *bld_base,
615 struct lp_build_emit_data *emit_data)
616 {
617 struct si_shader_context *ctx = si_shader_context(bld_base);
618 LLVMIntPredicate op;
619
620 switch (emit_data->info->opcode) {
621 default:
622 assert(0);
623 case TGSI_OPCODE_IMAX:
624 case TGSI_OPCODE_I64MAX:
625 op = LLVMIntSGT;
626 break;
627 case TGSI_OPCODE_IMIN:
628 case TGSI_OPCODE_I64MIN:
629 op = LLVMIntSLT;
630 break;
631 case TGSI_OPCODE_UMAX:
632 case TGSI_OPCODE_U64MAX:
633 op = LLVMIntUGT;
634 break;
635 case TGSI_OPCODE_UMIN:
636 case TGSI_OPCODE_U64MIN:
637 op = LLVMIntULT;
638 break;
639 }
640
641 emit_data->output[emit_data->chan] =
642 LLVMBuildSelect(ctx->ac.builder,
643 LLVMBuildICmp(ctx->ac.builder, op, emit_data->args[0],
644 emit_data->args[1], ""),
645 emit_data->args[0],
646 emit_data->args[1], "");
647 }
648
649 static void pk2h_fetch_args(struct lp_build_tgsi_context *bld_base,
650 struct lp_build_emit_data *emit_data)
651 {
652 emit_data->args[0] = lp_build_emit_fetch(bld_base, emit_data->inst,
653 0, TGSI_CHAN_X);
654 emit_data->args[1] = lp_build_emit_fetch(bld_base, emit_data->inst,
655 0, TGSI_CHAN_Y);
656 }
657
658 static void emit_pk2h(const struct lp_build_tgsi_action *action,
659 struct lp_build_tgsi_context *bld_base,
660 struct lp_build_emit_data *emit_data)
661 {
662 /* From the GLSL 4.50 spec:
663 * "The rounding mode cannot be set and is undefined."
664 *
665 * v_cvt_pkrtz_f16 rounds to zero, but it's fastest.
666 */
667 emit_data->output[emit_data->chan] =
668 ac_build_cvt_pkrtz_f16(&si_shader_context(bld_base)->ac,
669 emit_data->args);
670 }
671
672 static void up2h_fetch_args(struct lp_build_tgsi_context *bld_base,
673 struct lp_build_emit_data *emit_data)
674 {
675 emit_data->args[0] = lp_build_emit_fetch(bld_base, emit_data->inst,
676 0, TGSI_CHAN_X);
677 }
678
679 static void emit_up2h(const struct lp_build_tgsi_action *action,
680 struct lp_build_tgsi_context *bld_base,
681 struct lp_build_emit_data *emit_data)
682 {
683 struct si_shader_context *ctx = si_shader_context(bld_base);
684 LLVMContextRef context = bld_base->base.gallivm->context;
685 struct lp_build_context *uint_bld = &bld_base->uint_bld;
686 LLVMTypeRef i16;
687 LLVMValueRef const16, input, val;
688 unsigned i;
689
690 i16 = LLVMInt16TypeInContext(context);
691 const16 = lp_build_const_int32(uint_bld->gallivm, 16);
692 input = emit_data->args[0];
693
694 for (i = 0; i < 2; i++) {
695 val = i == 1 ? LLVMBuildLShr(ctx->ac.builder, input, const16, "") : input;
696 val = LLVMBuildTrunc(ctx->ac.builder, val, i16, "");
697 val = ac_to_float(&ctx->ac, val);
698 emit_data->output[i] = LLVMBuildFPExt(ctx->ac.builder, val, ctx->f32, "");
699 }
700 }
701
702 static void emit_fdiv(const struct lp_build_tgsi_action *action,
703 struct lp_build_tgsi_context *bld_base,
704 struct lp_build_emit_data *emit_data)
705 {
706 struct si_shader_context *ctx = si_shader_context(bld_base);
707
708 emit_data->output[emit_data->chan] =
709 LLVMBuildFDiv(ctx->ac.builder,
710 emit_data->args[0], emit_data->args[1], "");
711
712 /* Use v_rcp_f32 instead of precise division. */
713 if (!LLVMIsConstant(emit_data->output[emit_data->chan]))
714 LLVMSetMetadata(emit_data->output[emit_data->chan],
715 ctx->fpmath_md_kind, ctx->fpmath_md_2p5_ulp);
716 }
717
718 /* 1/sqrt is translated to rsq for f32 if fp32 denormals are not enabled in
719 * the target machine. f64 needs global unsafe math flags to get rsq. */
720 static void emit_rsq(const struct lp_build_tgsi_action *action,
721 struct lp_build_tgsi_context *bld_base,
722 struct lp_build_emit_data *emit_data)
723 {
724 LLVMValueRef sqrt =
725 lp_build_emit_llvm_unary(bld_base, TGSI_OPCODE_SQRT,
726 emit_data->args[0]);
727
728 emit_data->output[emit_data->chan] =
729 lp_build_emit_llvm_binary(bld_base, TGSI_OPCODE_DIV,
730 bld_base->base.one, sqrt);
731 }
732
733 static void dfracexp_fetch_args(struct lp_build_tgsi_context *bld_base,
734 struct lp_build_emit_data *emit_data)
735 {
736 emit_data->args[0] = lp_build_emit_fetch(bld_base, emit_data->inst, 0, TGSI_CHAN_X);
737 emit_data->arg_count = 1;
738 }
739
740 static void dfracexp_emit(const struct lp_build_tgsi_action *action,
741 struct lp_build_tgsi_context *bld_base,
742 struct lp_build_emit_data *emit_data)
743 {
744 struct si_shader_context *ctx = si_shader_context(bld_base);
745
746 emit_data->output[emit_data->chan] =
747 lp_build_intrinsic(ctx->ac.builder, "llvm.amdgcn.frexp.mant.f64",
748 ctx->ac.f64, &emit_data->args[0], 1, 0);
749 emit_data->output1[emit_data->chan] =
750 lp_build_intrinsic(ctx->ac.builder, "llvm.amdgcn.frexp.exp.i32.f64",
751 ctx->ac.i32, &emit_data->args[0], 1, 0);
752 }
753
754 void si_shader_context_init_alu(struct lp_build_tgsi_context *bld_base)
755 {
756 lp_set_default_actions(bld_base);
757
758 bld_base->op_actions[TGSI_OPCODE_AND].emit = emit_and;
759 bld_base->op_actions[TGSI_OPCODE_ARL].emit = emit_arl;
760 bld_base->op_actions[TGSI_OPCODE_BFI].emit = emit_bfi;
761 bld_base->op_actions[TGSI_OPCODE_BREV].emit = build_tgsi_intrinsic_nomem;
762 bld_base->op_actions[TGSI_OPCODE_BREV].intr_name = "llvm.bitreverse.i32";
763 bld_base->op_actions[TGSI_OPCODE_CEIL].emit = build_tgsi_intrinsic_nomem;
764 bld_base->op_actions[TGSI_OPCODE_CEIL].intr_name = "llvm.ceil.f32";
765 bld_base->op_actions[TGSI_OPCODE_CMP].emit = emit_cmp;
766 bld_base->op_actions[TGSI_OPCODE_COS].emit = build_tgsi_intrinsic_nomem;
767 bld_base->op_actions[TGSI_OPCODE_COS].intr_name = "llvm.cos.f32";
768 bld_base->op_actions[TGSI_OPCODE_DABS].emit = build_tgsi_intrinsic_nomem;
769 bld_base->op_actions[TGSI_OPCODE_DABS].intr_name = "llvm.fabs.f64";
770 bld_base->op_actions[TGSI_OPCODE_DCEIL].emit = build_tgsi_intrinsic_nomem;
771 bld_base->op_actions[TGSI_OPCODE_DCEIL].intr_name = "llvm.ceil.f64";
772 bld_base->op_actions[TGSI_OPCODE_DFLR].emit = build_tgsi_intrinsic_nomem;
773 bld_base->op_actions[TGSI_OPCODE_DFLR].intr_name = "llvm.floor.f64";
774 bld_base->op_actions[TGSI_OPCODE_DFMA].emit = build_tgsi_intrinsic_nomem;
775 bld_base->op_actions[TGSI_OPCODE_DFMA].intr_name = "llvm.fma.f64";
776 bld_base->op_actions[TGSI_OPCODE_DFRAC].emit = emit_frac;
777 bld_base->op_actions[TGSI_OPCODE_DIV].emit = emit_fdiv;
778 bld_base->op_actions[TGSI_OPCODE_DNEG].emit = emit_dneg;
779 bld_base->op_actions[TGSI_OPCODE_DROUND].emit = build_tgsi_intrinsic_nomem;
780 bld_base->op_actions[TGSI_OPCODE_DROUND].intr_name = "llvm.rint.f64";
781 bld_base->op_actions[TGSI_OPCODE_DSEQ].emit = emit_dcmp;
782 bld_base->op_actions[TGSI_OPCODE_DSGE].emit = emit_dcmp;
783 bld_base->op_actions[TGSI_OPCODE_DSLT].emit = emit_dcmp;
784 bld_base->op_actions[TGSI_OPCODE_DSNE].emit = emit_dcmp;
785 bld_base->op_actions[TGSI_OPCODE_DSSG].emit = emit_ssg;
786 bld_base->op_actions[TGSI_OPCODE_DRSQ].emit = build_tgsi_intrinsic_nomem;
787 bld_base->op_actions[TGSI_OPCODE_DRSQ].intr_name = "llvm.amdgcn.rsq.f64";
788 bld_base->op_actions[TGSI_OPCODE_DSQRT].emit = build_tgsi_intrinsic_nomem;
789 bld_base->op_actions[TGSI_OPCODE_DSQRT].intr_name = "llvm.sqrt.f64";
790 bld_base->op_actions[TGSI_OPCODE_DTRUNC].emit = build_tgsi_intrinsic_nomem;
791 bld_base->op_actions[TGSI_OPCODE_DTRUNC].intr_name = "llvm.trunc.f64";
792 bld_base->op_actions[TGSI_OPCODE_DFRACEXP].fetch_args = dfracexp_fetch_args;
793 bld_base->op_actions[TGSI_OPCODE_DFRACEXP].emit = dfracexp_emit;
794 bld_base->op_actions[TGSI_OPCODE_DLDEXP].emit = build_tgsi_intrinsic_nomem;
795 bld_base->op_actions[TGSI_OPCODE_DLDEXP].intr_name = "llvm.amdgcn.ldexp.f64";
796 bld_base->op_actions[TGSI_OPCODE_EX2].emit = build_tgsi_intrinsic_nomem;
797 bld_base->op_actions[TGSI_OPCODE_EX2].intr_name = "llvm.exp2.f32";
798 bld_base->op_actions[TGSI_OPCODE_FLR].emit = build_tgsi_intrinsic_nomem;
799 bld_base->op_actions[TGSI_OPCODE_FLR].intr_name = "llvm.floor.f32";
800 bld_base->op_actions[TGSI_OPCODE_FMA].emit =
801 bld_base->op_actions[TGSI_OPCODE_MAD].emit;
802 bld_base->op_actions[TGSI_OPCODE_FRC].emit = emit_frac;
803 bld_base->op_actions[TGSI_OPCODE_F2I].emit = emit_f2i;
804 bld_base->op_actions[TGSI_OPCODE_F2U].emit = emit_f2u;
805 bld_base->op_actions[TGSI_OPCODE_FSEQ].emit = emit_fcmp;
806 bld_base->op_actions[TGSI_OPCODE_FSGE].emit = emit_fcmp;
807 bld_base->op_actions[TGSI_OPCODE_FSLT].emit = emit_fcmp;
808 bld_base->op_actions[TGSI_OPCODE_FSNE].emit = emit_fcmp;
809 bld_base->op_actions[TGSI_OPCODE_IABS].emit = emit_iabs;
810 bld_base->op_actions[TGSI_OPCODE_IBFE].emit = emit_bfe;
811 bld_base->op_actions[TGSI_OPCODE_IDIV].emit = emit_idiv;
812 bld_base->op_actions[TGSI_OPCODE_IMAX].emit = emit_minmax_int;
813 bld_base->op_actions[TGSI_OPCODE_IMIN].emit = emit_minmax_int;
814 bld_base->op_actions[TGSI_OPCODE_IMSB].emit = emit_imsb;
815 bld_base->op_actions[TGSI_OPCODE_INEG].emit = emit_ineg;
816 bld_base->op_actions[TGSI_OPCODE_ISHR].emit = emit_ishr;
817 bld_base->op_actions[TGSI_OPCODE_ISGE].emit = emit_icmp;
818 bld_base->op_actions[TGSI_OPCODE_ISLT].emit = emit_icmp;
819 bld_base->op_actions[TGSI_OPCODE_ISSG].emit = emit_ssg;
820 bld_base->op_actions[TGSI_OPCODE_I2F].emit = emit_i2f;
821 bld_base->op_actions[TGSI_OPCODE_KILL_IF].fetch_args = kill_if_fetch_args;
822 bld_base->op_actions[TGSI_OPCODE_KILL_IF].emit = kil_emit;
823 bld_base->op_actions[TGSI_OPCODE_KILL].emit = kil_emit;
824 bld_base->op_actions[TGSI_OPCODE_LDEXP].emit = build_tgsi_intrinsic_nomem;
825 bld_base->op_actions[TGSI_OPCODE_LDEXP].intr_name = "llvm.amdgcn.ldexp.f32";
826 bld_base->op_actions[TGSI_OPCODE_LSB].emit = emit_lsb;
827 bld_base->op_actions[TGSI_OPCODE_LG2].emit = build_tgsi_intrinsic_nomem;
828 bld_base->op_actions[TGSI_OPCODE_LG2].intr_name = "llvm.log2.f32";
829 bld_base->op_actions[TGSI_OPCODE_MAX].emit = build_tgsi_intrinsic_nomem;
830 bld_base->op_actions[TGSI_OPCODE_MAX].intr_name = "llvm.maxnum.f32";
831 bld_base->op_actions[TGSI_OPCODE_MIN].emit = build_tgsi_intrinsic_nomem;
832 bld_base->op_actions[TGSI_OPCODE_MIN].intr_name = "llvm.minnum.f32";
833 bld_base->op_actions[TGSI_OPCODE_MOD].emit = emit_mod;
834 bld_base->op_actions[TGSI_OPCODE_UMSB].emit = emit_umsb;
835 bld_base->op_actions[TGSI_OPCODE_NOT].emit = emit_not;
836 bld_base->op_actions[TGSI_OPCODE_OR].emit = emit_or;
837 bld_base->op_actions[TGSI_OPCODE_PK2H].fetch_args = pk2h_fetch_args;
838 bld_base->op_actions[TGSI_OPCODE_PK2H].emit = emit_pk2h;
839 bld_base->op_actions[TGSI_OPCODE_POPC].emit = build_tgsi_intrinsic_nomem;
840 bld_base->op_actions[TGSI_OPCODE_POPC].intr_name = "llvm.ctpop.i32";
841 bld_base->op_actions[TGSI_OPCODE_POW].emit = build_tgsi_intrinsic_nomem;
842 bld_base->op_actions[TGSI_OPCODE_POW].intr_name = "llvm.pow.f32";
843 bld_base->op_actions[TGSI_OPCODE_ROUND].emit = build_tgsi_intrinsic_nomem;
844 bld_base->op_actions[TGSI_OPCODE_ROUND].intr_name = "llvm.rint.f32";
845 bld_base->op_actions[TGSI_OPCODE_RSQ].emit = emit_rsq;
846 bld_base->op_actions[TGSI_OPCODE_SGE].emit = emit_set_cond;
847 bld_base->op_actions[TGSI_OPCODE_SEQ].emit = emit_set_cond;
848 bld_base->op_actions[TGSI_OPCODE_SHL].emit = emit_shl;
849 bld_base->op_actions[TGSI_OPCODE_SLE].emit = emit_set_cond;
850 bld_base->op_actions[TGSI_OPCODE_SLT].emit = emit_set_cond;
851 bld_base->op_actions[TGSI_OPCODE_SNE].emit = emit_set_cond;
852 bld_base->op_actions[TGSI_OPCODE_SGT].emit = emit_set_cond;
853 bld_base->op_actions[TGSI_OPCODE_SIN].emit = build_tgsi_intrinsic_nomem;
854 bld_base->op_actions[TGSI_OPCODE_SIN].intr_name = "llvm.sin.f32";
855 bld_base->op_actions[TGSI_OPCODE_SQRT].emit = build_tgsi_intrinsic_nomem;
856 bld_base->op_actions[TGSI_OPCODE_SQRT].intr_name = "llvm.sqrt.f32";
857 bld_base->op_actions[TGSI_OPCODE_SSG].emit = emit_ssg;
858 bld_base->op_actions[TGSI_OPCODE_TRUNC].emit = build_tgsi_intrinsic_nomem;
859 bld_base->op_actions[TGSI_OPCODE_TRUNC].intr_name = "llvm.trunc.f32";
860 bld_base->op_actions[TGSI_OPCODE_UADD].emit = emit_uadd;
861 bld_base->op_actions[TGSI_OPCODE_UBFE].emit = emit_bfe;
862 bld_base->op_actions[TGSI_OPCODE_UDIV].emit = emit_udiv;
863 bld_base->op_actions[TGSI_OPCODE_UMAX].emit = emit_minmax_int;
864 bld_base->op_actions[TGSI_OPCODE_UMIN].emit = emit_minmax_int;
865 bld_base->op_actions[TGSI_OPCODE_UMOD].emit = emit_umod;
866 bld_base->op_actions[TGSI_OPCODE_USEQ].emit = emit_icmp;
867 bld_base->op_actions[TGSI_OPCODE_USGE].emit = emit_icmp;
868 bld_base->op_actions[TGSI_OPCODE_USHR].emit = emit_ushr;
869 bld_base->op_actions[TGSI_OPCODE_USLT].emit = emit_icmp;
870 bld_base->op_actions[TGSI_OPCODE_USNE].emit = emit_icmp;
871 bld_base->op_actions[TGSI_OPCODE_U2F].emit = emit_u2f;
872 bld_base->op_actions[TGSI_OPCODE_XOR].emit = emit_xor;
873 bld_base->op_actions[TGSI_OPCODE_UCMP].emit = emit_ucmp;
874 bld_base->op_actions[TGSI_OPCODE_UP2H].fetch_args = up2h_fetch_args;
875 bld_base->op_actions[TGSI_OPCODE_UP2H].emit = emit_up2h;
876
877 bld_base->op_actions[TGSI_OPCODE_I64MAX].emit = emit_minmax_int;
878 bld_base->op_actions[TGSI_OPCODE_I64MIN].emit = emit_minmax_int;
879 bld_base->op_actions[TGSI_OPCODE_U64MAX].emit = emit_minmax_int;
880 bld_base->op_actions[TGSI_OPCODE_U64MIN].emit = emit_minmax_int;
881 bld_base->op_actions[TGSI_OPCODE_I64ABS].emit = emit_iabs;
882 bld_base->op_actions[TGSI_OPCODE_I64SSG].emit = emit_ssg;
883 bld_base->op_actions[TGSI_OPCODE_I64NEG].emit = emit_ineg;
884
885 bld_base->op_actions[TGSI_OPCODE_U64SEQ].emit = emit_icmp;
886 bld_base->op_actions[TGSI_OPCODE_U64SNE].emit = emit_icmp;
887 bld_base->op_actions[TGSI_OPCODE_U64SGE].emit = emit_icmp;
888 bld_base->op_actions[TGSI_OPCODE_U64SLT].emit = emit_icmp;
889 bld_base->op_actions[TGSI_OPCODE_I64SGE].emit = emit_icmp;
890 bld_base->op_actions[TGSI_OPCODE_I64SLT].emit = emit_icmp;
891
892 bld_base->op_actions[TGSI_OPCODE_U64ADD].emit = emit_uadd;
893 bld_base->op_actions[TGSI_OPCODE_U64SHL].emit = emit_shl;
894 bld_base->op_actions[TGSI_OPCODE_U64SHR].emit = emit_ushr;
895 bld_base->op_actions[TGSI_OPCODE_I64SHR].emit = emit_ishr;
896
897 bld_base->op_actions[TGSI_OPCODE_U64MOD].emit = emit_umod;
898 bld_base->op_actions[TGSI_OPCODE_I64MOD].emit = emit_mod;
899 bld_base->op_actions[TGSI_OPCODE_U64DIV].emit = emit_udiv;
900 bld_base->op_actions[TGSI_OPCODE_I64DIV].emit = emit_idiv;
901 }