projects / mesa.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
history | raw | HEAD
freedreno/ir3: array offset can be negative
[mesa.git] / src / gallium / drivers / freedreno / ir3 / ir3_compiler_nir.c
1 /* -*- mode: C; c-file-style: "k&r"; tab-width 4; indent-tabs-mode: t; -*- */
2
3 /*
4 * Copyright (C) 2015 Rob Clark <robclark@freedesktop.org>
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the "Software"),
8 * to deal in the Software without restriction, including without limitation
9 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
10 * and/or sell copies of the Software, and to permit persons to whom the
11 * Software is furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice (including the next
14 * paragraph) shall be included in all copies or substantial portions of the
15 * Software.
16 *
17 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
18 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
19 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
20 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
21 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
22 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
23 * SOFTWARE.
24 *
25 * Authors:
26 * Rob Clark <robclark@freedesktop.org>
27 */
28
29 #include <stdarg.h>
30
31 #include "pipe/p_state.h"
32 #include "util/u_string.h"
33 #include "util/u_memory.h"
34 #include "util/u_inlines.h"
35
36 #include "freedreno_util.h"
37
38 #include "ir3_compiler.h"
39 #include "ir3_shader.h"
40 #include "ir3_nir.h"
41
42 #include "instr-a3xx.h"
43 #include "ir3.h"
44
45
46 struct ir3_compile {
47 struct ir3_compiler *compiler;
48
49 struct nir_shader *s;
50
51 struct ir3 *ir;
52 struct ir3_shader_variant *so;
53
54 struct ir3_block *block; /* the current block */
55 struct ir3_block *in_block; /* block created for shader inputs */
56
57 nir_function_impl *impl;
58
59 /* For fragment shaders, from the hw perspective the only
60 * actual input is r0.xy position register passed to bary.f.
61 * But TGSI doesn't know that, it still declares things as
62 * IN[] registers. So we do all the input tracking normally
63 * and fix things up after compile_instructions()
64 *
65 * NOTE that frag_pos is the hardware position (possibly it
66 * is actually an index or tag or some such.. it is *not*
67 * values that can be directly used for gl_FragCoord..)
68 */
69 struct ir3_instruction *frag_pos, *frag_face, *frag_coord[4];
70
71 /* For vertex shaders, keep track of the system values sources */
72 struct ir3_instruction *vertex_id, *basevertex, *instance_id;
73
74 /* mapping from nir_register to defining instruction: */
75 struct hash_table *def_ht;
76
77 unsigned num_arrays;
78
79 /* a common pattern for indirect addressing is to request the
80 * same address register multiple times. To avoid generating
81 * duplicate instruction sequences (which our backend does not
82 * try to clean up, since that should be done as the NIR stage)
83 * we cache the address value generated for a given src value:
84 */
85 struct hash_table *addr_ht;
86
87 /* maps nir_block to ir3_block, mostly for the purposes of
88 * figuring out the blocks successors
89 */
90 struct hash_table *block_ht;
91
92 /* a4xx (at least patchlevel 0) cannot seem to flat-interpolate
93 * so we need to use ldlv.u32 to load the varying directly:
94 */
95 bool flat_bypass;
96
97 /* on a3xx, we need to add one to # of array levels:
98 */
99 bool levels_add_one;
100
101 /* on a3xx, we need to scale up integer coords for isaml based
102 * on LoD:
103 */
104 bool unminify_coords;
105
106 /* for looking up which system value is which */
107 unsigned sysval_semantics[8];
108
109 /* set if we encounter something we can't handle yet, so we
110 * can bail cleanly and fallback to TGSI compiler f/e
111 */
112 bool error;
113 };
114
115
116 static struct ir3_instruction * create_immed(struct ir3_block *block, uint32_t val);
117 static struct ir3_block * get_block(struct ir3_compile *ctx, nir_block *nblock);
118
119
120 static struct ir3_compile *
121 compile_init(struct ir3_compiler *compiler,
122 struct ir3_shader_variant *so)
123 {
124 struct ir3_compile *ctx = rzalloc(NULL, struct ir3_compile);
125
126 if (compiler->gpu_id >= 400) {
127 /* need special handling for "flat" */
128 ctx->flat_bypass = true;
129 ctx->levels_add_one = false;
130 ctx->unminify_coords = false;
131 } else {
132 /* no special handling for "flat" */
133 ctx->flat_bypass = false;
134 ctx->levels_add_one = true;
135 ctx->unminify_coords = true;
136 }
137
138 ctx->compiler = compiler;
139 ctx->ir = so->ir;
140 ctx->so = so;
141 ctx->def_ht = _mesa_hash_table_create(ctx,
142 _mesa_hash_pointer, _mesa_key_pointer_equal);
143 ctx->block_ht = _mesa_hash_table_create(ctx,
144 _mesa_hash_pointer, _mesa_key_pointer_equal);
145
146 /* TODO: maybe generate some sort of bitmask of what key
147 * lowers vs what shader has (ie. no need to lower
148 * texture clamp lowering if no texture sample instrs)..
149 * although should be done further up the stack to avoid
150 * creating duplicate variants..
151 */
152
153 if (ir3_key_lowers_nir(&so->key)) {
154 nir_shader *s = nir_shader_clone(ctx, so->shader->nir);
155 ctx->s = ir3_optimize_nir(so->shader, s, &so->key);
156 } else {
157 /* fast-path for shader key that lowers nothing in NIR: */
158 ctx->s = so->shader->nir;
159 }
160
161 if (fd_mesa_debug & FD_DBG_DISASM) {
162 DBG("dump nir%dv%d: type=%d, k={bp=%u,cts=%u,hp=%u}",
163 so->shader->id, so->id, so->type,
164 so->key.binning_pass, so->key.color_two_side,
165 so->key.half_precision);
166 nir_print_shader(ctx->s, stdout);
167 }
168
169 so->first_driver_param = so->first_immediate = ctx->s->num_uniforms;
170
171 /* Layout of constant registers:
172 *
173 * num_uniform * vec4 - user consts
174 * 4 * vec4 - UBO addresses
175 * if (vertex shader) {
176 * N * vec4 - driver params (IR3_DP_*)
177 * 1 * vec4 - stream-out addresses
178 * }
179 *
180 * TODO this could be made more dynamic, to at least skip sections
181 * that we don't need..
182 */
183
184 /* reserve 4 (vec4) slots for ubo base addresses: */
185 so->first_immediate += 4;
186
187 if (so->type == SHADER_VERTEX) {
188 /* driver params (see ir3_driver_param): */
189 so->first_immediate += IR3_DP_COUNT/4; /* convert to vec4 */
190 /* one (vec4) slot for stream-output base addresses: */
191 so->first_immediate++;
192 }
193
194 return ctx;
195 }
196
197 static void
198 compile_error(struct ir3_compile *ctx, const char *format, ...)
199 {
200 va_list ap;
201 va_start(ap, format);
202 _debug_vprintf(format, ap);
203 va_end(ap);
204 nir_print_shader(ctx->s, stdout);
205 ctx->error = true;
206 debug_assert(0);
207 }
208
209 #define compile_assert(ctx, cond) do { \
210 if (!(cond)) compile_error((ctx), "failed assert: "#cond"\n"); \
211 } while (0)
212
213 static void
214 compile_free(struct ir3_compile *ctx)
215 {
216 ralloc_free(ctx);
217 }
218
219 static void
220 declare_var(struct ir3_compile *ctx, nir_variable *var)
221 {
222 unsigned length = glsl_get_length(var->type) * 4; /* always vec4, at least with ttn */
223 struct ir3_array *arr = ralloc(ctx, struct ir3_array);
224 arr->id = ++ctx->num_arrays;
225 arr->length = length;
226 arr->var = var;
227 list_addtail(&arr->node, &ctx->ir->array_list);
228 }
229
230 static struct ir3_array *
231 get_var(struct ir3_compile *ctx, nir_variable *var)
232 {
233 list_for_each_entry (struct ir3_array, arr, &ctx->ir->array_list, node) {
234 if (arr->var == var)
235 return arr;
236 }
237 compile_error(ctx, "bogus var: %s\n", var->name);
238 return NULL;
239 }
240
241 /* allocate a n element value array (to be populated by caller) and
242 * insert in def_ht
243 */
244 static struct ir3_instruction **
245 __get_dst(struct ir3_compile *ctx, void *key, unsigned n)
246 {
247 struct ir3_instruction **value =
248 ralloc_array(ctx->def_ht, struct ir3_instruction *, n);
249 _mesa_hash_table_insert(ctx->def_ht, key, value);
250 return value;
251 }
252
253 static struct ir3_instruction **
254 get_dst(struct ir3_compile *ctx, nir_dest *dst, unsigned n)
255 {
256 compile_assert(ctx, dst->is_ssa);
257 if (dst->is_ssa) {
258 return __get_dst(ctx, &dst->ssa, n);
259 } else {
260 return __get_dst(ctx, dst->reg.reg, n);
261 }
262 }
263
264 static struct ir3_instruction **
265 get_dst_ssa(struct ir3_compile *ctx, nir_ssa_def *dst, unsigned n)
266 {
267 return __get_dst(ctx, dst, n);
268 }
269
270 static struct ir3_instruction **
271 get_src(struct ir3_compile *ctx, nir_src *src)
272 {
273 struct hash_entry *entry;
274 compile_assert(ctx, src->is_ssa);
275 if (src->is_ssa) {
276 entry = _mesa_hash_table_search(ctx->def_ht, src->ssa);
277 } else {
278 entry = _mesa_hash_table_search(ctx->def_ht, src->reg.reg);
279 }
280 compile_assert(ctx, entry);
281 return entry->data;
282 }
283
284 static struct ir3_instruction *
285 create_immed(struct ir3_block *block, uint32_t val)
286 {
287 struct ir3_instruction *mov;
288
289 mov = ir3_instr_create(block, 1, 0);
290 mov->cat1.src_type = TYPE_U32;
291 mov->cat1.dst_type = TYPE_U32;
292 ir3_reg_create(mov, 0, 0);
293 ir3_reg_create(mov, 0, IR3_REG_IMMED)->uim_val = val;
294
295 return mov;
296 }
297
298 static struct ir3_instruction *
299 create_addr(struct ir3_block *block, struct ir3_instruction *src)
300 {
301 struct ir3_instruction *instr, *immed;
302
303 /* TODO in at least some cases, the backend could probably be
304 * made clever enough to propagate IR3_REG_HALF..
305 */
306 instr = ir3_COV(block, src, TYPE_U32, TYPE_S16);
307 instr->regs[0]->flags |= IR3_REG_HALF;
308
309 immed = create_immed(block, 2);
310 immed->regs[0]->flags |= IR3_REG_HALF;
311
312 instr = ir3_SHL_B(block, instr, 0, immed, 0);
313 instr->regs[0]->flags |= IR3_REG_HALF;
314 instr->regs[1]->flags |= IR3_REG_HALF;
315
316 instr = ir3_MOV(block, instr, TYPE_S16);
317 instr->regs[0]->num = regid(REG_A0, 0);
318 instr->regs[0]->flags |= IR3_REG_HALF;
319 instr->regs[1]->flags |= IR3_REG_HALF;
320
321 return instr;
322 }
323
324 /* caches addr values to avoid generating multiple cov/shl/mova
325 * sequences for each use of a given NIR level src as address
326 */
327 static struct ir3_instruction *
328 get_addr(struct ir3_compile *ctx, struct ir3_instruction *src)
329 {
330 struct ir3_instruction *addr;
331
332 if (!ctx->addr_ht) {
333 ctx->addr_ht = _mesa_hash_table_create(ctx,
334 _mesa_hash_pointer, _mesa_key_pointer_equal);
335 } else {
336 struct hash_entry *entry;
337 entry = _mesa_hash_table_search(ctx->addr_ht, src);
338 if (entry)
339 return entry->data;
340 }
341
342 addr = create_addr(ctx->block, src);
343 _mesa_hash_table_insert(ctx->addr_ht, src, addr);
344
345 return addr;
346 }
347
348 static struct ir3_instruction *
349 get_predicate(struct ir3_compile *ctx, struct ir3_instruction *src)
350 {
351 struct ir3_block *b = ctx->block;
352 struct ir3_instruction *cond;
353
354 /* NOTE: only cmps.*.* can write p0.x: */
355 cond = ir3_CMPS_S(b, src, 0, create_immed(b, 0), 0);
356 cond->cat2.condition = IR3_COND_NE;
357
358 /* condition always goes in predicate register: */
359 cond->regs[0]->num = regid(REG_P0, 0);
360
361 return cond;
362 }
363
364 static struct ir3_instruction *
365 create_uniform(struct ir3_compile *ctx, unsigned n)
366 {
367 struct ir3_instruction *mov;
368
369 mov = ir3_instr_create(ctx->block, 1, 0);
370 /* TODO get types right? */
371 mov->cat1.src_type = TYPE_F32;
372 mov->cat1.dst_type = TYPE_F32;
373 ir3_reg_create(mov, 0, 0);
374 ir3_reg_create(mov, n, IR3_REG_CONST);
375
376 return mov;
377 }
378
379 static struct ir3_instruction *
380 create_uniform_indirect(struct ir3_compile *ctx, int n,
381 struct ir3_instruction *address)
382 {
383 struct ir3_instruction *mov;
384
385 mov = ir3_instr_create(ctx->block, 1, 0);
386 mov->cat1.src_type = TYPE_U32;
387 mov->cat1.dst_type = TYPE_U32;
388 ir3_reg_create(mov, 0, 0);
389 ir3_reg_create(mov, 0, IR3_REG_CONST | IR3_REG_RELATIV)->array.offset = n;
390
391 ir3_instr_set_address(mov, address);
392
393 return mov;
394 }
395
396 static struct ir3_instruction *
397 create_collect(struct ir3_block *block, struct ir3_instruction **arr,
398 unsigned arrsz)
399 {
400 struct ir3_instruction *collect;
401
402 if (arrsz == 0)
403 return NULL;
404
405 collect = ir3_instr_create2(block, -1, OPC_META_FI, 1 + arrsz);
406 ir3_reg_create(collect, 0, 0); /* dst */
407 for (unsigned i = 0; i < arrsz; i++)
408 ir3_reg_create(collect, 0, IR3_REG_SSA)->instr = arr[i];
409
410 return collect;
411 }
412
413 static struct ir3_instruction *
414 create_indirect_load(struct ir3_compile *ctx, unsigned arrsz, int n,
415 struct ir3_instruction *address, struct ir3_instruction *collect)
416 {
417 struct ir3_block *block = ctx->block;
418 struct ir3_instruction *mov;
419 struct ir3_register *src;
420
421 mov = ir3_instr_create(block, 1, 0);
422 mov->cat1.src_type = TYPE_U32;
423 mov->cat1.dst_type = TYPE_U32;
424 ir3_reg_create(mov, 0, 0);
425 src = ir3_reg_create(mov, 0, IR3_REG_SSA | IR3_REG_RELATIV);
426 src->instr = collect;
427 src->size = arrsz;
428 src->array.offset = n;
429
430 ir3_instr_set_address(mov, address);
431
432 return mov;
433 }
434
435 /* relative (indirect) if address!=NULL */
436 static struct ir3_instruction *
437 create_var_load(struct ir3_compile *ctx, struct ir3_array *arr, int n,
438 struct ir3_instruction *address)
439 {
440 struct ir3_block *block = ctx->block;
441 struct ir3_instruction *mov;
442 struct ir3_register *src;
443
444 mov = ir3_instr_create(block, 1, 0);
445 mov->cat1.src_type = TYPE_U32;
446 mov->cat1.dst_type = TYPE_U32;
447 ir3_reg_create(mov, 0, 0);
448 src = ir3_reg_create(mov, 0, IR3_REG_ARRAY |
449 COND(address, IR3_REG_RELATIV));
450 src->instr = arr->last_write;
451 src->size = arr->length;
452 src->array.id = arr->id;
453 src->array.offset = n;
454
455 if (address)
456 ir3_instr_set_address(mov, address);
457
458 arr->last_access = mov;
459
460 return mov;
461 }
462
463 /* relative (indirect) if address!=NULL */
464 static struct ir3_instruction *
465 create_var_store(struct ir3_compile *ctx, struct ir3_array *arr, int n,
466 struct ir3_instruction *src, struct ir3_instruction *address)
467 {
468 struct ir3_block *block = ctx->block;
469 struct ir3_instruction *mov;
470 struct ir3_register *dst;
471
472 mov = ir3_instr_create(block, 1, 0);
473 mov->cat1.src_type = TYPE_U32;
474 mov->cat1.dst_type = TYPE_U32;
475 dst = ir3_reg_create(mov, 0, IR3_REG_ARRAY |
476 COND(address, IR3_REG_RELATIV));
477 dst->instr = arr->last_access;
478 dst->size = arr->length;
479 dst->array.id = arr->id;
480 dst->array.offset = n;
481 ir3_reg_create(mov, 0, IR3_REG_SSA)->instr = src;
482
483 ir3_instr_set_address(mov, address);
484
485 arr->last_write = arr->last_access = mov;
486
487 return mov;
488 }
489
490 static struct ir3_instruction *
491 create_input(struct ir3_block *block, unsigned n)
492 {
493 struct ir3_instruction *in;
494
495 in = ir3_instr_create(block, -1, OPC_META_INPUT);
496 in->inout.block = block;
497 ir3_reg_create(in, n, 0);
498
499 return in;
500 }
501
502 static struct ir3_instruction *
503 create_frag_input(struct ir3_compile *ctx, bool use_ldlv)
504 {
505 struct ir3_block *block = ctx->block;
506 struct ir3_instruction *instr;
507 /* actual inloc is assigned and fixed up later: */
508 struct ir3_instruction *inloc = create_immed(block, 0);
509
510 if (use_ldlv) {
511 instr = ir3_LDLV(block, inloc, 0, create_immed(block, 1), 0);
512 instr->cat6.type = TYPE_U32;
513 instr->cat6.iim_val = 1;
514 } else {
515 instr = ir3_BARY_F(block, inloc, 0, ctx->frag_pos, 0);
516 instr->regs[2]->wrmask = 0x3;
517 }
518
519 return instr;
520 }
521
522 static struct ir3_instruction *
523 create_frag_coord(struct ir3_compile *ctx, unsigned comp)
524 {
525 struct ir3_block *block = ctx->block;
526 struct ir3_instruction *instr;
527
528 compile_assert(ctx, !ctx->frag_coord[comp]);
529
530 ctx->frag_coord[comp] = create_input(ctx->block, 0);
531
532 switch (comp) {
533 case 0: /* .x */
534 case 1: /* .y */
535 /* for frag_coord, we get unsigned values.. we need
536 * to subtract (integer) 8 and divide by 16 (right-
537 * shift by 4) then convert to float:
538 *
539 * sub.s tmp, src, 8
540 * shr.b tmp, tmp, 4
541 * mov.u32f32 dst, tmp
542 *
543 */
544 instr = ir3_SUB_S(block, ctx->frag_coord[comp], 0,
545 create_immed(block, 8), 0);
546 instr = ir3_SHR_B(block, instr, 0,
547 create_immed(block, 4), 0);
548 instr = ir3_COV(block, instr, TYPE_U32, TYPE_F32);
549
550 return instr;
551 case 2: /* .z */
552 case 3: /* .w */
553 default:
554 /* seems that we can use these as-is: */
555 return ctx->frag_coord[comp];
556 }
557 }
558
559 static struct ir3_instruction *
560 create_frag_face(struct ir3_compile *ctx, unsigned comp)
561 {
562 struct ir3_block *block = ctx->block;
563 struct ir3_instruction *instr;
564
565 switch (comp) {
566 case 0: /* .x */
567 compile_assert(ctx, !ctx->frag_face);
568
569 ctx->frag_face = create_input(block, 0);
570 ctx->frag_face->regs[0]->flags |= IR3_REG_HALF;
571
572 /* for faceness, we always get -1 or 0 (int).. but TGSI expects
573 * positive vs negative float.. and piglit further seems to
574 * expect -1.0 or 1.0:
575 *
576 * mul.s tmp, hr0.x, 2
577 * add.s tmp, tmp, 1
578 * mov.s32f32, dst, tmp
579 *
580 */
581 instr = ir3_MUL_S(block, ctx->frag_face, 0,
582 create_immed(block, 2), 0);
583 instr = ir3_ADD_S(block, instr, 0,
584 create_immed(block, 1), 0);
585 instr = ir3_COV(block, instr, TYPE_S32, TYPE_F32);
586
587 return instr;
588 case 1: /* .y */
589 case 2: /* .z */
590 return create_immed(block, fui(0.0));
591 default:
592 case 3: /* .w */
593 return create_immed(block, fui(1.0));
594 }
595 }
596
597 static struct ir3_instruction *
598 create_driver_param(struct ir3_compile *ctx, enum ir3_driver_param dp)
599 {
600 /* first four vec4 sysval's reserved for UBOs: */
601 /* NOTE: dp is in scalar, but there can be >4 dp components: */
602 unsigned n = ctx->so->first_driver_param + IR3_DRIVER_PARAM_OFF;
603 unsigned r = regid(n + dp / 4, dp % 4);
604 return create_uniform(ctx, r);
605 }
606
607 /* helper for instructions that produce multiple consecutive scalar
608 * outputs which need to have a split/fanout meta instruction inserted
609 */
610 static void
611 split_dest(struct ir3_block *block, struct ir3_instruction **dst,
612 struct ir3_instruction *src, unsigned n)
613 {
614 struct ir3_instruction *prev = NULL;
615 for (int i = 0, j = 0; i < n; i++) {
616 struct ir3_instruction *split =
617 ir3_instr_create(block, -1, OPC_META_FO);
618 ir3_reg_create(split, 0, IR3_REG_SSA);
619 ir3_reg_create(split, 0, IR3_REG_SSA)->instr = src;
620 split->fo.off = i;
621
622 if (prev) {
623 split->cp.left = prev;
624 split->cp.left_cnt++;
625 prev->cp.right = split;
626 prev->cp.right_cnt++;
627 }
628 prev = split;
629
630 if (src->regs[0]->wrmask & (1 << i))
631 dst[j++] = split;
632 }
633 }
634
635 /*
636 * Adreno uses uint rather than having dedicated bool type,
637 * which (potentially) requires some conversion, in particular
638 * when using output of an bool instr to int input, or visa
639 * versa.
640 *
641 * | Adreno | NIR |
642 * -------+---------+-------+-
643 * true | 1 | ~0 |
644 * false | 0 | 0 |
645 *
646 * To convert from an adreno bool (uint) to nir, use:
647 *
648 * absneg.s dst, (neg)src
649 *
650 * To convert back in the other direction:
651 *
652 * absneg.s dst, (abs)arc
653 *
654 * The CP step can clean up the absneg.s that cancel each other
655 * out, and with a slight bit of extra cleverness (to recognize
656 * the instructions which produce either a 0 or 1) can eliminate
657 * the absneg.s's completely when an instruction that wants
658 * 0/1 consumes the result. For example, when a nir 'bcsel'
659 * consumes the result of 'feq'. So we should be able to get by
660 * without a boolean resolve step, and without incuring any
661 * extra penalty in instruction count.
662 */
663
664 /* NIR bool -> native (adreno): */
665 static struct ir3_instruction *
666 ir3_b2n(struct ir3_block *block, struct ir3_instruction *instr)
667 {
668 return ir3_ABSNEG_S(block, instr, IR3_REG_SABS);
669 }
670
671 /* native (adreno) -> NIR bool: */
672 static struct ir3_instruction *
673 ir3_n2b(struct ir3_block *block, struct ir3_instruction *instr)
674 {
675 return ir3_ABSNEG_S(block, instr, IR3_REG_SNEG);
676 }
677
678 /*
679 * alu/sfu instructions:
680 */
681
682 static void
683 emit_alu(struct ir3_compile *ctx, nir_alu_instr *alu)
684 {
685 const nir_op_info *info = &nir_op_infos[alu->op];
686 struct ir3_instruction **dst, *src[info->num_inputs];
687 struct ir3_block *b = ctx->block;
688
689 dst = get_dst(ctx, &alu->dest.dest, MAX2(info->output_size, 1));
690
691 /* Vectors are special in that they have non-scalarized writemasks,
692 * and just take the first swizzle channel for each argument in
693 * order into each writemask channel.
694 */
695 if ((alu->op == nir_op_vec2) ||
696 (alu->op == nir_op_vec3) ||
697 (alu->op == nir_op_vec4)) {
698
699 for (int i = 0; i < info->num_inputs; i++) {
700 nir_alu_src *asrc = &alu->src[i];
701
702 compile_assert(ctx, !asrc->abs);
703 compile_assert(ctx, !asrc->negate);
704
705 src[i] = get_src(ctx, &asrc->src)[asrc->swizzle[0]];
706 if (!src[i])
707 src[i] = create_immed(ctx->block, 0);
708 dst[i] = ir3_MOV(b, src[i], TYPE_U32);
709 }
710
711 return;
712 }
713
714 /* General case: We can just grab the one used channel per src. */
715 for (int i = 0; i < info->num_inputs; i++) {
716 unsigned chan = ffs(alu->dest.write_mask) - 1;
717 nir_alu_src *asrc = &alu->src[i];
718
719 compile_assert(ctx, !asrc->abs);
720 compile_assert(ctx, !asrc->negate);
721
722 src[i] = get_src(ctx, &asrc->src)[asrc->swizzle[chan]];
723
724 compile_assert(ctx, src[i]);
725 }
726
727 switch (alu->op) {
728 case nir_op_f2i:
729 dst[0] = ir3_COV(b, src[0], TYPE_F32, TYPE_S32);
730 break;
731 case nir_op_f2u:
732 dst[0] = ir3_COV(b, src[0], TYPE_F32, TYPE_U32);
733 break;
734 case nir_op_i2f:
735 dst[0] = ir3_COV(b, src[0], TYPE_S32, TYPE_F32);
736 break;
737 case nir_op_u2f:
738 dst[0] = ir3_COV(b, src[0], TYPE_U32, TYPE_F32);
739 break;
740 case nir_op_imov:
741 dst[0] = ir3_MOV(b, src[0], TYPE_S32);
742 break;
743 case nir_op_fmov:
744 dst[0] = ir3_MOV(b, src[0], TYPE_F32);
745 break;
746 case nir_op_f2b:
747 dst[0] = ir3_CMPS_F(b, src[0], 0, create_immed(b, fui(0.0)), 0);
748 dst[0]->cat2.condition = IR3_COND_NE;
749 dst[0] = ir3_n2b(b, dst[0]);
750 break;
751 case nir_op_b2f:
752 dst[0] = ir3_COV(b, ir3_b2n(b, src[0]), TYPE_U32, TYPE_F32);
753 break;
754 case nir_op_b2i:
755 dst[0] = ir3_b2n(b, src[0]);
756 break;
757 case nir_op_i2b:
758 dst[0] = ir3_CMPS_S(b, src[0], 0, create_immed(b, 0), 0);
759 dst[0]->cat2.condition = IR3_COND_NE;
760 dst[0] = ir3_n2b(b, dst[0]);
761 break;
762
763 case nir_op_fneg:
764 dst[0] = ir3_ABSNEG_F(b, src[0], IR3_REG_FNEG);
765 break;
766 case nir_op_fabs:
767 dst[0] = ir3_ABSNEG_F(b, src[0], IR3_REG_FABS);
768 break;
769 case nir_op_fmax:
770 dst[0] = ir3_MAX_F(b, src[0], 0, src[1], 0);
771 break;
772 case nir_op_fmin:
773 dst[0] = ir3_MIN_F(b, src[0], 0, src[1], 0);
774 break;
775 case nir_op_fmul:
776 dst[0] = ir3_MUL_F(b, src[0], 0, src[1], 0);
777 break;
778 case nir_op_fadd:
779 dst[0] = ir3_ADD_F(b, src[0], 0, src[1], 0);
780 break;
781 case nir_op_fsub:
782 dst[0] = ir3_ADD_F(b, src[0], 0, src[1], IR3_REG_FNEG);
783 break;
784 case nir_op_ffma:
785 dst[0] = ir3_MAD_F32(b, src[0], 0, src[1], 0, src[2], 0);
786 break;
787 case nir_op_fddx:
788 dst[0] = ir3_DSX(b, src[0], 0);
789 dst[0]->cat5.type = TYPE_F32;
790 break;
791 case nir_op_fddy:
792 dst[0] = ir3_DSY(b, src[0], 0);
793 dst[0]->cat5.type = TYPE_F32;
794 break;
795 break;
796 case nir_op_flt:
797 dst[0] = ir3_CMPS_F(b, src[0], 0, src[1], 0);
798 dst[0]->cat2.condition = IR3_COND_LT;
799 dst[0] = ir3_n2b(b, dst[0]);
800 break;
801 case nir_op_fge:
802 dst[0] = ir3_CMPS_F(b, src[0], 0, src[1], 0);
803 dst[0]->cat2.condition = IR3_COND_GE;
804 dst[0] = ir3_n2b(b, dst[0]);
805 break;
806 case nir_op_feq:
807 dst[0] = ir3_CMPS_F(b, src[0], 0, src[1], 0);
808 dst[0]->cat2.condition = IR3_COND_EQ;
809 dst[0] = ir3_n2b(b, dst[0]);
810 break;
811 case nir_op_fne:
812 dst[0] = ir3_CMPS_F(b, src[0], 0, src[1], 0);
813 dst[0]->cat2.condition = IR3_COND_NE;
814 dst[0] = ir3_n2b(b, dst[0]);
815 break;
816 case nir_op_fceil:
817 dst[0] = ir3_CEIL_F(b, src[0], 0);
818 break;
819 case nir_op_ffloor:
820 dst[0] = ir3_FLOOR_F(b, src[0], 0);
821 break;
822 case nir_op_ftrunc:
823 dst[0] = ir3_TRUNC_F(b, src[0], 0);
824 break;
825 case nir_op_fround_even:
826 dst[0] = ir3_RNDNE_F(b, src[0], 0);
827 break;
828 case nir_op_fsign:
829 dst[0] = ir3_SIGN_F(b, src[0], 0);
830 break;
831
832 case nir_op_fsin:
833 dst[0] = ir3_SIN(b, src[0], 0);
834 break;
835 case nir_op_fcos:
836 dst[0] = ir3_COS(b, src[0], 0);
837 break;
838 case nir_op_frsq:
839 dst[0] = ir3_RSQ(b, src[0], 0);
840 break;
841 case nir_op_frcp:
842 dst[0] = ir3_RCP(b, src[0], 0);
843 break;
844 case nir_op_flog2:
845 dst[0] = ir3_LOG2(b, src[0], 0);
846 break;
847 case nir_op_fexp2:
848 dst[0] = ir3_EXP2(b, src[0], 0);
849 break;
850 case nir_op_fsqrt:
851 dst[0] = ir3_SQRT(b, src[0], 0);
852 break;
853
854 case nir_op_iabs:
855 dst[0] = ir3_ABSNEG_S(b, src[0], IR3_REG_SABS);
856 break;
857 case nir_op_iadd:
858 dst[0] = ir3_ADD_U(b, src[0], 0, src[1], 0);
859 break;
860 case nir_op_iand:
861 dst[0] = ir3_AND_B(b, src[0], 0, src[1], 0);
862 break;
863 case nir_op_imax:
864 dst[0] = ir3_MAX_S(b, src[0], 0, src[1], 0);
865 break;
866 case nir_op_umax:
867 dst[0] = ir3_MAX_U(b, src[0], 0, src[1], 0);
868 break;
869 case nir_op_imin:
870 dst[0] = ir3_MIN_S(b, src[0], 0, src[1], 0);
871 break;
872 case nir_op_umin:
873 dst[0] = ir3_MIN_U(b, src[0], 0, src[1], 0);
874 break;
875 case nir_op_imul:
876 /*
877 * dst = (al * bl) + (ah * bl << 16) + (al * bh << 16)
878 * mull.u tmp0, a, b ; mul low, i.e. al * bl
879 * madsh.m16 tmp1, a, b, tmp0 ; mul-add shift high mix, i.e. ah * bl << 16
880 * madsh.m16 dst, b, a, tmp1 ; i.e. al * bh << 16
881 */
882 dst[0] = ir3_MADSH_M16(b, src[1], 0, src[0], 0,
883 ir3_MADSH_M16(b, src[0], 0, src[1], 0,
884 ir3_MULL_U(b, src[0], 0, src[1], 0), 0), 0);
885 break;
886 case nir_op_ineg:
887 dst[0] = ir3_ABSNEG_S(b, src[0], IR3_REG_SNEG);
888 break;
889 case nir_op_inot:
890 dst[0] = ir3_NOT_B(b, src[0], 0);
891 break;
892 case nir_op_ior:
893 dst[0] = ir3_OR_B(b, src[0], 0, src[1], 0);
894 break;
895 case nir_op_ishl:
896 dst[0] = ir3_SHL_B(b, src[0], 0, src[1], 0);
897 break;
898 case nir_op_ishr:
899 dst[0] = ir3_ASHR_B(b, src[0], 0, src[1], 0);
900 break;
901 case nir_op_isign: {
902 /* maybe this would be sane to lower in nir.. */
903 struct ir3_instruction *neg, *pos;
904
905 neg = ir3_CMPS_S(b, src[0], 0, create_immed(b, 0), 0);
906 neg->cat2.condition = IR3_COND_LT;
907
908 pos = ir3_CMPS_S(b, src[0], 0, create_immed(b, 0), 0);
909 pos->cat2.condition = IR3_COND_GT;
910
911 dst[0] = ir3_SUB_U(b, pos, 0, neg, 0);
912
913 break;
914 }
915 case nir_op_isub:
916 dst[0] = ir3_SUB_U(b, src[0], 0, src[1], 0);
917 break;
918 case nir_op_ixor:
919 dst[0] = ir3_XOR_B(b, src[0], 0, src[1], 0);
920 break;
921 case nir_op_ushr:
922 dst[0] = ir3_SHR_B(b, src[0], 0, src[1], 0);
923 break;
924 case nir_op_ilt:
925 dst[0] = ir3_CMPS_S(b, src[0], 0, src[1], 0);
926 dst[0]->cat2.condition = IR3_COND_LT;
927 dst[0] = ir3_n2b(b, dst[0]);
928 break;
929 case nir_op_ige:
930 dst[0] = ir3_CMPS_S(b, src[0], 0, src[1], 0);
931 dst[0]->cat2.condition = IR3_COND_GE;
932 dst[0] = ir3_n2b(b, dst[0]);
933 break;
934 case nir_op_ieq:
935 dst[0] = ir3_CMPS_S(b, src[0], 0, src[1], 0);
936 dst[0]->cat2.condition = IR3_COND_EQ;
937 dst[0] = ir3_n2b(b, dst[0]);
938 break;
939 case nir_op_ine:
940 dst[0] = ir3_CMPS_S(b, src[0], 0, src[1], 0);
941 dst[0]->cat2.condition = IR3_COND_NE;
942 dst[0] = ir3_n2b(b, dst[0]);
943 break;
944 case nir_op_ult:
945 dst[0] = ir3_CMPS_U(b, src[0], 0, src[1], 0);
946 dst[0]->cat2.condition = IR3_COND_LT;
947 dst[0] = ir3_n2b(b, dst[0]);
948 break;
949 case nir_op_uge:
950 dst[0] = ir3_CMPS_U(b, src[0], 0, src[1], 0);
951 dst[0]->cat2.condition = IR3_COND_GE;
952 dst[0] = ir3_n2b(b, dst[0]);
953 break;
954
955 case nir_op_bcsel:
956 dst[0] = ir3_SEL_B32(b, src[1], 0, ir3_b2n(b, src[0]), 0, src[2], 0);
957 break;
958
959 case nir_op_bit_count:
960 dst[0] = ir3_CBITS_B(b, src[0], 0);
961 break;
962 case nir_op_ifind_msb: {
963 struct ir3_instruction *cmp;
964 dst[0] = ir3_CLZ_S(b, src[0], 0);
965 cmp = ir3_CMPS_S(b, dst[0], 0, create_immed(b, 0), 0);
966 cmp->cat2.condition = IR3_COND_GE;
967 dst[0] = ir3_SEL_B32(b,
968 ir3_SUB_U(b, create_immed(b, 31), 0, dst[0], 0), 0,
969 cmp, 0, dst[0], 0);
970 break;
971 }
972 case nir_op_ufind_msb:
973 dst[0] = ir3_CLZ_B(b, src[0], 0);
974 dst[0] = ir3_SEL_B32(b,
975 ir3_SUB_U(b, create_immed(b, 31), 0, dst[0], 0), 0,
976 src[0], 0, dst[0], 0);
977 break;
978 case nir_op_find_lsb:
979 dst[0] = ir3_BFREV_B(b, src[0], 0);
980 dst[0] = ir3_CLZ_B(b, dst[0], 0);
981 break;
982 case nir_op_bitfield_reverse:
983 dst[0] = ir3_BFREV_B(b, src[0], 0);
984 break;
985
986 default:
987 compile_error(ctx, "Unhandled ALU op: %s\n",
988 nir_op_infos[alu->op].name);
989 break;
990 }
991 }
992
993 /* handles direct/indirect UBO reads: */
994 static void
995 emit_intrinsic_load_ubo(struct ir3_compile *ctx, nir_intrinsic_instr *intr,
996 struct ir3_instruction **dst)
997 {
998 struct ir3_block *b = ctx->block;
999 struct ir3_instruction *addr, *src0, *src1;
1000 nir_const_value *const_offset;
1001 /* UBO addresses are the first driver params: */
1002 unsigned ubo = regid(ctx->so->first_driver_param + IR3_UBOS_OFF, 0);
1003 int off = intr->const_index[0];
1004
1005 /* First src is ubo index, which could either be an immed or not: */
1006 src0 = get_src(ctx, &intr->src[0])[0];
1007 if (is_same_type_mov(src0) &&
1008 (src0->regs[1]->flags & IR3_REG_IMMED)) {
1009 addr = create_uniform(ctx, ubo + src0->regs[1]->iim_val);
1010 } else {
1011 addr = create_uniform_indirect(ctx, ubo, get_addr(ctx, src0));
1012 }
1013
1014 const_offset = nir_src_as_const_value(intr->src[1]);
1015 if (const_offset) {
1016 off += const_offset->u[0];
1017 } else {
1018 /* For load_ubo_indirect, second src is indirect offset: */
1019 src1 = get_src(ctx, &intr->src[1])[0];
1020
1021 /* and add offset to addr: */
1022 addr = ir3_ADD_S(b, addr, 0, src1, 0);
1023 }
1024
1025 /* if offset is to large to encode in the ldg, split it out: */
1026 if ((off + (intr->num_components * 4)) > 1024) {
1027 /* split out the minimal amount to improve the odds that
1028 * cp can fit the immediate in the add.s instruction:
1029 */
1030 unsigned off2 = off + (intr->num_components * 4) - 1024;
1031 addr = ir3_ADD_S(b, addr, 0, create_immed(b, off2), 0);
1032 off -= off2;
1033 }
1034
1035 for (int i = 0; i < intr->num_components; i++) {
1036 struct ir3_instruction *load =
1037 ir3_LDG(b, addr, 0, create_immed(b, 1), 0);
1038 load->cat6.type = TYPE_U32;
1039 load->cat6.src_offset = off + i * 4; /* byte offset */
1040 dst[i] = load;
1041 }
1042 }
1043
1044 /* handles array reads: */
1045 static void
1046 emit_intrinsic_load_var(struct ir3_compile *ctx, nir_intrinsic_instr *intr,
1047 struct ir3_instruction **dst)
1048 {
1049 nir_deref_var *dvar = intr->variables[0];
1050 nir_deref_array *darr = nir_deref_as_array(dvar->deref.child);
1051 struct ir3_array *arr = get_var(ctx, dvar->var);
1052
1053 compile_assert(ctx, dvar->deref.child &&
1054 (dvar->deref.child->deref_type == nir_deref_type_array));
1055
1056 switch (darr->deref_array_type) {
1057 case nir_deref_array_type_direct:
1058 /* direct access does not require anything special: */
1059 for (int i = 0; i < intr->num_components; i++) {
1060 unsigned n = darr->base_offset * 4 + i;
1061 compile_assert(ctx, n < arr->length);
1062 dst[i] = create_var_load(ctx, arr, n, NULL);
1063 }
1064 break;
1065 case nir_deref_array_type_indirect: {
1066 /* for indirect, we need to collect all the array elements: */
1067 struct ir3_instruction *addr =
1068 get_addr(ctx, get_src(ctx, &darr->indirect)[0]);
1069 for (int i = 0; i < intr->num_components; i++) {
1070 unsigned n = darr->base_offset * 4 + i;
1071 compile_assert(ctx, n < arr->length);
1072 dst[i] = create_var_load(ctx, arr, n, addr);
1073 }
1074 break;
1075 }
1076 default:
1077 compile_error(ctx, "Unhandled load deref type: %u\n",
1078 darr->deref_array_type);
1079 break;
1080 }
1081 }
1082
1083 /* handles array writes: */
1084 static void
1085 emit_intrinsic_store_var(struct ir3_compile *ctx, nir_intrinsic_instr *intr)
1086 {
1087 nir_deref_var *dvar = intr->variables[0];
1088 nir_deref_array *darr = nir_deref_as_array(dvar->deref.child);
1089 struct ir3_array *arr = get_var(ctx, dvar->var);
1090 struct ir3_instruction *addr, **src;
1091 unsigned wrmask = intr->const_index[0];
1092
1093 compile_assert(ctx, dvar->deref.child &&
1094 (dvar->deref.child->deref_type == nir_deref_type_array));
1095
1096 src = get_src(ctx, &intr->src[0]);
1097
1098 switch (darr->deref_array_type) {
1099 case nir_deref_array_type_direct:
1100 addr = NULL;
1101 break;
1102 case nir_deref_array_type_indirect:
1103 addr = get_addr(ctx, get_src(ctx, &darr->indirect)[0]);
1104 break;
1105 default:
1106 compile_error(ctx, "Unhandled store deref type: %u\n",
1107 darr->deref_array_type);
1108 break;
1109 }
1110
1111 for (int i = 0; i < intr->num_components; i++) {
1112 if (!(wrmask & (1 << i)))
1113 continue;
1114 unsigned n = darr->base_offset * 4 + i;
1115 compile_assert(ctx, n < arr->length);
1116 create_var_store(ctx, arr, n, src[i], addr);
1117 }
1118 }
1119
1120 static void add_sysval_input(struct ir3_compile *ctx, gl_system_value slot,
1121 struct ir3_instruction *instr)
1122 {
1123 struct ir3_shader_variant *so = ctx->so;
1124 unsigned r = regid(so->inputs_count, 0);
1125 unsigned n = so->inputs_count++;
1126
1127 so->inputs[n].sysval = true;
1128 so->inputs[n].slot = slot;
1129 so->inputs[n].compmask = 1;
1130 so->inputs[n].regid = r;
1131 so->inputs[n].interpolate = INTERP_QUALIFIER_FLAT;
1132 so->total_in++;
1133
1134 ctx->ir->ninputs = MAX2(ctx->ir->ninputs, r + 1);
1135 ctx->ir->inputs[r] = instr;
1136 }
1137
1138 static void
1139 emit_intrinsic(struct ir3_compile *ctx, nir_intrinsic_instr *intr)
1140 {
1141 const nir_intrinsic_info *info = &nir_intrinsic_infos[intr->intrinsic];
1142 struct ir3_instruction **dst, **src;
1143 struct ir3_block *b = ctx->block;
1144 int idx = intr->const_index[0];
1145 nir_const_value *const_offset;
1146
1147 if (info->has_dest) {
1148 dst = get_dst(ctx, &intr->dest, intr->num_components);
1149 } else {
1150 dst = NULL;
1151 }
1152
1153 switch (intr->intrinsic) {
1154 case nir_intrinsic_load_uniform:
1155 const_offset = nir_src_as_const_value(intr->src[0]);
1156 if (const_offset) {
1157 idx += const_offset->u[0];
1158 for (int i = 0; i < intr->num_components; i++) {
1159 unsigned n = idx * 4 + i;
1160 dst[i] = create_uniform(ctx, n);
1161 }
1162 } else {
1163 src = get_src(ctx, &intr->src[0]);
1164 for (int i = 0; i < intr->num_components; i++) {
1165 int n = idx * 4 + i;
1166 dst[i] = create_uniform_indirect(ctx, n,
1167 get_addr(ctx, src[0]));
1168 }
1169 /* NOTE: if relative addressing is used, we set
1170 * constlen in the compiler (to worst-case value)
1171 * since we don't know in the assembler what the max
1172 * addr reg value can be:
1173 */
1174 ctx->so->constlen = ctx->s->num_uniforms;
1175 }
1176 break;
1177 case nir_intrinsic_load_ubo:
1178 emit_intrinsic_load_ubo(ctx, intr, dst);
1179 break;
1180 case nir_intrinsic_load_input:
1181 const_offset = nir_src_as_const_value(intr->src[0]);
1182 if (const_offset) {
1183 idx += const_offset->u[0];
1184 for (int i = 0; i < intr->num_components; i++) {
1185 unsigned n = idx * 4 + i;
1186 dst[i] = ctx->ir->inputs[n];
1187 }
1188 } else {
1189 src = get_src(ctx, &intr->src[0]);
1190 struct ir3_instruction *collect =
1191 create_collect(b, ctx->ir->inputs, ctx->ir->ninputs);
1192 struct ir3_instruction *addr = get_addr(ctx, src[0]);
1193 for (int i = 0; i < intr->num_components; i++) {
1194 unsigned n = idx * 4 + i;
1195 dst[i] = create_indirect_load(ctx, ctx->ir->ninputs,
1196 n, addr, collect);
1197 }
1198 }
1199 break;
1200 case nir_intrinsic_load_var:
1201 emit_intrinsic_load_var(ctx, intr, dst);
1202 break;
1203 case nir_intrinsic_store_var:
1204 emit_intrinsic_store_var(ctx, intr);
1205 break;
1206 case nir_intrinsic_store_output:
1207 const_offset = nir_src_as_const_value(intr->src[1]);
1208 compile_assert(ctx, const_offset != NULL);
1209 idx += const_offset->u[0];
1210
1211 src = get_src(ctx, &intr->src[0]);
1212 for (int i = 0; i < intr->num_components; i++) {
1213 unsigned n = idx * 4 + i;
1214 ctx->ir->outputs[n] = src[i];
1215 }
1216 break;
1217 case nir_intrinsic_load_base_vertex:
1218 if (!ctx->basevertex) {
1219 ctx->basevertex = create_driver_param(ctx, IR3_DP_VTXID_BASE);
1220 add_sysval_input(ctx, SYSTEM_VALUE_BASE_VERTEX,
1221 ctx->basevertex);
1222 }
1223 dst[0] = ctx->basevertex;
1224 break;
1225 case nir_intrinsic_load_vertex_id_zero_base:
1226 if (!ctx->vertex_id) {
1227 ctx->vertex_id = create_input(ctx->block, 0);
1228 add_sysval_input(ctx, SYSTEM_VALUE_VERTEX_ID_ZERO_BASE,
1229 ctx->vertex_id);
1230 }
1231 dst[0] = ctx->vertex_id;
1232 break;
1233 case nir_intrinsic_load_instance_id:
1234 if (!ctx->instance_id) {
1235 ctx->instance_id = create_input(ctx->block, 0);
1236 add_sysval_input(ctx, SYSTEM_VALUE_INSTANCE_ID,
1237 ctx->instance_id);
1238 }
1239 dst[0] = ctx->instance_id;
1240 break;
1241 case nir_intrinsic_load_user_clip_plane:
1242 for (int i = 0; i < intr->num_components; i++) {
1243 unsigned n = idx * 4 + i;
1244 dst[i] = create_driver_param(ctx, IR3_DP_UCP0_X + n);
1245 }
1246 break;
1247 case nir_intrinsic_discard_if:
1248 case nir_intrinsic_discard: {
1249 struct ir3_instruction *cond, *kill;
1250
1251 if (intr->intrinsic == nir_intrinsic_discard_if) {
1252 /* conditional discard: */
1253 src = get_src(ctx, &intr->src[0]);
1254 cond = ir3_b2n(b, src[0]);
1255 } else {
1256 /* unconditional discard: */
1257 cond = create_immed(b, 1);
1258 }
1259
1260 /* NOTE: only cmps.*.* can write p0.x: */
1261 cond = ir3_CMPS_S(b, cond, 0, create_immed(b, 0), 0);
1262 cond->cat2.condition = IR3_COND_NE;
1263
1264 /* condition always goes in predicate register: */
1265 cond->regs[0]->num = regid(REG_P0, 0);
1266
1267 kill = ir3_KILL(b, cond, 0);
1268 array_insert(ctx->ir->predicates, kill);
1269
1270 array_insert(ctx->ir->keeps, kill);
1271 ctx->so->has_kill = true;
1272
1273 break;
1274 }
1275 default:
1276 compile_error(ctx, "Unhandled intrinsic type: %s\n",
1277 nir_intrinsic_infos[intr->intrinsic].name);
1278 break;
1279 }
1280 }
1281
1282 static void
1283 emit_load_const(struct ir3_compile *ctx, nir_load_const_instr *instr)
1284 {
1285 struct ir3_instruction **dst = get_dst_ssa(ctx, &instr->def,
1286 instr->def.num_components);
1287 for (int i = 0; i < instr->def.num_components; i++)
1288 dst[i] = create_immed(ctx->block, instr->value.u[i]);
1289 }
1290
1291 static void
1292 emit_undef(struct ir3_compile *ctx, nir_ssa_undef_instr *undef)
1293 {
1294 struct ir3_instruction **dst = get_dst_ssa(ctx, &undef->def,
1295 undef->def.num_components);
1296 /* backend doesn't want undefined instructions, so just plug
1297 * in 0.0..
1298 */
1299 for (int i = 0; i < undef->def.num_components; i++)
1300 dst[i] = create_immed(ctx->block, fui(0.0));
1301 }
1302
1303 /*
1304 * texture fetch/sample instructions:
1305 */
1306
1307 static void
1308 tex_info(nir_tex_instr *tex, unsigned *flagsp, unsigned *coordsp)
1309 {
1310 unsigned coords, flags = 0;
1311
1312 /* note: would use tex->coord_components.. except txs.. also,
1313 * since array index goes after shadow ref, we don't want to
1314 * count it:
1315 */
1316 switch (tex->sampler_dim) {
1317 case GLSL_SAMPLER_DIM_1D:
1318 case GLSL_SAMPLER_DIM_BUF:
1319 coords = 1;
1320 break;
1321 case GLSL_SAMPLER_DIM_2D:
1322 case GLSL_SAMPLER_DIM_RECT:
1323 case GLSL_SAMPLER_DIM_EXTERNAL:
1324 case GLSL_SAMPLER_DIM_MS:
1325 coords = 2;
1326 break;
1327 case GLSL_SAMPLER_DIM_3D:
1328 case GLSL_SAMPLER_DIM_CUBE:
1329 coords = 3;
1330 flags |= IR3_INSTR_3D;
1331 break;
1332 default:
1333 unreachable("bad sampler_dim");
1334 }
1335
1336 if (tex->is_shadow && tex->op != nir_texop_lod)
1337 flags |= IR3_INSTR_S;
1338
1339 if (tex->is_array && tex->op != nir_texop_lod)
1340 flags |= IR3_INSTR_A;
1341
1342 *flagsp = flags;
1343 *coordsp = coords;
1344 }
1345
1346 static void
1347 emit_tex(struct ir3_compile *ctx, nir_tex_instr *tex)
1348 {
1349 struct ir3_block *b = ctx->block;
1350 struct ir3_instruction **dst, *sam, *src0[12], *src1[4];
1351 struct ir3_instruction **coord, *lod, *compare, *proj, **off, **ddx, **ddy;
1352 bool has_bias = false, has_lod = false, has_proj = false, has_off = false;
1353 unsigned i, coords, flags;
1354 unsigned nsrc0 = 0, nsrc1 = 0;
1355 type_t type;
1356 opc_t opc = 0;
1357
1358 coord = off = ddx = ddy = NULL;
1359 lod = proj = compare = NULL;
1360
1361 /* TODO: might just be one component for gathers? */
1362 dst = get_dst(ctx, &tex->dest, 4);
1363
1364 for (unsigned i = 0; i < tex->num_srcs; i++) {
1365 switch (tex->src[i].src_type) {
1366 case nir_tex_src_coord:
1367 coord = get_src(ctx, &tex->src[i].src);
1368 break;
1369 case nir_tex_src_bias:
1370 lod = get_src(ctx, &tex->src[i].src)[0];
1371 has_bias = true;
1372 break;
1373 case nir_tex_src_lod:
1374 lod = get_src(ctx, &tex->src[i].src)[0];
1375 has_lod = true;
1376 break;
1377 case nir_tex_src_comparitor: /* shadow comparator */
1378 compare = get_src(ctx, &tex->src[i].src)[0];
1379 break;
1380 case nir_tex_src_projector:
1381 proj = get_src(ctx, &tex->src[i].src)[0];
1382 has_proj = true;
1383 break;
1384 case nir_tex_src_offset:
1385 off = get_src(ctx, &tex->src[i].src);
1386 has_off = true;
1387 break;
1388 case nir_tex_src_ddx:
1389 ddx = get_src(ctx, &tex->src[i].src);
1390 break;
1391 case nir_tex_src_ddy:
1392 ddy = get_src(ctx, &tex->src[i].src);
1393 break;
1394 default:
1395 compile_error(ctx, "Unhandled NIR tex serc type: %d\n",
1396 tex->src[i].src_type);
1397 return;
1398 }
1399 }
1400
1401 switch (tex->op) {
1402 case nir_texop_tex: opc = OPC_SAM; break;
1403 case nir_texop_txb: opc = OPC_SAMB; break;
1404 case nir_texop_txl: opc = OPC_SAML; break;
1405 case nir_texop_txd: opc = OPC_SAMGQ; break;
1406 case nir_texop_txf: opc = OPC_ISAML; break;
1407 case nir_texop_lod: opc = OPC_GETLOD; break;
1408 case nir_texop_txf_ms:
1409 case nir_texop_txs:
1410 case nir_texop_tg4:
1411 case nir_texop_query_levels:
1412 case nir_texop_texture_samples:
1413 case nir_texop_samples_identical:
1414 compile_error(ctx, "Unhandled NIR tex type: %d\n", tex->op);
1415 return;
1416 }
1417
1418 tex_info(tex, &flags, &coords);
1419
1420 /* scale up integer coords for TXF based on the LOD */
1421 if (ctx->unminify_coords && (opc == OPC_ISAML)) {
1422 assert(has_lod);
1423 for (i = 0; i < coords; i++)
1424 coord[i] = ir3_SHL_B(b, coord[i], 0, lod, 0);
1425 }
1426
1427 /* the array coord for cube arrays needs 0.5 added to it */
1428 if (tex->sampler_dim == GLSL_SAMPLER_DIM_CUBE && tex->is_array &&
1429 opc != OPC_ISAML)
1430 coord[3] = ir3_ADD_F(b, coord[3], 0, create_immed(b, fui(0.5)), 0);
1431
1432 /*
1433 * lay out the first argument in the proper order:
1434 * - actual coordinates first
1435 * - shadow reference
1436 * - array index
1437 * - projection w
1438 * - starting at offset 4, dpdx.xy, dpdy.xy
1439 *
1440 * bias/lod go into the second arg
1441 */
1442
1443 /* insert tex coords: */
1444 for (i = 0; i < coords; i++)
1445 src0[nsrc0++] = coord[i];
1446
1447 if (coords == 1) {
1448 /* hw doesn't do 1d, so we treat it as 2d with
1449 * height of 1, and patch up the y coord.
1450 * TODO: y coord should be (int)0 in some cases..
1451 */
1452 src0[nsrc0++] = create_immed(b, fui(0.5));
1453 }
1454
1455 if (tex->is_shadow && tex->op != nir_texop_lod)
1456 src0[nsrc0++] = compare;
1457
1458 if (tex->is_array && tex->op != nir_texop_lod)
1459 src0[nsrc0++] = coord[coords];
1460
1461 if (has_proj) {
1462 src0[nsrc0++] = proj;
1463 flags |= IR3_INSTR_P;
1464 }
1465
1466 /* pad to 4, then ddx/ddy: */
1467 if (tex->op == nir_texop_txd) {
1468 while (nsrc0 < 4)
1469 src0[nsrc0++] = create_immed(b, fui(0.0));
1470 for (i = 0; i < coords; i++)
1471 src0[nsrc0++] = ddx[i];
1472 if (coords < 2)
1473 src0[nsrc0++] = create_immed(b, fui(0.0));
1474 for (i = 0; i < coords; i++)
1475 src0[nsrc0++] = ddy[i];
1476 if (coords < 2)
1477 src0[nsrc0++] = create_immed(b, fui(0.0));
1478 }
1479
1480 /*
1481 * second argument (if applicable):
1482 * - offsets
1483 * - lod
1484 * - bias
1485 */
1486 if (has_off | has_lod | has_bias) {
1487 if (has_off) {
1488 for (i = 0; i < coords; i++)
1489 src1[nsrc1++] = off[i];
1490 if (coords < 2)
1491 src1[nsrc1++] = create_immed(b, fui(0.0));
1492 flags |= IR3_INSTR_O;
1493 }
1494
1495 if (has_lod | has_bias)
1496 src1[nsrc1++] = lod;
1497 }
1498
1499 switch (tex->dest_type) {
1500 case nir_type_invalid:
1501 case nir_type_float:
1502 type = TYPE_F32;
1503 break;
1504 case nir_type_int:
1505 type = TYPE_S32;
1506 break;
1507 case nir_type_uint:
1508 case nir_type_bool:
1509 type = TYPE_U32;
1510 break;
1511 default:
1512 unreachable("bad dest_type");
1513 }
1514
1515 if (opc == OPC_GETLOD)
1516 type = TYPE_U32;
1517
1518 sam = ir3_SAM(b, opc, type, TGSI_WRITEMASK_XYZW,
1519 flags, tex->sampler_index, tex->sampler_index,
1520 create_collect(b, src0, nsrc0),
1521 create_collect(b, src1, nsrc1));
1522
1523 split_dest(b, dst, sam, 4);
1524
1525 /* GETLOD returns results in 4.8 fixed point */
1526 if (opc == OPC_GETLOD) {
1527 struct ir3_instruction *factor = create_immed(b, fui(1.0 / 256));
1528
1529 compile_assert(ctx, tex->dest_type == nir_type_float);
1530 for (i = 0; i < 2; i++) {
1531 dst[i] = ir3_MUL_F(b, ir3_COV(b, dst[i], TYPE_U32, TYPE_F32), 0,
1532 factor, 0);
1533 }
1534 }
1535 }
1536
1537 static void
1538 emit_tex_query_levels(struct ir3_compile *ctx, nir_tex_instr *tex)
1539 {
1540 struct ir3_block *b = ctx->block;
1541 struct ir3_instruction **dst, *sam;
1542
1543 dst = get_dst(ctx, &tex->dest, 1);
1544
1545 sam = ir3_SAM(b, OPC_GETINFO, TYPE_U32, TGSI_WRITEMASK_Z, 0,
1546 tex->sampler_index, tex->sampler_index, NULL, NULL);
1547
1548 /* even though there is only one component, since it ends
1549 * up in .z rather than .x, we need a split_dest()
1550 */
1551 split_dest(b, dst, sam, 3);
1552
1553 /* The # of levels comes from getinfo.z. We need to add 1 to it, since
1554 * the value in TEX_CONST_0 is zero-based.
1555 */
1556 if (ctx->levels_add_one)
1557 dst[0] = ir3_ADD_U(b, dst[0], 0, create_immed(b, 1), 0);
1558 }
1559
1560 static void
1561 emit_tex_txs(struct ir3_compile *ctx, nir_tex_instr *tex)
1562 {
1563 struct ir3_block *b = ctx->block;
1564 struct ir3_instruction **dst, *sam, *lod;
1565 unsigned flags, coords;
1566
1567 tex_info(tex, &flags, &coords);
1568
1569 /* Actually we want the number of dimensions, not coordinates. This
1570 * distinction only matters for cubes.
1571 */
1572 if (tex->sampler_dim == GLSL_SAMPLER_DIM_CUBE)
1573 coords = 2;
1574
1575 dst = get_dst(ctx, &tex->dest, 4);
1576
1577 compile_assert(ctx, tex->num_srcs == 1);
1578 compile_assert(ctx, tex->src[0].src_type == nir_tex_src_lod);
1579
1580 lod = get_src(ctx, &tex->src[0].src)[0];
1581
1582 sam = ir3_SAM(b, OPC_GETSIZE, TYPE_U32, TGSI_WRITEMASK_XYZW, flags,
1583 tex->sampler_index, tex->sampler_index, lod, NULL);
1584
1585 split_dest(b, dst, sam, 4);
1586
1587 /* Array size actually ends up in .w rather than .z. This doesn't
1588 * matter for miplevel 0, but for higher mips the value in z is
1589 * minified whereas w stays. Also, the value in TEX_CONST_3_DEPTH is
1590 * returned, which means that we have to add 1 to it for arrays.
1591 */
1592 if (tex->is_array) {
1593 if (ctx->levels_add_one) {
1594 dst[coords] = ir3_ADD_U(b, dst[3], 0, create_immed(b, 1), 0);
1595 } else {
1596 dst[coords] = ir3_MOV(b, dst[3], TYPE_U32);
1597 }
1598 }
1599 }
1600
1601 static void
1602 emit_phi(struct ir3_compile *ctx, nir_phi_instr *nphi)
1603 {
1604 struct ir3_instruction *phi, **dst;
1605
1606 /* NOTE: phi's should be lowered to scalar at this point */
1607 compile_assert(ctx, nphi->dest.ssa.num_components == 1);
1608
1609 dst = get_dst(ctx, &nphi->dest, 1);
1610
1611 phi = ir3_instr_create2(ctx->block, -1, OPC_META_PHI,
1612 1 + exec_list_length(&nphi->srcs));
1613 ir3_reg_create(phi, 0, 0); /* dst */
1614 phi->phi.nphi = nphi;
1615
1616 dst[0] = phi;
1617 }
1618
1619 /* phi instructions are left partially constructed. We don't resolve
1620 * their srcs until the end of the block, since (eg. loops) one of
1621 * the phi's srcs might be defined after the phi due to back edges in
1622 * the CFG.
1623 */
1624 static void
1625 resolve_phis(struct ir3_compile *ctx, struct ir3_block *block)
1626 {
1627 list_for_each_entry (struct ir3_instruction, instr, &block->instr_list, node) {
1628 nir_phi_instr *nphi;
1629
1630 /* phi's only come at start of block: */
1631 if (!(is_meta(instr) && (instr->opc == OPC_META_PHI)))
1632 break;
1633
1634 if (!instr->phi.nphi)
1635 break;
1636
1637 nphi = instr->phi.nphi;
1638 instr->phi.nphi = NULL;
1639
1640 foreach_list_typed(nir_phi_src, nsrc, node, &nphi->srcs) {
1641 struct ir3_instruction *src = get_src(ctx, &nsrc->src)[0];
1642 ir3_reg_create(instr, 0, IR3_REG_SSA)->instr = src;
1643 }
1644 }
1645 }
1646
1647 static void
1648 emit_jump(struct ir3_compile *ctx, nir_jump_instr *jump)
1649 {
1650 switch (jump->type) {
1651 case nir_jump_break:
1652 case nir_jump_continue:
1653 /* I *think* we can simply just ignore this, and use the
1654 * successor block link to figure out where we need to
1655 * jump to for break/continue
1656 */
1657 break;
1658 default:
1659 compile_error(ctx, "Unhandled NIR jump type: %d\n", jump->type);
1660 break;
1661 }
1662 }
1663
1664 static void
1665 emit_instr(struct ir3_compile *ctx, nir_instr *instr)
1666 {
1667 switch (instr->type) {
1668 case nir_instr_type_alu:
1669 emit_alu(ctx, nir_instr_as_alu(instr));
1670 break;
1671 case nir_instr_type_intrinsic:
1672 emit_intrinsic(ctx, nir_instr_as_intrinsic(instr));
1673 break;
1674 case nir_instr_type_load_const:
1675 emit_load_const(ctx, nir_instr_as_load_const(instr));
1676 break;
1677 case nir_instr_type_ssa_undef:
1678 emit_undef(ctx, nir_instr_as_ssa_undef(instr));
1679 break;
1680 case nir_instr_type_tex: {
1681 nir_tex_instr *tex = nir_instr_as_tex(instr);
1682 /* couple tex instructions get special-cased:
1683 */
1684 switch (tex->op) {
1685 case nir_texop_txs:
1686 emit_tex_txs(ctx, tex);
1687 break;
1688 case nir_texop_query_levels:
1689 emit_tex_query_levels(ctx, tex);
1690 break;
1691 default:
1692 emit_tex(ctx, tex);
1693 break;
1694 }
1695 break;
1696 }
1697 case nir_instr_type_phi:
1698 emit_phi(ctx, nir_instr_as_phi(instr));
1699 break;
1700 case nir_instr_type_jump:
1701 emit_jump(ctx, nir_instr_as_jump(instr));
1702 break;
1703 case nir_instr_type_call:
1704 case nir_instr_type_parallel_copy:
1705 compile_error(ctx, "Unhandled NIR instruction type: %d\n", instr->type);
1706 break;
1707 }
1708 }
1709
1710 static struct ir3_block *
1711 get_block(struct ir3_compile *ctx, nir_block *nblock)
1712 {
1713 struct ir3_block *block;
1714 struct hash_entry *entry;
1715 entry = _mesa_hash_table_search(ctx->block_ht, nblock);
1716 if (entry)
1717 return entry->data;
1718
1719 block = ir3_block_create(ctx->ir);
1720 block->nblock = nblock;
1721 _mesa_hash_table_insert(ctx->block_ht, nblock, block);
1722
1723 return block;
1724 }
1725
1726 static void
1727 emit_block(struct ir3_compile *ctx, nir_block *nblock)
1728 {
1729 struct ir3_block *block = get_block(ctx, nblock);
1730
1731 for (int i = 0; i < ARRAY_SIZE(block->successors); i++) {
1732 if (nblock->successors[i]) {
1733 block->successors[i] =
1734 get_block(ctx, nblock->successors[i]);
1735 }
1736 }
1737
1738 ctx->block = block;
1739 list_addtail(&block->node, &ctx->ir->block_list);
1740
1741 /* re-emit addr register in each block if needed: */
1742 _mesa_hash_table_destroy(ctx->addr_ht, NULL);
1743 ctx->addr_ht = NULL;
1744
1745 nir_foreach_instr(nblock, instr) {
1746 emit_instr(ctx, instr);
1747 if (ctx->error)
1748 return;
1749 }
1750 }
1751
1752 static void emit_cf_list(struct ir3_compile *ctx, struct exec_list *list);
1753
1754 static void
1755 emit_if(struct ir3_compile *ctx, nir_if *nif)
1756 {
1757 struct ir3_instruction *condition = get_src(ctx, &nif->condition)[0];
1758
1759 ctx->block->condition =
1760 get_predicate(ctx, ir3_b2n(condition->block, condition));
1761
1762 emit_cf_list(ctx, &nif->then_list);
1763 emit_cf_list(ctx, &nif->else_list);
1764 }
1765
1766 static void
1767 emit_loop(struct ir3_compile *ctx, nir_loop *nloop)
1768 {
1769 emit_cf_list(ctx, &nloop->body);
1770 }
1771
1772 static void
1773 emit_cf_list(struct ir3_compile *ctx, struct exec_list *list)
1774 {
1775 foreach_list_typed(nir_cf_node, node, node, list) {
1776 switch (node->type) {
1777 case nir_cf_node_block:
1778 emit_block(ctx, nir_cf_node_as_block(node));
1779 break;
1780 case nir_cf_node_if:
1781 emit_if(ctx, nir_cf_node_as_if(node));
1782 break;
1783 case nir_cf_node_loop:
1784 emit_loop(ctx, nir_cf_node_as_loop(node));
1785 break;
1786 case nir_cf_node_function:
1787 compile_error(ctx, "TODO\n");
1788 break;
1789 }
1790 }
1791 }
1792
1793 /* emit stream-out code. At this point, the current block is the original
1794 * (nir) end block, and nir ensures that all flow control paths terminate
1795 * into the end block. We re-purpose the original end block to generate
1796 * the 'if (vtxcnt < maxvtxcnt)' condition, then append the conditional
1797 * block holding stream-out write instructions, followed by the new end
1798 * block:
1799 *
1800 * blockOrigEnd {
1801 * p0.x = (vtxcnt < maxvtxcnt)
1802 * // succs: blockStreamOut, blockNewEnd
1803 * }
1804 * blockStreamOut {
1805 * ... stream-out instructions ...
1806 * // succs: blockNewEnd
1807 * }
1808 * blockNewEnd {
1809 * }
1810 */
1811 static void
1812 emit_stream_out(struct ir3_compile *ctx)
1813 {
1814 struct ir3_shader_variant *v = ctx->so;
1815 struct ir3 *ir = ctx->ir;
1816 struct pipe_stream_output_info *strmout =
1817 &ctx->so->shader->stream_output;
1818 struct ir3_block *orig_end_block, *stream_out_block, *new_end_block;
1819 struct ir3_instruction *vtxcnt, *maxvtxcnt, *cond;
1820 struct ir3_instruction *bases[PIPE_MAX_SO_BUFFERS];
1821
1822 /* create vtxcnt input in input block at top of shader,
1823 * so that it is seen as live over the entire duration
1824 * of the shader:
1825 */
1826 vtxcnt = create_input(ctx->in_block, 0);
1827 add_sysval_input(ctx, SYSTEM_VALUE_VERTEX_CNT, vtxcnt);
1828
1829 maxvtxcnt = create_driver_param(ctx, IR3_DP_VTXCNT_MAX);
1830
1831 /* at this point, we are at the original 'end' block,
1832 * re-purpose this block to stream-out condition, then
1833 * append stream-out block and new-end block
1834 */
1835 orig_end_block = ctx->block;
1836
1837 stream_out_block = ir3_block_create(ir);
1838 list_addtail(&stream_out_block->node, &ir->block_list);
1839
1840 new_end_block = ir3_block_create(ir);
1841 list_addtail(&new_end_block->node, &ir->block_list);
1842
1843 orig_end_block->successors[0] = stream_out_block;
1844 orig_end_block->successors[1] = new_end_block;
1845 stream_out_block->successors[0] = new_end_block;
1846
1847 /* setup 'if (vtxcnt < maxvtxcnt)' condition: */
1848 cond = ir3_CMPS_S(ctx->block, vtxcnt, 0, maxvtxcnt, 0);
1849 cond->regs[0]->num = regid(REG_P0, 0);
1850 cond->cat2.condition = IR3_COND_LT;
1851
1852 /* condition goes on previous block to the conditional,
1853 * since it is used to pick which of the two successor
1854 * paths to take:
1855 */
1856 orig_end_block->condition = cond;
1857
1858 /* switch to stream_out_block to generate the stream-out
1859 * instructions:
1860 */
1861 ctx->block = stream_out_block;
1862
1863 /* Calculate base addresses based on vtxcnt. Instructions
1864 * generated for bases not used in following loop will be
1865 * stripped out in the backend.
1866 */
1867 for (unsigned i = 0; i < PIPE_MAX_SO_BUFFERS; i++) {
1868 unsigned stride = strmout->stride[i];
1869 struct ir3_instruction *base, *off;
1870
1871 base = create_uniform(ctx, regid(v->first_driver_param + IR3_TFBOS_OFF, i));
1872
1873 /* 24-bit should be enough: */
1874 off = ir3_MUL_U(ctx->block, vtxcnt, 0,
1875 create_immed(ctx->block, stride * 4), 0);
1876
1877 bases[i] = ir3_ADD_S(ctx->block, off, 0, base, 0);
1878 }
1879
1880 /* Generate the per-output store instructions: */
1881 for (unsigned i = 0; i < strmout->num_outputs; i++) {
1882 for (unsigned j = 0; j < strmout->output[i].num_components; j++) {
1883 unsigned c = j + strmout->output[i].start_component;
1884 struct ir3_instruction *base, *out, *stg;
1885
1886 base = bases[strmout->output[i].output_buffer];
1887 out = ctx->ir->outputs[regid(strmout->output[i].register_index, c)];
1888
1889 stg = ir3_STG(ctx->block, base, 0, out, 0,
1890 create_immed(ctx->block, 1), 0);
1891 stg->cat6.type = TYPE_U32;
1892 stg->cat6.dst_offset = (strmout->output[i].dst_offset + j) * 4;
1893
1894 array_insert(ctx->ir->keeps, stg);
1895 }
1896 }
1897
1898 /* and finally switch to the new_end_block: */
1899 ctx->block = new_end_block;
1900 }
1901
1902 static void
1903 emit_function(struct ir3_compile *ctx, nir_function_impl *impl)
1904 {
1905 nir_metadata_require(impl, nir_metadata_block_index);
1906
1907 emit_cf_list(ctx, &impl->body);
1908 emit_block(ctx, impl->end_block);
1909
1910 /* at this point, we should have a single empty block,
1911 * into which we emit the 'end' instruction.
1912 */
1913 compile_assert(ctx, list_empty(&ctx->block->instr_list));
1914
1915 /* If stream-out (aka transform-feedback) enabled, emit the
1916 * stream-out instructions, followed by a new empty block (into
1917 * which the 'end' instruction lands).
1918 *
1919 * NOTE: it is done in this order, rather than inserting before
1920 * we emit end_block, because NIR guarantees that all blocks
1921 * flow into end_block, and that end_block has no successors.
1922 * So by re-purposing end_block as the first block of stream-
1923 * out, we guarantee that all exit paths flow into the stream-
1924 * out instructions.
1925 */
1926 if ((ctx->so->shader->stream_output.num_outputs > 0) &&
1927 !ctx->so->key.binning_pass) {
1928 debug_assert(ctx->so->type == SHADER_VERTEX);
1929 emit_stream_out(ctx);
1930 }
1931
1932 ir3_END(ctx->block);
1933 }
1934
1935 static void
1936 setup_input(struct ir3_compile *ctx, nir_variable *in)
1937 {
1938 struct ir3_shader_variant *so = ctx->so;
1939 unsigned array_len = MAX2(glsl_get_length(in->type), 1);
1940 unsigned ncomp = glsl_get_components(in->type);
1941 unsigned n = in->data.driver_location;
1942 unsigned slot = in->data.location;
1943
1944 DBG("; in: slot=%u, len=%ux%u, drvloc=%u",
1945 slot, array_len, ncomp, n);
1946
1947 so->inputs[n].slot = slot;
1948 so->inputs[n].compmask = (1 << ncomp) - 1;
1949 so->inputs_count = MAX2(so->inputs_count, n + 1);
1950 so->inputs[n].interpolate = in->data.interpolation;
1951
1952 if (ctx->so->type == SHADER_FRAGMENT) {
1953 for (int i = 0; i < ncomp; i++) {
1954 struct ir3_instruction *instr = NULL;
1955 unsigned idx = (n * 4) + i;
1956
1957 if (slot == VARYING_SLOT_POS) {
1958 so->inputs[n].bary = false;
1959 so->frag_coord = true;
1960 instr = create_frag_coord(ctx, i);
1961 } else if (slot == VARYING_SLOT_FACE) {
1962 so->inputs[n].bary = false;
1963 so->frag_face = true;
1964 instr = create_frag_face(ctx, i);
1965 } else {
1966 bool use_ldlv = false;
1967
1968 /* detect the special case for front/back colors where
1969 * we need to do flat vs smooth shading depending on
1970 * rast state:
1971 */
1972 if (in->data.interpolation == INTERP_QUALIFIER_NONE) {
1973 switch (slot) {
1974 case VARYING_SLOT_COL0:
1975 case VARYING_SLOT_COL1:
1976 case VARYING_SLOT_BFC0:
1977 case VARYING_SLOT_BFC1:
1978 so->inputs[n].rasterflat = true;
1979 break;
1980 default:
1981 break;
1982 }
1983 }
1984
1985 if (ctx->flat_bypass) {
1986 if ((so->inputs[n].interpolate == INTERP_QUALIFIER_FLAT) ||
1987 (so->inputs[n].rasterflat && ctx->so->key.rasterflat))
1988 use_ldlv = true;
1989 }
1990
1991 so->inputs[n].bary = true;
1992
1993 instr = create_frag_input(ctx, use_ldlv);
1994 }
1995
1996 ctx->ir->inputs[idx] = instr;
1997 }
1998 } else if (ctx->so->type == SHADER_VERTEX) {
1999 for (int i = 0; i < ncomp; i++) {
2000 unsigned idx = (n * 4) + i;
2001 ctx->ir->inputs[idx] = create_input(ctx->block, idx);
2002 }
2003 } else {
2004 compile_error(ctx, "unknown shader type: %d\n", ctx->so->type);
2005 }
2006
2007 if (so->inputs[n].bary || (ctx->so->type == SHADER_VERTEX)) {
2008 so->total_in += ncomp;
2009 }
2010 }
2011
2012 static void
2013 setup_output(struct ir3_compile *ctx, nir_variable *out)
2014 {
2015 struct ir3_shader_variant *so = ctx->so;
2016 unsigned array_len = MAX2(glsl_get_length(out->type), 1);
2017 unsigned ncomp = glsl_get_components(out->type);
2018 unsigned n = out->data.driver_location;
2019 unsigned slot = out->data.location;
2020 unsigned comp = 0;
2021
2022 DBG("; out: slot=%u, len=%ux%u, drvloc=%u",
2023 slot, array_len, ncomp, n);
2024
2025 if (ctx->so->type == SHADER_FRAGMENT) {
2026 switch (slot) {
2027 case FRAG_RESULT_DEPTH:
2028 comp = 2; /* tgsi will write to .z component */
2029 so->writes_pos = true;
2030 break;
2031 case FRAG_RESULT_COLOR:
2032 so->color0_mrt = 1;
2033 break;
2034 default:
2035 if (slot >= FRAG_RESULT_DATA0)
2036 break;
2037 compile_error(ctx, "unknown FS output name: %s\n",
2038 gl_frag_result_name(slot));
2039 }
2040 } else if (ctx->so->type == SHADER_VERTEX) {
2041 switch (slot) {
2042 case VARYING_SLOT_POS:
2043 so->writes_pos = true;
2044 break;
2045 case VARYING_SLOT_PSIZ:
2046 so->writes_psize = true;
2047 break;
2048 case VARYING_SLOT_COL0:
2049 case VARYING_SLOT_COL1:
2050 case VARYING_SLOT_BFC0:
2051 case VARYING_SLOT_BFC1:
2052 case VARYING_SLOT_FOGC:
2053 case VARYING_SLOT_CLIP_DIST0:
2054 case VARYING_SLOT_CLIP_DIST1:
2055 break;
2056 default:
2057 if (slot >= VARYING_SLOT_VAR0)
2058 break;
2059 if ((VARYING_SLOT_TEX0 <= slot) && (slot <= VARYING_SLOT_TEX7))
2060 break;
2061 compile_error(ctx, "unknown VS output name: %s\n",
2062 gl_varying_slot_name(slot));
2063 }
2064 } else {
2065 compile_error(ctx, "unknown shader type: %d\n", ctx->so->type);
2066 }
2067
2068 compile_assert(ctx, n < ARRAY_SIZE(so->outputs));
2069
2070 so->outputs[n].slot = slot;
2071 so->outputs[n].regid = regid(n, comp);
2072 so->outputs_count = MAX2(so->outputs_count, n + 1);
2073
2074 for (int i = 0; i < ncomp; i++) {
2075 unsigned idx = (n * 4) + i;
2076
2077 ctx->ir->outputs[idx] = create_immed(ctx->block, fui(0.0));
2078 }
2079 }
2080
2081 static void
2082 emit_instructions(struct ir3_compile *ctx)
2083 {
2084 unsigned ninputs, noutputs;
2085 nir_function_impl *fxn = NULL;
2086
2087 /* Find the main function: */
2088 nir_foreach_function(ctx->s, function) {
2089 compile_assert(ctx, strcmp(function->name, "main") == 0);
2090 compile_assert(ctx, function->impl);
2091 fxn = function->impl;
2092 break;
2093 }
2094
2095 ninputs = exec_list_length(&ctx->s->inputs) * 4;
2096 noutputs = exec_list_length(&ctx->s->outputs) * 4;
2097
2098 /* or vtx shaders, we need to leave room for sysvals:
2099 */
2100 if (ctx->so->type == SHADER_VERTEX) {
2101 ninputs += 8;
2102 }
2103
2104 ctx->ir = ir3_create(ctx->compiler, ninputs, noutputs);
2105
2106 /* Create inputs in first block: */
2107 ctx->block = get_block(ctx, nir_start_block(fxn));
2108 ctx->in_block = ctx->block;
2109 list_addtail(&ctx->block->node, &ctx->ir->block_list);
2110
2111 if (ctx->so->type == SHADER_VERTEX) {
2112 ctx->ir->ninputs -= 8;
2113 }
2114
2115 /* for fragment shader, we have a single input register (usually
2116 * r0.xy) which is used as the base for bary.f varying fetch instrs:
2117 */
2118 if (ctx->so->type == SHADER_FRAGMENT) {
2119 // TODO maybe a helper for fi since we need it a few places..
2120 struct ir3_instruction *instr;
2121 instr = ir3_instr_create(ctx->block, -1, OPC_META_FI);
2122 ir3_reg_create(instr, 0, 0);
2123 ir3_reg_create(instr, 0, IR3_REG_SSA); /* r0.x */
2124 ir3_reg_create(instr, 0, IR3_REG_SSA); /* r0.y */
2125 ctx->frag_pos = instr;
2126 }
2127
2128 /* Setup inputs: */
2129 nir_foreach_variable(var, &ctx->s->inputs) {
2130 setup_input(ctx, var);
2131 }
2132
2133 /* Setup outputs: */
2134 nir_foreach_variable(var, &ctx->s->outputs) {
2135 setup_output(ctx, var);
2136 }
2137
2138 /* Setup variables (which should only be arrays): */
2139 nir_foreach_variable(var, &ctx->s->globals) {
2140 declare_var(ctx, var);
2141 }
2142
2143 /* And emit the body: */
2144 ctx->impl = fxn;
2145 emit_function(ctx, fxn);
2146
2147 list_for_each_entry (struct ir3_block, block, &ctx->ir->block_list, node) {
2148 resolve_phis(ctx, block);
2149 }
2150 }
2151
2152 /* from NIR perspective, we actually have inputs. But most of the "inputs"
2153 * for a fragment shader are just bary.f instructions. The *actual* inputs
2154 * from the hw perspective are the frag_pos and optionally frag_coord and
2155 * frag_face.
2156 */
2157 static void
2158 fixup_frag_inputs(struct ir3_compile *ctx)
2159 {
2160 struct ir3_shader_variant *so = ctx->so;
2161 struct ir3 *ir = ctx->ir;
2162 struct ir3_instruction **inputs;
2163 struct ir3_instruction *instr;
2164 int n, regid = 0;
2165
2166 ir->ninputs = 0;
2167
2168 n = 4; /* always have frag_pos */
2169 n += COND(so->frag_face, 4);
2170 n += COND(so->frag_coord, 4);
2171
2172 inputs = ir3_alloc(ctx->ir, n * (sizeof(struct ir3_instruction *)));
2173
2174 if (so->frag_face) {
2175 /* this ultimately gets assigned to hr0.x so doesn't conflict
2176 * with frag_coord/frag_pos..
2177 */
2178 inputs[ir->ninputs++] = ctx->frag_face;
2179 ctx->frag_face->regs[0]->num = 0;
2180
2181 /* remaining channels not used, but let's avoid confusing
2182 * other parts that expect inputs to come in groups of vec4
2183 */
2184 inputs[ir->ninputs++] = NULL;
2185 inputs[ir->ninputs++] = NULL;
2186 inputs[ir->ninputs++] = NULL;
2187 }
2188
2189 /* since we don't know where to set the regid for frag_coord,
2190 * we have to use r0.x for it. But we don't want to *always*
2191 * use r1.x for frag_pos as that could increase the register
2192 * footprint on simple shaders:
2193 */
2194 if (so->frag_coord) {
2195 ctx->frag_coord[0]->regs[0]->num = regid++;
2196 ctx->frag_coord[1]->regs[0]->num = regid++;
2197 ctx->frag_coord[2]->regs[0]->num = regid++;
2198 ctx->frag_coord[3]->regs[0]->num = regid++;
2199
2200 inputs[ir->ninputs++] = ctx->frag_coord[0];
2201 inputs[ir->ninputs++] = ctx->frag_coord[1];
2202 inputs[ir->ninputs++] = ctx->frag_coord[2];
2203 inputs[ir->ninputs++] = ctx->frag_coord[3];
2204 }
2205
2206 /* we always have frag_pos: */
2207 so->pos_regid = regid;
2208
2209 /* r0.x */
2210 instr = create_input(ctx->in_block, ir->ninputs);
2211 instr->regs[0]->num = regid++;
2212 inputs[ir->ninputs++] = instr;
2213 ctx->frag_pos->regs[1]->instr = instr;
2214
2215 /* r0.y */
2216 instr = create_input(ctx->in_block, ir->ninputs);
2217 instr->regs[0]->num = regid++;
2218 inputs[ir->ninputs++] = instr;
2219 ctx->frag_pos->regs[2]->instr = instr;
2220
2221 ir->inputs = inputs;
2222 }
2223
2224 int
2225 ir3_compile_shader_nir(struct ir3_compiler *compiler,
2226 struct ir3_shader_variant *so)
2227 {
2228 struct ir3_compile *ctx;
2229 struct ir3 *ir;
2230 struct ir3_instruction **inputs;
2231 unsigned i, j, actual_in, inloc;
2232 int ret = 0, max_bary;
2233
2234 assert(!so->ir);
2235
2236 ctx = compile_init(compiler, so);
2237 if (!ctx) {
2238 DBG("INIT failed!");
2239 ret = -1;
2240 goto out;
2241 }
2242
2243 emit_instructions(ctx);
2244
2245 if (ctx->error) {
2246 DBG("EMIT failed!");
2247 ret = -1;
2248 goto out;
2249 }
2250
2251 ir = so->ir = ctx->ir;
2252
2253 /* keep track of the inputs from TGSI perspective.. */
2254 inputs = ir->inputs;
2255
2256 /* but fixup actual inputs for frag shader: */
2257 if (so->type == SHADER_FRAGMENT)
2258 fixup_frag_inputs(ctx);
2259
2260 /* at this point, for binning pass, throw away unneeded outputs: */
2261 if (so->key.binning_pass) {
2262 for (i = 0, j = 0; i < so->outputs_count; i++) {
2263 unsigned slot = so->outputs[i].slot;
2264
2265 /* throw away everything but first position/psize */
2266 if ((slot == VARYING_SLOT_POS) || (slot == VARYING_SLOT_PSIZ)) {
2267 if (i != j) {
2268 so->outputs[j] = so->outputs[i];
2269 ir->outputs[(j*4)+0] = ir->outputs[(i*4)+0];
2270 ir->outputs[(j*4)+1] = ir->outputs[(i*4)+1];
2271 ir->outputs[(j*4)+2] = ir->outputs[(i*4)+2];
2272 ir->outputs[(j*4)+3] = ir->outputs[(i*4)+3];
2273 }
2274 j++;
2275 }
2276 }
2277 so->outputs_count = j;
2278 ir->noutputs = j * 4;
2279 }
2280
2281 /* if we want half-precision outputs, mark the output registers
2282 * as half:
2283 */
2284 if (so->key.half_precision) {
2285 for (i = 0; i < ir->noutputs; i++) {
2286 struct ir3_instruction *out = ir->outputs[i];
2287 if (!out)
2288 continue;
2289 out->regs[0]->flags |= IR3_REG_HALF;
2290 /* output could be a fanout (ie. texture fetch output)
2291 * in which case we need to propagate the half-reg flag
2292 * up to the definer so that RA sees it:
2293 */
2294 if (is_meta(out) && (out->opc == OPC_META_FO)) {
2295 out = out->regs[1]->instr;
2296 out->regs[0]->flags |= IR3_REG_HALF;
2297 }
2298
2299 if (out->category == 1) {
2300 out->cat1.dst_type = half_type(out->cat1.dst_type);
2301 }
2302 }
2303 }
2304
2305 if (fd_mesa_debug & FD_DBG_OPTMSGS) {
2306 printf("BEFORE CP:\n");
2307 ir3_print(ir);
2308 }
2309
2310 ir3_cp(ir);
2311
2312 if (fd_mesa_debug & FD_DBG_OPTMSGS) {
2313 printf("BEFORE GROUPING:\n");
2314 ir3_print(ir);
2315 }
2316
2317 /* Group left/right neighbors, inserting mov's where needed to
2318 * solve conflicts:
2319 */
2320 ir3_group(ir);
2321
2322 ir3_depth(ir);
2323
2324 if (fd_mesa_debug & FD_DBG_OPTMSGS) {
2325 printf("AFTER DEPTH:\n");
2326 ir3_print(ir);
2327 }
2328
2329 ret = ir3_sched(ir);
2330 if (ret) {
2331 DBG("SCHED failed!");
2332 goto out;
2333 }
2334
2335 if (fd_mesa_debug & FD_DBG_OPTMSGS) {
2336 printf("AFTER SCHED:\n");
2337 ir3_print(ir);
2338 }
2339
2340 ret = ir3_ra(ir, so->type, so->frag_coord, so->frag_face);
2341 if (ret) {
2342 DBG("RA failed!");
2343 goto out;
2344 }
2345
2346 if (fd_mesa_debug & FD_DBG_OPTMSGS) {
2347 printf("AFTER RA:\n");
2348 ir3_print(ir);
2349 }
2350
2351 /* fixup input/outputs: */
2352 for (i = 0; i < so->outputs_count; i++) {
2353 so->outputs[i].regid = ir->outputs[i*4]->regs[0]->num;
2354 /* preserve hack for depth output.. tgsi writes depth to .z,
2355 * but what we give the hw is the scalar register:
2356 */
2357 if ((so->type == SHADER_FRAGMENT) &&
2358 (so->outputs[i].slot == FRAG_RESULT_DEPTH))
2359 so->outputs[i].regid += 2;
2360 }
2361
2362 /* Note that some or all channels of an input may be unused: */
2363 actual_in = 0;
2364 inloc = 0;
2365 for (i = 0; i < so->inputs_count; i++) {
2366 unsigned j, regid = ~0, compmask = 0;
2367 so->inputs[i].ncomp = 0;
2368 so->inputs[i].inloc = inloc + 8;
2369 for (j = 0; j < 4; j++) {
2370 struct ir3_instruction *in = inputs[(i*4) + j];
2371 if (in && !(in->flags & IR3_INSTR_UNUSED)) {
2372 compmask |= (1 << j);
2373 regid = in->regs[0]->num - j;
2374 actual_in++;
2375 so->inputs[i].ncomp++;
2376 if ((so->type == SHADER_FRAGMENT) && so->inputs[i].bary) {
2377 /* assign inloc: */
2378 assert(in->regs[1]->flags & IR3_REG_IMMED);
2379 in->regs[1]->iim_val = inloc++;
2380 }
2381 }
2382 }
2383 if ((so->type == SHADER_FRAGMENT) && compmask && so->inputs[i].bary)
2384 so->varying_in++;
2385 so->inputs[i].regid = regid;
2386 so->inputs[i].compmask = compmask;
2387 }
2388
2389 /* We need to do legalize after (for frag shader's) the "bary.f"
2390 * offsets (inloc) have been assigned.
2391 */
2392 ir3_legalize(ir, &so->has_samp, &max_bary);
2393
2394 if (fd_mesa_debug & FD_DBG_OPTMSGS) {
2395 printf("AFTER LEGALIZE:\n");
2396 ir3_print(ir);
2397 }
2398
2399 /* Note that actual_in counts inputs that are not bary.f'd for FS: */
2400 if (so->type == SHADER_VERTEX)
2401 so->total_in = actual_in;
2402 else
2403 so->total_in = max_bary + 1;
2404
2405 out:
2406 if (ret) {
2407 if (so->ir)
2408 ir3_destroy(so->ir);
2409 so->ir = NULL;
2410 }
2411 compile_free(ctx);
2412
2413 return ret;
2414 }