projects / mesa.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
history | raw | HEAD
freedreno/ir3: support load_front_face intrinsic
[mesa.git] / src / gallium / drivers / freedreno / ir3 / ir3_compiler_nir.c
1 /* -*- mode: C; c-file-style: "k&r"; tab-width 4; indent-tabs-mode: t; -*- */
2
3 /*
4 * Copyright (C) 2015 Rob Clark <robclark@freedesktop.org>
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the "Software"),
8 * to deal in the Software without restriction, including without limitation
9 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
10 * and/or sell copies of the Software, and to permit persons to whom the
11 * Software is furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice (including the next
14 * paragraph) shall be included in all copies or substantial portions of the
15 * Software.
16 *
17 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
18 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
19 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
20 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
21 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
22 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
23 * SOFTWARE.
24 *
25 * Authors:
26 * Rob Clark <robclark@freedesktop.org>
27 */
28
29 #include <stdarg.h>
30
31 #include "pipe/p_state.h"
32 #include "util/u_string.h"
33 #include "util/u_memory.h"
34 #include "util/u_inlines.h"
35
36 #include "freedreno_util.h"
37
38 #include "ir3_compiler.h"
39 #include "ir3_shader.h"
40 #include "ir3_nir.h"
41
42 #include "instr-a3xx.h"
43 #include "ir3.h"
44
45
46 struct ir3_compile {
47 struct ir3_compiler *compiler;
48
49 struct nir_shader *s;
50
51 struct ir3 *ir;
52 struct ir3_shader_variant *so;
53
54 struct ir3_block *block; /* the current block */
55 struct ir3_block *in_block; /* block created for shader inputs */
56
57 nir_function_impl *impl;
58
59 /* For fragment shaders, from the hw perspective the only
60 * actual input is r0.xy position register passed to bary.f.
61 * But TGSI doesn't know that, it still declares things as
62 * IN[] registers. So we do all the input tracking normally
63 * and fix things up after compile_instructions()
64 *
65 * NOTE that frag_pos is the hardware position (possibly it
66 * is actually an index or tag or some such.. it is *not*
67 * values that can be directly used for gl_FragCoord..)
68 */
69 struct ir3_instruction *frag_pos, *frag_face, *frag_coord[4];
70
71 /* For vertex shaders, keep track of the system values sources */
72 struct ir3_instruction *vertex_id, *basevertex, *instance_id;
73
74 /* mapping from nir_register to defining instruction: */
75 struct hash_table *def_ht;
76
77 unsigned num_arrays;
78
79 /* a common pattern for indirect addressing is to request the
80 * same address register multiple times. To avoid generating
81 * duplicate instruction sequences (which our backend does not
82 * try to clean up, since that should be done as the NIR stage)
83 * we cache the address value generated for a given src value:
84 */
85 struct hash_table *addr_ht;
86
87 /* maps nir_block to ir3_block, mostly for the purposes of
88 * figuring out the blocks successors
89 */
90 struct hash_table *block_ht;
91
92 /* a4xx (at least patchlevel 0) cannot seem to flat-interpolate
93 * so we need to use ldlv.u32 to load the varying directly:
94 */
95 bool flat_bypass;
96
97 /* on a3xx, we need to add one to # of array levels:
98 */
99 bool levels_add_one;
100
101 /* on a3xx, we need to scale up integer coords for isaml based
102 * on LoD:
103 */
104 bool unminify_coords;
105
106 /* for looking up which system value is which */
107 unsigned sysval_semantics[8];
108
109 /* set if we encounter something we can't handle yet, so we
110 * can bail cleanly and fallback to TGSI compiler f/e
111 */
112 bool error;
113 };
114
115
116 static struct ir3_instruction * create_immed(struct ir3_block *block, uint32_t val);
117 static struct ir3_block * get_block(struct ir3_compile *ctx, nir_block *nblock);
118
119
120 static struct ir3_compile *
121 compile_init(struct ir3_compiler *compiler,
122 struct ir3_shader_variant *so)
123 {
124 struct ir3_compile *ctx = rzalloc(NULL, struct ir3_compile);
125
126 if (compiler->gpu_id >= 400) {
127 /* need special handling for "flat" */
128 ctx->flat_bypass = true;
129 ctx->levels_add_one = false;
130 ctx->unminify_coords = false;
131 } else {
132 /* no special handling for "flat" */
133 ctx->flat_bypass = false;
134 ctx->levels_add_one = true;
135 ctx->unminify_coords = true;
136 }
137
138 ctx->compiler = compiler;
139 ctx->ir = so->ir;
140 ctx->so = so;
141 ctx->def_ht = _mesa_hash_table_create(ctx,
142 _mesa_hash_pointer, _mesa_key_pointer_equal);
143 ctx->block_ht = _mesa_hash_table_create(ctx,
144 _mesa_hash_pointer, _mesa_key_pointer_equal);
145
146 /* TODO: maybe generate some sort of bitmask of what key
147 * lowers vs what shader has (ie. no need to lower
148 * texture clamp lowering if no texture sample instrs)..
149 * although should be done further up the stack to avoid
150 * creating duplicate variants..
151 */
152
153 if (ir3_key_lowers_nir(&so->key)) {
154 nir_shader *s = nir_shader_clone(ctx, so->shader->nir);
155 ctx->s = ir3_optimize_nir(so->shader, s, &so->key);
156 } else {
157 /* fast-path for shader key that lowers nothing in NIR: */
158 ctx->s = so->shader->nir;
159 }
160
161 if (fd_mesa_debug & FD_DBG_DISASM) {
162 DBG("dump nir%dv%d: type=%d, k={bp=%u,cts=%u,hp=%u}",
163 so->shader->id, so->id, so->type,
164 so->key.binning_pass, so->key.color_two_side,
165 so->key.half_precision);
166 nir_print_shader(ctx->s, stdout);
167 }
168
169 so->first_driver_param = so->first_immediate = ctx->s->num_uniforms;
170
171 /* Layout of constant registers:
172 *
173 * num_uniform * vec4 - user consts
174 * 4 * vec4 - UBO addresses
175 * if (vertex shader) {
176 * N * vec4 - driver params (IR3_DP_*)
177 * 1 * vec4 - stream-out addresses
178 * }
179 *
180 * TODO this could be made more dynamic, to at least skip sections
181 * that we don't need..
182 */
183
184 /* reserve 4 (vec4) slots for ubo base addresses: */
185 so->first_immediate += 4;
186
187 if (so->type == SHADER_VERTEX) {
188 /* driver params (see ir3_driver_param): */
189 so->first_immediate += IR3_DP_COUNT/4; /* convert to vec4 */
190 /* one (vec4) slot for stream-output base addresses: */
191 so->first_immediate++;
192 }
193
194 return ctx;
195 }
196
197 static void
198 compile_error(struct ir3_compile *ctx, const char *format, ...)
199 {
200 va_list ap;
201 va_start(ap, format);
202 _debug_vprintf(format, ap);
203 va_end(ap);
204 nir_print_shader(ctx->s, stdout);
205 ctx->error = true;
206 debug_assert(0);
207 }
208
209 #define compile_assert(ctx, cond) do { \
210 if (!(cond)) compile_error((ctx), "failed assert: "#cond"\n"); \
211 } while (0)
212
213 static void
214 compile_free(struct ir3_compile *ctx)
215 {
216 ralloc_free(ctx);
217 }
218
219 static void
220 declare_var(struct ir3_compile *ctx, nir_variable *var)
221 {
222 unsigned length = glsl_get_length(var->type) * 4; /* always vec4, at least with ttn */
223 struct ir3_array *arr = ralloc(ctx, struct ir3_array);
224 arr->id = ++ctx->num_arrays;
225 arr->length = length;
226 arr->var = var;
227 list_addtail(&arr->node, &ctx->ir->array_list);
228 }
229
230 static struct ir3_array *
231 get_var(struct ir3_compile *ctx, nir_variable *var)
232 {
233 list_for_each_entry (struct ir3_array, arr, &ctx->ir->array_list, node) {
234 if (arr->var == var)
235 return arr;
236 }
237 compile_error(ctx, "bogus var: %s\n", var->name);
238 return NULL;
239 }
240
241 /* allocate a n element value array (to be populated by caller) and
242 * insert in def_ht
243 */
244 static struct ir3_instruction **
245 __get_dst(struct ir3_compile *ctx, void *key, unsigned n)
246 {
247 struct ir3_instruction **value =
248 ralloc_array(ctx->def_ht, struct ir3_instruction *, n);
249 _mesa_hash_table_insert(ctx->def_ht, key, value);
250 return value;
251 }
252
253 static struct ir3_instruction **
254 get_dst(struct ir3_compile *ctx, nir_dest *dst, unsigned n)
255 {
256 compile_assert(ctx, dst->is_ssa);
257 if (dst->is_ssa) {
258 return __get_dst(ctx, &dst->ssa, n);
259 } else {
260 return __get_dst(ctx, dst->reg.reg, n);
261 }
262 }
263
264 static struct ir3_instruction **
265 get_dst_ssa(struct ir3_compile *ctx, nir_ssa_def *dst, unsigned n)
266 {
267 return __get_dst(ctx, dst, n);
268 }
269
270 static struct ir3_instruction **
271 get_src(struct ir3_compile *ctx, nir_src *src)
272 {
273 struct hash_entry *entry;
274 compile_assert(ctx, src->is_ssa);
275 if (src->is_ssa) {
276 entry = _mesa_hash_table_search(ctx->def_ht, src->ssa);
277 } else {
278 entry = _mesa_hash_table_search(ctx->def_ht, src->reg.reg);
279 }
280 compile_assert(ctx, entry);
281 return entry->data;
282 }
283
284 static struct ir3_instruction *
285 create_immed(struct ir3_block *block, uint32_t val)
286 {
287 struct ir3_instruction *mov;
288
289 mov = ir3_instr_create(block, 1, 0);
290 mov->cat1.src_type = TYPE_U32;
291 mov->cat1.dst_type = TYPE_U32;
292 ir3_reg_create(mov, 0, 0);
293 ir3_reg_create(mov, 0, IR3_REG_IMMED)->uim_val = val;
294
295 return mov;
296 }
297
298 static struct ir3_instruction *
299 create_addr(struct ir3_block *block, struct ir3_instruction *src)
300 {
301 struct ir3_instruction *instr, *immed;
302
303 /* TODO in at least some cases, the backend could probably be
304 * made clever enough to propagate IR3_REG_HALF..
305 */
306 instr = ir3_COV(block, src, TYPE_U32, TYPE_S16);
307 instr->regs[0]->flags |= IR3_REG_HALF;
308
309 immed = create_immed(block, 2);
310 immed->regs[0]->flags |= IR3_REG_HALF;
311
312 instr = ir3_SHL_B(block, instr, 0, immed, 0);
313 instr->regs[0]->flags |= IR3_REG_HALF;
314 instr->regs[1]->flags |= IR3_REG_HALF;
315
316 instr = ir3_MOV(block, instr, TYPE_S16);
317 instr->regs[0]->num = regid(REG_A0, 0);
318 instr->regs[0]->flags |= IR3_REG_HALF;
319 instr->regs[1]->flags |= IR3_REG_HALF;
320
321 return instr;
322 }
323
324 /* caches addr values to avoid generating multiple cov/shl/mova
325 * sequences for each use of a given NIR level src as address
326 */
327 static struct ir3_instruction *
328 get_addr(struct ir3_compile *ctx, struct ir3_instruction *src)
329 {
330 struct ir3_instruction *addr;
331
332 if (!ctx->addr_ht) {
333 ctx->addr_ht = _mesa_hash_table_create(ctx,
334 _mesa_hash_pointer, _mesa_key_pointer_equal);
335 } else {
336 struct hash_entry *entry;
337 entry = _mesa_hash_table_search(ctx->addr_ht, src);
338 if (entry)
339 return entry->data;
340 }
341
342 addr = create_addr(ctx->block, src);
343 _mesa_hash_table_insert(ctx->addr_ht, src, addr);
344
345 return addr;
346 }
347
348 static struct ir3_instruction *
349 get_predicate(struct ir3_compile *ctx, struct ir3_instruction *src)
350 {
351 struct ir3_block *b = ctx->block;
352 struct ir3_instruction *cond;
353
354 /* NOTE: only cmps.*.* can write p0.x: */
355 cond = ir3_CMPS_S(b, src, 0, create_immed(b, 0), 0);
356 cond->cat2.condition = IR3_COND_NE;
357
358 /* condition always goes in predicate register: */
359 cond->regs[0]->num = regid(REG_P0, 0);
360
361 return cond;
362 }
363
364 static struct ir3_instruction *
365 create_uniform(struct ir3_compile *ctx, unsigned n)
366 {
367 struct ir3_instruction *mov;
368
369 mov = ir3_instr_create(ctx->block, 1, 0);
370 /* TODO get types right? */
371 mov->cat1.src_type = TYPE_F32;
372 mov->cat1.dst_type = TYPE_F32;
373 ir3_reg_create(mov, 0, 0);
374 ir3_reg_create(mov, n, IR3_REG_CONST);
375
376 return mov;
377 }
378
379 static struct ir3_instruction *
380 create_uniform_indirect(struct ir3_compile *ctx, int n,
381 struct ir3_instruction *address)
382 {
383 struct ir3_instruction *mov;
384
385 mov = ir3_instr_create(ctx->block, 1, 0);
386 mov->cat1.src_type = TYPE_U32;
387 mov->cat1.dst_type = TYPE_U32;
388 ir3_reg_create(mov, 0, 0);
389 ir3_reg_create(mov, 0, IR3_REG_CONST | IR3_REG_RELATIV)->array.offset = n;
390
391 ir3_instr_set_address(mov, address);
392
393 return mov;
394 }
395
396 static struct ir3_instruction *
397 create_collect(struct ir3_block *block, struct ir3_instruction **arr,
398 unsigned arrsz)
399 {
400 struct ir3_instruction *collect;
401
402 if (arrsz == 0)
403 return NULL;
404
405 collect = ir3_instr_create2(block, -1, OPC_META_FI, 1 + arrsz);
406 ir3_reg_create(collect, 0, 0); /* dst */
407 for (unsigned i = 0; i < arrsz; i++)
408 ir3_reg_create(collect, 0, IR3_REG_SSA)->instr = arr[i];
409
410 return collect;
411 }
412
413 static struct ir3_instruction *
414 create_indirect_load(struct ir3_compile *ctx, unsigned arrsz, int n,
415 struct ir3_instruction *address, struct ir3_instruction *collect)
416 {
417 struct ir3_block *block = ctx->block;
418 struct ir3_instruction *mov;
419 struct ir3_register *src;
420
421 mov = ir3_instr_create(block, 1, 0);
422 mov->cat1.src_type = TYPE_U32;
423 mov->cat1.dst_type = TYPE_U32;
424 ir3_reg_create(mov, 0, 0);
425 src = ir3_reg_create(mov, 0, IR3_REG_SSA | IR3_REG_RELATIV);
426 src->instr = collect;
427 src->size = arrsz;
428 src->array.offset = n;
429
430 ir3_instr_set_address(mov, address);
431
432 return mov;
433 }
434
435 /* relative (indirect) if address!=NULL */
436 static struct ir3_instruction *
437 create_var_load(struct ir3_compile *ctx, struct ir3_array *arr, int n,
438 struct ir3_instruction *address)
439 {
440 struct ir3_block *block = ctx->block;
441 struct ir3_instruction *mov;
442 struct ir3_register *src;
443
444 mov = ir3_instr_create(block, 1, 0);
445 mov->cat1.src_type = TYPE_U32;
446 mov->cat1.dst_type = TYPE_U32;
447 ir3_reg_create(mov, 0, 0);
448 src = ir3_reg_create(mov, 0, IR3_REG_ARRAY |
449 COND(address, IR3_REG_RELATIV));
450 src->instr = arr->last_write;
451 src->size = arr->length;
452 src->array.id = arr->id;
453 src->array.offset = n;
454
455 if (address)
456 ir3_instr_set_address(mov, address);
457
458 arr->last_access = mov;
459
460 return mov;
461 }
462
463 /* relative (indirect) if address!=NULL */
464 static struct ir3_instruction *
465 create_var_store(struct ir3_compile *ctx, struct ir3_array *arr, int n,
466 struct ir3_instruction *src, struct ir3_instruction *address)
467 {
468 struct ir3_block *block = ctx->block;
469 struct ir3_instruction *mov;
470 struct ir3_register *dst;
471
472 mov = ir3_instr_create(block, 1, 0);
473 mov->cat1.src_type = TYPE_U32;
474 mov->cat1.dst_type = TYPE_U32;
475 dst = ir3_reg_create(mov, 0, IR3_REG_ARRAY |
476 COND(address, IR3_REG_RELATIV));
477 dst->instr = arr->last_access;
478 dst->size = arr->length;
479 dst->array.id = arr->id;
480 dst->array.offset = n;
481 ir3_reg_create(mov, 0, IR3_REG_SSA)->instr = src;
482
483 ir3_instr_set_address(mov, address);
484
485 arr->last_write = arr->last_access = mov;
486
487 return mov;
488 }
489
490 static struct ir3_instruction *
491 create_input(struct ir3_block *block, unsigned n)
492 {
493 struct ir3_instruction *in;
494
495 in = ir3_instr_create(block, -1, OPC_META_INPUT);
496 in->inout.block = block;
497 ir3_reg_create(in, n, 0);
498
499 return in;
500 }
501
502 static struct ir3_instruction *
503 create_frag_input(struct ir3_compile *ctx, bool use_ldlv)
504 {
505 struct ir3_block *block = ctx->block;
506 struct ir3_instruction *instr;
507 /* actual inloc is assigned and fixed up later: */
508 struct ir3_instruction *inloc = create_immed(block, 0);
509
510 if (use_ldlv) {
511 instr = ir3_LDLV(block, inloc, 0, create_immed(block, 1), 0);
512 instr->cat6.type = TYPE_U32;
513 instr->cat6.iim_val = 1;
514 } else {
515 instr = ir3_BARY_F(block, inloc, 0, ctx->frag_pos, 0);
516 instr->regs[2]->wrmask = 0x3;
517 }
518
519 return instr;
520 }
521
522 static struct ir3_instruction *
523 create_frag_coord(struct ir3_compile *ctx, unsigned comp)
524 {
525 struct ir3_block *block = ctx->block;
526 struct ir3_instruction *instr;
527
528 compile_assert(ctx, !ctx->frag_coord[comp]);
529
530 ctx->frag_coord[comp] = create_input(ctx->block, 0);
531
532 switch (comp) {
533 case 0: /* .x */
534 case 1: /* .y */
535 /* for frag_coord, we get unsigned values.. we need
536 * to subtract (integer) 8 and divide by 16 (right-
537 * shift by 4) then convert to float:
538 *
539 * sub.s tmp, src, 8
540 * shr.b tmp, tmp, 4
541 * mov.u32f32 dst, tmp
542 *
543 */
544 instr = ir3_SUB_S(block, ctx->frag_coord[comp], 0,
545 create_immed(block, 8), 0);
546 instr = ir3_SHR_B(block, instr, 0,
547 create_immed(block, 4), 0);
548 instr = ir3_COV(block, instr, TYPE_U32, TYPE_F32);
549
550 return instr;
551 case 2: /* .z */
552 case 3: /* .w */
553 default:
554 /* seems that we can use these as-is: */
555 return ctx->frag_coord[comp];
556 }
557 }
558
559 /* NOTE: this creates the "TGSI" style fragface (ie. input slot
560 * VARYING_SLOT_FACE). For NIR style nir_intrinsic_load_front_face
561 * we can just use the value from hw directly (since it is boolean)
562 */
563 static struct ir3_instruction *
564 create_frag_face(struct ir3_compile *ctx, unsigned comp)
565 {
566 struct ir3_block *block = ctx->block;
567 struct ir3_instruction *instr;
568
569 switch (comp) {
570 case 0: /* .x */
571 compile_assert(ctx, !ctx->frag_face);
572
573 ctx->frag_face = create_input(block, 0);
574 ctx->frag_face->regs[0]->flags |= IR3_REG_HALF;
575
576 /* for faceness, we always get -1 or 0 (int).. but TGSI expects
577 * positive vs negative float.. and piglit further seems to
578 * expect -1.0 or 1.0:
579 *
580 * mul.s tmp, hr0.x, 2
581 * add.s tmp, tmp, 1
582 * mov.s32f32, dst, tmp
583 *
584 */
585 instr = ir3_MUL_S(block, ctx->frag_face, 0,
586 create_immed(block, 2), 0);
587 instr = ir3_ADD_S(block, instr, 0,
588 create_immed(block, 1), 0);
589 instr = ir3_COV(block, instr, TYPE_S32, TYPE_F32);
590
591 return instr;
592 case 1: /* .y */
593 case 2: /* .z */
594 return create_immed(block, fui(0.0));
595 default:
596 case 3: /* .w */
597 return create_immed(block, fui(1.0));
598 }
599 }
600
601 static struct ir3_instruction *
602 create_driver_param(struct ir3_compile *ctx, enum ir3_driver_param dp)
603 {
604 /* first four vec4 sysval's reserved for UBOs: */
605 /* NOTE: dp is in scalar, but there can be >4 dp components: */
606 unsigned n = ctx->so->first_driver_param + IR3_DRIVER_PARAM_OFF;
607 unsigned r = regid(n + dp / 4, dp % 4);
608 return create_uniform(ctx, r);
609 }
610
611 /* helper for instructions that produce multiple consecutive scalar
612 * outputs which need to have a split/fanout meta instruction inserted
613 */
614 static void
615 split_dest(struct ir3_block *block, struct ir3_instruction **dst,
616 struct ir3_instruction *src, unsigned n)
617 {
618 struct ir3_instruction *prev = NULL;
619 for (int i = 0, j = 0; i < n; i++) {
620 struct ir3_instruction *split =
621 ir3_instr_create(block, -1, OPC_META_FO);
622 ir3_reg_create(split, 0, IR3_REG_SSA);
623 ir3_reg_create(split, 0, IR3_REG_SSA)->instr = src;
624 split->fo.off = i;
625
626 if (prev) {
627 split->cp.left = prev;
628 split->cp.left_cnt++;
629 prev->cp.right = split;
630 prev->cp.right_cnt++;
631 }
632 prev = split;
633
634 if (src->regs[0]->wrmask & (1 << i))
635 dst[j++] = split;
636 }
637 }
638
639 /*
640 * Adreno uses uint rather than having dedicated bool type,
641 * which (potentially) requires some conversion, in particular
642 * when using output of an bool instr to int input, or visa
643 * versa.
644 *
645 * | Adreno | NIR |
646 * -------+---------+-------+-
647 * true | 1 | ~0 |
648 * false | 0 | 0 |
649 *
650 * To convert from an adreno bool (uint) to nir, use:
651 *
652 * absneg.s dst, (neg)src
653 *
654 * To convert back in the other direction:
655 *
656 * absneg.s dst, (abs)arc
657 *
658 * The CP step can clean up the absneg.s that cancel each other
659 * out, and with a slight bit of extra cleverness (to recognize
660 * the instructions which produce either a 0 or 1) can eliminate
661 * the absneg.s's completely when an instruction that wants
662 * 0/1 consumes the result. For example, when a nir 'bcsel'
663 * consumes the result of 'feq'. So we should be able to get by
664 * without a boolean resolve step, and without incuring any
665 * extra penalty in instruction count.
666 */
667
668 /* NIR bool -> native (adreno): */
669 static struct ir3_instruction *
670 ir3_b2n(struct ir3_block *block, struct ir3_instruction *instr)
671 {
672 return ir3_ABSNEG_S(block, instr, IR3_REG_SABS);
673 }
674
675 /* native (adreno) -> NIR bool: */
676 static struct ir3_instruction *
677 ir3_n2b(struct ir3_block *block, struct ir3_instruction *instr)
678 {
679 return ir3_ABSNEG_S(block, instr, IR3_REG_SNEG);
680 }
681
682 /*
683 * alu/sfu instructions:
684 */
685
686 static void
687 emit_alu(struct ir3_compile *ctx, nir_alu_instr *alu)
688 {
689 const nir_op_info *info = &nir_op_infos[alu->op];
690 struct ir3_instruction **dst, *src[info->num_inputs];
691 struct ir3_block *b = ctx->block;
692
693 dst = get_dst(ctx, &alu->dest.dest, MAX2(info->output_size, 1));
694
695 /* Vectors are special in that they have non-scalarized writemasks,
696 * and just take the first swizzle channel for each argument in
697 * order into each writemask channel.
698 */
699 if ((alu->op == nir_op_vec2) ||
700 (alu->op == nir_op_vec3) ||
701 (alu->op == nir_op_vec4)) {
702
703 for (int i = 0; i < info->num_inputs; i++) {
704 nir_alu_src *asrc = &alu->src[i];
705
706 compile_assert(ctx, !asrc->abs);
707 compile_assert(ctx, !asrc->negate);
708
709 src[i] = get_src(ctx, &asrc->src)[asrc->swizzle[0]];
710 if (!src[i])
711 src[i] = create_immed(ctx->block, 0);
712 dst[i] = ir3_MOV(b, src[i], TYPE_U32);
713 }
714
715 return;
716 }
717
718 /* General case: We can just grab the one used channel per src. */
719 for (int i = 0; i < info->num_inputs; i++) {
720 unsigned chan = ffs(alu->dest.write_mask) - 1;
721 nir_alu_src *asrc = &alu->src[i];
722
723 compile_assert(ctx, !asrc->abs);
724 compile_assert(ctx, !asrc->negate);
725
726 src[i] = get_src(ctx, &asrc->src)[asrc->swizzle[chan]];
727
728 compile_assert(ctx, src[i]);
729 }
730
731 switch (alu->op) {
732 case nir_op_f2i:
733 dst[0] = ir3_COV(b, src[0], TYPE_F32, TYPE_S32);
734 break;
735 case nir_op_f2u:
736 dst[0] = ir3_COV(b, src[0], TYPE_F32, TYPE_U32);
737 break;
738 case nir_op_i2f:
739 dst[0] = ir3_COV(b, src[0], TYPE_S32, TYPE_F32);
740 break;
741 case nir_op_u2f:
742 dst[0] = ir3_COV(b, src[0], TYPE_U32, TYPE_F32);
743 break;
744 case nir_op_imov:
745 dst[0] = ir3_MOV(b, src[0], TYPE_S32);
746 break;
747 case nir_op_fmov:
748 dst[0] = ir3_MOV(b, src[0], TYPE_F32);
749 break;
750 case nir_op_f2b:
751 dst[0] = ir3_CMPS_F(b, src[0], 0, create_immed(b, fui(0.0)), 0);
752 dst[0]->cat2.condition = IR3_COND_NE;
753 dst[0] = ir3_n2b(b, dst[0]);
754 break;
755 case nir_op_b2f:
756 dst[0] = ir3_COV(b, ir3_b2n(b, src[0]), TYPE_U32, TYPE_F32);
757 break;
758 case nir_op_b2i:
759 dst[0] = ir3_b2n(b, src[0]);
760 break;
761 case nir_op_i2b:
762 dst[0] = ir3_CMPS_S(b, src[0], 0, create_immed(b, 0), 0);
763 dst[0]->cat2.condition = IR3_COND_NE;
764 dst[0] = ir3_n2b(b, dst[0]);
765 break;
766
767 case nir_op_fneg:
768 dst[0] = ir3_ABSNEG_F(b, src[0], IR3_REG_FNEG);
769 break;
770 case nir_op_fabs:
771 dst[0] = ir3_ABSNEG_F(b, src[0], IR3_REG_FABS);
772 break;
773 case nir_op_fmax:
774 dst[0] = ir3_MAX_F(b, src[0], 0, src[1], 0);
775 break;
776 case nir_op_fmin:
777 dst[0] = ir3_MIN_F(b, src[0], 0, src[1], 0);
778 break;
779 case nir_op_fmul:
780 dst[0] = ir3_MUL_F(b, src[0], 0, src[1], 0);
781 break;
782 case nir_op_fadd:
783 dst[0] = ir3_ADD_F(b, src[0], 0, src[1], 0);
784 break;
785 case nir_op_fsub:
786 dst[0] = ir3_ADD_F(b, src[0], 0, src[1], IR3_REG_FNEG);
787 break;
788 case nir_op_ffma:
789 dst[0] = ir3_MAD_F32(b, src[0], 0, src[1], 0, src[2], 0);
790 break;
791 case nir_op_fddx:
792 dst[0] = ir3_DSX(b, src[0], 0);
793 dst[0]->cat5.type = TYPE_F32;
794 break;
795 case nir_op_fddy:
796 dst[0] = ir3_DSY(b, src[0], 0);
797 dst[0]->cat5.type = TYPE_F32;
798 break;
799 break;
800 case nir_op_flt:
801 dst[0] = ir3_CMPS_F(b, src[0], 0, src[1], 0);
802 dst[0]->cat2.condition = IR3_COND_LT;
803 dst[0] = ir3_n2b(b, dst[0]);
804 break;
805 case nir_op_fge:
806 dst[0] = ir3_CMPS_F(b, src[0], 0, src[1], 0);
807 dst[0]->cat2.condition = IR3_COND_GE;
808 dst[0] = ir3_n2b(b, dst[0]);
809 break;
810 case nir_op_feq:
811 dst[0] = ir3_CMPS_F(b, src[0], 0, src[1], 0);
812 dst[0]->cat2.condition = IR3_COND_EQ;
813 dst[0] = ir3_n2b(b, dst[0]);
814 break;
815 case nir_op_fne:
816 dst[0] = ir3_CMPS_F(b, src[0], 0, src[1], 0);
817 dst[0]->cat2.condition = IR3_COND_NE;
818 dst[0] = ir3_n2b(b, dst[0]);
819 break;
820 case nir_op_fceil:
821 dst[0] = ir3_CEIL_F(b, src[0], 0);
822 break;
823 case nir_op_ffloor:
824 dst[0] = ir3_FLOOR_F(b, src[0], 0);
825 break;
826 case nir_op_ftrunc:
827 dst[0] = ir3_TRUNC_F(b, src[0], 0);
828 break;
829 case nir_op_fround_even:
830 dst[0] = ir3_RNDNE_F(b, src[0], 0);
831 break;
832 case nir_op_fsign:
833 dst[0] = ir3_SIGN_F(b, src[0], 0);
834 break;
835
836 case nir_op_fsin:
837 dst[0] = ir3_SIN(b, src[0], 0);
838 break;
839 case nir_op_fcos:
840 dst[0] = ir3_COS(b, src[0], 0);
841 break;
842 case nir_op_frsq:
843 dst[0] = ir3_RSQ(b, src[0], 0);
844 break;
845 case nir_op_frcp:
846 dst[0] = ir3_RCP(b, src[0], 0);
847 break;
848 case nir_op_flog2:
849 dst[0] = ir3_LOG2(b, src[0], 0);
850 break;
851 case nir_op_fexp2:
852 dst[0] = ir3_EXP2(b, src[0], 0);
853 break;
854 case nir_op_fsqrt:
855 dst[0] = ir3_SQRT(b, src[0], 0);
856 break;
857
858 case nir_op_iabs:
859 dst[0] = ir3_ABSNEG_S(b, src[0], IR3_REG_SABS);
860 break;
861 case nir_op_iadd:
862 dst[0] = ir3_ADD_U(b, src[0], 0, src[1], 0);
863 break;
864 case nir_op_iand:
865 dst[0] = ir3_AND_B(b, src[0], 0, src[1], 0);
866 break;
867 case nir_op_imax:
868 dst[0] = ir3_MAX_S(b, src[0], 0, src[1], 0);
869 break;
870 case nir_op_umax:
871 dst[0] = ir3_MAX_U(b, src[0], 0, src[1], 0);
872 break;
873 case nir_op_imin:
874 dst[0] = ir3_MIN_S(b, src[0], 0, src[1], 0);
875 break;
876 case nir_op_umin:
877 dst[0] = ir3_MIN_U(b, src[0], 0, src[1], 0);
878 break;
879 case nir_op_imul:
880 /*
881 * dst = (al * bl) + (ah * bl << 16) + (al * bh << 16)
882 * mull.u tmp0, a, b ; mul low, i.e. al * bl
883 * madsh.m16 tmp1, a, b, tmp0 ; mul-add shift high mix, i.e. ah * bl << 16
884 * madsh.m16 dst, b, a, tmp1 ; i.e. al * bh << 16
885 */
886 dst[0] = ir3_MADSH_M16(b, src[1], 0, src[0], 0,
887 ir3_MADSH_M16(b, src[0], 0, src[1], 0,
888 ir3_MULL_U(b, src[0], 0, src[1], 0), 0), 0);
889 break;
890 case nir_op_ineg:
891 dst[0] = ir3_ABSNEG_S(b, src[0], IR3_REG_SNEG);
892 break;
893 case nir_op_inot:
894 dst[0] = ir3_NOT_B(b, src[0], 0);
895 break;
896 case nir_op_ior:
897 dst[0] = ir3_OR_B(b, src[0], 0, src[1], 0);
898 break;
899 case nir_op_ishl:
900 dst[0] = ir3_SHL_B(b, src[0], 0, src[1], 0);
901 break;
902 case nir_op_ishr:
903 dst[0] = ir3_ASHR_B(b, src[0], 0, src[1], 0);
904 break;
905 case nir_op_isign: {
906 /* maybe this would be sane to lower in nir.. */
907 struct ir3_instruction *neg, *pos;
908
909 neg = ir3_CMPS_S(b, src[0], 0, create_immed(b, 0), 0);
910 neg->cat2.condition = IR3_COND_LT;
911
912 pos = ir3_CMPS_S(b, src[0], 0, create_immed(b, 0), 0);
913 pos->cat2.condition = IR3_COND_GT;
914
915 dst[0] = ir3_SUB_U(b, pos, 0, neg, 0);
916
917 break;
918 }
919 case nir_op_isub:
920 dst[0] = ir3_SUB_U(b, src[0], 0, src[1], 0);
921 break;
922 case nir_op_ixor:
923 dst[0] = ir3_XOR_B(b, src[0], 0, src[1], 0);
924 break;
925 case nir_op_ushr:
926 dst[0] = ir3_SHR_B(b, src[0], 0, src[1], 0);
927 break;
928 case nir_op_ilt:
929 dst[0] = ir3_CMPS_S(b, src[0], 0, src[1], 0);
930 dst[0]->cat2.condition = IR3_COND_LT;
931 dst[0] = ir3_n2b(b, dst[0]);
932 break;
933 case nir_op_ige:
934 dst[0] = ir3_CMPS_S(b, src[0], 0, src[1], 0);
935 dst[0]->cat2.condition = IR3_COND_GE;
936 dst[0] = ir3_n2b(b, dst[0]);
937 break;
938 case nir_op_ieq:
939 dst[0] = ir3_CMPS_S(b, src[0], 0, src[1], 0);
940 dst[0]->cat2.condition = IR3_COND_EQ;
941 dst[0] = ir3_n2b(b, dst[0]);
942 break;
943 case nir_op_ine:
944 dst[0] = ir3_CMPS_S(b, src[0], 0, src[1], 0);
945 dst[0]->cat2.condition = IR3_COND_NE;
946 dst[0] = ir3_n2b(b, dst[0]);
947 break;
948 case nir_op_ult:
949 dst[0] = ir3_CMPS_U(b, src[0], 0, src[1], 0);
950 dst[0]->cat2.condition = IR3_COND_LT;
951 dst[0] = ir3_n2b(b, dst[0]);
952 break;
953 case nir_op_uge:
954 dst[0] = ir3_CMPS_U(b, src[0], 0, src[1], 0);
955 dst[0]->cat2.condition = IR3_COND_GE;
956 dst[0] = ir3_n2b(b, dst[0]);
957 break;
958
959 case nir_op_bcsel:
960 dst[0] = ir3_SEL_B32(b, src[1], 0, ir3_b2n(b, src[0]), 0, src[2], 0);
961 break;
962
963 case nir_op_bit_count:
964 dst[0] = ir3_CBITS_B(b, src[0], 0);
965 break;
966 case nir_op_ifind_msb: {
967 struct ir3_instruction *cmp;
968 dst[0] = ir3_CLZ_S(b, src[0], 0);
969 cmp = ir3_CMPS_S(b, dst[0], 0, create_immed(b, 0), 0);
970 cmp->cat2.condition = IR3_COND_GE;
971 dst[0] = ir3_SEL_B32(b,
972 ir3_SUB_U(b, create_immed(b, 31), 0, dst[0], 0), 0,
973 cmp, 0, dst[0], 0);
974 break;
975 }
976 case nir_op_ufind_msb:
977 dst[0] = ir3_CLZ_B(b, src[0], 0);
978 dst[0] = ir3_SEL_B32(b,
979 ir3_SUB_U(b, create_immed(b, 31), 0, dst[0], 0), 0,
980 src[0], 0, dst[0], 0);
981 break;
982 case nir_op_find_lsb:
983 dst[0] = ir3_BFREV_B(b, src[0], 0);
984 dst[0] = ir3_CLZ_B(b, dst[0], 0);
985 break;
986 case nir_op_bitfield_reverse:
987 dst[0] = ir3_BFREV_B(b, src[0], 0);
988 break;
989
990 default:
991 compile_error(ctx, "Unhandled ALU op: %s\n",
992 nir_op_infos[alu->op].name);
993 break;
994 }
995 }
996
997 /* handles direct/indirect UBO reads: */
998 static void
999 emit_intrinsic_load_ubo(struct ir3_compile *ctx, nir_intrinsic_instr *intr,
1000 struct ir3_instruction **dst)
1001 {
1002 struct ir3_block *b = ctx->block;
1003 struct ir3_instruction *addr, *src0, *src1;
1004 nir_const_value *const_offset;
1005 /* UBO addresses are the first driver params: */
1006 unsigned ubo = regid(ctx->so->first_driver_param + IR3_UBOS_OFF, 0);
1007 int off = intr->const_index[0];
1008
1009 /* First src is ubo index, which could either be an immed or not: */
1010 src0 = get_src(ctx, &intr->src[0])[0];
1011 if (is_same_type_mov(src0) &&
1012 (src0->regs[1]->flags & IR3_REG_IMMED)) {
1013 addr = create_uniform(ctx, ubo + src0->regs[1]->iim_val);
1014 } else {
1015 addr = create_uniform_indirect(ctx, ubo, get_addr(ctx, src0));
1016 }
1017
1018 const_offset = nir_src_as_const_value(intr->src[1]);
1019 if (const_offset) {
1020 off += const_offset->u[0];
1021 } else {
1022 /* For load_ubo_indirect, second src is indirect offset: */
1023 src1 = get_src(ctx, &intr->src[1])[0];
1024
1025 /* and add offset to addr: */
1026 addr = ir3_ADD_S(b, addr, 0, src1, 0);
1027 }
1028
1029 /* if offset is to large to encode in the ldg, split it out: */
1030 if ((off + (intr->num_components * 4)) > 1024) {
1031 /* split out the minimal amount to improve the odds that
1032 * cp can fit the immediate in the add.s instruction:
1033 */
1034 unsigned off2 = off + (intr->num_components * 4) - 1024;
1035 addr = ir3_ADD_S(b, addr, 0, create_immed(b, off2), 0);
1036 off -= off2;
1037 }
1038
1039 for (int i = 0; i < intr->num_components; i++) {
1040 struct ir3_instruction *load =
1041 ir3_LDG(b, addr, 0, create_immed(b, 1), 0);
1042 load->cat6.type = TYPE_U32;
1043 load->cat6.src_offset = off + i * 4; /* byte offset */
1044 dst[i] = load;
1045 }
1046 }
1047
1048 /* handles array reads: */
1049 static void
1050 emit_intrinsic_load_var(struct ir3_compile *ctx, nir_intrinsic_instr *intr,
1051 struct ir3_instruction **dst)
1052 {
1053 nir_deref_var *dvar = intr->variables[0];
1054 nir_deref_array *darr = nir_deref_as_array(dvar->deref.child);
1055 struct ir3_array *arr = get_var(ctx, dvar->var);
1056
1057 compile_assert(ctx, dvar->deref.child &&
1058 (dvar->deref.child->deref_type == nir_deref_type_array));
1059
1060 switch (darr->deref_array_type) {
1061 case nir_deref_array_type_direct:
1062 /* direct access does not require anything special: */
1063 for (int i = 0; i < intr->num_components; i++) {
1064 unsigned n = darr->base_offset * 4 + i;
1065 compile_assert(ctx, n < arr->length);
1066 dst[i] = create_var_load(ctx, arr, n, NULL);
1067 }
1068 break;
1069 case nir_deref_array_type_indirect: {
1070 /* for indirect, we need to collect all the array elements: */
1071 struct ir3_instruction *addr =
1072 get_addr(ctx, get_src(ctx, &darr->indirect)[0]);
1073 for (int i = 0; i < intr->num_components; i++) {
1074 unsigned n = darr->base_offset * 4 + i;
1075 compile_assert(ctx, n < arr->length);
1076 dst[i] = create_var_load(ctx, arr, n, addr);
1077 }
1078 break;
1079 }
1080 default:
1081 compile_error(ctx, "Unhandled load deref type: %u\n",
1082 darr->deref_array_type);
1083 break;
1084 }
1085 }
1086
1087 /* handles array writes: */
1088 static void
1089 emit_intrinsic_store_var(struct ir3_compile *ctx, nir_intrinsic_instr *intr)
1090 {
1091 nir_deref_var *dvar = intr->variables[0];
1092 nir_deref_array *darr = nir_deref_as_array(dvar->deref.child);
1093 struct ir3_array *arr = get_var(ctx, dvar->var);
1094 struct ir3_instruction *addr, **src;
1095 unsigned wrmask = intr->const_index[0];
1096
1097 compile_assert(ctx, dvar->deref.child &&
1098 (dvar->deref.child->deref_type == nir_deref_type_array));
1099
1100 src = get_src(ctx, &intr->src[0]);
1101
1102 switch (darr->deref_array_type) {
1103 case nir_deref_array_type_direct:
1104 addr = NULL;
1105 break;
1106 case nir_deref_array_type_indirect:
1107 addr = get_addr(ctx, get_src(ctx, &darr->indirect)[0]);
1108 break;
1109 default:
1110 compile_error(ctx, "Unhandled store deref type: %u\n",
1111 darr->deref_array_type);
1112 break;
1113 }
1114
1115 for (int i = 0; i < intr->num_components; i++) {
1116 if (!(wrmask & (1 << i)))
1117 continue;
1118 unsigned n = darr->base_offset * 4 + i;
1119 compile_assert(ctx, n < arr->length);
1120 create_var_store(ctx, arr, n, src[i], addr);
1121 }
1122 }
1123
1124 static void add_sysval_input(struct ir3_compile *ctx, gl_system_value slot,
1125 struct ir3_instruction *instr)
1126 {
1127 struct ir3_shader_variant *so = ctx->so;
1128 unsigned r = regid(so->inputs_count, 0);
1129 unsigned n = so->inputs_count++;
1130
1131 so->inputs[n].sysval = true;
1132 so->inputs[n].slot = slot;
1133 so->inputs[n].compmask = 1;
1134 so->inputs[n].regid = r;
1135 so->inputs[n].interpolate = INTERP_QUALIFIER_FLAT;
1136 so->total_in++;
1137
1138 ctx->ir->ninputs = MAX2(ctx->ir->ninputs, r + 1);
1139 ctx->ir->inputs[r] = instr;
1140 }
1141
1142 static void
1143 emit_intrinsic(struct ir3_compile *ctx, nir_intrinsic_instr *intr)
1144 {
1145 const nir_intrinsic_info *info = &nir_intrinsic_infos[intr->intrinsic];
1146 struct ir3_instruction **dst, **src;
1147 struct ir3_block *b = ctx->block;
1148 int idx = intr->const_index[0];
1149 nir_const_value *const_offset;
1150
1151 if (info->has_dest) {
1152 dst = get_dst(ctx, &intr->dest, intr->num_components);
1153 } else {
1154 dst = NULL;
1155 }
1156
1157 switch (intr->intrinsic) {
1158 case nir_intrinsic_load_uniform:
1159 const_offset = nir_src_as_const_value(intr->src[0]);
1160 if (const_offset) {
1161 idx += const_offset->u[0];
1162 for (int i = 0; i < intr->num_components; i++) {
1163 unsigned n = idx * 4 + i;
1164 dst[i] = create_uniform(ctx, n);
1165 }
1166 } else {
1167 src = get_src(ctx, &intr->src[0]);
1168 for (int i = 0; i < intr->num_components; i++) {
1169 int n = idx * 4 + i;
1170 dst[i] = create_uniform_indirect(ctx, n,
1171 get_addr(ctx, src[0]));
1172 }
1173 /* NOTE: if relative addressing is used, we set
1174 * constlen in the compiler (to worst-case value)
1175 * since we don't know in the assembler what the max
1176 * addr reg value can be:
1177 */
1178 ctx->so->constlen = ctx->s->num_uniforms;
1179 }
1180 break;
1181 case nir_intrinsic_load_ubo:
1182 emit_intrinsic_load_ubo(ctx, intr, dst);
1183 break;
1184 case nir_intrinsic_load_input:
1185 const_offset = nir_src_as_const_value(intr->src[0]);
1186 if (const_offset) {
1187 idx += const_offset->u[0];
1188 for (int i = 0; i < intr->num_components; i++) {
1189 unsigned n = idx * 4 + i;
1190 dst[i] = ctx->ir->inputs[n];
1191 }
1192 } else {
1193 src = get_src(ctx, &intr->src[0]);
1194 struct ir3_instruction *collect =
1195 create_collect(b, ctx->ir->inputs, ctx->ir->ninputs);
1196 struct ir3_instruction *addr = get_addr(ctx, src[0]);
1197 for (int i = 0; i < intr->num_components; i++) {
1198 unsigned n = idx * 4 + i;
1199 dst[i] = create_indirect_load(ctx, ctx->ir->ninputs,
1200 n, addr, collect);
1201 }
1202 }
1203 break;
1204 case nir_intrinsic_load_var:
1205 emit_intrinsic_load_var(ctx, intr, dst);
1206 break;
1207 case nir_intrinsic_store_var:
1208 emit_intrinsic_store_var(ctx, intr);
1209 break;
1210 case nir_intrinsic_store_output:
1211 const_offset = nir_src_as_const_value(intr->src[1]);
1212 compile_assert(ctx, const_offset != NULL);
1213 idx += const_offset->u[0];
1214
1215 src = get_src(ctx, &intr->src[0]);
1216 for (int i = 0; i < intr->num_components; i++) {
1217 unsigned n = idx * 4 + i;
1218 ctx->ir->outputs[n] = src[i];
1219 }
1220 break;
1221 case nir_intrinsic_load_base_vertex:
1222 if (!ctx->basevertex) {
1223 ctx->basevertex = create_driver_param(ctx, IR3_DP_VTXID_BASE);
1224 add_sysval_input(ctx, SYSTEM_VALUE_BASE_VERTEX,
1225 ctx->basevertex);
1226 }
1227 dst[0] = ctx->basevertex;
1228 break;
1229 case nir_intrinsic_load_vertex_id_zero_base:
1230 if (!ctx->vertex_id) {
1231 ctx->vertex_id = create_input(b, 0);
1232 add_sysval_input(ctx, SYSTEM_VALUE_VERTEX_ID_ZERO_BASE,
1233 ctx->vertex_id);
1234 }
1235 dst[0] = ctx->vertex_id;
1236 break;
1237 case nir_intrinsic_load_instance_id:
1238 if (!ctx->instance_id) {
1239 ctx->instance_id = create_input(b, 0);
1240 add_sysval_input(ctx, SYSTEM_VALUE_INSTANCE_ID,
1241 ctx->instance_id);
1242 }
1243 dst[0] = ctx->instance_id;
1244 break;
1245 case nir_intrinsic_load_user_clip_plane:
1246 for (int i = 0; i < intr->num_components; i++) {
1247 unsigned n = idx * 4 + i;
1248 dst[i] = create_driver_param(ctx, IR3_DP_UCP0_X + n);
1249 }
1250 break;
1251 case nir_intrinsic_load_front_face:
1252 if (!ctx->frag_face) {
1253 ctx->so->frag_face = true;
1254 ctx->frag_face = create_input(b, 0);
1255 ctx->frag_face->regs[0]->flags |= IR3_REG_HALF;
1256 }
1257 dst[0] = ir3_ADD_S(b, ctx->frag_face, 0, create_immed(b, 1), 0);
1258 break;
1259 case nir_intrinsic_discard_if:
1260 case nir_intrinsic_discard: {
1261 struct ir3_instruction *cond, *kill;
1262
1263 if (intr->intrinsic == nir_intrinsic_discard_if) {
1264 /* conditional discard: */
1265 src = get_src(ctx, &intr->src[0]);
1266 cond = ir3_b2n(b, src[0]);
1267 } else {
1268 /* unconditional discard: */
1269 cond = create_immed(b, 1);
1270 }
1271
1272 /* NOTE: only cmps.*.* can write p0.x: */
1273 cond = ir3_CMPS_S(b, cond, 0, create_immed(b, 0), 0);
1274 cond->cat2.condition = IR3_COND_NE;
1275
1276 /* condition always goes in predicate register: */
1277 cond->regs[0]->num = regid(REG_P0, 0);
1278
1279 kill = ir3_KILL(b, cond, 0);
1280 array_insert(ctx->ir->predicates, kill);
1281
1282 array_insert(ctx->ir->keeps, kill);
1283 ctx->so->has_kill = true;
1284
1285 break;
1286 }
1287 default:
1288 compile_error(ctx, "Unhandled intrinsic type: %s\n",
1289 nir_intrinsic_infos[intr->intrinsic].name);
1290 break;
1291 }
1292 }
1293
1294 static void
1295 emit_load_const(struct ir3_compile *ctx, nir_load_const_instr *instr)
1296 {
1297 struct ir3_instruction **dst = get_dst_ssa(ctx, &instr->def,
1298 instr->def.num_components);
1299 for (int i = 0; i < instr->def.num_components; i++)
1300 dst[i] = create_immed(ctx->block, instr->value.u[i]);
1301 }
1302
1303 static void
1304 emit_undef(struct ir3_compile *ctx, nir_ssa_undef_instr *undef)
1305 {
1306 struct ir3_instruction **dst = get_dst_ssa(ctx, &undef->def,
1307 undef->def.num_components);
1308 /* backend doesn't want undefined instructions, so just plug
1309 * in 0.0..
1310 */
1311 for (int i = 0; i < undef->def.num_components; i++)
1312 dst[i] = create_immed(ctx->block, fui(0.0));
1313 }
1314
1315 /*
1316 * texture fetch/sample instructions:
1317 */
1318
1319 static void
1320 tex_info(nir_tex_instr *tex, unsigned *flagsp, unsigned *coordsp)
1321 {
1322 unsigned coords, flags = 0;
1323
1324 /* note: would use tex->coord_components.. except txs.. also,
1325 * since array index goes after shadow ref, we don't want to
1326 * count it:
1327 */
1328 switch (tex->sampler_dim) {
1329 case GLSL_SAMPLER_DIM_1D:
1330 case GLSL_SAMPLER_DIM_BUF:
1331 coords = 1;
1332 break;
1333 case GLSL_SAMPLER_DIM_2D:
1334 case GLSL_SAMPLER_DIM_RECT:
1335 case GLSL_SAMPLER_DIM_EXTERNAL:
1336 case GLSL_SAMPLER_DIM_MS:
1337 coords = 2;
1338 break;
1339 case GLSL_SAMPLER_DIM_3D:
1340 case GLSL_SAMPLER_DIM_CUBE:
1341 coords = 3;
1342 flags |= IR3_INSTR_3D;
1343 break;
1344 default:
1345 unreachable("bad sampler_dim");
1346 }
1347
1348 if (tex->is_shadow && tex->op != nir_texop_lod)
1349 flags |= IR3_INSTR_S;
1350
1351 if (tex->is_array && tex->op != nir_texop_lod)
1352 flags |= IR3_INSTR_A;
1353
1354 *flagsp = flags;
1355 *coordsp = coords;
1356 }
1357
1358 static void
1359 emit_tex(struct ir3_compile *ctx, nir_tex_instr *tex)
1360 {
1361 struct ir3_block *b = ctx->block;
1362 struct ir3_instruction **dst, *sam, *src0[12], *src1[4];
1363 struct ir3_instruction **coord, *lod, *compare, *proj, **off, **ddx, **ddy;
1364 bool has_bias = false, has_lod = false, has_proj = false, has_off = false;
1365 unsigned i, coords, flags;
1366 unsigned nsrc0 = 0, nsrc1 = 0;
1367 type_t type;
1368 opc_t opc = 0;
1369
1370 coord = off = ddx = ddy = NULL;
1371 lod = proj = compare = NULL;
1372
1373 /* TODO: might just be one component for gathers? */
1374 dst = get_dst(ctx, &tex->dest, 4);
1375
1376 for (unsigned i = 0; i < tex->num_srcs; i++) {
1377 switch (tex->src[i].src_type) {
1378 case nir_tex_src_coord:
1379 coord = get_src(ctx, &tex->src[i].src);
1380 break;
1381 case nir_tex_src_bias:
1382 lod = get_src(ctx, &tex->src[i].src)[0];
1383 has_bias = true;
1384 break;
1385 case nir_tex_src_lod:
1386 lod = get_src(ctx, &tex->src[i].src)[0];
1387 has_lod = true;
1388 break;
1389 case nir_tex_src_comparitor: /* shadow comparator */
1390 compare = get_src(ctx, &tex->src[i].src)[0];
1391 break;
1392 case nir_tex_src_projector:
1393 proj = get_src(ctx, &tex->src[i].src)[0];
1394 has_proj = true;
1395 break;
1396 case nir_tex_src_offset:
1397 off = get_src(ctx, &tex->src[i].src);
1398 has_off = true;
1399 break;
1400 case nir_tex_src_ddx:
1401 ddx = get_src(ctx, &tex->src[i].src);
1402 break;
1403 case nir_tex_src_ddy:
1404 ddy = get_src(ctx, &tex->src[i].src);
1405 break;
1406 default:
1407 compile_error(ctx, "Unhandled NIR tex serc type: %d\n",
1408 tex->src[i].src_type);
1409 return;
1410 }
1411 }
1412
1413 switch (tex->op) {
1414 case nir_texop_tex: opc = OPC_SAM; break;
1415 case nir_texop_txb: opc = OPC_SAMB; break;
1416 case nir_texop_txl: opc = OPC_SAML; break;
1417 case nir_texop_txd: opc = OPC_SAMGQ; break;
1418 case nir_texop_txf: opc = OPC_ISAML; break;
1419 case nir_texop_lod: opc = OPC_GETLOD; break;
1420 case nir_texop_txf_ms:
1421 case nir_texop_txs:
1422 case nir_texop_tg4:
1423 case nir_texop_query_levels:
1424 case nir_texop_texture_samples:
1425 case nir_texop_samples_identical:
1426 compile_error(ctx, "Unhandled NIR tex type: %d\n", tex->op);
1427 return;
1428 }
1429
1430 tex_info(tex, &flags, &coords);
1431
1432 /* scale up integer coords for TXF based on the LOD */
1433 if (ctx->unminify_coords && (opc == OPC_ISAML)) {
1434 assert(has_lod);
1435 for (i = 0; i < coords; i++)
1436 coord[i] = ir3_SHL_B(b, coord[i], 0, lod, 0);
1437 }
1438
1439 /* the array coord for cube arrays needs 0.5 added to it */
1440 if (tex->sampler_dim == GLSL_SAMPLER_DIM_CUBE && tex->is_array &&
1441 opc != OPC_ISAML)
1442 coord[3] = ir3_ADD_F(b, coord[3], 0, create_immed(b, fui(0.5)), 0);
1443
1444 /*
1445 * lay out the first argument in the proper order:
1446 * - actual coordinates first
1447 * - shadow reference
1448 * - array index
1449 * - projection w
1450 * - starting at offset 4, dpdx.xy, dpdy.xy
1451 *
1452 * bias/lod go into the second arg
1453 */
1454
1455 /* insert tex coords: */
1456 for (i = 0; i < coords; i++)
1457 src0[nsrc0++] = coord[i];
1458
1459 if (coords == 1) {
1460 /* hw doesn't do 1d, so we treat it as 2d with
1461 * height of 1, and patch up the y coord.
1462 * TODO: y coord should be (int)0 in some cases..
1463 */
1464 src0[nsrc0++] = create_immed(b, fui(0.5));
1465 }
1466
1467 if (tex->is_shadow && tex->op != nir_texop_lod)
1468 src0[nsrc0++] = compare;
1469
1470 if (tex->is_array && tex->op != nir_texop_lod)
1471 src0[nsrc0++] = coord[coords];
1472
1473 if (has_proj) {
1474 src0[nsrc0++] = proj;
1475 flags |= IR3_INSTR_P;
1476 }
1477
1478 /* pad to 4, then ddx/ddy: */
1479 if (tex->op == nir_texop_txd) {
1480 while (nsrc0 < 4)
1481 src0[nsrc0++] = create_immed(b, fui(0.0));
1482 for (i = 0; i < coords; i++)
1483 src0[nsrc0++] = ddx[i];
1484 if (coords < 2)
1485 src0[nsrc0++] = create_immed(b, fui(0.0));
1486 for (i = 0; i < coords; i++)
1487 src0[nsrc0++] = ddy[i];
1488 if (coords < 2)
1489 src0[nsrc0++] = create_immed(b, fui(0.0));
1490 }
1491
1492 /*
1493 * second argument (if applicable):
1494 * - offsets
1495 * - lod
1496 * - bias
1497 */
1498 if (has_off | has_lod | has_bias) {
1499 if (has_off) {
1500 for (i = 0; i < coords; i++)
1501 src1[nsrc1++] = off[i];
1502 if (coords < 2)
1503 src1[nsrc1++] = create_immed(b, fui(0.0));
1504 flags |= IR3_INSTR_O;
1505 }
1506
1507 if (has_lod | has_bias)
1508 src1[nsrc1++] = lod;
1509 }
1510
1511 switch (tex->dest_type) {
1512 case nir_type_invalid:
1513 case nir_type_float:
1514 type = TYPE_F32;
1515 break;
1516 case nir_type_int:
1517 type = TYPE_S32;
1518 break;
1519 case nir_type_uint:
1520 case nir_type_bool:
1521 type = TYPE_U32;
1522 break;
1523 default:
1524 unreachable("bad dest_type");
1525 }
1526
1527 if (opc == OPC_GETLOD)
1528 type = TYPE_U32;
1529
1530 sam = ir3_SAM(b, opc, type, TGSI_WRITEMASK_XYZW,
1531 flags, tex->sampler_index, tex->sampler_index,
1532 create_collect(b, src0, nsrc0),
1533 create_collect(b, src1, nsrc1));
1534
1535 split_dest(b, dst, sam, 4);
1536
1537 /* GETLOD returns results in 4.8 fixed point */
1538 if (opc == OPC_GETLOD) {
1539 struct ir3_instruction *factor = create_immed(b, fui(1.0 / 256));
1540
1541 compile_assert(ctx, tex->dest_type == nir_type_float);
1542 for (i = 0; i < 2; i++) {
1543 dst[i] = ir3_MUL_F(b, ir3_COV(b, dst[i], TYPE_U32, TYPE_F32), 0,
1544 factor, 0);
1545 }
1546 }
1547 }
1548
1549 static void
1550 emit_tex_query_levels(struct ir3_compile *ctx, nir_tex_instr *tex)
1551 {
1552 struct ir3_block *b = ctx->block;
1553 struct ir3_instruction **dst, *sam;
1554
1555 dst = get_dst(ctx, &tex->dest, 1);
1556
1557 sam = ir3_SAM(b, OPC_GETINFO, TYPE_U32, TGSI_WRITEMASK_Z, 0,
1558 tex->sampler_index, tex->sampler_index, NULL, NULL);
1559
1560 /* even though there is only one component, since it ends
1561 * up in .z rather than .x, we need a split_dest()
1562 */
1563 split_dest(b, dst, sam, 3);
1564
1565 /* The # of levels comes from getinfo.z. We need to add 1 to it, since
1566 * the value in TEX_CONST_0 is zero-based.
1567 */
1568 if (ctx->levels_add_one)
1569 dst[0] = ir3_ADD_U(b, dst[0], 0, create_immed(b, 1), 0);
1570 }
1571
1572 static void
1573 emit_tex_txs(struct ir3_compile *ctx, nir_tex_instr *tex)
1574 {
1575 struct ir3_block *b = ctx->block;
1576 struct ir3_instruction **dst, *sam, *lod;
1577 unsigned flags, coords;
1578
1579 tex_info(tex, &flags, &coords);
1580
1581 /* Actually we want the number of dimensions, not coordinates. This
1582 * distinction only matters for cubes.
1583 */
1584 if (tex->sampler_dim == GLSL_SAMPLER_DIM_CUBE)
1585 coords = 2;
1586
1587 dst = get_dst(ctx, &tex->dest, 4);
1588
1589 compile_assert(ctx, tex->num_srcs == 1);
1590 compile_assert(ctx, tex->src[0].src_type == nir_tex_src_lod);
1591
1592 lod = get_src(ctx, &tex->src[0].src)[0];
1593
1594 sam = ir3_SAM(b, OPC_GETSIZE, TYPE_U32, TGSI_WRITEMASK_XYZW, flags,
1595 tex->sampler_index, tex->sampler_index, lod, NULL);
1596
1597 split_dest(b, dst, sam, 4);
1598
1599 /* Array size actually ends up in .w rather than .z. This doesn't
1600 * matter for miplevel 0, but for higher mips the value in z is
1601 * minified whereas w stays. Also, the value in TEX_CONST_3_DEPTH is
1602 * returned, which means that we have to add 1 to it for arrays.
1603 */
1604 if (tex->is_array) {
1605 if (ctx->levels_add_one) {
1606 dst[coords] = ir3_ADD_U(b, dst[3], 0, create_immed(b, 1), 0);
1607 } else {
1608 dst[coords] = ir3_MOV(b, dst[3], TYPE_U32);
1609 }
1610 }
1611 }
1612
1613 static void
1614 emit_phi(struct ir3_compile *ctx, nir_phi_instr *nphi)
1615 {
1616 struct ir3_instruction *phi, **dst;
1617
1618 /* NOTE: phi's should be lowered to scalar at this point */
1619 compile_assert(ctx, nphi->dest.ssa.num_components == 1);
1620
1621 dst = get_dst(ctx, &nphi->dest, 1);
1622
1623 phi = ir3_instr_create2(ctx->block, -1, OPC_META_PHI,
1624 1 + exec_list_length(&nphi->srcs));
1625 ir3_reg_create(phi, 0, 0); /* dst */
1626 phi->phi.nphi = nphi;
1627
1628 dst[0] = phi;
1629 }
1630
1631 /* phi instructions are left partially constructed. We don't resolve
1632 * their srcs until the end of the block, since (eg. loops) one of
1633 * the phi's srcs might be defined after the phi due to back edges in
1634 * the CFG.
1635 */
1636 static void
1637 resolve_phis(struct ir3_compile *ctx, struct ir3_block *block)
1638 {
1639 list_for_each_entry (struct ir3_instruction, instr, &block->instr_list, node) {
1640 nir_phi_instr *nphi;
1641
1642 /* phi's only come at start of block: */
1643 if (!(is_meta(instr) && (instr->opc == OPC_META_PHI)))
1644 break;
1645
1646 if (!instr->phi.nphi)
1647 break;
1648
1649 nphi = instr->phi.nphi;
1650 instr->phi.nphi = NULL;
1651
1652 foreach_list_typed(nir_phi_src, nsrc, node, &nphi->srcs) {
1653 struct ir3_instruction *src = get_src(ctx, &nsrc->src)[0];
1654 ir3_reg_create(instr, 0, IR3_REG_SSA)->instr = src;
1655 }
1656 }
1657 }
1658
1659 static void
1660 emit_jump(struct ir3_compile *ctx, nir_jump_instr *jump)
1661 {
1662 switch (jump->type) {
1663 case nir_jump_break:
1664 case nir_jump_continue:
1665 /* I *think* we can simply just ignore this, and use the
1666 * successor block link to figure out where we need to
1667 * jump to for break/continue
1668 */
1669 break;
1670 default:
1671 compile_error(ctx, "Unhandled NIR jump type: %d\n", jump->type);
1672 break;
1673 }
1674 }
1675
1676 static void
1677 emit_instr(struct ir3_compile *ctx, nir_instr *instr)
1678 {
1679 switch (instr->type) {
1680 case nir_instr_type_alu:
1681 emit_alu(ctx, nir_instr_as_alu(instr));
1682 break;
1683 case nir_instr_type_intrinsic:
1684 emit_intrinsic(ctx, nir_instr_as_intrinsic(instr));
1685 break;
1686 case nir_instr_type_load_const:
1687 emit_load_const(ctx, nir_instr_as_load_const(instr));
1688 break;
1689 case nir_instr_type_ssa_undef:
1690 emit_undef(ctx, nir_instr_as_ssa_undef(instr));
1691 break;
1692 case nir_instr_type_tex: {
1693 nir_tex_instr *tex = nir_instr_as_tex(instr);
1694 /* couple tex instructions get special-cased:
1695 */
1696 switch (tex->op) {
1697 case nir_texop_txs:
1698 emit_tex_txs(ctx, tex);
1699 break;
1700 case nir_texop_query_levels:
1701 emit_tex_query_levels(ctx, tex);
1702 break;
1703 default:
1704 emit_tex(ctx, tex);
1705 break;
1706 }
1707 break;
1708 }
1709 case nir_instr_type_phi:
1710 emit_phi(ctx, nir_instr_as_phi(instr));
1711 break;
1712 case nir_instr_type_jump:
1713 emit_jump(ctx, nir_instr_as_jump(instr));
1714 break;
1715 case nir_instr_type_call:
1716 case nir_instr_type_parallel_copy:
1717 compile_error(ctx, "Unhandled NIR instruction type: %d\n", instr->type);
1718 break;
1719 }
1720 }
1721
1722 static struct ir3_block *
1723 get_block(struct ir3_compile *ctx, nir_block *nblock)
1724 {
1725 struct ir3_block *block;
1726 struct hash_entry *entry;
1727 entry = _mesa_hash_table_search(ctx->block_ht, nblock);
1728 if (entry)
1729 return entry->data;
1730
1731 block = ir3_block_create(ctx->ir);
1732 block->nblock = nblock;
1733 _mesa_hash_table_insert(ctx->block_ht, nblock, block);
1734
1735 return block;
1736 }
1737
1738 static void
1739 emit_block(struct ir3_compile *ctx, nir_block *nblock)
1740 {
1741 struct ir3_block *block = get_block(ctx, nblock);
1742
1743 for (int i = 0; i < ARRAY_SIZE(block->successors); i++) {
1744 if (nblock->successors[i]) {
1745 block->successors[i] =
1746 get_block(ctx, nblock->successors[i]);
1747 }
1748 }
1749
1750 ctx->block = block;
1751 list_addtail(&block->node, &ctx->ir->block_list);
1752
1753 /* re-emit addr register in each block if needed: */
1754 _mesa_hash_table_destroy(ctx->addr_ht, NULL);
1755 ctx->addr_ht = NULL;
1756
1757 nir_foreach_instr(nblock, instr) {
1758 emit_instr(ctx, instr);
1759 if (ctx->error)
1760 return;
1761 }
1762 }
1763
1764 static void emit_cf_list(struct ir3_compile *ctx, struct exec_list *list);
1765
1766 static void
1767 emit_if(struct ir3_compile *ctx, nir_if *nif)
1768 {
1769 struct ir3_instruction *condition = get_src(ctx, &nif->condition)[0];
1770
1771 ctx->block->condition =
1772 get_predicate(ctx, ir3_b2n(condition->block, condition));
1773
1774 emit_cf_list(ctx, &nif->then_list);
1775 emit_cf_list(ctx, &nif->else_list);
1776 }
1777
1778 static void
1779 emit_loop(struct ir3_compile *ctx, nir_loop *nloop)
1780 {
1781 emit_cf_list(ctx, &nloop->body);
1782 }
1783
1784 static void
1785 emit_cf_list(struct ir3_compile *ctx, struct exec_list *list)
1786 {
1787 foreach_list_typed(nir_cf_node, node, node, list) {
1788 switch (node->type) {
1789 case nir_cf_node_block:
1790 emit_block(ctx, nir_cf_node_as_block(node));
1791 break;
1792 case nir_cf_node_if:
1793 emit_if(ctx, nir_cf_node_as_if(node));
1794 break;
1795 case nir_cf_node_loop:
1796 emit_loop(ctx, nir_cf_node_as_loop(node));
1797 break;
1798 case nir_cf_node_function:
1799 compile_error(ctx, "TODO\n");
1800 break;
1801 }
1802 }
1803 }
1804
1805 /* emit stream-out code. At this point, the current block is the original
1806 * (nir) end block, and nir ensures that all flow control paths terminate
1807 * into the end block. We re-purpose the original end block to generate
1808 * the 'if (vtxcnt < maxvtxcnt)' condition, then append the conditional
1809 * block holding stream-out write instructions, followed by the new end
1810 * block:
1811 *
1812 * blockOrigEnd {
1813 * p0.x = (vtxcnt < maxvtxcnt)
1814 * // succs: blockStreamOut, blockNewEnd
1815 * }
1816 * blockStreamOut {
1817 * ... stream-out instructions ...
1818 * // succs: blockNewEnd
1819 * }
1820 * blockNewEnd {
1821 * }
1822 */
1823 static void
1824 emit_stream_out(struct ir3_compile *ctx)
1825 {
1826 struct ir3_shader_variant *v = ctx->so;
1827 struct ir3 *ir = ctx->ir;
1828 struct pipe_stream_output_info *strmout =
1829 &ctx->so->shader->stream_output;
1830 struct ir3_block *orig_end_block, *stream_out_block, *new_end_block;
1831 struct ir3_instruction *vtxcnt, *maxvtxcnt, *cond;
1832 struct ir3_instruction *bases[PIPE_MAX_SO_BUFFERS];
1833
1834 /* create vtxcnt input in input block at top of shader,
1835 * so that it is seen as live over the entire duration
1836 * of the shader:
1837 */
1838 vtxcnt = create_input(ctx->in_block, 0);
1839 add_sysval_input(ctx, SYSTEM_VALUE_VERTEX_CNT, vtxcnt);
1840
1841 maxvtxcnt = create_driver_param(ctx, IR3_DP_VTXCNT_MAX);
1842
1843 /* at this point, we are at the original 'end' block,
1844 * re-purpose this block to stream-out condition, then
1845 * append stream-out block and new-end block
1846 */
1847 orig_end_block = ctx->block;
1848
1849 stream_out_block = ir3_block_create(ir);
1850 list_addtail(&stream_out_block->node, &ir->block_list);
1851
1852 new_end_block = ir3_block_create(ir);
1853 list_addtail(&new_end_block->node, &ir->block_list);
1854
1855 orig_end_block->successors[0] = stream_out_block;
1856 orig_end_block->successors[1] = new_end_block;
1857 stream_out_block->successors[0] = new_end_block;
1858
1859 /* setup 'if (vtxcnt < maxvtxcnt)' condition: */
1860 cond = ir3_CMPS_S(ctx->block, vtxcnt, 0, maxvtxcnt, 0);
1861 cond->regs[0]->num = regid(REG_P0, 0);
1862 cond->cat2.condition = IR3_COND_LT;
1863
1864 /* condition goes on previous block to the conditional,
1865 * since it is used to pick which of the two successor
1866 * paths to take:
1867 */
1868 orig_end_block->condition = cond;
1869
1870 /* switch to stream_out_block to generate the stream-out
1871 * instructions:
1872 */
1873 ctx->block = stream_out_block;
1874
1875 /* Calculate base addresses based on vtxcnt. Instructions
1876 * generated for bases not used in following loop will be
1877 * stripped out in the backend.
1878 */
1879 for (unsigned i = 0; i < PIPE_MAX_SO_BUFFERS; i++) {
1880 unsigned stride = strmout->stride[i];
1881 struct ir3_instruction *base, *off;
1882
1883 base = create_uniform(ctx, regid(v->first_driver_param + IR3_TFBOS_OFF, i));
1884
1885 /* 24-bit should be enough: */
1886 off = ir3_MUL_U(ctx->block, vtxcnt, 0,
1887 create_immed(ctx->block, stride * 4), 0);
1888
1889 bases[i] = ir3_ADD_S(ctx->block, off, 0, base, 0);
1890 }
1891
1892 /* Generate the per-output store instructions: */
1893 for (unsigned i = 0; i < strmout->num_outputs; i++) {
1894 for (unsigned j = 0; j < strmout->output[i].num_components; j++) {
1895 unsigned c = j + strmout->output[i].start_component;
1896 struct ir3_instruction *base, *out, *stg;
1897
1898 base = bases[strmout->output[i].output_buffer];
1899 out = ctx->ir->outputs[regid(strmout->output[i].register_index, c)];
1900
1901 stg = ir3_STG(ctx->block, base, 0, out, 0,
1902 create_immed(ctx->block, 1), 0);
1903 stg->cat6.type = TYPE_U32;
1904 stg->cat6.dst_offset = (strmout->output[i].dst_offset + j) * 4;
1905
1906 array_insert(ctx->ir->keeps, stg);
1907 }
1908 }
1909
1910 /* and finally switch to the new_end_block: */
1911 ctx->block = new_end_block;
1912 }
1913
1914 static void
1915 emit_function(struct ir3_compile *ctx, nir_function_impl *impl)
1916 {
1917 nir_metadata_require(impl, nir_metadata_block_index);
1918
1919 emit_cf_list(ctx, &impl->body);
1920 emit_block(ctx, impl->end_block);
1921
1922 /* at this point, we should have a single empty block,
1923 * into which we emit the 'end' instruction.
1924 */
1925 compile_assert(ctx, list_empty(&ctx->block->instr_list));
1926
1927 /* If stream-out (aka transform-feedback) enabled, emit the
1928 * stream-out instructions, followed by a new empty block (into
1929 * which the 'end' instruction lands).
1930 *
1931 * NOTE: it is done in this order, rather than inserting before
1932 * we emit end_block, because NIR guarantees that all blocks
1933 * flow into end_block, and that end_block has no successors.
1934 * So by re-purposing end_block as the first block of stream-
1935 * out, we guarantee that all exit paths flow into the stream-
1936 * out instructions.
1937 */
1938 if ((ctx->so->shader->stream_output.num_outputs > 0) &&
1939 !ctx->so->key.binning_pass) {
1940 debug_assert(ctx->so->type == SHADER_VERTEX);
1941 emit_stream_out(ctx);
1942 }
1943
1944 ir3_END(ctx->block);
1945 }
1946
1947 static void
1948 setup_input(struct ir3_compile *ctx, nir_variable *in)
1949 {
1950 struct ir3_shader_variant *so = ctx->so;
1951 unsigned array_len = MAX2(glsl_get_length(in->type), 1);
1952 unsigned ncomp = glsl_get_components(in->type);
1953 unsigned n = in->data.driver_location;
1954 unsigned slot = in->data.location;
1955
1956 DBG("; in: slot=%u, len=%ux%u, drvloc=%u",
1957 slot, array_len, ncomp, n);
1958
1959 so->inputs[n].slot = slot;
1960 so->inputs[n].compmask = (1 << ncomp) - 1;
1961 so->inputs_count = MAX2(so->inputs_count, n + 1);
1962 so->inputs[n].interpolate = in->data.interpolation;
1963
1964 if (ctx->so->type == SHADER_FRAGMENT) {
1965 for (int i = 0; i < ncomp; i++) {
1966 struct ir3_instruction *instr = NULL;
1967 unsigned idx = (n * 4) + i;
1968
1969 if (slot == VARYING_SLOT_POS) {
1970 so->inputs[n].bary = false;
1971 so->frag_coord = true;
1972 instr = create_frag_coord(ctx, i);
1973 } else if (slot == VARYING_SLOT_FACE) {
1974 so->inputs[n].bary = false;
1975 so->frag_face = true;
1976 instr = create_frag_face(ctx, i);
1977 } else {
1978 bool use_ldlv = false;
1979
1980 /* detect the special case for front/back colors where
1981 * we need to do flat vs smooth shading depending on
1982 * rast state:
1983 */
1984 if (in->data.interpolation == INTERP_QUALIFIER_NONE) {
1985 switch (slot) {
1986 case VARYING_SLOT_COL0:
1987 case VARYING_SLOT_COL1:
1988 case VARYING_SLOT_BFC0:
1989 case VARYING_SLOT_BFC1:
1990 so->inputs[n].rasterflat = true;
1991 break;
1992 default:
1993 break;
1994 }
1995 }
1996
1997 if (ctx->flat_bypass) {
1998 if ((so->inputs[n].interpolate == INTERP_QUALIFIER_FLAT) ||
1999 (so->inputs[n].rasterflat && ctx->so->key.rasterflat))
2000 use_ldlv = true;
2001 }
2002
2003 so->inputs[n].bary = true;
2004
2005 instr = create_frag_input(ctx, use_ldlv);
2006 }
2007
2008 ctx->ir->inputs[idx] = instr;
2009 }
2010 } else if (ctx->so->type == SHADER_VERTEX) {
2011 for (int i = 0; i < ncomp; i++) {
2012 unsigned idx = (n * 4) + i;
2013 ctx->ir->inputs[idx] = create_input(ctx->block, idx);
2014 }
2015 } else {
2016 compile_error(ctx, "unknown shader type: %d\n", ctx->so->type);
2017 }
2018
2019 if (so->inputs[n].bary || (ctx->so->type == SHADER_VERTEX)) {
2020 so->total_in += ncomp;
2021 }
2022 }
2023
2024 static void
2025 setup_output(struct ir3_compile *ctx, nir_variable *out)
2026 {
2027 struct ir3_shader_variant *so = ctx->so;
2028 unsigned array_len = MAX2(glsl_get_length(out->type), 1);
2029 unsigned ncomp = glsl_get_components(out->type);
2030 unsigned n = out->data.driver_location;
2031 unsigned slot = out->data.location;
2032 unsigned comp = 0;
2033
2034 DBG("; out: slot=%u, len=%ux%u, drvloc=%u",
2035 slot, array_len, ncomp, n);
2036
2037 if (ctx->so->type == SHADER_FRAGMENT) {
2038 switch (slot) {
2039 case FRAG_RESULT_DEPTH:
2040 comp = 2; /* tgsi will write to .z component */
2041 so->writes_pos = true;
2042 break;
2043 case FRAG_RESULT_COLOR:
2044 so->color0_mrt = 1;
2045 break;
2046 default:
2047 if (slot >= FRAG_RESULT_DATA0)
2048 break;
2049 compile_error(ctx, "unknown FS output name: %s\n",
2050 gl_frag_result_name(slot));
2051 }
2052 } else if (ctx->so->type == SHADER_VERTEX) {
2053 switch (slot) {
2054 case VARYING_SLOT_POS:
2055 so->writes_pos = true;
2056 break;
2057 case VARYING_SLOT_PSIZ:
2058 so->writes_psize = true;
2059 break;
2060 case VARYING_SLOT_COL0:
2061 case VARYING_SLOT_COL1:
2062 case VARYING_SLOT_BFC0:
2063 case VARYING_SLOT_BFC1:
2064 case VARYING_SLOT_FOGC:
2065 case VARYING_SLOT_CLIP_DIST0:
2066 case VARYING_SLOT_CLIP_DIST1:
2067 break;
2068 default:
2069 if (slot >= VARYING_SLOT_VAR0)
2070 break;
2071 if ((VARYING_SLOT_TEX0 <= slot) && (slot <= VARYING_SLOT_TEX7))
2072 break;
2073 compile_error(ctx, "unknown VS output name: %s\n",
2074 gl_varying_slot_name(slot));
2075 }
2076 } else {
2077 compile_error(ctx, "unknown shader type: %d\n", ctx->so->type);
2078 }
2079
2080 compile_assert(ctx, n < ARRAY_SIZE(so->outputs));
2081
2082 so->outputs[n].slot = slot;
2083 so->outputs[n].regid = regid(n, comp);
2084 so->outputs_count = MAX2(so->outputs_count, n + 1);
2085
2086 for (int i = 0; i < ncomp; i++) {
2087 unsigned idx = (n * 4) + i;
2088
2089 ctx->ir->outputs[idx] = create_immed(ctx->block, fui(0.0));
2090 }
2091 }
2092
2093 static void
2094 emit_instructions(struct ir3_compile *ctx)
2095 {
2096 unsigned ninputs, noutputs;
2097 nir_function_impl *fxn = NULL;
2098
2099 /* Find the main function: */
2100 nir_foreach_function(ctx->s, function) {
2101 compile_assert(ctx, strcmp(function->name, "main") == 0);
2102 compile_assert(ctx, function->impl);
2103 fxn = function->impl;
2104 break;
2105 }
2106
2107 ninputs = exec_list_length(&ctx->s->inputs) * 4;
2108 noutputs = exec_list_length(&ctx->s->outputs) * 4;
2109
2110 /* or vtx shaders, we need to leave room for sysvals:
2111 */
2112 if (ctx->so->type == SHADER_VERTEX) {
2113 ninputs += 8;
2114 }
2115
2116 ctx->ir = ir3_create(ctx->compiler, ninputs, noutputs);
2117
2118 /* Create inputs in first block: */
2119 ctx->block = get_block(ctx, nir_start_block(fxn));
2120 ctx->in_block = ctx->block;
2121 list_addtail(&ctx->block->node, &ctx->ir->block_list);
2122
2123 if (ctx->so->type == SHADER_VERTEX) {
2124 ctx->ir->ninputs -= 8;
2125 }
2126
2127 /* for fragment shader, we have a single input register (usually
2128 * r0.xy) which is used as the base for bary.f varying fetch instrs:
2129 */
2130 if (ctx->so->type == SHADER_FRAGMENT) {
2131 // TODO maybe a helper for fi since we need it a few places..
2132 struct ir3_instruction *instr;
2133 instr = ir3_instr_create(ctx->block, -1, OPC_META_FI);
2134 ir3_reg_create(instr, 0, 0);
2135 ir3_reg_create(instr, 0, IR3_REG_SSA); /* r0.x */
2136 ir3_reg_create(instr, 0, IR3_REG_SSA); /* r0.y */
2137 ctx->frag_pos = instr;
2138 }
2139
2140 /* Setup inputs: */
2141 nir_foreach_variable(var, &ctx->s->inputs) {
2142 setup_input(ctx, var);
2143 }
2144
2145 /* Setup outputs: */
2146 nir_foreach_variable(var, &ctx->s->outputs) {
2147 setup_output(ctx, var);
2148 }
2149
2150 /* Setup variables (which should only be arrays): */
2151 nir_foreach_variable(var, &ctx->s->globals) {
2152 declare_var(ctx, var);
2153 }
2154
2155 /* And emit the body: */
2156 ctx->impl = fxn;
2157 emit_function(ctx, fxn);
2158
2159 list_for_each_entry (struct ir3_block, block, &ctx->ir->block_list, node) {
2160 resolve_phis(ctx, block);
2161 }
2162 }
2163
2164 /* from NIR perspective, we actually have inputs. But most of the "inputs"
2165 * for a fragment shader are just bary.f instructions. The *actual* inputs
2166 * from the hw perspective are the frag_pos and optionally frag_coord and
2167 * frag_face.
2168 */
2169 static void
2170 fixup_frag_inputs(struct ir3_compile *ctx)
2171 {
2172 struct ir3_shader_variant *so = ctx->so;
2173 struct ir3 *ir = ctx->ir;
2174 struct ir3_instruction **inputs;
2175 struct ir3_instruction *instr;
2176 int n, regid = 0;
2177
2178 ir->ninputs = 0;
2179
2180 n = 4; /* always have frag_pos */
2181 n += COND(so->frag_face, 4);
2182 n += COND(so->frag_coord, 4);
2183
2184 inputs = ir3_alloc(ctx->ir, n * (sizeof(struct ir3_instruction *)));
2185
2186 if (so->frag_face) {
2187 /* this ultimately gets assigned to hr0.x so doesn't conflict
2188 * with frag_coord/frag_pos..
2189 */
2190 inputs[ir->ninputs++] = ctx->frag_face;
2191 ctx->frag_face->regs[0]->num = 0;
2192
2193 /* remaining channels not used, but let's avoid confusing
2194 * other parts that expect inputs to come in groups of vec4
2195 */
2196 inputs[ir->ninputs++] = NULL;
2197 inputs[ir->ninputs++] = NULL;
2198 inputs[ir->ninputs++] = NULL;
2199 }
2200
2201 /* since we don't know where to set the regid for frag_coord,
2202 * we have to use r0.x for it. But we don't want to *always*
2203 * use r1.x for frag_pos as that could increase the register
2204 * footprint on simple shaders:
2205 */
2206 if (so->frag_coord) {
2207 ctx->frag_coord[0]->regs[0]->num = regid++;
2208 ctx->frag_coord[1]->regs[0]->num = regid++;
2209 ctx->frag_coord[2]->regs[0]->num = regid++;
2210 ctx->frag_coord[3]->regs[0]->num = regid++;
2211
2212 inputs[ir->ninputs++] = ctx->frag_coord[0];
2213 inputs[ir->ninputs++] = ctx->frag_coord[1];
2214 inputs[ir->ninputs++] = ctx->frag_coord[2];
2215 inputs[ir->ninputs++] = ctx->frag_coord[3];
2216 }
2217
2218 /* we always have frag_pos: */
2219 so->pos_regid = regid;
2220
2221 /* r0.x */
2222 instr = create_input(ctx->in_block, ir->ninputs);
2223 instr->regs[0]->num = regid++;
2224 inputs[ir->ninputs++] = instr;
2225 ctx->frag_pos->regs[1]->instr = instr;
2226
2227 /* r0.y */
2228 instr = create_input(ctx->in_block, ir->ninputs);
2229 instr->regs[0]->num = regid++;
2230 inputs[ir->ninputs++] = instr;
2231 ctx->frag_pos->regs[2]->instr = instr;
2232
2233 ir->inputs = inputs;
2234 }
2235
2236 int
2237 ir3_compile_shader_nir(struct ir3_compiler *compiler,
2238 struct ir3_shader_variant *so)
2239 {
2240 struct ir3_compile *ctx;
2241 struct ir3 *ir;
2242 struct ir3_instruction **inputs;
2243 unsigned i, j, actual_in, inloc;
2244 int ret = 0, max_bary;
2245
2246 assert(!so->ir);
2247
2248 ctx = compile_init(compiler, so);
2249 if (!ctx) {
2250 DBG("INIT failed!");
2251 ret = -1;
2252 goto out;
2253 }
2254
2255 emit_instructions(ctx);
2256
2257 if (ctx->error) {
2258 DBG("EMIT failed!");
2259 ret = -1;
2260 goto out;
2261 }
2262
2263 ir = so->ir = ctx->ir;
2264
2265 /* keep track of the inputs from TGSI perspective.. */
2266 inputs = ir->inputs;
2267
2268 /* but fixup actual inputs for frag shader: */
2269 if (so->type == SHADER_FRAGMENT)
2270 fixup_frag_inputs(ctx);
2271
2272 /* at this point, for binning pass, throw away unneeded outputs: */
2273 if (so->key.binning_pass) {
2274 for (i = 0, j = 0; i < so->outputs_count; i++) {
2275 unsigned slot = so->outputs[i].slot;
2276
2277 /* throw away everything but first position/psize */
2278 if ((slot == VARYING_SLOT_POS) || (slot == VARYING_SLOT_PSIZ)) {
2279 if (i != j) {
2280 so->outputs[j] = so->outputs[i];
2281 ir->outputs[(j*4)+0] = ir->outputs[(i*4)+0];
2282 ir->outputs[(j*4)+1] = ir->outputs[(i*4)+1];
2283 ir->outputs[(j*4)+2] = ir->outputs[(i*4)+2];
2284 ir->outputs[(j*4)+3] = ir->outputs[(i*4)+3];
2285 }
2286 j++;
2287 }
2288 }
2289 so->outputs_count = j;
2290 ir->noutputs = j * 4;
2291 }
2292
2293 /* if we want half-precision outputs, mark the output registers
2294 * as half:
2295 */
2296 if (so->key.half_precision) {
2297 for (i = 0; i < ir->noutputs; i++) {
2298 struct ir3_instruction *out = ir->outputs[i];
2299 if (!out)
2300 continue;
2301 out->regs[0]->flags |= IR3_REG_HALF;
2302 /* output could be a fanout (ie. texture fetch output)
2303 * in which case we need to propagate the half-reg flag
2304 * up to the definer so that RA sees it:
2305 */
2306 if (is_meta(out) && (out->opc == OPC_META_FO)) {
2307 out = out->regs[1]->instr;
2308 out->regs[0]->flags |= IR3_REG_HALF;
2309 }
2310
2311 if (out->category == 1) {
2312 out->cat1.dst_type = half_type(out->cat1.dst_type);
2313 }
2314 }
2315 }
2316
2317 if (fd_mesa_debug & FD_DBG_OPTMSGS) {
2318 printf("BEFORE CP:\n");
2319 ir3_print(ir);
2320 }
2321
2322 ir3_cp(ir);
2323
2324 if (fd_mesa_debug & FD_DBG_OPTMSGS) {
2325 printf("BEFORE GROUPING:\n");
2326 ir3_print(ir);
2327 }
2328
2329 /* Group left/right neighbors, inserting mov's where needed to
2330 * solve conflicts:
2331 */
2332 ir3_group(ir);
2333
2334 ir3_depth(ir);
2335
2336 if (fd_mesa_debug & FD_DBG_OPTMSGS) {
2337 printf("AFTER DEPTH:\n");
2338 ir3_print(ir);
2339 }
2340
2341 ret = ir3_sched(ir);
2342 if (ret) {
2343 DBG("SCHED failed!");
2344 goto out;
2345 }
2346
2347 if (fd_mesa_debug & FD_DBG_OPTMSGS) {
2348 printf("AFTER SCHED:\n");
2349 ir3_print(ir);
2350 }
2351
2352 ret = ir3_ra(ir, so->type, so->frag_coord, so->frag_face);
2353 if (ret) {
2354 DBG("RA failed!");
2355 goto out;
2356 }
2357
2358 if (fd_mesa_debug & FD_DBG_OPTMSGS) {
2359 printf("AFTER RA:\n");
2360 ir3_print(ir);
2361 }
2362
2363 /* fixup input/outputs: */
2364 for (i = 0; i < so->outputs_count; i++) {
2365 so->outputs[i].regid = ir->outputs[i*4]->regs[0]->num;
2366 /* preserve hack for depth output.. tgsi writes depth to .z,
2367 * but what we give the hw is the scalar register:
2368 */
2369 if ((so->type == SHADER_FRAGMENT) &&
2370 (so->outputs[i].slot == FRAG_RESULT_DEPTH))
2371 so->outputs[i].regid += 2;
2372 }
2373
2374 /* Note that some or all channels of an input may be unused: */
2375 actual_in = 0;
2376 inloc = 0;
2377 for (i = 0; i < so->inputs_count; i++) {
2378 unsigned j, regid = ~0, compmask = 0;
2379 so->inputs[i].ncomp = 0;
2380 so->inputs[i].inloc = inloc + 8;
2381 for (j = 0; j < 4; j++) {
2382 struct ir3_instruction *in = inputs[(i*4) + j];
2383 if (in && !(in->flags & IR3_INSTR_UNUSED)) {
2384 compmask |= (1 << j);
2385 regid = in->regs[0]->num - j;
2386 actual_in++;
2387 so->inputs[i].ncomp++;
2388 if ((so->type == SHADER_FRAGMENT) && so->inputs[i].bary) {
2389 /* assign inloc: */
2390 assert(in->regs[1]->flags & IR3_REG_IMMED);
2391 in->regs[1]->iim_val = inloc++;
2392 }
2393 }
2394 }
2395 if ((so->type == SHADER_FRAGMENT) && compmask && so->inputs[i].bary)
2396 so->varying_in++;
2397 so->inputs[i].regid = regid;
2398 so->inputs[i].compmask = compmask;
2399 }
2400
2401 /* We need to do legalize after (for frag shader's) the "bary.f"
2402 * offsets (inloc) have been assigned.
2403 */
2404 ir3_legalize(ir, &so->has_samp, &max_bary);
2405
2406 if (fd_mesa_debug & FD_DBG_OPTMSGS) {
2407 printf("AFTER LEGALIZE:\n");
2408 ir3_print(ir);
2409 }
2410
2411 /* Note that actual_in counts inputs that are not bary.f'd for FS: */
2412 if (so->type == SHADER_VERTEX)
2413 so->total_in = actual_in;
2414 else
2415 so->total_in = max_bary + 1;
2416
2417 out:
2418 if (ret) {
2419 if (so->ir)
2420 ir3_destroy(so->ir);
2421 so->ir = NULL;
2422 }
2423 compile_free(ctx);
2424
2425 return ret;
2426 }