projects / mesa.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
history | raw | HEAD
nir: Get rid of function overloads
[mesa.git] / src / gallium / drivers / freedreno / ir3 / ir3_compiler_nir.c
1 /* -*- mode: C; c-file-style: "k&r"; tab-width 4; indent-tabs-mode: t; -*- */
2
3 /*
4 * Copyright (C) 2015 Rob Clark <robclark@freedesktop.org>
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the "Software"),
8 * to deal in the Software without restriction, including without limitation
9 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
10 * and/or sell copies of the Software, and to permit persons to whom the
11 * Software is furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice (including the next
14 * paragraph) shall be included in all copies or substantial portions of the
15 * Software.
16 *
17 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
18 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
19 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
20 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
21 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
22 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
23 * SOFTWARE.
24 *
25 * Authors:
26 * Rob Clark <robclark@freedesktop.org>
27 */
28
29 #include <stdarg.h>
30
31 #include "pipe/p_state.h"
32 #include "util/u_string.h"
33 #include "util/u_memory.h"
34 #include "util/u_inlines.h"
35 #include "tgsi/tgsi_lowering.h"
36 #include "tgsi/tgsi_strings.h"
37
38 #include "nir/tgsi_to_nir.h"
39
40 #include "freedreno_util.h"
41
42 #include "ir3_compiler.h"
43 #include "ir3_shader.h"
44 #include "ir3_nir.h"
45
46 #include "instr-a3xx.h"
47 #include "ir3.h"
48
49
50 struct ir3_compile {
51 struct ir3_compiler *compiler;
52
53 const struct tgsi_token *tokens;
54 struct nir_shader *s;
55
56 struct ir3 *ir;
57 struct ir3_shader_variant *so;
58
59 struct ir3_block *block; /* the current block */
60 struct ir3_block *in_block; /* block created for shader inputs */
61
62 nir_function_impl *impl;
63
64 /* For fragment shaders, from the hw perspective the only
65 * actual input is r0.xy position register passed to bary.f.
66 * But TGSI doesn't know that, it still declares things as
67 * IN[] registers. So we do all the input tracking normally
68 * and fix things up after compile_instructions()
69 *
70 * NOTE that frag_pos is the hardware position (possibly it
71 * is actually an index or tag or some such.. it is *not*
72 * values that can be directly used for gl_FragCoord..)
73 */
74 struct ir3_instruction *frag_pos, *frag_face, *frag_coord[4];
75
76 /* For vertex shaders, keep track of the system values sources */
77 struct ir3_instruction *vertex_id, *basevertex, *instance_id;
78
79 /* mapping from nir_register to defining instruction: */
80 struct hash_table *def_ht;
81
82 /* mapping from nir_variable to ir3_array: */
83 struct hash_table *var_ht;
84 unsigned num_arrays;
85
86 /* a common pattern for indirect addressing is to request the
87 * same address register multiple times. To avoid generating
88 * duplicate instruction sequences (which our backend does not
89 * try to clean up, since that should be done as the NIR stage)
90 * we cache the address value generated for a given src value:
91 */
92 struct hash_table *addr_ht;
93
94 /* maps nir_block to ir3_block, mostly for the purposes of
95 * figuring out the blocks successors
96 */
97 struct hash_table *block_ht;
98
99 /* a4xx (at least patchlevel 0) cannot seem to flat-interpolate
100 * so we need to use ldlv.u32 to load the varying directly:
101 */
102 bool flat_bypass;
103
104 /* on a3xx, we need to add one to # of array levels:
105 */
106 bool levels_add_one;
107
108 /* on a3xx, we need to scale up integer coords for isaml based
109 * on LoD:
110 */
111 bool unminify_coords;
112
113 /* for looking up which system value is which */
114 unsigned sysval_semantics[8];
115
116 /* set if we encounter something we can't handle yet, so we
117 * can bail cleanly and fallback to TGSI compiler f/e
118 */
119 bool error;
120 };
121
122
123 static struct ir3_instruction * create_immed(struct ir3_block *block, uint32_t val);
124 static struct ir3_block * get_block(struct ir3_compile *ctx, nir_block *nblock);
125
126 static struct nir_shader *to_nir(struct ir3_compile *ctx,
127 const struct tgsi_token *tokens, struct ir3_shader_variant *so)
128 {
129 static const nir_shader_compiler_options options = {
130 .lower_fpow = true,
131 .lower_fsat = true,
132 .lower_scmp = true,
133 .lower_flrp = true,
134 .lower_ffract = true,
135 .native_integers = true,
136 };
137 struct nir_lower_tex_options tex_options = {
138 .lower_rect = 0,
139 };
140 bool progress;
141
142 switch (so->type) {
143 case SHADER_FRAGMENT:
144 case SHADER_COMPUTE:
145 tex_options.saturate_s = so->key.fsaturate_s;
146 tex_options.saturate_t = so->key.fsaturate_t;
147 tex_options.saturate_r = so->key.fsaturate_r;
148 break;
149 case SHADER_VERTEX:
150 tex_options.saturate_s = so->key.vsaturate_s;
151 tex_options.saturate_t = so->key.vsaturate_t;
152 tex_options.saturate_r = so->key.vsaturate_r;
153 break;
154 }
155
156 if (ctx->compiler->gpu_id >= 400) {
157 /* a4xx seems to have *no* sam.p */
158 tex_options.lower_txp = ~0; /* lower all txp */
159 } else {
160 /* a3xx just needs to avoid sam.p for 3d tex */
161 tex_options.lower_txp = (1 << GLSL_SAMPLER_DIM_3D);
162 }
163
164 struct nir_shader *s = tgsi_to_nir(tokens, &options);
165
166 if (fd_mesa_debug & FD_DBG_DISASM) {
167 debug_printf("----------------------\n");
168 nir_print_shader(s, stdout);
169 debug_printf("----------------------\n");
170 }
171
172 nir_opt_global_to_local(s);
173 nir_convert_to_ssa(s);
174 if (s->stage == MESA_SHADER_VERTEX) {
175 nir_lower_clip_vs(s, so->key.ucp_enables);
176 } else if (s->stage == MESA_SHADER_FRAGMENT) {
177 nir_lower_clip_fs(s, so->key.ucp_enables);
178 }
179 nir_lower_tex(s, &tex_options);
180 if (so->key.color_two_side)
181 nir_lower_two_sided_color(s);
182 nir_lower_idiv(s);
183 nir_lower_load_const_to_scalar(s);
184
185 do {
186 progress = false;
187
188 nir_lower_vars_to_ssa(s);
189 nir_lower_alu_to_scalar(s);
190 nir_lower_phis_to_scalar(s);
191
192 progress |= nir_copy_prop(s);
193 progress |= nir_opt_dce(s);
194 progress |= nir_opt_cse(s);
195 progress |= ir3_nir_lower_if_else(s);
196 progress |= nir_opt_algebraic(s);
197 progress |= nir_opt_constant_folding(s);
198
199 } while (progress);
200
201 nir_remove_dead_variables(s);
202 nir_validate_shader(s);
203
204 if (fd_mesa_debug & FD_DBG_DISASM) {
205 debug_printf("----------------------\n");
206 nir_print_shader(s, stdout);
207 debug_printf("----------------------\n");
208 }
209
210 return s;
211 }
212
213 static struct ir3_compile *
214 compile_init(struct ir3_compiler *compiler,
215 struct ir3_shader_variant *so,
216 const struct tgsi_token *tokens)
217 {
218 struct ir3_compile *ctx = rzalloc(NULL, struct ir3_compile);
219
220 if (compiler->gpu_id >= 400) {
221 /* need special handling for "flat" */
222 ctx->flat_bypass = true;
223 ctx->levels_add_one = false;
224 ctx->unminify_coords = false;
225 } else {
226 /* no special handling for "flat" */
227 ctx->flat_bypass = false;
228 ctx->levels_add_one = true;
229 ctx->unminify_coords = true;
230 }
231
232 ctx->compiler = compiler;
233 ctx->ir = so->ir;
234 ctx->so = so;
235 ctx->def_ht = _mesa_hash_table_create(ctx,
236 _mesa_hash_pointer, _mesa_key_pointer_equal);
237 ctx->var_ht = _mesa_hash_table_create(ctx,
238 _mesa_hash_pointer, _mesa_key_pointer_equal);
239 ctx->block_ht = _mesa_hash_table_create(ctx,
240 _mesa_hash_pointer, _mesa_key_pointer_equal);
241
242 ctx->s = to_nir(ctx, tokens, so);
243
244 so->first_driver_param = so->first_immediate = ctx->s->num_uniforms;
245
246 /* Layout of constant registers:
247 *
248 * num_uniform * vec4 - user consts
249 * 4 * vec4 - UBO addresses
250 * if (vertex shader) {
251 * N * vec4 - driver params (IR3_DP_*)
252 * 1 * vec4 - stream-out addresses
253 * }
254 *
255 * TODO this could be made more dynamic, to at least skip sections
256 * that we don't need..
257 */
258
259 /* reserve 4 (vec4) slots for ubo base addresses: */
260 so->first_immediate += 4;
261
262 if (so->type == SHADER_VERTEX) {
263 /* driver params (see ir3_driver_param): */
264 so->first_immediate += IR3_DP_COUNT/4; /* convert to vec4 */
265 /* one (vec4) slot for stream-output base addresses: */
266 so->first_immediate++;
267 }
268
269 return ctx;
270 }
271
272 static void
273 compile_error(struct ir3_compile *ctx, const char *format, ...)
274 {
275 va_list ap;
276 va_start(ap, format);
277 _debug_vprintf(format, ap);
278 va_end(ap);
279 nir_print_shader(ctx->s, stdout);
280 ctx->error = true;
281 debug_assert(0);
282 }
283
284 #define compile_assert(ctx, cond) do { \
285 if (!(cond)) compile_error((ctx), "failed assert: "#cond"\n"); \
286 } while (0)
287
288 static void
289 compile_free(struct ir3_compile *ctx)
290 {
291 ralloc_free(ctx);
292 }
293
294 /* global per-array information: */
295 struct ir3_array {
296 unsigned length, aid;
297 };
298
299 /* per-block array state: */
300 struct ir3_array_value {
301 /* TODO drop length/aid, and just have ptr back to ir3_array */
302 unsigned length, aid;
303 /* initial array element values are phi's, other than for the
304 * entry block. The phi src's get added later in a resolve step
305 * after we have visited all the blocks, to account for back
306 * edges in the cfg.
307 */
308 struct ir3_instruction **phis;
309 /* current array element values (as block is processed). When
310 * the array phi's are resolved, it will contain the array state
311 * at exit of block, so successor blocks can use it to add their
312 * phi srcs.
313 */
314 struct ir3_instruction *arr[];
315 };
316
317 /* track array assignments per basic block. When an array is read
318 * outside of the same basic block, we can use NIR's dominance-frontier
319 * information to figure out where phi nodes are needed.
320 */
321 struct ir3_nir_block_data {
322 unsigned foo;
323 /* indexed by array-id (aid): */
324 struct ir3_array_value *arrs[];
325 };
326
327 static struct ir3_nir_block_data *
328 get_block_data(struct ir3_compile *ctx, struct ir3_block *block)
329 {
330 if (!block->data) {
331 struct ir3_nir_block_data *bd = ralloc_size(ctx, sizeof(*bd) +
332 ((ctx->num_arrays + 1) * sizeof(bd->arrs[0])));
333 block->data = bd;
334 }
335 return block->data;
336 }
337
338 static void
339 declare_var(struct ir3_compile *ctx, nir_variable *var)
340 {
341 unsigned length = glsl_get_length(var->type) * 4; /* always vec4, at least with ttn */
342 struct ir3_array *arr = ralloc(ctx, struct ir3_array);
343 arr->length = length;
344 arr->aid = ++ctx->num_arrays;
345 _mesa_hash_table_insert(ctx->var_ht, var, arr);
346 }
347
348 static nir_block *
349 nir_block_pred(nir_block *block)
350 {
351 assert(block->predecessors->entries < 2);
352 if (block->predecessors->entries == 0)
353 return NULL;
354 return (nir_block *)_mesa_set_next_entry(block->predecessors, NULL)->key;
355 }
356
357 static struct ir3_array_value *
358 get_var(struct ir3_compile *ctx, nir_variable *var)
359 {
360 struct hash_entry *entry = _mesa_hash_table_search(ctx->var_ht, var);
361 struct ir3_block *block = ctx->block;
362 struct ir3_nir_block_data *bd = get_block_data(ctx, block);
363 struct ir3_array *arr = entry->data;
364
365 if (!bd->arrs[arr->aid]) {
366 struct ir3_array_value *av = ralloc_size(bd, sizeof(*av) +
367 (arr->length * sizeof(av->arr[0])));
368 struct ir3_array_value *defn = NULL;
369 nir_block *pred_block;
370
371 av->length = arr->length;
372 av->aid = arr->aid;
373
374 /* For loops, we have to consider that we have not visited some
375 * of the blocks who should feed into the phi (ie. back-edges in
376 * the cfg).. for example:
377 *
378 * loop {
379 * block { load_var; ... }
380 * if then block {} else block {}
381 * block { store_var; ... }
382 * if then block {} else block {}
383 * block {...}
384 * }
385 *
386 * We can skip the phi if we can chase the block predecessors
387 * until finding the block previously defining the array without
388 * crossing a block that has more than one predecessor.
389 *
390 * Otherwise create phi's and resolve them as a post-pass after
391 * all the blocks have been visited (to handle back-edges).
392 */
393
394 for (pred_block = block->nblock;
395 pred_block && (pred_block->predecessors->entries < 2) && !defn;
396 pred_block = nir_block_pred(pred_block)) {
397 struct ir3_block *pblock = get_block(ctx, pred_block);
398 struct ir3_nir_block_data *pbd = pblock->data;
399 if (!pbd)
400 continue;
401 defn = pbd->arrs[arr->aid];
402 }
403
404 if (defn) {
405 /* only one possible definer: */
406 for (unsigned i = 0; i < arr->length; i++)
407 av->arr[i] = defn->arr[i];
408 } else if (pred_block) {
409 /* not the first block, and multiple potential definers: */
410 av->phis = ralloc_size(av, arr->length * sizeof(av->phis[0]));
411
412 for (unsigned i = 0; i < arr->length; i++) {
413 struct ir3_instruction *phi;
414
415 phi = ir3_instr_create2(block, -1, OPC_META_PHI,
416 1 + ctx->impl->num_blocks);
417 ir3_reg_create(phi, 0, 0); /* dst */
418
419 /* phi's should go at head of block: */
420 list_delinit(&phi->node);
421 list_add(&phi->node, &block->instr_list);
422
423 av->phis[i] = av->arr[i] = phi;
424 }
425 } else {
426 /* Some shaders end up reading array elements without
427 * first writing.. so initialize things to prevent null
428 * instr ptrs later:
429 */
430 for (unsigned i = 0; i < arr->length; i++)
431 av->arr[i] = create_immed(block, 0);
432 }
433
434 bd->arrs[arr->aid] = av;
435 }
436
437 return bd->arrs[arr->aid];
438 }
439
440 static void
441 add_array_phi_srcs(struct ir3_compile *ctx, nir_block *nblock,
442 struct ir3_array_value *av, BITSET_WORD *visited)
443 {
444 struct ir3_block *block;
445 struct ir3_nir_block_data *bd;
446
447 if (BITSET_TEST(visited, nblock->index))
448 return;
449
450 BITSET_SET(visited, nblock->index);
451
452 block = get_block(ctx, nblock);
453 bd = block->data;
454
455 if (bd && bd->arrs[av->aid]) {
456 struct ir3_array_value *dav = bd->arrs[av->aid];
457 for (unsigned i = 0; i < av->length; i++) {
458 ir3_reg_create(av->phis[i], 0, IR3_REG_SSA)->instr =
459 dav->arr[i];
460 }
461 } else {
462 /* didn't find defn, recurse predecessors: */
463 struct set_entry *entry;
464 set_foreach(nblock->predecessors, entry) {
465 add_array_phi_srcs(ctx, (nir_block *)entry->key, av, visited);
466 }
467 }
468 }
469
470 static void
471 resolve_array_phis(struct ir3_compile *ctx, struct ir3_block *block)
472 {
473 struct ir3_nir_block_data *bd = block->data;
474 unsigned bitset_words = BITSET_WORDS(ctx->impl->num_blocks);
475
476 if (!bd)
477 return;
478
479 /* TODO use nir dom_frontier to help us with this? */
480
481 for (unsigned i = 1; i <= ctx->num_arrays; i++) {
482 struct ir3_array_value *av = bd->arrs[i];
483 BITSET_WORD visited[bitset_words];
484 struct set_entry *entry;
485
486 if (!(av && av->phis))
487 continue;
488
489 memset(visited, 0, sizeof(visited));
490 set_foreach(block->nblock->predecessors, entry) {
491 add_array_phi_srcs(ctx, (nir_block *)entry->key, av, visited);
492 }
493 }
494 }
495
496 /* allocate a n element value array (to be populated by caller) and
497 * insert in def_ht
498 */
499 static struct ir3_instruction **
500 __get_dst(struct ir3_compile *ctx, void *key, unsigned n)
501 {
502 struct ir3_instruction **value =
503 ralloc_array(ctx->def_ht, struct ir3_instruction *, n);
504 _mesa_hash_table_insert(ctx->def_ht, key, value);
505 return value;
506 }
507
508 static struct ir3_instruction **
509 get_dst(struct ir3_compile *ctx, nir_dest *dst, unsigned n)
510 {
511 if (dst->is_ssa) {
512 return __get_dst(ctx, &dst->ssa, n);
513 } else {
514 return __get_dst(ctx, dst->reg.reg, n);
515 }
516 }
517
518 static struct ir3_instruction **
519 get_dst_ssa(struct ir3_compile *ctx, nir_ssa_def *dst, unsigned n)
520 {
521 return __get_dst(ctx, dst, n);
522 }
523
524 static struct ir3_instruction **
525 get_src(struct ir3_compile *ctx, nir_src *src)
526 {
527 struct hash_entry *entry;
528 if (src->is_ssa) {
529 entry = _mesa_hash_table_search(ctx->def_ht, src->ssa);
530 } else {
531 entry = _mesa_hash_table_search(ctx->def_ht, src->reg.reg);
532 }
533 compile_assert(ctx, entry);
534 return entry->data;
535 }
536
537 static struct ir3_instruction *
538 create_immed(struct ir3_block *block, uint32_t val)
539 {
540 struct ir3_instruction *mov;
541
542 mov = ir3_instr_create(block, 1, 0);
543 mov->cat1.src_type = TYPE_U32;
544 mov->cat1.dst_type = TYPE_U32;
545 ir3_reg_create(mov, 0, 0);
546 ir3_reg_create(mov, 0, IR3_REG_IMMED)->uim_val = val;
547
548 return mov;
549 }
550
551 static struct ir3_instruction *
552 create_addr(struct ir3_block *block, struct ir3_instruction *src)
553 {
554 struct ir3_instruction *instr, *immed;
555
556 /* TODO in at least some cases, the backend could probably be
557 * made clever enough to propagate IR3_REG_HALF..
558 */
559 instr = ir3_COV(block, src, TYPE_U32, TYPE_S16);
560 instr->regs[0]->flags |= IR3_REG_HALF;
561
562 immed = create_immed(block, 2);
563 immed->regs[0]->flags |= IR3_REG_HALF;
564
565 instr = ir3_SHL_B(block, instr, 0, immed, 0);
566 instr->regs[0]->flags |= IR3_REG_HALF;
567 instr->regs[1]->flags |= IR3_REG_HALF;
568
569 instr = ir3_MOV(block, instr, TYPE_S16);
570 instr->regs[0]->num = regid(REG_A0, 0);
571 instr->regs[0]->flags |= IR3_REG_HALF;
572 instr->regs[1]->flags |= IR3_REG_HALF;
573
574 return instr;
575 }
576
577 /* caches addr values to avoid generating multiple cov/shl/mova
578 * sequences for each use of a given NIR level src as address
579 */
580 static struct ir3_instruction *
581 get_addr(struct ir3_compile *ctx, struct ir3_instruction *src)
582 {
583 struct ir3_instruction *addr;
584
585 if (!ctx->addr_ht) {
586 ctx->addr_ht = _mesa_hash_table_create(ctx,
587 _mesa_hash_pointer, _mesa_key_pointer_equal);
588 } else {
589 struct hash_entry *entry;
590 entry = _mesa_hash_table_search(ctx->addr_ht, src);
591 if (entry)
592 return entry->data;
593 }
594
595 addr = create_addr(ctx->block, src);
596 _mesa_hash_table_insert(ctx->addr_ht, src, addr);
597
598 return addr;
599 }
600
601 static struct ir3_instruction *
602 get_predicate(struct ir3_compile *ctx, struct ir3_instruction *src)
603 {
604 struct ir3_block *b = ctx->block;
605 struct ir3_instruction *cond;
606
607 /* NOTE: only cmps.*.* can write p0.x: */
608 cond = ir3_CMPS_S(b, src, 0, create_immed(b, 0), 0);
609 cond->cat2.condition = IR3_COND_NE;
610
611 /* condition always goes in predicate register: */
612 cond->regs[0]->num = regid(REG_P0, 0);
613
614 return cond;
615 }
616
617 static struct ir3_instruction *
618 create_uniform(struct ir3_compile *ctx, unsigned n)
619 {
620 struct ir3_instruction *mov;
621
622 mov = ir3_instr_create(ctx->block, 1, 0);
623 /* TODO get types right? */
624 mov->cat1.src_type = TYPE_F32;
625 mov->cat1.dst_type = TYPE_F32;
626 ir3_reg_create(mov, 0, 0);
627 ir3_reg_create(mov, n, IR3_REG_CONST);
628
629 return mov;
630 }
631
632 static struct ir3_instruction *
633 create_uniform_indirect(struct ir3_compile *ctx, unsigned n,
634 struct ir3_instruction *address)
635 {
636 struct ir3_instruction *mov;
637
638 mov = ir3_instr_create(ctx->block, 1, 0);
639 mov->cat1.src_type = TYPE_U32;
640 mov->cat1.dst_type = TYPE_U32;
641 ir3_reg_create(mov, 0, 0);
642 ir3_reg_create(mov, n, IR3_REG_CONST | IR3_REG_RELATIV);
643
644 ir3_instr_set_address(mov, address);
645
646 return mov;
647 }
648
649 static struct ir3_instruction *
650 create_collect(struct ir3_block *block, struct ir3_instruction **arr,
651 unsigned arrsz)
652 {
653 struct ir3_instruction *collect;
654
655 if (arrsz == 0)
656 return NULL;
657
658 collect = ir3_instr_create2(block, -1, OPC_META_FI, 1 + arrsz);
659 ir3_reg_create(collect, 0, 0); /* dst */
660 for (unsigned i = 0; i < arrsz; i++)
661 ir3_reg_create(collect, 0, IR3_REG_SSA)->instr = arr[i];
662
663 return collect;
664 }
665
666 static struct ir3_instruction *
667 create_indirect_load(struct ir3_compile *ctx, unsigned arrsz, unsigned n,
668 struct ir3_instruction *address, struct ir3_instruction *collect)
669 {
670 struct ir3_block *block = ctx->block;
671 struct ir3_instruction *mov;
672 struct ir3_register *src;
673
674 mov = ir3_instr_create(block, 1, 0);
675 mov->cat1.src_type = TYPE_U32;
676 mov->cat1.dst_type = TYPE_U32;
677 ir3_reg_create(mov, 0, 0);
678 src = ir3_reg_create(mov, 0, IR3_REG_SSA | IR3_REG_RELATIV);
679 src->instr = collect;
680 src->size = arrsz;
681 src->offset = n;
682
683 ir3_instr_set_address(mov, address);
684
685 return mov;
686 }
687
688 static struct ir3_instruction *
689 create_indirect_store(struct ir3_compile *ctx, unsigned arrsz, unsigned n,
690 struct ir3_instruction *src, struct ir3_instruction *address,
691 struct ir3_instruction *collect)
692 {
693 struct ir3_block *block = ctx->block;
694 struct ir3_instruction *mov;
695 struct ir3_register *dst;
696
697 mov = ir3_instr_create(block, 1, 0);
698 mov->cat1.src_type = TYPE_U32;
699 mov->cat1.dst_type = TYPE_U32;
700 dst = ir3_reg_create(mov, 0, IR3_REG_RELATIV);
701 dst->size = arrsz;
702 dst->offset = n;
703 ir3_reg_create(mov, 0, IR3_REG_SSA)->instr = src;
704 mov->fanin = collect;
705
706 ir3_instr_set_address(mov, address);
707
708 return mov;
709 }
710
711 static struct ir3_instruction *
712 create_input(struct ir3_block *block, unsigned n)
713 {
714 struct ir3_instruction *in;
715
716 in = ir3_instr_create(block, -1, OPC_META_INPUT);
717 in->inout.block = block;
718 ir3_reg_create(in, n, 0);
719
720 return in;
721 }
722
723 static struct ir3_instruction *
724 create_frag_input(struct ir3_compile *ctx, bool use_ldlv)
725 {
726 struct ir3_block *block = ctx->block;
727 struct ir3_instruction *instr;
728 /* actual inloc is assigned and fixed up later: */
729 struct ir3_instruction *inloc = create_immed(block, 0);
730
731 if (use_ldlv) {
732 instr = ir3_LDLV(block, inloc, 0, create_immed(block, 1), 0);
733 instr->cat6.type = TYPE_U32;
734 instr->cat6.iim_val = 1;
735 } else {
736 instr = ir3_BARY_F(block, inloc, 0, ctx->frag_pos, 0);
737 instr->regs[2]->wrmask = 0x3;
738 }
739
740 return instr;
741 }
742
743 static struct ir3_instruction *
744 create_frag_coord(struct ir3_compile *ctx, unsigned comp)
745 {
746 struct ir3_block *block = ctx->block;
747 struct ir3_instruction *instr;
748
749 compile_assert(ctx, !ctx->frag_coord[comp]);
750
751 ctx->frag_coord[comp] = create_input(ctx->block, 0);
752
753 switch (comp) {
754 case 0: /* .x */
755 case 1: /* .y */
756 /* for frag_coord, we get unsigned values.. we need
757 * to subtract (integer) 8 and divide by 16 (right-
758 * shift by 4) then convert to float:
759 *
760 * sub.s tmp, src, 8
761 * shr.b tmp, tmp, 4
762 * mov.u32f32 dst, tmp
763 *
764 */
765 instr = ir3_SUB_S(block, ctx->frag_coord[comp], 0,
766 create_immed(block, 8), 0);
767 instr = ir3_SHR_B(block, instr, 0,
768 create_immed(block, 4), 0);
769 instr = ir3_COV(block, instr, TYPE_U32, TYPE_F32);
770
771 return instr;
772 case 2: /* .z */
773 case 3: /* .w */
774 default:
775 /* seems that we can use these as-is: */
776 return ctx->frag_coord[comp];
777 }
778 }
779
780 static struct ir3_instruction *
781 create_frag_face(struct ir3_compile *ctx, unsigned comp)
782 {
783 struct ir3_block *block = ctx->block;
784 struct ir3_instruction *instr;
785
786 switch (comp) {
787 case 0: /* .x */
788 compile_assert(ctx, !ctx->frag_face);
789
790 ctx->frag_face = create_input(block, 0);
791 ctx->frag_face->regs[0]->flags |= IR3_REG_HALF;
792
793 /* for faceness, we always get -1 or 0 (int).. but TGSI expects
794 * positive vs negative float.. and piglit further seems to
795 * expect -1.0 or 1.0:
796 *
797 * mul.s tmp, hr0.x, 2
798 * add.s tmp, tmp, 1
799 * mov.s32f32, dst, tmp
800 *
801 */
802 instr = ir3_MUL_S(block, ctx->frag_face, 0,
803 create_immed(block, 2), 0);
804 instr = ir3_ADD_S(block, instr, 0,
805 create_immed(block, 1), 0);
806 instr = ir3_COV(block, instr, TYPE_S32, TYPE_F32);
807
808 return instr;
809 case 1: /* .y */
810 case 2: /* .z */
811 return create_immed(block, fui(0.0));
812 default:
813 case 3: /* .w */
814 return create_immed(block, fui(1.0));
815 }
816 }
817
818 static struct ir3_instruction *
819 create_driver_param(struct ir3_compile *ctx, enum ir3_driver_param dp)
820 {
821 /* first four vec4 sysval's reserved for UBOs: */
822 /* NOTE: dp is in scalar, but there can be >4 dp components: */
823 unsigned n = ctx->so->first_driver_param + IR3_DRIVER_PARAM_OFF;
824 unsigned r = regid(n + dp / 4, dp % 4);
825 return create_uniform(ctx, r);
826 }
827
828 /* helper for instructions that produce multiple consecutive scalar
829 * outputs which need to have a split/fanout meta instruction inserted
830 */
831 static void
832 split_dest(struct ir3_block *block, struct ir3_instruction **dst,
833 struct ir3_instruction *src, unsigned n)
834 {
835 struct ir3_instruction *prev = NULL;
836 for (int i = 0, j = 0; i < n; i++) {
837 struct ir3_instruction *split =
838 ir3_instr_create(block, -1, OPC_META_FO);
839 ir3_reg_create(split, 0, IR3_REG_SSA);
840 ir3_reg_create(split, 0, IR3_REG_SSA)->instr = src;
841 split->fo.off = i;
842
843 if (prev) {
844 split->cp.left = prev;
845 split->cp.left_cnt++;
846 prev->cp.right = split;
847 prev->cp.right_cnt++;
848 }
849 prev = split;
850
851 if (src->regs[0]->wrmask & (1 << i))
852 dst[j++] = split;
853 }
854 }
855
856 /*
857 * Adreno uses uint rather than having dedicated bool type,
858 * which (potentially) requires some conversion, in particular
859 * when using output of an bool instr to int input, or visa
860 * versa.
861 *
862 * | Adreno | NIR |
863 * -------+---------+-------+-
864 * true | 1 | ~0 |
865 * false | 0 | 0 |
866 *
867 * To convert from an adreno bool (uint) to nir, use:
868 *
869 * absneg.s dst, (neg)src
870 *
871 * To convert back in the other direction:
872 *
873 * absneg.s dst, (abs)arc
874 *
875 * The CP step can clean up the absneg.s that cancel each other
876 * out, and with a slight bit of extra cleverness (to recognize
877 * the instructions which produce either a 0 or 1) can eliminate
878 * the absneg.s's completely when an instruction that wants
879 * 0/1 consumes the result. For example, when a nir 'bcsel'
880 * consumes the result of 'feq'. So we should be able to get by
881 * without a boolean resolve step, and without incuring any
882 * extra penalty in instruction count.
883 */
884
885 /* NIR bool -> native (adreno): */
886 static struct ir3_instruction *
887 ir3_b2n(struct ir3_block *block, struct ir3_instruction *instr)
888 {
889 return ir3_ABSNEG_S(block, instr, IR3_REG_SABS);
890 }
891
892 /* native (adreno) -> NIR bool: */
893 static struct ir3_instruction *
894 ir3_n2b(struct ir3_block *block, struct ir3_instruction *instr)
895 {
896 return ir3_ABSNEG_S(block, instr, IR3_REG_SNEG);
897 }
898
899 /*
900 * alu/sfu instructions:
901 */
902
903 static void
904 emit_alu(struct ir3_compile *ctx, nir_alu_instr *alu)
905 {
906 const nir_op_info *info = &nir_op_infos[alu->op];
907 struct ir3_instruction **dst, *src[info->num_inputs];
908 struct ir3_block *b = ctx->block;
909
910 dst = get_dst(ctx, &alu->dest.dest, MAX2(info->output_size, 1));
911
912 /* Vectors are special in that they have non-scalarized writemasks,
913 * and just take the first swizzle channel for each argument in
914 * order into each writemask channel.
915 */
916 if ((alu->op == nir_op_vec2) ||
917 (alu->op == nir_op_vec3) ||
918 (alu->op == nir_op_vec4)) {
919
920 for (int i = 0; i < info->num_inputs; i++) {
921 nir_alu_src *asrc = &alu->src[i];
922
923 compile_assert(ctx, !asrc->abs);
924 compile_assert(ctx, !asrc->negate);
925
926 src[i] = get_src(ctx, &asrc->src)[asrc->swizzle[0]];
927 if (!src[i])
928 src[i] = create_immed(ctx->block, 0);
929 dst[i] = ir3_MOV(b, src[i], TYPE_U32);
930 }
931
932 return;
933 }
934
935 /* General case: We can just grab the one used channel per src. */
936 for (int i = 0; i < info->num_inputs; i++) {
937 unsigned chan = ffs(alu->dest.write_mask) - 1;
938 nir_alu_src *asrc = &alu->src[i];
939
940 compile_assert(ctx, !asrc->abs);
941 compile_assert(ctx, !asrc->negate);
942
943 src[i] = get_src(ctx, &asrc->src)[asrc->swizzle[chan]];
944
945 compile_assert(ctx, src[i]);
946 }
947
948 switch (alu->op) {
949 case nir_op_f2i:
950 dst[0] = ir3_COV(b, src[0], TYPE_F32, TYPE_S32);
951 break;
952 case nir_op_f2u:
953 dst[0] = ir3_COV(b, src[0], TYPE_F32, TYPE_U32);
954 break;
955 case nir_op_i2f:
956 dst[0] = ir3_COV(b, src[0], TYPE_S32, TYPE_F32);
957 break;
958 case nir_op_u2f:
959 dst[0] = ir3_COV(b, src[0], TYPE_U32, TYPE_F32);
960 break;
961 case nir_op_imov:
962 dst[0] = ir3_MOV(b, src[0], TYPE_S32);
963 break;
964 case nir_op_fmov:
965 dst[0] = ir3_MOV(b, src[0], TYPE_F32);
966 break;
967 case nir_op_f2b:
968 dst[0] = ir3_CMPS_F(b, src[0], 0, create_immed(b, fui(0.0)), 0);
969 dst[0]->cat2.condition = IR3_COND_NE;
970 dst[0] = ir3_n2b(b, dst[0]);
971 break;
972 case nir_op_b2f:
973 dst[0] = ir3_COV(b, ir3_b2n(b, src[0]), TYPE_U32, TYPE_F32);
974 break;
975 case nir_op_b2i:
976 dst[0] = ir3_b2n(b, src[0]);
977 break;
978 case nir_op_i2b:
979 dst[0] = ir3_CMPS_S(b, src[0], 0, create_immed(b, 0), 0);
980 dst[0]->cat2.condition = IR3_COND_NE;
981 dst[0] = ir3_n2b(b, dst[0]);
982 break;
983
984 case nir_op_fneg:
985 dst[0] = ir3_ABSNEG_F(b, src[0], IR3_REG_FNEG);
986 break;
987 case nir_op_fabs:
988 dst[0] = ir3_ABSNEG_F(b, src[0], IR3_REG_FABS);
989 break;
990 case nir_op_fmax:
991 dst[0] = ir3_MAX_F(b, src[0], 0, src[1], 0);
992 break;
993 case nir_op_fmin:
994 dst[0] = ir3_MIN_F(b, src[0], 0, src[1], 0);
995 break;
996 case nir_op_fmul:
997 dst[0] = ir3_MUL_F(b, src[0], 0, src[1], 0);
998 break;
999 case nir_op_fadd:
1000 dst[0] = ir3_ADD_F(b, src[0], 0, src[1], 0);
1001 break;
1002 case nir_op_fsub:
1003 dst[0] = ir3_ADD_F(b, src[0], 0, src[1], IR3_REG_FNEG);
1004 break;
1005 case nir_op_ffma:
1006 dst[0] = ir3_MAD_F32(b, src[0], 0, src[1], 0, src[2], 0);
1007 break;
1008 case nir_op_fddx:
1009 dst[0] = ir3_DSX(b, src[0], 0);
1010 dst[0]->cat5.type = TYPE_F32;
1011 break;
1012 case nir_op_fddy:
1013 dst[0] = ir3_DSY(b, src[0], 0);
1014 dst[0]->cat5.type = TYPE_F32;
1015 break;
1016 break;
1017 case nir_op_flt:
1018 dst[0] = ir3_CMPS_F(b, src[0], 0, src[1], 0);
1019 dst[0]->cat2.condition = IR3_COND_LT;
1020 dst[0] = ir3_n2b(b, dst[0]);
1021 break;
1022 case nir_op_fge:
1023 dst[0] = ir3_CMPS_F(b, src[0], 0, src[1], 0);
1024 dst[0]->cat2.condition = IR3_COND_GE;
1025 dst[0] = ir3_n2b(b, dst[0]);
1026 break;
1027 case nir_op_feq:
1028 dst[0] = ir3_CMPS_F(b, src[0], 0, src[1], 0);
1029 dst[0]->cat2.condition = IR3_COND_EQ;
1030 dst[0] = ir3_n2b(b, dst[0]);
1031 break;
1032 case nir_op_fne:
1033 dst[0] = ir3_CMPS_F(b, src[0], 0, src[1], 0);
1034 dst[0]->cat2.condition = IR3_COND_NE;
1035 dst[0] = ir3_n2b(b, dst[0]);
1036 break;
1037 case nir_op_fceil:
1038 dst[0] = ir3_CEIL_F(b, src[0], 0);
1039 break;
1040 case nir_op_ffloor:
1041 dst[0] = ir3_FLOOR_F(b, src[0], 0);
1042 break;
1043 case nir_op_ftrunc:
1044 dst[0] = ir3_TRUNC_F(b, src[0], 0);
1045 break;
1046 case nir_op_fround_even:
1047 dst[0] = ir3_RNDNE_F(b, src[0], 0);
1048 break;
1049 case nir_op_fsign:
1050 dst[0] = ir3_SIGN_F(b, src[0], 0);
1051 break;
1052
1053 case nir_op_fsin:
1054 dst[0] = ir3_SIN(b, src[0], 0);
1055 break;
1056 case nir_op_fcos:
1057 dst[0] = ir3_COS(b, src[0], 0);
1058 break;
1059 case nir_op_frsq:
1060 dst[0] = ir3_RSQ(b, src[0], 0);
1061 break;
1062 case nir_op_frcp:
1063 dst[0] = ir3_RCP(b, src[0], 0);
1064 break;
1065 case nir_op_flog2:
1066 dst[0] = ir3_LOG2(b, src[0], 0);
1067 break;
1068 case nir_op_fexp2:
1069 dst[0] = ir3_EXP2(b, src[0], 0);
1070 break;
1071 case nir_op_fsqrt:
1072 dst[0] = ir3_SQRT(b, src[0], 0);
1073 break;
1074
1075 case nir_op_iabs:
1076 dst[0] = ir3_ABSNEG_S(b, src[0], IR3_REG_SABS);
1077 break;
1078 case nir_op_iadd:
1079 dst[0] = ir3_ADD_U(b, src[0], 0, src[1], 0);
1080 break;
1081 case nir_op_iand:
1082 dst[0] = ir3_AND_B(b, src[0], 0, src[1], 0);
1083 break;
1084 case nir_op_imax:
1085 dst[0] = ir3_MAX_S(b, src[0], 0, src[1], 0);
1086 break;
1087 case nir_op_umax:
1088 dst[0] = ir3_MAX_U(b, src[0], 0, src[1], 0);
1089 break;
1090 case nir_op_imin:
1091 dst[0] = ir3_MIN_S(b, src[0], 0, src[1], 0);
1092 break;
1093 case nir_op_umin:
1094 dst[0] = ir3_MIN_U(b, src[0], 0, src[1], 0);
1095 break;
1096 case nir_op_imul:
1097 /*
1098 * dst = (al * bl) + (ah * bl << 16) + (al * bh << 16)
1099 * mull.u tmp0, a, b ; mul low, i.e. al * bl
1100 * madsh.m16 tmp1, a, b, tmp0 ; mul-add shift high mix, i.e. ah * bl << 16
1101 * madsh.m16 dst, b, a, tmp1 ; i.e. al * bh << 16
1102 */
1103 dst[0] = ir3_MADSH_M16(b, src[1], 0, src[0], 0,
1104 ir3_MADSH_M16(b, src[0], 0, src[1], 0,
1105 ir3_MULL_U(b, src[0], 0, src[1], 0), 0), 0);
1106 break;
1107 case nir_op_ineg:
1108 dst[0] = ir3_ABSNEG_S(b, src[0], IR3_REG_SNEG);
1109 break;
1110 case nir_op_inot:
1111 dst[0] = ir3_NOT_B(b, src[0], 0);
1112 break;
1113 case nir_op_ior:
1114 dst[0] = ir3_OR_B(b, src[0], 0, src[1], 0);
1115 break;
1116 case nir_op_ishl:
1117 dst[0] = ir3_SHL_B(b, src[0], 0, src[1], 0);
1118 break;
1119 case nir_op_ishr:
1120 dst[0] = ir3_ASHR_B(b, src[0], 0, src[1], 0);
1121 break;
1122 case nir_op_isign: {
1123 /* maybe this would be sane to lower in nir.. */
1124 struct ir3_instruction *neg, *pos;
1125
1126 neg = ir3_CMPS_S(b, src[0], 0, create_immed(b, 0), 0);
1127 neg->cat2.condition = IR3_COND_LT;
1128
1129 pos = ir3_CMPS_S(b, src[0], 0, create_immed(b, 0), 0);
1130 pos->cat2.condition = IR3_COND_GT;
1131
1132 dst[0] = ir3_SUB_U(b, pos, 0, neg, 0);
1133
1134 break;
1135 }
1136 case nir_op_isub:
1137 dst[0] = ir3_SUB_U(b, src[0], 0, src[1], 0);
1138 break;
1139 case nir_op_ixor:
1140 dst[0] = ir3_XOR_B(b, src[0], 0, src[1], 0);
1141 break;
1142 case nir_op_ushr:
1143 dst[0] = ir3_SHR_B(b, src[0], 0, src[1], 0);
1144 break;
1145 case nir_op_ilt:
1146 dst[0] = ir3_CMPS_S(b, src[0], 0, src[1], 0);
1147 dst[0]->cat2.condition = IR3_COND_LT;
1148 dst[0] = ir3_n2b(b, dst[0]);
1149 break;
1150 case nir_op_ige:
1151 dst[0] = ir3_CMPS_S(b, src[0], 0, src[1], 0);
1152 dst[0]->cat2.condition = IR3_COND_GE;
1153 dst[0] = ir3_n2b(b, dst[0]);
1154 break;
1155 case nir_op_ieq:
1156 dst[0] = ir3_CMPS_S(b, src[0], 0, src[1], 0);
1157 dst[0]->cat2.condition = IR3_COND_EQ;
1158 dst[0] = ir3_n2b(b, dst[0]);
1159 break;
1160 case nir_op_ine:
1161 dst[0] = ir3_CMPS_S(b, src[0], 0, src[1], 0);
1162 dst[0]->cat2.condition = IR3_COND_NE;
1163 dst[0] = ir3_n2b(b, dst[0]);
1164 break;
1165 case nir_op_ult:
1166 dst[0] = ir3_CMPS_U(b, src[0], 0, src[1], 0);
1167 dst[0]->cat2.condition = IR3_COND_LT;
1168 dst[0] = ir3_n2b(b, dst[0]);
1169 break;
1170 case nir_op_uge:
1171 dst[0] = ir3_CMPS_U(b, src[0], 0, src[1], 0);
1172 dst[0]->cat2.condition = IR3_COND_GE;
1173 dst[0] = ir3_n2b(b, dst[0]);
1174 break;
1175
1176 case nir_op_bcsel:
1177 dst[0] = ir3_SEL_B32(b, src[1], 0, ir3_b2n(b, src[0]), 0, src[2], 0);
1178 break;
1179
1180 case nir_op_bit_count:
1181 dst[0] = ir3_CBITS_B(b, src[0], 0);
1182 break;
1183 case nir_op_ifind_msb: {
1184 struct ir3_instruction *cmp;
1185 dst[0] = ir3_CLZ_S(b, src[0], 0);
1186 cmp = ir3_CMPS_S(b, dst[0], 0, create_immed(b, 0), 0);
1187 cmp->cat2.condition = IR3_COND_GE;
1188 dst[0] = ir3_SEL_B32(b,
1189 ir3_SUB_U(b, create_immed(b, 31), 0, dst[0], 0), 0,
1190 cmp, 0, dst[0], 0);
1191 break;
1192 }
1193 case nir_op_ufind_msb:
1194 dst[0] = ir3_CLZ_B(b, src[0], 0);
1195 dst[0] = ir3_SEL_B32(b,
1196 ir3_SUB_U(b, create_immed(b, 31), 0, dst[0], 0), 0,
1197 src[0], 0, dst[0], 0);
1198 break;
1199 case nir_op_find_lsb:
1200 dst[0] = ir3_BFREV_B(b, src[0], 0);
1201 dst[0] = ir3_CLZ_B(b, dst[0], 0);
1202 break;
1203 case nir_op_bitfield_reverse:
1204 dst[0] = ir3_BFREV_B(b, src[0], 0);
1205 break;
1206
1207 default:
1208 compile_error(ctx, "Unhandled ALU op: %s\n",
1209 nir_op_infos[alu->op].name);
1210 break;
1211 }
1212 }
1213
1214 /* handles direct/indirect UBO reads: */
1215 static void
1216 emit_intrinsic_load_ubo(struct ir3_compile *ctx, nir_intrinsic_instr *intr,
1217 struct ir3_instruction **dst)
1218 {
1219 struct ir3_block *b = ctx->block;
1220 struct ir3_instruction *addr, *src0, *src1;
1221 nir_const_value *const_offset;
1222 /* UBO addresses are the first driver params: */
1223 unsigned ubo = regid(ctx->so->first_driver_param + IR3_UBOS_OFF, 0);
1224 unsigned off = intr->const_index[0];
1225
1226 /* First src is ubo index, which could either be an immed or not: */
1227 src0 = get_src(ctx, &intr->src[0])[0];
1228 if (is_same_type_mov(src0) &&
1229 (src0->regs[1]->flags & IR3_REG_IMMED)) {
1230 addr = create_uniform(ctx, ubo + src0->regs[1]->iim_val);
1231 } else {
1232 addr = create_uniform_indirect(ctx, ubo, get_addr(ctx, src0));
1233 }
1234
1235 const_offset = nir_src_as_const_value(intr->src[1]);
1236 if (const_offset) {
1237 off += const_offset->u[0];
1238 } else {
1239 /* For load_ubo_indirect, second src is indirect offset: */
1240 src1 = get_src(ctx, &intr->src[1])[0];
1241
1242 /* and add offset to addr: */
1243 addr = ir3_ADD_S(b, addr, 0, src1, 0);
1244 }
1245
1246 /* if offset is to large to encode in the ldg, split it out: */
1247 if ((off + (intr->num_components * 4)) > 1024) {
1248 /* split out the minimal amount to improve the odds that
1249 * cp can fit the immediate in the add.s instruction:
1250 */
1251 unsigned off2 = off + (intr->num_components * 4) - 1024;
1252 addr = ir3_ADD_S(b, addr, 0, create_immed(b, off2), 0);
1253 off -= off2;
1254 }
1255
1256 for (int i = 0; i < intr->num_components; i++) {
1257 struct ir3_instruction *load =
1258 ir3_LDG(b, addr, 0, create_immed(b, 1), 0);
1259 load->cat6.type = TYPE_U32;
1260 load->cat6.src_offset = off + i * 4; /* byte offset */
1261 dst[i] = load;
1262 }
1263 }
1264
1265 /* handles array reads: */
1266 static void
1267 emit_intrinsic_load_var(struct ir3_compile *ctx, nir_intrinsic_instr *intr,
1268 struct ir3_instruction **dst)
1269 {
1270 nir_deref_var *dvar = intr->variables[0];
1271 nir_deref_array *darr = nir_deref_as_array(dvar->deref.child);
1272 struct ir3_array_value *arr = get_var(ctx, dvar->var);
1273
1274 compile_assert(ctx, dvar->deref.child &&
1275 (dvar->deref.child->deref_type == nir_deref_type_array));
1276
1277 switch (darr->deref_array_type) {
1278 case nir_deref_array_type_direct:
1279 /* direct access does not require anything special: */
1280 for (int i = 0; i < intr->num_components; i++) {
1281 unsigned n = darr->base_offset * 4 + i;
1282 compile_assert(ctx, n < arr->length);
1283 dst[i] = arr->arr[n];
1284 }
1285 break;
1286 case nir_deref_array_type_indirect: {
1287 /* for indirect, we need to collect all the array elements: */
1288 struct ir3_instruction *collect =
1289 create_collect(ctx->block, arr->arr, arr->length);
1290 struct ir3_instruction *addr =
1291 get_addr(ctx, get_src(ctx, &darr->indirect)[0]);
1292 for (int i = 0; i < intr->num_components; i++) {
1293 unsigned n = darr->base_offset * 4 + i;
1294 compile_assert(ctx, n < arr->length);
1295 dst[i] = create_indirect_load(ctx, arr->length, n, addr, collect);
1296 }
1297 break;
1298 }
1299 default:
1300 compile_error(ctx, "Unhandled load deref type: %u\n",
1301 darr->deref_array_type);
1302 break;
1303 }
1304 }
1305
1306 /* handles array writes: */
1307 static void
1308 emit_intrinsic_store_var(struct ir3_compile *ctx, nir_intrinsic_instr *intr)
1309 {
1310 nir_deref_var *dvar = intr->variables[0];
1311 nir_deref_array *darr = nir_deref_as_array(dvar->deref.child);
1312 struct ir3_array_value *arr = get_var(ctx, dvar->var);
1313 struct ir3_instruction **src;
1314
1315 compile_assert(ctx, dvar->deref.child &&
1316 (dvar->deref.child->deref_type == nir_deref_type_array));
1317
1318 src = get_src(ctx, &intr->src[0]);
1319
1320 switch (darr->deref_array_type) {
1321 case nir_deref_array_type_direct:
1322 /* direct access does not require anything special: */
1323 for (int i = 0; i < intr->num_components; i++) {
1324 /* ttn doesn't generate partial writemasks */
1325 assert(intr->const_index[0] ==
1326 (1 << intr->num_components) - 1);
1327
1328 unsigned n = darr->base_offset * 4 + i;
1329 compile_assert(ctx, n < arr->length);
1330 arr->arr[n] = src[i];
1331 }
1332 break;
1333 case nir_deref_array_type_indirect: {
1334 /* for indirect, create indirect-store and fan that out: */
1335 struct ir3_instruction *collect =
1336 create_collect(ctx->block, arr->arr, arr->length);
1337 struct ir3_instruction *addr =
1338 get_addr(ctx, get_src(ctx, &darr->indirect)[0]);
1339 for (int i = 0; i < intr->num_components; i++) {
1340 /* ttn doesn't generate partial writemasks */
1341 assert(intr->const_index[0] ==
1342 (1 << intr->num_components) - 1);
1343
1344 struct ir3_instruction *store;
1345 unsigned n = darr->base_offset * 4 + i;
1346 compile_assert(ctx, n < arr->length);
1347
1348 store = create_indirect_store(ctx, arr->length,
1349 n, src[i], addr, collect);
1350
1351 store->fanin->fi.aid = arr->aid;
1352
1353 /* TODO: probably split this out to be used for
1354 * store_output_indirect? or move this into
1355 * create_indirect_store()?
1356 */
1357 for (int j = i; j < arr->length; j += intr->num_components) {
1358 struct ir3_instruction *split;
1359
1360 split = ir3_instr_create(ctx->block, -1, OPC_META_FO);
1361 split->fo.off = j;
1362 ir3_reg_create(split, 0, 0);
1363 ir3_reg_create(split, 0, IR3_REG_SSA)->instr = store;
1364
1365 arr->arr[j] = split;
1366 }
1367 }
1368 /* fixup fanout/split neighbors: */
1369 for (int i = 0; i < arr->length; i++) {
1370 arr->arr[i]->cp.right = (i < (arr->length - 1)) ?
1371 arr->arr[i+1] : NULL;
1372 arr->arr[i]->cp.left = (i > 0) ?
1373 arr->arr[i-1] : NULL;
1374 }
1375 break;
1376 }
1377 default:
1378 compile_error(ctx, "Unhandled store deref type: %u\n",
1379 darr->deref_array_type);
1380 break;
1381 }
1382 }
1383
1384 static void add_sysval_input(struct ir3_compile *ctx, gl_system_value slot,
1385 struct ir3_instruction *instr)
1386 {
1387 struct ir3_shader_variant *so = ctx->so;
1388 unsigned r = regid(so->inputs_count, 0);
1389 unsigned n = so->inputs_count++;
1390
1391 so->inputs[n].sysval = true;
1392 so->inputs[n].slot = slot;
1393 so->inputs[n].compmask = 1;
1394 so->inputs[n].regid = r;
1395 so->inputs[n].interpolate = INTERP_QUALIFIER_FLAT;
1396 so->total_in++;
1397
1398 ctx->ir->ninputs = MAX2(ctx->ir->ninputs, r + 1);
1399 ctx->ir->inputs[r] = instr;
1400 }
1401
1402 static void
1403 emit_intrinsic(struct ir3_compile *ctx, nir_intrinsic_instr *intr)
1404 {
1405 const nir_intrinsic_info *info = &nir_intrinsic_infos[intr->intrinsic];
1406 struct ir3_instruction **dst, **src;
1407 struct ir3_block *b = ctx->block;
1408 unsigned idx = intr->const_index[0];
1409 nir_const_value *const_offset;
1410
1411 if (info->has_dest) {
1412 dst = get_dst(ctx, &intr->dest, intr->num_components);
1413 } else {
1414 dst = NULL;
1415 }
1416
1417 switch (intr->intrinsic) {
1418 case nir_intrinsic_load_uniform:
1419 const_offset = nir_src_as_const_value(intr->src[0]);
1420 if (const_offset) {
1421 idx += const_offset->u[0];
1422 for (int i = 0; i < intr->num_components; i++) {
1423 unsigned n = idx * 4 + i;
1424 dst[i] = create_uniform(ctx, n);
1425 }
1426 } else {
1427 src = get_src(ctx, &intr->src[0]);
1428 for (int i = 0; i < intr->num_components; i++) {
1429 unsigned n = idx * 4 + i;
1430 dst[i] = create_uniform_indirect(ctx, n,
1431 get_addr(ctx, src[0]));
1432 }
1433 /* NOTE: if relative addressing is used, we set
1434 * constlen in the compiler (to worst-case value)
1435 * since we don't know in the assembler what the max
1436 * addr reg value can be:
1437 */
1438 ctx->so->constlen = ctx->s->num_uniforms;
1439 }
1440 break;
1441 case nir_intrinsic_load_ubo:
1442 emit_intrinsic_load_ubo(ctx, intr, dst);
1443 break;
1444 case nir_intrinsic_load_input:
1445 const_offset = nir_src_as_const_value(intr->src[0]);
1446 if (const_offset) {
1447 idx += const_offset->u[0];
1448 for (int i = 0; i < intr->num_components; i++) {
1449 unsigned n = idx * 4 + i;
1450 dst[i] = ctx->ir->inputs[n];
1451 }
1452 } else {
1453 src = get_src(ctx, &intr->src[0]);
1454 struct ir3_instruction *collect =
1455 create_collect(b, ctx->ir->inputs, ctx->ir->ninputs);
1456 struct ir3_instruction *addr = get_addr(ctx, src[0]);
1457 for (int i = 0; i < intr->num_components; i++) {
1458 unsigned n = idx * 4 + i;
1459 dst[i] = create_indirect_load(ctx, ctx->ir->ninputs,
1460 n, addr, collect);
1461 }
1462 }
1463 break;
1464 case nir_intrinsic_load_var:
1465 emit_intrinsic_load_var(ctx, intr, dst);
1466 break;
1467 case nir_intrinsic_store_var:
1468 emit_intrinsic_store_var(ctx, intr);
1469 break;
1470 case nir_intrinsic_store_output:
1471 const_offset = nir_src_as_const_value(intr->src[1]);
1472 compile_assert(ctx, const_offset != NULL);
1473 idx += const_offset->u[0];
1474
1475 src = get_src(ctx, &intr->src[0]);
1476 for (int i = 0; i < intr->num_components; i++) {
1477 unsigned n = idx * 4 + i;
1478 ctx->ir->outputs[n] = src[i];
1479 }
1480 break;
1481 case nir_intrinsic_load_base_vertex:
1482 if (!ctx->basevertex) {
1483 ctx->basevertex = create_driver_param(ctx, IR3_DP_VTXID_BASE);
1484 add_sysval_input(ctx, SYSTEM_VALUE_BASE_VERTEX,
1485 ctx->basevertex);
1486 }
1487 dst[0] = ctx->basevertex;
1488 break;
1489 case nir_intrinsic_load_vertex_id_zero_base:
1490 if (!ctx->vertex_id) {
1491 ctx->vertex_id = create_input(ctx->block, 0);
1492 add_sysval_input(ctx, SYSTEM_VALUE_VERTEX_ID_ZERO_BASE,
1493 ctx->vertex_id);
1494 }
1495 dst[0] = ctx->vertex_id;
1496 break;
1497 case nir_intrinsic_load_instance_id:
1498 if (!ctx->instance_id) {
1499 ctx->instance_id = create_input(ctx->block, 0);
1500 add_sysval_input(ctx, SYSTEM_VALUE_INSTANCE_ID,
1501 ctx->instance_id);
1502 }
1503 dst[0] = ctx->instance_id;
1504 break;
1505 case nir_intrinsic_load_user_clip_plane:
1506 for (int i = 0; i < intr->num_components; i++) {
1507 unsigned n = idx * 4 + i;
1508 dst[i] = create_driver_param(ctx, IR3_DP_UCP0_X + n);
1509 }
1510 break;
1511 case nir_intrinsic_discard_if:
1512 case nir_intrinsic_discard: {
1513 struct ir3_instruction *cond, *kill;
1514
1515 if (intr->intrinsic == nir_intrinsic_discard_if) {
1516 /* conditional discard: */
1517 src = get_src(ctx, &intr->src[0]);
1518 cond = ir3_b2n(b, src[0]);
1519 } else {
1520 /* unconditional discard: */
1521 cond = create_immed(b, 1);
1522 }
1523
1524 /* NOTE: only cmps.*.* can write p0.x: */
1525 cond = ir3_CMPS_S(b, cond, 0, create_immed(b, 0), 0);
1526 cond->cat2.condition = IR3_COND_NE;
1527
1528 /* condition always goes in predicate register: */
1529 cond->regs[0]->num = regid(REG_P0, 0);
1530
1531 kill = ir3_KILL(b, cond, 0);
1532 array_insert(ctx->ir->predicates, kill);
1533
1534 array_insert(ctx->ir->keeps, kill);
1535 ctx->so->has_kill = true;
1536
1537 break;
1538 }
1539 default:
1540 compile_error(ctx, "Unhandled intrinsic type: %s\n",
1541 nir_intrinsic_infos[intr->intrinsic].name);
1542 break;
1543 }
1544 }
1545
1546 static void
1547 emit_load_const(struct ir3_compile *ctx, nir_load_const_instr *instr)
1548 {
1549 struct ir3_instruction **dst = get_dst_ssa(ctx, &instr->def,
1550 instr->def.num_components);
1551 for (int i = 0; i < instr->def.num_components; i++)
1552 dst[i] = create_immed(ctx->block, instr->value.u[i]);
1553 }
1554
1555 static void
1556 emit_undef(struct ir3_compile *ctx, nir_ssa_undef_instr *undef)
1557 {
1558 struct ir3_instruction **dst = get_dst_ssa(ctx, &undef->def,
1559 undef->def.num_components);
1560 /* backend doesn't want undefined instructions, so just plug
1561 * in 0.0..
1562 */
1563 for (int i = 0; i < undef->def.num_components; i++)
1564 dst[i] = create_immed(ctx->block, fui(0.0));
1565 }
1566
1567 /*
1568 * texture fetch/sample instructions:
1569 */
1570
1571 static void
1572 tex_info(nir_tex_instr *tex, unsigned *flagsp, unsigned *coordsp)
1573 {
1574 unsigned coords, flags = 0;
1575
1576 /* note: would use tex->coord_components.. except txs.. also,
1577 * since array index goes after shadow ref, we don't want to
1578 * count it:
1579 */
1580 switch (tex->sampler_dim) {
1581 case GLSL_SAMPLER_DIM_1D:
1582 case GLSL_SAMPLER_DIM_BUF:
1583 coords = 1;
1584 break;
1585 case GLSL_SAMPLER_DIM_2D:
1586 case GLSL_SAMPLER_DIM_RECT:
1587 case GLSL_SAMPLER_DIM_EXTERNAL:
1588 case GLSL_SAMPLER_DIM_MS:
1589 coords = 2;
1590 break;
1591 case GLSL_SAMPLER_DIM_3D:
1592 case GLSL_SAMPLER_DIM_CUBE:
1593 coords = 3;
1594 flags |= IR3_INSTR_3D;
1595 break;
1596 default:
1597 unreachable("bad sampler_dim");
1598 }
1599
1600 if (tex->is_shadow && tex->op != nir_texop_lod)
1601 flags |= IR3_INSTR_S;
1602
1603 if (tex->is_array && tex->op != nir_texop_lod)
1604 flags |= IR3_INSTR_A;
1605
1606 *flagsp = flags;
1607 *coordsp = coords;
1608 }
1609
1610 static void
1611 emit_tex(struct ir3_compile *ctx, nir_tex_instr *tex)
1612 {
1613 struct ir3_block *b = ctx->block;
1614 struct ir3_instruction **dst, *sam, *src0[12], *src1[4];
1615 struct ir3_instruction **coord, *lod, *compare, *proj, **off, **ddx, **ddy;
1616 bool has_bias = false, has_lod = false, has_proj = false, has_off = false;
1617 unsigned i, coords, flags;
1618 unsigned nsrc0 = 0, nsrc1 = 0;
1619 type_t type;
1620 opc_t opc = 0;
1621
1622 coord = off = ddx = ddy = NULL;
1623 lod = proj = compare = NULL;
1624
1625 /* TODO: might just be one component for gathers? */
1626 dst = get_dst(ctx, &tex->dest, 4);
1627
1628 for (unsigned i = 0; i < tex->num_srcs; i++) {
1629 switch (tex->src[i].src_type) {
1630 case nir_tex_src_coord:
1631 coord = get_src(ctx, &tex->src[i].src);
1632 break;
1633 case nir_tex_src_bias:
1634 lod = get_src(ctx, &tex->src[i].src)[0];
1635 has_bias = true;
1636 break;
1637 case nir_tex_src_lod:
1638 lod = get_src(ctx, &tex->src[i].src)[0];
1639 has_lod = true;
1640 break;
1641 case nir_tex_src_comparitor: /* shadow comparator */
1642 compare = get_src(ctx, &tex->src[i].src)[0];
1643 break;
1644 case nir_tex_src_projector:
1645 proj = get_src(ctx, &tex->src[i].src)[0];
1646 has_proj = true;
1647 break;
1648 case nir_tex_src_offset:
1649 off = get_src(ctx, &tex->src[i].src);
1650 has_off = true;
1651 break;
1652 case nir_tex_src_ddx:
1653 ddx = get_src(ctx, &tex->src[i].src);
1654 break;
1655 case nir_tex_src_ddy:
1656 ddy = get_src(ctx, &tex->src[i].src);
1657 break;
1658 default:
1659 compile_error(ctx, "Unhandled NIR tex serc type: %d\n",
1660 tex->src[i].src_type);
1661 return;
1662 }
1663 }
1664
1665 switch (tex->op) {
1666 case nir_texop_tex: opc = OPC_SAM; break;
1667 case nir_texop_txb: opc = OPC_SAMB; break;
1668 case nir_texop_txl: opc = OPC_SAML; break;
1669 case nir_texop_txd: opc = OPC_SAMGQ; break;
1670 case nir_texop_txf: opc = OPC_ISAML; break;
1671 case nir_texop_lod: opc = OPC_GETLOD; break;
1672 case nir_texop_txf_ms:
1673 case nir_texop_txs:
1674 case nir_texop_tg4:
1675 case nir_texop_query_levels:
1676 case nir_texop_texture_samples:
1677 case nir_texop_samples_identical:
1678 compile_error(ctx, "Unhandled NIR tex type: %d\n", tex->op);
1679 return;
1680 }
1681
1682 tex_info(tex, &flags, &coords);
1683
1684 /* scale up integer coords for TXF based on the LOD */
1685 if (ctx->unminify_coords && (opc == OPC_ISAML)) {
1686 assert(has_lod);
1687 for (i = 0; i < coords; i++)
1688 coord[i] = ir3_SHL_B(b, coord[i], 0, lod, 0);
1689 }
1690
1691 /* the array coord for cube arrays needs 0.5 added to it */
1692 if (tex->sampler_dim == GLSL_SAMPLER_DIM_CUBE && tex->is_array &&
1693 opc != OPC_ISAML)
1694 coord[3] = ir3_ADD_F(b, coord[3], 0, create_immed(b, fui(0.5)), 0);
1695
1696 /*
1697 * lay out the first argument in the proper order:
1698 * - actual coordinates first
1699 * - shadow reference
1700 * - array index
1701 * - projection w
1702 * - starting at offset 4, dpdx.xy, dpdy.xy
1703 *
1704 * bias/lod go into the second arg
1705 */
1706
1707 /* insert tex coords: */
1708 for (i = 0; i < coords; i++)
1709 src0[nsrc0++] = coord[i];
1710
1711 if (coords == 1) {
1712 /* hw doesn't do 1d, so we treat it as 2d with
1713 * height of 1, and patch up the y coord.
1714 * TODO: y coord should be (int)0 in some cases..
1715 */
1716 src0[nsrc0++] = create_immed(b, fui(0.5));
1717 }
1718
1719 if (tex->is_shadow && tex->op != nir_texop_lod)
1720 src0[nsrc0++] = compare;
1721
1722 if (tex->is_array && tex->op != nir_texop_lod)
1723 src0[nsrc0++] = coord[coords];
1724
1725 if (has_proj) {
1726 src0[nsrc0++] = proj;
1727 flags |= IR3_INSTR_P;
1728 }
1729
1730 /* pad to 4, then ddx/ddy: */
1731 if (tex->op == nir_texop_txd) {
1732 while (nsrc0 < 4)
1733 src0[nsrc0++] = create_immed(b, fui(0.0));
1734 for (i = 0; i < coords; i++)
1735 src0[nsrc0++] = ddx[i];
1736 if (coords < 2)
1737 src0[nsrc0++] = create_immed(b, fui(0.0));
1738 for (i = 0; i < coords; i++)
1739 src0[nsrc0++] = ddy[i];
1740 if (coords < 2)
1741 src0[nsrc0++] = create_immed(b, fui(0.0));
1742 }
1743
1744 /*
1745 * second argument (if applicable):
1746 * - offsets
1747 * - lod
1748 * - bias
1749 */
1750 if (has_off | has_lod | has_bias) {
1751 if (has_off) {
1752 for (i = 0; i < coords; i++)
1753 src1[nsrc1++] = off[i];
1754 if (coords < 2)
1755 src1[nsrc1++] = create_immed(b, fui(0.0));
1756 flags |= IR3_INSTR_O;
1757 }
1758
1759 if (has_lod | has_bias)
1760 src1[nsrc1++] = lod;
1761 }
1762
1763 switch (tex->dest_type) {
1764 case nir_type_invalid:
1765 case nir_type_float:
1766 type = TYPE_F32;
1767 break;
1768 case nir_type_int:
1769 type = TYPE_S32;
1770 break;
1771 case nir_type_uint:
1772 case nir_type_bool:
1773 type = TYPE_U32;
1774 break;
1775 default:
1776 unreachable("bad dest_type");
1777 }
1778
1779 if (opc == OPC_GETLOD)
1780 type = TYPE_U32;
1781
1782 sam = ir3_SAM(b, opc, type, TGSI_WRITEMASK_XYZW,
1783 flags, tex->sampler_index, tex->sampler_index,
1784 create_collect(b, src0, nsrc0),
1785 create_collect(b, src1, nsrc1));
1786
1787 split_dest(b, dst, sam, 4);
1788
1789 /* GETLOD returns results in 4.8 fixed point */
1790 if (opc == OPC_GETLOD) {
1791 struct ir3_instruction *factor = create_immed(b, fui(1.0 / 256));
1792
1793 compile_assert(ctx, tex->dest_type == nir_type_float);
1794 for (i = 0; i < 2; i++) {
1795 dst[i] = ir3_MUL_F(b, ir3_COV(b, dst[i], TYPE_U32, TYPE_F32), 0,
1796 factor, 0);
1797 }
1798 }
1799 }
1800
1801 static void
1802 emit_tex_query_levels(struct ir3_compile *ctx, nir_tex_instr *tex)
1803 {
1804 struct ir3_block *b = ctx->block;
1805 struct ir3_instruction **dst, *sam;
1806
1807 dst = get_dst(ctx, &tex->dest, 1);
1808
1809 sam = ir3_SAM(b, OPC_GETINFO, TYPE_U32, TGSI_WRITEMASK_Z, 0,
1810 tex->sampler_index, tex->sampler_index, NULL, NULL);
1811
1812 /* even though there is only one component, since it ends
1813 * up in .z rather than .x, we need a split_dest()
1814 */
1815 split_dest(b, dst, sam, 3);
1816
1817 /* The # of levels comes from getinfo.z. We need to add 1 to it, since
1818 * the value in TEX_CONST_0 is zero-based.
1819 */
1820 if (ctx->levels_add_one)
1821 dst[0] = ir3_ADD_U(b, dst[0], 0, create_immed(b, 1), 0);
1822 }
1823
1824 static void
1825 emit_tex_txs(struct ir3_compile *ctx, nir_tex_instr *tex)
1826 {
1827 struct ir3_block *b = ctx->block;
1828 struct ir3_instruction **dst, *sam, *lod;
1829 unsigned flags, coords;
1830
1831 tex_info(tex, &flags, &coords);
1832
1833 /* Actually we want the number of dimensions, not coordinates. This
1834 * distinction only matters for cubes.
1835 */
1836 if (tex->sampler_dim == GLSL_SAMPLER_DIM_CUBE)
1837 coords = 2;
1838
1839 dst = get_dst(ctx, &tex->dest, 4);
1840
1841 compile_assert(ctx, tex->num_srcs == 1);
1842 compile_assert(ctx, tex->src[0].src_type == nir_tex_src_lod);
1843
1844 lod = get_src(ctx, &tex->src[0].src)[0];
1845
1846 sam = ir3_SAM(b, OPC_GETSIZE, TYPE_U32, TGSI_WRITEMASK_XYZW, flags,
1847 tex->sampler_index, tex->sampler_index, lod, NULL);
1848
1849 split_dest(b, dst, sam, 4);
1850
1851 /* Array size actually ends up in .w rather than .z. This doesn't
1852 * matter for miplevel 0, but for higher mips the value in z is
1853 * minified whereas w stays. Also, the value in TEX_CONST_3_DEPTH is
1854 * returned, which means that we have to add 1 to it for arrays.
1855 */
1856 if (tex->is_array) {
1857 if (ctx->levels_add_one) {
1858 dst[coords] = ir3_ADD_U(b, dst[3], 0, create_immed(b, 1), 0);
1859 } else {
1860 dst[coords] = ir3_MOV(b, dst[3], TYPE_U32);
1861 }
1862 }
1863 }
1864
1865 static void
1866 emit_phi(struct ir3_compile *ctx, nir_phi_instr *nphi)
1867 {
1868 struct ir3_instruction *phi, **dst;
1869
1870 /* NOTE: phi's should be lowered to scalar at this point */
1871 compile_assert(ctx, nphi->dest.ssa.num_components == 1);
1872
1873 dst = get_dst(ctx, &nphi->dest, 1);
1874
1875 phi = ir3_instr_create2(ctx->block, -1, OPC_META_PHI,
1876 1 + exec_list_length(&nphi->srcs));
1877 ir3_reg_create(phi, 0, 0); /* dst */
1878 phi->phi.nphi = nphi;
1879
1880 dst[0] = phi;
1881 }
1882
1883 /* phi instructions are left partially constructed. We don't resolve
1884 * their srcs until the end of the block, since (eg. loops) one of
1885 * the phi's srcs might be defined after the phi due to back edges in
1886 * the CFG.
1887 */
1888 static void
1889 resolve_phis(struct ir3_compile *ctx, struct ir3_block *block)
1890 {
1891 list_for_each_entry (struct ir3_instruction, instr, &block->instr_list, node) {
1892 nir_phi_instr *nphi;
1893
1894 /* phi's only come at start of block: */
1895 if (!(is_meta(instr) && (instr->opc == OPC_META_PHI)))
1896 break;
1897
1898 if (!instr->phi.nphi)
1899 break;
1900
1901 nphi = instr->phi.nphi;
1902 instr->phi.nphi = NULL;
1903
1904 foreach_list_typed(nir_phi_src, nsrc, node, &nphi->srcs) {
1905 struct ir3_instruction *src = get_src(ctx, &nsrc->src)[0];
1906 ir3_reg_create(instr, 0, IR3_REG_SSA)->instr = src;
1907 }
1908 }
1909
1910 resolve_array_phis(ctx, block);
1911 }
1912
1913 static void
1914 emit_jump(struct ir3_compile *ctx, nir_jump_instr *jump)
1915 {
1916 switch (jump->type) {
1917 case nir_jump_break:
1918 case nir_jump_continue:
1919 /* I *think* we can simply just ignore this, and use the
1920 * successor block link to figure out where we need to
1921 * jump to for break/continue
1922 */
1923 break;
1924 default:
1925 compile_error(ctx, "Unhandled NIR jump type: %d\n", jump->type);
1926 break;
1927 }
1928 }
1929
1930 static void
1931 emit_instr(struct ir3_compile *ctx, nir_instr *instr)
1932 {
1933 switch (instr->type) {
1934 case nir_instr_type_alu:
1935 emit_alu(ctx, nir_instr_as_alu(instr));
1936 break;
1937 case nir_instr_type_intrinsic:
1938 emit_intrinsic(ctx, nir_instr_as_intrinsic(instr));
1939 break;
1940 case nir_instr_type_load_const:
1941 emit_load_const(ctx, nir_instr_as_load_const(instr));
1942 break;
1943 case nir_instr_type_ssa_undef:
1944 emit_undef(ctx, nir_instr_as_ssa_undef(instr));
1945 break;
1946 case nir_instr_type_tex: {
1947 nir_tex_instr *tex = nir_instr_as_tex(instr);
1948 /* couple tex instructions get special-cased:
1949 */
1950 switch (tex->op) {
1951 case nir_texop_txs:
1952 emit_tex_txs(ctx, tex);
1953 break;
1954 case nir_texop_query_levels:
1955 emit_tex_query_levels(ctx, tex);
1956 break;
1957 case nir_texop_samples_identical:
1958 unreachable("nir_texop_samples_identical");
1959 default:
1960 emit_tex(ctx, tex);
1961 break;
1962 }
1963 break;
1964 }
1965 case nir_instr_type_phi:
1966 emit_phi(ctx, nir_instr_as_phi(instr));
1967 break;
1968 case nir_instr_type_jump:
1969 emit_jump(ctx, nir_instr_as_jump(instr));
1970 break;
1971 case nir_instr_type_call:
1972 case nir_instr_type_parallel_copy:
1973 compile_error(ctx, "Unhandled NIR instruction type: %d\n", instr->type);
1974 break;
1975 }
1976 }
1977
1978 static struct ir3_block *
1979 get_block(struct ir3_compile *ctx, nir_block *nblock)
1980 {
1981 struct ir3_block *block;
1982 struct hash_entry *entry;
1983 entry = _mesa_hash_table_search(ctx->block_ht, nblock);
1984 if (entry)
1985 return entry->data;
1986
1987 block = ir3_block_create(ctx->ir);
1988 block->nblock = nblock;
1989 _mesa_hash_table_insert(ctx->block_ht, nblock, block);
1990
1991 return block;
1992 }
1993
1994 static void
1995 emit_block(struct ir3_compile *ctx, nir_block *nblock)
1996 {
1997 struct ir3_block *block = get_block(ctx, nblock);
1998
1999 for (int i = 0; i < ARRAY_SIZE(block->successors); i++) {
2000 if (nblock->successors[i]) {
2001 block->successors[i] =
2002 get_block(ctx, nblock->successors[i]);
2003 }
2004 }
2005
2006 ctx->block = block;
2007 list_addtail(&block->node, &ctx->ir->block_list);
2008
2009 /* re-emit addr register in each block if needed: */
2010 _mesa_hash_table_destroy(ctx->addr_ht, NULL);
2011 ctx->addr_ht = NULL;
2012
2013 nir_foreach_instr(nblock, instr) {
2014 emit_instr(ctx, instr);
2015 if (ctx->error)
2016 return;
2017 }
2018 }
2019
2020 static void emit_cf_list(struct ir3_compile *ctx, struct exec_list *list);
2021
2022 static void
2023 emit_if(struct ir3_compile *ctx, nir_if *nif)
2024 {
2025 struct ir3_instruction *condition = get_src(ctx, &nif->condition)[0];
2026
2027 ctx->block->condition =
2028 get_predicate(ctx, ir3_b2n(condition->block, condition));
2029
2030 emit_cf_list(ctx, &nif->then_list);
2031 emit_cf_list(ctx, &nif->else_list);
2032 }
2033
2034 static void
2035 emit_loop(struct ir3_compile *ctx, nir_loop *nloop)
2036 {
2037 emit_cf_list(ctx, &nloop->body);
2038 }
2039
2040 static void
2041 emit_cf_list(struct ir3_compile *ctx, struct exec_list *list)
2042 {
2043 foreach_list_typed(nir_cf_node, node, node, list) {
2044 switch (node->type) {
2045 case nir_cf_node_block:
2046 emit_block(ctx, nir_cf_node_as_block(node));
2047 break;
2048 case nir_cf_node_if:
2049 emit_if(ctx, nir_cf_node_as_if(node));
2050 break;
2051 case nir_cf_node_loop:
2052 emit_loop(ctx, nir_cf_node_as_loop(node));
2053 break;
2054 case nir_cf_node_function:
2055 compile_error(ctx, "TODO\n");
2056 break;
2057 }
2058 }
2059 }
2060
2061 /* emit stream-out code. At this point, the current block is the original
2062 * (nir) end block, and nir ensures that all flow control paths terminate
2063 * into the end block. We re-purpose the original end block to generate
2064 * the 'if (vtxcnt < maxvtxcnt)' condition, then append the conditional
2065 * block holding stream-out write instructions, followed by the new end
2066 * block:
2067 *
2068 * blockOrigEnd {
2069 * p0.x = (vtxcnt < maxvtxcnt)
2070 * // succs: blockStreamOut, blockNewEnd
2071 * }
2072 * blockStreamOut {
2073 * ... stream-out instructions ...
2074 * // succs: blockNewEnd
2075 * }
2076 * blockNewEnd {
2077 * }
2078 */
2079 static void
2080 emit_stream_out(struct ir3_compile *ctx)
2081 {
2082 struct ir3_shader_variant *v = ctx->so;
2083 struct ir3 *ir = ctx->ir;
2084 struct pipe_stream_output_info *strmout =
2085 &ctx->so->shader->stream_output;
2086 struct ir3_block *orig_end_block, *stream_out_block, *new_end_block;
2087 struct ir3_instruction *vtxcnt, *maxvtxcnt, *cond;
2088 struct ir3_instruction *bases[PIPE_MAX_SO_BUFFERS];
2089
2090 /* create vtxcnt input in input block at top of shader,
2091 * so that it is seen as live over the entire duration
2092 * of the shader:
2093 */
2094 vtxcnt = create_input(ctx->in_block, 0);
2095 add_sysval_input(ctx, SYSTEM_VALUE_VERTEX_CNT, vtxcnt);
2096
2097 maxvtxcnt = create_driver_param(ctx, IR3_DP_VTXCNT_MAX);
2098
2099 /* at this point, we are at the original 'end' block,
2100 * re-purpose this block to stream-out condition, then
2101 * append stream-out block and new-end block
2102 */
2103 orig_end_block = ctx->block;
2104
2105 stream_out_block = ir3_block_create(ir);
2106 list_addtail(&stream_out_block->node, &ir->block_list);
2107
2108 new_end_block = ir3_block_create(ir);
2109 list_addtail(&new_end_block->node, &ir->block_list);
2110
2111 orig_end_block->successors[0] = stream_out_block;
2112 orig_end_block->successors[1] = new_end_block;
2113 stream_out_block->successors[0] = new_end_block;
2114
2115 /* setup 'if (vtxcnt < maxvtxcnt)' condition: */
2116 cond = ir3_CMPS_S(ctx->block, vtxcnt, 0, maxvtxcnt, 0);
2117 cond->regs[0]->num = regid(REG_P0, 0);
2118 cond->cat2.condition = IR3_COND_LT;
2119
2120 /* condition goes on previous block to the conditional,
2121 * since it is used to pick which of the two successor
2122 * paths to take:
2123 */
2124 orig_end_block->condition = cond;
2125
2126 /* switch to stream_out_block to generate the stream-out
2127 * instructions:
2128 */
2129 ctx->block = stream_out_block;
2130
2131 /* Calculate base addresses based on vtxcnt. Instructions
2132 * generated for bases not used in following loop will be
2133 * stripped out in the backend.
2134 */
2135 for (unsigned i = 0; i < PIPE_MAX_SO_BUFFERS; i++) {
2136 unsigned stride = strmout->stride[i];
2137 struct ir3_instruction *base, *off;
2138
2139 base = create_uniform(ctx, regid(v->first_driver_param + IR3_TFBOS_OFF, i));
2140
2141 /* 24-bit should be enough: */
2142 off = ir3_MUL_U(ctx->block, vtxcnt, 0,
2143 create_immed(ctx->block, stride * 4), 0);
2144
2145 bases[i] = ir3_ADD_S(ctx->block, off, 0, base, 0);
2146 }
2147
2148 /* Generate the per-output store instructions: */
2149 for (unsigned i = 0; i < strmout->num_outputs; i++) {
2150 for (unsigned j = 0; j < strmout->output[i].num_components; j++) {
2151 unsigned c = j + strmout->output[i].start_component;
2152 struct ir3_instruction *base, *out, *stg;
2153
2154 base = bases[strmout->output[i].output_buffer];
2155 out = ctx->ir->outputs[regid(strmout->output[i].register_index, c)];
2156
2157 stg = ir3_STG(ctx->block, base, 0, out, 0,
2158 create_immed(ctx->block, 1), 0);
2159 stg->cat6.type = TYPE_U32;
2160 stg->cat6.dst_offset = (strmout->output[i].dst_offset + j) * 4;
2161
2162 array_insert(ctx->ir->keeps, stg);
2163 }
2164 }
2165
2166 /* and finally switch to the new_end_block: */
2167 ctx->block = new_end_block;
2168 }
2169
2170 static void
2171 emit_function(struct ir3_compile *ctx, nir_function_impl *impl)
2172 {
2173 emit_cf_list(ctx, &impl->body);
2174 emit_block(ctx, impl->end_block);
2175
2176 /* at this point, we should have a single empty block,
2177 * into which we emit the 'end' instruction.
2178 */
2179 compile_assert(ctx, list_empty(&ctx->block->instr_list));
2180
2181 /* If stream-out (aka transform-feedback) enabled, emit the
2182 * stream-out instructions, followed by a new empty block (into
2183 * which the 'end' instruction lands).
2184 *
2185 * NOTE: it is done in this order, rather than inserting before
2186 * we emit end_block, because NIR guarantees that all blocks
2187 * flow into end_block, and that end_block has no successors.
2188 * So by re-purposing end_block as the first block of stream-
2189 * out, we guarantee that all exit paths flow into the stream-
2190 * out instructions.
2191 */
2192 if ((ctx->so->shader->stream_output.num_outputs > 0) &&
2193 !ctx->so->key.binning_pass) {
2194 debug_assert(ctx->so->type == SHADER_VERTEX);
2195 emit_stream_out(ctx);
2196 }
2197
2198 ir3_END(ctx->block);
2199 }
2200
2201 static void
2202 setup_input(struct ir3_compile *ctx, nir_variable *in)
2203 {
2204 struct ir3_shader_variant *so = ctx->so;
2205 unsigned array_len = MAX2(glsl_get_length(in->type), 1);
2206 unsigned ncomp = glsl_get_components(in->type);
2207 unsigned n = in->data.driver_location;
2208 unsigned slot = in->data.location;
2209
2210 DBG("; in: slot=%u, len=%ux%u, drvloc=%u",
2211 slot, array_len, ncomp, n);
2212
2213 so->inputs[n].slot = slot;
2214 so->inputs[n].compmask = (1 << ncomp) - 1;
2215 so->inputs_count = MAX2(so->inputs_count, n + 1);
2216 so->inputs[n].interpolate = in->data.interpolation;
2217
2218 if (ctx->so->type == SHADER_FRAGMENT) {
2219 for (int i = 0; i < ncomp; i++) {
2220 struct ir3_instruction *instr = NULL;
2221 unsigned idx = (n * 4) + i;
2222
2223 if (slot == VARYING_SLOT_POS) {
2224 so->inputs[n].bary = false;
2225 so->frag_coord = true;
2226 instr = create_frag_coord(ctx, i);
2227 } else if (slot == VARYING_SLOT_FACE) {
2228 so->inputs[n].bary = false;
2229 so->frag_face = true;
2230 instr = create_frag_face(ctx, i);
2231 } else {
2232 bool use_ldlv = false;
2233
2234 /* detect the special case for front/back colors where
2235 * we need to do flat vs smooth shading depending on
2236 * rast state:
2237 */
2238 if (in->data.interpolation == INTERP_QUALIFIER_NONE) {
2239 switch (slot) {
2240 case VARYING_SLOT_COL0:
2241 case VARYING_SLOT_COL1:
2242 case VARYING_SLOT_BFC0:
2243 case VARYING_SLOT_BFC1:
2244 so->inputs[n].rasterflat = true;
2245 break;
2246 default:
2247 break;
2248 }
2249 }
2250
2251 if (ctx->flat_bypass) {
2252 if ((so->inputs[n].interpolate == INTERP_QUALIFIER_FLAT) ||
2253 (so->inputs[n].rasterflat && ctx->so->key.rasterflat))
2254 use_ldlv = true;
2255 }
2256
2257 so->inputs[n].bary = true;
2258
2259 instr = create_frag_input(ctx, use_ldlv);
2260 }
2261
2262 ctx->ir->inputs[idx] = instr;
2263 }
2264 } else if (ctx->so->type == SHADER_VERTEX) {
2265 for (int i = 0; i < ncomp; i++) {
2266 unsigned idx = (n * 4) + i;
2267 ctx->ir->inputs[idx] = create_input(ctx->block, idx);
2268 }
2269 } else {
2270 compile_error(ctx, "unknown shader type: %d\n", ctx->so->type);
2271 }
2272
2273 if (so->inputs[n].bary || (ctx->so->type == SHADER_VERTEX)) {
2274 so->total_in += ncomp;
2275 }
2276 }
2277
2278 static void
2279 setup_output(struct ir3_compile *ctx, nir_variable *out)
2280 {
2281 struct ir3_shader_variant *so = ctx->so;
2282 unsigned array_len = MAX2(glsl_get_length(out->type), 1);
2283 unsigned ncomp = glsl_get_components(out->type);
2284 unsigned n = out->data.driver_location;
2285 unsigned slot = out->data.location;
2286 unsigned comp = 0;
2287
2288 DBG("; out: slot=%u, len=%ux%u, drvloc=%u",
2289 slot, array_len, ncomp, n);
2290
2291 if (ctx->so->type == SHADER_FRAGMENT) {
2292 switch (slot) {
2293 case FRAG_RESULT_DEPTH:
2294 comp = 2; /* tgsi will write to .z component */
2295 so->writes_pos = true;
2296 break;
2297 case FRAG_RESULT_COLOR:
2298 so->color0_mrt = 1;
2299 break;
2300 default:
2301 if (slot >= FRAG_RESULT_DATA0)
2302 break;
2303 compile_error(ctx, "unknown FS output name: %s\n",
2304 gl_frag_result_name(slot));
2305 }
2306 } else if (ctx->so->type == SHADER_VERTEX) {
2307 switch (slot) {
2308 case VARYING_SLOT_POS:
2309 so->writes_pos = true;
2310 break;
2311 case VARYING_SLOT_PSIZ:
2312 so->writes_psize = true;
2313 break;
2314 case VARYING_SLOT_COL0:
2315 case VARYING_SLOT_COL1:
2316 case VARYING_SLOT_BFC0:
2317 case VARYING_SLOT_BFC1:
2318 case VARYING_SLOT_FOGC:
2319 case VARYING_SLOT_CLIP_DIST0:
2320 case VARYING_SLOT_CLIP_DIST1:
2321 break;
2322 default:
2323 if (slot >= VARYING_SLOT_VAR0)
2324 break;
2325 if ((VARYING_SLOT_TEX0 <= slot) && (slot <= VARYING_SLOT_TEX7))
2326 break;
2327 compile_error(ctx, "unknown VS output name: %s\n",
2328 gl_varying_slot_name(slot));
2329 }
2330 } else {
2331 compile_error(ctx, "unknown shader type: %d\n", ctx->so->type);
2332 }
2333
2334 compile_assert(ctx, n < ARRAY_SIZE(so->outputs));
2335
2336 so->outputs[n].slot = slot;
2337 so->outputs[n].regid = regid(n, comp);
2338 so->outputs_count = MAX2(so->outputs_count, n + 1);
2339
2340 for (int i = 0; i < ncomp; i++) {
2341 unsigned idx = (n * 4) + i;
2342
2343 ctx->ir->outputs[idx] = create_immed(ctx->block, fui(0.0));
2344 }
2345 }
2346
2347 static void
2348 emit_instructions(struct ir3_compile *ctx)
2349 {
2350 unsigned ninputs, noutputs;
2351 nir_function_impl *fxn = NULL;
2352
2353 /* Find the main function: */
2354 nir_foreach_function(ctx->s, function) {
2355 compile_assert(ctx, strcmp(function->name, "main") == 0);
2356 compile_assert(ctx, function->impl);
2357 fxn = function->impl;
2358 break;
2359 }
2360
2361 ninputs = exec_list_length(&ctx->s->inputs) * 4;
2362 noutputs = exec_list_length(&ctx->s->outputs) * 4;
2363
2364 /* or vtx shaders, we need to leave room for sysvals:
2365 */
2366 if (ctx->so->type == SHADER_VERTEX) {
2367 ninputs += 8;
2368 }
2369
2370 ctx->ir = ir3_create(ctx->compiler, ninputs, noutputs);
2371
2372 /* Create inputs in first block: */
2373 ctx->block = get_block(ctx, nir_start_block(fxn));
2374 ctx->in_block = ctx->block;
2375 list_addtail(&ctx->block->node, &ctx->ir->block_list);
2376
2377 if (ctx->so->type == SHADER_VERTEX) {
2378 ctx->ir->ninputs -= 8;
2379 }
2380
2381 /* for fragment shader, we have a single input register (usually
2382 * r0.xy) which is used as the base for bary.f varying fetch instrs:
2383 */
2384 if (ctx->so->type == SHADER_FRAGMENT) {
2385 // TODO maybe a helper for fi since we need it a few places..
2386 struct ir3_instruction *instr;
2387 instr = ir3_instr_create(ctx->block, -1, OPC_META_FI);
2388 ir3_reg_create(instr, 0, 0);
2389 ir3_reg_create(instr, 0, IR3_REG_SSA); /* r0.x */
2390 ir3_reg_create(instr, 0, IR3_REG_SSA); /* r0.y */
2391 ctx->frag_pos = instr;
2392 }
2393
2394 /* Setup inputs: */
2395 nir_foreach_variable(var, &ctx->s->inputs) {
2396 setup_input(ctx, var);
2397 }
2398
2399 /* Setup outputs: */
2400 nir_foreach_variable(var, &ctx->s->outputs) {
2401 setup_output(ctx, var);
2402 }
2403
2404 /* Setup variables (which should only be arrays): */
2405 nir_foreach_variable(var, &ctx->s->globals) {
2406 declare_var(ctx, var);
2407 }
2408
2409 /* And emit the body: */
2410 ctx->impl = fxn;
2411 emit_function(ctx, fxn);
2412
2413 list_for_each_entry (struct ir3_block, block, &ctx->ir->block_list, node) {
2414 resolve_phis(ctx, block);
2415 }
2416 }
2417
2418 /* from NIR perspective, we actually have inputs. But most of the "inputs"
2419 * for a fragment shader are just bary.f instructions. The *actual* inputs
2420 * from the hw perspective are the frag_pos and optionally frag_coord and
2421 * frag_face.
2422 */
2423 static void
2424 fixup_frag_inputs(struct ir3_compile *ctx)
2425 {
2426 struct ir3_shader_variant *so = ctx->so;
2427 struct ir3 *ir = ctx->ir;
2428 struct ir3_instruction **inputs;
2429 struct ir3_instruction *instr;
2430 int n, regid = 0;
2431
2432 ir->ninputs = 0;
2433
2434 n = 4; /* always have frag_pos */
2435 n += COND(so->frag_face, 4);
2436 n += COND(so->frag_coord, 4);
2437
2438 inputs = ir3_alloc(ctx->ir, n * (sizeof(struct ir3_instruction *)));
2439
2440 if (so->frag_face) {
2441 /* this ultimately gets assigned to hr0.x so doesn't conflict
2442 * with frag_coord/frag_pos..
2443 */
2444 inputs[ir->ninputs++] = ctx->frag_face;
2445 ctx->frag_face->regs[0]->num = 0;
2446
2447 /* remaining channels not used, but let's avoid confusing
2448 * other parts that expect inputs to come in groups of vec4
2449 */
2450 inputs[ir->ninputs++] = NULL;
2451 inputs[ir->ninputs++] = NULL;
2452 inputs[ir->ninputs++] = NULL;
2453 }
2454
2455 /* since we don't know where to set the regid for frag_coord,
2456 * we have to use r0.x for it. But we don't want to *always*
2457 * use r1.x for frag_pos as that could increase the register
2458 * footprint on simple shaders:
2459 */
2460 if (so->frag_coord) {
2461 ctx->frag_coord[0]->regs[0]->num = regid++;
2462 ctx->frag_coord[1]->regs[0]->num = regid++;
2463 ctx->frag_coord[2]->regs[0]->num = regid++;
2464 ctx->frag_coord[3]->regs[0]->num = regid++;
2465
2466 inputs[ir->ninputs++] = ctx->frag_coord[0];
2467 inputs[ir->ninputs++] = ctx->frag_coord[1];
2468 inputs[ir->ninputs++] = ctx->frag_coord[2];
2469 inputs[ir->ninputs++] = ctx->frag_coord[3];
2470 }
2471
2472 /* we always have frag_pos: */
2473 so->pos_regid = regid;
2474
2475 /* r0.x */
2476 instr = create_input(ctx->in_block, ir->ninputs);
2477 instr->regs[0]->num = regid++;
2478 inputs[ir->ninputs++] = instr;
2479 ctx->frag_pos->regs[1]->instr = instr;
2480
2481 /* r0.y */
2482 instr = create_input(ctx->in_block, ir->ninputs);
2483 instr->regs[0]->num = regid++;
2484 inputs[ir->ninputs++] = instr;
2485 ctx->frag_pos->regs[2]->instr = instr;
2486
2487 ir->inputs = inputs;
2488 }
2489
2490 int
2491 ir3_compile_shader_nir(struct ir3_compiler *compiler,
2492 struct ir3_shader_variant *so)
2493 {
2494 struct ir3_compile *ctx;
2495 struct ir3 *ir;
2496 struct ir3_instruction **inputs;
2497 unsigned i, j, actual_in, inloc;
2498 int ret = 0, max_bary;
2499
2500 assert(!so->ir);
2501
2502 ctx = compile_init(compiler, so, so->shader->tokens);
2503 if (!ctx) {
2504 DBG("INIT failed!");
2505 ret = -1;
2506 goto out;
2507 }
2508
2509 emit_instructions(ctx);
2510
2511 if (ctx->error) {
2512 DBG("EMIT failed!");
2513 ret = -1;
2514 goto out;
2515 }
2516
2517 ir = so->ir = ctx->ir;
2518
2519 /* keep track of the inputs from TGSI perspective.. */
2520 inputs = ir->inputs;
2521
2522 /* but fixup actual inputs for frag shader: */
2523 if (so->type == SHADER_FRAGMENT)
2524 fixup_frag_inputs(ctx);
2525
2526 /* at this point, for binning pass, throw away unneeded outputs: */
2527 if (so->key.binning_pass) {
2528 for (i = 0, j = 0; i < so->outputs_count; i++) {
2529 unsigned slot = so->outputs[i].slot;
2530
2531 /* throw away everything but first position/psize */
2532 if ((slot == VARYING_SLOT_POS) || (slot == VARYING_SLOT_PSIZ)) {
2533 if (i != j) {
2534 so->outputs[j] = so->outputs[i];
2535 ir->outputs[(j*4)+0] = ir->outputs[(i*4)+0];
2536 ir->outputs[(j*4)+1] = ir->outputs[(i*4)+1];
2537 ir->outputs[(j*4)+2] = ir->outputs[(i*4)+2];
2538 ir->outputs[(j*4)+3] = ir->outputs[(i*4)+3];
2539 }
2540 j++;
2541 }
2542 }
2543 so->outputs_count = j;
2544 ir->noutputs = j * 4;
2545 }
2546
2547 /* if we want half-precision outputs, mark the output registers
2548 * as half:
2549 */
2550 if (so->key.half_precision) {
2551 for (i = 0; i < ir->noutputs; i++) {
2552 struct ir3_instruction *out = ir->outputs[i];
2553 if (!out)
2554 continue;
2555 out->regs[0]->flags |= IR3_REG_HALF;
2556 /* output could be a fanout (ie. texture fetch output)
2557 * in which case we need to propagate the half-reg flag
2558 * up to the definer so that RA sees it:
2559 */
2560 if (is_meta(out) && (out->opc == OPC_META_FO)) {
2561 out = out->regs[1]->instr;
2562 out->regs[0]->flags |= IR3_REG_HALF;
2563 }
2564
2565 if (out->category == 1) {
2566 out->cat1.dst_type = half_type(out->cat1.dst_type);
2567 }
2568 }
2569 }
2570
2571 if (fd_mesa_debug & FD_DBG_OPTMSGS) {
2572 printf("BEFORE CP:\n");
2573 ir3_print(ir);
2574 }
2575
2576 ir3_cp(ir);
2577
2578 if (fd_mesa_debug & FD_DBG_OPTMSGS) {
2579 printf("BEFORE GROUPING:\n");
2580 ir3_print(ir);
2581 }
2582
2583 /* Group left/right neighbors, inserting mov's where needed to
2584 * solve conflicts:
2585 */
2586 ir3_group(ir);
2587
2588 ir3_depth(ir);
2589
2590 if (fd_mesa_debug & FD_DBG_OPTMSGS) {
2591 printf("AFTER DEPTH:\n");
2592 ir3_print(ir);
2593 }
2594
2595 ret = ir3_sched(ir);
2596 if (ret) {
2597 DBG("SCHED failed!");
2598 goto out;
2599 }
2600
2601 if (fd_mesa_debug & FD_DBG_OPTMSGS) {
2602 printf("AFTER SCHED:\n");
2603 ir3_print(ir);
2604 }
2605
2606 ret = ir3_ra(ir, so->type, so->frag_coord, so->frag_face);
2607 if (ret) {
2608 DBG("RA failed!");
2609 goto out;
2610 }
2611
2612 if (fd_mesa_debug & FD_DBG_OPTMSGS) {
2613 printf("AFTER RA:\n");
2614 ir3_print(ir);
2615 }
2616
2617 /* fixup input/outputs: */
2618 for (i = 0; i < so->outputs_count; i++) {
2619 so->outputs[i].regid = ir->outputs[i*4]->regs[0]->num;
2620 /* preserve hack for depth output.. tgsi writes depth to .z,
2621 * but what we give the hw is the scalar register:
2622 */
2623 if ((so->type == SHADER_FRAGMENT) &&
2624 (so->outputs[i].slot == FRAG_RESULT_DEPTH))
2625 so->outputs[i].regid += 2;
2626 }
2627
2628 /* Note that some or all channels of an input may be unused: */
2629 actual_in = 0;
2630 inloc = 0;
2631 for (i = 0; i < so->inputs_count; i++) {
2632 unsigned j, regid = ~0, compmask = 0;
2633 so->inputs[i].ncomp = 0;
2634 so->inputs[i].inloc = inloc + 8;
2635 for (j = 0; j < 4; j++) {
2636 struct ir3_instruction *in = inputs[(i*4) + j];
2637 if (in && !(in->flags & IR3_INSTR_UNUSED)) {
2638 compmask |= (1 << j);
2639 regid = in->regs[0]->num - j;
2640 actual_in++;
2641 so->inputs[i].ncomp++;
2642 if ((so->type == SHADER_FRAGMENT) && so->inputs[i].bary) {
2643 /* assign inloc: */
2644 assert(in->regs[1]->flags & IR3_REG_IMMED);
2645 in->regs[1]->iim_val = inloc++;
2646 }
2647 }
2648 }
2649 if ((so->type == SHADER_FRAGMENT) && compmask && so->inputs[i].bary)
2650 so->varying_in++;
2651 so->inputs[i].regid = regid;
2652 so->inputs[i].compmask = compmask;
2653 }
2654
2655 /* We need to do legalize after (for frag shader's) the "bary.f"
2656 * offsets (inloc) have been assigned.
2657 */
2658 ir3_legalize(ir, &so->has_samp, &max_bary);
2659
2660 if (fd_mesa_debug & FD_DBG_OPTMSGS) {
2661 printf("AFTER LEGALIZE:\n");
2662 ir3_print(ir);
2663 }
2664
2665 /* Note that actual_in counts inputs that are not bary.f'd for FS: */
2666 if (so->type == SHADER_VERTEX)
2667 so->total_in = actual_in;
2668 else
2669 so->total_in = max_bary + 1;
2670
2671 out:
2672 if (ret) {
2673 if (so->ir)
2674 ir3_destroy(so->ir);
2675 so->ir = NULL;
2676 }
2677 compile_free(ctx);
2678
2679 return ret;
2680 }