projects / mesa.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
history | raw | HEAD
freedreno/ir3: remove ir3_instruction::category
[mesa.git] / src / gallium / drivers / freedreno / ir3 / ir3_compiler_nir.c
1 /* -*- mode: C; c-file-style: "k&r"; tab-width 4; indent-tabs-mode: t; -*- */
2
3 /*
4 * Copyright (C) 2015 Rob Clark <robclark@freedesktop.org>
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the "Software"),
8 * to deal in the Software without restriction, including without limitation
9 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
10 * and/or sell copies of the Software, and to permit persons to whom the
11 * Software is furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice (including the next
14 * paragraph) shall be included in all copies or substantial portions of the
15 * Software.
16 *
17 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
18 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
19 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
20 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
21 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
22 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
23 * SOFTWARE.
24 *
25 * Authors:
26 * Rob Clark <robclark@freedesktop.org>
27 */
28
29 #include <stdarg.h>
30
31 #include "pipe/p_state.h"
32 #include "util/u_string.h"
33 #include "util/u_memory.h"
34 #include "util/u_inlines.h"
35
36 #include "freedreno_util.h"
37
38 #include "ir3_compiler.h"
39 #include "ir3_shader.h"
40 #include "ir3_nir.h"
41
42 #include "instr-a3xx.h"
43 #include "ir3.h"
44
45
46 struct ir3_compile {
47 struct ir3_compiler *compiler;
48
49 struct nir_shader *s;
50
51 struct ir3 *ir;
52 struct ir3_shader_variant *so;
53
54 struct ir3_block *block; /* the current block */
55 struct ir3_block *in_block; /* block created for shader inputs */
56
57 nir_function_impl *impl;
58
59 /* For fragment shaders, from the hw perspective the only
60 * actual input is r0.xy position register passed to bary.f.
61 * But TGSI doesn't know that, it still declares things as
62 * IN[] registers. So we do all the input tracking normally
63 * and fix things up after compile_instructions()
64 *
65 * NOTE that frag_pos is the hardware position (possibly it
66 * is actually an index or tag or some such.. it is *not*
67 * values that can be directly used for gl_FragCoord..)
68 */
69 struct ir3_instruction *frag_pos, *frag_face, *frag_coord[4];
70
71 /* For vertex shaders, keep track of the system values sources */
72 struct ir3_instruction *vertex_id, *basevertex, *instance_id;
73
74 /* mapping from nir_register to defining instruction: */
75 struct hash_table *def_ht;
76
77 unsigned num_arrays;
78
79 /* a common pattern for indirect addressing is to request the
80 * same address register multiple times. To avoid generating
81 * duplicate instruction sequences (which our backend does not
82 * try to clean up, since that should be done as the NIR stage)
83 * we cache the address value generated for a given src value:
84 */
85 struct hash_table *addr_ht;
86
87 /* maps nir_block to ir3_block, mostly for the purposes of
88 * figuring out the blocks successors
89 */
90 struct hash_table *block_ht;
91
92 /* a4xx (at least patchlevel 0) cannot seem to flat-interpolate
93 * so we need to use ldlv.u32 to load the varying directly:
94 */
95 bool flat_bypass;
96
97 /* on a3xx, we need to add one to # of array levels:
98 */
99 bool levels_add_one;
100
101 /* on a3xx, we need to scale up integer coords for isaml based
102 * on LoD:
103 */
104 bool unminify_coords;
105
106 /* for looking up which system value is which */
107 unsigned sysval_semantics[8];
108
109 /* set if we encounter something we can't handle yet, so we
110 * can bail cleanly and fallback to TGSI compiler f/e
111 */
112 bool error;
113 };
114
115
116 static struct ir3_instruction * create_immed(struct ir3_block *block, uint32_t val);
117 static struct ir3_block * get_block(struct ir3_compile *ctx, nir_block *nblock);
118
119
120 static struct ir3_compile *
121 compile_init(struct ir3_compiler *compiler,
122 struct ir3_shader_variant *so)
123 {
124 struct ir3_compile *ctx = rzalloc(NULL, struct ir3_compile);
125
126 if (compiler->gpu_id >= 400) {
127 /* need special handling for "flat" */
128 ctx->flat_bypass = true;
129 ctx->levels_add_one = false;
130 ctx->unminify_coords = false;
131 } else {
132 /* no special handling for "flat" */
133 ctx->flat_bypass = false;
134 ctx->levels_add_one = true;
135 ctx->unminify_coords = true;
136 }
137
138 ctx->compiler = compiler;
139 ctx->ir = so->ir;
140 ctx->so = so;
141 ctx->def_ht = _mesa_hash_table_create(ctx,
142 _mesa_hash_pointer, _mesa_key_pointer_equal);
143 ctx->block_ht = _mesa_hash_table_create(ctx,
144 _mesa_hash_pointer, _mesa_key_pointer_equal);
145
146 /* TODO: maybe generate some sort of bitmask of what key
147 * lowers vs what shader has (ie. no need to lower
148 * texture clamp lowering if no texture sample instrs)..
149 * although should be done further up the stack to avoid
150 * creating duplicate variants..
151 */
152
153 if (ir3_key_lowers_nir(&so->key)) {
154 nir_shader *s = nir_shader_clone(ctx, so->shader->nir);
155 ctx->s = ir3_optimize_nir(so->shader, s, &so->key);
156 } else {
157 /* fast-path for shader key that lowers nothing in NIR: */
158 ctx->s = so->shader->nir;
159 }
160
161 if (fd_mesa_debug & FD_DBG_DISASM) {
162 DBG("dump nir%dv%d: type=%d, k={bp=%u,cts=%u,hp=%u}",
163 so->shader->id, so->id, so->type,
164 so->key.binning_pass, so->key.color_two_side,
165 so->key.half_precision);
166 nir_print_shader(ctx->s, stdout);
167 }
168
169 so->first_driver_param = so->first_immediate = ctx->s->num_uniforms;
170
171 /* Layout of constant registers:
172 *
173 * num_uniform * vec4 - user consts
174 * 4 * vec4 - UBO addresses
175 * if (vertex shader) {
176 * N * vec4 - driver params (IR3_DP_*)
177 * 1 * vec4 - stream-out addresses
178 * }
179 *
180 * TODO this could be made more dynamic, to at least skip sections
181 * that we don't need..
182 */
183
184 /* reserve 4 (vec4) slots for ubo base addresses: */
185 so->first_immediate += 4;
186
187 if (so->type == SHADER_VERTEX) {
188 /* driver params (see ir3_driver_param): */
189 so->first_immediate += IR3_DP_COUNT/4; /* convert to vec4 */
190 /* one (vec4) slot for stream-output base addresses: */
191 so->first_immediate++;
192 }
193
194 return ctx;
195 }
196
197 static void
198 compile_error(struct ir3_compile *ctx, const char *format, ...)
199 {
200 va_list ap;
201 va_start(ap, format);
202 _debug_vprintf(format, ap);
203 va_end(ap);
204 nir_print_shader(ctx->s, stdout);
205 ctx->error = true;
206 debug_assert(0);
207 }
208
209 #define compile_assert(ctx, cond) do { \
210 if (!(cond)) compile_error((ctx), "failed assert: "#cond"\n"); \
211 } while (0)
212
213 static void
214 compile_free(struct ir3_compile *ctx)
215 {
216 ralloc_free(ctx);
217 }
218
219 static void
220 declare_var(struct ir3_compile *ctx, nir_variable *var)
221 {
222 unsigned length = glsl_get_length(var->type) * 4; /* always vec4, at least with ttn */
223 struct ir3_array *arr = ralloc(ctx, struct ir3_array);
224 arr->id = ++ctx->num_arrays;
225 arr->length = length;
226 arr->var = var;
227 list_addtail(&arr->node, &ctx->ir->array_list);
228 }
229
230 static struct ir3_array *
231 get_var(struct ir3_compile *ctx, nir_variable *var)
232 {
233 list_for_each_entry (struct ir3_array, arr, &ctx->ir->array_list, node) {
234 if (arr->var == var)
235 return arr;
236 }
237 compile_error(ctx, "bogus var: %s\n", var->name);
238 return NULL;
239 }
240
241 /* allocate a n element value array (to be populated by caller) and
242 * insert in def_ht
243 */
244 static struct ir3_instruction **
245 __get_dst(struct ir3_compile *ctx, void *key, unsigned n)
246 {
247 struct ir3_instruction **value =
248 ralloc_array(ctx->def_ht, struct ir3_instruction *, n);
249 _mesa_hash_table_insert(ctx->def_ht, key, value);
250 return value;
251 }
252
253 static struct ir3_instruction **
254 get_dst(struct ir3_compile *ctx, nir_dest *dst, unsigned n)
255 {
256 compile_assert(ctx, dst->is_ssa);
257 if (dst->is_ssa) {
258 return __get_dst(ctx, &dst->ssa, n);
259 } else {
260 return __get_dst(ctx, dst->reg.reg, n);
261 }
262 }
263
264 static struct ir3_instruction **
265 get_dst_ssa(struct ir3_compile *ctx, nir_ssa_def *dst, unsigned n)
266 {
267 return __get_dst(ctx, dst, n);
268 }
269
270 static struct ir3_instruction **
271 get_src(struct ir3_compile *ctx, nir_src *src)
272 {
273 struct hash_entry *entry;
274 compile_assert(ctx, src->is_ssa);
275 if (src->is_ssa) {
276 entry = _mesa_hash_table_search(ctx->def_ht, src->ssa);
277 } else {
278 entry = _mesa_hash_table_search(ctx->def_ht, src->reg.reg);
279 }
280 compile_assert(ctx, entry);
281 return entry->data;
282 }
283
284 static struct ir3_instruction *
285 create_immed(struct ir3_block *block, uint32_t val)
286 {
287 struct ir3_instruction *mov;
288
289 mov = ir3_instr_create(block, 1, OPC_MOV);
290 mov->cat1.src_type = TYPE_U32;
291 mov->cat1.dst_type = TYPE_U32;
292 ir3_reg_create(mov, 0, 0);
293 ir3_reg_create(mov, 0, IR3_REG_IMMED)->uim_val = val;
294
295 return mov;
296 }
297
298 static struct ir3_instruction *
299 create_addr(struct ir3_block *block, struct ir3_instruction *src)
300 {
301 struct ir3_instruction *instr, *immed;
302
303 /* TODO in at least some cases, the backend could probably be
304 * made clever enough to propagate IR3_REG_HALF..
305 */
306 instr = ir3_COV(block, src, TYPE_U32, TYPE_S16);
307 instr->regs[0]->flags |= IR3_REG_HALF;
308
309 immed = create_immed(block, 2);
310 immed->regs[0]->flags |= IR3_REG_HALF;
311
312 instr = ir3_SHL_B(block, instr, 0, immed, 0);
313 instr->regs[0]->flags |= IR3_REG_HALF;
314 instr->regs[1]->flags |= IR3_REG_HALF;
315
316 instr = ir3_MOV(block, instr, TYPE_S16);
317 instr->regs[0]->num = regid(REG_A0, 0);
318 instr->regs[0]->flags |= IR3_REG_HALF;
319 instr->regs[1]->flags |= IR3_REG_HALF;
320
321 return instr;
322 }
323
324 /* caches addr values to avoid generating multiple cov/shl/mova
325 * sequences for each use of a given NIR level src as address
326 */
327 static struct ir3_instruction *
328 get_addr(struct ir3_compile *ctx, struct ir3_instruction *src)
329 {
330 struct ir3_instruction *addr;
331
332 if (!ctx->addr_ht) {
333 ctx->addr_ht = _mesa_hash_table_create(ctx,
334 _mesa_hash_pointer, _mesa_key_pointer_equal);
335 } else {
336 struct hash_entry *entry;
337 entry = _mesa_hash_table_search(ctx->addr_ht, src);
338 if (entry)
339 return entry->data;
340 }
341
342 addr = create_addr(ctx->block, src);
343 _mesa_hash_table_insert(ctx->addr_ht, src, addr);
344
345 return addr;
346 }
347
348 static struct ir3_instruction *
349 get_predicate(struct ir3_compile *ctx, struct ir3_instruction *src)
350 {
351 struct ir3_block *b = ctx->block;
352 struct ir3_instruction *cond;
353
354 /* NOTE: only cmps.*.* can write p0.x: */
355 cond = ir3_CMPS_S(b, src, 0, create_immed(b, 0), 0);
356 cond->cat2.condition = IR3_COND_NE;
357
358 /* condition always goes in predicate register: */
359 cond->regs[0]->num = regid(REG_P0, 0);
360
361 return cond;
362 }
363
364 static struct ir3_instruction *
365 create_uniform(struct ir3_compile *ctx, unsigned n)
366 {
367 struct ir3_instruction *mov;
368
369 mov = ir3_instr_create(ctx->block, 1, OPC_MOV);
370 /* TODO get types right? */
371 mov->cat1.src_type = TYPE_F32;
372 mov->cat1.dst_type = TYPE_F32;
373 ir3_reg_create(mov, 0, 0);
374 ir3_reg_create(mov, n, IR3_REG_CONST);
375
376 return mov;
377 }
378
379 static struct ir3_instruction *
380 create_uniform_indirect(struct ir3_compile *ctx, int n,
381 struct ir3_instruction *address)
382 {
383 struct ir3_instruction *mov;
384
385 mov = ir3_instr_create(ctx->block, 1, OPC_MOV);
386 mov->cat1.src_type = TYPE_U32;
387 mov->cat1.dst_type = TYPE_U32;
388 ir3_reg_create(mov, 0, 0);
389 ir3_reg_create(mov, 0, IR3_REG_CONST | IR3_REG_RELATIV)->array.offset = n;
390
391 ir3_instr_set_address(mov, address);
392
393 return mov;
394 }
395
396 static struct ir3_instruction *
397 create_collect(struct ir3_block *block, struct ir3_instruction **arr,
398 unsigned arrsz)
399 {
400 struct ir3_instruction *collect;
401
402 if (arrsz == 0)
403 return NULL;
404
405 collect = ir3_instr_create2(block, -1, OPC_META_FI, 1 + arrsz);
406 ir3_reg_create(collect, 0, 0); /* dst */
407 for (unsigned i = 0; i < arrsz; i++)
408 ir3_reg_create(collect, 0, IR3_REG_SSA)->instr = arr[i];
409
410 return collect;
411 }
412
413 static struct ir3_instruction *
414 create_indirect_load(struct ir3_compile *ctx, unsigned arrsz, int n,
415 struct ir3_instruction *address, struct ir3_instruction *collect)
416 {
417 struct ir3_block *block = ctx->block;
418 struct ir3_instruction *mov;
419 struct ir3_register *src;
420
421 mov = ir3_instr_create(block, 1, OPC_MOV);
422 mov->cat1.src_type = TYPE_U32;
423 mov->cat1.dst_type = TYPE_U32;
424 ir3_reg_create(mov, 0, 0);
425 src = ir3_reg_create(mov, 0, IR3_REG_SSA | IR3_REG_RELATIV);
426 src->instr = collect;
427 src->size = arrsz;
428 src->array.offset = n;
429
430 ir3_instr_set_address(mov, address);
431
432 return mov;
433 }
434
435 /* relative (indirect) if address!=NULL */
436 static struct ir3_instruction *
437 create_var_load(struct ir3_compile *ctx, struct ir3_array *arr, int n,
438 struct ir3_instruction *address)
439 {
440 struct ir3_block *block = ctx->block;
441 struct ir3_instruction *mov;
442 struct ir3_register *src;
443
444 mov = ir3_instr_create(block, 1, OPC_MOV);
445 mov->cat1.src_type = TYPE_U32;
446 mov->cat1.dst_type = TYPE_U32;
447 ir3_reg_create(mov, 0, 0);
448 src = ir3_reg_create(mov, 0, IR3_REG_ARRAY |
449 COND(address, IR3_REG_RELATIV));
450 src->instr = arr->last_write;
451 src->size = arr->length;
452 src->array.id = arr->id;
453 src->array.offset = n;
454
455 if (address)
456 ir3_instr_set_address(mov, address);
457
458 arr->last_access = mov;
459
460 return mov;
461 }
462
463 /* relative (indirect) if address!=NULL */
464 static struct ir3_instruction *
465 create_var_store(struct ir3_compile *ctx, struct ir3_array *arr, int n,
466 struct ir3_instruction *src, struct ir3_instruction *address)
467 {
468 struct ir3_block *block = ctx->block;
469 struct ir3_instruction *mov;
470 struct ir3_register *dst;
471
472 mov = ir3_instr_create(block, 1, OPC_MOV);
473 mov->cat1.src_type = TYPE_U32;
474 mov->cat1.dst_type = TYPE_U32;
475 dst = ir3_reg_create(mov, 0, IR3_REG_ARRAY |
476 COND(address, IR3_REG_RELATIV));
477 dst->instr = arr->last_access;
478 dst->size = arr->length;
479 dst->array.id = arr->id;
480 dst->array.offset = n;
481 ir3_reg_create(mov, 0, IR3_REG_SSA)->instr = src;
482
483 ir3_instr_set_address(mov, address);
484
485 arr->last_write = arr->last_access = mov;
486
487 return mov;
488 }
489
490 static struct ir3_instruction *
491 create_input(struct ir3_block *block, unsigned n)
492 {
493 struct ir3_instruction *in;
494
495 in = ir3_instr_create(block, -1, OPC_META_INPUT);
496 in->inout.block = block;
497 ir3_reg_create(in, n, 0);
498
499 return in;
500 }
501
502 static struct ir3_instruction *
503 create_frag_input(struct ir3_compile *ctx, bool use_ldlv)
504 {
505 struct ir3_block *block = ctx->block;
506 struct ir3_instruction *instr;
507 /* actual inloc is assigned and fixed up later: */
508 struct ir3_instruction *inloc = create_immed(block, 0);
509
510 if (use_ldlv) {
511 instr = ir3_LDLV(block, inloc, 0, create_immed(block, 1), 0);
512 instr->cat6.type = TYPE_U32;
513 instr->cat6.iim_val = 1;
514 } else {
515 instr = ir3_BARY_F(block, inloc, 0, ctx->frag_pos, 0);
516 instr->regs[2]->wrmask = 0x3;
517 }
518
519 return instr;
520 }
521
522 static struct ir3_instruction *
523 create_frag_coord(struct ir3_compile *ctx, unsigned comp)
524 {
525 struct ir3_block *block = ctx->block;
526 struct ir3_instruction *instr;
527
528 compile_assert(ctx, !ctx->frag_coord[comp]);
529
530 ctx->frag_coord[comp] = create_input(ctx->block, 0);
531
532 switch (comp) {
533 case 0: /* .x */
534 case 1: /* .y */
535 /* for frag_coord, we get unsigned values.. we need
536 * to subtract (integer) 8 and divide by 16 (right-
537 * shift by 4) then convert to float:
538 *
539 * sub.s tmp, src, 8
540 * shr.b tmp, tmp, 4
541 * mov.u32f32 dst, tmp
542 *
543 */
544 instr = ir3_SUB_S(block, ctx->frag_coord[comp], 0,
545 create_immed(block, 8), 0);
546 instr = ir3_SHR_B(block, instr, 0,
547 create_immed(block, 4), 0);
548 instr = ir3_COV(block, instr, TYPE_U32, TYPE_F32);
549
550 return instr;
551 case 2: /* .z */
552 case 3: /* .w */
553 default:
554 /* seems that we can use these as-is: */
555 return ctx->frag_coord[comp];
556 }
557 }
558
559 /* NOTE: this creates the "TGSI" style fragface (ie. input slot
560 * VARYING_SLOT_FACE). For NIR style nir_intrinsic_load_front_face
561 * we can just use the value from hw directly (since it is boolean)
562 */
563 static struct ir3_instruction *
564 create_frag_face(struct ir3_compile *ctx, unsigned comp)
565 {
566 struct ir3_block *block = ctx->block;
567 struct ir3_instruction *instr;
568
569 switch (comp) {
570 case 0: /* .x */
571 compile_assert(ctx, !ctx->frag_face);
572
573 ctx->frag_face = create_input(block, 0);
574 ctx->frag_face->regs[0]->flags |= IR3_REG_HALF;
575
576 /* for faceness, we always get -1 or 0 (int).. but TGSI expects
577 * positive vs negative float.. and piglit further seems to
578 * expect -1.0 or 1.0:
579 *
580 * mul.s tmp, hr0.x, 2
581 * add.s tmp, tmp, 1
582 * mov.s32f32, dst, tmp
583 *
584 */
585 instr = ir3_MUL_S(block, ctx->frag_face, 0,
586 create_immed(block, 2), 0);
587 instr = ir3_ADD_S(block, instr, 0,
588 create_immed(block, 1), 0);
589 instr = ir3_COV(block, instr, TYPE_S32, TYPE_F32);
590
591 return instr;
592 case 1: /* .y */
593 case 2: /* .z */
594 return create_immed(block, fui(0.0));
595 default:
596 case 3: /* .w */
597 return create_immed(block, fui(1.0));
598 }
599 }
600
601 static struct ir3_instruction *
602 create_driver_param(struct ir3_compile *ctx, enum ir3_driver_param dp)
603 {
604 /* first four vec4 sysval's reserved for UBOs: */
605 /* NOTE: dp is in scalar, but there can be >4 dp components: */
606 unsigned n = ctx->so->first_driver_param + IR3_DRIVER_PARAM_OFF;
607 unsigned r = regid(n + dp / 4, dp % 4);
608 return create_uniform(ctx, r);
609 }
610
611 /* helper for instructions that produce multiple consecutive scalar
612 * outputs which need to have a split/fanout meta instruction inserted
613 */
614 static void
615 split_dest(struct ir3_block *block, struct ir3_instruction **dst,
616 struct ir3_instruction *src, unsigned n)
617 {
618 struct ir3_instruction *prev = NULL;
619 for (int i = 0, j = 0; i < n; i++) {
620 struct ir3_instruction *split =
621 ir3_instr_create(block, -1, OPC_META_FO);
622 ir3_reg_create(split, 0, IR3_REG_SSA);
623 ir3_reg_create(split, 0, IR3_REG_SSA)->instr = src;
624 split->fo.off = i;
625
626 if (prev) {
627 split->cp.left = prev;
628 split->cp.left_cnt++;
629 prev->cp.right = split;
630 prev->cp.right_cnt++;
631 }
632 prev = split;
633
634 if (src->regs[0]->wrmask & (1 << i))
635 dst[j++] = split;
636 }
637 }
638
639 /*
640 * Adreno uses uint rather than having dedicated bool type,
641 * which (potentially) requires some conversion, in particular
642 * when using output of an bool instr to int input, or visa
643 * versa.
644 *
645 * | Adreno | NIR |
646 * -------+---------+-------+-
647 * true | 1 | ~0 |
648 * false | 0 | 0 |
649 *
650 * To convert from an adreno bool (uint) to nir, use:
651 *
652 * absneg.s dst, (neg)src
653 *
654 * To convert back in the other direction:
655 *
656 * absneg.s dst, (abs)arc
657 *
658 * The CP step can clean up the absneg.s that cancel each other
659 * out, and with a slight bit of extra cleverness (to recognize
660 * the instructions which produce either a 0 or 1) can eliminate
661 * the absneg.s's completely when an instruction that wants
662 * 0/1 consumes the result. For example, when a nir 'bcsel'
663 * consumes the result of 'feq'. So we should be able to get by
664 * without a boolean resolve step, and without incuring any
665 * extra penalty in instruction count.
666 */
667
668 /* NIR bool -> native (adreno): */
669 static struct ir3_instruction *
670 ir3_b2n(struct ir3_block *block, struct ir3_instruction *instr)
671 {
672 return ir3_ABSNEG_S(block, instr, IR3_REG_SABS);
673 }
674
675 /* native (adreno) -> NIR bool: */
676 static struct ir3_instruction *
677 ir3_n2b(struct ir3_block *block, struct ir3_instruction *instr)
678 {
679 return ir3_ABSNEG_S(block, instr, IR3_REG_SNEG);
680 }
681
682 /*
683 * alu/sfu instructions:
684 */
685
686 static void
687 emit_alu(struct ir3_compile *ctx, nir_alu_instr *alu)
688 {
689 const nir_op_info *info = &nir_op_infos[alu->op];
690 struct ir3_instruction **dst, *src[info->num_inputs];
691 struct ir3_block *b = ctx->block;
692
693 dst = get_dst(ctx, &alu->dest.dest, MAX2(info->output_size, 1));
694
695 /* Vectors are special in that they have non-scalarized writemasks,
696 * and just take the first swizzle channel for each argument in
697 * order into each writemask channel.
698 */
699 if ((alu->op == nir_op_vec2) ||
700 (alu->op == nir_op_vec3) ||
701 (alu->op == nir_op_vec4)) {
702
703 for (int i = 0; i < info->num_inputs; i++) {
704 nir_alu_src *asrc = &alu->src[i];
705
706 compile_assert(ctx, !asrc->abs);
707 compile_assert(ctx, !asrc->negate);
708
709 src[i] = get_src(ctx, &asrc->src)[asrc->swizzle[0]];
710 if (!src[i])
711 src[i] = create_immed(ctx->block, 0);
712 dst[i] = ir3_MOV(b, src[i], TYPE_U32);
713 }
714
715 return;
716 }
717
718 /* General case: We can just grab the one used channel per src. */
719 for (int i = 0; i < info->num_inputs; i++) {
720 unsigned chan = ffs(alu->dest.write_mask) - 1;
721 nir_alu_src *asrc = &alu->src[i];
722
723 compile_assert(ctx, !asrc->abs);
724 compile_assert(ctx, !asrc->negate);
725
726 src[i] = get_src(ctx, &asrc->src)[asrc->swizzle[chan]];
727
728 compile_assert(ctx, src[i]);
729 }
730
731 switch (alu->op) {
732 case nir_op_f2i:
733 dst[0] = ir3_COV(b, src[0], TYPE_F32, TYPE_S32);
734 break;
735 case nir_op_f2u:
736 dst[0] = ir3_COV(b, src[0], TYPE_F32, TYPE_U32);
737 break;
738 case nir_op_i2f:
739 dst[0] = ir3_COV(b, src[0], TYPE_S32, TYPE_F32);
740 break;
741 case nir_op_u2f:
742 dst[0] = ir3_COV(b, src[0], TYPE_U32, TYPE_F32);
743 break;
744 case nir_op_imov:
745 dst[0] = ir3_MOV(b, src[0], TYPE_S32);
746 break;
747 case nir_op_fmov:
748 dst[0] = ir3_MOV(b, src[0], TYPE_F32);
749 break;
750 case nir_op_f2b:
751 dst[0] = ir3_CMPS_F(b, src[0], 0, create_immed(b, fui(0.0)), 0);
752 dst[0]->cat2.condition = IR3_COND_NE;
753 dst[0] = ir3_n2b(b, dst[0]);
754 break;
755 case nir_op_b2f:
756 dst[0] = ir3_COV(b, ir3_b2n(b, src[0]), TYPE_U32, TYPE_F32);
757 break;
758 case nir_op_b2i:
759 dst[0] = ir3_b2n(b, src[0]);
760 break;
761 case nir_op_i2b:
762 dst[0] = ir3_CMPS_S(b, src[0], 0, create_immed(b, 0), 0);
763 dst[0]->cat2.condition = IR3_COND_NE;
764 dst[0] = ir3_n2b(b, dst[0]);
765 break;
766
767 case nir_op_fneg:
768 dst[0] = ir3_ABSNEG_F(b, src[0], IR3_REG_FNEG);
769 break;
770 case nir_op_fabs:
771 dst[0] = ir3_ABSNEG_F(b, src[0], IR3_REG_FABS);
772 break;
773 case nir_op_fmax:
774 dst[0] = ir3_MAX_F(b, src[0], 0, src[1], 0);
775 break;
776 case nir_op_fmin:
777 dst[0] = ir3_MIN_F(b, src[0], 0, src[1], 0);
778 break;
779 case nir_op_fmul:
780 dst[0] = ir3_MUL_F(b, src[0], 0, src[1], 0);
781 break;
782 case nir_op_fadd:
783 dst[0] = ir3_ADD_F(b, src[0], 0, src[1], 0);
784 break;
785 case nir_op_fsub:
786 dst[0] = ir3_ADD_F(b, src[0], 0, src[1], IR3_REG_FNEG);
787 break;
788 case nir_op_ffma:
789 dst[0] = ir3_MAD_F32(b, src[0], 0, src[1], 0, src[2], 0);
790 break;
791 case nir_op_fddx:
792 dst[0] = ir3_DSX(b, src[0], 0);
793 dst[0]->cat5.type = TYPE_F32;
794 break;
795 case nir_op_fddy:
796 dst[0] = ir3_DSY(b, src[0], 0);
797 dst[0]->cat5.type = TYPE_F32;
798 break;
799 break;
800 case nir_op_flt:
801 dst[0] = ir3_CMPS_F(b, src[0], 0, src[1], 0);
802 dst[0]->cat2.condition = IR3_COND_LT;
803 dst[0] = ir3_n2b(b, dst[0]);
804 break;
805 case nir_op_fge:
806 dst[0] = ir3_CMPS_F(b, src[0], 0, src[1], 0);
807 dst[0]->cat2.condition = IR3_COND_GE;
808 dst[0] = ir3_n2b(b, dst[0]);
809 break;
810 case nir_op_feq:
811 dst[0] = ir3_CMPS_F(b, src[0], 0, src[1], 0);
812 dst[0]->cat2.condition = IR3_COND_EQ;
813 dst[0] = ir3_n2b(b, dst[0]);
814 break;
815 case nir_op_fne:
816 dst[0] = ir3_CMPS_F(b, src[0], 0, src[1], 0);
817 dst[0]->cat2.condition = IR3_COND_NE;
818 dst[0] = ir3_n2b(b, dst[0]);
819 break;
820 case nir_op_fceil:
821 dst[0] = ir3_CEIL_F(b, src[0], 0);
822 break;
823 case nir_op_ffloor:
824 dst[0] = ir3_FLOOR_F(b, src[0], 0);
825 break;
826 case nir_op_ftrunc:
827 dst[0] = ir3_TRUNC_F(b, src[0], 0);
828 break;
829 case nir_op_fround_even:
830 dst[0] = ir3_RNDNE_F(b, src[0], 0);
831 break;
832 case nir_op_fsign:
833 dst[0] = ir3_SIGN_F(b, src[0], 0);
834 break;
835
836 case nir_op_fsin:
837 dst[0] = ir3_SIN(b, src[0], 0);
838 break;
839 case nir_op_fcos:
840 dst[0] = ir3_COS(b, src[0], 0);
841 break;
842 case nir_op_frsq:
843 dst[0] = ir3_RSQ(b, src[0], 0);
844 break;
845 case nir_op_frcp:
846 dst[0] = ir3_RCP(b, src[0], 0);
847 break;
848 case nir_op_flog2:
849 dst[0] = ir3_LOG2(b, src[0], 0);
850 break;
851 case nir_op_fexp2:
852 dst[0] = ir3_EXP2(b, src[0], 0);
853 break;
854 case nir_op_fsqrt:
855 dst[0] = ir3_SQRT(b, src[0], 0);
856 break;
857
858 case nir_op_iabs:
859 dst[0] = ir3_ABSNEG_S(b, src[0], IR3_REG_SABS);
860 break;
861 case nir_op_iadd:
862 dst[0] = ir3_ADD_U(b, src[0], 0, src[1], 0);
863 break;
864 case nir_op_iand:
865 dst[0] = ir3_AND_B(b, src[0], 0, src[1], 0);
866 break;
867 case nir_op_imax:
868 dst[0] = ir3_MAX_S(b, src[0], 0, src[1], 0);
869 break;
870 case nir_op_umax:
871 dst[0] = ir3_MAX_U(b, src[0], 0, src[1], 0);
872 break;
873 case nir_op_imin:
874 dst[0] = ir3_MIN_S(b, src[0], 0, src[1], 0);
875 break;
876 case nir_op_umin:
877 dst[0] = ir3_MIN_U(b, src[0], 0, src[1], 0);
878 break;
879 case nir_op_imul:
880 /*
881 * dst = (al * bl) + (ah * bl << 16) + (al * bh << 16)
882 * mull.u tmp0, a, b ; mul low, i.e. al * bl
883 * madsh.m16 tmp1, a, b, tmp0 ; mul-add shift high mix, i.e. ah * bl << 16
884 * madsh.m16 dst, b, a, tmp1 ; i.e. al * bh << 16
885 */
886 dst[0] = ir3_MADSH_M16(b, src[1], 0, src[0], 0,
887 ir3_MADSH_M16(b, src[0], 0, src[1], 0,
888 ir3_MULL_U(b, src[0], 0, src[1], 0), 0), 0);
889 break;
890 case nir_op_ineg:
891 dst[0] = ir3_ABSNEG_S(b, src[0], IR3_REG_SNEG);
892 break;
893 case nir_op_inot:
894 dst[0] = ir3_NOT_B(b, src[0], 0);
895 break;
896 case nir_op_ior:
897 dst[0] = ir3_OR_B(b, src[0], 0, src[1], 0);
898 break;
899 case nir_op_ishl:
900 dst[0] = ir3_SHL_B(b, src[0], 0, src[1], 0);
901 break;
902 case nir_op_ishr:
903 dst[0] = ir3_ASHR_B(b, src[0], 0, src[1], 0);
904 break;
905 case nir_op_isign: {
906 /* maybe this would be sane to lower in nir.. */
907 struct ir3_instruction *neg, *pos;
908
909 neg = ir3_CMPS_S(b, src[0], 0, create_immed(b, 0), 0);
910 neg->cat2.condition = IR3_COND_LT;
911
912 pos = ir3_CMPS_S(b, src[0], 0, create_immed(b, 0), 0);
913 pos->cat2.condition = IR3_COND_GT;
914
915 dst[0] = ir3_SUB_U(b, pos, 0, neg, 0);
916
917 break;
918 }
919 case nir_op_isub:
920 dst[0] = ir3_SUB_U(b, src[0], 0, src[1], 0);
921 break;
922 case nir_op_ixor:
923 dst[0] = ir3_XOR_B(b, src[0], 0, src[1], 0);
924 break;
925 case nir_op_ushr:
926 dst[0] = ir3_SHR_B(b, src[0], 0, src[1], 0);
927 break;
928 case nir_op_ilt:
929 dst[0] = ir3_CMPS_S(b, src[0], 0, src[1], 0);
930 dst[0]->cat2.condition = IR3_COND_LT;
931 dst[0] = ir3_n2b(b, dst[0]);
932 break;
933 case nir_op_ige:
934 dst[0] = ir3_CMPS_S(b, src[0], 0, src[1], 0);
935 dst[0]->cat2.condition = IR3_COND_GE;
936 dst[0] = ir3_n2b(b, dst[0]);
937 break;
938 case nir_op_ieq:
939 dst[0] = ir3_CMPS_S(b, src[0], 0, src[1], 0);
940 dst[0]->cat2.condition = IR3_COND_EQ;
941 dst[0] = ir3_n2b(b, dst[0]);
942 break;
943 case nir_op_ine:
944 dst[0] = ir3_CMPS_S(b, src[0], 0, src[1], 0);
945 dst[0]->cat2.condition = IR3_COND_NE;
946 dst[0] = ir3_n2b(b, dst[0]);
947 break;
948 case nir_op_ult:
949 dst[0] = ir3_CMPS_U(b, src[0], 0, src[1], 0);
950 dst[0]->cat2.condition = IR3_COND_LT;
951 dst[0] = ir3_n2b(b, dst[0]);
952 break;
953 case nir_op_uge:
954 dst[0] = ir3_CMPS_U(b, src[0], 0, src[1], 0);
955 dst[0]->cat2.condition = IR3_COND_GE;
956 dst[0] = ir3_n2b(b, dst[0]);
957 break;
958
959 case nir_op_bcsel:
960 dst[0] = ir3_SEL_B32(b, src[1], 0, ir3_b2n(b, src[0]), 0, src[2], 0);
961 break;
962
963 case nir_op_bit_count:
964 dst[0] = ir3_CBITS_B(b, src[0], 0);
965 break;
966 case nir_op_ifind_msb: {
967 struct ir3_instruction *cmp;
968 dst[0] = ir3_CLZ_S(b, src[0], 0);
969 cmp = ir3_CMPS_S(b, dst[0], 0, create_immed(b, 0), 0);
970 cmp->cat2.condition = IR3_COND_GE;
971 dst[0] = ir3_SEL_B32(b,
972 ir3_SUB_U(b, create_immed(b, 31), 0, dst[0], 0), 0,
973 cmp, 0, dst[0], 0);
974 break;
975 }
976 case nir_op_ufind_msb:
977 dst[0] = ir3_CLZ_B(b, src[0], 0);
978 dst[0] = ir3_SEL_B32(b,
979 ir3_SUB_U(b, create_immed(b, 31), 0, dst[0], 0), 0,
980 src[0], 0, dst[0], 0);
981 break;
982 case nir_op_find_lsb:
983 dst[0] = ir3_BFREV_B(b, src[0], 0);
984 dst[0] = ir3_CLZ_B(b, dst[0], 0);
985 break;
986 case nir_op_bitfield_reverse:
987 dst[0] = ir3_BFREV_B(b, src[0], 0);
988 break;
989
990 default:
991 compile_error(ctx, "Unhandled ALU op: %s\n",
992 nir_op_infos[alu->op].name);
993 break;
994 }
995 }
996
997 /* handles direct/indirect UBO reads: */
998 static void
999 emit_intrinsic_load_ubo(struct ir3_compile *ctx, nir_intrinsic_instr *intr,
1000 struct ir3_instruction **dst)
1001 {
1002 struct ir3_block *b = ctx->block;
1003 struct ir3_instruction *addr, *src0, *src1;
1004 nir_const_value *const_offset;
1005 /* UBO addresses are the first driver params: */
1006 unsigned ubo = regid(ctx->so->first_driver_param + IR3_UBOS_OFF, 0);
1007 int off = 0;
1008
1009 /* First src is ubo index, which could either be an immed or not: */
1010 src0 = get_src(ctx, &intr->src[0])[0];
1011 if (is_same_type_mov(src0) &&
1012 (src0->regs[1]->flags & IR3_REG_IMMED)) {
1013 addr = create_uniform(ctx, ubo + src0->regs[1]->iim_val);
1014 } else {
1015 addr = create_uniform_indirect(ctx, ubo, get_addr(ctx, src0));
1016 }
1017
1018 const_offset = nir_src_as_const_value(intr->src[1]);
1019 if (const_offset) {
1020 off += const_offset->u32[0];
1021 } else {
1022 /* For load_ubo_indirect, second src is indirect offset: */
1023 src1 = get_src(ctx, &intr->src[1])[0];
1024
1025 /* and add offset to addr: */
1026 addr = ir3_ADD_S(b, addr, 0, src1, 0);
1027 }
1028
1029 /* if offset is to large to encode in the ldg, split it out: */
1030 if ((off + (intr->num_components * 4)) > 1024) {
1031 /* split out the minimal amount to improve the odds that
1032 * cp can fit the immediate in the add.s instruction:
1033 */
1034 unsigned off2 = off + (intr->num_components * 4) - 1024;
1035 addr = ir3_ADD_S(b, addr, 0, create_immed(b, off2), 0);
1036 off -= off2;
1037 }
1038
1039 for (int i = 0; i < intr->num_components; i++) {
1040 struct ir3_instruction *load =
1041 ir3_LDG(b, addr, 0, create_immed(b, 1), 0);
1042 load->cat6.type = TYPE_U32;
1043 load->cat6.src_offset = off + i * 4; /* byte offset */
1044 dst[i] = load;
1045 }
1046 }
1047
1048 /* handles array reads: */
1049 static void
1050 emit_intrinsic_load_var(struct ir3_compile *ctx, nir_intrinsic_instr *intr,
1051 struct ir3_instruction **dst)
1052 {
1053 nir_deref_var *dvar = intr->variables[0];
1054 nir_deref_array *darr = nir_deref_as_array(dvar->deref.child);
1055 struct ir3_array *arr = get_var(ctx, dvar->var);
1056
1057 compile_assert(ctx, dvar->deref.child &&
1058 (dvar->deref.child->deref_type == nir_deref_type_array));
1059
1060 switch (darr->deref_array_type) {
1061 case nir_deref_array_type_direct:
1062 /* direct access does not require anything special: */
1063 for (int i = 0; i < intr->num_components; i++) {
1064 unsigned n = darr->base_offset * 4 + i;
1065 compile_assert(ctx, n < arr->length);
1066 dst[i] = create_var_load(ctx, arr, n, NULL);
1067 }
1068 break;
1069 case nir_deref_array_type_indirect: {
1070 /* for indirect, we need to collect all the array elements: */
1071 struct ir3_instruction *addr =
1072 get_addr(ctx, get_src(ctx, &darr->indirect)[0]);
1073 for (int i = 0; i < intr->num_components; i++) {
1074 unsigned n = darr->base_offset * 4 + i;
1075 compile_assert(ctx, n < arr->length);
1076 dst[i] = create_var_load(ctx, arr, n, addr);
1077 }
1078 break;
1079 }
1080 default:
1081 compile_error(ctx, "Unhandled load deref type: %u\n",
1082 darr->deref_array_type);
1083 break;
1084 }
1085 }
1086
1087 /* handles array writes: */
1088 static void
1089 emit_intrinsic_store_var(struct ir3_compile *ctx, nir_intrinsic_instr *intr)
1090 {
1091 nir_deref_var *dvar = intr->variables[0];
1092 nir_deref_array *darr = nir_deref_as_array(dvar->deref.child);
1093 struct ir3_array *arr = get_var(ctx, dvar->var);
1094 struct ir3_instruction *addr, **src;
1095 unsigned wrmask = nir_intrinsic_write_mask(intr);
1096
1097 compile_assert(ctx, dvar->deref.child &&
1098 (dvar->deref.child->deref_type == nir_deref_type_array));
1099
1100 src = get_src(ctx, &intr->src[0]);
1101
1102 switch (darr->deref_array_type) {
1103 case nir_deref_array_type_direct:
1104 addr = NULL;
1105 break;
1106 case nir_deref_array_type_indirect:
1107 addr = get_addr(ctx, get_src(ctx, &darr->indirect)[0]);
1108 break;
1109 default:
1110 compile_error(ctx, "Unhandled store deref type: %u\n",
1111 darr->deref_array_type);
1112 return;
1113 }
1114
1115 for (int i = 0; i < intr->num_components; i++) {
1116 if (!(wrmask & (1 << i)))
1117 continue;
1118 unsigned n = darr->base_offset * 4 + i;
1119 compile_assert(ctx, n < arr->length);
1120 create_var_store(ctx, arr, n, src[i], addr);
1121 }
1122 }
1123
1124 static void add_sysval_input(struct ir3_compile *ctx, gl_system_value slot,
1125 struct ir3_instruction *instr)
1126 {
1127 struct ir3_shader_variant *so = ctx->so;
1128 unsigned r = regid(so->inputs_count, 0);
1129 unsigned n = so->inputs_count++;
1130
1131 so->inputs[n].sysval = true;
1132 so->inputs[n].slot = slot;
1133 so->inputs[n].compmask = 1;
1134 so->inputs[n].regid = r;
1135 so->inputs[n].interpolate = INTERP_QUALIFIER_FLAT;
1136 so->total_in++;
1137
1138 ctx->ir->ninputs = MAX2(ctx->ir->ninputs, r + 1);
1139 ctx->ir->inputs[r] = instr;
1140 }
1141
1142 static void
1143 emit_intrinsic(struct ir3_compile *ctx, nir_intrinsic_instr *intr)
1144 {
1145 const nir_intrinsic_info *info = &nir_intrinsic_infos[intr->intrinsic];
1146 struct ir3_instruction **dst, **src;
1147 struct ir3_block *b = ctx->block;
1148 nir_const_value *const_offset;
1149 int idx;
1150
1151 if (info->has_dest) {
1152 dst = get_dst(ctx, &intr->dest, intr->num_components);
1153 } else {
1154 dst = NULL;
1155 }
1156
1157 switch (intr->intrinsic) {
1158 case nir_intrinsic_load_uniform:
1159 idx = nir_intrinsic_base(intr);
1160 const_offset = nir_src_as_const_value(intr->src[0]);
1161 if (const_offset) {
1162 idx += const_offset->u32[0];
1163 for (int i = 0; i < intr->num_components; i++) {
1164 unsigned n = idx * 4 + i;
1165 dst[i] = create_uniform(ctx, n);
1166 }
1167 } else {
1168 src = get_src(ctx, &intr->src[0]);
1169 for (int i = 0; i < intr->num_components; i++) {
1170 int n = idx * 4 + i;
1171 dst[i] = create_uniform_indirect(ctx, n,
1172 get_addr(ctx, src[0]));
1173 }
1174 /* NOTE: if relative addressing is used, we set
1175 * constlen in the compiler (to worst-case value)
1176 * since we don't know in the assembler what the max
1177 * addr reg value can be:
1178 */
1179 ctx->so->constlen = ctx->s->num_uniforms;
1180 }
1181 break;
1182 case nir_intrinsic_load_ubo:
1183 emit_intrinsic_load_ubo(ctx, intr, dst);
1184 break;
1185 case nir_intrinsic_load_input:
1186 idx = nir_intrinsic_base(intr);
1187 const_offset = nir_src_as_const_value(intr->src[0]);
1188 if (const_offset) {
1189 idx += const_offset->u32[0];
1190 for (int i = 0; i < intr->num_components; i++) {
1191 unsigned n = idx * 4 + i;
1192 dst[i] = ctx->ir->inputs[n];
1193 }
1194 } else {
1195 src = get_src(ctx, &intr->src[0]);
1196 struct ir3_instruction *collect =
1197 create_collect(b, ctx->ir->inputs, ctx->ir->ninputs);
1198 struct ir3_instruction *addr = get_addr(ctx, src[0]);
1199 for (int i = 0; i < intr->num_components; i++) {
1200 unsigned n = idx * 4 + i;
1201 dst[i] = create_indirect_load(ctx, ctx->ir->ninputs,
1202 n, addr, collect);
1203 }
1204 }
1205 break;
1206 case nir_intrinsic_load_var:
1207 emit_intrinsic_load_var(ctx, intr, dst);
1208 break;
1209 case nir_intrinsic_store_var:
1210 emit_intrinsic_store_var(ctx, intr);
1211 break;
1212 case nir_intrinsic_store_output:
1213 idx = nir_intrinsic_base(intr);
1214 const_offset = nir_src_as_const_value(intr->src[1]);
1215 compile_assert(ctx, const_offset != NULL);
1216 idx += const_offset->u32[0];
1217
1218 src = get_src(ctx, &intr->src[0]);
1219 for (int i = 0; i < intr->num_components; i++) {
1220 unsigned n = idx * 4 + i;
1221 ctx->ir->outputs[n] = src[i];
1222 }
1223 break;
1224 case nir_intrinsic_load_base_vertex:
1225 if (!ctx->basevertex) {
1226 ctx->basevertex = create_driver_param(ctx, IR3_DP_VTXID_BASE);
1227 add_sysval_input(ctx, SYSTEM_VALUE_BASE_VERTEX,
1228 ctx->basevertex);
1229 }
1230 dst[0] = ctx->basevertex;
1231 break;
1232 case nir_intrinsic_load_vertex_id_zero_base:
1233 if (!ctx->vertex_id) {
1234 ctx->vertex_id = create_input(b, 0);
1235 add_sysval_input(ctx, SYSTEM_VALUE_VERTEX_ID_ZERO_BASE,
1236 ctx->vertex_id);
1237 }
1238 dst[0] = ctx->vertex_id;
1239 break;
1240 case nir_intrinsic_load_instance_id:
1241 if (!ctx->instance_id) {
1242 ctx->instance_id = create_input(b, 0);
1243 add_sysval_input(ctx, SYSTEM_VALUE_INSTANCE_ID,
1244 ctx->instance_id);
1245 }
1246 dst[0] = ctx->instance_id;
1247 break;
1248 case nir_intrinsic_load_user_clip_plane:
1249 idx = nir_intrinsic_ucp_id(intr);
1250 for (int i = 0; i < intr->num_components; i++) {
1251 unsigned n = idx * 4 + i;
1252 dst[i] = create_driver_param(ctx, IR3_DP_UCP0_X + n);
1253 }
1254 break;
1255 case nir_intrinsic_load_front_face:
1256 if (!ctx->frag_face) {
1257 ctx->so->frag_face = true;
1258 ctx->frag_face = create_input(b, 0);
1259 ctx->frag_face->regs[0]->flags |= IR3_REG_HALF;
1260 }
1261 /* for fragface, we always get -1 or 0, but that is inverse
1262 * of what nir expects (where ~0 is true). Unfortunately
1263 * trying to widen from half to full in add.s seems to do a
1264 * non-sign-extending widen (resulting in something that
1265 * gets interpreted as float Inf??)
1266 */
1267 dst[0] = ir3_COV(b, ctx->frag_face, TYPE_S16, TYPE_S32);
1268 dst[0] = ir3_ADD_S(b, dst[0], 0, create_immed(b, 1), 0);
1269 break;
1270 case nir_intrinsic_discard_if:
1271 case nir_intrinsic_discard: {
1272 struct ir3_instruction *cond, *kill;
1273
1274 if (intr->intrinsic == nir_intrinsic_discard_if) {
1275 /* conditional discard: */
1276 src = get_src(ctx, &intr->src[0]);
1277 cond = ir3_b2n(b, src[0]);
1278 } else {
1279 /* unconditional discard: */
1280 cond = create_immed(b, 1);
1281 }
1282
1283 /* NOTE: only cmps.*.* can write p0.x: */
1284 cond = ir3_CMPS_S(b, cond, 0, create_immed(b, 0), 0);
1285 cond->cat2.condition = IR3_COND_NE;
1286
1287 /* condition always goes in predicate register: */
1288 cond->regs[0]->num = regid(REG_P0, 0);
1289
1290 kill = ir3_KILL(b, cond, 0);
1291 array_insert(ctx->ir->predicates, kill);
1292
1293 array_insert(ctx->ir->keeps, kill);
1294 ctx->so->has_kill = true;
1295
1296 break;
1297 }
1298 default:
1299 compile_error(ctx, "Unhandled intrinsic type: %s\n",
1300 nir_intrinsic_infos[intr->intrinsic].name);
1301 break;
1302 }
1303 }
1304
1305 static void
1306 emit_load_const(struct ir3_compile *ctx, nir_load_const_instr *instr)
1307 {
1308 struct ir3_instruction **dst = get_dst_ssa(ctx, &instr->def,
1309 instr->def.num_components);
1310 for (int i = 0; i < instr->def.num_components; i++)
1311 dst[i] = create_immed(ctx->block, instr->value.u32[i]);
1312 }
1313
1314 static void
1315 emit_undef(struct ir3_compile *ctx, nir_ssa_undef_instr *undef)
1316 {
1317 struct ir3_instruction **dst = get_dst_ssa(ctx, &undef->def,
1318 undef->def.num_components);
1319 /* backend doesn't want undefined instructions, so just plug
1320 * in 0.0..
1321 */
1322 for (int i = 0; i < undef->def.num_components; i++)
1323 dst[i] = create_immed(ctx->block, fui(0.0));
1324 }
1325
1326 /*
1327 * texture fetch/sample instructions:
1328 */
1329
1330 static void
1331 tex_info(nir_tex_instr *tex, unsigned *flagsp, unsigned *coordsp)
1332 {
1333 unsigned coords, flags = 0;
1334
1335 /* note: would use tex->coord_components.. except txs.. also,
1336 * since array index goes after shadow ref, we don't want to
1337 * count it:
1338 */
1339 switch (tex->sampler_dim) {
1340 case GLSL_SAMPLER_DIM_1D:
1341 case GLSL_SAMPLER_DIM_BUF:
1342 coords = 1;
1343 break;
1344 case GLSL_SAMPLER_DIM_2D:
1345 case GLSL_SAMPLER_DIM_RECT:
1346 case GLSL_SAMPLER_DIM_EXTERNAL:
1347 case GLSL_SAMPLER_DIM_MS:
1348 coords = 2;
1349 break;
1350 case GLSL_SAMPLER_DIM_3D:
1351 case GLSL_SAMPLER_DIM_CUBE:
1352 coords = 3;
1353 flags |= IR3_INSTR_3D;
1354 break;
1355 default:
1356 unreachable("bad sampler_dim");
1357 }
1358
1359 if (tex->is_shadow && tex->op != nir_texop_lod)
1360 flags |= IR3_INSTR_S;
1361
1362 if (tex->is_array && tex->op != nir_texop_lod)
1363 flags |= IR3_INSTR_A;
1364
1365 *flagsp = flags;
1366 *coordsp = coords;
1367 }
1368
1369 static void
1370 emit_tex(struct ir3_compile *ctx, nir_tex_instr *tex)
1371 {
1372 struct ir3_block *b = ctx->block;
1373 struct ir3_instruction **dst, *sam, *src0[12], *src1[4];
1374 struct ir3_instruction **coord, *lod, *compare, *proj, **off, **ddx, **ddy;
1375 bool has_bias = false, has_lod = false, has_proj = false, has_off = false;
1376 unsigned i, coords, flags;
1377 unsigned nsrc0 = 0, nsrc1 = 0;
1378 type_t type;
1379 opc_t opc = 0;
1380
1381 coord = off = ddx = ddy = NULL;
1382 lod = proj = compare = NULL;
1383
1384 /* TODO: might just be one component for gathers? */
1385 dst = get_dst(ctx, &tex->dest, 4);
1386
1387 for (unsigned i = 0; i < tex->num_srcs; i++) {
1388 switch (tex->src[i].src_type) {
1389 case nir_tex_src_coord:
1390 coord = get_src(ctx, &tex->src[i].src);
1391 break;
1392 case nir_tex_src_bias:
1393 lod = get_src(ctx, &tex->src[i].src)[0];
1394 has_bias = true;
1395 break;
1396 case nir_tex_src_lod:
1397 lod = get_src(ctx, &tex->src[i].src)[0];
1398 has_lod = true;
1399 break;
1400 case nir_tex_src_comparitor: /* shadow comparator */
1401 compare = get_src(ctx, &tex->src[i].src)[0];
1402 break;
1403 case nir_tex_src_projector:
1404 proj = get_src(ctx, &tex->src[i].src)[0];
1405 has_proj = true;
1406 break;
1407 case nir_tex_src_offset:
1408 off = get_src(ctx, &tex->src[i].src);
1409 has_off = true;
1410 break;
1411 case nir_tex_src_ddx:
1412 ddx = get_src(ctx, &tex->src[i].src);
1413 break;
1414 case nir_tex_src_ddy:
1415 ddy = get_src(ctx, &tex->src[i].src);
1416 break;
1417 default:
1418 compile_error(ctx, "Unhandled NIR tex src type: %d\n",
1419 tex->src[i].src_type);
1420 return;
1421 }
1422 }
1423
1424 switch (tex->op) {
1425 case nir_texop_tex: opc = OPC_SAM; break;
1426 case nir_texop_txb: opc = OPC_SAMB; break;
1427 case nir_texop_txl: opc = OPC_SAML; break;
1428 case nir_texop_txd: opc = OPC_SAMGQ; break;
1429 case nir_texop_txf: opc = OPC_ISAML; break;
1430 case nir_texop_lod: opc = OPC_GETLOD; break;
1431 case nir_texop_txf_ms:
1432 case nir_texop_txs:
1433 case nir_texop_tg4:
1434 case nir_texop_query_levels:
1435 case nir_texop_texture_samples:
1436 case nir_texop_samples_identical:
1437 compile_error(ctx, "Unhandled NIR tex type: %d\n", tex->op);
1438 return;
1439 }
1440
1441 tex_info(tex, &flags, &coords);
1442
1443 /* scale up integer coords for TXF based on the LOD */
1444 if (ctx->unminify_coords && (opc == OPC_ISAML)) {
1445 assert(has_lod);
1446 for (i = 0; i < coords; i++)
1447 coord[i] = ir3_SHL_B(b, coord[i], 0, lod, 0);
1448 }
1449
1450 /* the array coord for cube arrays needs 0.5 added to it */
1451 if (tex->sampler_dim == GLSL_SAMPLER_DIM_CUBE && tex->is_array &&
1452 opc != OPC_ISAML)
1453 coord[3] = ir3_ADD_F(b, coord[3], 0, create_immed(b, fui(0.5)), 0);
1454
1455 /*
1456 * lay out the first argument in the proper order:
1457 * - actual coordinates first
1458 * - shadow reference
1459 * - array index
1460 * - projection w
1461 * - starting at offset 4, dpdx.xy, dpdy.xy
1462 *
1463 * bias/lod go into the second arg
1464 */
1465
1466 /* insert tex coords: */
1467 for (i = 0; i < coords; i++)
1468 src0[nsrc0++] = coord[i];
1469
1470 if (coords == 1) {
1471 /* hw doesn't do 1d, so we treat it as 2d with
1472 * height of 1, and patch up the y coord.
1473 * TODO: y coord should be (int)0 in some cases..
1474 */
1475 src0[nsrc0++] = create_immed(b, fui(0.5));
1476 }
1477
1478 if (tex->is_shadow && tex->op != nir_texop_lod)
1479 src0[nsrc0++] = compare;
1480
1481 if (tex->is_array && tex->op != nir_texop_lod)
1482 src0[nsrc0++] = coord[coords];
1483
1484 if (has_proj) {
1485 src0[nsrc0++] = proj;
1486 flags |= IR3_INSTR_P;
1487 }
1488
1489 /* pad to 4, then ddx/ddy: */
1490 if (tex->op == nir_texop_txd) {
1491 while (nsrc0 < 4)
1492 src0[nsrc0++] = create_immed(b, fui(0.0));
1493 for (i = 0; i < coords; i++)
1494 src0[nsrc0++] = ddx[i];
1495 if (coords < 2)
1496 src0[nsrc0++] = create_immed(b, fui(0.0));
1497 for (i = 0; i < coords; i++)
1498 src0[nsrc0++] = ddy[i];
1499 if (coords < 2)
1500 src0[nsrc0++] = create_immed(b, fui(0.0));
1501 }
1502
1503 /*
1504 * second argument (if applicable):
1505 * - offsets
1506 * - lod
1507 * - bias
1508 */
1509 if (has_off | has_lod | has_bias) {
1510 if (has_off) {
1511 for (i = 0; i < coords; i++)
1512 src1[nsrc1++] = off[i];
1513 if (coords < 2)
1514 src1[nsrc1++] = create_immed(b, fui(0.0));
1515 flags |= IR3_INSTR_O;
1516 }
1517
1518 if (has_lod | has_bias)
1519 src1[nsrc1++] = lod;
1520 }
1521
1522 switch (tex->dest_type) {
1523 case nir_type_invalid:
1524 case nir_type_float:
1525 type = TYPE_F32;
1526 break;
1527 case nir_type_int:
1528 type = TYPE_S32;
1529 break;
1530 case nir_type_uint:
1531 case nir_type_bool:
1532 type = TYPE_U32;
1533 break;
1534 default:
1535 unreachable("bad dest_type");
1536 }
1537
1538 if (opc == OPC_GETLOD)
1539 type = TYPE_U32;
1540
1541 sam = ir3_SAM(b, opc, type, TGSI_WRITEMASK_XYZW,
1542 flags, tex->texture_index, tex->texture_index,
1543 create_collect(b, src0, nsrc0),
1544 create_collect(b, src1, nsrc1));
1545
1546 split_dest(b, dst, sam, 4);
1547
1548 /* GETLOD returns results in 4.8 fixed point */
1549 if (opc == OPC_GETLOD) {
1550 struct ir3_instruction *factor = create_immed(b, fui(1.0 / 256));
1551
1552 compile_assert(ctx, tex->dest_type == nir_type_float);
1553 for (i = 0; i < 2; i++) {
1554 dst[i] = ir3_MUL_F(b, ir3_COV(b, dst[i], TYPE_U32, TYPE_F32), 0,
1555 factor, 0);
1556 }
1557 }
1558 }
1559
1560 static void
1561 emit_tex_query_levels(struct ir3_compile *ctx, nir_tex_instr *tex)
1562 {
1563 struct ir3_block *b = ctx->block;
1564 struct ir3_instruction **dst, *sam;
1565
1566 dst = get_dst(ctx, &tex->dest, 1);
1567
1568 sam = ir3_SAM(b, OPC_GETINFO, TYPE_U32, TGSI_WRITEMASK_Z, 0,
1569 tex->texture_index, tex->texture_index, NULL, NULL);
1570
1571 /* even though there is only one component, since it ends
1572 * up in .z rather than .x, we need a split_dest()
1573 */
1574 split_dest(b, dst, sam, 3);
1575
1576 /* The # of levels comes from getinfo.z. We need to add 1 to it, since
1577 * the value in TEX_CONST_0 is zero-based.
1578 */
1579 if (ctx->levels_add_one)
1580 dst[0] = ir3_ADD_U(b, dst[0], 0, create_immed(b, 1), 0);
1581 }
1582
1583 static void
1584 emit_tex_txs(struct ir3_compile *ctx, nir_tex_instr *tex)
1585 {
1586 struct ir3_block *b = ctx->block;
1587 struct ir3_instruction **dst, *sam, *lod;
1588 unsigned flags, coords;
1589
1590 tex_info(tex, &flags, &coords);
1591
1592 /* Actually we want the number of dimensions, not coordinates. This
1593 * distinction only matters for cubes.
1594 */
1595 if (tex->sampler_dim == GLSL_SAMPLER_DIM_CUBE)
1596 coords = 2;
1597
1598 dst = get_dst(ctx, &tex->dest, 4);
1599
1600 compile_assert(ctx, tex->num_srcs == 1);
1601 compile_assert(ctx, tex->src[0].src_type == nir_tex_src_lod);
1602
1603 lod = get_src(ctx, &tex->src[0].src)[0];
1604
1605 sam = ir3_SAM(b, OPC_GETSIZE, TYPE_U32, TGSI_WRITEMASK_XYZW, flags,
1606 tex->texture_index, tex->texture_index, lod, NULL);
1607
1608 split_dest(b, dst, sam, 4);
1609
1610 /* Array size actually ends up in .w rather than .z. This doesn't
1611 * matter for miplevel 0, but for higher mips the value in z is
1612 * minified whereas w stays. Also, the value in TEX_CONST_3_DEPTH is
1613 * returned, which means that we have to add 1 to it for arrays.
1614 */
1615 if (tex->is_array) {
1616 if (ctx->levels_add_one) {
1617 dst[coords] = ir3_ADD_U(b, dst[3], 0, create_immed(b, 1), 0);
1618 } else {
1619 dst[coords] = ir3_MOV(b, dst[3], TYPE_U32);
1620 }
1621 }
1622 }
1623
1624 static void
1625 emit_phi(struct ir3_compile *ctx, nir_phi_instr *nphi)
1626 {
1627 struct ir3_instruction *phi, **dst;
1628
1629 /* NOTE: phi's should be lowered to scalar at this point */
1630 compile_assert(ctx, nphi->dest.ssa.num_components == 1);
1631
1632 dst = get_dst(ctx, &nphi->dest, 1);
1633
1634 phi = ir3_instr_create2(ctx->block, -1, OPC_META_PHI,
1635 1 + exec_list_length(&nphi->srcs));
1636 ir3_reg_create(phi, 0, 0); /* dst */
1637 phi->phi.nphi = nphi;
1638
1639 dst[0] = phi;
1640 }
1641
1642 /* phi instructions are left partially constructed. We don't resolve
1643 * their srcs until the end of the block, since (eg. loops) one of
1644 * the phi's srcs might be defined after the phi due to back edges in
1645 * the CFG.
1646 */
1647 static void
1648 resolve_phis(struct ir3_compile *ctx, struct ir3_block *block)
1649 {
1650 list_for_each_entry (struct ir3_instruction, instr, &block->instr_list, node) {
1651 nir_phi_instr *nphi;
1652
1653 /* phi's only come at start of block: */
1654 if (instr->opc != OPC_META_PHI)
1655 break;
1656
1657 if (!instr->phi.nphi)
1658 break;
1659
1660 nphi = instr->phi.nphi;
1661 instr->phi.nphi = NULL;
1662
1663 foreach_list_typed(nir_phi_src, nsrc, node, &nphi->srcs) {
1664 struct ir3_instruction *src = get_src(ctx, &nsrc->src)[0];
1665 ir3_reg_create(instr, 0, IR3_REG_SSA)->instr = src;
1666 }
1667 }
1668 }
1669
1670 static void
1671 emit_jump(struct ir3_compile *ctx, nir_jump_instr *jump)
1672 {
1673 switch (jump->type) {
1674 case nir_jump_break:
1675 case nir_jump_continue:
1676 /* I *think* we can simply just ignore this, and use the
1677 * successor block link to figure out where we need to
1678 * jump to for break/continue
1679 */
1680 break;
1681 default:
1682 compile_error(ctx, "Unhandled NIR jump type: %d\n", jump->type);
1683 break;
1684 }
1685 }
1686
1687 static void
1688 emit_instr(struct ir3_compile *ctx, nir_instr *instr)
1689 {
1690 switch (instr->type) {
1691 case nir_instr_type_alu:
1692 emit_alu(ctx, nir_instr_as_alu(instr));
1693 break;
1694 case nir_instr_type_intrinsic:
1695 emit_intrinsic(ctx, nir_instr_as_intrinsic(instr));
1696 break;
1697 case nir_instr_type_load_const:
1698 emit_load_const(ctx, nir_instr_as_load_const(instr));
1699 break;
1700 case nir_instr_type_ssa_undef:
1701 emit_undef(ctx, nir_instr_as_ssa_undef(instr));
1702 break;
1703 case nir_instr_type_tex: {
1704 nir_tex_instr *tex = nir_instr_as_tex(instr);
1705 /* couple tex instructions get special-cased:
1706 */
1707 switch (tex->op) {
1708 case nir_texop_txs:
1709 emit_tex_txs(ctx, tex);
1710 break;
1711 case nir_texop_query_levels:
1712 emit_tex_query_levels(ctx, tex);
1713 break;
1714 default:
1715 emit_tex(ctx, tex);
1716 break;
1717 }
1718 break;
1719 }
1720 case nir_instr_type_phi:
1721 emit_phi(ctx, nir_instr_as_phi(instr));
1722 break;
1723 case nir_instr_type_jump:
1724 emit_jump(ctx, nir_instr_as_jump(instr));
1725 break;
1726 case nir_instr_type_call:
1727 case nir_instr_type_parallel_copy:
1728 compile_error(ctx, "Unhandled NIR instruction type: %d\n", instr->type);
1729 break;
1730 }
1731 }
1732
1733 static struct ir3_block *
1734 get_block(struct ir3_compile *ctx, nir_block *nblock)
1735 {
1736 struct ir3_block *block;
1737 struct hash_entry *entry;
1738 entry = _mesa_hash_table_search(ctx->block_ht, nblock);
1739 if (entry)
1740 return entry->data;
1741
1742 block = ir3_block_create(ctx->ir);
1743 block->nblock = nblock;
1744 _mesa_hash_table_insert(ctx->block_ht, nblock, block);
1745
1746 return block;
1747 }
1748
1749 static void
1750 emit_block(struct ir3_compile *ctx, nir_block *nblock)
1751 {
1752 struct ir3_block *block = get_block(ctx, nblock);
1753
1754 for (int i = 0; i < ARRAY_SIZE(block->successors); i++) {
1755 if (nblock->successors[i]) {
1756 block->successors[i] =
1757 get_block(ctx, nblock->successors[i]);
1758 }
1759 }
1760
1761 ctx->block = block;
1762 list_addtail(&block->node, &ctx->ir->block_list);
1763
1764 /* re-emit addr register in each block if needed: */
1765 _mesa_hash_table_destroy(ctx->addr_ht, NULL);
1766 ctx->addr_ht = NULL;
1767
1768 nir_foreach_instr(nblock, instr) {
1769 emit_instr(ctx, instr);
1770 if (ctx->error)
1771 return;
1772 }
1773 }
1774
1775 static void emit_cf_list(struct ir3_compile *ctx, struct exec_list *list);
1776
1777 static void
1778 emit_if(struct ir3_compile *ctx, nir_if *nif)
1779 {
1780 struct ir3_instruction *condition = get_src(ctx, &nif->condition)[0];
1781
1782 ctx->block->condition =
1783 get_predicate(ctx, ir3_b2n(condition->block, condition));
1784
1785 emit_cf_list(ctx, &nif->then_list);
1786 emit_cf_list(ctx, &nif->else_list);
1787 }
1788
1789 static void
1790 emit_loop(struct ir3_compile *ctx, nir_loop *nloop)
1791 {
1792 emit_cf_list(ctx, &nloop->body);
1793 }
1794
1795 static void
1796 emit_cf_list(struct ir3_compile *ctx, struct exec_list *list)
1797 {
1798 foreach_list_typed(nir_cf_node, node, node, list) {
1799 switch (node->type) {
1800 case nir_cf_node_block:
1801 emit_block(ctx, nir_cf_node_as_block(node));
1802 break;
1803 case nir_cf_node_if:
1804 emit_if(ctx, nir_cf_node_as_if(node));
1805 break;
1806 case nir_cf_node_loop:
1807 emit_loop(ctx, nir_cf_node_as_loop(node));
1808 break;
1809 case nir_cf_node_function:
1810 compile_error(ctx, "TODO\n");
1811 break;
1812 }
1813 }
1814 }
1815
1816 /* emit stream-out code. At this point, the current block is the original
1817 * (nir) end block, and nir ensures that all flow control paths terminate
1818 * into the end block. We re-purpose the original end block to generate
1819 * the 'if (vtxcnt < maxvtxcnt)' condition, then append the conditional
1820 * block holding stream-out write instructions, followed by the new end
1821 * block:
1822 *
1823 * blockOrigEnd {
1824 * p0.x = (vtxcnt < maxvtxcnt)
1825 * // succs: blockStreamOut, blockNewEnd
1826 * }
1827 * blockStreamOut {
1828 * ... stream-out instructions ...
1829 * // succs: blockNewEnd
1830 * }
1831 * blockNewEnd {
1832 * }
1833 */
1834 static void
1835 emit_stream_out(struct ir3_compile *ctx)
1836 {
1837 struct ir3_shader_variant *v = ctx->so;
1838 struct ir3 *ir = ctx->ir;
1839 struct pipe_stream_output_info *strmout =
1840 &ctx->so->shader->stream_output;
1841 struct ir3_block *orig_end_block, *stream_out_block, *new_end_block;
1842 struct ir3_instruction *vtxcnt, *maxvtxcnt, *cond;
1843 struct ir3_instruction *bases[PIPE_MAX_SO_BUFFERS];
1844
1845 /* create vtxcnt input in input block at top of shader,
1846 * so that it is seen as live over the entire duration
1847 * of the shader:
1848 */
1849 vtxcnt = create_input(ctx->in_block, 0);
1850 add_sysval_input(ctx, SYSTEM_VALUE_VERTEX_CNT, vtxcnt);
1851
1852 maxvtxcnt = create_driver_param(ctx, IR3_DP_VTXCNT_MAX);
1853
1854 /* at this point, we are at the original 'end' block,
1855 * re-purpose this block to stream-out condition, then
1856 * append stream-out block and new-end block
1857 */
1858 orig_end_block = ctx->block;
1859
1860 stream_out_block = ir3_block_create(ir);
1861 list_addtail(&stream_out_block->node, &ir->block_list);
1862
1863 new_end_block = ir3_block_create(ir);
1864 list_addtail(&new_end_block->node, &ir->block_list);
1865
1866 orig_end_block->successors[0] = stream_out_block;
1867 orig_end_block->successors[1] = new_end_block;
1868 stream_out_block->successors[0] = new_end_block;
1869
1870 /* setup 'if (vtxcnt < maxvtxcnt)' condition: */
1871 cond = ir3_CMPS_S(ctx->block, vtxcnt, 0, maxvtxcnt, 0);
1872 cond->regs[0]->num = regid(REG_P0, 0);
1873 cond->cat2.condition = IR3_COND_LT;
1874
1875 /* condition goes on previous block to the conditional,
1876 * since it is used to pick which of the two successor
1877 * paths to take:
1878 */
1879 orig_end_block->condition = cond;
1880
1881 /* switch to stream_out_block to generate the stream-out
1882 * instructions:
1883 */
1884 ctx->block = stream_out_block;
1885
1886 /* Calculate base addresses based on vtxcnt. Instructions
1887 * generated for bases not used in following loop will be
1888 * stripped out in the backend.
1889 */
1890 for (unsigned i = 0; i < PIPE_MAX_SO_BUFFERS; i++) {
1891 unsigned stride = strmout->stride[i];
1892 struct ir3_instruction *base, *off;
1893
1894 base = create_uniform(ctx, regid(v->first_driver_param + IR3_TFBOS_OFF, i));
1895
1896 /* 24-bit should be enough: */
1897 off = ir3_MUL_U(ctx->block, vtxcnt, 0,
1898 create_immed(ctx->block, stride * 4), 0);
1899
1900 bases[i] = ir3_ADD_S(ctx->block, off, 0, base, 0);
1901 }
1902
1903 /* Generate the per-output store instructions: */
1904 for (unsigned i = 0; i < strmout->num_outputs; i++) {
1905 for (unsigned j = 0; j < strmout->output[i].num_components; j++) {
1906 unsigned c = j + strmout->output[i].start_component;
1907 struct ir3_instruction *base, *out, *stg;
1908
1909 base = bases[strmout->output[i].output_buffer];
1910 out = ctx->ir->outputs[regid(strmout->output[i].register_index, c)];
1911
1912 stg = ir3_STG(ctx->block, base, 0, out, 0,
1913 create_immed(ctx->block, 1), 0);
1914 stg->cat6.type = TYPE_U32;
1915 stg->cat6.dst_offset = (strmout->output[i].dst_offset + j) * 4;
1916
1917 array_insert(ctx->ir->keeps, stg);
1918 }
1919 }
1920
1921 /* and finally switch to the new_end_block: */
1922 ctx->block = new_end_block;
1923 }
1924
1925 static void
1926 emit_function(struct ir3_compile *ctx, nir_function_impl *impl)
1927 {
1928 nir_metadata_require(impl, nir_metadata_block_index);
1929
1930 emit_cf_list(ctx, &impl->body);
1931 emit_block(ctx, impl->end_block);
1932
1933 /* at this point, we should have a single empty block,
1934 * into which we emit the 'end' instruction.
1935 */
1936 compile_assert(ctx, list_empty(&ctx->block->instr_list));
1937
1938 /* If stream-out (aka transform-feedback) enabled, emit the
1939 * stream-out instructions, followed by a new empty block (into
1940 * which the 'end' instruction lands).
1941 *
1942 * NOTE: it is done in this order, rather than inserting before
1943 * we emit end_block, because NIR guarantees that all blocks
1944 * flow into end_block, and that end_block has no successors.
1945 * So by re-purposing end_block as the first block of stream-
1946 * out, we guarantee that all exit paths flow into the stream-
1947 * out instructions.
1948 */
1949 if ((ctx->so->shader->stream_output.num_outputs > 0) &&
1950 !ctx->so->key.binning_pass) {
1951 debug_assert(ctx->so->type == SHADER_VERTEX);
1952 emit_stream_out(ctx);
1953 }
1954
1955 ir3_END(ctx->block);
1956 }
1957
1958 static void
1959 setup_input(struct ir3_compile *ctx, nir_variable *in)
1960 {
1961 struct ir3_shader_variant *so = ctx->so;
1962 unsigned array_len = MAX2(glsl_get_length(in->type), 1);
1963 unsigned ncomp = glsl_get_components(in->type);
1964 unsigned n = in->data.driver_location;
1965 unsigned slot = in->data.location;
1966
1967 DBG("; in: slot=%u, len=%ux%u, drvloc=%u",
1968 slot, array_len, ncomp, n);
1969
1970 so->inputs[n].slot = slot;
1971 so->inputs[n].compmask = (1 << ncomp) - 1;
1972 so->inputs_count = MAX2(so->inputs_count, n + 1);
1973 so->inputs[n].interpolate = in->data.interpolation;
1974
1975 if (ctx->so->type == SHADER_FRAGMENT) {
1976 for (int i = 0; i < ncomp; i++) {
1977 struct ir3_instruction *instr = NULL;
1978 unsigned idx = (n * 4) + i;
1979
1980 if (slot == VARYING_SLOT_POS) {
1981 so->inputs[n].bary = false;
1982 so->frag_coord = true;
1983 instr = create_frag_coord(ctx, i);
1984 } else if (slot == VARYING_SLOT_FACE) {
1985 so->inputs[n].bary = false;
1986 so->frag_face = true;
1987 instr = create_frag_face(ctx, i);
1988 } else {
1989 bool use_ldlv = false;
1990
1991 /* detect the special case for front/back colors where
1992 * we need to do flat vs smooth shading depending on
1993 * rast state:
1994 */
1995 if (in->data.interpolation == INTERP_QUALIFIER_NONE) {
1996 switch (slot) {
1997 case VARYING_SLOT_COL0:
1998 case VARYING_SLOT_COL1:
1999 case VARYING_SLOT_BFC0:
2000 case VARYING_SLOT_BFC1:
2001 so->inputs[n].rasterflat = true;
2002 break;
2003 default:
2004 break;
2005 }
2006 }
2007
2008 if (ctx->flat_bypass) {
2009 if ((so->inputs[n].interpolate == INTERP_QUALIFIER_FLAT) ||
2010 (so->inputs[n].rasterflat && ctx->so->key.rasterflat))
2011 use_ldlv = true;
2012 }
2013
2014 so->inputs[n].bary = true;
2015
2016 instr = create_frag_input(ctx, use_ldlv);
2017 }
2018
2019 ctx->ir->inputs[idx] = instr;
2020 }
2021 } else if (ctx->so->type == SHADER_VERTEX) {
2022 for (int i = 0; i < ncomp; i++) {
2023 unsigned idx = (n * 4) + i;
2024 ctx->ir->inputs[idx] = create_input(ctx->block, idx);
2025 }
2026 } else {
2027 compile_error(ctx, "unknown shader type: %d\n", ctx->so->type);
2028 }
2029
2030 if (so->inputs[n].bary || (ctx->so->type == SHADER_VERTEX)) {
2031 so->total_in += ncomp;
2032 }
2033 }
2034
2035 static void
2036 setup_output(struct ir3_compile *ctx, nir_variable *out)
2037 {
2038 struct ir3_shader_variant *so = ctx->so;
2039 unsigned array_len = MAX2(glsl_get_length(out->type), 1);
2040 unsigned ncomp = glsl_get_components(out->type);
2041 unsigned n = out->data.driver_location;
2042 unsigned slot = out->data.location;
2043 unsigned comp = 0;
2044
2045 DBG("; out: slot=%u, len=%ux%u, drvloc=%u",
2046 slot, array_len, ncomp, n);
2047
2048 if (ctx->so->type == SHADER_FRAGMENT) {
2049 switch (slot) {
2050 case FRAG_RESULT_DEPTH:
2051 comp = 2; /* tgsi will write to .z component */
2052 so->writes_pos = true;
2053 break;
2054 case FRAG_RESULT_COLOR:
2055 so->color0_mrt = 1;
2056 break;
2057 default:
2058 if (slot >= FRAG_RESULT_DATA0)
2059 break;
2060 compile_error(ctx, "unknown FS output name: %s\n",
2061 gl_frag_result_name(slot));
2062 }
2063 } else if (ctx->so->type == SHADER_VERTEX) {
2064 switch (slot) {
2065 case VARYING_SLOT_POS:
2066 so->writes_pos = true;
2067 break;
2068 case VARYING_SLOT_PSIZ:
2069 so->writes_psize = true;
2070 break;
2071 case VARYING_SLOT_COL0:
2072 case VARYING_SLOT_COL1:
2073 case VARYING_SLOT_BFC0:
2074 case VARYING_SLOT_BFC1:
2075 case VARYING_SLOT_FOGC:
2076 case VARYING_SLOT_CLIP_DIST0:
2077 case VARYING_SLOT_CLIP_DIST1:
2078 break;
2079 case VARYING_SLOT_CLIP_VERTEX:
2080 /* handled entirely in nir_lower_clip: */
2081 return;
2082 default:
2083 if (slot >= VARYING_SLOT_VAR0)
2084 break;
2085 if ((VARYING_SLOT_TEX0 <= slot) && (slot <= VARYING_SLOT_TEX7))
2086 break;
2087 compile_error(ctx, "unknown VS output name: %s\n",
2088 gl_varying_slot_name(slot));
2089 }
2090 } else {
2091 compile_error(ctx, "unknown shader type: %d\n", ctx->so->type);
2092 }
2093
2094 compile_assert(ctx, n < ARRAY_SIZE(so->outputs));
2095
2096 so->outputs[n].slot = slot;
2097 so->outputs[n].regid = regid(n, comp);
2098 so->outputs_count = MAX2(so->outputs_count, n + 1);
2099
2100 for (int i = 0; i < ncomp; i++) {
2101 unsigned idx = (n * 4) + i;
2102
2103 ctx->ir->outputs[idx] = create_immed(ctx->block, fui(0.0));
2104 }
2105 }
2106
2107 static void
2108 emit_instructions(struct ir3_compile *ctx)
2109 {
2110 unsigned ninputs, noutputs;
2111 nir_function_impl *fxn = NULL;
2112
2113 /* Find the main function: */
2114 nir_foreach_function(ctx->s, function) {
2115 compile_assert(ctx, strcmp(function->name, "main") == 0);
2116 compile_assert(ctx, function->impl);
2117 fxn = function->impl;
2118 break;
2119 }
2120
2121 ninputs = exec_list_length(&ctx->s->inputs) * 4;
2122 noutputs = exec_list_length(&ctx->s->outputs) * 4;
2123
2124 /* or vtx shaders, we need to leave room for sysvals:
2125 */
2126 if (ctx->so->type == SHADER_VERTEX) {
2127 ninputs += 8;
2128 }
2129
2130 ctx->ir = ir3_create(ctx->compiler, ninputs, noutputs);
2131
2132 /* Create inputs in first block: */
2133 ctx->block = get_block(ctx, nir_start_block(fxn));
2134 ctx->in_block = ctx->block;
2135 list_addtail(&ctx->block->node, &ctx->ir->block_list);
2136
2137 if (ctx->so->type == SHADER_VERTEX) {
2138 ctx->ir->ninputs -= 8;
2139 }
2140
2141 /* for fragment shader, we have a single input register (usually
2142 * r0.xy) which is used as the base for bary.f varying fetch instrs:
2143 */
2144 if (ctx->so->type == SHADER_FRAGMENT) {
2145 // TODO maybe a helper for fi since we need it a few places..
2146 struct ir3_instruction *instr;
2147 instr = ir3_instr_create(ctx->block, -1, OPC_META_FI);
2148 ir3_reg_create(instr, 0, 0);
2149 ir3_reg_create(instr, 0, IR3_REG_SSA); /* r0.x */
2150 ir3_reg_create(instr, 0, IR3_REG_SSA); /* r0.y */
2151 ctx->frag_pos = instr;
2152 }
2153
2154 /* Setup inputs: */
2155 nir_foreach_variable(var, &ctx->s->inputs) {
2156 setup_input(ctx, var);
2157 }
2158
2159 /* Setup outputs: */
2160 nir_foreach_variable(var, &ctx->s->outputs) {
2161 setup_output(ctx, var);
2162 }
2163
2164 /* Setup global variables (which should only be arrays): */
2165 nir_foreach_variable(var, &ctx->s->globals) {
2166 declare_var(ctx, var);
2167 }
2168
2169 /* Setup local variables (which should only be arrays): */
2170 /* NOTE: need to do something more clever when we support >1 fxn */
2171 nir_foreach_variable(var, &fxn->locals) {
2172 declare_var(ctx, var);
2173 }
2174
2175 /* And emit the body: */
2176 ctx->impl = fxn;
2177 emit_function(ctx, fxn);
2178
2179 list_for_each_entry (struct ir3_block, block, &ctx->ir->block_list, node) {
2180 resolve_phis(ctx, block);
2181 }
2182 }
2183
2184 /* from NIR perspective, we actually have inputs. But most of the "inputs"
2185 * for a fragment shader are just bary.f instructions. The *actual* inputs
2186 * from the hw perspective are the frag_pos and optionally frag_coord and
2187 * frag_face.
2188 */
2189 static void
2190 fixup_frag_inputs(struct ir3_compile *ctx)
2191 {
2192 struct ir3_shader_variant *so = ctx->so;
2193 struct ir3 *ir = ctx->ir;
2194 struct ir3_instruction **inputs;
2195 struct ir3_instruction *instr;
2196 int n, regid = 0;
2197
2198 ir->ninputs = 0;
2199
2200 n = 4; /* always have frag_pos */
2201 n += COND(so->frag_face, 4);
2202 n += COND(so->frag_coord, 4);
2203
2204 inputs = ir3_alloc(ctx->ir, n * (sizeof(struct ir3_instruction *)));
2205
2206 if (so->frag_face) {
2207 /* this ultimately gets assigned to hr0.x so doesn't conflict
2208 * with frag_coord/frag_pos..
2209 */
2210 inputs[ir->ninputs++] = ctx->frag_face;
2211 ctx->frag_face->regs[0]->num = 0;
2212
2213 /* remaining channels not used, but let's avoid confusing
2214 * other parts that expect inputs to come in groups of vec4
2215 */
2216 inputs[ir->ninputs++] = NULL;
2217 inputs[ir->ninputs++] = NULL;
2218 inputs[ir->ninputs++] = NULL;
2219 }
2220
2221 /* since we don't know where to set the regid for frag_coord,
2222 * we have to use r0.x for it. But we don't want to *always*
2223 * use r1.x for frag_pos as that could increase the register
2224 * footprint on simple shaders:
2225 */
2226 if (so->frag_coord) {
2227 ctx->frag_coord[0]->regs[0]->num = regid++;
2228 ctx->frag_coord[1]->regs[0]->num = regid++;
2229 ctx->frag_coord[2]->regs[0]->num = regid++;
2230 ctx->frag_coord[3]->regs[0]->num = regid++;
2231
2232 inputs[ir->ninputs++] = ctx->frag_coord[0];
2233 inputs[ir->ninputs++] = ctx->frag_coord[1];
2234 inputs[ir->ninputs++] = ctx->frag_coord[2];
2235 inputs[ir->ninputs++] = ctx->frag_coord[3];
2236 }
2237
2238 /* we always have frag_pos: */
2239 so->pos_regid = regid;
2240
2241 /* r0.x */
2242 instr = create_input(ctx->in_block, ir->ninputs);
2243 instr->regs[0]->num = regid++;
2244 inputs[ir->ninputs++] = instr;
2245 ctx->frag_pos->regs[1]->instr = instr;
2246
2247 /* r0.y */
2248 instr = create_input(ctx->in_block, ir->ninputs);
2249 instr->regs[0]->num = regid++;
2250 inputs[ir->ninputs++] = instr;
2251 ctx->frag_pos->regs[2]->instr = instr;
2252
2253 ir->inputs = inputs;
2254 }
2255
2256 int
2257 ir3_compile_shader_nir(struct ir3_compiler *compiler,
2258 struct ir3_shader_variant *so)
2259 {
2260 struct ir3_compile *ctx;
2261 struct ir3 *ir;
2262 struct ir3_instruction **inputs;
2263 unsigned i, j, actual_in, inloc;
2264 int ret = 0, max_bary;
2265
2266 assert(!so->ir);
2267
2268 ctx = compile_init(compiler, so);
2269 if (!ctx) {
2270 DBG("INIT failed!");
2271 ret = -1;
2272 goto out;
2273 }
2274
2275 emit_instructions(ctx);
2276
2277 if (ctx->error) {
2278 DBG("EMIT failed!");
2279 ret = -1;
2280 goto out;
2281 }
2282
2283 ir = so->ir = ctx->ir;
2284
2285 /* keep track of the inputs from TGSI perspective.. */
2286 inputs = ir->inputs;
2287
2288 /* but fixup actual inputs for frag shader: */
2289 if (so->type == SHADER_FRAGMENT)
2290 fixup_frag_inputs(ctx);
2291
2292 /* at this point, for binning pass, throw away unneeded outputs: */
2293 if (so->key.binning_pass) {
2294 for (i = 0, j = 0; i < so->outputs_count; i++) {
2295 unsigned slot = so->outputs[i].slot;
2296
2297 /* throw away everything but first position/psize */
2298 if ((slot == VARYING_SLOT_POS) || (slot == VARYING_SLOT_PSIZ)) {
2299 if (i != j) {
2300 so->outputs[j] = so->outputs[i];
2301 ir->outputs[(j*4)+0] = ir->outputs[(i*4)+0];
2302 ir->outputs[(j*4)+1] = ir->outputs[(i*4)+1];
2303 ir->outputs[(j*4)+2] = ir->outputs[(i*4)+2];
2304 ir->outputs[(j*4)+3] = ir->outputs[(i*4)+3];
2305 }
2306 j++;
2307 }
2308 }
2309 so->outputs_count = j;
2310 ir->noutputs = j * 4;
2311 }
2312
2313 /* if we want half-precision outputs, mark the output registers
2314 * as half:
2315 */
2316 if (so->key.half_precision) {
2317 for (i = 0; i < ir->noutputs; i++) {
2318 struct ir3_instruction *out = ir->outputs[i];
2319 if (!out)
2320 continue;
2321 out->regs[0]->flags |= IR3_REG_HALF;
2322 /* output could be a fanout (ie. texture fetch output)
2323 * in which case we need to propagate the half-reg flag
2324 * up to the definer so that RA sees it:
2325 */
2326 if (out->opc == OPC_META_FO) {
2327 out = out->regs[1]->instr;
2328 out->regs[0]->flags |= IR3_REG_HALF;
2329 }
2330
2331 if (out->opc == OPC_MOV) {
2332 out->cat1.dst_type = half_type(out->cat1.dst_type);
2333 }
2334 }
2335 }
2336
2337 if (fd_mesa_debug & FD_DBG_OPTMSGS) {
2338 printf("BEFORE CP:\n");
2339 ir3_print(ir);
2340 }
2341
2342 ir3_cp(ir);
2343
2344 if (fd_mesa_debug & FD_DBG_OPTMSGS) {
2345 printf("BEFORE GROUPING:\n");
2346 ir3_print(ir);
2347 }
2348
2349 /* Group left/right neighbors, inserting mov's where needed to
2350 * solve conflicts:
2351 */
2352 ir3_group(ir);
2353
2354 ir3_depth(ir);
2355
2356 if (fd_mesa_debug & FD_DBG_OPTMSGS) {
2357 printf("AFTER DEPTH:\n");
2358 ir3_print(ir);
2359 }
2360
2361 ret = ir3_sched(ir);
2362 if (ret) {
2363 DBG("SCHED failed!");
2364 goto out;
2365 }
2366
2367 if (fd_mesa_debug & FD_DBG_OPTMSGS) {
2368 printf("AFTER SCHED:\n");
2369 ir3_print(ir);
2370 }
2371
2372 ret = ir3_ra(ir, so->type, so->frag_coord, so->frag_face);
2373 if (ret) {
2374 DBG("RA failed!");
2375 goto out;
2376 }
2377
2378 if (fd_mesa_debug & FD_DBG_OPTMSGS) {
2379 printf("AFTER RA:\n");
2380 ir3_print(ir);
2381 }
2382
2383 /* fixup input/outputs: */
2384 for (i = 0; i < so->outputs_count; i++) {
2385 so->outputs[i].regid = ir->outputs[i*4]->regs[0]->num;
2386 /* preserve hack for depth output.. tgsi writes depth to .z,
2387 * but what we give the hw is the scalar register:
2388 */
2389 if ((so->type == SHADER_FRAGMENT) &&
2390 (so->outputs[i].slot == FRAG_RESULT_DEPTH))
2391 so->outputs[i].regid += 2;
2392 }
2393
2394 /* Note that some or all channels of an input may be unused: */
2395 actual_in = 0;
2396 inloc = 0;
2397 for (i = 0; i < so->inputs_count; i++) {
2398 unsigned j, regid = ~0, compmask = 0;
2399 so->inputs[i].ncomp = 0;
2400 so->inputs[i].inloc = inloc + 8;
2401 for (j = 0; j < 4; j++) {
2402 struct ir3_instruction *in = inputs[(i*4) + j];
2403 if (in && !(in->flags & IR3_INSTR_UNUSED)) {
2404 compmask |= (1 << j);
2405 regid = in->regs[0]->num - j;
2406 actual_in++;
2407 so->inputs[i].ncomp++;
2408 if ((so->type == SHADER_FRAGMENT) && so->inputs[i].bary) {
2409 /* assign inloc: */
2410 assert(in->regs[1]->flags & IR3_REG_IMMED);
2411 in->regs[1]->iim_val = inloc++;
2412 }
2413 }
2414 }
2415 if ((so->type == SHADER_FRAGMENT) && compmask && so->inputs[i].bary)
2416 so->varying_in++;
2417 so->inputs[i].regid = regid;
2418 so->inputs[i].compmask = compmask;
2419 }
2420
2421 /* We need to do legalize after (for frag shader's) the "bary.f"
2422 * offsets (inloc) have been assigned.
2423 */
2424 ir3_legalize(ir, &so->has_samp, &max_bary);
2425
2426 if (fd_mesa_debug & FD_DBG_OPTMSGS) {
2427 printf("AFTER LEGALIZE:\n");
2428 ir3_print(ir);
2429 }
2430
2431 /* Note that actual_in counts inputs that are not bary.f'd for FS: */
2432 if (so->type == SHADER_VERTEX)
2433 so->total_in = actual_in;
2434 else
2435 so->total_in = max_bary + 1;
2436
2437 out:
2438 if (ret) {
2439 if (so->ir)
2440 ir3_destroy(so->ir);
2441 so->ir = NULL;
2442 }
2443 compile_free(ctx);
2444
2445 return ret;
2446 }