lima/ppir: clone ld_{uni,tex,var} into each block
[mesa.git] / src / gallium / drivers / lima / ir / pp / nir.c
1 /*
2 * Copyright (c) 2017 Lima Project
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sub license,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the
12 * next paragraph) shall be included in all copies or substantial portions
13 * of the Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
21 * DEALINGS IN THE SOFTWARE.
22 *
23 */
24
25 #include <string.h>
26
27 #include "util/ralloc.h"
28 #include "util/bitscan.h"
29 #include "compiler/nir/nir.h"
30 #include "pipe/p_state.h"
31
32
33 #include "ppir.h"
34
35 static void *ppir_node_create_ssa(ppir_block *block, ppir_op op, nir_ssa_def *ssa)
36 {
37 ppir_node *node = ppir_node_create(block, op, ssa->index, 0);
38 if (!node)
39 return NULL;
40
41 ppir_dest *dest = ppir_node_get_dest(node);
42 dest->type = ppir_target_ssa;
43 dest->ssa.num_components = ssa->num_components;
44 dest->ssa.live_in = INT_MAX;
45 dest->ssa.live_out = 0;
46 dest->write_mask = u_bit_consecutive(0, ssa->num_components);
47
48 if (node->type == ppir_node_type_load ||
49 node->type == ppir_node_type_store)
50 dest->ssa.is_head = true;
51
52 return node;
53 }
54
55 static void *ppir_node_create_reg(ppir_block *block, ppir_op op,
56 nir_reg_dest *reg, unsigned mask)
57 {
58 ppir_node *node = ppir_node_create(block, op, reg->reg->index, mask);
59 if (!node)
60 return NULL;
61
62 ppir_dest *dest = ppir_node_get_dest(node);
63
64 list_for_each_entry(ppir_reg, r, &block->comp->reg_list, list) {
65 if (r->index == reg->reg->index) {
66 dest->reg = r;
67 break;
68 }
69 }
70
71 dest->type = ppir_target_register;
72 dest->write_mask = mask;
73
74 if (node->type == ppir_node_type_load ||
75 node->type == ppir_node_type_store)
76 dest->reg->is_head = true;
77
78 return node;
79 }
80
81 static void *ppir_node_create_dest(ppir_block *block, ppir_op op,
82 nir_dest *dest, unsigned mask)
83 {
84 unsigned index = -1;
85
86 if (dest) {
87 if (dest->is_ssa)
88 return ppir_node_create_ssa(block, op, &dest->ssa);
89 else
90 return ppir_node_create_reg(block, op, &dest->reg, mask);
91 }
92
93 return ppir_node_create(block, op, index, 0);
94 }
95
96 static void ppir_node_add_src(ppir_compiler *comp, ppir_node *node,
97 ppir_src *ps, nir_src *ns, unsigned mask)
98 {
99 ppir_node *child = NULL;
100
101 if (ns->is_ssa) {
102 child = comp->var_nodes[ns->ssa->index];
103 /* Clone consts for each successor */
104 switch (child->op) {
105 case ppir_op_const:
106 child = ppir_node_clone(node->block, child);
107 break;
108 /* Clone uniforms and load textures for each block */
109 case ppir_op_load_texture:
110 case ppir_op_load_uniform:
111 case ppir_op_load_varying:
112 if (child->block != node->block) {
113 child = ppir_node_clone(node->block, child);
114 comp->var_nodes[ns->ssa->index] = child;
115 }
116 break;
117 default:
118 break;
119 }
120
121 ppir_node_add_dep(node, child);
122 }
123 else {
124 nir_register *reg = ns->reg.reg;
125 while (mask) {
126 int swizzle = ps->swizzle[u_bit_scan(&mask)];
127 child = comp->var_nodes[(reg->index << 2) + comp->reg_base + swizzle];
128 ppir_node_add_dep(node, child);
129 }
130 }
131
132 ppir_node_target_assign(ps, child);
133 }
134
135 static int nir_to_ppir_opcodes[nir_num_opcodes] = {
136 /* not supported */
137 [0 ... nir_last_opcode] = -1,
138
139 [nir_op_mov] = ppir_op_mov,
140 [nir_op_fmul] = ppir_op_mul,
141 [nir_op_fabs] = ppir_op_abs,
142 [nir_op_fneg] = ppir_op_neg,
143 [nir_op_fadd] = ppir_op_add,
144 [nir_op_fsum3] = ppir_op_sum3,
145 [nir_op_fsum4] = ppir_op_sum4,
146 [nir_op_frsq] = ppir_op_rsqrt,
147 [nir_op_flog2] = ppir_op_log2,
148 [nir_op_fexp2] = ppir_op_exp2,
149 [nir_op_fsqrt] = ppir_op_sqrt,
150 [nir_op_fsin] = ppir_op_sin,
151 [nir_op_fcos] = ppir_op_cos,
152 [nir_op_fmax] = ppir_op_max,
153 [nir_op_fmin] = ppir_op_min,
154 [nir_op_frcp] = ppir_op_rcp,
155 [nir_op_ffloor] = ppir_op_floor,
156 [nir_op_fceil] = ppir_op_ceil,
157 [nir_op_ffract] = ppir_op_fract,
158 [nir_op_sge] = ppir_op_ge,
159 [nir_op_fge] = ppir_op_ge,
160 [nir_op_slt] = ppir_op_lt,
161 [nir_op_flt] = ppir_op_lt,
162 [nir_op_seq] = ppir_op_eq,
163 [nir_op_feq] = ppir_op_eq,
164 [nir_op_sne] = ppir_op_ne,
165 [nir_op_fne] = ppir_op_ne,
166 [nir_op_fcsel] = ppir_op_select,
167 [nir_op_inot] = ppir_op_not,
168 [nir_op_ftrunc] = ppir_op_trunc,
169 [nir_op_fsat] = ppir_op_sat,
170 [nir_op_fddx] = ppir_op_ddx,
171 [nir_op_fddy] = ppir_op_ddy,
172 };
173
174 static ppir_node *ppir_emit_alu(ppir_block *block, nir_instr *ni)
175 {
176 nir_alu_instr *instr = nir_instr_as_alu(ni);
177 int op = nir_to_ppir_opcodes[instr->op];
178
179 if (op < 0) {
180 ppir_error("unsupported nir_op: %s\n", nir_op_infos[instr->op].name);
181 return NULL;
182 }
183
184 ppir_alu_node *node = ppir_node_create_dest(block, op, &instr->dest.dest,
185 instr->dest.write_mask);
186 if (!node)
187 return NULL;
188
189 ppir_dest *pd = &node->dest;
190 nir_alu_dest *nd = &instr->dest;
191 if (nd->saturate)
192 pd->modifier = ppir_outmod_clamp_fraction;
193
194 unsigned src_mask;
195 switch (op) {
196 case ppir_op_sum3:
197 src_mask = 0b0111;
198 break;
199 case ppir_op_sum4:
200 src_mask = 0b1111;
201 break;
202 default:
203 src_mask = pd->write_mask;
204 break;
205 }
206
207 unsigned num_child = nir_op_infos[instr->op].num_inputs;
208 node->num_src = num_child;
209
210 for (int i = 0; i < num_child; i++) {
211 nir_alu_src *ns = instr->src + i;
212 ppir_src *ps = node->src + i;
213 memcpy(ps->swizzle, ns->swizzle, sizeof(ps->swizzle));
214 ppir_node_add_src(block->comp, &node->node, ps, &ns->src, src_mask);
215
216 ps->absolute = ns->abs;
217 ps->negate = ns->negate;
218 }
219
220 return &node->node;
221 }
222
223 static ppir_block *ppir_block_create(ppir_compiler *comp);
224
225 static bool ppir_emit_discard_block(ppir_compiler *comp)
226 {
227 ppir_block *block = ppir_block_create(comp);
228 ppir_discard_node *discard;
229 if (!block)
230 return false;
231
232 comp->discard_block = block;
233 block->comp = comp;
234
235 discard = ppir_node_create(block, ppir_op_discard, -1, 0);
236 if (discard)
237 list_addtail(&discard->node.list, &block->node_list);
238 else
239 return false;
240
241 return true;
242 }
243
244 static ppir_node *ppir_emit_discard_if(ppir_block *block, nir_instr *ni)
245 {
246 nir_intrinsic_instr *instr = nir_instr_as_intrinsic(ni);
247 ppir_node *node;
248 ppir_compiler *comp = block->comp;
249 ppir_branch_node *branch;
250
251 if (!comp->discard_block && !ppir_emit_discard_block(comp))
252 return NULL;
253
254 node = ppir_node_create(block, ppir_op_branch, -1, 0);
255 if (!node)
256 return NULL;
257 branch = ppir_node_to_branch(node);
258
259 /* second src and condition will be updated during lowering */
260 ppir_node_add_src(block->comp, node, &branch->src[0],
261 &instr->src[0], u_bit_consecutive(0, instr->num_components));
262 branch->target = comp->discard_block;
263
264 return node;
265 }
266
267 static ppir_node *ppir_emit_discard(ppir_block *block, nir_instr *ni)
268 {
269 ppir_node *node = ppir_node_create(block, ppir_op_discard, -1, 0);
270
271 return node;
272 }
273
274 static ppir_node *ppir_emit_intrinsic(ppir_block *block, nir_instr *ni)
275 {
276 nir_intrinsic_instr *instr = nir_instr_as_intrinsic(ni);
277 unsigned mask = 0;
278 ppir_load_node *lnode;
279 ppir_store_node *snode;
280
281 switch (instr->intrinsic) {
282 case nir_intrinsic_load_input:
283 if (!instr->dest.is_ssa)
284 mask = u_bit_consecutive(0, instr->num_components);
285
286 lnode = ppir_node_create_dest(block, ppir_op_load_varying, &instr->dest, mask);
287 if (!lnode)
288 return NULL;
289
290 lnode->num_components = instr->num_components;
291 lnode->index = nir_intrinsic_base(instr) * 4 + nir_intrinsic_component(instr);
292 return &lnode->node;
293
294 case nir_intrinsic_load_frag_coord:
295 case nir_intrinsic_load_point_coord:
296 case nir_intrinsic_load_front_face:
297 if (!instr->dest.is_ssa)
298 mask = u_bit_consecutive(0, instr->num_components);
299
300 ppir_op op;
301 switch (instr->intrinsic) {
302 case nir_intrinsic_load_frag_coord:
303 op = ppir_op_load_fragcoord;
304 break;
305 case nir_intrinsic_load_point_coord:
306 op = ppir_op_load_pointcoord;
307 break;
308 case nir_intrinsic_load_front_face:
309 op = ppir_op_load_frontface;
310 break;
311 default:
312 assert(0);
313 break;
314 }
315
316 lnode = ppir_node_create_dest(block, op, &instr->dest, mask);
317 if (!lnode)
318 return NULL;
319
320 lnode->num_components = instr->num_components;
321 return &lnode->node;
322
323 case nir_intrinsic_load_uniform:
324 if (!instr->dest.is_ssa)
325 mask = u_bit_consecutive(0, instr->num_components);
326
327 lnode = ppir_node_create_dest(block, ppir_op_load_uniform, &instr->dest, mask);
328 if (!lnode)
329 return NULL;
330
331 lnode->num_components = instr->num_components;
332 lnode->index = nir_intrinsic_base(instr);
333 lnode->index += (uint32_t)nir_src_as_float(instr->src[0]);
334
335 return &lnode->node;
336
337 case nir_intrinsic_store_output:
338 snode = ppir_node_create_dest(block, ppir_op_store_color, NULL, 0);
339 if (!snode)
340 return NULL;
341
342 snode->index = nir_intrinsic_base(instr);
343
344 for (int i = 0; i < instr->num_components; i++)
345 snode->src.swizzle[i] = i;
346
347 ppir_node_add_src(block->comp, &snode->node, &snode->src, instr->src,
348 u_bit_consecutive(0, instr->num_components));
349
350 return &snode->node;
351
352 case nir_intrinsic_discard:
353 return ppir_emit_discard(block, ni);
354
355 case nir_intrinsic_discard_if:
356 return ppir_emit_discard_if(block, ni);
357
358 default:
359 ppir_error("unsupported nir_intrinsic_instr %s\n",
360 nir_intrinsic_infos[instr->intrinsic].name);
361 return NULL;
362 }
363 }
364
365 static ppir_node *ppir_emit_load_const(ppir_block *block, nir_instr *ni)
366 {
367 nir_load_const_instr *instr = nir_instr_as_load_const(ni);
368 ppir_const_node *node = ppir_node_create_ssa(block, ppir_op_const, &instr->def);
369 if (!node)
370 return NULL;
371
372 assert(instr->def.bit_size == 32);
373
374 for (int i = 0; i < instr->def.num_components; i++)
375 node->constant.value[i].i = instr->value[i].i32;
376 node->constant.num = instr->def.num_components;
377
378 return &node->node;
379 }
380
381 static ppir_node *ppir_emit_ssa_undef(ppir_block *block, nir_instr *ni)
382 {
383 ppir_error("nir_ssa_undef_instr not support\n");
384 return NULL;
385 }
386
387 static ppir_node *ppir_emit_tex(ppir_block *block, nir_instr *ni)
388 {
389 nir_tex_instr *instr = nir_instr_as_tex(ni);
390 ppir_load_texture_node *node;
391
392 if (instr->op != nir_texop_tex) {
393 ppir_error("unsupported texop %d\n", instr->op);
394 return NULL;
395 }
396
397 node = ppir_node_create_dest(block, ppir_op_load_texture, &instr->dest, 0);
398 if (!node)
399 return NULL;
400
401 node->sampler = instr->texture_index;
402
403 switch (instr->sampler_dim) {
404 case GLSL_SAMPLER_DIM_2D:
405 case GLSL_SAMPLER_DIM_RECT:
406 case GLSL_SAMPLER_DIM_EXTERNAL:
407 break;
408 default:
409 ppir_error("unsupported sampler dim: %d\n", instr->sampler_dim);
410 return NULL;
411 }
412
413 node->sampler_dim = instr->sampler_dim;
414
415 for (int i = 0; i < instr->coord_components; i++)
416 node->src_coords.swizzle[i] = i;
417
418 for (int i = 0; i < instr->num_srcs; i++) {
419 switch (instr->src[i].src_type) {
420 case nir_tex_src_coord:
421 ppir_node_add_src(block->comp, &node->node, &node->src_coords, &instr->src[i].src,
422 u_bit_consecutive(0, instr->coord_components));
423 break;
424 default:
425 ppir_error("unsupported texture source type\n");
426 assert(0);
427 return NULL;
428 }
429 }
430
431 return &node->node;
432 }
433
434 static ppir_node *ppir_emit_jump(ppir_block *block, nir_instr *ni)
435 {
436 ppir_error("nir_jump_instr not support\n");
437 return NULL;
438 }
439
440 static ppir_node *(*ppir_emit_instr[nir_instr_type_phi])(ppir_block *, nir_instr *) = {
441 [nir_instr_type_alu] = ppir_emit_alu,
442 [nir_instr_type_intrinsic] = ppir_emit_intrinsic,
443 [nir_instr_type_load_const] = ppir_emit_load_const,
444 [nir_instr_type_ssa_undef] = ppir_emit_ssa_undef,
445 [nir_instr_type_tex] = ppir_emit_tex,
446 [nir_instr_type_jump] = ppir_emit_jump,
447 };
448
449 static ppir_block *ppir_block_create(ppir_compiler *comp)
450 {
451 ppir_block *block = rzalloc(comp, ppir_block);
452 if (!block)
453 return NULL;
454
455 list_inithead(&block->node_list);
456 list_inithead(&block->instr_list);
457
458 return block;
459 }
460
461 static bool ppir_emit_block(ppir_compiler *comp, nir_block *nblock)
462 {
463 ppir_block *block = ppir_block_create(comp);
464 if (!block)
465 return false;
466
467 list_addtail(&block->list, &comp->block_list);
468 block->comp = comp;
469
470 nir_foreach_instr(instr, nblock) {
471 assert(instr->type < nir_instr_type_phi);
472 ppir_node *node = ppir_emit_instr[instr->type](block, instr);
473 if (!node)
474 return false;
475
476 list_addtail(&node->list, &block->node_list);
477 }
478
479 return true;
480 }
481
482 static bool ppir_emit_if(ppir_compiler *comp, nir_if *nif)
483 {
484 ppir_error("if nir_cf_node not support\n");
485 return false;
486 }
487
488 static bool ppir_emit_loop(ppir_compiler *comp, nir_loop *nloop)
489 {
490 ppir_error("loop nir_cf_node not support\n");
491 return false;
492 }
493
494 static bool ppir_emit_function(ppir_compiler *comp, nir_function_impl *nfunc)
495 {
496 ppir_error("function nir_cf_node not support\n");
497 return false;
498 }
499
500 static bool ppir_emit_cf_list(ppir_compiler *comp, struct exec_list *list)
501 {
502 foreach_list_typed(nir_cf_node, node, node, list) {
503 bool ret;
504
505 switch (node->type) {
506 case nir_cf_node_block:
507 ret = ppir_emit_block(comp, nir_cf_node_as_block(node));
508 break;
509 case nir_cf_node_if:
510 ret = ppir_emit_if(comp, nir_cf_node_as_if(node));
511 break;
512 case nir_cf_node_loop:
513 ret = ppir_emit_loop(comp, nir_cf_node_as_loop(node));
514 break;
515 case nir_cf_node_function:
516 ret = ppir_emit_function(comp, nir_cf_node_as_function(node));
517 break;
518 default:
519 ppir_error("unknown NIR node type %d\n", node->type);
520 return false;
521 }
522
523 if (!ret)
524 return false;
525 }
526
527 return true;
528 }
529
530 static ppir_compiler *ppir_compiler_create(void *prog, unsigned num_reg, unsigned num_ssa)
531 {
532 ppir_compiler *comp = rzalloc_size(
533 prog, sizeof(*comp) + ((num_reg << 2) + num_ssa) * sizeof(ppir_node *));
534 if (!comp)
535 return NULL;
536
537 list_inithead(&comp->block_list);
538 list_inithead(&comp->reg_list);
539
540 comp->var_nodes = (ppir_node **)(comp + 1);
541 comp->reg_base = num_ssa;
542 comp->prog = prog;
543 return comp;
544 }
545
546 static void ppir_add_ordering_deps(ppir_compiler *comp)
547 {
548 /* Some intrinsics do not have explicit dependencies and thus depend
549 * on instructions order. Consider discard_if and store_ouput as
550 * example. If we don't add fake dependency of discard_if to store_output
551 * scheduler may put store_output first and since store_output terminates
552 * shader on Utgard PP, rest of it will never be executed.
553 * Add fake dependencies for discard/branch/store to preserve
554 * instruction order.
555 *
556 * TODO: scheduler should schedule discard_if as early as possible otherwise
557 * we may end up with suboptimal code for cases like this:
558 *
559 * s3 = s1 < s2
560 * discard_if s3
561 * s4 = s1 + s2
562 * store s4
563 *
564 * In this case store depends on discard_if and s4, but since dependencies can
565 * be scheduled in any order it can result in code like this:
566 *
567 * instr1: s3 = s1 < s3
568 * instr2: s4 = s1 + s2
569 * instr3: discard_if s3
570 * instr4: store s4
571 */
572 list_for_each_entry(ppir_block, block, &comp->block_list, list) {
573 ppir_node *prev_node = NULL;
574 list_for_each_entry(ppir_node, node, &block->node_list, list) {
575 if (node->type == ppir_node_type_discard ||
576 node->type == ppir_node_type_store ||
577 node->type == ppir_node_type_branch) {
578 if (prev_node)
579 ppir_node_add_dep(node, prev_node);
580 prev_node = node;
581 }
582 }
583 }
584 }
585
586 static void ppir_print_shader_db(struct nir_shader *nir, ppir_compiler *comp,
587 struct pipe_debug_callback *debug)
588 {
589 const struct shader_info *info = &nir->info;
590 char *shaderdb;
591 int ret = asprintf(&shaderdb,
592 "%s shader: %d inst, %d loops, %d:%d spills:fills\n",
593 gl_shader_stage_name(info->stage),
594 comp->cur_instr_index,
595 comp->num_loops,
596 comp->num_spills,
597 comp->num_fills);
598 assert(ret >= 0);
599
600 if (lima_debug & LIMA_DEBUG_SHADERDB)
601 fprintf(stderr, "SHADER-DB: %s\n", shaderdb);
602
603 pipe_debug_message(debug, SHADER_INFO, "%s", shaderdb);
604 free(shaderdb);
605 }
606
607 bool ppir_compile_nir(struct lima_fs_shader_state *prog, struct nir_shader *nir,
608 struct ra_regs *ra,
609 struct pipe_debug_callback *debug)
610 {
611 nir_function_impl *func = nir_shader_get_entrypoint(nir);
612 ppir_compiler *comp = ppir_compiler_create(prog, func->reg_alloc, func->ssa_alloc);
613 if (!comp)
614 return false;
615
616 comp->ra = ra;
617
618 foreach_list_typed(nir_register, reg, node, &func->registers) {
619 ppir_reg *r = rzalloc(comp, ppir_reg);
620 if (!r)
621 return false;
622
623 r->index = reg->index;
624 r->num_components = reg->num_components;
625 r->live_in = INT_MAX;
626 r->live_out = 0;
627 r->is_head = false;
628 list_addtail(&r->list, &comp->reg_list);
629 }
630
631 if (!ppir_emit_cf_list(comp, &func->body))
632 goto err_out0;
633
634 /* If we have discard block add it to the very end */
635 if (comp->discard_block)
636 list_addtail(&comp->discard_block->list, &comp->block_list);
637
638 ppir_add_ordering_deps(comp);
639
640 ppir_node_print_prog(comp);
641
642 if (!ppir_lower_prog(comp))
643 goto err_out0;
644
645 ppir_node_print_prog(comp);
646
647 if (!ppir_node_to_instr(comp))
648 goto err_out0;
649
650 if (!ppir_schedule_prog(comp))
651 goto err_out0;
652
653 if (!ppir_regalloc_prog(comp))
654 goto err_out0;
655
656 if (!ppir_codegen_prog(comp))
657 goto err_out0;
658
659 ppir_print_shader_db(nir, comp, debug);
660
661 ralloc_free(comp);
662 return true;
663
664 err_out0:
665 ralloc_free(comp);
666 return false;
667 }
668