projects / mesa.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
history | raw | HEAD
etnaviv: update Android build files
[mesa.git] / src / gallium / drivers / etnaviv / etnaviv_compiler_nir.c
1 /*
2 * Copyright (c) 2012-2019 Etnaviv Project
3 * Copyright (c) 2019 Zodiac Inflight Innovations
4 *
5 * Permission is hereby granted, free of charge, to any person obtaining a
6 * copy of this software and associated documentation files (the "Software"),
7 * to deal in the Software without restriction, including without limitation
8 * the rights to use, copy, modify, merge, publish, distribute, sub license,
9 * and/or sell copies of the Software, and to permit persons to whom the
10 * Software is furnished to do so, subject to the following conditions:
11 *
12 * The above copyright notice and this permission notice (including the
13 * next paragraph) shall be included in all copies or substantial portions
14 * of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
21 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
22 * DEALINGS IN THE SOFTWARE.
23 *
24 * Authors:
25 * Jonathan Marek <jonathan@marek.ca>
26 * Wladimir J. van der Laan <laanwj@gmail.com>
27 */
28
29 #include "etnaviv_compiler.h"
30 #include "etnaviv_asm.h"
31 #include "etnaviv_context.h"
32 #include "etnaviv_debug.h"
33 #include "etnaviv_disasm.h"
34 #include "etnaviv_uniforms.h"
35 #include "etnaviv_util.h"
36
37 #include <math.h>
38 #include "util/u_memory.h"
39 #include "util/register_allocate.h"
40 #include "compiler/nir/nir_builder.h"
41 #include "compiler/nir/nir_worklist.h"
42
43 #include "tgsi/tgsi_strings.h"
44 #include "util/u_half.h"
45
46 struct etna_compile {
47 nir_shader *nir;
48 #define is_fs(c) ((c)->nir->info.stage == MESA_SHADER_FRAGMENT)
49 const struct etna_specs *specs;
50 struct etna_shader_variant *variant;
51
52 /* block # to instr index */
53 unsigned *block_ptr;
54
55 /* Code generation */
56 int inst_ptr; /* current instruction pointer */
57 struct etna_inst code[ETNA_MAX_INSTRUCTIONS * ETNA_INST_SIZE];
58
59 /* constants */
60 uint64_t consts[ETNA_MAX_IMM];
61
62 /* There was an error during compilation */
63 bool error;
64 };
65
66 /* io related lowering
67 * run after lower_int_to_float because it adds i2f/f2i ops
68 */
69 static void
70 etna_lower_io(nir_shader *shader, struct etna_shader_variant *v)
71 {
72 nir_foreach_function(function, shader) {
73 nir_builder b;
74 nir_builder_init(&b, function->impl);
75
76 nir_foreach_block(block, function->impl) {
77 nir_foreach_instr_safe(instr, block) {
78 if (instr->type == nir_instr_type_intrinsic) {
79 nir_intrinsic_instr *intr = nir_instr_as_intrinsic(instr);
80
81 switch (intr->intrinsic) {
82 case nir_intrinsic_load_front_face: {
83 /* HW front_face is 0.0/1.0, not 0/~0u for bool
84 * lower with a comparison with 0
85 */
86 intr->dest.ssa.bit_size = 32;
87
88 b.cursor = nir_after_instr(instr);
89
90 nir_ssa_def *ssa = nir_ine(&b, &intr->dest.ssa, nir_imm_int(&b, 0));
91 if (v->key.front_ccw)
92 nir_instr_as_alu(ssa->parent_instr)->op = nir_op_ieq;
93
94 nir_ssa_def_rewrite_uses_after(&intr->dest.ssa,
95 nir_src_for_ssa(ssa),
96 ssa->parent_instr);
97 } break;
98 case nir_intrinsic_store_deref: {
99 nir_deref_instr *deref = nir_src_as_deref(intr->src[0]);
100 if (shader->info.stage != MESA_SHADER_FRAGMENT || !v->key.frag_rb_swap)
101 break;
102
103 assert(deref->deref_type == nir_deref_type_var);
104
105 if (deref->var->data.location != FRAG_RESULT_COLOR &&
106 deref->var->data.location != FRAG_RESULT_DATA0)
107 break;
108
109 b.cursor = nir_before_instr(instr);
110
111 nir_ssa_def *ssa = nir_mov(&b, intr->src[1].ssa);
112 nir_alu_instr *alu = nir_instr_as_alu(ssa->parent_instr);
113 alu->src[0].swizzle[0] = 2;
114 alu->src[0].swizzle[2] = 0;
115 nir_instr_rewrite_src(instr, &intr->src[1], nir_src_for_ssa(ssa));
116 } break;
117 case nir_intrinsic_load_uniform: {
118 /* multiply by 16 and convert to int */
119 b.cursor = nir_before_instr(instr);
120 nir_ssa_def *ssa = nir_imul(&b, intr->src[0].ssa, nir_imm_int(&b, 16));
121 nir_instr_rewrite_src(instr, &intr->src[0], nir_src_for_ssa(ssa));
122 } break;
123 case nir_intrinsic_load_vertex_id:
124 case nir_intrinsic_load_instance_id:
125 /* detect use of vertex_id/instance_id */
126 v->vs_id_in_reg = v->infile.num_reg;
127 break;
128 default:
129 break;
130 }
131 }
132
133 if (instr->type != nir_instr_type_tex)
134 continue;
135
136 nir_tex_instr *tex = nir_instr_as_tex(instr);
137 nir_src *coord = NULL;
138 nir_src *lod_bias = NULL;
139 unsigned lod_bias_idx;
140
141 assert(tex->sampler_index == tex->texture_index);
142
143 for (unsigned i = 0; i < tex->num_srcs; i++) {
144 switch (tex->src[i].src_type) {
145 case nir_tex_src_coord:
146 coord = &tex->src[i].src;
147 break;
148 case nir_tex_src_bias:
149 case nir_tex_src_lod:
150 assert(!lod_bias);
151 lod_bias = &tex->src[i].src;
152 lod_bias_idx = i;
153 break;
154 case nir_tex_src_comparator:
155 break;
156 default:
157 assert(0);
158 break;
159 }
160 }
161
162 if (tex->sampler_dim == GLSL_SAMPLER_DIM_RECT) {
163 /* use a dummy load_uniform here to represent texcoord scale */
164 b.cursor = nir_before_instr(instr);
165 nir_intrinsic_instr *load =
166 nir_intrinsic_instr_create(b.shader, nir_intrinsic_load_uniform);
167 nir_intrinsic_set_base(load, ~tex->sampler_index);
168 load->num_components = 2;
169 load->src[0] = nir_src_for_ssa(nir_imm_float(&b, 0.0f));
170 nir_ssa_dest_init(&load->instr, &load->dest, 2, 32, NULL);
171 nir_intrinsic_set_type(load, nir_type_float);
172
173 nir_builder_instr_insert(&b, &load->instr);
174
175 nir_ssa_def *new_coord = nir_fmul(&b, coord->ssa, &load->dest.ssa);
176 nir_instr_rewrite_src(&tex->instr, coord, nir_src_for_ssa(new_coord));
177 }
178
179 /* pre HALTI5 needs texture sources in a single source */
180
181 if (!lod_bias || v->shader->specs->halti >= 5)
182 continue;
183
184 assert(coord && lod_bias && tex->coord_components < 4);
185
186 nir_alu_instr *vec = nir_alu_instr_create(shader, nir_op_vec4);
187 for (unsigned i = 0; i < tex->coord_components; i++) {
188 vec->src[i].src = nir_src_for_ssa(coord->ssa);
189 vec->src[i].swizzle[0] = i;
190 }
191 for (unsigned i = tex->coord_components; i < 4; i++)
192 vec->src[i].src = nir_src_for_ssa(lod_bias->ssa);
193
194 vec->dest.write_mask = 0xf;
195 nir_ssa_dest_init(&vec->instr, &vec->dest.dest, 4, 32, NULL);
196
197 nir_tex_instr_remove_src(tex, lod_bias_idx);
198 nir_instr_rewrite_src(&tex->instr, coord, nir_src_for_ssa(&vec->dest.dest.ssa));
199 tex->coord_components = 4;
200
201 nir_instr_insert_before(&tex->instr, &vec->instr);
202 }
203 }
204 }
205 }
206
207 static bool
208 etna_alu_to_scalar_filter_cb(const nir_instr *instr, const void *data)
209 {
210 const struct etna_specs *specs = data;
211
212 if (instr->type != nir_instr_type_alu)
213 return false;
214
215 nir_alu_instr *alu = nir_instr_as_alu(instr);
216 switch (alu->op) {
217 case nir_op_frsq:
218 case nir_op_frcp:
219 case nir_op_flog2:
220 case nir_op_fexp2:
221 case nir_op_fsqrt:
222 case nir_op_fcos:
223 case nir_op_fsin:
224 case nir_op_fdiv:
225 case nir_op_imul:
226 return true;
227 /* TODO: can do better than alu_to_scalar for vector compares */
228 case nir_op_b32all_fequal2:
229 case nir_op_b32all_fequal3:
230 case nir_op_b32all_fequal4:
231 case nir_op_b32any_fnequal2:
232 case nir_op_b32any_fnequal3:
233 case nir_op_b32any_fnequal4:
234 case nir_op_b32all_iequal2:
235 case nir_op_b32all_iequal3:
236 case nir_op_b32all_iequal4:
237 case nir_op_b32any_inequal2:
238 case nir_op_b32any_inequal3:
239 case nir_op_b32any_inequal4:
240 return true;
241 case nir_op_fdot2:
242 if (!specs->has_halti2_instructions)
243 return true;
244 break;
245 default:
246 break;
247 }
248
249 return false;
250 }
251
252 static void
253 etna_lower_alu_impl(nir_function_impl *impl, struct etna_compile *c)
254 {
255 nir_shader *shader = impl->function->shader;
256
257 nir_builder b;
258 nir_builder_init(&b, impl);
259
260 /* in a seperate loop so we can apply the multiple-uniform logic to the new fmul */
261 nir_foreach_block(block, impl) {
262 nir_foreach_instr_safe(instr, block) {
263 if (instr->type != nir_instr_type_alu)
264 continue;
265
266 nir_alu_instr *alu = nir_instr_as_alu(instr);
267 /* multiply sin/cos src by constant
268 * TODO: do this earlier (but it breaks const_prop opt)
269 */
270 if (alu->op == nir_op_fsin || alu->op == nir_op_fcos) {
271 b.cursor = nir_before_instr(instr);
272
273 nir_ssa_def *imm = c->specs->has_new_transcendentals ?
274 nir_imm_float(&b, 1.0 / M_PI) :
275 nir_imm_float(&b, 2.0 / M_PI);
276
277 nir_instr_rewrite_src(instr, &alu->src[0].src,
278 nir_src_for_ssa(nir_fmul(&b, alu->src[0].src.ssa, imm)));
279 }
280
281 /* change transcendental ops to vec2 and insert vec1 mul for the result
282 * TODO: do this earlier (but it breaks with optimizations)
283 */
284 if (c->specs->has_new_transcendentals && (
285 alu->op == nir_op_fdiv || alu->op == nir_op_flog2 ||
286 alu->op == nir_op_fsin || alu->op == nir_op_fcos)) {
287 nir_ssa_def *ssa = &alu->dest.dest.ssa;
288
289 assert(ssa->num_components == 1);
290
291 nir_alu_instr *mul = nir_alu_instr_create(shader, nir_op_fmul);
292 mul->src[0].src = mul->src[1].src = nir_src_for_ssa(ssa);
293 mul->src[1].swizzle[0] = 1;
294
295 mul->dest.write_mask = 1;
296 nir_ssa_dest_init(&mul->instr, &mul->dest.dest, 1, 32, NULL);
297
298 ssa->num_components = 2;
299
300 mul->dest.saturate = alu->dest.saturate;
301 alu->dest.saturate = 0;
302
303 nir_instr_insert_after(instr, &mul->instr);
304
305 nir_ssa_def_rewrite_uses_after(ssa, nir_src_for_ssa(&mul->dest.dest.ssa), &mul->instr);
306 }
307 }
308 }
309 }
310
311 static void etna_lower_alu(nir_shader *shader, struct etna_compile *c)
312 {
313 nir_foreach_function(function, shader) {
314 if (function->impl)
315 etna_lower_alu_impl(function->impl, c);
316 }
317 }
318
319 static void
320 emit_inst(struct etna_compile *c, struct etna_inst *inst)
321 {
322 c->code[c->inst_ptr++] = *inst;
323 }
324
325 /* to map nir srcs should to etna_inst srcs */
326 enum {
327 SRC_0_1_2 = (0 << 0) | (1 << 2) | (2 << 4),
328 SRC_0_1_X = (0 << 0) | (1 << 2) | (3 << 4),
329 SRC_0_X_X = (0 << 0) | (3 << 2) | (3 << 4),
330 SRC_0_X_1 = (0 << 0) | (3 << 2) | (1 << 4),
331 SRC_0_1_0 = (0 << 0) | (1 << 2) | (0 << 4),
332 SRC_X_X_0 = (3 << 0) | (3 << 2) | (0 << 4),
333 SRC_0_X_0 = (0 << 0) | (3 << 2) | (0 << 4),
334 };
335
336 /* info to translate a nir op to etna_inst */
337 struct etna_op_info {
338 uint8_t opcode; /* INST_OPCODE_ */
339 uint8_t src; /* SRC_ enum */
340 uint8_t cond; /* INST_CONDITION_ */
341 uint8_t type; /* INST_TYPE_ */
342 };
343
344 static const struct etna_op_info etna_ops[] = {
345 [0 ... nir_num_opcodes - 1] = {0xff},
346 #undef TRUE
347 #undef FALSE
348 #define OPCT(nir, op, src, cond, type) [nir_op_##nir] = { \
349 INST_OPCODE_##op, \
350 SRC_##src, \
351 INST_CONDITION_##cond, \
352 INST_TYPE_##type \
353 }
354 #define OPC(nir, op, src, cond) OPCT(nir, op, src, cond, F32)
355 #define IOPC(nir, op, src, cond) OPCT(nir, op, src, cond, S32)
356 #define UOPC(nir, op, src, cond) OPCT(nir, op, src, cond, U32)
357 #define OP(nir, op, src) OPC(nir, op, src, TRUE)
358 #define IOP(nir, op, src) IOPC(nir, op, src, TRUE)
359 #define UOP(nir, op, src) UOPC(nir, op, src, TRUE)
360 OP(mov, MOV, X_X_0), OP(fneg, MOV, X_X_0), OP(fabs, MOV, X_X_0), OP(fsat, MOV, X_X_0),
361 OP(fmul, MUL, 0_1_X), OP(fadd, ADD, 0_X_1), OP(ffma, MAD, 0_1_2),
362 OP(fdot2, DP2, 0_1_X), OP(fdot3, DP3, 0_1_X), OP(fdot4, DP4, 0_1_X),
363 OPC(fmin, SELECT, 0_1_0, GT), OPC(fmax, SELECT, 0_1_0, LT),
364 OP(ffract, FRC, X_X_0), OP(frcp, RCP, X_X_0), OP(frsq, RSQ, X_X_0),
365 OP(fsqrt, SQRT, X_X_0), OP(fsin, SIN, X_X_0), OP(fcos, COS, X_X_0),
366 OP(fsign, SIGN, X_X_0), OP(ffloor, FLOOR, X_X_0), OP(fceil, CEIL, X_X_0),
367 OP(flog2, LOG, X_X_0), OP(fexp2, EXP, X_X_0),
368 OPC(seq, SET, 0_1_X, EQ), OPC(sne, SET, 0_1_X, NE), OPC(sge, SET, 0_1_X, GE), OPC(slt, SET, 0_1_X, LT),
369 OPC(fcsel, SELECT, 0_1_2, NZ),
370 OP(fdiv, DIV, 0_1_X),
371 OP(fddx, DSX, 0_X_0), OP(fddy, DSY, 0_X_0),
372
373 /* type convert */
374 IOP(i2f32, I2F, 0_X_X),
375 UOP(u2f32, I2F, 0_X_X),
376 IOP(f2i32, F2I, 0_X_X),
377 UOP(f2u32, F2I, 0_X_X),
378 UOP(b2f32, AND, 0_X_X), /* AND with fui(1.0f) */
379 UOP(b2i32, AND, 0_X_X), /* AND with 1 */
380 OPC(f2b32, CMP, 0_X_X, NE), /* != 0.0 */
381 UOPC(i2b32, CMP, 0_X_X, NE), /* != 0 */
382
383 /* arithmetic */
384 IOP(iadd, ADD, 0_X_1),
385 IOP(imul, IMULLO0, 0_1_X),
386 /* IOP(imad, IMADLO0, 0_1_2), */
387 IOP(ineg, ADD, X_X_0), /* ADD 0, -x */
388 IOP(iabs, IABS, X_X_0),
389 IOP(isign, SIGN, X_X_0),
390 IOPC(imin, SELECT, 0_1_0, GT),
391 IOPC(imax, SELECT, 0_1_0, LT),
392 UOPC(umin, SELECT, 0_1_0, GT),
393 UOPC(umax, SELECT, 0_1_0, LT),
394
395 /* select */
396 UOPC(b32csel, SELECT, 0_1_2, NZ),
397
398 /* compare with int result */
399 OPC(feq32, CMP, 0_1_X, EQ),
400 OPC(fne32, CMP, 0_1_X, NE),
401 OPC(fge32, CMP, 0_1_X, GE),
402 OPC(flt32, CMP, 0_1_X, LT),
403 IOPC(ieq32, CMP, 0_1_X, EQ),
404 IOPC(ine32, CMP, 0_1_X, NE),
405 IOPC(ige32, CMP, 0_1_X, GE),
406 IOPC(ilt32, CMP, 0_1_X, LT),
407 UOPC(uge32, CMP, 0_1_X, GE),
408 UOPC(ult32, CMP, 0_1_X, LT),
409
410 /* bit ops */
411 IOP(ior, OR, 0_X_1),
412 IOP(iand, AND, 0_X_1),
413 IOP(ixor, XOR, 0_X_1),
414 IOP(inot, NOT, X_X_0),
415 IOP(ishl, LSHIFT, 0_X_1),
416 IOP(ishr, RSHIFT, 0_X_1),
417 UOP(ushr, RSHIFT, 0_X_1),
418 };
419
420 static void
421 etna_emit_block_start(struct etna_compile *c, unsigned block)
422 {
423 c->block_ptr[block] = c->inst_ptr;
424 }
425
426 static void
427 etna_emit_alu(struct etna_compile *c, nir_op op, struct etna_inst_dst dst,
428 struct etna_inst_src src[3], bool saturate)
429 {
430 struct etna_op_info ei = etna_ops[op];
431 unsigned swiz_scalar = INST_SWIZ_BROADCAST(ffs(dst.write_mask) - 1);
432
433 assert(ei.opcode != 0xff);
434
435 struct etna_inst inst = {
436 .opcode = ei.opcode,
437 .type = ei.type,
438 .cond = ei.cond,
439 .dst = dst,
440 .sat = saturate,
441 };
442
443 switch (op) {
444 case nir_op_fdiv:
445 case nir_op_flog2:
446 case nir_op_fsin:
447 case nir_op_fcos:
448 if (c->specs->has_new_transcendentals)
449 inst.tex.amode = 1;
450 /* fall through */
451 case nir_op_frsq:
452 case nir_op_frcp:
453 case nir_op_fexp2:
454 case nir_op_fsqrt:
455 case nir_op_imul:
456 /* scalar instructions we want src to be in x component */
457 src[0].swiz = inst_swiz_compose(src[0].swiz, swiz_scalar);
458 src[1].swiz = inst_swiz_compose(src[1].swiz, swiz_scalar);
459 break;
460 /* deal with instructions which don't have 1:1 mapping */
461 case nir_op_b2f32:
462 inst.src[2] = etna_immediate_float(1.0f);
463 break;
464 case nir_op_b2i32:
465 inst.src[2] = etna_immediate_int(1);
466 break;
467 case nir_op_f2b32:
468 inst.src[1] = etna_immediate_float(0.0f);
469 break;
470 case nir_op_i2b32:
471 inst.src[1] = etna_immediate_int(0);
472 break;
473 case nir_op_ineg:
474 inst.src[0] = etna_immediate_int(0);
475 src[0].neg = 1;
476 break;
477 default:
478 break;
479 }
480
481 /* set the "true" value for CMP instructions */
482 if (inst.opcode == INST_OPCODE_CMP)
483 inst.src[2] = etna_immediate_int(-1);
484
485 for (unsigned j = 0; j < 3; j++) {
486 unsigned i = ((ei.src >> j*2) & 3);
487 if (i < 3)
488 inst.src[j] = src[i];
489 }
490
491 emit_inst(c, &inst);
492 }
493
494 static void
495 etna_emit_tex(struct etna_compile *c, nir_texop op, unsigned texid, unsigned dst_swiz,
496 struct etna_inst_dst dst, struct etna_inst_src coord,
497 struct etna_inst_src lod_bias, struct etna_inst_src compare)
498 {
499 struct etna_inst inst = {
500 .dst = dst,
501 .tex.id = texid + (is_fs(c) ? 0 : c->specs->vertex_sampler_offset),
502 .tex.swiz = dst_swiz,
503 .src[0] = coord,
504 };
505
506 if (lod_bias.use)
507 inst.src[1] = lod_bias;
508
509 if (compare.use)
510 inst.src[2] = compare;
511
512 switch (op) {
513 case nir_texop_tex: inst.opcode = INST_OPCODE_TEXLD; break;
514 case nir_texop_txb: inst.opcode = INST_OPCODE_TEXLDB; break;
515 case nir_texop_txl: inst.opcode = INST_OPCODE_TEXLDL; break;
516 default:
517 assert(0);
518 }
519
520 emit_inst(c, &inst);
521 }
522
523 static void
524 etna_emit_jump(struct etna_compile *c, unsigned block, struct etna_inst_src condition)
525 {
526 if (!condition.use) {
527 emit_inst(c, &(struct etna_inst) {.opcode = INST_OPCODE_BRANCH, .imm = block });
528 return;
529 }
530
531 struct etna_inst inst = {
532 .opcode = INST_OPCODE_BRANCH,
533 .cond = INST_CONDITION_NOT,
534 .type = INST_TYPE_U32,
535 .src[0] = condition,
536 .imm = block,
537 };
538 inst.src[0].swiz = INST_SWIZ_BROADCAST(inst.src[0].swiz & 3);
539 emit_inst(c, &inst);
540 }
541
542 static void
543 etna_emit_discard(struct etna_compile *c, struct etna_inst_src condition)
544 {
545 if (!condition.use) {
546 emit_inst(c, &(struct etna_inst) { .opcode = INST_OPCODE_TEXKILL });
547 return;
548 }
549
550 struct etna_inst inst = {
551 .opcode = INST_OPCODE_TEXKILL,
552 .cond = INST_CONDITION_NZ,
553 .type = (c->specs->halti < 2) ? INST_TYPE_F32 : INST_TYPE_U32,
554 .src[0] = condition,
555 };
556 inst.src[0].swiz = INST_SWIZ_BROADCAST(inst.src[0].swiz & 3);
557 emit_inst(c, &inst);
558 }
559
560 static void
561 etna_emit_output(struct etna_compile *c, nir_variable *var, struct etna_inst_src src)
562 {
563 struct etna_shader_io_file *sf = &c->variant->outfile;
564
565 if (is_fs(c)) {
566 switch (var->data.location) {
567 case FRAG_RESULT_COLOR:
568 case FRAG_RESULT_DATA0: /* DATA0 is used by gallium shaders for color */
569 c->variant->ps_color_out_reg = src.reg;
570 break;
571 case FRAG_RESULT_DEPTH:
572 c->variant->ps_depth_out_reg = src.reg;
573 break;
574 default:
575 unreachable("Unsupported fs output");
576 }
577 return;
578 }
579
580 switch (var->data.location) {
581 case VARYING_SLOT_POS:
582 c->variant->vs_pos_out_reg = src.reg;
583 break;
584 case VARYING_SLOT_PSIZ:
585 c->variant->vs_pointsize_out_reg = src.reg;
586 break;
587 default:
588 sf->reg[sf->num_reg].reg = src.reg;
589 sf->reg[sf->num_reg].slot = var->data.location;
590 sf->reg[sf->num_reg].num_components = glsl_get_components(var->type);
591 sf->num_reg++;
592 break;
593 }
594 }
595
596 static void
597 etna_emit_load_ubo(struct etna_compile *c, struct etna_inst_dst dst,
598 struct etna_inst_src src, struct etna_inst_src base)
599 {
600 /* convert float offset back to integer */
601 if (c->specs->halti < 2) {
602 emit_inst(c, &(struct etna_inst) {
603 .opcode = INST_OPCODE_F2I,
604 .type = INST_TYPE_U32,
605 .dst = dst,
606 .src[0] = src,
607 });
608
609 emit_inst(c, &(struct etna_inst) {
610 .opcode = INST_OPCODE_LOAD,
611 .type = INST_TYPE_U32,
612 .dst = dst,
613 .src[0] = {
614 .use = 1,
615 .rgroup = INST_RGROUP_TEMP,
616 .reg = dst.reg,
617 .swiz = INST_SWIZ_BROADCAST(ffs(dst.write_mask) - 1)
618 },
619 .src[1] = base,
620 });
621
622 return;
623 }
624
625 emit_inst(c, &(struct etna_inst) {
626 .opcode = INST_OPCODE_LOAD,
627 .type = INST_TYPE_U32,
628 .dst = dst,
629 .src[0] = src,
630 .src[1] = base,
631 });
632 }
633
634 #define OPT(nir, pass, ...) ({ \
635 bool this_progress = false; \
636 NIR_PASS(this_progress, nir, pass, ##__VA_ARGS__); \
637 this_progress; \
638 })
639 #define OPT_V(nir, pass, ...) NIR_PASS_V(nir, pass, ##__VA_ARGS__)
640
641 static void
642 etna_optimize_loop(nir_shader *s)
643 {
644 bool progress;
645 do {
646 progress = false;
647
648 OPT_V(s, nir_lower_vars_to_ssa);
649 progress |= OPT(s, nir_opt_copy_prop_vars);
650 progress |= OPT(s, nir_copy_prop);
651 progress |= OPT(s, nir_opt_dce);
652 progress |= OPT(s, nir_opt_cse);
653 progress |= OPT(s, nir_opt_peephole_select, 16, true, true);
654 progress |= OPT(s, nir_opt_intrinsics);
655 progress |= OPT(s, nir_opt_algebraic);
656 progress |= OPT(s, nir_opt_constant_folding);
657 progress |= OPT(s, nir_opt_dead_cf);
658 if (OPT(s, nir_opt_trivial_continues)) {
659 progress = true;
660 /* If nir_opt_trivial_continues makes progress, then we need to clean
661 * things up if we want any hope of nir_opt_if or nir_opt_loop_unroll
662 * to make progress.
663 */
664 OPT(s, nir_copy_prop);
665 OPT(s, nir_opt_dce);
666 }
667 progress |= OPT(s, nir_opt_loop_unroll, nir_var_all);
668 progress |= OPT(s, nir_opt_if, false);
669 progress |= OPT(s, nir_opt_remove_phis);
670 progress |= OPT(s, nir_opt_undef);
671 }
672 while (progress);
673 }
674
675 static int
676 etna_glsl_type_size(const struct glsl_type *type, bool bindless)
677 {
678 return glsl_count_attribute_slots(type, false);
679 }
680
681 static void
682 copy_uniform_state_to_shader(struct etna_shader_variant *sobj, uint64_t *consts, unsigned count)
683 {
684 struct etna_shader_uniform_info *uinfo = &sobj->uniforms;
685
686 uinfo->imm_count = count * 4;
687 uinfo->imm_data = MALLOC(uinfo->imm_count * sizeof(*uinfo->imm_data));
688 uinfo->imm_contents = MALLOC(uinfo->imm_count * sizeof(*uinfo->imm_contents));
689
690 for (unsigned i = 0; i < uinfo->imm_count; i++) {
691 uinfo->imm_data[i] = consts[i];
692 uinfo->imm_contents[i] = consts[i] >> 32;
693 }
694
695 etna_set_shader_uniforms_dirty_flags(sobj);
696 }
697
698 #include "etnaviv_compiler_nir_emit.h"
699
700 bool
701 etna_compile_shader_nir(struct etna_shader_variant *v)
702 {
703 if (unlikely(!v))
704 return false;
705
706 struct etna_compile *c = CALLOC_STRUCT(etna_compile);
707 if (!c)
708 return false;
709
710 c->variant = v;
711 c->specs = v->shader->specs;
712 c->nir = nir_shader_clone(NULL, v->shader->nir);
713
714 nir_shader *s = c->nir;
715 const struct etna_specs *specs = c->specs;
716
717 v->stage = s->info.stage;
718 v->num_loops = 0; /* TODO */
719 v->vs_id_in_reg = -1;
720 v->vs_pos_out_reg = -1;
721 v->vs_pointsize_out_reg = -1;
722 v->ps_color_out_reg = 0; /* 0 for shader that doesn't write fragcolor.. */
723 v->ps_depth_out_reg = -1;
724
725 /* setup input linking */
726 struct etna_shader_io_file *sf = &v->infile;
727 if (s->info.stage == MESA_SHADER_VERTEX) {
728 nir_foreach_variable(var, &s->inputs) {
729 unsigned idx = var->data.driver_location;
730 sf->reg[idx].reg = idx;
731 sf->reg[idx].slot = var->data.location;
732 sf->reg[idx].num_components = glsl_get_components(var->type);
733 sf->num_reg = MAX2(sf->num_reg, idx+1);
734 }
735 } else {
736 unsigned count = 0;
737 nir_foreach_variable(var, &s->inputs) {
738 unsigned idx = var->data.driver_location;
739 sf->reg[idx].reg = idx + 1;
740 sf->reg[idx].slot = var->data.location;
741 sf->reg[idx].num_components = glsl_get_components(var->type);
742 sf->num_reg = MAX2(sf->num_reg, idx+1);
743 count++;
744 }
745 assert(sf->num_reg == count);
746 }
747
748 NIR_PASS_V(s, nir_lower_io, ~nir_var_shader_out, etna_glsl_type_size,
749 (nir_lower_io_options)0);
750
751 OPT_V(s, nir_lower_regs_to_ssa);
752 OPT_V(s, nir_lower_vars_to_ssa);
753 OPT_V(s, nir_lower_indirect_derefs, nir_var_all);
754 OPT_V(s, nir_lower_tex, &(struct nir_lower_tex_options) { .lower_txp = ~0u });
755 OPT_V(s, nir_lower_alu_to_scalar, etna_alu_to_scalar_filter_cb, specs);
756
757 etna_optimize_loop(s);
758
759 OPT_V(s, etna_lower_io, v);
760
761 if (v->shader->specs->vs_need_z_div)
762 NIR_PASS_V(s, nir_lower_clip_halfz);
763
764 /* lower pre-halti2 to float (halti0 has integers, but only scalar..) */
765 if (c->specs->halti < 2) {
766 /* use opt_algebraic between int_to_float and boot_to_float because
767 * int_to_float emits ftrunc, and ftrunc lowering generates bool ops
768 */
769 OPT_V(s, nir_lower_int_to_float);
770 OPT_V(s, nir_opt_algebraic);
771 OPT_V(s, nir_lower_bool_to_float);
772 } else {
773 OPT_V(s, nir_lower_idiv, nir_lower_idiv_fast);
774 OPT_V(s, nir_lower_bool_to_int32);
775 }
776
777 etna_optimize_loop(s);
778
779 if (DBG_ENABLED(ETNA_DBG_DUMP_SHADERS))
780 nir_print_shader(s, stdout);
781
782 while( OPT(s, nir_opt_vectorize) );
783 OPT_V(s, nir_lower_alu_to_scalar, etna_alu_to_scalar_filter_cb, specs);
784
785 NIR_PASS_V(s, nir_remove_dead_variables, nir_var_function_temp);
786 NIR_PASS_V(s, nir_opt_algebraic_late);
787
788 NIR_PASS_V(s, nir_move_vec_src_uses_to_dest);
789 NIR_PASS_V(s, nir_copy_prop);
790 /* only HW supported integer source mod is ineg for iadd instruction (?) */
791 NIR_PASS_V(s, nir_lower_to_source_mods, ~nir_lower_int_source_mods);
792 /* need copy prop after uses_to_dest, and before src mods: see
793 * dEQP-GLES2.functional.shaders.random.all_features.fragment.95
794 */
795
796 NIR_PASS_V(s, nir_opt_dce);
797
798 NIR_PASS_V(s, etna_lower_alu, c);
799
800 if (DBG_ENABLED(ETNA_DBG_DUMP_SHADERS))
801 nir_print_shader(s, stdout);
802
803 unsigned block_ptr[nir_shader_get_entrypoint(s)->num_blocks];
804 c->block_ptr = block_ptr;
805
806 unsigned num_consts;
807 ASSERTED bool ok = emit_shader(c, &v->num_temps, &num_consts);
808 assert(ok);
809
810 /* empty shader, emit NOP */
811 if (!c->inst_ptr)
812 emit_inst(c, &(struct etna_inst) { .opcode = INST_OPCODE_NOP });
813
814 /* assemble instructions, fixing up labels */
815 uint32_t *code = MALLOC(c->inst_ptr * 16);
816 for (unsigned i = 0; i < c->inst_ptr; i++) {
817 struct etna_inst *inst = &c->code[i];
818 if (inst->opcode == INST_OPCODE_BRANCH)
819 inst->imm = block_ptr[inst->imm];
820
821 inst->halti5 = specs->halti >= 5;
822 etna_assemble(&code[i * 4], inst);
823 }
824
825 v->code_size = c->inst_ptr * 4;
826 v->code = code;
827 v->needs_icache = c->inst_ptr > specs->max_instructions;
828
829 copy_uniform_state_to_shader(v, c->consts, num_consts);
830
831 if (s->info.stage == MESA_SHADER_FRAGMENT) {
832 v->input_count_unk8 = 31; /* XXX what is this */
833 assert(v->ps_depth_out_reg <= 0);
834 ralloc_free(c->nir);
835 FREE(c);
836 return true;
837 }
838
839 v->input_count_unk8 = DIV_ROUND_UP(v->infile.num_reg + 4, 16); /* XXX what is this */
840
841 /* fill in "mystery meat" load balancing value. This value determines how
842 * work is scheduled between VS and PS
843 * in the unified shader architecture. More precisely, it is determined from
844 * the number of VS outputs, as well as chip-specific
845 * vertex output buffer size, vertex cache size, and the number of shader
846 * cores.
847 *
848 * XXX this is a conservative estimate, the "optimal" value is only known for
849 * sure at link time because some
850 * outputs may be unused and thus unmapped. Then again, in the general use
851 * case with GLSL the vertex and fragment
852 * shaders are linked already before submitting to Gallium, thus all outputs
853 * are used.
854 *
855 * note: TGSI compiler counts all outputs (including position and pointsize), here
856 * v->outfile.num_reg only counts varyings, +1 to compensate for the position output
857 * TODO: might have a problem that we don't count pointsize when it is used
858 */
859
860 int half_out = v->outfile.num_reg / 2 + 1;
861 assert(half_out);
862
863 uint32_t b = ((20480 / (specs->vertex_output_buffer_size -
864 2 * half_out * specs->vertex_cache_size)) +
865 9) /
866 10;
867 uint32_t a = (b + 256 / (specs->shader_core_count * half_out)) / 2;
868 v->vs_load_balancing = VIVS_VS_LOAD_BALANCING_A(MIN2(a, 255)) |
869 VIVS_VS_LOAD_BALANCING_B(MIN2(b, 255)) |
870 VIVS_VS_LOAD_BALANCING_C(0x3f) |
871 VIVS_VS_LOAD_BALANCING_D(0x0f);
872
873 ralloc_free(c->nir);
874 FREE(c);
875 return true;
876 }
877
878 void
879 etna_destroy_shader_nir(struct etna_shader_variant *shader)
880 {
881 assert(shader);
882
883 FREE(shader->code);
884 FREE(shader->uniforms.imm_data);
885 FREE(shader->uniforms.imm_contents);
886 FREE(shader);
887 }
888
889 extern const char *tgsi_swizzle_names[];
890 void
891 etna_dump_shader_nir(const struct etna_shader_variant *shader)
892 {
893 if (shader->stage == MESA_SHADER_VERTEX)
894 printf("VERT\n");
895 else
896 printf("FRAG\n");
897
898 etna_disasm(shader->code, shader->code_size, PRINT_RAW);
899
900 printf("num loops: %i\n", shader->num_loops);
901 printf("num temps: %i\n", shader->num_temps);
902 printf("immediates:\n");
903 for (int idx = 0; idx < shader->uniforms.imm_count; ++idx) {
904 printf(" [%i].%s = %f (0x%08x) (%d)\n",
905 idx / 4,
906 tgsi_swizzle_names[idx % 4],
907 *((float *)&shader->uniforms.imm_data[idx]),
908 shader->uniforms.imm_data[idx],
909 shader->uniforms.imm_contents[idx]);
910 }
911 printf("inputs:\n");
912 for (int idx = 0; idx < shader->infile.num_reg; ++idx) {
913 printf(" [%i] name=%s comps=%i\n", shader->infile.reg[idx].reg,
914 (shader->stage == MESA_SHADER_VERTEX) ?
915 gl_vert_attrib_name(shader->infile.reg[idx].slot) :
916 gl_varying_slot_name(shader->infile.reg[idx].slot),
917 shader->infile.reg[idx].num_components);
918 }
919 printf("outputs:\n");
920 for (int idx = 0; idx < shader->outfile.num_reg; ++idx) {
921 printf(" [%i] name=%s comps=%i\n", shader->outfile.reg[idx].reg,
922 (shader->stage == MESA_SHADER_VERTEX) ?
923 gl_varying_slot_name(shader->outfile.reg[idx].slot) :
924 gl_frag_result_name(shader->outfile.reg[idx].slot),
925 shader->outfile.reg[idx].num_components);
926 }
927 printf("special:\n");
928 if (shader->stage == MESA_SHADER_VERTEX) {
929 printf(" vs_pos_out_reg=%i\n", shader->vs_pos_out_reg);
930 printf(" vs_pointsize_out_reg=%i\n", shader->vs_pointsize_out_reg);
931 printf(" vs_load_balancing=0x%08x\n", shader->vs_load_balancing);
932 } else {
933 printf(" ps_color_out_reg=%i\n", shader->ps_color_out_reg);
934 printf(" ps_depth_out_reg=%i\n", shader->ps_depth_out_reg);
935 }
936 printf(" input_count_unk8=0x%08x\n", shader->input_count_unk8);
937 }
938
939 static const struct etna_shader_inout *
940 etna_shader_vs_lookup(const struct etna_shader_variant *sobj,
941 const struct etna_shader_inout *in)
942 {
943 for (int i = 0; i < sobj->outfile.num_reg; i++)
944 if (sobj->outfile.reg[i].slot == in->slot)
945 return &sobj->outfile.reg[i];
946
947 return NULL;
948 }
949
950 bool
951 etna_link_shader_nir(struct etna_shader_link_info *info,
952 const struct etna_shader_variant *vs,
953 const struct etna_shader_variant *fs)
954 {
955 int comp_ofs = 0;
956 /* For each fragment input we need to find the associated vertex shader
957 * output, which can be found by matching on semantic name and index. A
958 * binary search could be used because the vs outputs are sorted by their
959 * semantic index and grouped by semantic type by fill_in_vs_outputs.
960 */
961 assert(fs->infile.num_reg < ETNA_NUM_INPUTS);
962 info->pcoord_varying_comp_ofs = -1;
963
964 for (int idx = 0; idx < fs->infile.num_reg; ++idx) {
965 const struct etna_shader_inout *fsio = &fs->infile.reg[idx];
966 const struct etna_shader_inout *vsio = etna_shader_vs_lookup(vs, fsio);
967 struct etna_varying *varying;
968 bool interpolate_always = true;
969
970 assert(fsio->reg > 0 && fsio->reg <= ARRAY_SIZE(info->varyings));
971
972 if (fsio->reg > info->num_varyings)
973 info->num_varyings = fsio->reg;
974
975 varying = &info->varyings[fsio->reg - 1];
976 varying->num_components = fsio->num_components;
977
978 if (!interpolate_always) /* colors affected by flat shading */
979 varying->pa_attributes = 0x200;
980 else /* texture coord or other bypasses flat shading */
981 varying->pa_attributes = 0x2f1;
982
983 varying->use[0] = VARYING_COMPONENT_USE_UNUSED;
984 varying->use[1] = VARYING_COMPONENT_USE_UNUSED;
985 varying->use[2] = VARYING_COMPONENT_USE_UNUSED;
986 varying->use[3] = VARYING_COMPONENT_USE_UNUSED;
987
988 /* point coord is an input to the PS without matching VS output,
989 * so it gets a varying slot without being assigned a VS register.
990 */
991 if (fsio->slot == VARYING_SLOT_PNTC) {
992 varying->use[0] = VARYING_COMPONENT_USE_POINTCOORD_X;
993 varying->use[1] = VARYING_COMPONENT_USE_POINTCOORD_Y;
994
995 info->pcoord_varying_comp_ofs = comp_ofs;
996 } else {
997 if (vsio == NULL) { /* not found -- link error */
998 BUG("Semantic value not found in vertex shader outputs\n");
999 return true;
1000 }
1001 varying->reg = vsio->reg;
1002 }
1003
1004 comp_ofs += varying->num_components;
1005 }
1006
1007 assert(info->num_varyings == fs->infile.num_reg);
1008
1009 return false;
1010 }