projects / mesa.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
history | raw | HEAD
gallium: s/unsigned/enum pipe_shader_type/ for get_compiler_options()
[mesa.git] / src / gallium / drivers / vc4 / vc4_program.c
1 /*
2 * Copyright (c) 2014 Scott Mansell
3 * Copyright © 2014 Broadcom
4 *
5 * Permission is hereby granted, free of charge, to any person obtaining a
6 * copy of this software and associated documentation files (the "Software"),
7 * to deal in the Software without restriction, including without limitation
8 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
9 * and/or sell copies of the Software, and to permit persons to whom the
10 * Software is furnished to do so, subject to the following conditions:
11 *
12 * The above copyright notice and this permission notice (including the next
13 * paragraph) shall be included in all copies or substantial portions of the
14 * Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
21 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
22 * IN THE SOFTWARE.
23 */
24
25 #include <inttypes.h>
26 #include "util/u_format.h"
27 #include "util/crc32.h"
28 #include "util/u_math.h"
29 #include "util/u_memory.h"
30 #include "util/ralloc.h"
31 #include "util/hash_table.h"
32 #include "tgsi/tgsi_dump.h"
33 #include "tgsi/tgsi_parse.h"
34 #include "compiler/nir/nir.h"
35 #include "compiler/nir/nir_builder.h"
36 #include "nir/tgsi_to_nir.h"
37 #include "vc4_context.h"
38 #include "vc4_qpu.h"
39 #include "vc4_qir.h"
40 #include "mesa/state_tracker/st_glsl_types.h"
41
42 static struct qreg
43 ntq_get_src(struct vc4_compile *c, nir_src src, int i);
44 static void
45 ntq_emit_cf_list(struct vc4_compile *c, struct exec_list *list);
46
47 static void
48 resize_qreg_array(struct vc4_compile *c,
49 struct qreg **regs,
50 uint32_t *size,
51 uint32_t decl_size)
52 {
53 if (*size >= decl_size)
54 return;
55
56 uint32_t old_size = *size;
57 *size = MAX2(*size * 2, decl_size);
58 *regs = reralloc(c, *regs, struct qreg, *size);
59 if (!*regs) {
60 fprintf(stderr, "Malloc failure\n");
61 abort();
62 }
63
64 for (uint32_t i = old_size; i < *size; i++)
65 (*regs)[i] = c->undef;
66 }
67
68 static void
69 ntq_emit_thrsw(struct vc4_compile *c)
70 {
71 if (!c->fs_threaded)
72 return;
73
74 /* Always thread switch after each texture operation for now.
75 *
76 * We could do better by batching a bunch of texture fetches up and
77 * then doing one thread switch and collecting all their results
78 * afterward.
79 */
80 qir_emit_nondef(c, qir_inst(QOP_THRSW, c->undef,
81 c->undef, c->undef));
82 c->last_thrsw_at_top_level = (c->execute.file == QFILE_NULL);
83 }
84
85 static struct qreg
86 indirect_uniform_load(struct vc4_compile *c, nir_intrinsic_instr *intr)
87 {
88 struct qreg indirect_offset = ntq_get_src(c, intr->src[0], 0);
89 uint32_t offset = nir_intrinsic_base(intr);
90 struct vc4_compiler_ubo_range *range = NULL;
91 unsigned i;
92 for (i = 0; i < c->num_uniform_ranges; i++) {
93 range = &c->ubo_ranges[i];
94 if (offset >= range->src_offset &&
95 offset < range->src_offset + range->size) {
96 break;
97 }
98 }
99 /* The driver-location-based offset always has to be within a declared
100 * uniform range.
101 */
102 assert(range);
103 if (!range->used) {
104 range->used = true;
105 range->dst_offset = c->next_ubo_dst_offset;
106 c->next_ubo_dst_offset += range->size;
107 c->num_ubo_ranges++;
108 }
109
110 offset -= range->src_offset;
111
112 /* Adjust for where we stored the TGSI register base. */
113 indirect_offset = qir_ADD(c, indirect_offset,
114 qir_uniform_ui(c, (range->dst_offset +
115 offset)));
116
117 /* Clamp to [0, array size). Note that MIN/MAX are signed. */
118 indirect_offset = qir_MAX(c, indirect_offset, qir_uniform_ui(c, 0));
119 indirect_offset = qir_MIN_NOIMM(c, indirect_offset,
120 qir_uniform_ui(c, (range->dst_offset +
121 range->size - 4)));
122
123 qir_ADD_dest(c, qir_reg(QFILE_TEX_S_DIRECT, 0),
124 indirect_offset,
125 qir_uniform(c, QUNIFORM_UBO_ADDR, 0));
126
127 c->num_texture_samples++;
128
129 ntq_emit_thrsw(c);
130
131 return qir_TEX_RESULT(c);
132 }
133
134 nir_ssa_def *
135 vc4_nir_get_swizzled_channel(nir_builder *b, nir_ssa_def **srcs, int swiz)
136 {
137 switch (swiz) {
138 default:
139 case PIPE_SWIZZLE_NONE:
140 fprintf(stderr, "warning: unknown swizzle\n");
141 /* FALLTHROUGH */
142 case PIPE_SWIZZLE_0:
143 return nir_imm_float(b, 0.0);
144 case PIPE_SWIZZLE_1:
145 return nir_imm_float(b, 1.0);
146 case PIPE_SWIZZLE_X:
147 case PIPE_SWIZZLE_Y:
148 case PIPE_SWIZZLE_Z:
149 case PIPE_SWIZZLE_W:
150 return srcs[swiz];
151 }
152 }
153
154 static struct qreg *
155 ntq_init_ssa_def(struct vc4_compile *c, nir_ssa_def *def)
156 {
157 struct qreg *qregs = ralloc_array(c->def_ht, struct qreg,
158 def->num_components);
159 _mesa_hash_table_insert(c->def_ht, def, qregs);
160 return qregs;
161 }
162
163 /**
164 * This function is responsible for getting QIR results into the associated
165 * storage for a NIR instruction.
166 *
167 * If it's a NIR SSA def, then we just set the associated hash table entry to
168 * the new result.
169 *
170 * If it's a NIR reg, then we need to update the existing qreg assigned to the
171 * NIR destination with the incoming value. To do that without introducing
172 * new MOVs, we require that the incoming qreg either be a uniform, or be
173 * SSA-defined by the previous QIR instruction in the block and rewritable by
174 * this function. That lets us sneak ahead and insert the SF flag beforehand
175 * (knowing that the previous instruction doesn't depend on flags) and rewrite
176 * its destination to be the NIR reg's destination
177 */
178 static void
179 ntq_store_dest(struct vc4_compile *c, nir_dest *dest, int chan,
180 struct qreg result)
181 {
182 struct qinst *last_inst = NULL;
183 if (!list_empty(&c->cur_block->instructions))
184 last_inst = (struct qinst *)c->cur_block->instructions.prev;
185
186 assert(result.file == QFILE_UNIF ||
187 (result.file == QFILE_TEMP &&
188 last_inst && last_inst == c->defs[result.index]));
189
190 if (dest->is_ssa) {
191 assert(chan < dest->ssa.num_components);
192
193 struct qreg *qregs;
194 struct hash_entry *entry =
195 _mesa_hash_table_search(c->def_ht, &dest->ssa);
196
197 if (entry)
198 qregs = entry->data;
199 else
200 qregs = ntq_init_ssa_def(c, &dest->ssa);
201
202 qregs[chan] = result;
203 } else {
204 nir_register *reg = dest->reg.reg;
205 assert(dest->reg.base_offset == 0);
206 assert(reg->num_array_elems == 0);
207 struct hash_entry *entry =
208 _mesa_hash_table_search(c->def_ht, reg);
209 struct qreg *qregs = entry->data;
210
211 /* Insert a MOV if the source wasn't an SSA def in the
212 * previous instruction.
213 */
214 if (result.file == QFILE_UNIF) {
215 result = qir_MOV(c, result);
216 last_inst = c->defs[result.index];
217 }
218
219 /* We know they're both temps, so just rewrite index. */
220 c->defs[last_inst->dst.index] = NULL;
221 last_inst->dst.index = qregs[chan].index;
222
223 /* If we're in control flow, then make this update of the reg
224 * conditional on the execution mask.
225 */
226 if (c->execute.file != QFILE_NULL) {
227 last_inst->dst.index = qregs[chan].index;
228
229 /* Set the flags to the current exec mask. To insert
230 * the SF, we temporarily remove our SSA instruction.
231 */
232 list_del(&last_inst->link);
233 qir_SF(c, c->execute);
234 list_addtail(&last_inst->link,
235 &c->cur_block->instructions);
236
237 last_inst->cond = QPU_COND_ZS;
238 last_inst->cond_is_exec_mask = true;
239 }
240 }
241 }
242
243 static struct qreg *
244 ntq_get_dest(struct vc4_compile *c, nir_dest *dest)
245 {
246 if (dest->is_ssa) {
247 struct qreg *qregs = ntq_init_ssa_def(c, &dest->ssa);
248 for (int i = 0; i < dest->ssa.num_components; i++)
249 qregs[i] = c->undef;
250 return qregs;
251 } else {
252 nir_register *reg = dest->reg.reg;
253 assert(dest->reg.base_offset == 0);
254 assert(reg->num_array_elems == 0);
255 struct hash_entry *entry =
256 _mesa_hash_table_search(c->def_ht, reg);
257 return entry->data;
258 }
259 }
260
261 static struct qreg
262 ntq_get_src(struct vc4_compile *c, nir_src src, int i)
263 {
264 struct hash_entry *entry;
265 if (src.is_ssa) {
266 entry = _mesa_hash_table_search(c->def_ht, src.ssa);
267 assert(i < src.ssa->num_components);
268 } else {
269 nir_register *reg = src.reg.reg;
270 entry = _mesa_hash_table_search(c->def_ht, reg);
271 assert(reg->num_array_elems == 0);
272 assert(src.reg.base_offset == 0);
273 assert(i < reg->num_components);
274 }
275
276 struct qreg *qregs = entry->data;
277 return qregs[i];
278 }
279
280 static struct qreg
281 ntq_get_alu_src(struct vc4_compile *c, nir_alu_instr *instr,
282 unsigned src)
283 {
284 assert(util_is_power_of_two(instr->dest.write_mask));
285 unsigned chan = ffs(instr->dest.write_mask) - 1;
286 struct qreg r = ntq_get_src(c, instr->src[src].src,
287 instr->src[src].swizzle[chan]);
288
289 assert(!instr->src[src].abs);
290 assert(!instr->src[src].negate);
291
292 return r;
293 };
294
295 static inline struct qreg
296 qir_SAT(struct vc4_compile *c, struct qreg val)
297 {
298 return qir_FMAX(c,
299 qir_FMIN(c, val, qir_uniform_f(c, 1.0)),
300 qir_uniform_f(c, 0.0));
301 }
302
303 static struct qreg
304 ntq_rcp(struct vc4_compile *c, struct qreg x)
305 {
306 struct qreg r = qir_RCP(c, x);
307
308 /* Apply a Newton-Raphson step to improve the accuracy. */
309 r = qir_FMUL(c, r, qir_FSUB(c,
310 qir_uniform_f(c, 2.0),
311 qir_FMUL(c, x, r)));
312
313 return r;
314 }
315
316 static struct qreg
317 ntq_rsq(struct vc4_compile *c, struct qreg x)
318 {
319 struct qreg r = qir_RSQ(c, x);
320
321 /* Apply a Newton-Raphson step to improve the accuracy. */
322 r = qir_FMUL(c, r, qir_FSUB(c,
323 qir_uniform_f(c, 1.5),
324 qir_FMUL(c,
325 qir_uniform_f(c, 0.5),
326 qir_FMUL(c, x,
327 qir_FMUL(c, r, r)))));
328
329 return r;
330 }
331
332 static struct qreg
333 ntq_umul(struct vc4_compile *c, struct qreg src0, struct qreg src1)
334 {
335 struct qreg src0_hi = qir_SHR(c, src0,
336 qir_uniform_ui(c, 24));
337 struct qreg src1_hi = qir_SHR(c, src1,
338 qir_uniform_ui(c, 24));
339
340 struct qreg hilo = qir_MUL24(c, src0_hi, src1);
341 struct qreg lohi = qir_MUL24(c, src0, src1_hi);
342 struct qreg lolo = qir_MUL24(c, src0, src1);
343
344 return qir_ADD(c, lolo, qir_SHL(c,
345 qir_ADD(c, hilo, lohi),
346 qir_uniform_ui(c, 24)));
347 }
348
349 static struct qreg
350 ntq_scale_depth_texture(struct vc4_compile *c, struct qreg src)
351 {
352 struct qreg depthf = qir_ITOF(c, qir_SHR(c, src,
353 qir_uniform_ui(c, 8)));
354 return qir_FMUL(c, depthf, qir_uniform_f(c, 1.0f/0xffffff));
355 }
356
357 /**
358 * Emits a lowered TXF_MS from an MSAA texture.
359 *
360 * The addressing math has been lowered in NIR, and now we just need to read
361 * it like a UBO.
362 */
363 static void
364 ntq_emit_txf(struct vc4_compile *c, nir_tex_instr *instr)
365 {
366 uint32_t tile_width = 32;
367 uint32_t tile_height = 32;
368 uint32_t tile_size = (tile_height * tile_width *
369 VC4_MAX_SAMPLES * sizeof(uint32_t));
370
371 unsigned unit = instr->texture_index;
372 uint32_t w = align(c->key->tex[unit].msaa_width, tile_width);
373 uint32_t w_tiles = w / tile_width;
374 uint32_t h = align(c->key->tex[unit].msaa_height, tile_height);
375 uint32_t h_tiles = h / tile_height;
376 uint32_t size = w_tiles * h_tiles * tile_size;
377
378 struct qreg addr;
379 assert(instr->num_srcs == 1);
380 assert(instr->src[0].src_type == nir_tex_src_coord);
381 addr = ntq_get_src(c, instr->src[0].src, 0);
382
383 /* Perform the clamping required by kernel validation. */
384 addr = qir_MAX(c, addr, qir_uniform_ui(c, 0));
385 addr = qir_MIN_NOIMM(c, addr, qir_uniform_ui(c, size - 4));
386
387 qir_ADD_dest(c, qir_reg(QFILE_TEX_S_DIRECT, 0),
388 addr, qir_uniform(c, QUNIFORM_TEXTURE_MSAA_ADDR, unit));
389
390 ntq_emit_thrsw(c);
391
392 struct qreg tex = qir_TEX_RESULT(c);
393 c->num_texture_samples++;
394
395 enum pipe_format format = c->key->tex[unit].format;
396 if (util_format_is_depth_or_stencil(format)) {
397 struct qreg scaled = ntq_scale_depth_texture(c, tex);
398 for (int i = 0; i < 4; i++)
399 ntq_store_dest(c, &instr->dest, i, qir_MOV(c, scaled));
400 } else {
401 for (int i = 0; i < 4; i++)
402 ntq_store_dest(c, &instr->dest, i,
403 qir_UNPACK_8_F(c, tex, i));
404 }
405 }
406
407 static void
408 ntq_emit_tex(struct vc4_compile *c, nir_tex_instr *instr)
409 {
410 struct qreg s, t, r, lod, compare;
411 bool is_txb = false, is_txl = false;
412 unsigned unit = instr->texture_index;
413
414 if (instr->op == nir_texop_txf) {
415 ntq_emit_txf(c, instr);
416 return;
417 }
418
419 for (unsigned i = 0; i < instr->num_srcs; i++) {
420 switch (instr->src[i].src_type) {
421 case nir_tex_src_coord:
422 s = ntq_get_src(c, instr->src[i].src, 0);
423 if (instr->sampler_dim == GLSL_SAMPLER_DIM_1D)
424 t = qir_uniform_f(c, 0.5);
425 else
426 t = ntq_get_src(c, instr->src[i].src, 1);
427 if (instr->sampler_dim == GLSL_SAMPLER_DIM_CUBE)
428 r = ntq_get_src(c, instr->src[i].src, 2);
429 break;
430 case nir_tex_src_bias:
431 lod = ntq_get_src(c, instr->src[i].src, 0);
432 is_txb = true;
433 break;
434 case nir_tex_src_lod:
435 lod = ntq_get_src(c, instr->src[i].src, 0);
436 is_txl = true;
437 break;
438 case nir_tex_src_comparator:
439 compare = ntq_get_src(c, instr->src[i].src, 0);
440 break;
441 default:
442 unreachable("unknown texture source");
443 }
444 }
445
446 if (c->stage != QSTAGE_FRAG && !is_txl) {
447 /* From the GLSL 1.20 spec:
448 *
449 * "If it is mip-mapped and running on the vertex shader,
450 * then the base texture is used."
451 */
452 is_txl = true;
453 lod = qir_uniform_ui(c, 0);
454 }
455
456 if (c->key->tex[unit].force_first_level) {
457 lod = qir_uniform(c, QUNIFORM_TEXTURE_FIRST_LEVEL, unit);
458 is_txl = true;
459 is_txb = false;
460 }
461
462 struct qreg texture_u[] = {
463 qir_uniform(c, QUNIFORM_TEXTURE_CONFIG_P0, unit),
464 qir_uniform(c, QUNIFORM_TEXTURE_CONFIG_P1, unit),
465 qir_uniform(c, QUNIFORM_CONSTANT, 0),
466 qir_uniform(c, QUNIFORM_CONSTANT, 0),
467 };
468 uint32_t next_texture_u = 0;
469
470 /* There is no native support for GL texture rectangle coordinates, so
471 * we have to rescale from ([0, width], [0, height]) to ([0, 1], [0,
472 * 1]).
473 */
474 if (instr->sampler_dim == GLSL_SAMPLER_DIM_RECT) {
475 s = qir_FMUL(c, s,
476 qir_uniform(c, QUNIFORM_TEXRECT_SCALE_X, unit));
477 t = qir_FMUL(c, t,
478 qir_uniform(c, QUNIFORM_TEXRECT_SCALE_Y, unit));
479 }
480
481 if (instr->sampler_dim == GLSL_SAMPLER_DIM_CUBE || is_txl) {
482 texture_u[2] = qir_uniform(c, QUNIFORM_TEXTURE_CONFIG_P2,
483 unit | (is_txl << 16));
484 }
485
486 struct qinst *tmu;
487 if (instr->sampler_dim == GLSL_SAMPLER_DIM_CUBE) {
488 tmu = qir_MOV_dest(c, qir_reg(QFILE_TEX_R, 0), r);
489 tmu->src[qir_get_tex_uniform_src(tmu)] =
490 texture_u[next_texture_u++];
491 } else if (c->key->tex[unit].wrap_s == PIPE_TEX_WRAP_CLAMP_TO_BORDER ||
492 c->key->tex[unit].wrap_s == PIPE_TEX_WRAP_CLAMP ||
493 c->key->tex[unit].wrap_t == PIPE_TEX_WRAP_CLAMP_TO_BORDER ||
494 c->key->tex[unit].wrap_t == PIPE_TEX_WRAP_CLAMP) {
495 tmu = qir_MOV_dest(c, qir_reg(QFILE_TEX_R, 0),
496 qir_uniform(c, QUNIFORM_TEXTURE_BORDER_COLOR,
497 unit));
498 tmu->src[qir_get_tex_uniform_src(tmu)] =
499 texture_u[next_texture_u++];
500 }
501
502 if (c->key->tex[unit].wrap_s == PIPE_TEX_WRAP_CLAMP) {
503 s = qir_SAT(c, s);
504 }
505
506 if (c->key->tex[unit].wrap_t == PIPE_TEX_WRAP_CLAMP) {
507 t = qir_SAT(c, t);
508 }
509
510 tmu = qir_MOV_dest(c, qir_reg(QFILE_TEX_T, 0), t);
511 tmu->src[qir_get_tex_uniform_src(tmu)] =
512 texture_u[next_texture_u++];
513
514 if (is_txl || is_txb) {
515 tmu = qir_MOV_dest(c, qir_reg(QFILE_TEX_B, 0), lod);
516 tmu->src[qir_get_tex_uniform_src(tmu)] =
517 texture_u[next_texture_u++];
518 }
519
520 tmu = qir_MOV_dest(c, qir_reg(QFILE_TEX_S, 0), s);
521 tmu->src[qir_get_tex_uniform_src(tmu)] = texture_u[next_texture_u++];
522
523 c->num_texture_samples++;
524
525 ntq_emit_thrsw(c);
526
527 struct qreg tex = qir_TEX_RESULT(c);
528
529 enum pipe_format format = c->key->tex[unit].format;
530
531 struct qreg *dest = ntq_get_dest(c, &instr->dest);
532 if (util_format_is_depth_or_stencil(format)) {
533 struct qreg normalized = ntq_scale_depth_texture(c, tex);
534 struct qreg depth_output;
535
536 struct qreg u0 = qir_uniform_f(c, 0.0f);
537 struct qreg u1 = qir_uniform_f(c, 1.0f);
538 if (c->key->tex[unit].compare_mode) {
539 /* From the GL_ARB_shadow spec:
540 *
541 * "Let Dt (D subscript t) be the depth texture
542 * value, in the range [0, 1]. Let R be the
543 * interpolated texture coordinate clamped to the
544 * range [0, 1]."
545 */
546 compare = qir_SAT(c, compare);
547
548 switch (c->key->tex[unit].compare_func) {
549 case PIPE_FUNC_NEVER:
550 depth_output = qir_uniform_f(c, 0.0f);
551 break;
552 case PIPE_FUNC_ALWAYS:
553 depth_output = u1;
554 break;
555 case PIPE_FUNC_EQUAL:
556 qir_SF(c, qir_FSUB(c, compare, normalized));
557 depth_output = qir_SEL(c, QPU_COND_ZS, u1, u0);
558 break;
559 case PIPE_FUNC_NOTEQUAL:
560 qir_SF(c, qir_FSUB(c, compare, normalized));
561 depth_output = qir_SEL(c, QPU_COND_ZC, u1, u0);
562 break;
563 case PIPE_FUNC_GREATER:
564 qir_SF(c, qir_FSUB(c, compare, normalized));
565 depth_output = qir_SEL(c, QPU_COND_NC, u1, u0);
566 break;
567 case PIPE_FUNC_GEQUAL:
568 qir_SF(c, qir_FSUB(c, normalized, compare));
569 depth_output = qir_SEL(c, QPU_COND_NS, u1, u0);
570 break;
571 case PIPE_FUNC_LESS:
572 qir_SF(c, qir_FSUB(c, compare, normalized));
573 depth_output = qir_SEL(c, QPU_COND_NS, u1, u0);
574 break;
575 case PIPE_FUNC_LEQUAL:
576 qir_SF(c, qir_FSUB(c, normalized, compare));
577 depth_output = qir_SEL(c, QPU_COND_NC, u1, u0);
578 break;
579 }
580 } else {
581 depth_output = normalized;
582 }
583
584 for (int i = 0; i < 4; i++)
585 dest[i] = depth_output;
586 } else {
587 for (int i = 0; i < 4; i++)
588 dest[i] = qir_UNPACK_8_F(c, tex, i);
589 }
590 }
591
592 /**
593 * Computes x - floor(x), which is tricky because our FTOI truncates (rounds
594 * to zero).
595 */
596 static struct qreg
597 ntq_ffract(struct vc4_compile *c, struct qreg src)
598 {
599 struct qreg trunc = qir_ITOF(c, qir_FTOI(c, src));
600 struct qreg diff = qir_FSUB(c, src, trunc);
601 qir_SF(c, diff);
602
603 qir_FADD_dest(c, diff,
604 diff, qir_uniform_f(c, 1.0))->cond = QPU_COND_NS;
605
606 return qir_MOV(c, diff);
607 }
608
609 /**
610 * Computes floor(x), which is tricky because our FTOI truncates (rounds to
611 * zero).
612 */
613 static struct qreg
614 ntq_ffloor(struct vc4_compile *c, struct qreg src)
615 {
616 struct qreg result = qir_ITOF(c, qir_FTOI(c, src));
617
618 /* This will be < 0 if we truncated and the truncation was of a value
619 * that was < 0 in the first place.
620 */
621 qir_SF(c, qir_FSUB(c, src, result));
622
623 struct qinst *sub = qir_FSUB_dest(c, result,
624 result, qir_uniform_f(c, 1.0));
625 sub->cond = QPU_COND_NS;
626
627 return qir_MOV(c, result);
628 }
629
630 /**
631 * Computes ceil(x), which is tricky because our FTOI truncates (rounds to
632 * zero).
633 */
634 static struct qreg
635 ntq_fceil(struct vc4_compile *c, struct qreg src)
636 {
637 struct qreg result = qir_ITOF(c, qir_FTOI(c, src));
638
639 /* This will be < 0 if we truncated and the truncation was of a value
640 * that was > 0 in the first place.
641 */
642 qir_SF(c, qir_FSUB(c, result, src));
643
644 qir_FADD_dest(c, result,
645 result, qir_uniform_f(c, 1.0))->cond = QPU_COND_NS;
646
647 return qir_MOV(c, result);
648 }
649
650 static struct qreg
651 ntq_fsin(struct vc4_compile *c, struct qreg src)
652 {
653 float coeff[] = {
654 -2.0 * M_PI,
655 pow(2.0 * M_PI, 3) / (3 * 2 * 1),
656 -pow(2.0 * M_PI, 5) / (5 * 4 * 3 * 2 * 1),
657 pow(2.0 * M_PI, 7) / (7 * 6 * 5 * 4 * 3 * 2 * 1),
658 -pow(2.0 * M_PI, 9) / (9 * 8 * 7 * 6 * 5 * 4 * 3 * 2 * 1),
659 };
660
661 struct qreg scaled_x =
662 qir_FMUL(c,
663 src,
664 qir_uniform_f(c, 1.0 / (M_PI * 2.0)));
665
666 struct qreg x = qir_FADD(c,
667 ntq_ffract(c, scaled_x),
668 qir_uniform_f(c, -0.5));
669 struct qreg x2 = qir_FMUL(c, x, x);
670 struct qreg sum = qir_FMUL(c, x, qir_uniform_f(c, coeff[0]));
671 for (int i = 1; i < ARRAY_SIZE(coeff); i++) {
672 x = qir_FMUL(c, x, x2);
673 sum = qir_FADD(c,
674 sum,
675 qir_FMUL(c,
676 x,
677 qir_uniform_f(c, coeff[i])));
678 }
679 return sum;
680 }
681
682 static struct qreg
683 ntq_fcos(struct vc4_compile *c, struct qreg src)
684 {
685 float coeff[] = {
686 -1.0f,
687 pow(2.0 * M_PI, 2) / (2 * 1),
688 -pow(2.0 * M_PI, 4) / (4 * 3 * 2 * 1),
689 pow(2.0 * M_PI, 6) / (6 * 5 * 4 * 3 * 2 * 1),
690 -pow(2.0 * M_PI, 8) / (8 * 7 * 6 * 5 * 4 * 3 * 2 * 1),
691 pow(2.0 * M_PI, 10) / (10 * 9 * 8 * 7 * 6 * 5 * 4 * 3 * 2 * 1),
692 };
693
694 struct qreg scaled_x =
695 qir_FMUL(c, src,
696 qir_uniform_f(c, 1.0f / (M_PI * 2.0f)));
697 struct qreg x_frac = qir_FADD(c,
698 ntq_ffract(c, scaled_x),
699 qir_uniform_f(c, -0.5));
700
701 struct qreg sum = qir_uniform_f(c, coeff[0]);
702 struct qreg x2 = qir_FMUL(c, x_frac, x_frac);
703 struct qreg x = x2; /* Current x^2, x^4, or x^6 */
704 for (int i = 1; i < ARRAY_SIZE(coeff); i++) {
705 if (i != 1)
706 x = qir_FMUL(c, x, x2);
707
708 struct qreg mul = qir_FMUL(c,
709 x,
710 qir_uniform_f(c, coeff[i]));
711 if (i == 0)
712 sum = mul;
713 else
714 sum = qir_FADD(c, sum, mul);
715 }
716 return sum;
717 }
718
719 static struct qreg
720 ntq_fsign(struct vc4_compile *c, struct qreg src)
721 {
722 struct qreg t = qir_get_temp(c);
723
724 qir_SF(c, src);
725 qir_MOV_dest(c, t, qir_uniform_f(c, 0.0));
726 qir_MOV_dest(c, t, qir_uniform_f(c, 1.0))->cond = QPU_COND_ZC;
727 qir_MOV_dest(c, t, qir_uniform_f(c, -1.0))->cond = QPU_COND_NS;
728 return qir_MOV(c, t);
729 }
730
731 static void
732 emit_vertex_input(struct vc4_compile *c, int attr)
733 {
734 enum pipe_format format = c->vs_key->attr_formats[attr];
735 uint32_t attr_size = util_format_get_blocksize(format);
736 uint32_t vpm_attr = c->next_vpm_input++;
737
738 c->vpm_input_order[vpm_attr] = attr;
739
740 c->vattr_sizes[vpm_attr] = align(attr_size, 4);
741 for (int i = 0; i < align(attr_size, 4) / 4; i++) {
742 c->inputs[attr * 4 + i] =
743 qir_MOV(c, qir_reg(QFILE_VPM, vpm_attr * 4 + i));
744 c->num_inputs++;
745 }
746 }
747
748 static void
749 emit_fragcoord_input(struct vc4_compile *c, int attr)
750 {
751 c->inputs[attr * 4 + 0] = qir_ITOF(c, qir_reg(QFILE_FRAG_X, 0));
752 c->inputs[attr * 4 + 1] = qir_ITOF(c, qir_reg(QFILE_FRAG_Y, 0));
753 c->inputs[attr * 4 + 2] =
754 qir_FMUL(c,
755 qir_ITOF(c, qir_FRAG_Z(c)),
756 qir_uniform_f(c, 1.0 / 0xffffff));
757 c->inputs[attr * 4 + 3] = qir_RCP(c, qir_FRAG_W(c));
758 }
759
760 static struct qreg
761 emit_fragment_varying(struct vc4_compile *c, gl_varying_slot slot,
762 uint8_t swizzle)
763 {
764 uint32_t i = c->num_input_slots++;
765 struct qreg vary = {
766 QFILE_VARY,
767 i
768 };
769
770 if (c->num_input_slots >= c->input_slots_array_size) {
771 c->input_slots_array_size =
772 MAX2(4, c->input_slots_array_size * 2);
773
774 c->input_slots = reralloc(c, c->input_slots,
775 struct vc4_varying_slot,
776 c->input_slots_array_size);
777 }
778
779 c->input_slots[i].slot = slot;
780 c->input_slots[i].swizzle = swizzle;
781
782 return qir_VARY_ADD_C(c, qir_FMUL(c, vary, qir_FRAG_W(c)));
783 }
784
785 static void
786 emit_fragment_input(struct vc4_compile *c, int attr, gl_varying_slot slot)
787 {
788 for (int i = 0; i < 4; i++) {
789 c->inputs[attr * 4 + i] =
790 emit_fragment_varying(c, slot, i);
791 c->num_inputs++;
792 }
793 }
794
795 static void
796 add_output(struct vc4_compile *c,
797 uint32_t decl_offset,
798 uint8_t slot,
799 uint8_t swizzle)
800 {
801 uint32_t old_array_size = c->outputs_array_size;
802 resize_qreg_array(c, &c->outputs, &c->outputs_array_size,
803 decl_offset + 1);
804
805 if (old_array_size != c->outputs_array_size) {
806 c->output_slots = reralloc(c,
807 c->output_slots,
808 struct vc4_varying_slot,
809 c->outputs_array_size);
810 }
811
812 c->output_slots[decl_offset].slot = slot;
813 c->output_slots[decl_offset].swizzle = swizzle;
814 }
815
816 static void
817 declare_uniform_range(struct vc4_compile *c, uint32_t start, uint32_t size)
818 {
819 unsigned array_id = c->num_uniform_ranges++;
820 if (array_id >= c->ubo_ranges_array_size) {
821 c->ubo_ranges_array_size = MAX2(c->ubo_ranges_array_size * 2,
822 array_id + 1);
823 c->ubo_ranges = reralloc(c, c->ubo_ranges,
824 struct vc4_compiler_ubo_range,
825 c->ubo_ranges_array_size);
826 }
827
828 c->ubo_ranges[array_id].dst_offset = 0;
829 c->ubo_ranges[array_id].src_offset = start;
830 c->ubo_ranges[array_id].size = size;
831 c->ubo_ranges[array_id].used = false;
832 }
833
834 static bool
835 ntq_src_is_only_ssa_def_user(nir_src *src)
836 {
837 if (!src->is_ssa)
838 return false;
839
840 if (!list_empty(&src->ssa->if_uses))
841 return false;
842
843 return (src->ssa->uses.next == &src->use_link &&
844 src->ssa->uses.next->next == &src->ssa->uses);
845 }
846
847 /**
848 * In general, emits a nir_pack_unorm_4x8 as a series of MOVs with the pack
849 * bit set.
850 *
851 * However, as an optimization, it tries to find the instructions generating
852 * the sources to be packed and just emit the pack flag there, if possible.
853 */
854 static void
855 ntq_emit_pack_unorm_4x8(struct vc4_compile *c, nir_alu_instr *instr)
856 {
857 struct qreg result = qir_get_temp(c);
858 struct nir_alu_instr *vec4 = NULL;
859
860 /* If packing from a vec4 op (as expected), identify it so that we can
861 * peek back at what generated its sources.
862 */
863 if (instr->src[0].src.is_ssa &&
864 instr->src[0].src.ssa->parent_instr->type == nir_instr_type_alu &&
865 nir_instr_as_alu(instr->src[0].src.ssa->parent_instr)->op ==
866 nir_op_vec4) {
867 vec4 = nir_instr_as_alu(instr->src[0].src.ssa->parent_instr);
868 }
869
870 /* If the pack is replicating the same channel 4 times, use the 8888
871 * pack flag. This is common for blending using the alpha
872 * channel.
873 */
874 if (instr->src[0].swizzle[0] == instr->src[0].swizzle[1] &&
875 instr->src[0].swizzle[0] == instr->src[0].swizzle[2] &&
876 instr->src[0].swizzle[0] == instr->src[0].swizzle[3]) {
877 struct qreg rep = ntq_get_src(c,
878 instr->src[0].src,
879 instr->src[0].swizzle[0]);
880 ntq_store_dest(c, &instr->dest.dest, 0, qir_PACK_8888_F(c, rep));
881 return;
882 }
883
884 for (int i = 0; i < 4; i++) {
885 int swiz = instr->src[0].swizzle[i];
886 struct qreg src;
887 if (vec4) {
888 src = ntq_get_src(c, vec4->src[swiz].src,
889 vec4->src[swiz].swizzle[0]);
890 } else {
891 src = ntq_get_src(c, instr->src[0].src, swiz);
892 }
893
894 if (vec4 &&
895 ntq_src_is_only_ssa_def_user(&vec4->src[swiz].src) &&
896 src.file == QFILE_TEMP &&
897 c->defs[src.index] &&
898 qir_is_mul(c->defs[src.index]) &&
899 !c->defs[src.index]->dst.pack) {
900 struct qinst *rewrite = c->defs[src.index];
901 c->defs[src.index] = NULL;
902 rewrite->dst = result;
903 rewrite->dst.pack = QPU_PACK_MUL_8A + i;
904 continue;
905 }
906
907 qir_PACK_8_F(c, result, src, i);
908 }
909
910 ntq_store_dest(c, &instr->dest.dest, 0, qir_MOV(c, result));
911 }
912
913 /** Handles sign-extended bitfield extracts for 16 bits. */
914 static struct qreg
915 ntq_emit_ibfe(struct vc4_compile *c, struct qreg base, struct qreg offset,
916 struct qreg bits)
917 {
918 assert(bits.file == QFILE_UNIF &&
919 c->uniform_contents[bits.index] == QUNIFORM_CONSTANT &&
920 c->uniform_data[bits.index] == 16);
921
922 assert(offset.file == QFILE_UNIF &&
923 c->uniform_contents[offset.index] == QUNIFORM_CONSTANT);
924 int offset_bit = c->uniform_data[offset.index];
925 assert(offset_bit % 16 == 0);
926
927 return qir_UNPACK_16_I(c, base, offset_bit / 16);
928 }
929
930 /** Handles unsigned bitfield extracts for 8 bits. */
931 static struct qreg
932 ntq_emit_ubfe(struct vc4_compile *c, struct qreg base, struct qreg offset,
933 struct qreg bits)
934 {
935 assert(bits.file == QFILE_UNIF &&
936 c->uniform_contents[bits.index] == QUNIFORM_CONSTANT &&
937 c->uniform_data[bits.index] == 8);
938
939 assert(offset.file == QFILE_UNIF &&
940 c->uniform_contents[offset.index] == QUNIFORM_CONSTANT);
941 int offset_bit = c->uniform_data[offset.index];
942 assert(offset_bit % 8 == 0);
943
944 return qir_UNPACK_8_I(c, base, offset_bit / 8);
945 }
946
947 /**
948 * If compare_instr is a valid comparison instruction, emits the
949 * compare_instr's comparison and returns the sel_instr's return value based
950 * on the compare_instr's result.
951 */
952 static bool
953 ntq_emit_comparison(struct vc4_compile *c, struct qreg *dest,
954 nir_alu_instr *compare_instr,
955 nir_alu_instr *sel_instr)
956 {
957 enum qpu_cond cond;
958
959 switch (compare_instr->op) {
960 case nir_op_feq:
961 case nir_op_ieq:
962 case nir_op_seq:
963 cond = QPU_COND_ZS;
964 break;
965 case nir_op_fne:
966 case nir_op_ine:
967 case nir_op_sne:
968 cond = QPU_COND_ZC;
969 break;
970 case nir_op_fge:
971 case nir_op_ige:
972 case nir_op_uge:
973 case nir_op_sge:
974 cond = QPU_COND_NC;
975 break;
976 case nir_op_flt:
977 case nir_op_ilt:
978 case nir_op_slt:
979 cond = QPU_COND_NS;
980 break;
981 default:
982 return false;
983 }
984
985 struct qreg src0 = ntq_get_alu_src(c, compare_instr, 0);
986 struct qreg src1 = ntq_get_alu_src(c, compare_instr, 1);
987
988 unsigned unsized_type =
989 nir_alu_type_get_base_type(nir_op_infos[compare_instr->op].input_types[0]);
990 if (unsized_type == nir_type_float)
991 qir_SF(c, qir_FSUB(c, src0, src1));
992 else
993 qir_SF(c, qir_SUB(c, src0, src1));
994
995 switch (sel_instr->op) {
996 case nir_op_seq:
997 case nir_op_sne:
998 case nir_op_sge:
999 case nir_op_slt:
1000 *dest = qir_SEL(c, cond,
1001 qir_uniform_f(c, 1.0), qir_uniform_f(c, 0.0));
1002 break;
1003
1004 case nir_op_bcsel:
1005 *dest = qir_SEL(c, cond,
1006 ntq_get_alu_src(c, sel_instr, 1),
1007 ntq_get_alu_src(c, sel_instr, 2));
1008 break;
1009
1010 default:
1011 *dest = qir_SEL(c, cond,
1012 qir_uniform_ui(c, ~0), qir_uniform_ui(c, 0));
1013 break;
1014 }
1015
1016 /* Make the temporary for nir_store_dest(). */
1017 *dest = qir_MOV(c, *dest);
1018
1019 return true;
1020 }
1021
1022 /**
1023 * Attempts to fold a comparison generating a boolean result into the
1024 * condition code for selecting between two values, instead of comparing the
1025 * boolean result against 0 to generate the condition code.
1026 */
1027 static struct qreg ntq_emit_bcsel(struct vc4_compile *c, nir_alu_instr *instr,
1028 struct qreg *src)
1029 {
1030 if (!instr->src[0].src.is_ssa)
1031 goto out;
1032 if (instr->src[0].src.ssa->parent_instr->type != nir_instr_type_alu)
1033 goto out;
1034 nir_alu_instr *compare =
1035 nir_instr_as_alu(instr->src[0].src.ssa->parent_instr);
1036 if (!compare)
1037 goto out;
1038
1039 struct qreg dest;
1040 if (ntq_emit_comparison(c, &dest, compare, instr))
1041 return dest;
1042
1043 out:
1044 qir_SF(c, src[0]);
1045 return qir_MOV(c, qir_SEL(c, QPU_COND_NS, src[1], src[2]));
1046 }
1047
1048 static struct qreg
1049 ntq_fddx(struct vc4_compile *c, struct qreg src)
1050 {
1051 /* Make sure that we have a bare temp to use for MUL rotation, so it
1052 * can be allocated to an accumulator.
1053 */
1054 if (src.pack || src.file != QFILE_TEMP)
1055 src = qir_MOV(c, src);
1056
1057 struct qreg from_left = qir_ROT_MUL(c, src, 1);
1058 struct qreg from_right = qir_ROT_MUL(c, src, 15);
1059
1060 /* Distinguish left/right pixels of the quad. */
1061 qir_SF(c, qir_AND(c, qir_reg(QFILE_QPU_ELEMENT, 0),
1062 qir_uniform_ui(c, 1)));
1063
1064 return qir_MOV(c, qir_SEL(c, QPU_COND_ZS,
1065 qir_FSUB(c, from_right, src),
1066 qir_FSUB(c, src, from_left)));
1067 }
1068
1069 static struct qreg
1070 ntq_fddy(struct vc4_compile *c, struct qreg src)
1071 {
1072 if (src.pack || src.file != QFILE_TEMP)
1073 src = qir_MOV(c, src);
1074
1075 struct qreg from_bottom = qir_ROT_MUL(c, src, 2);
1076 struct qreg from_top = qir_ROT_MUL(c, src, 14);
1077
1078 /* Distinguish top/bottom pixels of the quad. */
1079 qir_SF(c, qir_AND(c,
1080 qir_reg(QFILE_QPU_ELEMENT, 0),
1081 qir_uniform_ui(c, 2)));
1082
1083 return qir_MOV(c, qir_SEL(c, QPU_COND_ZS,
1084 qir_FSUB(c, from_top, src),
1085 qir_FSUB(c, src, from_bottom)));
1086 }
1087
1088 static void
1089 ntq_emit_alu(struct vc4_compile *c, nir_alu_instr *instr)
1090 {
1091 /* This should always be lowered to ALU operations for VC4. */
1092 assert(!instr->dest.saturate);
1093
1094 /* Vectors are special in that they have non-scalarized writemasks,
1095 * and just take the first swizzle channel for each argument in order
1096 * into each writemask channel.
1097 */
1098 if (instr->op == nir_op_vec2 ||
1099 instr->op == nir_op_vec3 ||
1100 instr->op == nir_op_vec4) {
1101 struct qreg srcs[4];
1102 for (int i = 0; i < nir_op_infos[instr->op].num_inputs; i++)
1103 srcs[i] = ntq_get_src(c, instr->src[i].src,
1104 instr->src[i].swizzle[0]);
1105 for (int i = 0; i < nir_op_infos[instr->op].num_inputs; i++)
1106 ntq_store_dest(c, &instr->dest.dest, i,
1107 qir_MOV(c, srcs[i]));
1108 return;
1109 }
1110
1111 if (instr->op == nir_op_pack_unorm_4x8) {
1112 ntq_emit_pack_unorm_4x8(c, instr);
1113 return;
1114 }
1115
1116 if (instr->op == nir_op_unpack_unorm_4x8) {
1117 struct qreg src = ntq_get_src(c, instr->src[0].src,
1118 instr->src[0].swizzle[0]);
1119 for (int i = 0; i < 4; i++) {
1120 if (instr->dest.write_mask & (1 << i))
1121 ntq_store_dest(c, &instr->dest.dest, i,
1122 qir_UNPACK_8_F(c, src, i));
1123 }
1124 return;
1125 }
1126
1127 /* General case: We can just grab the one used channel per src. */
1128 struct qreg src[nir_op_infos[instr->op].num_inputs];
1129 for (int i = 0; i < nir_op_infos[instr->op].num_inputs; i++) {
1130 src[i] = ntq_get_alu_src(c, instr, i);
1131 }
1132
1133 struct qreg result;
1134
1135 switch (instr->op) {
1136 case nir_op_fmov:
1137 case nir_op_imov:
1138 result = qir_MOV(c, src[0]);
1139 break;
1140 case nir_op_fmul:
1141 result = qir_FMUL(c, src[0], src[1]);
1142 break;
1143 case nir_op_fadd:
1144 result = qir_FADD(c, src[0], src[1]);
1145 break;
1146 case nir_op_fsub:
1147 result = qir_FSUB(c, src[0], src[1]);
1148 break;
1149 case nir_op_fmin:
1150 result = qir_FMIN(c, src[0], src[1]);
1151 break;
1152 case nir_op_fmax:
1153 result = qir_FMAX(c, src[0], src[1]);
1154 break;
1155
1156 case nir_op_f2i:
1157 case nir_op_f2u:
1158 result = qir_FTOI(c, src[0]);
1159 break;
1160 case nir_op_i2f:
1161 case nir_op_u2f:
1162 result = qir_ITOF(c, src[0]);
1163 break;
1164 case nir_op_b2f:
1165 result = qir_AND(c, src[0], qir_uniform_f(c, 1.0));
1166 break;
1167 case nir_op_b2i:
1168 result = qir_AND(c, src[0], qir_uniform_ui(c, 1));
1169 break;
1170 case nir_op_i2b:
1171 case nir_op_f2b:
1172 qir_SF(c, src[0]);
1173 result = qir_MOV(c, qir_SEL(c, QPU_COND_ZC,
1174 qir_uniform_ui(c, ~0),
1175 qir_uniform_ui(c, 0)));
1176 break;
1177
1178 case nir_op_iadd:
1179 result = qir_ADD(c, src[0], src[1]);
1180 break;
1181 case nir_op_ushr:
1182 result = qir_SHR(c, src[0], src[1]);
1183 break;
1184 case nir_op_isub:
1185 result = qir_SUB(c, src[0], src[1]);
1186 break;
1187 case nir_op_ishr:
1188 result = qir_ASR(c, src[0], src[1]);
1189 break;
1190 case nir_op_ishl:
1191 result = qir_SHL(c, src[0], src[1]);
1192 break;
1193 case nir_op_imin:
1194 result = qir_MIN(c, src[0], src[1]);
1195 break;
1196 case nir_op_imax:
1197 result = qir_MAX(c, src[0], src[1]);
1198 break;
1199 case nir_op_iand:
1200 result = qir_AND(c, src[0], src[1]);
1201 break;
1202 case nir_op_ior:
1203 result = qir_OR(c, src[0], src[1]);
1204 break;
1205 case nir_op_ixor:
1206 result = qir_XOR(c, src[0], src[1]);
1207 break;
1208 case nir_op_inot:
1209 result = qir_NOT(c, src[0]);
1210 break;
1211
1212 case nir_op_imul:
1213 result = ntq_umul(c, src[0], src[1]);
1214 break;
1215
1216 case nir_op_seq:
1217 case nir_op_sne:
1218 case nir_op_sge:
1219 case nir_op_slt:
1220 case nir_op_feq:
1221 case nir_op_fne:
1222 case nir_op_fge:
1223 case nir_op_flt:
1224 case nir_op_ieq:
1225 case nir_op_ine:
1226 case nir_op_ige:
1227 case nir_op_uge:
1228 case nir_op_ilt:
1229 if (!ntq_emit_comparison(c, &result, instr, instr)) {
1230 fprintf(stderr, "Bad comparison instruction\n");
1231 }
1232 break;
1233
1234 case nir_op_bcsel:
1235 result = ntq_emit_bcsel(c, instr, src);
1236 break;
1237 case nir_op_fcsel:
1238 qir_SF(c, src[0]);
1239 result = qir_MOV(c, qir_SEL(c, QPU_COND_ZC, src[1], src[2]));
1240 break;
1241
1242 case nir_op_frcp:
1243 result = ntq_rcp(c, src[0]);
1244 break;
1245 case nir_op_frsq:
1246 result = ntq_rsq(c, src[0]);
1247 break;
1248 case nir_op_fexp2:
1249 result = qir_EXP2(c, src[0]);
1250 break;
1251 case nir_op_flog2:
1252 result = qir_LOG2(c, src[0]);
1253 break;
1254
1255 case nir_op_ftrunc:
1256 result = qir_ITOF(c, qir_FTOI(c, src[0]));
1257 break;
1258 case nir_op_fceil:
1259 result = ntq_fceil(c, src[0]);
1260 break;
1261 case nir_op_ffract:
1262 result = ntq_ffract(c, src[0]);
1263 break;
1264 case nir_op_ffloor:
1265 result = ntq_ffloor(c, src[0]);
1266 break;
1267
1268 case nir_op_fsin:
1269 result = ntq_fsin(c, src[0]);
1270 break;
1271 case nir_op_fcos:
1272 result = ntq_fcos(c, src[0]);
1273 break;
1274
1275 case nir_op_fsign:
1276 result = ntq_fsign(c, src[0]);
1277 break;
1278
1279 case nir_op_fabs:
1280 result = qir_FMAXABS(c, src[0], src[0]);
1281 break;
1282 case nir_op_iabs:
1283 result = qir_MAX(c, src[0],
1284 qir_SUB(c, qir_uniform_ui(c, 0), src[0]));
1285 break;
1286
1287 case nir_op_ibitfield_extract:
1288 result = ntq_emit_ibfe(c, src[0], src[1], src[2]);
1289 break;
1290
1291 case nir_op_ubitfield_extract:
1292 result = ntq_emit_ubfe(c, src[0], src[1], src[2]);
1293 break;
1294
1295 case nir_op_usadd_4x8:
1296 result = qir_V8ADDS(c, src[0], src[1]);
1297 break;
1298
1299 case nir_op_ussub_4x8:
1300 result = qir_V8SUBS(c, src[0], src[1]);
1301 break;
1302
1303 case nir_op_umin_4x8:
1304 result = qir_V8MIN(c, src[0], src[1]);
1305 break;
1306
1307 case nir_op_umax_4x8:
1308 result = qir_V8MAX(c, src[0], src[1]);
1309 break;
1310
1311 case nir_op_umul_unorm_4x8:
1312 result = qir_V8MULD(c, src[0], src[1]);
1313 break;
1314
1315 case nir_op_fddx:
1316 case nir_op_fddx_coarse:
1317 case nir_op_fddx_fine:
1318 result = ntq_fddx(c, src[0]);
1319 break;
1320
1321 case nir_op_fddy:
1322 case nir_op_fddy_coarse:
1323 case nir_op_fddy_fine:
1324 result = ntq_fddy(c, src[0]);
1325 break;
1326
1327 default:
1328 fprintf(stderr, "unknown NIR ALU inst: ");
1329 nir_print_instr(&instr->instr, stderr);
1330 fprintf(stderr, "\n");
1331 abort();
1332 }
1333
1334 /* We have a scalar result, so the instruction should only have a
1335 * single channel written to.
1336 */
1337 assert(util_is_power_of_two(instr->dest.write_mask));
1338 ntq_store_dest(c, &instr->dest.dest,
1339 ffs(instr->dest.write_mask) - 1, result);
1340 }
1341
1342 static void
1343 emit_frag_end(struct vc4_compile *c)
1344 {
1345 struct qreg color;
1346 if (c->output_color_index != -1) {
1347 color = c->outputs[c->output_color_index];
1348 } else {
1349 color = qir_uniform_ui(c, 0);
1350 }
1351
1352 uint32_t discard_cond = QPU_COND_ALWAYS;
1353 if (c->s->info->fs.uses_discard) {
1354 qir_SF(c, c->discard);
1355 discard_cond = QPU_COND_ZS;
1356 }
1357
1358 if (c->fs_key->stencil_enabled) {
1359 qir_MOV_dest(c, qir_reg(QFILE_TLB_STENCIL_SETUP, 0),
1360 qir_uniform(c, QUNIFORM_STENCIL, 0));
1361 if (c->fs_key->stencil_twoside) {
1362 qir_MOV_dest(c, qir_reg(QFILE_TLB_STENCIL_SETUP, 0),
1363 qir_uniform(c, QUNIFORM_STENCIL, 1));
1364 }
1365 if (c->fs_key->stencil_full_writemasks) {
1366 qir_MOV_dest(c, qir_reg(QFILE_TLB_STENCIL_SETUP, 0),
1367 qir_uniform(c, QUNIFORM_STENCIL, 2));
1368 }
1369 }
1370
1371 if (c->output_sample_mask_index != -1) {
1372 qir_MS_MASK(c, c->outputs[c->output_sample_mask_index]);
1373 }
1374
1375 if (c->fs_key->depth_enabled) {
1376 if (c->output_position_index != -1) {
1377 qir_FTOI_dest(c, qir_reg(QFILE_TLB_Z_WRITE, 0),
1378 qir_FMUL(c,
1379 c->outputs[c->output_position_index],
1380 qir_uniform_f(c, 0xffffff)))->cond = discard_cond;
1381 } else {
1382 qir_MOV_dest(c, qir_reg(QFILE_TLB_Z_WRITE, 0),
1383 qir_FRAG_Z(c))->cond = discard_cond;
1384 }
1385 }
1386
1387 if (!c->msaa_per_sample_output) {
1388 qir_MOV_dest(c, qir_reg(QFILE_TLB_COLOR_WRITE, 0),
1389 color)->cond = discard_cond;
1390 } else {
1391 for (int i = 0; i < VC4_MAX_SAMPLES; i++) {
1392 qir_MOV_dest(c, qir_reg(QFILE_TLB_COLOR_WRITE_MS, 0),
1393 c->sample_colors[i])->cond = discard_cond;
1394 }
1395 }
1396 }
1397
1398 static void
1399 emit_scaled_viewport_write(struct vc4_compile *c, struct qreg rcp_w)
1400 {
1401 struct qreg packed = qir_get_temp(c);
1402
1403 for (int i = 0; i < 2; i++) {
1404 struct qreg scale =
1405 qir_uniform(c, QUNIFORM_VIEWPORT_X_SCALE + i, 0);
1406
1407 struct qreg packed_chan = packed;
1408 packed_chan.pack = QPU_PACK_A_16A + i;
1409
1410 qir_FTOI_dest(c, packed_chan,
1411 qir_FMUL(c,
1412 qir_FMUL(c,
1413 c->outputs[c->output_position_index + i],
1414 scale),
1415 rcp_w));
1416 }
1417
1418 qir_VPM_WRITE(c, packed);
1419 }
1420
1421 static void
1422 emit_zs_write(struct vc4_compile *c, struct qreg rcp_w)
1423 {
1424 struct qreg zscale = qir_uniform(c, QUNIFORM_VIEWPORT_Z_SCALE, 0);
1425 struct qreg zoffset = qir_uniform(c, QUNIFORM_VIEWPORT_Z_OFFSET, 0);
1426
1427 qir_VPM_WRITE(c, qir_FADD(c, qir_FMUL(c, qir_FMUL(c,
1428 c->outputs[c->output_position_index + 2],
1429 zscale),
1430 rcp_w),
1431 zoffset));
1432 }
1433
1434 static void
1435 emit_rcp_wc_write(struct vc4_compile *c, struct qreg rcp_w)
1436 {
1437 qir_VPM_WRITE(c, rcp_w);
1438 }
1439
1440 static void
1441 emit_point_size_write(struct vc4_compile *c)
1442 {
1443 struct qreg point_size;
1444
1445 if (c->output_point_size_index != -1)
1446 point_size = c->outputs[c->output_point_size_index];
1447 else
1448 point_size = qir_uniform_f(c, 1.0);
1449
1450 /* Workaround: HW-2726 PTB does not handle zero-size points (BCM2835,
1451 * BCM21553).
1452 */
1453 point_size = qir_FMAX(c, point_size, qir_uniform_f(c, .125));
1454
1455 qir_VPM_WRITE(c, point_size);
1456 }
1457
1458 /**
1459 * Emits a VPM read of the stub vertex attribute set up by vc4_draw.c.
1460 *
1461 * The simulator insists that there be at least one vertex attribute, so
1462 * vc4_draw.c will emit one if it wouldn't have otherwise. The simulator also
1463 * insists that all vertex attributes loaded get read by the VS/CS, so we have
1464 * to consume it here.
1465 */
1466 static void
1467 emit_stub_vpm_read(struct vc4_compile *c)
1468 {
1469 if (c->num_inputs)
1470 return;
1471
1472 c->next_vpm_input++;
1473 c->vattr_sizes[0] = 4;
1474 (void)qir_MOV(c, qir_reg(QFILE_VPM, 0));
1475 c->num_inputs++;
1476 }
1477
1478 static void
1479 emit_vert_end(struct vc4_compile *c,
1480 struct vc4_varying_slot *fs_inputs,
1481 uint32_t num_fs_inputs)
1482 {
1483 struct qreg rcp_w = ntq_rcp(c, c->outputs[c->output_position_index + 3]);
1484
1485 emit_stub_vpm_read(c);
1486
1487 emit_scaled_viewport_write(c, rcp_w);
1488 emit_zs_write(c, rcp_w);
1489 emit_rcp_wc_write(c, rcp_w);
1490 if (c->vs_key->per_vertex_point_size)
1491 emit_point_size_write(c);
1492
1493 for (int i = 0; i < num_fs_inputs; i++) {
1494 struct vc4_varying_slot *input = &fs_inputs[i];
1495 int j;
1496
1497 for (j = 0; j < c->num_outputs; j++) {
1498 struct vc4_varying_slot *output =
1499 &c->output_slots[j];
1500
1501 if (input->slot == output->slot &&
1502 input->swizzle == output->swizzle) {
1503 qir_VPM_WRITE(c, c->outputs[j]);
1504 break;
1505 }
1506 }
1507 /* Emit padding if we didn't find a declared VS output for
1508 * this FS input.
1509 */
1510 if (j == c->num_outputs)
1511 qir_VPM_WRITE(c, qir_uniform_f(c, 0.0));
1512 }
1513 }
1514
1515 static void
1516 emit_coord_end(struct vc4_compile *c)
1517 {
1518 struct qreg rcp_w = ntq_rcp(c, c->outputs[c->output_position_index + 3]);
1519
1520 emit_stub_vpm_read(c);
1521
1522 for (int i = 0; i < 4; i++)
1523 qir_VPM_WRITE(c, c->outputs[c->output_position_index + i]);
1524
1525 emit_scaled_viewport_write(c, rcp_w);
1526 emit_zs_write(c, rcp_w);
1527 emit_rcp_wc_write(c, rcp_w);
1528 if (c->vs_key->per_vertex_point_size)
1529 emit_point_size_write(c);
1530 }
1531
1532 static void
1533 vc4_optimize_nir(struct nir_shader *s)
1534 {
1535 bool progress;
1536
1537 do {
1538 progress = false;
1539
1540 NIR_PASS_V(s, nir_lower_vars_to_ssa);
1541 NIR_PASS(progress, s, nir_lower_alu_to_scalar);
1542 NIR_PASS(progress, s, nir_lower_phis_to_scalar);
1543 NIR_PASS(progress, s, nir_copy_prop);
1544 NIR_PASS(progress, s, nir_opt_remove_phis);
1545 NIR_PASS(progress, s, nir_opt_dce);
1546 NIR_PASS(progress, s, nir_opt_dead_cf);
1547 NIR_PASS(progress, s, nir_opt_cse);
1548 NIR_PASS(progress, s, nir_opt_peephole_select, 8);
1549 NIR_PASS(progress, s, nir_opt_algebraic);
1550 NIR_PASS(progress, s, nir_opt_constant_folding);
1551 NIR_PASS(progress, s, nir_opt_undef);
1552 NIR_PASS(progress, s, nir_opt_loop_unroll,
1553 nir_var_shader_in |
1554 nir_var_shader_out |
1555 nir_var_local);
1556 } while (progress);
1557 }
1558
1559 static void
1560 ntq_setup_outputs(struct vc4_compile *c)
1561 {
1562 nir_foreach_variable(var, &c->s->outputs) {
1563 unsigned array_len = MAX2(glsl_get_length(var->type), 1);
1564 unsigned loc = var->data.driver_location * 4;
1565
1566 assert(array_len == 1);
1567 (void)array_len;
1568
1569 for (int i = 0; i < 4; i++)
1570 add_output(c, loc + i, var->data.location, i);
1571
1572 if (c->stage == QSTAGE_FRAG) {
1573 switch (var->data.location) {
1574 case FRAG_RESULT_COLOR:
1575 case FRAG_RESULT_DATA0:
1576 c->output_color_index = loc;
1577 break;
1578 case FRAG_RESULT_DEPTH:
1579 c->output_position_index = loc;
1580 break;
1581 case FRAG_RESULT_SAMPLE_MASK:
1582 c->output_sample_mask_index = loc;
1583 break;
1584 }
1585 } else {
1586 switch (var->data.location) {
1587 case VARYING_SLOT_POS:
1588 c->output_position_index = loc;
1589 break;
1590 case VARYING_SLOT_PSIZ:
1591 c->output_point_size_index = loc;
1592 break;
1593 }
1594 }
1595 }
1596 }
1597
1598 static void
1599 ntq_setup_uniforms(struct vc4_compile *c)
1600 {
1601 nir_foreach_variable(var, &c->s->uniforms) {
1602 uint32_t vec4_count = st_glsl_type_size(var->type);
1603 unsigned vec4_size = 4 * sizeof(float);
1604
1605 declare_uniform_range(c, var->data.driver_location * vec4_size,
1606 vec4_count * vec4_size);
1607
1608 }
1609 }
1610
1611 /**
1612 * Sets up the mapping from nir_register to struct qreg *.
1613 *
1614 * Each nir_register gets a struct qreg per 32-bit component being stored.
1615 */
1616 static void
1617 ntq_setup_registers(struct vc4_compile *c, struct exec_list *list)
1618 {
1619 foreach_list_typed(nir_register, nir_reg, node, list) {
1620 unsigned array_len = MAX2(nir_reg->num_array_elems, 1);
1621 struct qreg *qregs = ralloc_array(c->def_ht, struct qreg,
1622 array_len *
1623 nir_reg->num_components);
1624
1625 _mesa_hash_table_insert(c->def_ht, nir_reg, qregs);
1626
1627 for (int i = 0; i < array_len * nir_reg->num_components; i++)
1628 qregs[i] = qir_get_temp(c);
1629 }
1630 }
1631
1632 static void
1633 ntq_emit_load_const(struct vc4_compile *c, nir_load_const_instr *instr)
1634 {
1635 struct qreg *qregs = ntq_init_ssa_def(c, &instr->def);
1636 for (int i = 0; i < instr->def.num_components; i++)
1637 qregs[i] = qir_uniform_ui(c, instr->value.u32[i]);
1638
1639 _mesa_hash_table_insert(c->def_ht, &instr->def, qregs);
1640 }
1641
1642 static void
1643 ntq_emit_ssa_undef(struct vc4_compile *c, nir_ssa_undef_instr *instr)
1644 {
1645 struct qreg *qregs = ntq_init_ssa_def(c, &instr->def);
1646
1647 /* QIR needs there to be *some* value, so pick 0 (same as for
1648 * ntq_setup_registers().
1649 */
1650 for (int i = 0; i < instr->def.num_components; i++)
1651 qregs[i] = qir_uniform_ui(c, 0);
1652 }
1653
1654 static void
1655 ntq_emit_color_read(struct vc4_compile *c, nir_intrinsic_instr *instr)
1656 {
1657 nir_const_value *const_offset = nir_src_as_const_value(instr->src[0]);
1658 assert(const_offset->u32[0] == 0);
1659
1660 /* Reads of the per-sample color need to be done in
1661 * order.
1662 */
1663 int sample_index = (nir_intrinsic_base(instr) -
1664 VC4_NIR_TLB_COLOR_READ_INPUT);
1665 for (int i = 0; i <= sample_index; i++) {
1666 if (c->color_reads[i].file == QFILE_NULL) {
1667 c->color_reads[i] =
1668 qir_TLB_COLOR_READ(c);
1669 }
1670 }
1671 ntq_store_dest(c, &instr->dest, 0,
1672 qir_MOV(c, c->color_reads[sample_index]));
1673 }
1674
1675 static void
1676 ntq_emit_load_input(struct vc4_compile *c, nir_intrinsic_instr *instr)
1677 {
1678 assert(instr->num_components == 1);
1679
1680 nir_const_value *const_offset = nir_src_as_const_value(instr->src[0]);
1681 assert(const_offset && "vc4 doesn't support indirect inputs");
1682
1683 if (c->stage == QSTAGE_FRAG &&
1684 nir_intrinsic_base(instr) >= VC4_NIR_TLB_COLOR_READ_INPUT) {
1685 ntq_emit_color_read(c, instr);
1686 return;
1687 }
1688
1689 /* Size our inputs array as far as this input. Input arrays are
1690 * small, and we don't have a shader_info field that tells us up front
1691 * what the maximum driver_location is.
1692 */
1693 uint32_t loc = nir_intrinsic_base(instr) + const_offset->u32[0];
1694 if ((loc + 1) * 4 > c->inputs_array_size) {
1695 resize_qreg_array(c, &c->inputs, &c->inputs_array_size,
1696 (loc + 1) * 4);
1697 }
1698
1699 /* If we've already loaded this input, just return it. This would
1700 * happen for VPM loads, where we load an entire vertex attribute at
1701 * once, or possibly also in the FS if we haven't CSEed away repeated
1702 * loads.
1703 */
1704 int comp = nir_intrinsic_component(instr);
1705 if (c->inputs[loc * 4 + comp].file != QFILE_NULL) {
1706 ntq_store_dest(c, &instr->dest, 0,
1707 qir_MOV(c, c->inputs[loc * 4 + comp]));
1708 return;
1709 }
1710
1711 /* In the FS, we always have to fully drain our FS FIFO before
1712 * terminating the shader. For the VS we only have to drain whatever
1713 * VPM setup we configure, but vc4_qpu_emit.c configures it for the
1714 * entire vertex attribute space. Because of this, we emit our lazy
1715 * varying/VPM loads at the last top level basic block.
1716 */
1717 struct qblock *saved_cur_block = c->cur_block;
1718 c->cur_block = c->last_top_block;
1719
1720 /* Look up the NIR variable for this input, so we can see how big the
1721 * input is, or what sort of interpolation is necessary.
1722 */
1723 nir_variable *var = NULL;
1724 nir_foreach_variable(search_var, &c->s->inputs) {
1725 unsigned search_len = MAX2(glsl_get_length(search_var->type), 1);
1726 unsigned search_loc = search_var->data.driver_location;
1727
1728 if (loc >= search_loc && loc < search_loc + search_len) {
1729 var = search_var;
1730 break;
1731 }
1732 }
1733 assert(var);
1734
1735 if (c->stage == QSTAGE_FRAG) {
1736 if (var->data.location == VARYING_SLOT_POS) {
1737 emit_fragcoord_input(c, loc);
1738 } else if (var->data.location == VARYING_SLOT_PNTC ||
1739 (var->data.location >= VARYING_SLOT_VAR0 &&
1740 (c->fs_key->point_sprite_mask &
1741 (1 << (var->data.location -
1742 VARYING_SLOT_VAR0))))) {
1743 c->inputs[loc * 4 + 0] = c->point_x;
1744 c->inputs[loc * 4 + 1] = c->point_y;
1745 } else {
1746 emit_fragment_input(c, loc, var->data.location);
1747 }
1748 } else {
1749 emit_vertex_input(c, loc);
1750 }
1751
1752 c->cur_block = saved_cur_block;
1753
1754 ntq_store_dest(c, &instr->dest, 0,
1755 qir_MOV(c, c->inputs[loc * 4 + comp]));
1756 }
1757
1758 static void
1759 ntq_emit_intrinsic(struct vc4_compile *c, nir_intrinsic_instr *instr)
1760 {
1761 nir_const_value *const_offset;
1762 unsigned offset;
1763
1764 switch (instr->intrinsic) {
1765 case nir_intrinsic_load_uniform:
1766 assert(instr->num_components == 1);
1767 const_offset = nir_src_as_const_value(instr->src[0]);
1768 if (const_offset) {
1769 offset = nir_intrinsic_base(instr) + const_offset->u32[0];
1770 assert(offset % 4 == 0);
1771 /* We need dwords */
1772 offset = offset / 4;
1773 ntq_store_dest(c, &instr->dest, 0,
1774 qir_uniform(c, QUNIFORM_UNIFORM,
1775 offset));
1776 } else {
1777 ntq_store_dest(c, &instr->dest, 0,
1778 indirect_uniform_load(c, instr));
1779 }
1780 break;
1781
1782 case nir_intrinsic_load_user_clip_plane:
1783 for (int i = 0; i < instr->num_components; i++) {
1784 ntq_store_dest(c, &instr->dest, i,
1785 qir_uniform(c, QUNIFORM_USER_CLIP_PLANE,
1786 nir_intrinsic_ucp_id(instr) *
1787 4 + i));
1788 }
1789 break;
1790
1791 case nir_intrinsic_load_blend_const_color_r_float:
1792 case nir_intrinsic_load_blend_const_color_g_float:
1793 case nir_intrinsic_load_blend_const_color_b_float:
1794 case nir_intrinsic_load_blend_const_color_a_float:
1795 ntq_store_dest(c, &instr->dest, 0,
1796 qir_uniform(c, QUNIFORM_BLEND_CONST_COLOR_X +
1797 (instr->intrinsic -
1798 nir_intrinsic_load_blend_const_color_r_float),
1799 0));
1800 break;
1801
1802 case nir_intrinsic_load_blend_const_color_rgba8888_unorm:
1803 ntq_store_dest(c, &instr->dest, 0,
1804 qir_uniform(c, QUNIFORM_BLEND_CONST_COLOR_RGBA,
1805 0));
1806 break;
1807
1808 case nir_intrinsic_load_blend_const_color_aaaa8888_unorm:
1809 ntq_store_dest(c, &instr->dest, 0,
1810 qir_uniform(c, QUNIFORM_BLEND_CONST_COLOR_AAAA,
1811 0));
1812 break;
1813
1814 case nir_intrinsic_load_alpha_ref_float:
1815 ntq_store_dest(c, &instr->dest, 0,
1816 qir_uniform(c, QUNIFORM_ALPHA_REF, 0));
1817 break;
1818
1819 case nir_intrinsic_load_sample_mask_in:
1820 ntq_store_dest(c, &instr->dest, 0,
1821 qir_uniform(c, QUNIFORM_SAMPLE_MASK, 0));
1822 break;
1823
1824 case nir_intrinsic_load_front_face:
1825 /* The register contains 0 (front) or 1 (back), and we need to
1826 * turn it into a NIR bool where true means front.
1827 */
1828 ntq_store_dest(c, &instr->dest, 0,
1829 qir_ADD(c,
1830 qir_uniform_ui(c, -1),
1831 qir_reg(QFILE_FRAG_REV_FLAG, 0)));
1832 break;
1833
1834 case nir_intrinsic_load_input:
1835 ntq_emit_load_input(c, instr);
1836 break;
1837
1838 case nir_intrinsic_store_output:
1839 const_offset = nir_src_as_const_value(instr->src[1]);
1840 assert(const_offset && "vc4 doesn't support indirect outputs");
1841 offset = nir_intrinsic_base(instr) + const_offset->u32[0];
1842
1843 /* MSAA color outputs are the only case where we have an
1844 * output that's not lowered to being a store of a single 32
1845 * bit value.
1846 */
1847 if (c->stage == QSTAGE_FRAG && instr->num_components == 4) {
1848 assert(offset == c->output_color_index);
1849 for (int i = 0; i < 4; i++) {
1850 c->sample_colors[i] =
1851 qir_MOV(c, ntq_get_src(c, instr->src[0],
1852 i));
1853 }
1854 } else {
1855 offset = offset * 4 + nir_intrinsic_component(instr);
1856 assert(instr->num_components == 1);
1857 c->outputs[offset] =
1858 qir_MOV(c, ntq_get_src(c, instr->src[0], 0));
1859 c->num_outputs = MAX2(c->num_outputs, offset + 1);
1860 }
1861 break;
1862
1863 case nir_intrinsic_discard:
1864 if (c->execute.file != QFILE_NULL) {
1865 qir_SF(c, c->execute);
1866 qir_MOV_cond(c, QPU_COND_ZS, c->discard,
1867 qir_uniform_ui(c, ~0));
1868 } else {
1869 qir_MOV_dest(c, c->discard, qir_uniform_ui(c, ~0));
1870 }
1871 break;
1872
1873 case nir_intrinsic_discard_if: {
1874 /* true (~0) if we're discarding */
1875 struct qreg cond = ntq_get_src(c, instr->src[0], 0);
1876
1877 if (c->execute.file != QFILE_NULL) {
1878 /* execute == 0 means the channel is active. Invert
1879 * the condition so that we can use zero as "executing
1880 * and discarding."
1881 */
1882 qir_SF(c, qir_AND(c, c->execute, qir_NOT(c, cond)));
1883 qir_MOV_cond(c, QPU_COND_ZS, c->discard, cond);
1884 } else {
1885 qir_OR_dest(c, c->discard, c->discard,
1886 ntq_get_src(c, instr->src[0], 0));
1887 }
1888
1889 break;
1890 }
1891
1892 default:
1893 fprintf(stderr, "Unknown intrinsic: ");
1894 nir_print_instr(&instr->instr, stderr);
1895 fprintf(stderr, "\n");
1896 break;
1897 }
1898 }
1899
1900 /* Clears (activates) the execute flags for any channels whose jump target
1901 * matches this block.
1902 */
1903 static void
1904 ntq_activate_execute_for_block(struct vc4_compile *c)
1905 {
1906 qir_SF(c, qir_SUB(c,
1907 c->execute,
1908 qir_uniform_ui(c, c->cur_block->index)));
1909 qir_MOV_cond(c, QPU_COND_ZS, c->execute, qir_uniform_ui(c, 0));
1910 }
1911
1912 static void
1913 ntq_emit_if(struct vc4_compile *c, nir_if *if_stmt)
1914 {
1915 if (!c->vc4->screen->has_control_flow) {
1916 fprintf(stderr,
1917 "IF statement support requires updated kernel.\n");
1918 return;
1919 }
1920
1921 nir_block *nir_else_block = nir_if_first_else_block(if_stmt);
1922 bool empty_else_block =
1923 (nir_else_block == nir_if_last_else_block(if_stmt) &&
1924 exec_list_is_empty(&nir_else_block->instr_list));
1925
1926 struct qblock *then_block = qir_new_block(c);
1927 struct qblock *after_block = qir_new_block(c);
1928 struct qblock *else_block;
1929 if (empty_else_block)
1930 else_block = after_block;
1931 else
1932 else_block = qir_new_block(c);
1933
1934 bool was_top_level = false;
1935 if (c->execute.file == QFILE_NULL) {
1936 c->execute = qir_MOV(c, qir_uniform_ui(c, 0));
1937 was_top_level = true;
1938 }
1939
1940 /* Set ZS for executing (execute == 0) and jumping (if->condition ==
1941 * 0) channels, and then update execute flags for those to point to
1942 * the ELSE block.
1943 */
1944 qir_SF(c, qir_OR(c,
1945 c->execute,
1946 ntq_get_src(c, if_stmt->condition, 0)));
1947 qir_MOV_cond(c, QPU_COND_ZS, c->execute,
1948 qir_uniform_ui(c, else_block->index));
1949
1950 /* Jump to ELSE if nothing is active for THEN, otherwise fall
1951 * through.
1952 */
1953 qir_SF(c, c->execute);
1954 qir_BRANCH(c, QPU_COND_BRANCH_ALL_ZC);
1955 qir_link_blocks(c->cur_block, else_block);
1956 qir_link_blocks(c->cur_block, then_block);
1957
1958 /* Process the THEN block. */
1959 qir_set_emit_block(c, then_block);
1960 ntq_emit_cf_list(c, &if_stmt->then_list);
1961
1962 if (!empty_else_block) {
1963 /* Handle the end of the THEN block. First, all currently
1964 * active channels update their execute flags to point to
1965 * ENDIF
1966 */
1967 qir_SF(c, c->execute);
1968 qir_MOV_cond(c, QPU_COND_ZS, c->execute,
1969 qir_uniform_ui(c, after_block->index));
1970
1971 /* If everything points at ENDIF, then jump there immediately. */
1972 qir_SF(c, qir_SUB(c, c->execute, qir_uniform_ui(c, after_block->index)));
1973 qir_BRANCH(c, QPU_COND_BRANCH_ALL_ZS);
1974 qir_link_blocks(c->cur_block, after_block);
1975 qir_link_blocks(c->cur_block, else_block);
1976
1977 qir_set_emit_block(c, else_block);
1978 ntq_activate_execute_for_block(c);
1979 ntq_emit_cf_list(c, &if_stmt->else_list);
1980 }
1981
1982 qir_link_blocks(c->cur_block, after_block);
1983
1984 qir_set_emit_block(c, after_block);
1985 if (was_top_level) {
1986 c->execute = c->undef;
1987 c->last_top_block = c->cur_block;
1988 } else {
1989 ntq_activate_execute_for_block(c);
1990 }
1991 }
1992
1993 static void
1994 ntq_emit_jump(struct vc4_compile *c, nir_jump_instr *jump)
1995 {
1996 struct qblock *jump_block;
1997 switch (jump->type) {
1998 case nir_jump_break:
1999 jump_block = c->loop_break_block;
2000 break;
2001 case nir_jump_continue:
2002 jump_block = c->loop_cont_block;
2003 break;
2004 default:
2005 unreachable("Unsupported jump type\n");
2006 }
2007
2008 qir_SF(c, c->execute);
2009 qir_MOV_cond(c, QPU_COND_ZS, c->execute,
2010 qir_uniform_ui(c, jump_block->index));
2011
2012 /* Jump to the destination block if everyone has taken the jump. */
2013 qir_SF(c, qir_SUB(c, c->execute, qir_uniform_ui(c, jump_block->index)));
2014 qir_BRANCH(c, QPU_COND_BRANCH_ALL_ZS);
2015 struct qblock *new_block = qir_new_block(c);
2016 qir_link_blocks(c->cur_block, jump_block);
2017 qir_link_blocks(c->cur_block, new_block);
2018 qir_set_emit_block(c, new_block);
2019 }
2020
2021 static void
2022 ntq_emit_instr(struct vc4_compile *c, nir_instr *instr)
2023 {
2024 switch (instr->type) {
2025 case nir_instr_type_alu:
2026 ntq_emit_alu(c, nir_instr_as_alu(instr));
2027 break;
2028
2029 case nir_instr_type_intrinsic:
2030 ntq_emit_intrinsic(c, nir_instr_as_intrinsic(instr));
2031 break;
2032
2033 case nir_instr_type_load_const:
2034 ntq_emit_load_const(c, nir_instr_as_load_const(instr));
2035 break;
2036
2037 case nir_instr_type_ssa_undef:
2038 ntq_emit_ssa_undef(c, nir_instr_as_ssa_undef(instr));
2039 break;
2040
2041 case nir_instr_type_tex:
2042 ntq_emit_tex(c, nir_instr_as_tex(instr));
2043 break;
2044
2045 case nir_instr_type_jump:
2046 ntq_emit_jump(c, nir_instr_as_jump(instr));
2047 break;
2048
2049 default:
2050 fprintf(stderr, "Unknown NIR instr type: ");
2051 nir_print_instr(instr, stderr);
2052 fprintf(stderr, "\n");
2053 abort();
2054 }
2055 }
2056
2057 static void
2058 ntq_emit_block(struct vc4_compile *c, nir_block *block)
2059 {
2060 nir_foreach_instr(instr, block) {
2061 ntq_emit_instr(c, instr);
2062 }
2063 }
2064
2065 static void ntq_emit_cf_list(struct vc4_compile *c, struct exec_list *list);
2066
2067 static void
2068 ntq_emit_loop(struct vc4_compile *c, nir_loop *loop)
2069 {
2070 if (!c->vc4->screen->has_control_flow) {
2071 fprintf(stderr,
2072 "loop support requires updated kernel.\n");
2073 ntq_emit_cf_list(c, &loop->body);
2074 return;
2075 }
2076
2077 bool was_top_level = false;
2078 if (c->execute.file == QFILE_NULL) {
2079 c->execute = qir_MOV(c, qir_uniform_ui(c, 0));
2080 was_top_level = true;
2081 }
2082
2083 struct qblock *save_loop_cont_block = c->loop_cont_block;
2084 struct qblock *save_loop_break_block = c->loop_break_block;
2085
2086 c->loop_cont_block = qir_new_block(c);
2087 c->loop_break_block = qir_new_block(c);
2088
2089 qir_link_blocks(c->cur_block, c->loop_cont_block);
2090 qir_set_emit_block(c, c->loop_cont_block);
2091 ntq_activate_execute_for_block(c);
2092
2093 ntq_emit_cf_list(c, &loop->body);
2094
2095 /* If anything had explicitly continued, or is here at the end of the
2096 * loop, then we need to loop again. SF updates are masked by the
2097 * instruction's condition, so we can do the OR of the two conditions
2098 * within SF.
2099 */
2100 qir_SF(c, c->execute);
2101 struct qinst *cont_check =
2102 qir_SUB_dest(c,
2103 c->undef,
2104 c->execute,
2105 qir_uniform_ui(c, c->loop_cont_block->index));
2106 cont_check->cond = QPU_COND_ZC;
2107 cont_check->sf = true;
2108
2109 qir_BRANCH(c, QPU_COND_BRANCH_ANY_ZS);
2110 qir_link_blocks(c->cur_block, c->loop_cont_block);
2111 qir_link_blocks(c->cur_block, c->loop_break_block);
2112
2113 qir_set_emit_block(c, c->loop_break_block);
2114 if (was_top_level) {
2115 c->execute = c->undef;
2116 c->last_top_block = c->cur_block;
2117 } else {
2118 ntq_activate_execute_for_block(c);
2119 }
2120
2121 c->loop_break_block = save_loop_break_block;
2122 c->loop_cont_block = save_loop_cont_block;
2123 }
2124
2125 static void
2126 ntq_emit_function(struct vc4_compile *c, nir_function_impl *func)
2127 {
2128 fprintf(stderr, "FUNCTIONS not handled.\n");
2129 abort();
2130 }
2131
2132 static void
2133 ntq_emit_cf_list(struct vc4_compile *c, struct exec_list *list)
2134 {
2135 foreach_list_typed(nir_cf_node, node, node, list) {
2136 switch (node->type) {
2137 case nir_cf_node_block:
2138 ntq_emit_block(c, nir_cf_node_as_block(node));
2139 break;
2140
2141 case nir_cf_node_if:
2142 ntq_emit_if(c, nir_cf_node_as_if(node));
2143 break;
2144
2145 case nir_cf_node_loop:
2146 ntq_emit_loop(c, nir_cf_node_as_loop(node));
2147 break;
2148
2149 case nir_cf_node_function:
2150 ntq_emit_function(c, nir_cf_node_as_function(node));
2151 break;
2152
2153 default:
2154 fprintf(stderr, "Unknown NIR node type\n");
2155 abort();
2156 }
2157 }
2158 }
2159
2160 static void
2161 ntq_emit_impl(struct vc4_compile *c, nir_function_impl *impl)
2162 {
2163 ntq_setup_registers(c, &impl->registers);
2164 ntq_emit_cf_list(c, &impl->body);
2165 }
2166
2167 static void
2168 nir_to_qir(struct vc4_compile *c)
2169 {
2170 if (c->stage == QSTAGE_FRAG && c->s->info->fs.uses_discard)
2171 c->discard = qir_MOV(c, qir_uniform_ui(c, 0));
2172
2173 ntq_setup_outputs(c);
2174 ntq_setup_uniforms(c);
2175 ntq_setup_registers(c, &c->s->registers);
2176
2177 /* Find the main function and emit the body. */
2178 nir_foreach_function(function, c->s) {
2179 assert(strcmp(function->name, "main") == 0);
2180 assert(function->impl);
2181 ntq_emit_impl(c, function->impl);
2182 }
2183 }
2184
2185 static const nir_shader_compiler_options nir_options = {
2186 .lower_extract_byte = true,
2187 .lower_extract_word = true,
2188 .lower_ffma = true,
2189 .lower_flrp32 = true,
2190 .lower_fpow = true,
2191 .lower_fsat = true,
2192 .lower_fsqrt = true,
2193 .lower_negate = true,
2194 .native_integers = true,
2195 .max_unroll_iterations = 32,
2196 };
2197
2198 const void *
2199 vc4_screen_get_compiler_options(struct pipe_screen *pscreen,
2200 enum pipe_shader_ir ir,
2201 enum pipe_shader_type shader)
2202 {
2203 return &nir_options;
2204 }
2205
2206 static int
2207 count_nir_instrs(nir_shader *nir)
2208 {
2209 int count = 0;
2210 nir_foreach_function(function, nir) {
2211 if (!function->impl)
2212 continue;
2213 nir_foreach_block(block, function->impl) {
2214 nir_foreach_instr(instr, block)
2215 count++;
2216 }
2217 }
2218 return count;
2219 }
2220
2221 static struct vc4_compile *
2222 vc4_shader_ntq(struct vc4_context *vc4, enum qstage stage,
2223 struct vc4_key *key, bool fs_threaded)
2224 {
2225 struct vc4_compile *c = qir_compile_init();
2226
2227 c->vc4 = vc4;
2228 c->stage = stage;
2229 c->shader_state = &key->shader_state->base;
2230 c->program_id = key->shader_state->program_id;
2231 c->variant_id =
2232 p_atomic_inc_return(&key->shader_state->compiled_variant_count);
2233 c->fs_threaded = fs_threaded;
2234
2235 c->key = key;
2236 switch (stage) {
2237 case QSTAGE_FRAG:
2238 c->fs_key = (struct vc4_fs_key *)key;
2239 if (c->fs_key->is_points) {
2240 c->point_x = emit_fragment_varying(c, ~0, 0);
2241 c->point_y = emit_fragment_varying(c, ~0, 0);
2242 } else if (c->fs_key->is_lines) {
2243 c->line_x = emit_fragment_varying(c, ~0, 0);
2244 }
2245 break;
2246 case QSTAGE_VERT:
2247 c->vs_key = (struct vc4_vs_key *)key;
2248 break;
2249 case QSTAGE_COORD:
2250 c->vs_key = (struct vc4_vs_key *)key;
2251 break;
2252 }
2253
2254 c->s = nir_shader_clone(c, key->shader_state->base.ir.nir);
2255
2256 if (stage == QSTAGE_FRAG)
2257 NIR_PASS_V(c->s, vc4_nir_lower_blend, c);
2258
2259 struct nir_lower_tex_options tex_options = {
2260 /* We would need to implement txs, but we don't want the
2261 * int/float conversions
2262 */
2263 .lower_rect = false,
2264
2265 .lower_txp = ~0,
2266
2267 /* Apply swizzles to all samplers. */
2268 .swizzle_result = ~0,
2269 };
2270
2271 /* Lower the format swizzle and ARB_texture_swizzle-style swizzle.
2272 * The format swizzling applies before sRGB decode, and
2273 * ARB_texture_swizzle is the last thing before returning the sample.
2274 */
2275 for (int i = 0; i < ARRAY_SIZE(key->tex); i++) {
2276 enum pipe_format format = c->key->tex[i].format;
2277
2278 if (!format)
2279 continue;
2280
2281 const uint8_t *format_swizzle = vc4_get_format_swizzle(format);
2282
2283 for (int j = 0; j < 4; j++) {
2284 uint8_t arb_swiz = c->key->tex[i].swizzle[j];
2285
2286 if (arb_swiz <= 3) {
2287 tex_options.swizzles[i][j] =
2288 format_swizzle[arb_swiz];
2289 } else {
2290 tex_options.swizzles[i][j] = arb_swiz;
2291 }
2292 }
2293
2294 if (util_format_is_srgb(format))
2295 tex_options.lower_srgb |= (1 << i);
2296 }
2297
2298 NIR_PASS_V(c->s, nir_lower_tex, &tex_options);
2299
2300 if (c->fs_key && c->fs_key->light_twoside)
2301 NIR_PASS_V(c->s, nir_lower_two_sided_color);
2302
2303 if (c->vs_key && c->vs_key->clamp_color)
2304 NIR_PASS_V(c->s, nir_lower_clamp_color_outputs);
2305
2306 if (c->key->ucp_enables) {
2307 if (stage == QSTAGE_FRAG) {
2308 NIR_PASS_V(c->s, nir_lower_clip_fs, c->key->ucp_enables);
2309 } else {
2310 NIR_PASS_V(c->s, nir_lower_clip_vs, c->key->ucp_enables);
2311 NIR_PASS_V(c->s, nir_lower_io_to_scalar,
2312 nir_var_shader_out);
2313 }
2314 }
2315
2316 /* FS input scalarizing must happen after nir_lower_two_sided_color,
2317 * which only handles a vec4 at a time. Similarly, VS output
2318 * scalarizing must happen after nir_lower_clip_vs.
2319 */
2320 if (c->stage == QSTAGE_FRAG)
2321 NIR_PASS_V(c->s, nir_lower_io_to_scalar, nir_var_shader_in);
2322 else
2323 NIR_PASS_V(c->s, nir_lower_io_to_scalar, nir_var_shader_out);
2324
2325 NIR_PASS_V(c->s, vc4_nir_lower_io, c);
2326 NIR_PASS_V(c->s, vc4_nir_lower_txf_ms, c);
2327 NIR_PASS_V(c->s, nir_lower_idiv);
2328
2329 vc4_optimize_nir(c->s);
2330
2331 NIR_PASS_V(c->s, nir_convert_from_ssa, true);
2332
2333 if (vc4_debug & VC4_DEBUG_SHADERDB) {
2334 fprintf(stderr, "SHADER-DB: %s prog %d/%d: %d NIR instructions\n",
2335 qir_get_stage_name(c->stage),
2336 c->program_id, c->variant_id,
2337 count_nir_instrs(c->s));
2338 }
2339
2340 if (vc4_debug & VC4_DEBUG_NIR) {
2341 fprintf(stderr, "%s prog %d/%d NIR:\n",
2342 qir_get_stage_name(c->stage),
2343 c->program_id, c->variant_id);
2344 nir_print_shader(c->s, stderr);
2345 }
2346
2347 nir_to_qir(c);
2348
2349 switch (stage) {
2350 case QSTAGE_FRAG:
2351 /* FS threading requires that the thread execute
2352 * QPU_SIG_LAST_THREAD_SWITCH exactly once before terminating
2353 * (with no other THRSW afterwards, obviously). If we didn't
2354 * fetch a texture at a top level block, this wouldn't be
2355 * true.
2356 */
2357 if (c->fs_threaded && !c->last_thrsw_at_top_level) {
2358 c->failed = true;
2359 return c;
2360 }
2361
2362 emit_frag_end(c);
2363 break;
2364 case QSTAGE_VERT:
2365 emit_vert_end(c,
2366 c->vs_key->fs_inputs->input_slots,
2367 c->vs_key->fs_inputs->num_inputs);
2368 break;
2369 case QSTAGE_COORD:
2370 emit_coord_end(c);
2371 break;
2372 }
2373
2374 if (vc4_debug & VC4_DEBUG_QIR) {
2375 fprintf(stderr, "%s prog %d/%d pre-opt QIR:\n",
2376 qir_get_stage_name(c->stage),
2377 c->program_id, c->variant_id);
2378 qir_dump(c);
2379 fprintf(stderr, "\n");
2380 }
2381
2382 qir_optimize(c);
2383 qir_lower_uniforms(c);
2384
2385 qir_schedule_instructions(c);
2386 qir_emit_uniform_stream_resets(c);
2387
2388 if (vc4_debug & VC4_DEBUG_QIR) {
2389 fprintf(stderr, "%s prog %d/%d QIR:\n",
2390 qir_get_stage_name(c->stage),
2391 c->program_id, c->variant_id);
2392 qir_dump(c);
2393 fprintf(stderr, "\n");
2394 }
2395
2396 qir_reorder_uniforms(c);
2397 vc4_generate_code(vc4, c);
2398
2399 if (vc4_debug & VC4_DEBUG_SHADERDB) {
2400 fprintf(stderr, "SHADER-DB: %s prog %d/%d: %d instructions\n",
2401 qir_get_stage_name(c->stage),
2402 c->program_id, c->variant_id,
2403 c->qpu_inst_count);
2404 fprintf(stderr, "SHADER-DB: %s prog %d/%d: %d uniforms\n",
2405 qir_get_stage_name(c->stage),
2406 c->program_id, c->variant_id,
2407 c->num_uniforms);
2408 }
2409
2410 ralloc_free(c->s);
2411
2412 return c;
2413 }
2414
2415 static void *
2416 vc4_shader_state_create(struct pipe_context *pctx,
2417 const struct pipe_shader_state *cso)
2418 {
2419 struct vc4_context *vc4 = vc4_context(pctx);
2420 struct vc4_uncompiled_shader *so = CALLOC_STRUCT(vc4_uncompiled_shader);
2421 if (!so)
2422 return NULL;
2423
2424 so->program_id = vc4->next_uncompiled_program_id++;
2425
2426 nir_shader *s;
2427
2428 if (cso->type == PIPE_SHADER_IR_NIR) {
2429 /* The backend takes ownership of the NIR shader on state
2430 * creation.
2431 */
2432 s = cso->ir.nir;
2433 } else {
2434 assert(cso->type == PIPE_SHADER_IR_TGSI);
2435
2436 if (vc4_debug & VC4_DEBUG_TGSI) {
2437 fprintf(stderr, "prog %d TGSI:\n",
2438 so->program_id);
2439 tgsi_dump(cso->tokens, 0);
2440 fprintf(stderr, "\n");
2441 }
2442 s = tgsi_to_nir(cso->tokens, &nir_options);
2443 }
2444
2445 NIR_PASS_V(s, nir_opt_global_to_local);
2446 NIR_PASS_V(s, nir_lower_regs_to_ssa);
2447 NIR_PASS_V(s, nir_normalize_cubemap_coords);
2448
2449 NIR_PASS_V(s, nir_lower_load_const_to_scalar);
2450
2451 vc4_optimize_nir(s);
2452
2453 NIR_PASS_V(s, nir_remove_dead_variables, nir_var_local);
2454
2455 /* Garbage collect dead instructions */
2456 nir_sweep(s);
2457
2458 so->base.type = PIPE_SHADER_IR_NIR;
2459 so->base.ir.nir = s;
2460
2461 if (vc4_debug & VC4_DEBUG_NIR) {
2462 fprintf(stderr, "%s prog %d NIR:\n",
2463 gl_shader_stage_name(s->stage),
2464 so->program_id);
2465 nir_print_shader(s, stderr);
2466 fprintf(stderr, "\n");
2467 }
2468
2469 return so;
2470 }
2471
2472 static void
2473 copy_uniform_state_to_shader(struct vc4_compiled_shader *shader,
2474 struct vc4_compile *c)
2475 {
2476 int count = c->num_uniforms;
2477 struct vc4_shader_uniform_info *uinfo = &shader->uniforms;
2478
2479 uinfo->count = count;
2480 uinfo->data = ralloc_array(shader, uint32_t, count);
2481 memcpy(uinfo->data, c->uniform_data,
2482 count * sizeof(*uinfo->data));
2483 uinfo->contents = ralloc_array(shader, enum quniform_contents, count);
2484 memcpy(uinfo->contents, c->uniform_contents,
2485 count * sizeof(*uinfo->contents));
2486 uinfo->num_texture_samples = c->num_texture_samples;
2487
2488 vc4_set_shader_uniform_dirty_flags(shader);
2489 }
2490
2491 static void
2492 vc4_setup_compiled_fs_inputs(struct vc4_context *vc4, struct vc4_compile *c,
2493 struct vc4_compiled_shader *shader)
2494 {
2495 struct vc4_fs_inputs inputs;
2496
2497 memset(&inputs, 0, sizeof(inputs));
2498 inputs.input_slots = ralloc_array(shader,
2499 struct vc4_varying_slot,
2500 c->num_input_slots);
2501
2502 bool input_live[c->num_input_slots];
2503
2504 memset(input_live, 0, sizeof(input_live));
2505 qir_for_each_inst_inorder(inst, c) {
2506 for (int i = 0; i < qir_get_nsrc(inst); i++) {
2507 if (inst->src[i].file == QFILE_VARY)
2508 input_live[inst->src[i].index] = true;
2509 }
2510 }
2511
2512 for (int i = 0; i < c->num_input_slots; i++) {
2513 struct vc4_varying_slot *slot = &c->input_slots[i];
2514
2515 if (!input_live[i])
2516 continue;
2517
2518 /* Skip non-VS-output inputs. */
2519 if (slot->slot == (uint8_t)~0)
2520 continue;
2521
2522 if (slot->slot == VARYING_SLOT_COL0 ||
2523 slot->slot == VARYING_SLOT_COL1 ||
2524 slot->slot == VARYING_SLOT_BFC0 ||
2525 slot->slot == VARYING_SLOT_BFC1) {
2526 shader->color_inputs |= (1 << inputs.num_inputs);
2527 }
2528
2529 inputs.input_slots[inputs.num_inputs] = *slot;
2530 inputs.num_inputs++;
2531 }
2532 shader->num_inputs = inputs.num_inputs;
2533
2534 /* Add our set of inputs to the set of all inputs seen. This way, we
2535 * can have a single pointer that identifies an FS inputs set,
2536 * allowing VS to avoid recompiling when the FS is recompiled (or a
2537 * new one is bound using separate shader objects) but the inputs
2538 * don't change.
2539 */
2540 struct set_entry *entry = _mesa_set_search(vc4->fs_inputs_set, &inputs);
2541 if (entry) {
2542 shader->fs_inputs = entry->key;
2543 ralloc_free(inputs.input_slots);
2544 } else {
2545 struct vc4_fs_inputs *alloc_inputs;
2546
2547 alloc_inputs = rzalloc(vc4->fs_inputs_set, struct vc4_fs_inputs);
2548 memcpy(alloc_inputs, &inputs, sizeof(inputs));
2549 ralloc_steal(alloc_inputs, inputs.input_slots);
2550 _mesa_set_add(vc4->fs_inputs_set, alloc_inputs);
2551
2552 shader->fs_inputs = alloc_inputs;
2553 }
2554 }
2555
2556 static struct vc4_compiled_shader *
2557 vc4_get_compiled_shader(struct vc4_context *vc4, enum qstage stage,
2558 struct vc4_key *key)
2559 {
2560 struct hash_table *ht;
2561 uint32_t key_size;
2562 bool try_threading;
2563
2564 if (stage == QSTAGE_FRAG) {
2565 ht = vc4->fs_cache;
2566 key_size = sizeof(struct vc4_fs_key);
2567 try_threading = vc4->screen->has_threaded_fs;
2568 } else {
2569 ht = vc4->vs_cache;
2570 key_size = sizeof(struct vc4_vs_key);
2571 try_threading = false;
2572 }
2573
2574 struct vc4_compiled_shader *shader;
2575 struct hash_entry *entry = _mesa_hash_table_search(ht, key);
2576 if (entry)
2577 return entry->data;
2578
2579 struct vc4_compile *c = vc4_shader_ntq(vc4, stage, key, try_threading);
2580 /* If the FS failed to compile threaded, fall back to single threaded. */
2581 if (try_threading && c->failed) {
2582 qir_compile_destroy(c);
2583 c = vc4_shader_ntq(vc4, stage, key, false);
2584 }
2585
2586 shader = rzalloc(NULL, struct vc4_compiled_shader);
2587
2588 shader->program_id = vc4->next_compiled_program_id++;
2589 if (stage == QSTAGE_FRAG) {
2590 vc4_setup_compiled_fs_inputs(vc4, c, shader);
2591
2592 /* Note: the temporary clone in c->s has been freed. */
2593 nir_shader *orig_shader = key->shader_state->base.ir.nir;
2594 if (orig_shader->info->outputs_written & (1 << FRAG_RESULT_DEPTH))
2595 shader->disable_early_z = true;
2596 } else {
2597 shader->num_inputs = c->num_inputs;
2598
2599 uint8_t next_vattr_offset = 0;
2600 for (int i = 0; i < c->next_vpm_input; i++) {
2601 if (!c->vattr_sizes[i])
2602 continue;
2603
2604 uint32_t nir_attr = c->vpm_input_order[i];
2605 shader->vattr_offsets[nir_attr] = next_vattr_offset;
2606 next_vattr_offset += c->vattr_sizes[i];
2607 shader->vattrs_live |= (1 << nir_attr);
2608 }
2609 shader->vattr_total_size = next_vattr_offset;
2610 }
2611
2612 shader->failed = c->failed;
2613 if (c->failed) {
2614 shader->failed = true;
2615 } else {
2616 copy_uniform_state_to_shader(shader, c);
2617 shader->bo = vc4_bo_alloc_shader(vc4->screen, c->qpu_insts,
2618 c->qpu_inst_count *
2619 sizeof(uint64_t));
2620 }
2621
2622 shader->fs_threaded = c->fs_threaded;
2623
2624 /* Copy the compiler UBO range state to the compiled shader, dropping
2625 * out arrays that were never referenced by an indirect load.
2626 *
2627 * (Note that QIR dead code elimination of an array access still
2628 * leaves that array alive, though)
2629 */
2630 if (c->num_ubo_ranges) {
2631 shader->num_ubo_ranges = c->num_ubo_ranges;
2632 shader->ubo_ranges = ralloc_array(shader, struct vc4_ubo_range,
2633 c->num_ubo_ranges);
2634 uint32_t j = 0;
2635 for (int i = 0; i < c->num_uniform_ranges; i++) {
2636 struct vc4_compiler_ubo_range *range =
2637 &c->ubo_ranges[i];
2638 if (!range->used)
2639 continue;
2640
2641 shader->ubo_ranges[j].dst_offset = range->dst_offset;
2642 shader->ubo_ranges[j].src_offset = range->src_offset;
2643 shader->ubo_ranges[j].size = range->size;
2644 shader->ubo_size += c->ubo_ranges[i].size;
2645 j++;
2646 }
2647 }
2648 if (shader->ubo_size) {
2649 if (vc4_debug & VC4_DEBUG_SHADERDB) {
2650 fprintf(stderr, "SHADER-DB: %s prog %d/%d: %d UBO uniforms\n",
2651 qir_get_stage_name(c->stage),
2652 c->program_id, c->variant_id,
2653 shader->ubo_size / 4);
2654 }
2655 }
2656
2657 qir_compile_destroy(c);
2658
2659 struct vc4_key *dup_key;
2660 dup_key = rzalloc_size(shader, key_size); /* TODO: don't use rzalloc */
2661 memcpy(dup_key, key, key_size);
2662 _mesa_hash_table_insert(ht, dup_key, shader);
2663
2664 return shader;
2665 }
2666
2667 static void
2668 vc4_setup_shared_key(struct vc4_context *vc4, struct vc4_key *key,
2669 struct vc4_texture_stateobj *texstate)
2670 {
2671 for (int i = 0; i < texstate->num_textures; i++) {
2672 struct pipe_sampler_view *sampler = texstate->textures[i];
2673 struct vc4_sampler_view *vc4_sampler = vc4_sampler_view(sampler);
2674 struct pipe_sampler_state *sampler_state =
2675 texstate->samplers[i];
2676
2677 if (!sampler)
2678 continue;
2679
2680 key->tex[i].format = sampler->format;
2681 key->tex[i].swizzle[0] = sampler->swizzle_r;
2682 key->tex[i].swizzle[1] = sampler->swizzle_g;
2683 key->tex[i].swizzle[2] = sampler->swizzle_b;
2684 key->tex[i].swizzle[3] = sampler->swizzle_a;
2685
2686 if (sampler->texture->nr_samples > 1) {
2687 key->tex[i].msaa_width = sampler->texture->width0;
2688 key->tex[i].msaa_height = sampler->texture->height0;
2689 } else if (sampler){
2690 key->tex[i].compare_mode = sampler_state->compare_mode;
2691 key->tex[i].compare_func = sampler_state->compare_func;
2692 key->tex[i].wrap_s = sampler_state->wrap_s;
2693 key->tex[i].wrap_t = sampler_state->wrap_t;
2694 key->tex[i].force_first_level =
2695 vc4_sampler->force_first_level;
2696 }
2697 }
2698
2699 key->ucp_enables = vc4->rasterizer->base.clip_plane_enable;
2700 }
2701
2702 static void
2703 vc4_update_compiled_fs(struct vc4_context *vc4, uint8_t prim_mode)
2704 {
2705 struct vc4_job *job = vc4->job;
2706 struct vc4_fs_key local_key;
2707 struct vc4_fs_key *key = &local_key;
2708
2709 if (!(vc4->dirty & (VC4_DIRTY_PRIM_MODE |
2710 VC4_DIRTY_BLEND |
2711 VC4_DIRTY_FRAMEBUFFER |
2712 VC4_DIRTY_ZSA |
2713 VC4_DIRTY_RASTERIZER |
2714 VC4_DIRTY_SAMPLE_MASK |
2715 VC4_DIRTY_FRAGTEX |
2716 VC4_DIRTY_UNCOMPILED_FS))) {
2717 return;
2718 }
2719
2720 memset(key, 0, sizeof(*key));
2721 vc4_setup_shared_key(vc4, &key->base, &vc4->fragtex);
2722 key->base.shader_state = vc4->prog.bind_fs;
2723 key->is_points = (prim_mode == PIPE_PRIM_POINTS);
2724 key->is_lines = (prim_mode >= PIPE_PRIM_LINES &&
2725 prim_mode <= PIPE_PRIM_LINE_STRIP);
2726 key->blend = vc4->blend->rt[0];
2727 if (vc4->blend->logicop_enable) {
2728 key->logicop_func = vc4->blend->logicop_func;
2729 } else {
2730 key->logicop_func = PIPE_LOGICOP_COPY;
2731 }
2732 if (job->msaa) {
2733 key->msaa = vc4->rasterizer->base.multisample;
2734 key->sample_coverage = (vc4->sample_mask != (1 << VC4_MAX_SAMPLES) - 1);
2735 key->sample_alpha_to_coverage = vc4->blend->alpha_to_coverage;
2736 key->sample_alpha_to_one = vc4->blend->alpha_to_one;
2737 }
2738
2739 if (vc4->framebuffer.cbufs[0])
2740 key->color_format = vc4->framebuffer.cbufs[0]->format;
2741
2742 key->stencil_enabled = vc4->zsa->stencil_uniforms[0] != 0;
2743 key->stencil_twoside = vc4->zsa->stencil_uniforms[1] != 0;
2744 key->stencil_full_writemasks = vc4->zsa->stencil_uniforms[2] != 0;
2745 key->depth_enabled = (vc4->zsa->base.depth.enabled ||
2746 key->stencil_enabled);
2747 if (vc4->zsa->base.alpha.enabled) {
2748 key->alpha_test = true;
2749 key->alpha_test_func = vc4->zsa->base.alpha.func;
2750 }
2751
2752 if (key->is_points) {
2753 key->point_sprite_mask =
2754 vc4->rasterizer->base.sprite_coord_enable;
2755 key->point_coord_upper_left =
2756 (vc4->rasterizer->base.sprite_coord_mode ==
2757 PIPE_SPRITE_COORD_UPPER_LEFT);
2758 }
2759
2760 key->light_twoside = vc4->rasterizer->base.light_twoside;
2761
2762 struct vc4_compiled_shader *old_fs = vc4->prog.fs;
2763 vc4->prog.fs = vc4_get_compiled_shader(vc4, QSTAGE_FRAG, &key->base);
2764 if (vc4->prog.fs == old_fs)
2765 return;
2766
2767 vc4->dirty |= VC4_DIRTY_COMPILED_FS;
2768
2769 if (vc4->rasterizer->base.flatshade &&
2770 old_fs && vc4->prog.fs->color_inputs != old_fs->color_inputs) {
2771 vc4->dirty |= VC4_DIRTY_FLAT_SHADE_FLAGS;
2772 }
2773
2774 if (old_fs && vc4->prog.fs->fs_inputs != old_fs->fs_inputs)
2775 vc4->dirty |= VC4_DIRTY_FS_INPUTS;
2776 }
2777
2778 static void
2779 vc4_update_compiled_vs(struct vc4_context *vc4, uint8_t prim_mode)
2780 {
2781 struct vc4_vs_key local_key;
2782 struct vc4_vs_key *key = &local_key;
2783
2784 if (!(vc4->dirty & (VC4_DIRTY_PRIM_MODE |
2785 VC4_DIRTY_RASTERIZER |
2786 VC4_DIRTY_VERTTEX |
2787 VC4_DIRTY_VTXSTATE |
2788 VC4_DIRTY_UNCOMPILED_VS |
2789 VC4_DIRTY_FS_INPUTS))) {
2790 return;
2791 }
2792
2793 memset(key, 0, sizeof(*key));
2794 vc4_setup_shared_key(vc4, &key->base, &vc4->verttex);
2795 key->base.shader_state = vc4->prog.bind_vs;
2796 key->fs_inputs = vc4->prog.fs->fs_inputs;
2797 key->clamp_color = vc4->rasterizer->base.clamp_vertex_color;
2798
2799 for (int i = 0; i < ARRAY_SIZE(key->attr_formats); i++)
2800 key->attr_formats[i] = vc4->vtx->pipe[i].src_format;
2801
2802 key->per_vertex_point_size =
2803 (prim_mode == PIPE_PRIM_POINTS &&
2804 vc4->rasterizer->base.point_size_per_vertex);
2805
2806 struct vc4_compiled_shader *vs =
2807 vc4_get_compiled_shader(vc4, QSTAGE_VERT, &key->base);
2808 if (vs != vc4->prog.vs) {
2809 vc4->prog.vs = vs;
2810 vc4->dirty |= VC4_DIRTY_COMPILED_VS;
2811 }
2812
2813 key->is_coord = true;
2814 /* Coord shaders don't care what the FS inputs are. */
2815 key->fs_inputs = NULL;
2816 struct vc4_compiled_shader *cs =
2817 vc4_get_compiled_shader(vc4, QSTAGE_COORD, &key->base);
2818 if (cs != vc4->prog.cs) {
2819 vc4->prog.cs = cs;
2820 vc4->dirty |= VC4_DIRTY_COMPILED_CS;
2821 }
2822 }
2823
2824 bool
2825 vc4_update_compiled_shaders(struct vc4_context *vc4, uint8_t prim_mode)
2826 {
2827 vc4_update_compiled_fs(vc4, prim_mode);
2828 vc4_update_compiled_vs(vc4, prim_mode);
2829
2830 return !(vc4->prog.cs->failed ||
2831 vc4->prog.vs->failed ||
2832 vc4->prog.fs->failed);
2833 }
2834
2835 static uint32_t
2836 fs_cache_hash(const void *key)
2837 {
2838 return _mesa_hash_data(key, sizeof(struct vc4_fs_key));
2839 }
2840
2841 static uint32_t
2842 vs_cache_hash(const void *key)
2843 {
2844 return _mesa_hash_data(key, sizeof(struct vc4_vs_key));
2845 }
2846
2847 static bool
2848 fs_cache_compare(const void *key1, const void *key2)
2849 {
2850 return memcmp(key1, key2, sizeof(struct vc4_fs_key)) == 0;
2851 }
2852
2853 static bool
2854 vs_cache_compare(const void *key1, const void *key2)
2855 {
2856 return memcmp(key1, key2, sizeof(struct vc4_vs_key)) == 0;
2857 }
2858
2859 static uint32_t
2860 fs_inputs_hash(const void *key)
2861 {
2862 const struct vc4_fs_inputs *inputs = key;
2863
2864 return _mesa_hash_data(inputs->input_slots,
2865 sizeof(*inputs->input_slots) *
2866 inputs->num_inputs);
2867 }
2868
2869 static bool
2870 fs_inputs_compare(const void *key1, const void *key2)
2871 {
2872 const struct vc4_fs_inputs *inputs1 = key1;
2873 const struct vc4_fs_inputs *inputs2 = key2;
2874
2875 return (inputs1->num_inputs == inputs2->num_inputs &&
2876 memcmp(inputs1->input_slots,
2877 inputs2->input_slots,
2878 sizeof(*inputs1->input_slots) *
2879 inputs1->num_inputs) == 0);
2880 }
2881
2882 static void
2883 delete_from_cache_if_matches(struct hash_table *ht,
2884 struct hash_entry *entry,
2885 struct vc4_uncompiled_shader *so)
2886 {
2887 const struct vc4_key *key = entry->key;
2888
2889 if (key->shader_state == so) {
2890 struct vc4_compiled_shader *shader = entry->data;
2891 _mesa_hash_table_remove(ht, entry);
2892 vc4_bo_unreference(&shader->bo);
2893 ralloc_free(shader);
2894 }
2895 }
2896
2897 static void
2898 vc4_shader_state_delete(struct pipe_context *pctx, void *hwcso)
2899 {
2900 struct vc4_context *vc4 = vc4_context(pctx);
2901 struct vc4_uncompiled_shader *so = hwcso;
2902
2903 struct hash_entry *entry;
2904 hash_table_foreach(vc4->fs_cache, entry)
2905 delete_from_cache_if_matches(vc4->fs_cache, entry, so);
2906 hash_table_foreach(vc4->vs_cache, entry)
2907 delete_from_cache_if_matches(vc4->vs_cache, entry, so);
2908
2909 ralloc_free(so->base.ir.nir);
2910 free(so);
2911 }
2912
2913 static void
2914 vc4_fp_state_bind(struct pipe_context *pctx, void *hwcso)
2915 {
2916 struct vc4_context *vc4 = vc4_context(pctx);
2917 vc4->prog.bind_fs = hwcso;
2918 vc4->dirty |= VC4_DIRTY_UNCOMPILED_FS;
2919 }
2920
2921 static void
2922 vc4_vp_state_bind(struct pipe_context *pctx, void *hwcso)
2923 {
2924 struct vc4_context *vc4 = vc4_context(pctx);
2925 vc4->prog.bind_vs = hwcso;
2926 vc4->dirty |= VC4_DIRTY_UNCOMPILED_VS;
2927 }
2928
2929 void
2930 vc4_program_init(struct pipe_context *pctx)
2931 {
2932 struct vc4_context *vc4 = vc4_context(pctx);
2933
2934 pctx->create_vs_state = vc4_shader_state_create;
2935 pctx->delete_vs_state = vc4_shader_state_delete;
2936
2937 pctx->create_fs_state = vc4_shader_state_create;
2938 pctx->delete_fs_state = vc4_shader_state_delete;
2939
2940 pctx->bind_fs_state = vc4_fp_state_bind;
2941 pctx->bind_vs_state = vc4_vp_state_bind;
2942
2943 vc4->fs_cache = _mesa_hash_table_create(pctx, fs_cache_hash,
2944 fs_cache_compare);
2945 vc4->vs_cache = _mesa_hash_table_create(pctx, vs_cache_hash,
2946 vs_cache_compare);
2947 vc4->fs_inputs_set = _mesa_set_create(pctx, fs_inputs_hash,
2948 fs_inputs_compare);
2949 }
2950
2951 void
2952 vc4_program_fini(struct pipe_context *pctx)
2953 {
2954 struct vc4_context *vc4 = vc4_context(pctx);
2955
2956 struct hash_entry *entry;
2957 hash_table_foreach(vc4->fs_cache, entry) {
2958 struct vc4_compiled_shader *shader = entry->data;
2959 vc4_bo_unreference(&shader->bo);
2960 ralloc_free(shader);
2961 _mesa_hash_table_remove(vc4->fs_cache, entry);
2962 }
2963
2964 hash_table_foreach(vc4->vs_cache, entry) {
2965 struct vc4_compiled_shader *shader = entry->data;
2966 vc4_bo_unreference(&shader->bo);
2967 ralloc_free(shader);
2968 _mesa_hash_table_remove(vc4->vs_cache, entry);
2969 }
2970 }