81307e0996fe32d2c709601efe7341fcbc0f2d27
[mesa.git] / src / broadcom / compiler / v3d_compiler.h
1 /*
2 * Copyright © 2016 Broadcom
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 */
23
24 #ifndef V3D_COMPILER_H
25 #define V3D_COMPILER_H
26
27 #include <assert.h>
28 #include <stdio.h>
29 #include <stdlib.h>
30 #include <stdbool.h>
31 #include <stdint.h>
32 #include <string.h>
33
34 #include "util/macros.h"
35 #include "common/v3d_debug.h"
36 #include "common/v3d_device_info.h"
37 #include "common/v3d_limits.h"
38 #include "compiler/nir/nir.h"
39 #include "util/list.h"
40 #include "util/u_math.h"
41
42 #include "qpu/qpu_instr.h"
43 #include "pipe/p_state.h"
44
45 struct nir_builder;
46
47 struct v3d_fs_inputs {
48 /**
49 * Array of the meanings of the VPM inputs this shader needs.
50 *
51 * It doesn't include those that aren't part of the VPM, like
52 * point/line coordinates.
53 */
54 struct v3d_varying_slot *input_slots;
55 uint32_t num_inputs;
56 };
57
58 enum qfile {
59 /** An unused source or destination register. */
60 QFILE_NULL,
61
62 /** A physical register, such as the W coordinate payload. */
63 QFILE_REG,
64 /** One of the regsiters for fixed function interactions. */
65 QFILE_MAGIC,
66
67 /**
68 * A virtual register, that will be allocated to actual accumulator
69 * or physical registers later.
70 */
71 QFILE_TEMP,
72
73 /**
74 * VPM reads use this with an index value to say what part of the VPM
75 * is being read.
76 */
77 QFILE_VPM,
78
79 /**
80 * Stores an immediate value in the index field that will be used
81 * directly by qpu_load_imm().
82 */
83 QFILE_LOAD_IMM,
84
85 /**
86 * Stores an immediate value in the index field that can be turned
87 * into a small immediate field by qpu_encode_small_immediate().
88 */
89 QFILE_SMALL_IMM,
90 };
91
92 /**
93 * A reference to a QPU register or a virtual temp register.
94 */
95 struct qreg {
96 enum qfile file;
97 uint32_t index;
98 };
99
100 static inline struct qreg vir_reg(enum qfile file, uint32_t index)
101 {
102 return (struct qreg){file, index};
103 }
104
105 static inline struct qreg vir_magic_reg(uint32_t index)
106 {
107 return (struct qreg){QFILE_MAGIC, index};
108 }
109
110 static inline struct qreg vir_nop_reg(void)
111 {
112 return (struct qreg){QFILE_NULL, 0};
113 }
114
115 /**
116 * A reference to an actual register at the QPU level, for register
117 * allocation.
118 */
119 struct qpu_reg {
120 bool magic;
121 bool smimm;
122 int index;
123 };
124
125 struct qinst {
126 /** Entry in qblock->instructions */
127 struct list_head link;
128
129 /**
130 * The instruction being wrapped. Its condition codes, pack flags,
131 * signals, etc. will all be used, with just the register references
132 * being replaced by the contents of qinst->dst and qinst->src[].
133 */
134 struct v3d_qpu_instr qpu;
135
136 /* Pre-register-allocation references to src/dst registers */
137 struct qreg dst;
138 struct qreg src[3];
139 bool is_last_thrsw;
140
141 /* If the instruction reads a uniform (other than through src[i].file
142 * == QFILE_UNIF), that uniform's index in c->uniform_contents. ~0
143 * otherwise.
144 */
145 int uniform;
146 };
147
148 enum quniform_contents {
149 /**
150 * Indicates that a constant 32-bit value is copied from the program's
151 * uniform contents.
152 */
153 QUNIFORM_CONSTANT,
154 /**
155 * Indicates that the program's uniform contents are used as an index
156 * into the GL uniform storage.
157 */
158 QUNIFORM_UNIFORM,
159
160 /** @{
161 * Scaling factors from clip coordinates to relative to the viewport
162 * center.
163 *
164 * This is used by the coordinate and vertex shaders to produce the
165 * 32-bit entry consisting of 2 16-bit fields with 12.4 signed fixed
166 * point offsets from the viewport ccenter.
167 */
168 QUNIFORM_VIEWPORT_X_SCALE,
169 QUNIFORM_VIEWPORT_Y_SCALE,
170 /** @} */
171
172 QUNIFORM_VIEWPORT_Z_OFFSET,
173 QUNIFORM_VIEWPORT_Z_SCALE,
174
175 QUNIFORM_USER_CLIP_PLANE,
176
177 /**
178 * A reference to a V3D 3.x texture config parameter 0 uniform.
179 *
180 * This is a uniform implicitly loaded with a QPU_W_TMU* write, which
181 * defines texture type, miplevels, and such. It will be found as a
182 * parameter to the first QOP_TEX_[STRB] instruction in a sequence.
183 */
184 QUNIFORM_TEXTURE_CONFIG_P0_0,
185 QUNIFORM_TEXTURE_CONFIG_P0_1,
186 QUNIFORM_TEXTURE_CONFIG_P0_2,
187 QUNIFORM_TEXTURE_CONFIG_P0_3,
188 QUNIFORM_TEXTURE_CONFIG_P0_4,
189 QUNIFORM_TEXTURE_CONFIG_P0_5,
190 QUNIFORM_TEXTURE_CONFIG_P0_6,
191 QUNIFORM_TEXTURE_CONFIG_P0_7,
192 QUNIFORM_TEXTURE_CONFIG_P0_8,
193 QUNIFORM_TEXTURE_CONFIG_P0_9,
194 QUNIFORM_TEXTURE_CONFIG_P0_10,
195 QUNIFORM_TEXTURE_CONFIG_P0_11,
196 QUNIFORM_TEXTURE_CONFIG_P0_12,
197 QUNIFORM_TEXTURE_CONFIG_P0_13,
198 QUNIFORM_TEXTURE_CONFIG_P0_14,
199 QUNIFORM_TEXTURE_CONFIG_P0_15,
200 QUNIFORM_TEXTURE_CONFIG_P0_16,
201 QUNIFORM_TEXTURE_CONFIG_P0_17,
202 QUNIFORM_TEXTURE_CONFIG_P0_18,
203 QUNIFORM_TEXTURE_CONFIG_P0_19,
204 QUNIFORM_TEXTURE_CONFIG_P0_20,
205 QUNIFORM_TEXTURE_CONFIG_P0_21,
206 QUNIFORM_TEXTURE_CONFIG_P0_22,
207 QUNIFORM_TEXTURE_CONFIG_P0_23,
208 QUNIFORM_TEXTURE_CONFIG_P0_24,
209 QUNIFORM_TEXTURE_CONFIG_P0_25,
210 QUNIFORM_TEXTURE_CONFIG_P0_26,
211 QUNIFORM_TEXTURE_CONFIG_P0_27,
212 QUNIFORM_TEXTURE_CONFIG_P0_28,
213 QUNIFORM_TEXTURE_CONFIG_P0_29,
214 QUNIFORM_TEXTURE_CONFIG_P0_30,
215 QUNIFORM_TEXTURE_CONFIG_P0_31,
216 QUNIFORM_TEXTURE_CONFIG_P0_32,
217
218 /**
219 * A reference to a V3D 3.x texture config parameter 1 uniform.
220 *
221 * This is a uniform implicitly loaded with a QPU_W_TMU* write, which
222 * has the pointer to the indirect texture state. Our data[] field
223 * will have a packed p1 value, but the address field will be just
224 * which texture unit's texture should be referenced.
225 */
226 QUNIFORM_TEXTURE_CONFIG_P1,
227
228 /* A V3D 4.x texture config parameter. The high 8 bits will be
229 * which texture or sampler is being sampled, and the driver must
230 * replace the address field with the appropriate address.
231 */
232 QUNIFORM_TMU_CONFIG_P0,
233 QUNIFORM_TMU_CONFIG_P1,
234
235 QUNIFORM_IMAGE_TMU_CONFIG_P0,
236
237 QUNIFORM_TEXTURE_FIRST_LEVEL,
238
239 QUNIFORM_TEXTURE_WIDTH,
240 QUNIFORM_TEXTURE_HEIGHT,
241 QUNIFORM_TEXTURE_DEPTH,
242 QUNIFORM_TEXTURE_ARRAY_SIZE,
243 QUNIFORM_TEXTURE_LEVELS,
244
245 QUNIFORM_UBO_ADDR,
246
247 QUNIFORM_TEXRECT_SCALE_X,
248 QUNIFORM_TEXRECT_SCALE_Y,
249
250 /* Returns the base offset of the SSBO given by the data value. */
251 QUNIFORM_SSBO_OFFSET,
252
253 /* Returns the size of the SSBO given by the data value. */
254 QUNIFORM_GET_BUFFER_SIZE,
255
256 /* Sizes (in pixels) of a shader image given by the data value. */
257 QUNIFORM_IMAGE_WIDTH,
258 QUNIFORM_IMAGE_HEIGHT,
259 QUNIFORM_IMAGE_DEPTH,
260 QUNIFORM_IMAGE_ARRAY_SIZE,
261
262 QUNIFORM_ALPHA_REF,
263
264 /* Number of workgroups passed to glDispatchCompute in the dimension
265 * selected by the data value.
266 */
267 QUNIFORM_NUM_WORK_GROUPS,
268
269 /**
270 * Returns the the offset of the scratch buffer for register spilling.
271 */
272 QUNIFORM_SPILL_OFFSET,
273 QUNIFORM_SPILL_SIZE_PER_THREAD,
274
275 /**
276 * Returns the offset of the shared memory for compute shaders.
277 *
278 * This will be accessed using TMU general memory operations, so the
279 * L2T cache will effectively be the shared memory area.
280 */
281 QUNIFORM_SHARED_OFFSET,
282 };
283
284 static inline uint32_t v3d_unit_data_create(uint32_t unit, uint32_t value)
285 {
286 return unit << 24 | value;
287 }
288
289 static inline uint32_t v3d_unit_data_get_unit(uint32_t data)
290 {
291 return data >> 24;
292 }
293
294 static inline uint32_t v3d_unit_data_get_offset(uint32_t data)
295 {
296 return data & 0xffffff;
297 }
298
299 struct v3d_varying_slot {
300 uint8_t slot_and_component;
301 };
302
303 static inline struct v3d_varying_slot
304 v3d_slot_from_slot_and_component(uint8_t slot, uint8_t component)
305 {
306 assert(slot < 255 / 4);
307 return (struct v3d_varying_slot){ (slot << 2) + component };
308 }
309
310 static inline uint8_t v3d_slot_get_slot(struct v3d_varying_slot slot)
311 {
312 return slot.slot_and_component >> 2;
313 }
314
315 static inline uint8_t v3d_slot_get_component(struct v3d_varying_slot slot)
316 {
317 return slot.slot_and_component & 3;
318 }
319
320 struct v3d_ubo_range {
321 /**
322 * offset in bytes from the start of the ubo where this range is
323 * uploaded.
324 *
325 * Only set once used is set.
326 */
327 uint32_t dst_offset;
328
329 /**
330 * offset in bytes from the start of the gallium uniforms where the
331 * data comes from.
332 */
333 uint32_t src_offset;
334
335 /** size in bytes of this ubo range */
336 uint32_t size;
337 };
338
339 struct v3d_key {
340 void *shader_state;
341 struct {
342 uint8_t swizzle[4];
343 uint8_t return_size;
344 uint8_t return_channels;
345 bool clamp_s:1;
346 bool clamp_t:1;
347 bool clamp_r:1;
348 } tex[V3D_MAX_TEXTURE_SAMPLERS];
349 uint8_t ucp_enables;
350 };
351
352 struct v3d_fs_key {
353 struct v3d_key base;
354 bool depth_enabled;
355 bool is_points;
356 bool is_lines;
357 bool alpha_test;
358 bool point_coord_upper_left;
359 bool light_twoside;
360 bool msaa;
361 bool sample_coverage;
362 bool sample_alpha_to_coverage;
363 bool sample_alpha_to_one;
364 bool clamp_color;
365 bool shade_model_flat;
366 /* Mask of which color render targets are present. */
367 uint8_t cbufs;
368 uint8_t swap_color_rb;
369 /* Mask of which render targets need to be written as 32-bit floats */
370 uint8_t f32_color_rb;
371 /* Masks of which render targets need to be written as ints/uints.
372 * Used by gallium to work around lost information in TGSI.
373 */
374 uint8_t int_color_rb;
375 uint8_t uint_color_rb;
376 uint8_t alpha_test_func;
377 uint8_t logicop_func;
378 uint32_t point_sprite_mask;
379
380 struct pipe_rt_blend_state blend;
381 };
382
383 struct v3d_vs_key {
384 struct v3d_key base;
385
386 struct v3d_varying_slot fs_inputs[V3D_MAX_FS_INPUTS];
387 uint8_t num_fs_inputs;
388
389 bool is_coord;
390 bool per_vertex_point_size;
391 bool clamp_color;
392 };
393
394 /** A basic block of VIR intructions. */
395 struct qblock {
396 struct list_head link;
397
398 struct list_head instructions;
399
400 struct set *predecessors;
401 struct qblock *successors[2];
402
403 int index;
404
405 /* Instruction IPs for the first and last instruction of the block.
406 * Set by qpu_schedule.c.
407 */
408 uint32_t start_qpu_ip;
409 uint32_t end_qpu_ip;
410
411 /* Instruction IP for the branch instruction of the block. Set by
412 * qpu_schedule.c.
413 */
414 uint32_t branch_qpu_ip;
415
416 /** Offset within the uniform stream at the start of the block. */
417 uint32_t start_uniform;
418 /** Offset within the uniform stream of the branch instruction */
419 uint32_t branch_uniform;
420
421 /** @{ used by v3d_vir_live_variables.c */
422 BITSET_WORD *def;
423 BITSET_WORD *defin;
424 BITSET_WORD *defout;
425 BITSET_WORD *use;
426 BITSET_WORD *live_in;
427 BITSET_WORD *live_out;
428 int start_ip, end_ip;
429 /** @} */
430 };
431
432 /** Which util/list.h add mode we should use when inserting an instruction. */
433 enum vir_cursor_mode {
434 vir_cursor_add,
435 vir_cursor_addtail,
436 };
437
438 /**
439 * Tracking structure for where new instructions should be inserted. Create
440 * with one of the vir_after_inst()-style helper functions.
441 *
442 * This does not protect against removal of the block or instruction, so we
443 * have an assert in instruction removal to try to catch it.
444 */
445 struct vir_cursor {
446 enum vir_cursor_mode mode;
447 struct list_head *link;
448 };
449
450 static inline struct vir_cursor
451 vir_before_inst(struct qinst *inst)
452 {
453 return (struct vir_cursor){ vir_cursor_addtail, &inst->link };
454 }
455
456 static inline struct vir_cursor
457 vir_after_inst(struct qinst *inst)
458 {
459 return (struct vir_cursor){ vir_cursor_add, &inst->link };
460 }
461
462 static inline struct vir_cursor
463 vir_before_block(struct qblock *block)
464 {
465 return (struct vir_cursor){ vir_cursor_add, &block->instructions };
466 }
467
468 static inline struct vir_cursor
469 vir_after_block(struct qblock *block)
470 {
471 return (struct vir_cursor){ vir_cursor_addtail, &block->instructions };
472 }
473
474 /**
475 * Compiler state saved across compiler invocations, for any expensive global
476 * setup.
477 */
478 struct v3d_compiler {
479 const struct v3d_device_info *devinfo;
480 struct ra_regs *regs;
481 unsigned int reg_class_any[3];
482 unsigned int reg_class_r5[3];
483 unsigned int reg_class_phys[3];
484 unsigned int reg_class_phys_or_acc[3];
485 };
486
487 struct v3d_compile {
488 const struct v3d_device_info *devinfo;
489 nir_shader *s;
490 nir_function_impl *impl;
491 struct exec_list *cf_node_list;
492 const struct v3d_compiler *compiler;
493
494 void (*debug_output)(const char *msg,
495 void *debug_output_data);
496 void *debug_output_data;
497
498 /**
499 * Mapping from nir_register * or nir_ssa_def * to array of struct
500 * qreg for the values.
501 */
502 struct hash_table *def_ht;
503
504 /* For each temp, the instruction generating its value. */
505 struct qinst **defs;
506 uint32_t defs_array_size;
507
508 /**
509 * Inputs to the shader, arranged by TGSI declaration order.
510 *
511 * Not all fragment shader QFILE_VARY reads are present in this array.
512 */
513 struct qreg *inputs;
514 struct qreg *outputs;
515 bool msaa_per_sample_output;
516 struct qreg color_reads[V3D_MAX_SAMPLES];
517 struct qreg sample_colors[V3D_MAX_SAMPLES];
518 uint32_t inputs_array_size;
519 uint32_t outputs_array_size;
520 uint32_t uniforms_array_size;
521
522 /* Booleans for whether the corresponding QFILE_VARY[i] is
523 * flat-shaded. This includes gl_FragColor flat-shading, which is
524 * customized based on the shademodel_flat shader key.
525 */
526 uint32_t flat_shade_flags[BITSET_WORDS(V3D_MAX_FS_INPUTS)];
527
528 uint32_t noperspective_flags[BITSET_WORDS(V3D_MAX_FS_INPUTS)];
529
530 uint32_t centroid_flags[BITSET_WORDS(V3D_MAX_FS_INPUTS)];
531
532 bool uses_center_w;
533 bool writes_z;
534
535 struct v3d_ubo_range *ubo_ranges;
536 bool *ubo_range_used;
537 uint32_t ubo_ranges_array_size;
538 /** Number of uniform areas tracked in ubo_ranges. */
539 uint32_t num_ubo_ranges;
540 uint32_t next_ubo_dst_offset;
541
542 /* State for whether we're executing on each channel currently. 0 if
543 * yes, otherwise a block number + 1 that the channel jumped to.
544 */
545 struct qreg execute;
546 bool in_control_flow;
547
548 struct qreg line_x, point_x, point_y;
549
550 /**
551 * Instance ID, which comes in before the vertex attribute payload if
552 * the shader record requests it.
553 */
554 struct qreg iid;
555
556 /**
557 * Vertex ID, which comes in before the vertex attribute payload
558 * (after Instance ID) if the shader record requests it.
559 */
560 struct qreg vid;
561
562 /* Fragment shader payload regs. */
563 struct qreg payload_w, payload_w_centroid, payload_z;
564
565 struct qreg cs_payload[2];
566 struct qreg cs_shared_offset;
567 int local_invocation_index_bits;
568
569 uint8_t vattr_sizes[V3D_MAX_VS_INPUTS / 4];
570 uint32_t vpm_output_size;
571
572 /* Size in bytes of registers that have been spilled. This is how much
573 * space needs to be available in the spill BO per thread per QPU.
574 */
575 uint32_t spill_size;
576 /* Shader-db stats */
577 uint32_t spills, fills, loops;
578 /**
579 * Register spilling's per-thread base address, shared between each
580 * spill/fill's addressing calculations.
581 */
582 struct qreg spill_base;
583 /* Bit vector of which temps may be spilled */
584 BITSET_WORD *spillable;
585
586 /**
587 * Array of the VARYING_SLOT_* of all FS QFILE_VARY reads.
588 *
589 * This includes those that aren't part of the VPM varyings, like
590 * point/line coordinates.
591 */
592 struct v3d_varying_slot input_slots[V3D_MAX_FS_INPUTS];
593
594 /**
595 * An entry per outputs[] in the VS indicating what the VARYING_SLOT_*
596 * of the output is. Used to emit from the VS in the order that the
597 * FS needs.
598 */
599 struct v3d_varying_slot *output_slots;
600
601 struct pipe_shader_state *shader_state;
602 struct v3d_key *key;
603 struct v3d_fs_key *fs_key;
604 struct v3d_vs_key *vs_key;
605
606 /* Live ranges of temps. */
607 int *temp_start, *temp_end;
608 bool live_intervals_valid;
609
610 uint32_t *uniform_data;
611 enum quniform_contents *uniform_contents;
612 uint32_t uniform_array_size;
613 uint32_t num_uniforms;
614 uint32_t output_position_index;
615 nir_variable *output_color_var[4];
616 uint32_t output_sample_mask_index;
617
618 struct qreg undef;
619 uint32_t num_temps;
620
621 struct vir_cursor cursor;
622 struct list_head blocks;
623 int next_block_index;
624 struct qblock *cur_block;
625 struct qblock *loop_cont_block;
626 struct qblock *loop_break_block;
627
628 uint64_t *qpu_insts;
629 uint32_t qpu_inst_count;
630 uint32_t qpu_inst_size;
631
632 /* For the FS, the number of varying inputs not counting the
633 * point/line varyings payload
634 */
635 uint32_t num_inputs;
636
637 /**
638 * Number of inputs from num_inputs remaining to be queued to the read
639 * FIFO in the VS/CS.
640 */
641 uint32_t num_inputs_remaining;
642
643 /* Number of inputs currently in the read FIFO for the VS/CS */
644 uint32_t num_inputs_in_fifo;
645
646 /** Next offset in the VPM to read from in the VS/CS */
647 uint32_t vpm_read_offset;
648
649 uint32_t program_id;
650 uint32_t variant_id;
651
652 /* Set to compile program in in 1x, 2x, or 4x threaded mode, where
653 * SIG_THREAD_SWITCH is used to hide texturing latency at the cost of
654 * limiting ourselves to the part of the physical reg space.
655 *
656 * On V3D 3.x, 2x or 4x divide the physical reg space by 2x or 4x. On
657 * V3D 4.x, all shaders are 2x threaded, and 4x only divides the
658 * physical reg space in half.
659 */
660 uint8_t threads;
661 struct qinst *last_thrsw;
662 bool last_thrsw_at_top_level;
663
664 bool failed;
665 };
666
667 struct v3d_uniform_list {
668 enum quniform_contents *contents;
669 uint32_t *data;
670 uint32_t count;
671 };
672
673 struct v3d_prog_data {
674 struct v3d_uniform_list uniforms;
675
676 struct v3d_ubo_range *ubo_ranges;
677 uint32_t num_ubo_ranges;
678 uint32_t ubo_size;
679 uint32_t spill_size;
680
681 uint8_t threads;
682
683 /* For threads > 1, whether the program should be dispatched in the
684 * after-final-THRSW state.
685 */
686 bool single_seg;
687 };
688
689 struct v3d_vs_prog_data {
690 struct v3d_prog_data base;
691
692 bool uses_iid, uses_vid;
693
694 /* Number of components read from each vertex attribute. */
695 uint8_t vattr_sizes[V3D_MAX_VS_INPUTS / 4];
696
697 /* Total number of components read, for the shader state record. */
698 uint32_t vpm_input_size;
699
700 /* Total number of components written, for the shader state record. */
701 uint32_t vpm_output_size;
702
703 /* Set if there should be separate VPM segments for input and output.
704 * If unset, vpm_input_size will be 0.
705 */
706 bool separate_segments;
707
708 /* Value to be programmed in VCM_CACHE_SIZE. */
709 uint8_t vcm_cache_size;
710 };
711
712 struct v3d_fs_prog_data {
713 struct v3d_prog_data base;
714
715 struct v3d_varying_slot input_slots[V3D_MAX_FS_INPUTS];
716
717 /* Array of flat shade flags.
718 *
719 * Each entry is only 24 bits (high 8 bits 0), to match the hardware
720 * packet layout.
721 */
722 uint32_t flat_shade_flags[((V3D_MAX_FS_INPUTS - 1) / 24) + 1];
723
724 uint32_t noperspective_flags[((V3D_MAX_FS_INPUTS - 1) / 24) + 1];
725
726 uint32_t centroid_flags[((V3D_MAX_FS_INPUTS - 1) / 24) + 1];
727
728 uint8_t num_inputs;
729 bool writes_z;
730 bool disable_ez;
731 bool uses_center_w;
732 };
733
734 static inline bool
735 vir_has_uniform(struct qinst *inst)
736 {
737 return inst->uniform != ~0;
738 }
739
740 /* Special nir_load_input intrinsic index for loading the current TLB
741 * destination color.
742 */
743 #define V3D_NIR_TLB_COLOR_READ_INPUT 2000000000
744
745 #define V3D_NIR_MS_MASK_OUTPUT 2000000000
746
747 extern const nir_shader_compiler_options v3d_nir_options;
748
749 const struct v3d_compiler *v3d_compiler_init(const struct v3d_device_info *devinfo);
750 void v3d_compiler_free(const struct v3d_compiler *compiler);
751 void v3d_optimize_nir(struct nir_shader *s);
752
753 uint64_t *v3d_compile(const struct v3d_compiler *compiler,
754 struct v3d_key *key,
755 struct v3d_prog_data **prog_data,
756 nir_shader *s,
757 void (*debug_output)(const char *msg,
758 void *debug_output_data),
759 void *debug_output_data,
760 int program_id, int variant_id,
761 uint32_t *final_assembly_size);
762
763 void v3d_nir_to_vir(struct v3d_compile *c);
764
765 void vir_compile_destroy(struct v3d_compile *c);
766 const char *vir_get_stage_name(struct v3d_compile *c);
767 struct qblock *vir_new_block(struct v3d_compile *c);
768 void vir_set_emit_block(struct v3d_compile *c, struct qblock *block);
769 void vir_link_blocks(struct qblock *predecessor, struct qblock *successor);
770 struct qblock *vir_entry_block(struct v3d_compile *c);
771 struct qblock *vir_exit_block(struct v3d_compile *c);
772 struct qinst *vir_add_inst(enum v3d_qpu_add_op op, struct qreg dst,
773 struct qreg src0, struct qreg src1);
774 struct qinst *vir_mul_inst(enum v3d_qpu_mul_op op, struct qreg dst,
775 struct qreg src0, struct qreg src1);
776 struct qinst *vir_branch_inst(struct v3d_compile *c,
777 enum v3d_qpu_branch_cond cond);
778 void vir_remove_instruction(struct v3d_compile *c, struct qinst *qinst);
779 uint32_t vir_get_uniform_index(struct v3d_compile *c,
780 enum quniform_contents contents,
781 uint32_t data);
782 struct qreg vir_uniform(struct v3d_compile *c,
783 enum quniform_contents contents,
784 uint32_t data);
785 void vir_schedule_instructions(struct v3d_compile *c);
786 struct v3d_qpu_instr v3d_qpu_nop(void);
787
788 struct qreg vir_emit_def(struct v3d_compile *c, struct qinst *inst);
789 struct qinst *vir_emit_nondef(struct v3d_compile *c, struct qinst *inst);
790 void vir_set_cond(struct qinst *inst, enum v3d_qpu_cond cond);
791 void vir_set_pf(struct qinst *inst, enum v3d_qpu_pf pf);
792 void vir_set_uf(struct qinst *inst, enum v3d_qpu_uf uf);
793 void vir_set_unpack(struct qinst *inst, int src,
794 enum v3d_qpu_input_unpack unpack);
795
796 struct qreg vir_get_temp(struct v3d_compile *c);
797 void vir_emit_last_thrsw(struct v3d_compile *c);
798 void vir_calculate_live_intervals(struct v3d_compile *c);
799 int vir_get_nsrc(struct qinst *inst);
800 bool vir_has_side_effects(struct v3d_compile *c, struct qinst *inst);
801 bool vir_get_add_op(struct qinst *inst, enum v3d_qpu_add_op *op);
802 bool vir_get_mul_op(struct qinst *inst, enum v3d_qpu_mul_op *op);
803 bool vir_is_raw_mov(struct qinst *inst);
804 bool vir_is_tex(struct qinst *inst);
805 bool vir_is_add(struct qinst *inst);
806 bool vir_is_mul(struct qinst *inst);
807 bool vir_writes_r3(const struct v3d_device_info *devinfo, struct qinst *inst);
808 bool vir_writes_r4(const struct v3d_device_info *devinfo, struct qinst *inst);
809 struct qreg vir_follow_movs(struct v3d_compile *c, struct qreg reg);
810 uint8_t vir_channels_written(struct qinst *inst);
811 struct qreg ntq_get_src(struct v3d_compile *c, nir_src src, int i);
812 void ntq_store_dest(struct v3d_compile *c, nir_dest *dest, int chan,
813 struct qreg result);
814 void vir_emit_thrsw(struct v3d_compile *c);
815
816 void vir_dump(struct v3d_compile *c);
817 void vir_dump_inst(struct v3d_compile *c, struct qinst *inst);
818 void vir_dump_uniform(enum quniform_contents contents, uint32_t data);
819
820 void vir_validate(struct v3d_compile *c);
821
822 void vir_optimize(struct v3d_compile *c);
823 bool vir_opt_algebraic(struct v3d_compile *c);
824 bool vir_opt_constant_folding(struct v3d_compile *c);
825 bool vir_opt_copy_propagate(struct v3d_compile *c);
826 bool vir_opt_dead_code(struct v3d_compile *c);
827 bool vir_opt_peephole_sf(struct v3d_compile *c);
828 bool vir_opt_small_immediates(struct v3d_compile *c);
829 bool vir_opt_vpm(struct v3d_compile *c);
830 void v3d_nir_lower_blend(nir_shader *s, struct v3d_compile *c);
831 void v3d_nir_lower_io(nir_shader *s, struct v3d_compile *c);
832 void v3d_nir_lower_txf_ms(nir_shader *s, struct v3d_compile *c);
833 void v3d_nir_lower_image_load_store(nir_shader *s);
834 void vir_lower_uniforms(struct v3d_compile *c);
835
836 void v3d33_vir_vpm_read_setup(struct v3d_compile *c, int num_components);
837 void v3d33_vir_vpm_write_setup(struct v3d_compile *c);
838 void v3d33_vir_emit_tex(struct v3d_compile *c, nir_tex_instr *instr);
839 void v3d40_vir_emit_tex(struct v3d_compile *c, nir_tex_instr *instr);
840 void v3d40_vir_emit_image_load_store(struct v3d_compile *c,
841 nir_intrinsic_instr *instr);
842
843 void v3d_vir_to_qpu(struct v3d_compile *c, struct qpu_reg *temp_registers);
844 uint32_t v3d_qpu_schedule_instructions(struct v3d_compile *c);
845 void qpu_validate(struct v3d_compile *c);
846 struct qpu_reg *v3d_register_allocate(struct v3d_compile *c, bool *spilled);
847 bool vir_init_reg_sets(struct v3d_compiler *compiler);
848
849 bool v3d_gl_format_is_return_32(GLenum format);
850
851 static inline bool
852 quniform_contents_is_texture_p0(enum quniform_contents contents)
853 {
854 return (contents >= QUNIFORM_TEXTURE_CONFIG_P0_0 &&
855 contents < (QUNIFORM_TEXTURE_CONFIG_P0_0 +
856 V3D_MAX_TEXTURE_SAMPLERS));
857 }
858
859 static inline bool
860 vir_in_nonuniform_control_flow(struct v3d_compile *c)
861 {
862 return c->execute.file != QFILE_NULL;
863 }
864
865 static inline struct qreg
866 vir_uniform_ui(struct v3d_compile *c, uint32_t ui)
867 {
868 return vir_uniform(c, QUNIFORM_CONSTANT, ui);
869 }
870
871 static inline struct qreg
872 vir_uniform_f(struct v3d_compile *c, float f)
873 {
874 return vir_uniform(c, QUNIFORM_CONSTANT, fui(f));
875 }
876
877 #define VIR_ALU0(name, vir_inst, op) \
878 static inline struct qreg \
879 vir_##name(struct v3d_compile *c) \
880 { \
881 return vir_emit_def(c, vir_inst(op, c->undef, \
882 c->undef, c->undef)); \
883 } \
884 static inline struct qinst * \
885 vir_##name##_dest(struct v3d_compile *c, struct qreg dest) \
886 { \
887 return vir_emit_nondef(c, vir_inst(op, dest, \
888 c->undef, c->undef)); \
889 }
890
891 #define VIR_ALU1(name, vir_inst, op) \
892 static inline struct qreg \
893 vir_##name(struct v3d_compile *c, struct qreg a) \
894 { \
895 return vir_emit_def(c, vir_inst(op, c->undef, \
896 a, c->undef)); \
897 } \
898 static inline struct qinst * \
899 vir_##name##_dest(struct v3d_compile *c, struct qreg dest, \
900 struct qreg a) \
901 { \
902 return vir_emit_nondef(c, vir_inst(op, dest, a, \
903 c->undef)); \
904 }
905
906 #define VIR_ALU2(name, vir_inst, op) \
907 static inline struct qreg \
908 vir_##name(struct v3d_compile *c, struct qreg a, struct qreg b) \
909 { \
910 return vir_emit_def(c, vir_inst(op, c->undef, a, b)); \
911 } \
912 static inline struct qinst * \
913 vir_##name##_dest(struct v3d_compile *c, struct qreg dest, \
914 struct qreg a, struct qreg b) \
915 { \
916 return vir_emit_nondef(c, vir_inst(op, dest, a, b)); \
917 }
918
919 #define VIR_NODST_0(name, vir_inst, op) \
920 static inline struct qinst * \
921 vir_##name(struct v3d_compile *c) \
922 { \
923 return vir_emit_nondef(c, vir_inst(op, c->undef, \
924 c->undef, c->undef)); \
925 }
926
927 #define VIR_NODST_1(name, vir_inst, op) \
928 static inline struct qinst * \
929 vir_##name(struct v3d_compile *c, struct qreg a) \
930 { \
931 return vir_emit_nondef(c, vir_inst(op, c->undef, \
932 a, c->undef)); \
933 }
934
935 #define VIR_NODST_2(name, vir_inst, op) \
936 static inline struct qinst * \
937 vir_##name(struct v3d_compile *c, struct qreg a, struct qreg b) \
938 { \
939 return vir_emit_nondef(c, vir_inst(op, c->undef, \
940 a, b)); \
941 }
942
943 #define VIR_SFU(name) \
944 static inline struct qreg \
945 vir_##name(struct v3d_compile *c, struct qreg a) \
946 { \
947 if (c->devinfo->ver >= 41) { \
948 return vir_emit_def(c, vir_add_inst(V3D_QPU_A_##name, \
949 c->undef, \
950 a, c->undef)); \
951 } else { \
952 vir_FMOV_dest(c, vir_reg(QFILE_MAGIC, V3D_QPU_WADDR_##name), a); \
953 return vir_FMOV(c, vir_reg(QFILE_MAGIC, V3D_QPU_WADDR_R4)); \
954 } \
955 } \
956 static inline struct qinst * \
957 vir_##name##_dest(struct v3d_compile *c, struct qreg dest, \
958 struct qreg a) \
959 { \
960 if (c->devinfo->ver >= 41) { \
961 return vir_emit_nondef(c, vir_add_inst(V3D_QPU_A_##name, \
962 dest, \
963 a, c->undef)); \
964 } else { \
965 vir_FMOV_dest(c, vir_reg(QFILE_MAGIC, V3D_QPU_WADDR_##name), a); \
966 return vir_FMOV_dest(c, dest, vir_reg(QFILE_MAGIC, V3D_QPU_WADDR_R4)); \
967 } \
968 }
969
970 #define VIR_A_ALU2(name) VIR_ALU2(name, vir_add_inst, V3D_QPU_A_##name)
971 #define VIR_M_ALU2(name) VIR_ALU2(name, vir_mul_inst, V3D_QPU_M_##name)
972 #define VIR_A_ALU1(name) VIR_ALU1(name, vir_add_inst, V3D_QPU_A_##name)
973 #define VIR_M_ALU1(name) VIR_ALU1(name, vir_mul_inst, V3D_QPU_M_##name)
974 #define VIR_A_ALU0(name) VIR_ALU0(name, vir_add_inst, V3D_QPU_A_##name)
975 #define VIR_M_ALU0(name) VIR_ALU0(name, vir_mul_inst, V3D_QPU_M_##name)
976 #define VIR_A_NODST_2(name) VIR_NODST_2(name, vir_add_inst, V3D_QPU_A_##name)
977 #define VIR_M_NODST_2(name) VIR_NODST_2(name, vir_mul_inst, V3D_QPU_M_##name)
978 #define VIR_A_NODST_1(name) VIR_NODST_1(name, vir_add_inst, V3D_QPU_A_##name)
979 #define VIR_M_NODST_1(name) VIR_NODST_1(name, vir_mul_inst, V3D_QPU_M_##name)
980 #define VIR_A_NODST_0(name) VIR_NODST_0(name, vir_add_inst, V3D_QPU_A_##name)
981
982 VIR_A_ALU2(FADD)
983 VIR_A_ALU2(VFPACK)
984 VIR_A_ALU2(FSUB)
985 VIR_A_ALU2(FMIN)
986 VIR_A_ALU2(FMAX)
987
988 VIR_A_ALU2(ADD)
989 VIR_A_ALU2(SUB)
990 VIR_A_ALU2(SHL)
991 VIR_A_ALU2(SHR)
992 VIR_A_ALU2(ASR)
993 VIR_A_ALU2(ROR)
994 VIR_A_ALU2(MIN)
995 VIR_A_ALU2(MAX)
996 VIR_A_ALU2(UMIN)
997 VIR_A_ALU2(UMAX)
998 VIR_A_ALU2(AND)
999 VIR_A_ALU2(OR)
1000 VIR_A_ALU2(XOR)
1001 VIR_A_ALU2(VADD)
1002 VIR_A_ALU2(VSUB)
1003 VIR_A_NODST_2(STVPMV)
1004 VIR_A_ALU1(NOT)
1005 VIR_A_ALU1(NEG)
1006 VIR_A_ALU1(FLAPUSH)
1007 VIR_A_ALU1(FLBPUSH)
1008 VIR_A_ALU1(FLPOP)
1009 VIR_A_ALU1(SETMSF)
1010 VIR_A_ALU1(SETREVF)
1011 VIR_A_ALU0(TIDX)
1012 VIR_A_ALU0(EIDX)
1013 VIR_A_ALU1(LDVPMV_IN)
1014 VIR_A_ALU1(LDVPMV_OUT)
1015 VIR_A_ALU0(TMUWT)
1016
1017 VIR_A_ALU0(FXCD)
1018 VIR_A_ALU0(XCD)
1019 VIR_A_ALU0(FYCD)
1020 VIR_A_ALU0(YCD)
1021 VIR_A_ALU0(MSF)
1022 VIR_A_ALU0(REVF)
1023 VIR_A_ALU0(BARRIERID)
1024 VIR_A_NODST_1(VPMSETUP)
1025 VIR_A_NODST_0(VPMWT)
1026 VIR_A_ALU2(FCMP)
1027 VIR_A_ALU2(VFMAX)
1028
1029 VIR_A_ALU1(FROUND)
1030 VIR_A_ALU1(FTOIN)
1031 VIR_A_ALU1(FTRUNC)
1032 VIR_A_ALU1(FTOIZ)
1033 VIR_A_ALU1(FFLOOR)
1034 VIR_A_ALU1(FTOUZ)
1035 VIR_A_ALU1(FCEIL)
1036 VIR_A_ALU1(FTOC)
1037
1038 VIR_A_ALU1(FDX)
1039 VIR_A_ALU1(FDY)
1040
1041 VIR_A_ALU1(ITOF)
1042 VIR_A_ALU1(CLZ)
1043 VIR_A_ALU1(UTOF)
1044
1045 VIR_M_ALU2(UMUL24)
1046 VIR_M_ALU2(FMUL)
1047 VIR_M_ALU2(SMUL24)
1048 VIR_M_NODST_2(MULTOP)
1049
1050 VIR_M_ALU1(MOV)
1051 VIR_M_ALU1(FMOV)
1052
1053 VIR_SFU(RECIP)
1054 VIR_SFU(RSQRT)
1055 VIR_SFU(EXP)
1056 VIR_SFU(LOG)
1057 VIR_SFU(SIN)
1058 VIR_SFU(RSQRT2)
1059
1060 static inline struct qinst *
1061 vir_MOV_cond(struct v3d_compile *c, enum v3d_qpu_cond cond,
1062 struct qreg dest, struct qreg src)
1063 {
1064 struct qinst *mov = vir_MOV_dest(c, dest, src);
1065 vir_set_cond(mov, cond);
1066 return mov;
1067 }
1068
1069 static inline struct qreg
1070 vir_SEL(struct v3d_compile *c, enum v3d_qpu_cond cond,
1071 struct qreg src0, struct qreg src1)
1072 {
1073 struct qreg t = vir_get_temp(c);
1074 vir_MOV_dest(c, t, src1);
1075 vir_MOV_cond(c, cond, t, src0);
1076 return t;
1077 }
1078
1079 static inline struct qinst *
1080 vir_NOP(struct v3d_compile *c)
1081 {
1082 return vir_emit_nondef(c, vir_add_inst(V3D_QPU_A_NOP,
1083 c->undef, c->undef, c->undef));
1084 }
1085
1086 static inline struct qreg
1087 vir_LDTMU(struct v3d_compile *c)
1088 {
1089 if (c->devinfo->ver >= 41) {
1090 struct qinst *ldtmu = vir_add_inst(V3D_QPU_A_NOP, c->undef,
1091 c->undef, c->undef);
1092 ldtmu->qpu.sig.ldtmu = true;
1093
1094 return vir_emit_def(c, ldtmu);
1095 } else {
1096 vir_NOP(c)->qpu.sig.ldtmu = true;
1097 return vir_MOV(c, vir_reg(QFILE_MAGIC, V3D_QPU_WADDR_R4));
1098 }
1099 }
1100
1101 static inline struct qreg
1102 vir_UMUL(struct v3d_compile *c, struct qreg src0, struct qreg src1)
1103 {
1104 vir_MULTOP(c, src0, src1);
1105 return vir_UMUL24(c, src0, src1);
1106 }
1107
1108 /*
1109 static inline struct qreg
1110 vir_LOAD_IMM(struct v3d_compile *c, uint32_t val)
1111 {
1112 return vir_emit_def(c, vir_inst(QOP_LOAD_IMM, c->undef,
1113 vir_reg(QFILE_LOAD_IMM, val), c->undef));
1114 }
1115
1116 static inline struct qreg
1117 vir_LOAD_IMM_U2(struct v3d_compile *c, uint32_t val)
1118 {
1119 return vir_emit_def(c, vir_inst(QOP_LOAD_IMM_U2, c->undef,
1120 vir_reg(QFILE_LOAD_IMM, val),
1121 c->undef));
1122 }
1123 static inline struct qreg
1124 vir_LOAD_IMM_I2(struct v3d_compile *c, uint32_t val)
1125 {
1126 return vir_emit_def(c, vir_inst(QOP_LOAD_IMM_I2, c->undef,
1127 vir_reg(QFILE_LOAD_IMM, val),
1128 c->undef));
1129 }
1130 */
1131
1132 static inline struct qinst *
1133 vir_BRANCH(struct v3d_compile *c, enum v3d_qpu_branch_cond cond)
1134 {
1135 /* The actual uniform_data value will be set at scheduling time */
1136 return vir_emit_nondef(c, vir_branch_inst(c, cond));
1137 }
1138
1139 #define vir_for_each_block(block, c) \
1140 list_for_each_entry(struct qblock, block, &c->blocks, link)
1141
1142 #define vir_for_each_block_rev(block, c) \
1143 list_for_each_entry_rev(struct qblock, block, &c->blocks, link)
1144
1145 /* Loop over the non-NULL members of the successors array. */
1146 #define vir_for_each_successor(succ, block) \
1147 for (struct qblock *succ = block->successors[0]; \
1148 succ != NULL; \
1149 succ = (succ == block->successors[1] ? NULL : \
1150 block->successors[1]))
1151
1152 #define vir_for_each_inst(inst, block) \
1153 list_for_each_entry(struct qinst, inst, &block->instructions, link)
1154
1155 #define vir_for_each_inst_rev(inst, block) \
1156 list_for_each_entry_rev(struct qinst, inst, &block->instructions, link)
1157
1158 #define vir_for_each_inst_safe(inst, block) \
1159 list_for_each_entry_safe(struct qinst, inst, &block->instructions, link)
1160
1161 #define vir_for_each_inst_inorder(inst, c) \
1162 vir_for_each_block(_block, c) \
1163 vir_for_each_inst(inst, _block)
1164
1165 #define vir_for_each_inst_inorder_safe(inst, c) \
1166 vir_for_each_block(_block, c) \
1167 vir_for_each_inst_safe(inst, _block)
1168
1169 #endif /* V3D_COMPILER_H */