2 * Copyright © 2014 Broadcom
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
34 #include "util/macros.h"
35 #include "compiler/nir/nir.h"
36 #include "util/list.h"
37 #include "util/u_math.h"
39 #include "vc4_screen.h"
40 #include "vc4_qpu_defines.h"
42 #include "kernel/vc4_packet.h"
43 #include "pipe/p_state.h"
53 QFILE_TLB_COLOR_WRITE
,
54 QFILE_TLB_COLOR_WRITE_MS
,
56 QFILE_TLB_STENCIL_SETUP
,
58 /* Payload registers that aren't in the physical register file, so we
59 * can just use the corresponding qpu_reg at qpu_emit time.
66 * Stores an immediate value in the index field that will be used
67 * directly by qpu_load_imm().
72 * Stores an immediate value in the index field that can be turned
73 * into a small immediate field by qpu_encode_small_immediate().
84 static inline struct qreg
qir_reg(enum qfile file
, uint32_t index
)
86 return (struct qreg
){file
, index
};
134 /** Texture x coordinate parameter write */
136 /** Texture y coordinate parameter write */
138 /** Texture border color parameter or cube map z coordinate write */
140 /** Texture LOD bias parameter write */
144 * Texture-unit 4-byte read with address provided direct in S
147 * The first operand is the offset from the start of the UBO, and the
148 * second is the uniform that has the UBO's base pointer.
153 * Signal of texture read being necessary and then reading r4 into
160 /* Jumps to block->successor[0] if the qinst->cond (as a
161 * QPU_COND_BRANCH_*) passes, or block->successor[1] if not. Note
162 * that block->successor[1] may be unset if the condition is ALWAYS.
167 struct queued_qpu_inst
{
168 struct list_head link
;
173 struct list_head link
;
184 * Coordinate shader, runs during binning, before the VS, and just
192 enum quniform_contents
{
194 * Indicates that a constant 32-bit value is copied from the program's
199 * Indicates that the program's uniform contents are used as an index
200 * into the GL uniform storage.
205 * Scaling factors from clip coordinates to relative to the viewport
208 * This is used by the coordinate and vertex shaders to produce the
209 * 32-bit entry consisting of 2 16-bit fields with 12.4 signed fixed
210 * point offsets from the viewport ccenter.
212 QUNIFORM_VIEWPORT_X_SCALE
,
213 QUNIFORM_VIEWPORT_Y_SCALE
,
216 QUNIFORM_VIEWPORT_Z_OFFSET
,
217 QUNIFORM_VIEWPORT_Z_SCALE
,
219 QUNIFORM_USER_CLIP_PLANE
,
222 * A reference to a texture config parameter 0 uniform.
224 * This is a uniform implicitly loaded with a QPU_W_TMU* write, which
225 * defines texture type, miplevels, and such. It will be found as a
226 * parameter to the first QOP_TEX_[STRB] instruction in a sequence.
228 QUNIFORM_TEXTURE_CONFIG_P0
,
231 * A reference to a texture config parameter 1 uniform.
233 * This is a uniform implicitly loaded with a QPU_W_TMU* write, which
234 * defines texture width, height, filters, and wrap modes. It will be
235 * found as a parameter to the second QOP_TEX_[STRB] instruction in a
238 QUNIFORM_TEXTURE_CONFIG_P1
,
240 /** A reference to a texture config parameter 2 cubemap stride uniform */
241 QUNIFORM_TEXTURE_CONFIG_P2
,
243 QUNIFORM_TEXTURE_MSAA_ADDR
,
247 QUNIFORM_TEXRECT_SCALE_X
,
248 QUNIFORM_TEXRECT_SCALE_Y
,
250 QUNIFORM_TEXTURE_BORDER_COLOR
,
252 QUNIFORM_BLEND_CONST_COLOR_X
,
253 QUNIFORM_BLEND_CONST_COLOR_Y
,
254 QUNIFORM_BLEND_CONST_COLOR_Z
,
255 QUNIFORM_BLEND_CONST_COLOR_W
,
256 QUNIFORM_BLEND_CONST_COLOR_RGBA
,
257 QUNIFORM_BLEND_CONST_COLOR_AAAA
,
262 QUNIFORM_SAMPLE_MASK
,
265 struct vc4_varying_slot
{
270 struct vc4_compiler_ubo_range
{
272 * offset in bytes from the start of the ubo where this range is
275 * Only set once used is set.
280 * offset in bytes from the start of the gallium uniforms where the
285 /** size in bytes of this ubo range */
289 * Set if this range is used by the shader for indirect uniforms
296 struct vc4_uncompiled_shader
*shader_state
;
298 enum pipe_format format
;
302 unsigned compare_mode
:1;
303 unsigned compare_func
:3;
308 uint16_t msaa_width
, msaa_height
;
311 } tex
[VC4_MAX_TEXTURE_SAMPLERS
];
317 enum pipe_format color_format
;
319 bool stencil_enabled
;
320 bool stencil_twoside
;
321 bool stencil_full_writemasks
;
325 bool point_coord_upper_left
;
328 bool sample_coverage
;
329 bool sample_alpha_to_coverage
;
330 bool sample_alpha_to_one
;
331 uint8_t alpha_test_func
;
332 uint8_t logicop_func
;
333 uint32_t point_sprite_mask
;
335 struct pipe_rt_blend_state blend
;
342 * This is a proxy for the array of FS input semantics, which is
343 * larger than we would want to put in the key.
345 uint64_t compiled_fs_id
;
347 enum pipe_format attr_formats
[8];
349 bool per_vertex_point_size
;
353 /** A basic block of QIR intructions. */
355 struct list_head link
;
357 struct list_head instructions
;
359 struct set
*predecessors
;
360 struct qblock
*successors
[2];
364 /** @{ used by vc4_qir_live_variables.c */
367 BITSET_WORD
*live_in
;
368 BITSET_WORD
*live_out
;
369 int start_ip
, end_ip
;
374 struct vc4_context
*vc4
;
376 nir_function_impl
*impl
;
377 struct exec_list
*cf_node_list
;
380 * Mapping from nir_register * or nir_ssa_def * to array of struct
381 * qreg for the values.
383 struct hash_table
*def_ht
;
385 /* For each temp, the instruction generating its value. */
387 uint32_t defs_array_size
;
390 * Inputs to the shader, arranged by TGSI declaration order.
392 * Not all fragment shader QFILE_VARY reads are present in this array.
395 struct qreg
*outputs
;
396 bool msaa_per_sample_output
;
397 struct qreg color_reads
[VC4_MAX_SAMPLES
];
398 struct qreg sample_colors
[VC4_MAX_SAMPLES
];
399 uint32_t inputs_array_size
;
400 uint32_t outputs_array_size
;
401 uint32_t uniforms_array_size
;
403 struct vc4_compiler_ubo_range
*ubo_ranges
;
404 uint32_t ubo_ranges_array_size
;
405 /** Number of uniform areas declared in ubo_ranges. */
406 uint32_t num_uniform_ranges
;
407 /** Number of uniform areas used for indirect addressed loads. */
408 uint32_t num_ubo_ranges
;
409 uint32_t next_ubo_dst_offset
;
411 struct qreg line_x
, point_x
, point_y
;
413 struct qreg payload_FRAG_Z
;
414 struct qreg payload_FRAG_W
;
416 uint8_t vattr_sizes
[8];
419 * Array of the VARYING_SLOT_* of all FS QFILE_VARY reads.
421 * This includes those that aren't part of the VPM varyings, like
422 * point/line coordinates.
424 struct vc4_varying_slot
*input_slots
;
425 uint32_t num_input_slots
;
426 uint32_t input_slots_array_size
;
429 * An entry per outputs[] in the VS indicating what the VARYING_SLOT_*
430 * of the output is. Used to emit from the VS in the order that the
433 struct vc4_varying_slot
*output_slots
;
435 struct pipe_shader_state
*shader_state
;
437 struct vc4_fs_key
*fs_key
;
438 struct vc4_vs_key
*vs_key
;
440 /* Live ranges of temps. */
441 int *temp_start
, *temp_end
;
443 uint32_t *uniform_data
;
444 enum quniform_contents
*uniform_contents
;
445 uint32_t uniform_array_size
;
446 uint32_t num_uniforms
;
447 uint32_t num_outputs
;
448 uint32_t num_texture_samples
;
449 uint32_t output_position_index
;
450 uint32_t output_color_index
;
451 uint32_t output_point_size_index
;
452 uint32_t output_sample_mask_index
;
458 struct list_head blocks
;
459 int next_block_index
;
460 struct qblock
*cur_block
;
462 struct list_head qpu_inst_list
;
464 uint32_t qpu_inst_count
;
465 uint32_t qpu_inst_size
;
472 /* Special nir_load_input intrinsic index for loading the current TLB
475 #define VC4_NIR_TLB_COLOR_READ_INPUT 2000000000
477 #define VC4_NIR_MS_MASK_OUTPUT 2000000000
479 /* Special offset for nir_load_uniform values to get a QUNIFORM_*
480 * state-dependent value.
482 #define VC4_NIR_STATE_UNIFORM_OFFSET 1000000000
484 struct vc4_compile
*qir_compile_init(void);
485 void qir_compile_destroy(struct vc4_compile
*c
);
486 struct qblock
*qir_new_block(struct vc4_compile
*c
);
487 void qir_set_emit_block(struct vc4_compile
*c
, struct qblock
*block
);
488 void qir_link_blocks(struct qblock
*predecessor
, struct qblock
*successor
);
489 struct qblock
*qir_entry_block(struct vc4_compile
*c
);
490 struct qblock
*qir_exit_block(struct vc4_compile
*c
);
491 struct qinst
*qir_inst(enum qop op
, struct qreg dst
,
492 struct qreg src0
, struct qreg src1
);
493 struct qinst
*qir_inst4(enum qop op
, struct qreg dst
,
498 void qir_remove_instruction(struct vc4_compile
*c
, struct qinst
*qinst
);
499 struct qreg
qir_uniform(struct vc4_compile
*c
,
500 enum quniform_contents contents
,
502 void qir_schedule_instructions(struct vc4_compile
*c
);
503 void qir_reorder_uniforms(struct vc4_compile
*c
);
505 struct qreg
qir_emit_def(struct vc4_compile
*c
, struct qinst
*inst
);
506 struct qinst
*qir_emit_nondef(struct vc4_compile
*c
, struct qinst
*inst
);
508 struct qreg
qir_get_temp(struct vc4_compile
*c
);
509 void qir_calculate_live_intervals(struct vc4_compile
*c
);
510 int qir_get_op_nsrc(enum qop qop
);
511 bool qir_reg_equals(struct qreg a
, struct qreg b
);
512 bool qir_has_side_effects(struct vc4_compile
*c
, struct qinst
*inst
);
513 bool qir_has_side_effect_reads(struct vc4_compile
*c
, struct qinst
*inst
);
514 bool qir_is_mul(struct qinst
*inst
);
515 bool qir_is_raw_mov(struct qinst
*inst
);
516 bool qir_is_tex(struct qinst
*inst
);
517 bool qir_is_float_input(struct qinst
*inst
);
518 bool qir_depends_on_flags(struct qinst
*inst
);
519 bool qir_writes_r4(struct qinst
*inst
);
520 struct qreg
qir_follow_movs(struct vc4_compile
*c
, struct qreg reg
);
521 uint8_t qir_channels_written(struct qinst
*inst
);
523 void qir_dump(struct vc4_compile
*c
);
524 void qir_dump_inst(struct vc4_compile
*c
, struct qinst
*inst
);
525 const char *qir_get_stage_name(enum qstage stage
);
527 void qir_validate(struct vc4_compile
*c
);
529 void qir_optimize(struct vc4_compile
*c
);
530 bool qir_opt_algebraic(struct vc4_compile
*c
);
531 bool qir_opt_constant_folding(struct vc4_compile
*c
);
532 bool qir_opt_copy_propagation(struct vc4_compile
*c
);
533 bool qir_opt_dead_code(struct vc4_compile
*c
);
534 bool qir_opt_peephole_sf(struct vc4_compile
*c
);
535 bool qir_opt_small_immediates(struct vc4_compile
*c
);
536 bool qir_opt_vpm(struct vc4_compile
*c
);
537 void vc4_nir_lower_blend(nir_shader
*s
, struct vc4_compile
*c
);
538 void vc4_nir_lower_io(nir_shader
*s
, struct vc4_compile
*c
);
539 nir_ssa_def
*vc4_nir_get_state_uniform(struct nir_builder
*b
,
540 enum quniform_contents contents
);
541 nir_ssa_def
*vc4_nir_get_swizzled_channel(struct nir_builder
*b
,
542 nir_ssa_def
**srcs
, int swiz
);
543 void vc4_nir_lower_txf_ms(nir_shader
*s
, struct vc4_compile
*c
);
544 void qir_lower_uniforms(struct vc4_compile
*c
);
546 uint32_t qpu_schedule_instructions(struct vc4_compile
*c
);
548 void qir_SF(struct vc4_compile
*c
, struct qreg src
);
550 static inline struct qreg
551 qir_uniform_ui(struct vc4_compile
*c
, uint32_t ui
)
553 return qir_uniform(c
, QUNIFORM_CONSTANT
, ui
);
556 static inline struct qreg
557 qir_uniform_f(struct vc4_compile
*c
, float f
)
559 return qir_uniform(c
, QUNIFORM_CONSTANT
, fui(f
));
562 #define QIR_ALU0(name) \
563 static inline struct qreg \
564 qir_##name(struct vc4_compile *c) \
566 return qir_emit_def(c, qir_inst(QOP_##name, c->undef, \
567 c->undef, c->undef)); \
569 static inline struct qinst * \
570 qir_##name##_dest(struct vc4_compile *c, struct qreg dest) \
572 return qir_emit_nondef(c, qir_inst(QOP_##name, dest, \
573 c->undef, c->undef)); \
576 #define QIR_ALU1(name) \
577 static inline struct qreg \
578 qir_##name(struct vc4_compile *c, struct qreg a) \
580 return qir_emit_def(c, qir_inst(QOP_##name, c->undef, \
583 static inline struct qinst * \
584 qir_##name##_dest(struct vc4_compile *c, struct qreg dest, \
587 return qir_emit_nondef(c, qir_inst(QOP_##name, dest, a, \
591 #define QIR_ALU2(name) \
592 static inline struct qreg \
593 qir_##name(struct vc4_compile *c, struct qreg a, struct qreg b) \
595 return qir_emit_def(c, qir_inst(QOP_##name, c->undef, a, b)); \
597 static inline struct qinst * \
598 qir_##name##_dest(struct vc4_compile *c, struct qreg dest, \
599 struct qreg a, struct qreg b) \
601 return qir_emit_nondef(c, qir_inst(QOP_##name, dest, a, b)); \
604 #define QIR_NODST_1(name) \
605 static inline struct qinst * \
606 qir_##name(struct vc4_compile *c, struct qreg a) \
608 return qir_emit_nondef(c, qir_inst(QOP_##name, c->undef, \
612 #define QIR_NODST_2(name) \
613 static inline struct qinst * \
614 qir_##name(struct vc4_compile *c, struct qreg a, struct qreg b) \
616 return qir_emit_nondef(c, qir_inst(QOP_##name, c->undef, \
620 #define QIR_PAYLOAD(name) \
621 static inline struct qreg \
622 qir_##name(struct vc4_compile *c) \
624 struct qreg *payload = &c->payload_##name; \
625 if (payload->file != QFILE_NULL) \
627 *payload = qir_get_temp(c); \
628 struct qinst *inst = qir_inst(QOP_##name, *payload, \
629 c->undef, c->undef); \
630 struct qblock *entry = qir_entry_block(c); \
631 list_add(&inst->link, &entry->instructions); \
632 c->defs[payload->index] = inst; \
676 QIR_NODST_2(TEX_DIRECT
)
680 QIR_ALU0(TLB_COLOR_READ
)
683 static inline struct qreg
684 qir_SEL(struct vc4_compile
*c
, uint8_t cond
, struct qreg src0
, struct qreg src1
)
686 struct qreg t
= qir_get_temp(c
);
687 struct qinst
*a
= qir_MOV_dest(c
, t
, src0
);
688 struct qinst
*b
= qir_MOV_dest(c
, t
, src1
);
690 b
->cond
= qpu_cond_complement(cond
);
694 static inline struct qreg
695 qir_UNPACK_8_F(struct vc4_compile
*c
, struct qreg src
, int i
)
697 struct qreg t
= qir_FMOV(c
, src
);
698 c
->defs
[t
.index
]->src
[0].pack
= QPU_UNPACK_8A
+ i
;
702 static inline struct qreg
703 qir_UNPACK_8_I(struct vc4_compile
*c
, struct qreg src
, int i
)
705 struct qreg t
= qir_MOV(c
, src
);
706 c
->defs
[t
.index
]->src
[0].pack
= QPU_UNPACK_8A
+ i
;
710 static inline struct qreg
711 qir_UNPACK_16_F(struct vc4_compile
*c
, struct qreg src
, int i
)
713 struct qreg t
= qir_FMOV(c
, src
);
714 c
->defs
[t
.index
]->src
[0].pack
= QPU_UNPACK_16A
+ i
;
718 static inline struct qreg
719 qir_UNPACK_16_I(struct vc4_compile
*c
, struct qreg src
, int i
)
721 struct qreg t
= qir_MOV(c
, src
);
722 c
->defs
[t
.index
]->src
[0].pack
= QPU_UNPACK_16A
+ i
;
727 qir_PACK_8_F(struct vc4_compile
*c
, struct qreg dest
, struct qreg val
, int chan
)
730 dest
.pack
= QPU_PACK_MUL_8A
+ chan
;
731 qir_emit_nondef(c
, qir_inst(QOP_MMOV
, dest
, val
, c
->undef
));
734 static inline struct qreg
735 qir_PACK_8888_F(struct vc4_compile
*c
, struct qreg val
)
737 struct qreg dest
= qir_MMOV(c
, val
);
738 c
->defs
[dest
.index
]->dst
.pack
= QPU_PACK_MUL_8888
;
742 static inline struct qreg
743 qir_POW(struct vc4_compile
*c
, struct qreg x
, struct qreg y
)
745 return qir_EXP2(c
, qir_FMUL(c
,
751 qir_VPM_WRITE(struct vc4_compile
*c
, struct qreg val
)
753 qir_MOV_dest(c
, qir_reg(QFILE_VPM
, 0), val
);
756 static inline struct qreg
757 qir_LOAD_IMM(struct vc4_compile
*c
, uint32_t val
)
759 return qir_emit_def(c
, qir_inst(QOP_LOAD_IMM
, c
->undef
,
760 qir_reg(QFILE_LOAD_IMM
, val
), c
->undef
));
763 static inline struct qinst
*
764 qir_BRANCH(struct vc4_compile
*c
, uint8_t cond
)
766 struct qinst
*inst
= qir_inst(QOP_BRANCH
, c
->undef
, c
->undef
, c
->undef
);
768 qir_emit_nondef(c
, inst
);
772 #define qir_for_each_block(block, c) \
773 list_for_each_entry(struct qblock, block, &c->blocks, link)
775 #define qir_for_each_block_rev(block, c) \
776 list_for_each_entry_rev(struct qblock, block, &c->blocks, link)
778 /* Loop over the non-NULL members of the successors array. */
779 #define qir_for_each_successor(succ, block) \
780 for (struct qblock *succ = block->successors[0]; \
782 succ = (succ == block->successors[1] ? NULL : \
783 block->successors[1]))
785 #define qir_for_each_inst(inst, block) \
786 list_for_each_entry(struct qinst, inst, &block->instructions, link)
788 #define qir_for_each_inst_rev(inst, block) \
789 list_for_each_entry_rev(struct qinst, inst, &block->instructions, link)
791 #define qir_for_each_inst_safe(inst, block) \
792 list_for_each_entry_safe(struct qinst, inst, &block->instructions, link)
794 #define qir_for_each_inst_inorder(inst, c) \
795 qir_for_each_block(_block, c) \
796 qir_for_each_inst(inst, _block)
798 #endif /* VC4_QIR_H */