#include "midgard.h"
#include "helpers.h"
#include "midgard_compile.h"
+#include "midgard_ops.h"
+#include "lcra.h"
#include "util/hash_table.h"
#include "util/u_dynarray.h"
#include "main/mtypes.h"
#include "compiler/nir_types.h"
#include "compiler/nir/nir.h"
+#include "panfrost/util/pan_ir.h"
/* Forward declare */
struct midgard_block;
* emitted before the register allocation pass.
*/
+#define MIR_SRC_COUNT 4
+#define MIR_VEC_COMPONENTS 16
+
typedef struct midgard_instruction {
/* Must be first for casting */
struct list_head link;
/* Instruction arguments represented as block-local SSA
* indices, rather than registers. ~0 means unused. */
- unsigned src[3];
+ unsigned src[MIR_SRC_COUNT];
unsigned dest;
- /* Swizzle for the conditional for a csel */
- unsigned csel_swizzle;
+ /* vec16 swizzle, unpacked, per source */
+ unsigned swizzle[MIR_SRC_COUNT][MIR_VEC_COMPONENTS];
/* Special fields for an ALU instruction */
midgard_reg_info registers;
/* I.e. (1 << alu_bit) */
int unit;
- /* When emitting bundle, should this instruction have a break forced
- * before it? Used for r31 writes which are valid only within a single
- * bundle and *need* to happen as early as possible... this is a hack,
- * TODO remove when we have a scheduler */
- bool precede_break;
-
bool has_constants;
- uint32_t constants[4];
+ midgard_constants constants;
uint16_t inline_constant;
bool has_blend_constant;
bool has_inline_constant;
bool compact_branch;
bool writeout;
- bool prepacked_branch;
+ bool writeout_depth;
+ bool writeout_stencil;
+ bool last_writeout;
/* Kind of a hack, but hint against aggressive DCE */
bool dont_eliminate;
uint16_t mask;
/* For ALU ops only: set to true to invert (bitwise NOT) the
- * destination of an integer-out op. Not imeplemented in hardware but
+ * destination of an integer-out op. Not implemented in hardware but
* allows more optimizations */
bool invert;
/* Hint for the register allocator not to spill the destination written
- * from this instruction (because it is a spill/unspill node itself) */
+ * from this instruction (because it is a spill/unspill node itself).
+ * Bitmask of spilled classes */
- bool no_spill;
+ unsigned no_spill;
/* Generic hint for intra-pass use */
bool hint;
+ /* During scheduling, the backwards dependency graph
+ * (DAG). nr_dependencies is the number of unscheduled
+ * instructions that must still be scheduled after
+ * (before) this instruction. dependents are which
+ * instructions need to be scheduled before (after) this
+ * instruction. */
+
+ unsigned nr_dependencies;
+ BITSET_WORD *dependents;
+
+ /* For load/store ops.. force 64-bit destination */
+ bool load_64;
+
union {
midgard_load_store_word load_store;
midgard_vector_alu alu;
/* In liveness analysis, these are live masks (per-component) for
* indices for the block. Scalar compilers have the luxury of using
- * simple bit fields, but for us, liveness is a vector idea. We use
- * 8-bit to allow finegrained tracking up to vec8. If you're
- * implementing vec16 on Panfrost... I'm sorry. */
- uint8_t *live_in;
- uint8_t *live_out;
+ * simple bit fields, but for us, liveness is a vector idea. */
+ uint16_t *live_in;
+ uint16_t *live_out;
+
+ /* Indicates this is a fixed-function fragment epilogue block */
+ bool epilogue;
} midgard_block;
typedef struct midgard_bundle {
/* Tag for the overall bundle */
int tag;
- /* Instructions contained by the bundle */
+ /* Instructions contained by the bundle. instruction_count <= 6 (vmul,
+ * sadd, vadd, smul, vlut, branch) */
int instruction_count;
- midgard_instruction *instructions[5];
+ midgard_instruction *instructions[6];
/* Bundle-wide ALU configuration */
int padding;
int control;
bool has_embedded_constants;
- float constants[4];
+ midgard_constants constants;
bool has_blend_constant;
+ bool last_writeout;
} midgard_bundle;
+enum midgard_rt_id {
+ MIDGARD_COLOR_RT0,
+ MIDGARD_COLOR_RT1,
+ MIDGARD_COLOR_RT2,
+ MIDGARD_COLOR_RT3,
+ MIDGARD_ZS_RT,
+ MIDGARD_NUM_RTS,
+};
+
+struct panfrost_sysvals {
+ /* The mapping of sysvals to uniforms, the count, and the off-by-one inverse */
+ unsigned sysvals[MAX_SYSVAL_COUNT];
+ unsigned sysval_count;
+ struct hash_table_u64 *sysval_to_id;
+};
+
typedef struct compiler_context {
nir_shader *nir;
gl_shader_stage stage;
- /* The screen we correspond to */
- struct midgard_screen *screen;
-
/* Is internally a blend shader? Depends on stage == FRAGMENT */
bool is_blend;
+ /* Render target number for a keyed blend shader. Depends on is_blend */
+ unsigned blend_rt;
+
/* Tracking for blend constant patching */
int blend_constant_offset;
* Decrease when a tex op is removed. */
int texture_op_count;
- /* Mapping of texture register -> SSA index for unaliasing */
- int texture_index[2];
-
/* The number of uniforms allowable for the fast path */
int uniform_cutoff;
unsigned quadword_count;
- /* The mapping of sysvals to uniforms, the count, and the off-by-one inverse */
- unsigned sysvals[MAX_SYSVAL_COUNT];
- unsigned sysval_count;
- struct hash_table_u64 *sysval_to_id;
+ /* Bitmask of valid metadata */
+ unsigned metadata;
+
+ /* Model-specific quirk set */
+ uint32_t quirks;
+
+ /* Writeout instructions for each render target */
+ midgard_instruction *writeout_branch[MIDGARD_NUM_RTS];
+
+ struct panfrost_sysvals sysvals;
} compiler_context;
+/* Per-block live_in/live_out */
+#define MIDGARD_METADATA_LIVENESS (1 << 0)
+
/* Helpers for manipulating the above structures (forming the driver IR) */
/* Append instruction to end of current block */
#define mir_foreach_block_from(ctx, from, v) \
list_for_each_entry_from(struct midgard_block, v, from, &ctx->blocks, link)
-#define mir_foreach_instr(ctx, v) \
- list_for_each_entry(struct midgard_instruction, v, &ctx->current_block->instructions, link)
-
-#define mir_foreach_instr_safe(ctx, v) \
- list_for_each_entry_safe(struct midgard_instruction, v, &ctx->current_block->instructions, link)
-
#define mir_foreach_instr_in_block(block, v) \
list_for_each_entry(struct midgard_instruction, v, &block->instructions, link)
#define mir_foreach_instr_in_block_rev(block, v) \
mir_foreach_bundle_in_block_rev(block, _bundle) \
for (i = (_bundle->instruction_count - 1), v = _bundle->instructions[i]; \
i >= 0; \
- --i, v = _bundle->instructions[i]) \
+ --i, v = (i >= 0) ? _bundle->instructions[i] : NULL) \
#define mir_foreach_instr_global(ctx, v) \
mir_foreach_block(ctx, v_block) \
return ((ctx->func->impl->reg_alloc + ctx->temp_alloc++) << 1) | IS_REG;
}
+static inline unsigned
+nir_ssa_index(nir_ssa_def *ssa)
+{
+ return (ssa->index << 1) | 0;
+}
+
static inline unsigned
nir_src_index(compiler_context *ctx, nir_src *src)
{
if (src->is_ssa)
- return (src->ssa->index << 1) | 0;
+ return nir_ssa_index(src->ssa);
else {
assert(!src->reg.indirect);
return (src->reg.reg->index << 1) | IS_REG;
}
static inline unsigned
-nir_dest_index(compiler_context *ctx, nir_dest *dst)
+nir_dest_index(nir_dest *dst)
{
if (dst->is_ssa)
return (dst->ssa.index << 1) | 0;
/* MIR manipulation */
-unsigned mir_get_swizzle(midgard_instruction *ins, unsigned idx);
-void mir_set_swizzle(midgard_instruction *ins, unsigned idx, unsigned new);
void mir_rewrite_index(compiler_context *ctx, unsigned old, unsigned new);
void mir_rewrite_index_src(compiler_context *ctx, unsigned old, unsigned new);
void mir_rewrite_index_dst(compiler_context *ctx, unsigned old, unsigned new);
-void mir_rewrite_index_dst_tag(compiler_context *ctx, unsigned old, unsigned new, unsigned tag);
void mir_rewrite_index_dst_single(midgard_instruction *ins, unsigned old, unsigned new);
void mir_rewrite_index_src_single(midgard_instruction *ins, unsigned old, unsigned new);
-void mir_rewrite_index_src_tag(compiler_context *ctx, unsigned old, unsigned new, unsigned tag);
-void mir_rewrite_index_src_swizzle(compiler_context *ctx, unsigned old, unsigned new, unsigned swizzle);
+void mir_rewrite_index_src_swizzle(compiler_context *ctx, unsigned old, unsigned new, unsigned *swizzle);
bool mir_single_use(compiler_context *ctx, unsigned value);
bool mir_special_index(compiler_context *ctx, unsigned idx);
unsigned mir_use_count(compiler_context *ctx, unsigned value);
bool mir_is_written_before(compiler_context *ctx, midgard_instruction *ins, unsigned node);
-unsigned mir_mask_of_read_components(midgard_instruction *ins, unsigned node);
-unsigned mir_ubo_shift(midgard_load_store_op op);
+uint16_t mir_bytemask_of_read_components(midgard_instruction *ins, unsigned node);
+uint16_t mir_bytemask_of_read_components_index(midgard_instruction *ins, unsigned i);
+midgard_reg_mode mir_typesize(midgard_instruction *ins);
+midgard_reg_mode mir_srcsize(midgard_instruction *ins, unsigned i);
+unsigned mir_bytes_for_mode(midgard_reg_mode mode);
+midgard_reg_mode mir_mode_for_destsize(unsigned size);
+uint16_t mir_from_bytemask(uint16_t bytemask, midgard_reg_mode mode);
+uint16_t mir_bytemask(midgard_instruction *ins);
+uint16_t mir_round_bytemask_up(uint16_t mask, midgard_reg_mode mode);
+void mir_set_bytemask(midgard_instruction *ins, uint16_t bytemask);
+unsigned mir_upper_override(midgard_instruction *ins);
/* MIR printing */
void mir_print_shader(compiler_context *ctx);
bool mir_nontrivial_source2_mod(midgard_instruction *ins);
bool mir_nontrivial_source2_mod_simple(midgard_instruction *ins);
-bool mir_nontrivial_mod(midgard_vector_alu_src src, bool is_int, unsigned mask);
bool mir_nontrivial_outmod(midgard_instruction *ins);
void mir_insert_instruction_before_scheduled(compiler_context *ctx, midgard_block *block, midgard_instruction *tag, midgard_instruction ins);
void mir_insert_instruction_after_scheduled(compiler_context *ctx, midgard_block *block, midgard_instruction *tag, midgard_instruction ins);
+void mir_flip(midgard_instruction *ins);
+void mir_compute_temp_count(compiler_context *ctx);
-/* MIR goodies */
-
-static const midgard_vector_alu_src blank_alu_src = {
- .swizzle = SWIZZLE(COMPONENT_X, COMPONENT_Y, COMPONENT_Z, COMPONENT_W),
-};
-
-static const midgard_vector_alu_src blank_alu_src_xxxx = {
- .swizzle = SWIZZLE(COMPONENT_X, COMPONENT_X, COMPONENT_X, COMPONENT_X),
-};
-
-static const midgard_scalar_alu_src blank_scalar_alu_src = {
- .full = true
-};
-
-/* Used for encoding the unused source of 1-op instructions */
-static const midgard_vector_alu_src zero_alu_src = { 0 };
+void mir_set_offset(compiler_context *ctx, midgard_instruction *ins, nir_src *offset, bool is_shared);
/* 'Intrinsic' move for aliasing */
static inline midgard_instruction
-v_mov(unsigned src, midgard_vector_alu_src mod, unsigned dest)
+v_mov(unsigned src, unsigned dest)
{
midgard_instruction ins = {
.type = TAG_ALU_4,
.mask = 0xF,
- .src = { SSA_UNUSED, src, SSA_UNUSED },
+ .src = { ~0, src, ~0, ~0 },
+ .swizzle = SWIZZLE_IDENTITY,
.dest = dest,
.alu = {
.op = midgard_alu_op_imov,
.reg_mode = midgard_reg_mode_32,
.dest_override = midgard_dest_override_none,
- .outmod = midgard_outmod_int_wrap,
- .src1 = vector_alu_srco_unsigned(zero_alu_src),
- .src2 = vector_alu_srco_unsigned(mod)
+ .outmod = midgard_outmod_int_wrap
},
};
return ins;
}
+/* Broad types of register classes so we can handle special
+ * registers */
+
+#define REG_CLASS_WORK 0
+#define REG_CLASS_LDST 1
+#define REG_CLASS_TEXR 3
+#define REG_CLASS_TEXW 4
+
+/* Like a move, but to thread local storage! */
+
+static inline midgard_instruction
+v_load_store_scratch(
+ unsigned srcdest,
+ unsigned index,
+ bool is_store,
+ unsigned mask)
+{
+ /* We index by 32-bit vec4s */
+ unsigned byte = (index * 4 * 4);
+
+ midgard_instruction ins = {
+ .type = TAG_LOAD_STORE_4,
+ .mask = mask,
+ .dest = ~0,
+ .src = { ~0, ~0, ~0, ~0 },
+ .swizzle = SWIZZLE_IDENTITY_4,
+ .load_store = {
+ .op = is_store ? midgard_op_st_int4 : midgard_op_ld_int4,
+
+ /* For register spilling - to thread local storage */
+ .arg_1 = 0xEA,
+ .arg_2 = 0x1E,
+ },
+
+ /* If we spill an unspill, RA goes into an infinite loop */
+ .no_spill = (1 << REG_CLASS_WORK)
+ };
+
+ ins.constants.u32[0] = byte;
+
+ if (is_store) {
+ ins.src[0] = srcdest;
+
+ /* Ensure we are tightly swizzled so liveness analysis is
+ * correct */
+
+ for (unsigned i = 0; i < 4; ++i) {
+ if (!(mask & (1 << i)))
+ ins.swizzle[0][i] = COMPONENT_X;
+ }
+ } else
+ ins.dest = srcdest;
+
+ return ins;
+}
+
static inline bool
mir_has_arg(midgard_instruction *ins, unsigned arg)
{
/* Scheduling */
-void schedule_program(compiler_context *ctx);
-
-/* Register allocation */
-
-struct ra_graph;
-
-/* Broad types of register classes so we can handle special
- * registers */
-
-#define NR_REG_CLASSES 5
-
-#define REG_CLASS_WORK 0
-#define REG_CLASS_LDST 1
-#define REG_CLASS_LDST27 2
-#define REG_CLASS_TEXR 3
-#define REG_CLASS_TEXW 4
+void midgard_schedule_program(compiler_context *ctx);
+void mir_ra(compiler_context *ctx);
+void mir_squeeze_index(compiler_context *ctx);
void mir_lower_special_reads(compiler_context *ctx);
-struct ra_graph* allocate_registers(compiler_context *ctx, bool *spilled);
-void install_registers(compiler_context *ctx, struct ra_graph *g);
+void mir_liveness_ins_update(uint16_t *live, midgard_instruction *ins, unsigned max);
+void mir_compute_liveness(compiler_context *ctx);
+void mir_invalidate_liveness(compiler_context *ctx);
bool mir_is_live_after(compiler_context *ctx, midgard_block *block, midgard_instruction *start, int src);
-bool mir_has_multiple_writes(compiler_context *ctx, int src);
void mir_create_pipeline_registers(compiler_context *ctx);
-
-void
-midgard_promote_uniforms(compiler_context *ctx, unsigned promoted_count);
-
-midgard_instruction *
-emit_ubo_read(
- compiler_context *ctx,
- nir_instr *instr,
- unsigned dest,
- unsigned offset,
- nir_src *indirect_offset,
- unsigned index);
-
-void
-emit_sysval_read(compiler_context *ctx, nir_instr *instr, signed dest_override, unsigned nr_components);
+void midgard_promote_uniforms(compiler_context *ctx);
void
midgard_emit_derivatives(compiler_context *ctx, nir_alu_instr *instr);
void
midgard_lower_derivatives(compiler_context *ctx, midgard_block *block);
-bool mir_op_computes_derivatives(unsigned op);
+bool mir_op_computes_derivatives(gl_shader_stage stage, unsigned op);
/* Final emission */
struct util_dynarray *emission,
int next_tag);
-/* NIR stuff. TODO: Move? Share? Something? */
-
bool
nir_undef_to_zero(nir_shader *shader);
-void
-nir_clamp_psiz(nir_shader *shader, float min_size, float max_size);
+void midgard_nir_lod_errata(nir_shader *shader);
/* Optimizations */
bool midgard_opt_varying_projection(compiler_context *ctx, midgard_block *block);
bool midgard_opt_dead_code_eliminate(compiler_context *ctx, midgard_block *block);
bool midgard_opt_dead_move_eliminate(compiler_context *ctx, midgard_block *block);
-void midgard_opt_post_move_eliminate(compiler_context *ctx, midgard_block *block, struct ra_graph *g);
void midgard_lower_invert(compiler_context *ctx, midgard_block *block);
bool midgard_opt_not_propagate(compiler_context *ctx, midgard_block *block);
bool midgard_opt_fuse_src_invert(compiler_context *ctx, midgard_block *block);
bool midgard_opt_fuse_dest_invert(compiler_context *ctx, midgard_block *block);
+bool midgard_opt_csel_invert(compiler_context *ctx, midgard_block *block);
bool midgard_opt_promote_fmov(compiler_context *ctx, midgard_block *block);
+bool midgard_opt_drop_cmp_invert(compiler_context *ctx, midgard_block *block);
+bool midgard_opt_invert_branch(compiler_context *ctx, midgard_block *block);
#endif
projects / mesa.git / blobdiff