X-Git-Url: https://git.libre-soc.org/?a=blobdiff_plain;f=src%2Fpanfrost%2Fmidgard%2Fmidgard_ra.c;h=e4901eed6386667b3e95320e8ffa434545003de3;hb=553c2cf16b7612d4a70bd96230dad63777ec867e;hp=2fe406d6e7aec77e495942ead488594136428f36;hpb=8c79467a0d4d1c605bb93cbed844330f2fd4cbeb;p=mesa.git diff --git a/src/panfrost/midgard/midgard_ra.c b/src/panfrost/midgard/midgard_ra.c index 2fe406d6e7a..e4901eed638 100644 --- a/src/panfrost/midgard/midgard_ra.c +++ b/src/panfrost/midgard/midgard_ra.c @@ -26,7 +26,6 @@ #include "midgard_ops.h" #include "util/u_math.h" #include "util/u_memory.h" -#include "lcra.h" #include "midgard_quirks.h" struct phys_reg { @@ -43,17 +42,17 @@ struct phys_reg { /* Shift up by reg_offset and horizontally by dst_offset. */ static void -offset_swizzle(unsigned *swizzle, unsigned reg_offset, unsigned srcsize, unsigned dst_offset) +offset_swizzle(unsigned *swizzle, unsigned reg_offset, unsigned srcsize, unsigned dstsize, unsigned dst_offset) { unsigned out[MIR_VEC_COMPONENTS]; signed reg_comp = reg_offset / srcsize; - signed dst_comp = dst_offset / srcsize; + signed dst_comp = dst_offset / dstsize; unsigned max_component = (16 / srcsize) - 1; assert(reg_comp * srcsize == reg_offset); - assert(dst_comp * srcsize == dst_offset); + assert(dst_comp * dstsize == dst_offset); for (signed c = 0; c < MIR_VEC_COMPONENTS; ++c) { signed comp = MAX2(c - dst_comp, 0); @@ -66,12 +65,12 @@ offset_swizzle(unsigned *swizzle, unsigned reg_offset, unsigned srcsize, unsigne /* Helper to return the default phys_reg for a given register */ static struct phys_reg -default_phys_reg(int reg, midgard_reg_mode size) +default_phys_reg(int reg, unsigned size) { struct phys_reg r = { .reg = reg, .offset = 0, - .size = mir_bytes_for_mode(size) + .size = size }; return r; @@ -81,7 +80,7 @@ default_phys_reg(int reg, midgard_reg_mode size) * register corresponds to */ static struct phys_reg -index_to_reg(compiler_context *ctx, struct lcra_state *l, unsigned reg, midgard_reg_mode size) +index_to_reg(compiler_context *ctx, struct lcra_state *l, unsigned reg, unsigned size) { /* Check for special cases */ if (reg == ~0) @@ -94,7 +93,7 @@ index_to_reg(compiler_context *ctx, struct lcra_state *l, unsigned reg, midgard_ struct phys_reg r = { .reg = l->solutions[reg] / 16, .offset = l->solutions[reg] & 0xF, - .size = mir_bytes_for_mode(size) + .size = size }; /* Report that we actually use this register, and return it */ @@ -293,7 +292,7 @@ mir_lower_special_reads(compiler_context *ctx) } else { idx = spill_idx++; m = v_mov(i, idx); - m.mask = mir_from_bytemask(mir_bytemask_of_read_components(pre_use, i), midgard_reg_mode_32); + m.mask = mir_from_bytemask(mir_bytemask_of_read_components(pre_use, i), 32); mir_insert_instruction_before(ctx, pre_use, m); mir_rewrite_index_src_single(pre_use, i, idx); } @@ -380,12 +379,35 @@ mir_compute_interference( /* First, we need liveness information to be computed per block */ mir_compute_liveness(ctx); + /* We need to force r1.w live throughout a blend shader */ + + if (ctx->is_blend) { + unsigned r1w = ~0; + + mir_foreach_block(ctx, _block) { + midgard_block *block = (midgard_block *) _block; + mir_foreach_instr_in_block_rev(block, ins) { + if (ins->writeout) + r1w = ins->src[2]; + } + + if (r1w != ~0) + break; + } + + mir_foreach_instr_global(ctx, ins) { + if (ins->dest < ctx->temp_count) + lcra_add_node_interference(l, ins->dest, mir_bytemask(ins), r1w, 0xF); + } + } + /* Now that every block has live_in/live_out computed, we can determine * interference by walking each block linearly. Take live_out at the * end of each block and walk the block backwards. */ - mir_foreach_block(ctx, blk) { - uint16_t *live = mem_dup(blk->live_out, ctx->temp_count * sizeof(uint16_t)); + mir_foreach_block(ctx, _blk) { + midgard_block *blk = (midgard_block *) _blk; + uint16_t *live = mem_dup(_blk->live_out, ctx->temp_count * sizeof(uint16_t)); mir_foreach_instr_in_block_rev(blk, ins) { /* Mark all registers live after the instruction as @@ -427,7 +449,7 @@ allocate_registers(compiler_context *ctx, bool *spilled) if (!ctx->temp_count) return NULL; - struct lcra_state *l = lcra_alloc_equations(ctx->temp_count, 1, 8, 16, 5); + struct lcra_state *l = lcra_alloc_equations(ctx->temp_count, 5); /* Starts of classes, in bytes */ l->class_start[REG_CLASS_WORK] = 16 * 0; @@ -454,34 +476,85 @@ allocate_registers(compiler_context *ctx, bool *spilled) unsigned *found_class = calloc(sizeof(unsigned), ctx->temp_count); unsigned *min_alignment = calloc(sizeof(unsigned), ctx->temp_count); + unsigned *min_bound = calloc(sizeof(unsigned), ctx->temp_count); mir_foreach_instr_global(ctx, ins) { + /* Swizzles of 32-bit sources on 64-bit instructions need to be + * aligned to either bottom (xy) or top (zw). More general + * swizzle lowering should happen prior to scheduling (TODO), + * but once we get RA we shouldn't disrupt this further. Align + * sources of 64-bit instructions. */ + + if (ins->type == TAG_ALU_4 && ins->alu.reg_mode == midgard_reg_mode_64) { + mir_foreach_src(ins, v) { + unsigned s = ins->src[v]; + + if (s < ctx->temp_count) + min_alignment[s] = 3; + } + } + + if (ins->type == TAG_LOAD_STORE_4 && OP_HAS_ADDRESS(ins->load_store.op)) { + mir_foreach_src(ins, v) { + unsigned s = ins->src[v]; + unsigned size = nir_alu_type_get_type_size(ins->src_types[v]); + + if (s < ctx->temp_count) + min_alignment[s] = (size == 64) ? 3 : 2; + } + } + if (ins->dest >= SSA_FIXED_MINIMUM) continue; + unsigned size = nir_alu_type_get_type_size(ins->dest_type); + /* 0 for x, 1 for xy, 2 for xyz, 3 for xyzw */ - int class = util_logbase2(ins->mask); + int comps1 = util_logbase2(ins->mask); + + int bytes = (comps1 + 1) * (size / 8); /* Use the largest class if there's ambiguity, this * handles partial writes */ int dest = ins->dest; - found_class[dest] = MAX2(found_class[dest], class); + found_class[dest] = MAX2(found_class[dest], bytes); - /* XXX: Ensure swizzles align the right way with more LCRA constraints? */ - if (ins->type == TAG_ALU_4 && ins->alu.reg_mode != midgard_reg_mode_32) - min_alignment[dest] = 3; /* (1 << 3) = 8 */ + min_alignment[dest] = + (size == 16) ? 1 : /* (1 << 1) = 2-byte */ + (size == 32) ? 2 : /* (1 << 2) = 4-byte */ + (size == 64) ? 3 : /* (1 << 3) = 8-byte */ + 3; /* 8-bit todo */ - if (ins->type == TAG_LOAD_STORE_4 && ins->load_64) - min_alignment[dest] = 3; + /* We can't cross xy/zw boundaries. TODO: vec8 can */ + if (size == 16) + min_bound[dest] = 8; + + /* We don't have a swizzle for the conditional and we don't + * want to muck with the conditional itself, so just force + * alignment for now */ + + if (ins->type == TAG_ALU_4 && OP_IS_CSEL_V(ins->alu.op)) { + min_alignment[dest] = 4; /* 1 << 4= 16-byte = vec4 */ + + /* LCRA assumes bound >= alignment */ + min_bound[dest] = 16; + } + + /* Since ld/st swizzles and masks are 32-bit only, we need them + * aligned to enable final packing */ + if (ins->type == TAG_LOAD_STORE_4) + min_alignment[dest] = MAX2(min_alignment[dest], 2); } for (unsigned i = 0; i < ctx->temp_count; ++i) { - lcra_set_alignment(l, i, min_alignment[i] ? min_alignment[i] : 2); - lcra_restrict_range(l, i, (found_class[i] + 1) * 4); + lcra_set_alignment(l, i, min_alignment[i] ? min_alignment[i] : 2, + min_bound[i] ? min_bound[i] : 16); + lcra_restrict_range(l, i, found_class[i]); } free(found_class); free(min_alignment); + free(min_bound); /* Next, we'll determine semantic class. We default to zero (work). * But, if we're used with a special operation, that will force us to a @@ -498,13 +571,18 @@ allocate_registers(compiler_context *ctx, bool *spilled) set_class(l->class, ins->src[1], REG_CLASS_LDST); set_class(l->class, ins->src[2], REG_CLASS_LDST); - if (OP_IS_VEC4_ONLY(ins->load_store.op)) + if (OP_IS_VEC4_ONLY(ins->load_store.op)) { lcra_restrict_range(l, ins->dest, 16); + lcra_restrict_range(l, ins->src[0], 16); + lcra_restrict_range(l, ins->src[1], 16); + lcra_restrict_range(l, ins->src[2], 16); + } } else if (ins->type == TAG_TEXTURE_4) { set_class(l->class, ins->dest, REG_CLASS_TEXW); set_class(l->class, ins->src[0], REG_CLASS_TEXR); set_class(l->class, ins->src[1], REG_CLASS_TEXR); set_class(l->class, ins->src[2], REG_CLASS_TEXR); + set_class(l->class, ins->src[3], REG_CLASS_TEXR); } } @@ -516,10 +594,24 @@ allocate_registers(compiler_context *ctx, bool *spilled) assert(check_read_class(l->class, ins->type, ins->src[2])); } - /* Mark writeout to r0 */ + /* Mark writeout to r0, render target to r1.z, unknown to r1.w */ mir_foreach_instr_global(ctx, ins) { - if (ins->compact_branch && ins->writeout && ins->src[0] < ctx->temp_count) - l->solutions[ins->src[0]] = 0; + if (!(ins->compact_branch && ins->writeout)) continue; + + if (ins->src[0] < ctx->temp_count) { + if (ins->writeout_depth) + l->solutions[ins->src[0]] = (16 * 1) + COMPONENT_X * 4; + else if (ins->writeout_stencil) + l->solutions[ins->src[0]] = (16 * 1) + COMPONENT_Y * 4; + else + l->solutions[ins->src[0]] = 0; + } + + if (ins->src[1] < ctx->temp_count) + l->solutions[ins->src[1]] = (16 * 1) + COMPONENT_Z * 4; + + if (ins->src[2] < ctx->temp_count) + l->solutions[ins->src[2]] = (16 * 1) + COMPONENT_W * 4; } mir_compute_interference(ctx, l); @@ -528,6 +620,7 @@ allocate_registers(compiler_context *ctx, bool *spilled) return l; } + /* Once registers have been decided via register allocation * (allocate_registers), we need to rewrite the MIR to use registers instead of * indices */ @@ -538,6 +631,13 @@ install_registers_instr( struct lcra_state *l, midgard_instruction *ins) { + unsigned src_size[MIR_SRC_COUNT]; + + for (unsigned i = 0; i < MIR_SRC_COUNT; ++i) + src_size[i] = MAX2(nir_alu_type_get_type_size(ins->src_types[i]) / 8, 1); + + unsigned dest_size = MAX2(nir_alu_type_get_type_size(ins->dest_type) / 8, 1); + switch (ins->type) { case TAG_ALU_4: case TAG_ALU_8: @@ -546,9 +646,9 @@ install_registers_instr( if (ins->compact_branch) return; - struct phys_reg src1 = index_to_reg(ctx, l, ins->src[0], mir_srcsize(ins, 0)); - struct phys_reg src2 = index_to_reg(ctx, l, ins->src[1], mir_srcsize(ins, 1)); - struct phys_reg dest = index_to_reg(ctx, l, ins->dest, mir_typesize(ins)); + struct phys_reg src1 = index_to_reg(ctx, l, ins->src[0], src_size[0]); + struct phys_reg src2 = index_to_reg(ctx, l, ins->src[1], src_size[1]); + struct phys_reg dest = index_to_reg(ctx, l, ins->dest, dest_size); mir_set_bytemask(ins, mir_bytemask(ins) << dest.offset); @@ -556,7 +656,7 @@ install_registers_instr( GET_CHANNEL_COUNT(alu_opcode_props[ins->alu.op].props) ? 0 : dest.offset; - offset_swizzle(ins->swizzle[0], src1.offset, src1.size, dest_offset); + offset_swizzle(ins->swizzle[0], src1.offset, src1.size, dest.size, dest_offset); ins->registers.src1_reg = src1.reg; @@ -574,10 +674,7 @@ install_registers_instr( ins->alu.src2 = imm << 2; } else { - midgard_vector_alu_src mod2 = - vector_alu_from_unsigned(ins->alu.src2); - offset_swizzle(ins->swizzle[1], src2.offset, src2.size, dest_offset); - ins->alu.src2 = vector_alu_srco_unsigned(mod2); + offset_swizzle(ins->swizzle[1], src2.offset, src2.size, dest.size, dest_offset); ins->registers.src2_reg = src2.reg; } @@ -594,16 +691,16 @@ install_registers_instr( bool encodes_src = OP_IS_STORE(ins->load_store.op); if (encodes_src) { - struct phys_reg src = index_to_reg(ctx, l, ins->src[0], mir_srcsize(ins, 0)); + struct phys_reg src = index_to_reg(ctx, l, ins->src[0], src_size[0]); assert(src.reg == 26 || src.reg == 27); ins->load_store.reg = src.reg - 26; - offset_swizzle(ins->swizzle[0], src.offset, src.size, 0); + offset_swizzle(ins->swizzle[0], src.offset, src.size, 1, 0); } else { - struct phys_reg dst = index_to_reg(ctx, l, ins->dest, mir_typesize(ins)); + struct phys_reg dst = index_to_reg(ctx, l, ins->dest, dest_size); ins->load_store.reg = dst.reg; - offset_swizzle(ins->swizzle[0], 0, 4, dst.offset); + offset_swizzle(ins->swizzle[0], 0, 4, 4, dst.offset); mir_set_bytemask(ins, mir_bytemask(ins) << dst.offset); } @@ -613,14 +710,14 @@ install_registers_instr( unsigned src3 = ins->src[2]; if (src2 != ~0) { - struct phys_reg src = index_to_reg(ctx, l, src2, mir_srcsize(ins, 1)); + struct phys_reg src = index_to_reg(ctx, l, src2, 4); unsigned component = src.offset / src.size; assert(component * src.size == src.offset); ins->load_store.arg_1 |= midgard_ldst_reg(src.reg, component); } if (src3 != ~0) { - struct phys_reg src = index_to_reg(ctx, l, src3, mir_srcsize(ins, 2)); + struct phys_reg src = index_to_reg(ctx, l, src3, 4); unsigned component = src.offset / src.size; assert(component * src.size == src.offset); ins->load_store.arg_2 |= midgard_ldst_reg(src.reg, component); @@ -630,29 +727,31 @@ install_registers_instr( } case TAG_TEXTURE_4: { + if (ins->texture.op == TEXTURE_OP_BARRIER) + break; + /* Grab RA results */ - struct phys_reg dest = index_to_reg(ctx, l, ins->dest, mir_typesize(ins)); - struct phys_reg coord = index_to_reg(ctx, l, ins->src[1], mir_srcsize(ins, 1)); - struct phys_reg lod = index_to_reg(ctx, l, ins->src[2], mir_srcsize(ins, 2)); + struct phys_reg dest = index_to_reg(ctx, l, ins->dest, dest_size); + struct phys_reg coord = index_to_reg(ctx, l, ins->src[1], src_size[1]); + struct phys_reg lod = index_to_reg(ctx, l, ins->src[2], src_size[2]); + struct phys_reg offset = index_to_reg(ctx, l, ins->src[3], src_size[3]); /* First, install the texture coordinate */ - ins->texture.in_reg_full = 1; - ins->texture.in_reg_upper = 0; ins->texture.in_reg_select = coord.reg & 1; - offset_swizzle(ins->swizzle[1], coord.offset, coord.size, 0); + offset_swizzle(ins->swizzle[1], coord.offset, coord.size, dest.size, 0); /* Next, install the destination */ - ins->texture.out_full = 1; - ins->texture.out_upper = 0; ins->texture.out_reg_select = dest.reg & 1; - offset_swizzle(ins->swizzle[0], 0, 4, dest.offset); + offset_swizzle(ins->swizzle[0], 0, 4, dest.size, + dest_size == 2 ? dest.offset % 8 : + dest.offset); mir_set_bytemask(ins, mir_bytemask(ins) << dest.offset); /* If there is a register LOD/bias, use it */ if (ins->src[2] != ~0) { assert(!(lod.offset & 3)); midgard_tex_register_select sel = { - .select = lod.reg, + .select = lod.reg & 1, .full = 1, .component = lod.offset / 4 }; @@ -662,6 +761,24 @@ install_registers_instr( ins->texture.bias = packed; } + /* If there is an offset register, install it */ + if (ins->src[3] != ~0) { + unsigned x = offset.offset / 4; + unsigned y = x + 1; + unsigned z = x + 2; + + /* Check range, TODO: half-registers */ + assert(z < 4); + + ins->texture.offset = + (1) | /* full */ + (offset.reg & 1) << 1 | /* select */ + (0 << 2) | /* upper */ + (x << 3) | /* swizzle */ + (y << 5) | /* swizzle */ + (z << 7); /* swizzle */ + } + break; } @@ -730,7 +847,8 @@ mir_spill_register( if (is_special_w) spill_slot = spill_index++; - mir_foreach_block(ctx, block) { + mir_foreach_block(ctx, _block) { + midgard_block *block = (midgard_block *) _block; mir_foreach_instr_in_block_safe(block, ins) { if (ins->dest != spill_node) continue; @@ -772,7 +890,8 @@ mir_spill_register( * work registers to back special registers; TLS * spilling is to use memory to back work registers) */ - mir_foreach_block(ctx, block) { + mir_foreach_block(ctx, _block) { + midgard_block *block = (midgard_block *) _block; mir_foreach_instr_in_block(block, ins) { /* We can't rewrite the moves used to spill in the * first place. These moves are hinted. */ @@ -801,7 +920,7 @@ mir_spill_register( /* Mask the load based on the component count * actually needed to prevent RA loops */ - st.mask = mir_from_bytemask(read_bytemask, midgard_reg_mode_32); + st.mask = mir_from_bytemask(read_bytemask, 32); mir_insert_instruction_before_scheduled(ctx, block, before, st); } else {