X-Git-Url: https://git.libre-soc.org/?p=mesa.git;a=blobdiff_plain;f=src%2Famd%2Fcompiler%2Faco_instruction_selection.cpp;h=0a92788567a80382161667f5bcdfd7da002c6aa2;hp=f83aae2b179a05ce1bab6fb001dd70e0a5a8809b;hb=ccfe9813fb80ebff5d1d3f70cb95f1831be8844e;hpb=462a5fe6f44e03e1f39238174628beda3e43fa18 diff --git a/src/amd/compiler/aco_instruction_selection.cpp b/src/amd/compiler/aco_instruction_selection.cpp index f83aae2b179..0a92788567a 100644 --- a/src/amd/compiler/aco_instruction_selection.cpp +++ b/src/amd/compiler/aco_instruction_selection.cpp @@ -136,8 +136,11 @@ Temp emit_mbcnt(isel_context *ctx, Definition dst, if (ctx->program->wave_size == 32) { return thread_id_lo; + } else if (ctx->program->chip_class <= GFX7) { + Temp thread_id_hi = bld.vop2(aco_opcode::v_mbcnt_hi_u32_b32, dst, mask_hi, thread_id_lo); + return thread_id_hi; } else { - Temp thread_id_hi = bld.vop3(aco_opcode::v_mbcnt_hi_u32_b32, dst, mask_hi, thread_id_lo); + Temp thread_id_hi = bld.vop3(aco_opcode::v_mbcnt_hi_u32_b32_e64, dst, mask_hi, thread_id_lo); return thread_id_hi; } } @@ -169,33 +172,69 @@ static Temp emit_bpermute(isel_context *ctx, Builder &bld, Temp index, Temp data if (index.regClass() == s1) return bld.readlane(bld.def(s1), data, index); - Temp index_x4 = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(2u), index); - - /* Currently not implemented on GFX6-7 */ - assert(ctx->options->chip_class >= GFX8); - - if (ctx->options->chip_class <= GFX9 || ctx->program->wave_size == 32) { + if (ctx->options->chip_class <= GFX7) { + /* GFX6-7: there is no bpermute instruction */ + Operand index_op(index); + Operand input_data(data); + index_op.setLateKill(true); + input_data.setLateKill(true); + + return bld.pseudo(aco_opcode::p_bpermute, bld.def(v1), bld.def(bld.lm), bld.def(bld.lm, vcc), index_op, input_data); + } else if (ctx->options->chip_class >= GFX10 && ctx->program->wave_size == 64) { + /* GFX10 wave64 mode: emulate full-wave bpermute */ + if (!ctx->has_gfx10_wave64_bpermute) { + ctx->has_gfx10_wave64_bpermute = true; + ctx->program->config->num_shared_vgprs = 8; /* Shared VGPRs are allocated in groups of 8 */ + ctx->program->vgpr_limit -= 4; /* We allocate 8 shared VGPRs, so we'll have 4 fewer normal VGPRs */ + } + + Temp index_is_lo = bld.vopc(aco_opcode::v_cmp_ge_u32, bld.def(bld.lm), Operand(31u), index); + Builder::Result index_is_lo_split = bld.pseudo(aco_opcode::p_split_vector, bld.def(s1), bld.def(s1), index_is_lo); + Temp index_is_lo_n1 = bld.sop1(aco_opcode::s_not_b32, bld.def(s1), bld.def(s1, scc), index_is_lo_split.def(1).getTemp()); + Operand same_half = bld.pseudo(aco_opcode::p_create_vector, bld.def(s2), index_is_lo_split.def(0).getTemp(), index_is_lo_n1); + Operand index_x4 = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(2u), index); + Operand input_data(data); + + index_x4.setLateKill(true); + input_data.setLateKill(true); + same_half.setLateKill(true); + + return bld.pseudo(aco_opcode::p_bpermute, bld.def(v1), bld.def(s2), bld.def(s1, scc), index_x4, input_data, same_half); + } else { + /* GFX8-9 or GFX10 wave32: bpermute works normally */ + Temp index_x4 = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(2u), index); return bld.ds(aco_opcode::ds_bpermute_b32, bld.def(v1), index_x4, data); } +} - /* GFX10, wave64 mode: - * The bpermute instruction is limited to half-wave operation, which means that it can't - * properly support subgroup shuffle like older generations (or wave32 mode), so we - * emulate it here. - */ - if (!ctx->has_gfx10_wave64_bpermute) { - ctx->has_gfx10_wave64_bpermute = true; - ctx->program->config->num_shared_vgprs = 8; /* Shared VGPRs are allocated in groups of 8 */ - ctx->program->vgpr_limit -= 4; /* We allocate 8 shared VGPRs, so we'll have 4 fewer normal VGPRs */ - } +static Temp emit_masked_swizzle(isel_context *ctx, Builder &bld, Temp src, unsigned mask) +{ + if (ctx->options->chip_class >= GFX8) { + unsigned and_mask = mask & 0x1f; + unsigned or_mask = (mask >> 5) & 0x1f; + unsigned xor_mask = (mask >> 10) & 0x1f; - Temp lane_id = emit_mbcnt(ctx, bld.def(v1)); - Temp lane_is_hi = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(0x20u), lane_id); - Temp index_is_hi = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(0x20u), index); - Temp cmp = bld.vopc(aco_opcode::v_cmp_eq_u32, bld.def(bld.lm, vcc), lane_is_hi, index_is_hi); + uint16_t dpp_ctrl = 0xffff; - return bld.reduction(aco_opcode::p_wave64_bpermute, bld.def(v1), bld.def(s2), bld.def(s1, scc), - bld.vcc(cmp), Operand(v2.as_linear()), index_x4, data, gfx10_wave64_bpermute); + // TODO: we could use DPP8 for some swizzles + if (and_mask == 0x1f && or_mask < 4 && xor_mask < 4) { + unsigned res[4] = {0, 1, 2, 3}; + for (unsigned i = 0; i < 4; i++) + res[i] = ((res[i] | or_mask) ^ xor_mask) & 0x3; + dpp_ctrl = dpp_quad_perm(res[0], res[1], res[2], res[3]); + } else if (and_mask == 0x1f && !or_mask && xor_mask == 8) { + dpp_ctrl = dpp_row_rr(8); + } else if (and_mask == 0x1f && !or_mask && xor_mask == 0xf) { + dpp_ctrl = dpp_row_mirror; + } else if (and_mask == 0x1f && !or_mask && xor_mask == 0x7) { + dpp_ctrl = dpp_row_half_mirror; + } + + if (dpp_ctrl != 0xffff) + return bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), src, dpp_ctrl); + } + + return bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), src, mask, 0, false); } Temp as_vgpr(isel_context *ctx, Temp val) @@ -396,7 +435,7 @@ void byte_align_scalar(isel_context *ctx, Temp vec, Operand offset, Temp dst) bld.pseudo(aco_opcode::p_split_vector, Definition(lo), Definition(hi), vec); hi = bld.pseudo(aco_opcode::p_extract_vector, bld.def(s1), hi, Operand(0u)); if (select != Temp()) - hi = bld.sop2(aco_opcode::s_cselect_b32, bld.def(s1), hi, Operand(0u), select); + hi = bld.sop2(aco_opcode::s_cselect_b32, bld.def(s1), hi, Operand(0u), bld.scc(select)); lo = bld.sop2(aco_opcode::s_lshr_b64, bld.def(s2), bld.def(s1, scc), lo, shift); Temp mid = bld.tmp(s1); lo = bld.pseudo(aco_opcode::p_split_vector, bld.def(s1), Definition(mid), lo); @@ -407,38 +446,67 @@ void byte_align_scalar(isel_context *ctx, Temp vec, Operand offset, Temp dst) } } -/* this function trims subdword vectors: - * if dst is vgpr - split the src and create a shrunk version according to the mask. - * if dst is sgpr - split the src, but move the original to sgpr. */ -void trim_subdword_vector(isel_context *ctx, Temp vec_src, Temp dst, unsigned num_components, unsigned mask) +void byte_align_vector(isel_context *ctx, Temp vec, Operand offset, Temp dst, unsigned component_size) { - assert(vec_src.type() == RegType::vgpr); - emit_split_vector(ctx, vec_src, num_components); - Builder bld(ctx->program, ctx->block); - std::array elems; - unsigned component_size = vec_src.bytes() / num_components; - RegClass rc = RegClass(RegType::vgpr, component_size).as_subdword(); + if (offset.isTemp()) { + Temp tmp[4] = {vec, vec, vec, vec}; - unsigned k = 0; - for (unsigned i = 0; i < num_components; i++) { - if (mask & (1 << i)) - elems[k++] = emit_extract_vector(ctx, vec_src, i, rc); + if (vec.size() == 4) { + tmp[0] = bld.tmp(v1), tmp[1] = bld.tmp(v1), tmp[2] = bld.tmp(v1), tmp[3] = bld.tmp(v1); + bld.pseudo(aco_opcode::p_split_vector, Definition(tmp[0]), Definition(tmp[1]), Definition(tmp[2]), Definition(tmp[3]), vec); + } else if (vec.size() == 3) { + tmp[0] = bld.tmp(v1), tmp[1] = bld.tmp(v1), tmp[2] = bld.tmp(v1); + bld.pseudo(aco_opcode::p_split_vector, Definition(tmp[0]), Definition(tmp[1]), Definition(tmp[2]), vec); + } else if (vec.size() == 2) { + tmp[0] = bld.tmp(v1), tmp[1] = bld.tmp(v1), tmp[2] = tmp[1]; + bld.pseudo(aco_opcode::p_split_vector, Definition(tmp[0]), Definition(tmp[1]), vec); + } + for (unsigned i = 0; i < dst.size(); i++) + tmp[i] = bld.vop3(aco_opcode::v_alignbyte_b32, bld.def(v1), tmp[i + 1], tmp[i], offset); + + vec = tmp[0]; + if (dst.size() == 2) + vec = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), tmp[0], tmp[1]); + + offset = Operand(0u); + } + + unsigned num_components = vec.bytes() / component_size; + if (vec.regClass() == dst.regClass()) { + assert(offset.constantValue() == 0); + bld.copy(Definition(dst), vec); + emit_split_vector(ctx, dst, num_components); + return; } + emit_split_vector(ctx, vec, num_components); + std::array elems; + RegClass rc = RegClass(RegType::vgpr, component_size).as_subdword(); + + assert(offset.constantValue() % component_size == 0); + unsigned skip = offset.constantValue() / component_size; + for (unsigned i = skip; i < num_components; i++) + elems[i - skip] = emit_extract_vector(ctx, vec, i, rc); + + /* if dst is vgpr - split the src and create a shrunk version according to the mask. */ if (dst.type() == RegType::vgpr) { - assert(dst.bytes() == k * component_size); - aco_ptr vec{create_instruction(aco_opcode::p_create_vector, Format::PSEUDO, k, 1)}; - for (unsigned i = 0; i < k; i++) - vec->operands[i] = Operand(elems[i]); - vec->definitions[0] = Definition(dst); - bld.insert(std::move(vec)); + num_components = dst.bytes() / component_size; + aco_ptr create_vec{create_instruction(aco_opcode::p_create_vector, Format::PSEUDO, num_components, 1)}; + for (unsigned i = 0; i < num_components; i++) + create_vec->operands[i] = Operand(elems[i]); + create_vec->definitions[0] = Definition(dst); + bld.insert(std::move(create_vec)); + + /* if dst is sgpr - split the src, but move the original to sgpr. */ + } else if (skip) { + vec = bld.pseudo(aco_opcode::p_as_uniform, bld.def(RegClass(RegType::sgpr, vec.size())), vec); + byte_align_scalar(ctx, vec, offset, dst); } else { - // TODO: alignbyte if mask doesn't start with 1? - assert(mask & 1); - assert(dst.size() == vec_src.size()); - bld.pseudo(aco_opcode::p_as_uniform, Definition(dst), vec_src); + assert(dst.size() == vec.size()); + bld.pseudo(aco_opcode::p_as_uniform, Definition(dst), vec); } + ctx->allocated_vec.emplace(dst.id(), elems); } @@ -547,6 +615,8 @@ void emit_sop2_instruction(isel_context *ctx, nir_alu_instr *instr, aco_opcode o sop2->operands[0] = Operand(get_alu_src(ctx, instr->src[0])); sop2->operands[1] = Operand(get_alu_src(ctx, instr->src[1])); sop2->definitions[0] = Definition(dst); + if (instr->no_unsigned_wrap) + sop2->definitions[0].setNUW(true); if (writes_scc) sop2->definitions[1] = Definition(ctx->program->allocateId(), scc, s1); ctx->block->instructions.emplace_back(std::move(sop2)); @@ -556,6 +626,8 @@ void emit_vop2_instruction(isel_context *ctx, nir_alu_instr *instr, aco_opcode o bool commutative, bool swap_srcs=false, bool flush_denorms = false) { Builder bld(ctx->program, ctx->block); + bld.is_precise = instr->exact; + Temp src0 = get_alu_src(ctx, instr->src[swap_srcs ? 1 : 0]); Temp src1 = get_alu_src(ctx, instr->src[swap_srcs ? 0 : 1]); if (src1.type() == RegType::sgpr) { @@ -577,6 +649,31 @@ void emit_vop2_instruction(isel_context *ctx, nir_alu_instr *instr, aco_opcode o } } +void emit_vop2_instruction_logic64(isel_context *ctx, nir_alu_instr *instr, + aco_opcode op, Temp dst) +{ + Builder bld(ctx->program, ctx->block); + bld.is_precise = instr->exact; + + Temp src0 = get_alu_src(ctx, instr->src[0]); + Temp src1 = get_alu_src(ctx, instr->src[1]); + + if (src1.type() == RegType::sgpr) { + assert(src0.type() == RegType::vgpr); + std::swap(src0, src1); + } + + Temp src00 = bld.tmp(src0.type(), 1); + Temp src01 = bld.tmp(src0.type(), 1); + bld.pseudo(aco_opcode::p_split_vector, Definition(src00), Definition(src01), src0); + Temp src10 = bld.tmp(v1); + Temp src11 = bld.tmp(v1); + bld.pseudo(aco_opcode::p_split_vector, Definition(src10), Definition(src11), src1); + Temp lo = bld.vop2(op, bld.def(v1), src00, src10); + Temp hi = bld.vop2(op, bld.def(v1), src01, src11); + bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lo, hi); +} + void emit_vop3a_instruction(isel_context *ctx, nir_alu_instr *instr, aco_opcode op, Temp dst, bool flush_denorms = false) { @@ -594,6 +691,7 @@ void emit_vop3a_instruction(isel_context *ctx, nir_alu_instr *instr, aco_opcode src2 = as_vgpr(ctx, src2); Builder bld(ctx->program, ctx->block); + bld.is_precise = instr->exact; if (flush_denorms && ctx->program->chip_class < GFX9) { assert(dst.size() == 1); Temp tmp = bld.vop3(op, Definition(dst), src0, src1, src2); @@ -606,7 +704,12 @@ void emit_vop3a_instruction(isel_context *ctx, nir_alu_instr *instr, aco_opcode void emit_vop1_instruction(isel_context *ctx, nir_alu_instr *instr, aco_opcode op, Temp dst) { Builder bld(ctx->program, ctx->block); - bld.vop1(op, Definition(dst), get_alu_src(ctx, instr->src[0])); + bld.is_precise = instr->exact; + if (dst.type() == RegType::sgpr) + bld.pseudo(aco_opcode::p_as_uniform, Definition(dst), + bld.vop1(op, bld.def(RegType::vgpr, dst.size()), get_alu_src(ctx, instr->src[0]))); + else + bld.vop1(op, Definition(dst), get_alu_src(ctx, instr->src[0])); } void emit_vopc_instruction(isel_context *ctx, nir_alu_instr *instr, aco_opcode op, Temp dst) @@ -912,7 +1015,8 @@ Temp emit_floor_f64(isel_context *ctx, Builder& bld, Definition dst, Temp val) if (ctx->options->chip_class >= GFX7) return bld.vop1(aco_opcode::v_floor_f64, Definition(dst), val); - /* GFX6 doesn't support V_FLOOR_F64, lower it. */ + /* GFX6 doesn't support V_FLOOR_F64, lower it (note that it's actually + * lowered at NIR level for precision reasons). */ Temp src0 = as_vgpr(ctx, val); Temp mask = bld.copy(bld.def(s1), Operand(3u)); /* isnan */ @@ -938,7 +1042,7 @@ Temp emit_floor_f64(isel_context *ctx, Builder& bld, Definition dst, Temp val) return add->definitions[0].getTemp(); } -Temp convert_int(Builder& bld, Temp src, unsigned src_bits, unsigned dst_bits, bool is_signed, Temp dst=Temp()) { +Temp convert_int(isel_context *ctx, Builder& bld, Temp src, unsigned src_bits, unsigned dst_bits, bool is_signed, Temp dst=Temp()) { if (!dst.id()) { if (dst_bits % 32 == 0 || src.type() == RegType::sgpr) dst = bld.tmp(src.type(), DIV_ROUND_UP(dst_bits, 32u)); @@ -961,7 +1065,7 @@ Temp convert_int(Builder& bld, Temp src, unsigned src_bits, unsigned dst_bits, b bld.sop1(src_bits == 8 ? aco_opcode::s_sext_i32_i8 : aco_opcode::s_sext_i32_i16, Definition(tmp), src); else bld.sop2(aco_opcode::s_and_b32, Definition(tmp), bld.def(s1, scc), Operand(src_bits == 8 ? 0xFFu : 0xFFFFu), src); - } else { + } else if (ctx->options->chip_class >= GFX8) { assert(src_bits != 8 || src.regClass() == v1b); assert(src_bits != 16 || src.regClass() == v2b); aco_ptr sdwa{create_instruction(aco_opcode::v_mov_b32, asSDWA(Format::VOP1), 1, 1)}; @@ -973,6 +1077,10 @@ Temp convert_int(Builder& bld, Temp src, unsigned src_bits, unsigned dst_bits, b sdwa->sel[0] = src_bits == 8 ? sdwa_ubyte : sdwa_uword; sdwa->dst_sel = tmp.bytes() == 2 ? sdwa_uword : sdwa_udword; bld.insert(std::move(sdwa)); + } else { + assert(ctx->options->chip_class == GFX6 || ctx->options->chip_class == GFX7); + aco_opcode opcode = is_signed ? aco_opcode::v_bfe_i32 : aco_opcode::v_bfe_u32; + bld.vop3(opcode, Definition(tmp), src, Operand(0u), Operand(src_bits == 8 ? 8u : 16u)); } if (dst_bits == 64) { @@ -999,6 +1107,7 @@ void visit_alu_instr(isel_context *ctx, nir_alu_instr *instr) abort(); } Builder bld(ctx->program, ctx->block); + bld.is_precise = instr->exact; Temp dst = get_ssa_temp(ctx, &instr->dest.dest.ssa); switch(instr->op) { case nir_op_vec2: @@ -1056,13 +1165,15 @@ void visit_alu_instr(isel_context *ctx, nir_alu_instr *instr) bld.sop1(aco_opcode::s_mov_b64, Definition(dst), src); else unreachable("wrong src register class for nir_op_imov"); - } else if (dst.regClass() == v1) { - bld.vop1(aco_opcode::v_mov_b32, Definition(dst), src); - } else if (dst.regClass() == v2) { - bld.pseudo(aco_opcode::p_create_vector, Definition(dst), src); } else { - nir_print_instr(&instr->instr, stderr); - unreachable("Should have been lowered to scalar."); + if (dst.regClass() == v1) + bld.vop1(aco_opcode::v_mov_b32, Definition(dst), src); + else if (dst.regClass() == v1b || + dst.regClass() == v2b || + dst.regClass() == v2) + bld.pseudo(aco_opcode::p_create_vector, Definition(dst), src); + else + unreachable("wrong src register class for nir_op_imov"); } break; } @@ -1076,6 +1187,12 @@ void visit_alu_instr(isel_context *ctx, nir_alu_instr *instr) bld.sop2(Builder::s_and, Definition(dst), bld.def(s1, scc), tmp, Operand(exec, bld.lm)); } else if (dst.regClass() == v1) { emit_vop1_instruction(ctx, instr, aco_opcode::v_not_b32, dst); + } else if (dst.regClass() == v2) { + Temp lo = bld.tmp(v1), hi = bld.tmp(v1); + bld.pseudo(aco_opcode::p_split_vector, Definition(lo), Definition(hi), src); + lo = bld.vop1(aco_opcode::v_not_b32, bld.def(v1), lo); + hi = bld.vop1(aco_opcode::v_not_b32, bld.def(v1), hi); + bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lo, hi); } else if (dst.type() == RegType::sgpr) { aco_opcode opcode = dst.size() == 1 ? aco_opcode::s_not_b32 : aco_opcode::s_not_b64; bld.sop1(opcode, Definition(dst), bld.def(s1, scc), src); @@ -1211,6 +1328,8 @@ void visit_alu_instr(isel_context *ctx, nir_alu_instr *instr) emit_boolean_logic(ctx, instr, Builder::s_or, dst); } else if (dst.regClass() == v1) { emit_vop2_instruction(ctx, instr, aco_opcode::v_or_b32, dst, true); + } else if (dst.regClass() == v2) { + emit_vop2_instruction_logic64(ctx, instr, aco_opcode::v_or_b32, dst); } else if (dst.regClass() == s1) { emit_sop2_instruction(ctx, instr, aco_opcode::s_or_b32, dst, true); } else if (dst.regClass() == s2) { @@ -1227,6 +1346,8 @@ void visit_alu_instr(isel_context *ctx, nir_alu_instr *instr) emit_boolean_logic(ctx, instr, Builder::s_and, dst); } else if (dst.regClass() == v1) { emit_vop2_instruction(ctx, instr, aco_opcode::v_and_b32, dst, true); + } else if (dst.regClass() == v2) { + emit_vop2_instruction_logic64(ctx, instr, aco_opcode::v_and_b32, dst); } else if (dst.regClass() == s1) { emit_sop2_instruction(ctx, instr, aco_opcode::s_and_b32, dst, true); } else if (dst.regClass() == s2) { @@ -1243,6 +1364,8 @@ void visit_alu_instr(isel_context *ctx, nir_alu_instr *instr) emit_boolean_logic(ctx, instr, Builder::s_xor, dst); } else if (dst.regClass() == v1) { emit_vop2_instruction(ctx, instr, aco_opcode::v_xor_b32, dst, true); + } else if (dst.regClass() == v2) { + emit_vop2_instruction_logic64(ctx, instr, aco_opcode::v_xor_b32, dst); } else if (dst.regClass() == s1) { emit_sop2_instruction(ctx, instr, aco_opcode::s_xor_b32, dst, true); } else if (dst.regClass() == s2) { @@ -1821,6 +1944,7 @@ void visit_alu_instr(isel_context *ctx, nir_alu_instr *instr) } else if (dst.regClass() == v1) { emit_rsq(ctx, bld, Definition(dst), src); } else if (dst.regClass() == v2) { + /* Lowered at NIR level for precision reasons. */ emit_vop1_instruction(ctx, instr, aco_opcode::v_rsq_f64, dst); } else { fprintf(stderr, "Unimplemented NIR instr bit size: "); @@ -1832,6 +1956,8 @@ void visit_alu_instr(isel_context *ctx, nir_alu_instr *instr) case nir_op_fneg: { Temp src = get_alu_src(ctx, instr->src[0]); if (dst.regClass() == v2b) { + if (ctx->block->fp_mode.must_flush_denorms16_64) + src = bld.vop2(aco_opcode::v_mul_f16, bld.def(v2b), Operand((uint16_t)0x3C00), as_vgpr(ctx, src)); bld.vop2(aco_opcode::v_xor_b32, Definition(dst), Operand(0x8000u), as_vgpr(ctx, src)); } else if (dst.regClass() == v1) { if (ctx->block->fp_mode.must_flush_denorms32) @@ -1854,6 +1980,8 @@ void visit_alu_instr(isel_context *ctx, nir_alu_instr *instr) case nir_op_fabs: { Temp src = get_alu_src(ctx, instr->src[0]); if (dst.regClass() == v2b) { + if (ctx->block->fp_mode.must_flush_denorms16_64) + src = bld.vop2(aco_opcode::v_mul_f16, bld.def(v2b), Operand((uint16_t)0x3C00), as_vgpr(ctx, src)); bld.vop2(aco_opcode::v_and_b32, Definition(dst), Operand(0x7FFFu), as_vgpr(ctx, src)); } else if (dst.regClass() == v1) { if (ctx->block->fp_mode.must_flush_denorms32) @@ -1876,7 +2004,7 @@ void visit_alu_instr(isel_context *ctx, nir_alu_instr *instr) case nir_op_fsat: { Temp src = get_alu_src(ctx, instr->src[0]); if (dst.regClass() == v2b) { - bld.vop3(aco_opcode::v_med3_f16, Definition(dst), Operand(0u), Operand(0x3f800000u), src); + bld.vop3(aco_opcode::v_med3_f16, Definition(dst), Operand((uint16_t)0u), Operand((uint16_t)0x3c00), src); } else if (dst.regClass() == v1) { bld.vop3(aco_opcode::v_med3_f32, Definition(dst), Operand(0u), Operand(0x3f800000u), src); /* apparently, it is not necessary to flush denorms if this instruction is used with these operands */ @@ -1912,6 +2040,7 @@ void visit_alu_instr(isel_context *ctx, nir_alu_instr *instr) } else if (dst.regClass() == v1) { emit_rcp(ctx, bld, Definition(dst), src); } else if (dst.regClass() == v2) { + /* Lowered at NIR level for precision reasons. */ emit_vop1_instruction(ctx, instr, aco_opcode::v_rcp_f64, dst); } else { fprintf(stderr, "Unimplemented NIR instr bit size: "); @@ -1939,6 +2068,7 @@ void visit_alu_instr(isel_context *ctx, nir_alu_instr *instr) } else if (dst.regClass() == v1) { emit_sqrt(ctx, bld, Definition(dst), src); } else if (dst.regClass() == v2) { + /* Lowered at NIR level for precision reasons. */ emit_vop1_instruction(ctx, instr, aco_opcode::v_sqrt_f64, dst); } else { fprintf(stderr, "Unimplemented NIR instr bit size: "); @@ -2065,12 +2195,13 @@ void visit_alu_instr(isel_context *ctx, nir_alu_instr *instr) case nir_op_fcos: { Temp src = as_vgpr(ctx, get_alu_src(ctx, instr->src[0])); aco_ptr norm; - Temp half_pi = bld.copy(bld.def(s1), Operand(0x3e22f983u)); if (dst.regClass() == v2b) { + Temp half_pi = bld.copy(bld.def(s1), Operand(0x3118u)); Temp tmp = bld.vop2(aco_opcode::v_mul_f16, bld.def(v1), half_pi, src); aco_opcode opcode = instr->op == nir_op_fsin ? aco_opcode::v_sin_f16 : aco_opcode::v_cos_f16; bld.vop1(opcode, Definition(dst), tmp); } else if (dst.regClass() == v1) { + Temp half_pi = bld.copy(bld.def(s1), Operand(0x3e22f983u)); Temp tmp = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), half_pi, src); /* before GFX9, v_sin_f32 and v_cos_f32 had a valid input domain of [-256, +256] */ @@ -2122,7 +2253,7 @@ void visit_alu_instr(isel_context *ctx, nir_alu_instr *instr) if (instr->src[0].src.ssa->bit_size == 16) { Temp tmp = bld.vop1(aco_opcode::v_frexp_exp_i16_f16, bld.def(v1), src); tmp = bld.pseudo(aco_opcode::p_extract_vector, bld.def(v1b), tmp, Operand(0u)); - convert_int(bld, tmp, 8, 32, true, dst); + convert_int(ctx, bld, tmp, 8, 32, true, dst); } else if (instr->src[0].src.ssa->bit_size == 32) { bld.vop1(aco_opcode::v_frexp_exp_i32_f32, Definition(dst), src); } else if (instr->src[0].src.ssa->bit_size == 64) { @@ -2170,7 +2301,13 @@ void visit_alu_instr(isel_context *ctx, nir_alu_instr *instr) Temp src = get_alu_src(ctx, instr->src[0]); if (instr->src[0].src.ssa->bit_size == 64) src = bld.vop1(aco_opcode::v_cvt_f32_f64, bld.def(v1), src); - bld.vop1(aco_opcode::v_cvt_f16_f32, Definition(dst), src); + if (instr->op == nir_op_f2f16_rtne && ctx->block->fp_mode.round16_64 != fp_round_ne) + /* We emit s_round_mode/s_setreg_imm32 in lower_to_hw_instr to + * keep value numbering and the scheduler simpler. + */ + bld.vop1(aco_opcode::p_cvt_f16_f32_rtne, Definition(dst), src); + else + bld.vop1(aco_opcode::v_cvt_f16_f32, Definition(dst), src); break; } case nir_op_f2f16_rtz: { @@ -2203,7 +2340,9 @@ void visit_alu_instr(isel_context *ctx, nir_alu_instr *instr) assert(dst.regClass() == v2b); Temp src = get_alu_src(ctx, instr->src[0]); if (instr->src[0].src.ssa->bit_size == 8) - src = convert_int(bld, src, 8, 16, true); + src = convert_int(ctx, bld, src, 8, 16, true); + else if (instr->src[0].src.ssa->bit_size == 64) + src = convert_int(ctx, bld, src, 64, 32, false); bld.vop1(aco_opcode::v_cvt_f16_i16, Definition(dst), src); break; } @@ -2211,7 +2350,7 @@ void visit_alu_instr(isel_context *ctx, nir_alu_instr *instr) assert(dst.size() == 1); Temp src = get_alu_src(ctx, instr->src[0]); if (instr->src[0].src.ssa->bit_size <= 16) - src = convert_int(bld, src, instr->src[0].src.ssa->bit_size, 32, true); + src = convert_int(ctx, bld, src, instr->src[0].src.ssa->bit_size, 32, true); bld.vop1(aco_opcode::v_cvt_f32_i32, Definition(dst), src); break; } @@ -2219,7 +2358,7 @@ void visit_alu_instr(isel_context *ctx, nir_alu_instr *instr) if (instr->src[0].src.ssa->bit_size <= 32) { Temp src = get_alu_src(ctx, instr->src[0]); if (instr->src[0].src.ssa->bit_size <= 16) - src = convert_int(bld, src, instr->src[0].src.ssa->bit_size, 32, true); + src = convert_int(ctx, bld, src, instr->src[0].src.ssa->bit_size, 32, true); bld.vop1(aco_opcode::v_cvt_f64_i32, Definition(dst), src); } else if (instr->src[0].src.ssa->bit_size == 64) { Temp src = get_alu_src(ctx, instr->src[0]); @@ -2242,7 +2381,9 @@ void visit_alu_instr(isel_context *ctx, nir_alu_instr *instr) assert(dst.regClass() == v2b); Temp src = get_alu_src(ctx, instr->src[0]); if (instr->src[0].src.ssa->bit_size == 8) - src = convert_int(bld, src, 8, 16, false); + src = convert_int(ctx, bld, src, 8, 16, false); + else if (instr->src[0].src.ssa->bit_size == 64) + src = convert_int(ctx, bld, src, 64, 32, false); bld.vop1(aco_opcode::v_cvt_f16_u16, Definition(dst), src); break; } @@ -2250,11 +2391,10 @@ void visit_alu_instr(isel_context *ctx, nir_alu_instr *instr) assert(dst.size() == 1); Temp src = get_alu_src(ctx, instr->src[0]); if (instr->src[0].src.ssa->bit_size == 8) { - //TODO: we should use v_cvt_f32_ubyte1/v_cvt_f32_ubyte2/etc depending on the register assignment bld.vop1(aco_opcode::v_cvt_f32_ubyte0, Definition(dst), src); } else { if (instr->src[0].src.ssa->bit_size == 16) - src = convert_int(bld, src, instr->src[0].src.ssa->bit_size, 32, true); + src = convert_int(ctx, bld, src, instr->src[0].src.ssa->bit_size, 32, true); bld.vop1(aco_opcode::v_cvt_f32_u32, Definition(dst), src); } break; @@ -2263,7 +2403,7 @@ void visit_alu_instr(isel_context *ctx, nir_alu_instr *instr) if (instr->src[0].src.ssa->bit_size <= 32) { Temp src = get_alu_src(ctx, instr->src[0]); if (instr->src[0].src.ssa->bit_size <= 16) - src = convert_int(bld, src, instr->src[0].src.ssa->bit_size, 32, false); + src = convert_int(ctx, bld, src, instr->src[0].src.ssa->bit_size, 32, false); bld.vop1(aco_opcode::v_cvt_f64_u32, Definition(dst), src); } else if (instr->src[0].src.ssa->bit_size == 64) { Temp src = get_alu_src(ctx, instr->src[0]); @@ -2283,34 +2423,22 @@ void visit_alu_instr(isel_context *ctx, nir_alu_instr *instr) } case nir_op_f2i8: case nir_op_f2i16: { - Temp src = get_alu_src(ctx, instr->src[0]); if (instr->src[0].src.ssa->bit_size == 16) - src = bld.vop1(aco_opcode::v_cvt_i16_f16, bld.def(v1), src); + emit_vop1_instruction(ctx, instr, aco_opcode::v_cvt_i16_f16, dst); else if (instr->src[0].src.ssa->bit_size == 32) - src = bld.vop1(aco_opcode::v_cvt_i32_f32, bld.def(v1), src); + emit_vop1_instruction(ctx, instr, aco_opcode::v_cvt_i32_f32, dst); else - src = bld.vop1(aco_opcode::v_cvt_i32_f64, bld.def(v1), src); - - if (dst.type() == RegType::vgpr) - bld.pseudo(aco_opcode::p_extract_vector, Definition(dst), src, Operand(0u)); - else - bld.pseudo(aco_opcode::p_as_uniform, Definition(dst), src); + emit_vop1_instruction(ctx, instr, aco_opcode::v_cvt_i32_f64, dst); break; } case nir_op_f2u8: case nir_op_f2u16: { - Temp src = get_alu_src(ctx, instr->src[0]); if (instr->src[0].src.ssa->bit_size == 16) - src = bld.vop1(aco_opcode::v_cvt_u16_f16, bld.def(v1), src); + emit_vop1_instruction(ctx, instr, aco_opcode::v_cvt_u16_f16, dst); else if (instr->src[0].src.ssa->bit_size == 32) - src = bld.vop1(aco_opcode::v_cvt_u32_f32, bld.def(v1), src); - else - src = bld.vop1(aco_opcode::v_cvt_u32_f64, bld.def(v1), src); - - if (dst.type() == RegType::vgpr) - bld.pseudo(aco_opcode::p_extract_vector, Definition(dst), src, Operand(0u)); + emit_vop1_instruction(ctx, instr, aco_opcode::v_cvt_u32_f32, dst); else - bld.pseudo(aco_opcode::p_as_uniform, Definition(dst), src); + emit_vop1_instruction(ctx, instr, aco_opcode::v_cvt_u32_f64, dst); break; } case nir_op_f2i32: { @@ -2324,19 +2452,9 @@ void visit_alu_instr(isel_context *ctx, nir_alu_instr *instr) bld.vop1(aco_opcode::v_cvt_i32_f32, bld.def(v1), tmp)); } } else if (instr->src[0].src.ssa->bit_size == 32) { - if (dst.type() == RegType::vgpr) - bld.vop1(aco_opcode::v_cvt_i32_f32, Definition(dst), src); - else - bld.pseudo(aco_opcode::p_as_uniform, Definition(dst), - bld.vop1(aco_opcode::v_cvt_i32_f32, bld.def(v1), src)); - + emit_vop1_instruction(ctx, instr, aco_opcode::v_cvt_i32_f32, dst); } else if (instr->src[0].src.ssa->bit_size == 64) { - if (dst.type() == RegType::vgpr) - bld.vop1(aco_opcode::v_cvt_i32_f64, Definition(dst), src); - else - bld.pseudo(aco_opcode::p_as_uniform, Definition(dst), - bld.vop1(aco_opcode::v_cvt_i32_f64, bld.def(v1), src)); - + emit_vop1_instruction(ctx, instr, aco_opcode::v_cvt_i32_f64, dst); } else { fprintf(stderr, "Unimplemented NIR instr bit size: "); nir_print_instr(&instr->instr, stderr); @@ -2355,19 +2473,9 @@ void visit_alu_instr(isel_context *ctx, nir_alu_instr *instr) bld.vop1(aco_opcode::v_cvt_u32_f32, bld.def(v1), tmp)); } } else if (instr->src[0].src.ssa->bit_size == 32) { - if (dst.type() == RegType::vgpr) - bld.vop1(aco_opcode::v_cvt_u32_f32, Definition(dst), src); - else - bld.pseudo(aco_opcode::p_as_uniform, Definition(dst), - bld.vop1(aco_opcode::v_cvt_u32_f32, bld.def(v1), src)); - + emit_vop1_instruction(ctx, instr, aco_opcode::v_cvt_u32_f32, dst); } else if (instr->src[0].src.ssa->bit_size == 64) { - if (dst.type() == RegType::vgpr) - bld.vop1(aco_opcode::v_cvt_u32_f64, Definition(dst), src); - else - bld.pseudo(aco_opcode::p_as_uniform, Definition(dst), - bld.vop1(aco_opcode::v_cvt_u32_f64, bld.def(v1), src)); - + emit_vop1_instruction(ctx, instr, aco_opcode::v_cvt_u32_f64, dst); } else { fprintf(stderr, "Unimplemented NIR instr bit size: "); nir_print_instr(&instr->instr, stderr); @@ -2575,7 +2683,7 @@ void visit_alu_instr(isel_context *ctx, nir_alu_instr *instr) case nir_op_i2i16: case nir_op_i2i32: case nir_op_i2i64: { - convert_int(bld, get_alu_src(ctx, instr->src[0]), + convert_int(ctx, bld, get_alu_src(ctx, instr->src[0]), instr->src[0].src.ssa->bit_size, instr->dest.dest.ssa.bit_size, true, dst); break; } @@ -2583,23 +2691,30 @@ void visit_alu_instr(isel_context *ctx, nir_alu_instr *instr) case nir_op_u2u16: case nir_op_u2u32: case nir_op_u2u64: { - convert_int(bld, get_alu_src(ctx, instr->src[0]), + convert_int(ctx, bld, get_alu_src(ctx, instr->src[0]), instr->src[0].src.ssa->bit_size, instr->dest.dest.ssa.bit_size, false, dst); break; } case nir_op_b2b32: - case nir_op_b2i32: { + case nir_op_b2i8: + case nir_op_b2i16: + case nir_op_b2i32: + case nir_op_b2i64: { Temp src = get_alu_src(ctx, instr->src[0]); assert(src.regClass() == bld.lm); - if (dst.regClass() == s1) { + Temp tmp = dst.bytes() == 8 ? bld.tmp(RegClass::get(dst.type(), 4)) : dst; + if (tmp.regClass() == s1) { // TODO: in a post-RA optimization, we can check if src is in VCC, and directly use VCCNZ - bool_to_scalar_condition(ctx, src, dst); - } else if (dst.regClass() == v1) { - bld.vop2_e64(aco_opcode::v_cndmask_b32, Definition(dst), Operand(0u), Operand(1u), src); + bool_to_scalar_condition(ctx, src, tmp); + } else if (tmp.type() == RegType::vgpr) { + bld.vop2_e64(aco_opcode::v_cndmask_b32, Definition(tmp), Operand(0u), Operand(1u), src); } else { unreachable("Invalid register class for b2i32"); } + + if (tmp != dst) + bld.pseudo(aco_opcode::p_create_vector, Definition(dst), tmp, Operand(0u)); break; } case nir_op_b2b1: @@ -2688,7 +2803,6 @@ void visit_alu_instr(isel_context *ctx, nir_alu_instr *instr) } case nir_op_unpack_half_2x16_split_x: { if (dst.regClass() == v1) { - Builder bld(ctx->program, ctx->block); bld.vop1(aco_opcode::v_cvt_f32_f16, Definition(dst), get_alu_src(ctx, instr->src[0])); } else { fprintf(stderr, "Unimplemented NIR instr bit size: "); @@ -2699,7 +2813,6 @@ void visit_alu_instr(isel_context *ctx, nir_alu_instr *instr) } case nir_op_unpack_half_2x16_split_y: { if (dst.regClass() == v1) { - Builder bld(ctx->program, ctx->block); /* TODO: use SDWA here */ bld.vop1(aco_opcode::v_cvt_f32_f16, Definition(dst), bld.vop2(aco_opcode::v_lshrrev_b32, bld.def(v1), Operand(16u), as_vgpr(ctx, get_alu_src(ctx, instr->src[0])))); @@ -3011,35 +3124,6 @@ uint32_t widen_mask(uint32_t mask, unsigned multiplier) return new_mask; } -void byte_align_vector(isel_context *ctx, Temp vec, Operand offset, Temp dst) -{ - Builder bld(ctx->program, ctx->block); - if (offset.isTemp()) { - Temp tmp[3] = {vec, vec, vec}; - - if (vec.size() == 3) { - tmp[0] = bld.tmp(v1), tmp[1] = bld.tmp(v1), tmp[2] = bld.tmp(v1); - bld.pseudo(aco_opcode::p_split_vector, Definition(tmp[0]), Definition(tmp[1]), Definition(tmp[2]), vec); - } else if (vec.size() == 2) { - tmp[0] = bld.tmp(v1), tmp[1] = bld.tmp(v1), tmp[2] = tmp[1]; - bld.pseudo(aco_opcode::p_split_vector, Definition(tmp[0]), Definition(tmp[1]), vec); - } - for (unsigned i = 0; i < dst.size(); i++) - tmp[i] = bld.vop3(aco_opcode::v_alignbyte_b32, bld.def(v1), tmp[i + 1], tmp[i], offset); - - vec = tmp[0]; - if (dst.size() == 2) - vec = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), tmp[0], tmp[1]); - - offset = Operand(0u); - } - - if (vec.bytes() == dst.bytes() && offset.constantValue() == 0) - bld.copy(Definition(dst), vec); - else - trim_subdword_vector(ctx, vec, dst, vec.bytes(), ((1 << dst.bytes()) - 1) << offset.constantValue()); -} - struct LoadEmitInfo { Operand offset; Temp dst; @@ -3053,8 +3137,7 @@ struct LoadEmitInfo { bool glc = false; unsigned swizzle_component_size = 0; - barrier_interaction barrier = barrier_none; - bool can_reorder = true; + memory_sync_info sync; Temp soffset = Temp(0, s1); }; @@ -3084,7 +3167,9 @@ void emit_load(isel_context *ctx, Builder& bld, const LoadEmitInfo *info) int byte_align = align_mul % 4 == 0 ? align_offset % 4 : -1; if (byte_align) { - if ((bytes_needed > 2 || !supports_8bit_16bit_loads) && byte_align_loads) { + if ((bytes_needed > 2 || + (bytes_needed == 2 && (align_mul % 2 || align_offset % 2)) || + !supports_8bit_16bit_loads) && byte_align_loads) { if (info->component_stride) { assert(supports_8bit_16bit_loads && "unimplemented"); bytes_needed = 2; @@ -3146,7 +3231,9 @@ void emit_load(isel_context *ctx, Builder& bld, const LoadEmitInfo *info) /* align offset down if needed */ Operand aligned_offset = offset; + unsigned align = align_offset ? 1 << (ffs(align_offset) - 1) : align_mul; if (need_to_align_offset) { + align = 4; Temp offset_tmp = offset.isTemp() ? offset.getTemp() : Temp(); if (offset.isConstant()) { aligned_offset = Operand(offset.constantValue() & 0xfffffffcu); @@ -3166,12 +3253,18 @@ void emit_load(isel_context *ctx, Builder& bld, const LoadEmitInfo *info) Temp aligned_offset_tmp = aligned_offset.isTemp() ? aligned_offset.getTemp() : bld.copy(bld.def(s1), aligned_offset); - unsigned align = align_offset ? 1 << (ffs(align_offset) - 1) : align_mul; Temp val = callback(bld, info, aligned_offset_tmp, bytes_needed, align, reduced_const_offset, byte_align ? Temp() : info->dst); + /* the callback wrote directly to dst */ + if (val == info->dst) { + assert(num_vals == 0); + emit_split_vector(ctx, info->dst, info->num_components); + return; + } + /* shift result right if needed */ - if (byte_align) { + if (info->component_size < 4 && byte_align_loads) { Operand align((uint32_t)byte_align); if (byte_align == -1) { if (offset.isConstant()) @@ -3182,15 +3275,12 @@ void emit_load(isel_context *ctx, Builder& bld, const LoadEmitInfo *info) align = offset; } - if (align.isTemp() || align.constantValue()) { - assert(val.bytes() >= load_size && "unimplemented"); - Temp new_val = bld.tmp(RegClass::get(val.type(), load_size)); - if (val.type() == RegType::sgpr) - byte_align_scalar(ctx, val, align, new_val); - else - byte_align_vector(ctx, val, align, new_val); - val = new_val; - } + assert(val.bytes() >= load_size && "unimplemented"); + if (val.type() == RegType::sgpr) + byte_align_scalar(ctx, val, align, info->dst); + else + byte_align_vector(ctx, val, align, info->dst, component_size); + return; } /* add result to list and advance */ @@ -3206,13 +3296,6 @@ void emit_load(isel_context *ctx, Builder& bld, const LoadEmitInfo *info) vals[num_vals++] = val; } - /* the callback wrote directly to dst */ - if (vals[0] == info->dst) { - assert(num_vals == 1); - emit_split_vector(ctx, info->dst, info->num_components); - return; - } - /* create array of components */ unsigned components_split = 0; std::array allocated_vec; @@ -3231,7 +3314,7 @@ void emit_load(isel_context *ctx, Builder& bld, const LoadEmitInfo *info) if (num_tmps > 1) { aco_ptr vec{create_instruction( aco_opcode::p_create_vector, Format::PSEUDO, num_tmps, 1)}; - for (unsigned i = 0; i < num_vals; i++) + for (unsigned i = 0; i < num_tmps; i++) vec->operands[i] = Operand(tmp[i]); tmp[0] = bld.tmp(RegClass::get(reg_type, tmp_size)); vec->definitions[0] = Definition(tmp[0]); @@ -3357,10 +3440,12 @@ Temp lds_load_callback(Builder& bld, const LoadEmitInfo *info, RegClass rc = RegClass(RegType::vgpr, DIV_ROUND_UP(size, 4)); Temp val = rc == info->dst.regClass() && dst_hint.id() ? dst_hint : bld.tmp(rc); + Instruction *instr; if (read2) - bld.ds(op, Definition(val), offset, m, const_offset, const_offset + 1); + instr = bld.ds(op, Definition(val), offset, m, const_offset, const_offset + 1); else - bld.ds(op, Definition(val), offset, m, const_offset); + instr = bld.ds(op, Definition(val), offset, m, const_offset); + static_cast(instr)->sync = info->sync; if (size < 4) val = bld.pseudo(aco_opcode::p_extract_vector, bld.def(RegClass::get(RegType::vgpr, size)), val, Operand(0u)); @@ -3406,8 +3491,7 @@ Temp smem_load_callback(Builder& bld, const LoadEmitInfo *info, load->definitions[0] = Definition(val); load->glc = info->glc; load->dlc = info->glc && bld.program->chip_class >= GFX10; - load->barrier = info->barrier; - load->can_reorder = false; // FIXME: currently, it doesn't seem beneficial due to how our scheduler works + load->sync = info->sync; bld.insert(std::move(load)); return val; } @@ -3430,10 +3514,10 @@ Temp mubuf_load_callback(Builder& bld, const LoadEmitInfo *info, unsigned bytes_size = 0; aco_opcode op; - if (bytes_needed == 1) { + if (bytes_needed == 1 || align_ % 2) { bytes_size = 1; op = aco_opcode::buffer_load_ubyte; - } else if (bytes_needed == 2) { + } else if (bytes_needed == 2 || align_ % 4) { bytes_size = 2; op = aco_opcode::buffer_load_ushort; } else if (bytes_needed <= 4) { @@ -3456,21 +3540,19 @@ Temp mubuf_load_callback(Builder& bld, const LoadEmitInfo *info, mubuf->offen = (offset.type() == RegType::vgpr); mubuf->glc = info->glc; mubuf->dlc = info->glc && bld.program->chip_class >= GFX10; - mubuf->barrier = info->barrier; - mubuf->can_reorder = info->can_reorder; + mubuf->sync = info->sync; mubuf->offset = const_offset; - RegClass rc = RegClass::get(RegType::vgpr, align(bytes_size, 4)); + mubuf->swizzled = info->swizzle_component_size != 0; + RegClass rc = RegClass::get(RegType::vgpr, bytes_size); Temp val = dst_hint.id() && rc == dst_hint.regClass() ? dst_hint : bld.tmp(rc); mubuf->definitions[0] = Definition(val); bld.insert(std::move(mubuf)); - if (bytes_size < 4) - val = bld.pseudo(aco_opcode::p_extract_vector, bld.def(RegClass::get(RegType::vgpr, bytes_size)), val, Operand(0u)); - return val; } static auto emit_mubuf_load = emit_load; +static auto emit_scratch_load = emit_load; Temp get_gfx6_global_rsrc(Builder& bld, Temp addr) { @@ -3522,7 +3604,7 @@ Temp global_load_callback(Builder& bld, const LoadEmitInfo *info, mubuf->offset = 0; mubuf->addr64 = offset.type() == RegType::vgpr; mubuf->disable_wqm = false; - mubuf->barrier = info->barrier; + mubuf->sync = info->sync; mubuf->definitions[0] = Definition(val); bld.insert(std::move(mubuf)); } else { @@ -3533,15 +3615,12 @@ Temp global_load_callback(Builder& bld, const LoadEmitInfo *info, flat->operands[1] = Operand(s1); flat->glc = info->glc; flat->dlc = info->glc && bld.program->chip_class >= GFX10; - flat->barrier = info->barrier; + flat->sync = info->sync; flat->offset = 0u; flat->definitions[0] = Definition(val); bld.insert(std::move(flat)); } - if (bytes_size < 4) - val = bld.pseudo(aco_opcode::p_extract_vector, bld.def(RegClass::get(RegType::vgpr, bytes_size)), val, Operand(0u)); - return val; } @@ -3558,8 +3637,7 @@ Temp load_lds(isel_context *ctx, unsigned elem_size_bytes, Temp dst, LoadEmitInfo info = {Operand(as_vgpr(ctx, address)), dst, num_components, elem_size_bytes}; info.align_mul = align; info.align_offset = 0; - info.barrier = barrier_shared; - info.can_reorder = false; + info.sync = memory_sync_info(storage_shared); info.const_offset = base_offset; emit_lds_load(ctx, bld, &info); @@ -3598,13 +3676,15 @@ void split_store_data(isel_context *ctx, RegType dst_type, unsigned count, Temp /* use allocated_vec if possible */ auto it = ctx->allocated_vec.find(src.id()); if (it != ctx->allocated_vec.end()) { - unsigned total_size = 0; - for (unsigned i = 0; it->second[i].bytes() && (i < NIR_MAX_VEC_COMPONENTS); i++) - total_size += it->second[i].bytes(); - if (total_size != src.bytes()) + if (!it->second[0].id()) goto split; - unsigned elem_size = it->second[0].bytes(); + assert(src.bytes() % elem_size == 0); + + for (unsigned i = 0; i < src.bytes() / elem_size; i++) { + if (!it->second[i].id()) + goto split; + } for (unsigned i = 0; i < count; i++) { if (offsets[i] % elem_size || dst[i].bytes() % elem_size) @@ -3636,10 +3716,11 @@ void split_store_data(isel_context *ctx, RegType dst_type, unsigned count, Temp } } + split: + if (dst_type == RegType::sgpr) src = bld.as_uniform(src); - split: /* just split it */ aco_ptr split{create_instruction(aco_opcode::p_split_vector, Format::PSEUDO, 1, count)}; split->operands[0] = Operand(src); @@ -3765,13 +3846,16 @@ void store_lds(isel_context *ctx, unsigned elem_size_bytes, Temp data, uint32_t } assert(inline_offset <= max_offset); /* offsets[i] shouldn't be large enough for this to happen */ + Instruction *instr; if (write2) { Temp second_data = write_datas[second]; inline_offset /= data.bytes(); - bld.ds(op, address_offset, data, second_data, m, inline_offset, inline_offset + write2_off); + instr = bld.ds(op, address_offset, data, second_data, m, inline_offset, inline_offset + write2_off); } else { - bld.ds(op, address_offset, data, m, inline_offset); + instr = bld.ds(op, address_offset, data, m, inline_offset); } + static_cast(instr)->sync = + memory_sync_info(storage_shared); } } @@ -3839,10 +3923,10 @@ void split_buffer_store(isel_context *ctx, nir_intrinsic_instr *instr, bool smem /* dword or larger stores have to be dword-aligned */ unsigned align_mul = instr ? nir_intrinsic_align_mul(instr) : 4; - unsigned align_offset = instr ? nir_intrinsic_align_mul(instr) : 0; - bool dword_aligned = (align_offset + offset) % 4 == 0 && align_mul % 4 == 0; - if (bytes >= 4 && !dword_aligned) - bytes = MIN2(bytes, 2); + unsigned align_offset = (instr ? nir_intrinsic_align_offset(instr) : 0) + offset; + bool dword_aligned = align_offset % 4 == 0 && align_mul % 4 == 0; + if (!dword_aligned) + bytes = MIN2(bytes, (align_offset % 2 == 0 && align_mul % 2 == 0) ? 2 : 1); advance_write_mask(&todo, offset, bytes); write_count_with_skips++; @@ -3916,7 +4000,8 @@ inline unsigned resolve_excess_vmem_const_offset(Builder &bld, Temp &voffset, un } void emit_single_mubuf_store(isel_context *ctx, Temp descriptor, Temp voffset, Temp soffset, Temp vdata, - unsigned const_offset = 0u, bool allow_reorder = true, bool slc = false) + unsigned const_offset = 0u, memory_sync_info sync=memory_sync_info(), + bool slc = false, bool swizzled = false) { assert(vdata.id()); assert(vdata.size() != 3 || ctx->program->chip_class != GFX6); @@ -3929,18 +4014,19 @@ void emit_single_mubuf_store(isel_context *ctx, Temp descriptor, Temp voffset, T Operand voffset_op = voffset.id() ? Operand(as_vgpr(ctx, voffset)) : Operand(v1); Operand soffset_op = soffset.id() ? Operand(soffset) : Operand(0u); Builder::Result r = bld.mubuf(op, Operand(descriptor), voffset_op, soffset_op, Operand(vdata), const_offset, - /* offen */ !voffset_op.isUndefined(), /* idxen*/ false, /* addr64 */ false, - /* disable_wqm */ false, /* glc */ true, /* dlc*/ false, /* slc */ slc); + /* offen */ !voffset_op.isUndefined(), /* swizzled */ swizzled, + /* idxen*/ false, /* addr64 */ false, /* disable_wqm */ false, /* glc */ true, + /* dlc*/ false, /* slc */ slc); - static_cast(r.instr)->can_reorder = allow_reorder; + static_cast(r.instr)->sync = sync; } void store_vmem_mubuf(isel_context *ctx, Temp src, Temp descriptor, Temp voffset, Temp soffset, unsigned base_const_offset, unsigned elem_size_bytes, unsigned write_mask, - bool allow_combining = true, bool reorder = true, bool slc = false) + bool allow_combining = true, memory_sync_info sync=memory_sync_info(), bool slc = false) { Builder bld(ctx->program, ctx->block); - assert(elem_size_bytes == 4 || elem_size_bytes == 8); + assert(elem_size_bytes == 2 || elem_size_bytes == 4 || elem_size_bytes == 8); assert(write_mask); write_mask = widen_mask(write_mask, elem_size_bytes); @@ -3952,7 +4038,7 @@ void store_vmem_mubuf(isel_context *ctx, Temp src, Temp descriptor, Temp voffset for (unsigned i = 0; i < write_count; i++) { unsigned const_offset = offsets[i] + base_const_offset; - emit_single_mubuf_store(ctx, descriptor, voffset, soffset, write_datas[i], const_offset, reorder, slc); + emit_single_mubuf_store(ctx, descriptor, voffset, soffset, write_datas[i], const_offset, sync, slc, !allow_combining); } } @@ -3960,8 +4046,8 @@ void load_vmem_mubuf(isel_context *ctx, Temp dst, Temp descriptor, Temp voffset, unsigned base_const_offset, unsigned elem_size_bytes, unsigned num_components, unsigned stride = 0u, bool allow_combining = true, bool allow_reorder = true) { - assert(elem_size_bytes == 4 || elem_size_bytes == 8); - assert((num_components * elem_size_bytes / 4) == dst.size()); + assert(elem_size_bytes == 2 || elem_size_bytes == 4 || elem_size_bytes == 8); + assert((num_components * elem_size_bytes) == dst.bytes()); assert(!!stride != allow_combining); Builder bld(ctx->program, ctx->block); @@ -4272,7 +4358,7 @@ void visit_store_ls_or_es_output(isel_context *ctx, nir_intrinsic_instr *instr) /* GFX6-8: ES stage is not merged into GS, data is passed from ES to GS in VMEM. */ Temp esgs_ring = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), ctx->program->private_segment_buffer, Operand(RING_ESGS_VS * 16u)); Temp es2gs_offset = get_arg(ctx, ctx->args->es2gs_offset); - store_vmem_mubuf(ctx, src, esgs_ring, offs.first, es2gs_offset, offs.second, elem_size_bytes, write_mask, false, true, true); + store_vmem_mubuf(ctx, src, esgs_ring, offs.first, es2gs_offset, offs.second, elem_size_bytes, write_mask, false, memory_sync_info(), true); } else { Temp lds_base; @@ -4357,7 +4443,7 @@ void visit_store_tcs_output(isel_context *ctx, nir_intrinsic_instr *instr, bool Temp hs_ring_tess_offchip = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), ctx->program->private_segment_buffer, Operand(RING_HS_TESS_OFFCHIP * 16u)); Temp oc_lds = get_arg(ctx, ctx->args->oc_lds); - store_vmem_mubuf(ctx, store_val, hs_ring_tess_offchip, vmem_offs.first, oc_lds, vmem_offs.second, elem_size_bytes, write_mask, true, false); + store_vmem_mubuf(ctx, store_val, hs_ring_tess_offchip, vmem_offs.first, oc_lds, vmem_offs.second, elem_size_bytes, write_mask, true, memory_sync_info(storage_vmem_output)); } if (write_to_lds) { @@ -4422,10 +4508,40 @@ void emit_interp_instr(isel_context *ctx, unsigned idx, unsigned component, Temp Temp coord2 = emit_extract_vector(ctx, src, 1, v1); Builder bld(ctx->program, ctx->block); - Builder::Result interp_p1 = bld.vintrp(aco_opcode::v_interp_p1_f32, bld.def(v1), coord1, bld.m0(prim_mask), idx, component); - if (ctx->program->has_16bank_lds) - interp_p1.instr->operands[0].setLateKill(true); - bld.vintrp(aco_opcode::v_interp_p2_f32, Definition(dst), coord2, bld.m0(prim_mask), interp_p1, idx, component); + + if (dst.regClass() == v2b) { + if (ctx->program->has_16bank_lds) { + assert(ctx->options->chip_class <= GFX8); + Builder::Result interp_p1 = + bld.vintrp(aco_opcode::v_interp_mov_f32, bld.def(v1), + Operand(2u) /* P0 */, bld.m0(prim_mask), idx, component); + interp_p1 = bld.vintrp(aco_opcode::v_interp_p1lv_f16, bld.def(v2b), + coord1, bld.m0(prim_mask), interp_p1, idx, component); + bld.vintrp(aco_opcode::v_interp_p2_legacy_f16, Definition(dst), coord2, + bld.m0(prim_mask), interp_p1, idx, component); + } else { + aco_opcode interp_p2_op = aco_opcode::v_interp_p2_f16; + + if (ctx->options->chip_class == GFX8) + interp_p2_op = aco_opcode::v_interp_p2_legacy_f16; + + Builder::Result interp_p1 = + bld.vintrp(aco_opcode::v_interp_p1ll_f16, bld.def(v1), + coord1, bld.m0(prim_mask), idx, component); + bld.vintrp(interp_p2_op, Definition(dst), coord2, bld.m0(prim_mask), + interp_p1, idx, component); + } + } else { + Builder::Result interp_p1 = + bld.vintrp(aco_opcode::v_interp_p1_f32, bld.def(v1), coord1, + bld.m0(prim_mask), idx, component); + + if (ctx->program->has_16bank_lds) + interp_p1.instr->operands[0].setLateKill(true); + + bld.vintrp(aco_opcode::v_interp_p2_f32, Definition(dst), coord2, + bld.m0(prim_mask), interp_p1, idx, component); + } } void emit_load_frag_coord(isel_context *ctx, Temp dst, unsigned num_components) @@ -4591,6 +4707,7 @@ void visit_load_input(isel_context *ctx, nir_intrinsic_instr *instr) unsigned location = nir_intrinsic_base(instr) / 4 - VERT_ATTRIB_GENERIC0 + offset; unsigned component = nir_intrinsic_component(instr); + unsigned bitsize = instr->dest.ssa.bit_size; unsigned attrib_binding = ctx->options->key.vs.vertex_attribute_bindings[location]; uint32_t attrib_offset = ctx->options->key.vs.vertex_attribute_offsets[location]; uint32_t attrib_stride = ctx->options->key.vs.vertex_attribute_strides[location]; @@ -4647,7 +4764,7 @@ void visit_load_input(isel_context *ctx, nir_intrinsic_instr *instr) /* load channels */ while (channel_start < num_channels) { - unsigned fetch_size = num_channels - channel_start; + unsigned fetch_component = num_channels - channel_start; unsigned fetch_offset = attrib_offset + channel_start * vtx_info->chan_byte_size; bool expanded = false; @@ -4659,15 +4776,17 @@ void visit_load_input(isel_context *ctx, nir_intrinsic_instr *instr) vtx_info->chan_byte_size == 4; unsigned fetch_dfmt = V_008F0C_BUF_DATA_FORMAT_INVALID; if (!use_mubuf) { - fetch_dfmt = get_fetch_data_format(ctx, vtx_info, fetch_offset, attrib_stride, &fetch_size); + fetch_dfmt = get_fetch_data_format(ctx, vtx_info, fetch_offset, attrib_stride, &fetch_component); } else { - if (fetch_size == 3 && ctx->options->chip_class == GFX6) { + if (fetch_component == 3 && ctx->options->chip_class == GFX6) { /* GFX6 only supports loading vec3 with MTBUF, expand to vec4. */ - fetch_size = 4; + fetch_component = 4; expanded = true; } } + unsigned fetch_bytes = fetch_component * bitsize / 8; + Temp fetch_index = index; if (attrib_stride != 0 && fetch_offset > attrib_stride) { fetch_index = bld.vadd32(bld.def(v1), Operand(fetch_offset / attrib_stride), fetch_index); @@ -4681,19 +4800,37 @@ void visit_load_input(isel_context *ctx, nir_intrinsic_instr *instr) } aco_opcode opcode; - switch (fetch_size) { - case 1: - opcode = use_mubuf ? aco_opcode::buffer_load_dword : aco_opcode::tbuffer_load_format_x; - break; + switch (fetch_bytes) { case 2: - opcode = use_mubuf ? aco_opcode::buffer_load_dwordx2 : aco_opcode::tbuffer_load_format_xy; + assert(!use_mubuf && bitsize == 16); + opcode = aco_opcode::tbuffer_load_format_d16_x; + break; + case 4: + if (bitsize == 16) { + assert(!use_mubuf); + opcode = aco_opcode::tbuffer_load_format_d16_xy; + } else { + opcode = use_mubuf ? aco_opcode::buffer_load_dword : aco_opcode::tbuffer_load_format_x; + } + break; + case 6: + assert(!use_mubuf && bitsize == 16); + opcode = aco_opcode::tbuffer_load_format_d16_xyz; break; - case 3: + case 8: + if (bitsize == 16) { + assert(!use_mubuf); + opcode = aco_opcode::tbuffer_load_format_d16_xyzw; + } else { + opcode = use_mubuf ? aco_opcode::buffer_load_dwordx2 : aco_opcode::tbuffer_load_format_xy; + } + break; + case 12: assert(ctx->options->chip_class >= GFX7 || (!use_mubuf && ctx->options->chip_class == GFX6)); opcode = use_mubuf ? aco_opcode::buffer_load_dwordx3 : aco_opcode::tbuffer_load_format_xyz; break; - case 4: + case 16: opcode = use_mubuf ? aco_opcode::buffer_load_dwordx4 : aco_opcode::tbuffer_load_format_xyzw; break; default: @@ -4701,37 +4838,36 @@ void visit_load_input(isel_context *ctx, nir_intrinsic_instr *instr) } Temp fetch_dst; - if (channel_start == 0 && fetch_size == dst.size() && !post_shuffle && + if (channel_start == 0 && fetch_bytes == dst.bytes() && !post_shuffle && !expanded && (alpha_adjust == RADV_ALPHA_ADJUST_NONE || num_channels <= 3)) { direct_fetch = true; fetch_dst = dst; } else { - fetch_dst = bld.tmp(RegType::vgpr, fetch_size); + fetch_dst = bld.tmp(RegClass::get(RegType::vgpr, fetch_bytes)); } if (use_mubuf) { - Instruction *mubuf = bld.mubuf(opcode, - Definition(fetch_dst), list, fetch_index, soffset, - fetch_offset, false, true).instr; - static_cast(mubuf)->can_reorder = true; + bld.mubuf(opcode, + Definition(fetch_dst), list, fetch_index, soffset, + fetch_offset, false, false, true).instr; } else { - Instruction *mtbuf = bld.mtbuf(opcode, - Definition(fetch_dst), list, fetch_index, soffset, - fetch_dfmt, nfmt, fetch_offset, false, true).instr; - static_cast(mtbuf)->can_reorder = true; + bld.mtbuf(opcode, + Definition(fetch_dst), list, fetch_index, soffset, + fetch_dfmt, nfmt, fetch_offset, false, true).instr; } emit_split_vector(ctx, fetch_dst, fetch_dst.size()); - if (fetch_size == 1) { + if (fetch_component == 1) { channels[channel_start] = fetch_dst; } else { - for (unsigned i = 0; i < MIN2(fetch_size, num_channels - channel_start); i++) - channels[channel_start + i] = emit_extract_vector(ctx, fetch_dst, i, v1); + for (unsigned i = 0; i < MIN2(fetch_component, num_channels - channel_start); i++) + channels[channel_start + i] = emit_extract_vector(ctx, fetch_dst, i, + bitsize == 16 ? v2b : v1); } - channel_start += fetch_size; + channel_start += fetch_component; } if (!direct_fetch) { @@ -5071,18 +5207,17 @@ void visit_load_resource(isel_context *ctx, nir_intrinsic_instr *instr) void load_buffer(isel_context *ctx, unsigned num_components, unsigned component_size, Temp dst, Temp rsrc, Temp offset, unsigned align_mul, unsigned align_offset, - bool glc=false, bool readonly=true) + bool glc=false, bool allow_smem=true, memory_sync_info sync=memory_sync_info()) { Builder bld(ctx->program, ctx->block); - bool use_smem = dst.type() != RegType::vgpr && ((ctx->options->chip_class >= GFX8 && component_size >= 4) || readonly); + bool use_smem = dst.type() != RegType::vgpr && (!glc || ctx->options->chip_class >= GFX8) && allow_smem; if (use_smem) offset = bld.as_uniform(offset); LoadEmitInfo info = {Operand(offset), dst, num_components, component_size, rsrc}; info.glc = glc; - info.barrier = readonly ? barrier_none : barrier_buffer; - info.can_reorder = readonly; + info.sync = sync; info.align_mul = align_mul; info.align_offset = align_offset; if (use_smem) @@ -5158,7 +5293,7 @@ void visit_load_push_constant(isel_context *ctx, nir_intrinsic_instr *instr) Temp index = bld.as_uniform(get_ssa_temp(ctx, instr->src[0].ssa)); if (offset != 0) // TODO check if index != 0 as well - index = bld.sop2(aco_opcode::s_add_i32, bld.def(s1), bld.def(s1, scc), Operand(offset), index); + index = bld.nuw().sop2(aco_opcode::s_add_i32, bld.def(s1), bld.def(s1, scc), Operand(offset), index); Temp ptr = convert_pointer_to_64_bit(ctx, get_arg(ctx, ctx->args->ac.push_constants)); Temp vec = dst; bool trim = false; @@ -5200,7 +5335,7 @@ void visit_load_push_constant(isel_context *ctx, nir_intrinsic_instr *instr) unreachable("unimplemented or forbidden load_push_constant."); } - bld.smem(op, Definition(vec), ptr, index); + static_cast(bld.smem(op, Definition(vec), ptr, index).instr)->prevent_overflow = true; if (!aligned) { Operand byte_offset = index_cv ? Operand((offset + index_cv->u32) % 4) : Operand(index); @@ -5244,7 +5379,7 @@ void visit_load_constant(isel_context *ctx, nir_intrinsic_instr *instr) Temp offset = get_ssa_temp(ctx, instr->src[0].ssa); if (base && offset.type() == RegType::sgpr) - offset = bld.sop2(aco_opcode::s_add_u32, bld.def(s1), bld.def(s1, scc), offset, Operand(base)); + offset = bld.nuw().sop2(aco_opcode::s_add_u32, bld.def(s1), bld.def(s1, scc), offset, Operand(base)); else if (base && offset.type() == RegType::vgpr) offset = bld.vadd32(bld.def(v1), Operand(base), offset); @@ -5600,7 +5735,6 @@ static Temp adjust_sample_index_using_fmask(isel_context *ctx, bool da, std::vec load->unrm = true; load->da = da; load->dim = dim; - load->can_reorder = true; /* fmask images shouldn't be modified */ ctx->block->instructions.emplace_back(std::move(load)); Operand sample_index4; @@ -5700,6 +5834,22 @@ static Temp get_image_coords(isel_context *ctx, const nir_intrinsic_instr *instr } +memory_sync_info get_memory_sync_info(nir_intrinsic_instr *instr, storage_class storage, unsigned semantics) +{ + /* atomicrmw might not have NIR_INTRINSIC_ACCESS and there's nothing interesting there anyway */ + if (semantics & semantic_atomicrmw) + return memory_sync_info(storage, semantics); + + unsigned access = nir_intrinsic_access(instr); + + if (access & ACCESS_VOLATILE) + semantics |= semantic_volatile; + if (access & ACCESS_CAN_REORDER) + semantics |= semantic_can_reorder | semantic_private; + + return memory_sync_info(storage, semantics); +} + void visit_image_load(isel_context *ctx, nir_intrinsic_instr *instr) { Builder bld(ctx->program, ctx->block); @@ -5709,6 +5859,9 @@ void visit_image_load(isel_context *ctx, nir_intrinsic_instr *instr) bool is_array = glsl_sampler_type_is_array(type); Temp dst = get_ssa_temp(ctx, &instr->dest.ssa); + memory_sync_info sync = get_memory_sync_info(instr, storage_image, 0); + unsigned access = var->data.access | nir_intrinsic_access(instr); + if (dim == GLSL_SAMPLER_DIM_BUF) { unsigned mask = nir_ssa_def_components_read(&instr->dest.ssa); unsigned num_channels = util_last_bit(mask); @@ -5743,9 +5896,9 @@ void visit_image_load(isel_context *ctx, nir_intrinsic_instr *instr) tmp = {ctx->program->allocateId(), RegClass(RegType::vgpr, num_channels)}; load->definitions[0] = Definition(tmp); load->idxen = true; - load->glc = var->data.access & (ACCESS_VOLATILE | ACCESS_COHERENT); + load->glc = access & (ACCESS_VOLATILE | ACCESS_COHERENT); load->dlc = load->glc && ctx->options->chip_class >= GFX10; - load->barrier = barrier_image; + load->sync = sync; ctx->block->instructions.emplace_back(std::move(load)); expand_vector(ctx, tmp, dst, instr->dest.ssa.num_components, (1 << num_channels) - 1); @@ -5771,13 +5924,13 @@ void visit_image_load(isel_context *ctx, nir_intrinsic_instr *instr) load->operands[1] = Operand(s4); /* no sampler */ load->operands[2] = Operand(coords); load->definitions[0] = Definition(tmp); - load->glc = var->data.access & (ACCESS_VOLATILE | ACCESS_COHERENT) ? 1 : 0; + load->glc = access & (ACCESS_VOLATILE | ACCESS_COHERENT) ? 1 : 0; load->dlc = load->glc && ctx->options->chip_class >= GFX10; load->dim = ac_get_image_dim(ctx->options->chip_class, dim, is_array); load->dmask = dmask; load->unrm = true; load->da = should_declare_array(ctx, dim, glsl_sampler_type_is_array(type)); - load->barrier = barrier_image; + load->sync = sync; ctx->block->instructions.emplace_back(std::move(load)); expand_vector(ctx, tmp, dst, instr->dest.ssa.num_components, dmask); @@ -5792,7 +5945,9 @@ void visit_image_store(isel_context *ctx, nir_intrinsic_instr *instr) bool is_array = glsl_sampler_type_is_array(type); Temp data = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[3].ssa)); - bool glc = ctx->options->chip_class == GFX6 || var->data.access & (ACCESS_VOLATILE | ACCESS_COHERENT | ACCESS_NON_READABLE) ? 1 : 0; + memory_sync_info sync = get_memory_sync_info(instr, storage_image, 0); + unsigned access = var->data.access | nir_intrinsic_access(instr); + bool glc = ctx->options->chip_class == GFX6 || access & (ACCESS_VOLATILE | ACCESS_COHERENT | ACCESS_NON_READABLE) ? 1 : 0; if (dim == GLSL_SAMPLER_DIM_BUF) { Temp rsrc = get_sampler_desc(ctx, nir_instr_as_deref(instr->src[0].ssa->parent_instr), ACO_DESC_BUFFER, nullptr, true, true); @@ -5823,7 +5978,7 @@ void visit_image_store(isel_context *ctx, nir_intrinsic_instr *instr) store->glc = glc; store->dlc = false; store->disable_wqm = true; - store->barrier = barrier_image; + store->sync = sync; ctx->program->needs_exact = true; ctx->block->instructions.emplace_back(std::move(store)); return; @@ -5847,7 +6002,7 @@ void visit_image_store(isel_context *ctx, nir_intrinsic_instr *instr) store->unrm = true; store->da = should_declare_array(ctx, dim, glsl_sampler_type_is_array(type)); store->disable_wqm = true; - store->barrier = barrier_image; + store->sync = sync; ctx->program->needs_exact = true; ctx->block->instructions.emplace_back(std::move(store)); return; @@ -5925,6 +6080,7 @@ void visit_image_atomic(isel_context *ctx, nir_intrinsic_instr *instr) } Temp dst = get_ssa_temp(ctx, &instr->dest.ssa); + memory_sync_info sync = get_memory_sync_info(instr, storage_image, semantic_atomicrmw); if (dim == GLSL_SAMPLER_DIM_BUF) { Temp vindex = emit_extract_vector(ctx, get_ssa_temp(ctx, instr->src[1].ssa), 0, v1); @@ -5942,7 +6098,7 @@ void visit_image_atomic(isel_context *ctx, nir_intrinsic_instr *instr) mubuf->glc = return_previous; mubuf->dlc = false; /* Not needed for atomics */ mubuf->disable_wqm = true; - mubuf->barrier = barrier_image; + mubuf->sync = sync; ctx->program->needs_exact = true; ctx->block->instructions.emplace_back(std::move(mubuf)); return; @@ -5963,7 +6119,7 @@ void visit_image_atomic(isel_context *ctx, nir_intrinsic_instr *instr) mimg->unrm = true; mimg->da = should_declare_array(ctx, dim, glsl_sampler_type_is_array(type)); mimg->disable_wqm = true; - mimg->barrier = barrier_image; + mimg->sync = sync; ctx->program->needs_exact = true; ctx->block->instructions.emplace_back(std::move(mimg)); return; @@ -6027,7 +6183,6 @@ void visit_image_size(isel_context *ctx, nir_intrinsic_instr *instr) mimg->dim = ac_get_image_dim(ctx->options->chip_class, dim, is_array); mimg->dmask = (1 << instr->dest.ssa.num_components) - 1; mimg->da = glsl_sampler_type_is_array(type); - mimg->can_reorder = true; Definition& def = mimg->definitions[0]; ctx->block->instructions.emplace_back(std::move(mimg)); @@ -6070,10 +6225,20 @@ void visit_load_ssbo(isel_context *ctx, nir_intrinsic_instr *instr) Temp rsrc = convert_pointer_to_64_bit(ctx, get_ssa_temp(ctx, instr->src[0].ssa)); rsrc = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), rsrc, Operand(0u)); - bool glc = nir_intrinsic_access(instr) & (ACCESS_VOLATILE | ACCESS_COHERENT); + unsigned access = nir_intrinsic_access(instr); + bool glc = access & (ACCESS_VOLATILE | ACCESS_COHERENT); unsigned size = instr->dest.ssa.bit_size / 8; + + uint32_t flags = get_all_buffer_resource_flags(ctx, instr->src[0].ssa, access); + /* GLC bypasses VMEM/SMEM caches, so GLC SMEM loads/stores are coherent with GLC VMEM loads/stores + * TODO: this optimization is disabled for now because we still need to ensure correct ordering + */ + bool allow_smem = !(flags & (0 && glc ? has_nonglc_vmem_store : has_vmem_store)); + allow_smem |= ((access & ACCESS_RESTRICT) && (access & ACCESS_NON_WRITEABLE)) || (access & ACCESS_CAN_REORDER); + load_buffer(ctx, num_components, size, dst, rsrc, get_ssa_temp(ctx, instr->src[1].ssa), - nir_intrinsic_align_mul(instr), nir_intrinsic_align_offset(instr), glc, false); + nir_intrinsic_align_mul(instr), nir_intrinsic_align_offset(instr), glc, allow_smem, + get_memory_sync_info(instr, storage_buffer, 0)); } void visit_store_ssbo(isel_context *ctx, nir_intrinsic_instr *instr) @@ -6087,9 +6252,18 @@ void visit_store_ssbo(isel_context *ctx, nir_intrinsic_instr *instr) Temp rsrc = convert_pointer_to_64_bit(ctx, get_ssa_temp(ctx, instr->src[1].ssa)); rsrc = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), rsrc, Operand(0u)); + memory_sync_info sync = get_memory_sync_info(instr, storage_buffer, 0); + bool glc = nir_intrinsic_access(instr) & (ACCESS_VOLATILE | ACCESS_COHERENT | ACCESS_NON_READABLE); + uint32_t flags = get_all_buffer_resource_flags(ctx, instr->src[1].ssa, nir_intrinsic_access(instr)); + /* GLC bypasses VMEM/SMEM caches, so GLC SMEM loads/stores are coherent with GLC VMEM loads/stores + * TODO: this optimization is disabled for now because we still need to ensure correct ordering + */ + bool allow_smem = !(flags & (0 && glc ? has_nonglc_vmem_loadstore : has_vmem_loadstore)); + bool smem = !nir_src_is_divergent(instr->src[2]) && ctx->options->chip_class >= GFX8 && - elem_size_bytes >= 4; + (elem_size_bytes >= 4 || can_subdword_ssbo_store_use_smem(instr)) && + allow_smem; if (smem) offset = bld.as_uniform(offset); bool smem_nonfs = smem && ctx->stage != fragment_fs; @@ -6109,8 +6283,8 @@ void visit_store_ssbo(isel_context *ctx, nir_intrinsic_instr *instr) aco_ptr store{create_instruction(op, Format::SMEM, 3, 0)}; store->operands[0] = Operand(rsrc); if (offsets[i]) { - Temp off = bld.sop2(aco_opcode::s_add_i32, bld.def(s1), bld.def(s1, scc), - offset, Operand(offsets[i])); + Temp off = bld.nuw().sop2(aco_opcode::s_add_i32, bld.def(s1), bld.def(s1, scc), + offset, Operand(offsets[i])); store->operands[1] = Operand(off); } else { store->operands[1] = Operand(offset); @@ -6118,10 +6292,10 @@ void visit_store_ssbo(isel_context *ctx, nir_intrinsic_instr *instr) if (op != aco_opcode::p_fs_buffer_store_smem) store->operands[1].setFixed(m0); store->operands[2] = Operand(write_datas[i]); - store->glc = nir_intrinsic_access(instr) & (ACCESS_VOLATILE | ACCESS_COHERENT | ACCESS_NON_READABLE); + store->glc = glc; store->dlc = false; store->disable_wqm = true; - store->barrier = barrier_buffer; + store->sync = sync; ctx->block->instructions.emplace_back(std::move(store)); ctx->program->wb_smem_l1_on_end = true; if (op == aco_opcode::p_fs_buffer_store_smem) { @@ -6136,10 +6310,10 @@ void visit_store_ssbo(isel_context *ctx, nir_intrinsic_instr *instr) store->operands[3] = Operand(write_datas[i]); store->offset = offsets[i]; store->offen = (offset.type() == RegType::vgpr); - store->glc = nir_intrinsic_access(instr) & (ACCESS_VOLATILE | ACCESS_COHERENT | ACCESS_NON_READABLE); + store->glc = glc; store->dlc = false; store->disable_wqm = true; - store->barrier = barrier_buffer; + store->sync = sync; ctx->program->needs_exact = true; ctx->block->instructions.emplace_back(std::move(store)); } @@ -6230,7 +6404,7 @@ void visit_atomic_ssbo(isel_context *ctx, nir_intrinsic_instr *instr) mubuf->glc = return_previous; mubuf->dlc = false; /* Not needed for atomics */ mubuf->disable_wqm = true; - mubuf->barrier = barrier_buffer; + mubuf->sync = get_memory_sync_info(instr, storage_buffer, semantic_atomicrmw); ctx->program->needs_exact = true; ctx->block->instructions.emplace_back(std::move(mubuf)); } @@ -6255,8 +6429,7 @@ void visit_load_global(isel_context *ctx, nir_intrinsic_instr *instr) info.glc = nir_intrinsic_access(instr) & (ACCESS_VOLATILE | ACCESS_COHERENT); info.align_mul = nir_intrinsic_align_mul(instr); info.align_offset = nir_intrinsic_align_offset(instr); - info.barrier = barrier_buffer; - info.can_reorder = false; + info.sync = get_memory_sync_info(instr, storage_buffer, 0); /* VMEM stores don't update the SMEM cache and it's difficult to prove that * it's safe to use SMEM */ bool can_use_smem = nir_intrinsic_access(instr) & ACCESS_NON_WRITEABLE; @@ -6276,6 +6449,7 @@ void visit_store_global(isel_context *ctx, nir_intrinsic_instr *instr) Temp data = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[0].ssa)); Temp addr = get_ssa_temp(ctx, instr->src[1].ssa); + memory_sync_info sync = get_memory_sync_info(instr, storage_buffer, 0); bool glc = nir_intrinsic_access(instr) & (ACCESS_VOLATILE | ACCESS_COHERENT | ACCESS_NON_READABLE); if (ctx->options->chip_class >= GFX7) @@ -6341,7 +6515,7 @@ void visit_store_global(isel_context *ctx, nir_intrinsic_instr *instr) flat->dlc = false; flat->offset = offset; flat->disable_wqm = true; - flat->barrier = barrier_buffer; + flat->sync = sync; ctx->program->needs_exact = true; ctx->block->instructions.emplace_back(std::move(flat)); } else { @@ -6361,7 +6535,7 @@ void visit_store_global(isel_context *ctx, nir_intrinsic_instr *instr) mubuf->offset = offsets[i]; mubuf->addr64 = addr.type() == RegType::vgpr; mubuf->disable_wqm = true; - mubuf->barrier = barrier_buffer; + mubuf->sync = sync; ctx->program->needs_exact = true; ctx->block->instructions.emplace_back(std::move(mubuf)); } @@ -6454,7 +6628,7 @@ void visit_global_atomic(isel_context *ctx, nir_intrinsic_instr *instr) flat->dlc = false; /* Not needed for atomics */ flat->offset = 0; flat->disable_wqm = true; - flat->barrier = barrier_buffer; + flat->sync = get_memory_sync_info(instr, storage_buffer, semantic_atomicrmw); ctx->program->needs_exact = true; ctx->block->instructions.emplace_back(std::move(flat)); } else { @@ -6521,29 +6695,81 @@ void visit_global_atomic(isel_context *ctx, nir_intrinsic_instr *instr) mubuf->offset = 0; mubuf->addr64 = addr.type() == RegType::vgpr; mubuf->disable_wqm = true; - mubuf->barrier = barrier_buffer; + mubuf->sync = get_memory_sync_info(instr, storage_buffer, semantic_atomicrmw); ctx->program->needs_exact = true; ctx->block->instructions.emplace_back(std::move(mubuf)); } } +sync_scope translate_nir_scope(nir_scope scope) +{ + switch (scope) { + case NIR_SCOPE_NONE: + case NIR_SCOPE_INVOCATION: + return scope_invocation; + case NIR_SCOPE_SUBGROUP: + return scope_subgroup; + case NIR_SCOPE_WORKGROUP: + return scope_workgroup; + case NIR_SCOPE_QUEUE_FAMILY: + return scope_queuefamily; + case NIR_SCOPE_DEVICE: + return scope_device; + } + unreachable("invalid scope"); +} + void emit_memory_barrier(isel_context *ctx, nir_intrinsic_instr *instr) { Builder bld(ctx->program, ctx->block); + storage_class all_mem = (storage_class)(storage_buffer | storage_image | storage_atomic_counter | storage_shared); switch(instr->intrinsic) { case nir_intrinsic_group_memory_barrier: + bld.barrier(aco_opcode::p_barrier, + memory_sync_info(all_mem, semantic_acqrel, scope_workgroup)); + break; case nir_intrinsic_memory_barrier: - bld.barrier(aco_opcode::p_memory_barrier_common); + bld.barrier(aco_opcode::p_barrier, + memory_sync_info(all_mem, semantic_acqrel, scope_device)); break; case nir_intrinsic_memory_barrier_buffer: - bld.barrier(aco_opcode::p_memory_barrier_buffer); - break; + bld.barrier(aco_opcode::p_barrier, + memory_sync_info((storage_class)storage_buffer, semantic_acqrel, scope_device)); case nir_intrinsic_memory_barrier_image: - bld.barrier(aco_opcode::p_memory_barrier_image); + bld.barrier(aco_opcode::p_barrier, + memory_sync_info((storage_class)storage_image, semantic_acqrel, scope_device)); break; case nir_intrinsic_memory_barrier_tcs_patch: case nir_intrinsic_memory_barrier_shared: - bld.barrier(aco_opcode::p_memory_barrier_shared); + bld.barrier(aco_opcode::p_barrier, + memory_sync_info(storage_shared, semantic_acqrel, scope_workgroup)); + break; + case nir_intrinsic_scoped_barrier: { + unsigned semantics = 0; + unsigned storage = 0; + sync_scope mem_scope = translate_nir_scope(nir_intrinsic_memory_scope(instr)); + sync_scope exec_scope = translate_nir_scope(nir_intrinsic_execution_scope(instr)); + + unsigned nir_storage = nir_intrinsic_memory_modes(instr); + if (nir_storage & (nir_var_mem_ssbo | nir_var_mem_global)) + storage |= storage_buffer | storage_image; //TODO: split this when NIR gets nir_var_mem_image + if (ctx->shader->info.stage == MESA_SHADER_COMPUTE && (nir_storage & nir_var_mem_shared)) + storage |= storage_shared; + if (ctx->shader->info.stage == MESA_SHADER_TESS_CTRL && (nir_storage & nir_var_shader_out)) + storage |= storage_shared; + + unsigned nir_semantics = nir_intrinsic_memory_semantics(instr); + if (nir_semantics & NIR_MEMORY_ACQUIRE) + semantics |= semantic_acquire | semantic_release; + if (nir_semantics & NIR_MEMORY_RELEASE) + semantics |= semantic_acquire | semantic_release; + + assert(!(nir_semantics & (NIR_MEMORY_MAKE_AVAILABLE | NIR_MEMORY_MAKE_VISIBLE))); + + bld.barrier(aco_opcode::p_barrier, + memory_sync_info((storage_class)storage, (memory_semantics)semantics, mem_scope), + exec_scope); break; + } default: unreachable("Unimplemented memory barrier intrinsic"); break; @@ -6636,7 +6862,7 @@ void visit_shared_atomic(isel_context *ctx, nir_intrinsic_instr *instr) op32 = aco_opcode::ds_write_b32; op64 = aco_opcode::ds_write_b64; op32_rtn = aco_opcode::ds_wrxchg_rtn_b32; - op64_rtn = aco_opcode::ds_wrxchg2_rtn_b64; + op64_rtn = aco_opcode::ds_wrxchg_rtn_b64; break; case nir_intrinsic_shared_atomic_comp_swap: op32 = aco_opcode::ds_cmpst_b32; @@ -6645,6 +6871,12 @@ void visit_shared_atomic(isel_context *ctx, nir_intrinsic_instr *instr) op64_rtn = aco_opcode::ds_cmpst_rtn_b64; num_operands = 4; break; + case nir_intrinsic_shared_atomic_fadd: + op32 = aco_opcode::ds_add_f32; + op32_rtn = aco_opcode::ds_add_rtn_f32; + op64 = aco_opcode::num_opcodes; + op64_rtn = aco_opcode::num_opcodes; + break; default: unreachable("Unhandled shared atomic intrinsic"); } @@ -6684,6 +6916,7 @@ void visit_shared_atomic(isel_context *ctx, nir_intrinsic_instr *instr) ds->offset0 = offset; if (return_previous) ds->definitions[0] = Definition(get_ssa_temp(ctx, &instr->dest.ssa)); + ds->sync = memory_sync_info(storage_shared, semantic_atomicrmw); ctx->block->instructions.emplace_back(std::move(ds)); } @@ -6695,7 +6928,7 @@ Temp get_scratch_resource(isel_context *ctx) scratch_addr = bld.smem(aco_opcode::s_load_dwordx2, bld.def(s2), scratch_addr, Operand(0u)); uint32_t rsrc_conf = S_008F0C_ADD_TID_ENABLE(1) | - S_008F0C_INDEX_STRIDE(ctx->program->wave_size == 64 ? 3 : 2);; + S_008F0C_INDEX_STRIDE(ctx->program->wave_size == 64 ? 3 : 2); if (ctx->program->chip_class >= GFX10) { rsrc_conf |= S_008F0C_FORMAT(V_008F0C_IMG_FORMAT_32_FLOAT) | @@ -6706,9 +6939,9 @@ Temp get_scratch_resource(isel_context *ctx) S_008F0C_DATA_FORMAT(V_008F0C_BUF_DATA_FORMAT_32); } - /* older generations need element size = 16 bytes. element size removed in GFX9 */ + /* older generations need element size = 4 bytes. element size removed in GFX9 */ if (ctx->program->chip_class <= GFX8) - rsrc_conf |= S_008F0C_ELEMENT_SIZE(3); + rsrc_conf |= S_008F0C_ELEMENT_SIZE(1); return bld.pseudo(aco_opcode::p_create_vector, bld.def(s4), scratch_addr, Operand(-1u), Operand(rsrc_conf)); } @@ -6723,10 +6956,10 @@ void visit_load_scratch(isel_context *ctx, nir_intrinsic_instr *instr) { instr->dest.ssa.bit_size / 8u, rsrc}; info.align_mul = nir_intrinsic_align_mul(instr); info.align_offset = nir_intrinsic_align_offset(instr); - info.swizzle_component_size = 16; - info.can_reorder = false; + info.swizzle_component_size = ctx->program->chip_class <= GFX8 ? 4 : 0; + info.sync = memory_sync_info(storage_scratch, semantic_private); info.soffset = ctx->program->scratch_offset; - emit_mubuf_load(ctx, bld, &info); + emit_scratch_load(ctx, bld, &info); } void visit_store_scratch(isel_context *ctx, nir_intrinsic_instr *instr) { @@ -6741,12 +6974,14 @@ void visit_store_scratch(isel_context *ctx, nir_intrinsic_instr *instr) { unsigned write_count = 0; Temp write_datas[32]; unsigned offsets[32]; + unsigned swizzle_component_size = ctx->program->chip_class <= GFX8 ? 4 : 16; split_buffer_store(ctx, instr, false, RegType::vgpr, data, writemask, - 16, &write_count, write_datas, offsets); + swizzle_component_size, &write_count, write_datas, offsets); for (unsigned i = 0; i < write_count; i++) { aco_opcode op = get_buffer_store_op(false, write_datas[i].bytes()); - bld.mubuf(op, rsrc, offset, ctx->program->scratch_offset, write_datas[i], offsets[i], true); + Instruction *instr = bld.mubuf(op, rsrc, offset, ctx->program->scratch_offset, write_datas[i], offsets[i], true, true); + static_cast(instr)->sync = memory_sync_info(storage_scratch, semantic_private); } } @@ -6860,8 +7095,7 @@ void visit_emit_vertex_with_counter(isel_context *ctx, nir_intrinsic_instr *inst mtbuf->offset = const_offset; mtbuf->glc = true; mtbuf->slc = true; - mtbuf->barrier = barrier_gs_data; - mtbuf->can_reorder = true; + mtbuf->sync = memory_sync_info(storage_vmem_output, semantic_can_reorder); bld.insert(std::move(mtbuf)); } @@ -7127,6 +7361,7 @@ void visit_intrinsic(isel_context *ctx, nir_intrinsic_instr *instr) Temp addr = get_ssa_temp(ctx, instr->src[0].ssa); nir_const_value* const_addr = nir_src_as_const_value(instr->src[0]); Temp private_segment_buffer = ctx->program->private_segment_buffer; + //TODO: bounds checking? if (addr.type() == RegType::sgpr) { Operand offset; if (const_addr) { @@ -7185,8 +7420,6 @@ void visit_intrinsic(isel_context *ctx, nir_intrinsic_instr *instr) load->glc = false; load->dlc = false; load->disable_wqm = false; - load->barrier = barrier_none; - load->can_reorder = true; ctx->block->instructions.emplace_back(std::move(load)); } @@ -7299,6 +7532,7 @@ void visit_intrinsic(isel_context *ctx, nir_intrinsic_instr *instr) case nir_intrinsic_shared_atomic_xor: case nir_intrinsic_shared_atomic_exchange: case nir_intrinsic_shared_atomic_comp_swap: + case nir_intrinsic_shared_atomic_fadd: visit_shared_atomic(ctx, instr); break; case nir_intrinsic_image_deref_load: @@ -7368,17 +7602,7 @@ void visit_intrinsic(isel_context *ctx, nir_intrinsic_instr *instr) visit_get_buffer_size(ctx, instr); break; case nir_intrinsic_control_barrier: { - if (ctx->program->chip_class == GFX6 && ctx->shader->info.stage == MESA_SHADER_TESS_CTRL) { - /* GFX6 only (thanks to a hw bug workaround): - * The real barrier instruction isn’t needed, because an entire patch - * always fits into a single wave. - */ - break; - } - - if (ctx->program->workgroup_size > ctx->program->wave_size) - bld.sopp(aco_opcode::s_barrier); - + bld.barrier(aco_opcode::p_barrier, memory_sync_info(0, 0, scope_invocation), scope_workgroup); break; } case nir_intrinsic_memory_barrier_tcs_patch: @@ -7387,6 +7611,7 @@ void visit_intrinsic(isel_context *ctx, nir_intrinsic_instr *instr) case nir_intrinsic_memory_barrier_buffer: case nir_intrinsic_memory_barrier_image: case nir_intrinsic_memory_barrier_shared: + case nir_intrinsic_scoped_barrier: emit_memory_barrier(ctx, instr); break; case nir_intrinsic_load_num_work_groups: { @@ -7485,7 +7710,14 @@ void visit_intrinsic(isel_context *ctx, nir_intrinsic_instr *instr) if (instr->intrinsic == nir_intrinsic_read_invocation || !nir_src_is_divergent(instr->src[1])) tid = bld.as_uniform(tid); Temp dst = get_ssa_temp(ctx, &instr->dest.ssa); - if (src.regClass() == v1) { + if (src.regClass() == v1b || src.regClass() == v2b) { + Temp tmp = bld.tmp(v1); + tmp = emit_wqm(ctx, emit_bpermute(ctx, bld, tid, src), tmp); + if (dst.type() == RegType::vgpr) + bld.pseudo(aco_opcode::p_split_vector, Definition(dst), bld.def(src.regClass() == v1b ? v3b : v2b), tmp); + else + bld.pseudo(aco_opcode::p_as_uniform, Definition(dst), tmp); + } else if (src.regClass() == v1) { emit_wqm(ctx, emit_bpermute(ctx, bld, tid, src), dst); } else if (src.regClass() == v2) { Temp lo = bld.tmp(v1), hi = bld.tmp(v1); @@ -7530,7 +7762,7 @@ void visit_intrinsic(isel_context *ctx, nir_intrinsic_instr *instr) case nir_intrinsic_read_first_invocation: { Temp src = get_ssa_temp(ctx, instr->src[0].ssa); Temp dst = get_ssa_temp(ctx, &instr->dest.ssa); - if (src.regClass() == v1) { + if (src.regClass() == v1b || src.regClass() == v2b || src.regClass() == v1) { emit_wqm(ctx, bld.vop1(aco_opcode::v_readfirstlane_b32, bld.def(s1), src), dst); @@ -7615,27 +7847,31 @@ void visit_intrinsic(isel_context *ctx, nir_intrinsic_instr *instr) } else if (cluster_size == 1) { bld.copy(Definition(dst), src); } else { - src = as_vgpr(ctx, src); + unsigned bit_size = instr->src[0].ssa->bit_size; + + src = emit_extract_vector(ctx, src, 0, RegClass::get(RegType::vgpr, bit_size / 8)); ReduceOp reduce_op; switch (op) { - #define CASE(name) case nir_op_##name: reduce_op = (src.regClass() == v1) ? name##32 : name##64; break; - CASE(iadd) - CASE(imul) - CASE(fadd) - CASE(fmul) - CASE(imin) - CASE(umin) - CASE(fmin) - CASE(imax) - CASE(umax) - CASE(fmax) - CASE(iand) - CASE(ior) - CASE(ixor) + #define CASEI(name) case nir_op_##name: reduce_op = (bit_size == 32) ? name##32 : (bit_size == 16) ? name##16 : (bit_size == 8) ? name##8 : name##64; break; + #define CASEF(name) case nir_op_##name: reduce_op = (bit_size == 32) ? name##32 : (bit_size == 16) ? name##16 : name##64; break; + CASEI(iadd) + CASEI(imul) + CASEI(imin) + CASEI(umin) + CASEI(imax) + CASEI(umax) + CASEI(iand) + CASEI(ior) + CASEI(ixor) + CASEF(fadd) + CASEF(fmul) + CASEF(fmin) + CASEF(fmax) default: unreachable("unknown reduction op"); - #undef CASE + #undef CASEI + #undef CASEF } aco_opcode aco_op; @@ -7688,6 +7924,20 @@ void visit_intrinsic(isel_context *ctx, nir_intrinsic_instr *instr) bld.sop2(Builder::s_and, bld.def(bld.lm), bld.def(s1, scc), mask_tmp, bld.sop2(Builder::s_and, bld.def(bld.lm), bld.def(s1, scc), src, Operand(exec, bld.lm)))); emit_wqm(ctx, tmp, dst); + } else if (instr->dest.ssa.bit_size == 8) { + Temp tmp = bld.tmp(v1); + if (ctx->program->chip_class >= GFX8) + emit_wqm(ctx, bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), src, dpp_ctrl), tmp); + else + emit_wqm(ctx, bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), src, (1 << 15) | dpp_ctrl), tmp); + bld.pseudo(aco_opcode::p_split_vector, Definition(dst), bld.def(v3b), tmp); + } else if (instr->dest.ssa.bit_size == 16) { + Temp tmp = bld.tmp(v1); + if (ctx->program->chip_class >= GFX8) + emit_wqm(ctx, bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), src, dpp_ctrl), tmp); + else + emit_wqm(ctx, bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), src, (1 << 15) | dpp_ctrl), tmp); + bld.pseudo(aco_opcode::p_split_vector, Definition(dst), bld.def(v2b), tmp); } else if (instr->dest.ssa.bit_size == 32) { if (ctx->program->chip_class >= GFX8) emit_wqm(ctx, bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), src, dpp_ctrl), dst); @@ -7752,6 +8002,20 @@ void visit_intrinsic(isel_context *ctx, nir_intrinsic_instr *instr) src = bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), src, dpp_ctrl); Temp tmp = bld.vopc(aco_opcode::v_cmp_lg_u32, bld.def(bld.lm), Operand(0u), src); emit_wqm(ctx, tmp, dst); + } else if (instr->dest.ssa.bit_size == 8) { + Temp tmp = bld.tmp(v1); + if (ctx->program->chip_class >= GFX8) + emit_wqm(ctx, bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), src, dpp_ctrl), tmp); + else + emit_wqm(ctx, bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), src, dpp_ctrl), tmp); + bld.pseudo(aco_opcode::p_split_vector, Definition(dst), bld.def(v3b), tmp); + } else if (instr->dest.ssa.bit_size == 16) { + Temp tmp = bld.tmp(v1); + if (ctx->program->chip_class >= GFX8) + emit_wqm(ctx, bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), src, dpp_ctrl), tmp); + else + emit_wqm(ctx, bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), src, dpp_ctrl), tmp); + bld.pseudo(aco_opcode::p_split_vector, Definition(dst), bld.def(v2b), tmp); } else if (instr->dest.ssa.bit_size == 32) { Temp tmp; if (ctx->program->chip_class >= GFX8) @@ -7786,15 +8050,25 @@ void visit_intrinsic(isel_context *ctx, nir_intrinsic_instr *instr) } Temp dst = get_ssa_temp(ctx, &instr->dest.ssa); uint32_t mask = nir_intrinsic_swizzle_mask(instr); - if (dst.regClass() == v1) { - emit_wqm(ctx, - bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), src, mask, 0, false), - dst); + if (instr->dest.ssa.bit_size == 1) { + assert(src.regClass() == bld.lm); + src = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(0u), Operand((uint32_t)-1), src); + src = emit_masked_swizzle(ctx, bld, src, mask); + Temp tmp = bld.vopc(aco_opcode::v_cmp_lg_u32, bld.def(bld.lm), Operand(0u), src); + emit_wqm(ctx, tmp, dst); + } else if (dst.regClass() == v1b) { + Temp tmp = emit_wqm(ctx, emit_masked_swizzle(ctx, bld, src, mask)); + emit_extract_vector(ctx, tmp, 0, dst); + } else if (dst.regClass() == v2b) { + Temp tmp = emit_wqm(ctx, emit_masked_swizzle(ctx, bld, src, mask)); + emit_extract_vector(ctx, tmp, 0, dst); + } else if (dst.regClass() == v1) { + emit_wqm(ctx, emit_masked_swizzle(ctx, bld, src, mask), dst); } else if (dst.regClass() == v2) { Temp lo = bld.tmp(v1), hi = bld.tmp(v1); bld.pseudo(aco_opcode::p_split_vector, Definition(lo), Definition(hi), src); - lo = emit_wqm(ctx, bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), lo, mask, 0, false)); - hi = emit_wqm(ctx, bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), hi, mask, 0, false)); + lo = emit_wqm(ctx, emit_masked_swizzle(ctx, bld, lo, mask)); + hi = emit_wqm(ctx, emit_masked_swizzle(ctx, bld, hi, mask)); bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lo, hi); emit_split_vector(ctx, dst, 2); } else { @@ -7877,10 +8151,14 @@ void visit_intrinsic(isel_context *ctx, nir_intrinsic_instr *instr) get_ssa_temp(ctx, &instr->dest.ssa)); break; } - case nir_intrinsic_shader_clock: - bld.smem(aco_opcode::s_memtime, Definition(get_ssa_temp(ctx, &instr->dest.ssa)), false); + case nir_intrinsic_shader_clock: { + aco_opcode opcode = + nir_intrinsic_memory_scope(instr) == NIR_SCOPE_DEVICE ? + aco_opcode::s_memrealtime : aco_opcode::s_memtime; + bld.smem(opcode, Definition(get_ssa_temp(ctx, &instr->dest.ssa)), memory_sync_info(0, semantic_volatile)); emit_split_vector(ctx, get_ssa_temp(ctx, &instr->dest.ssa), 2); break; + } case nir_intrinsic_load_vertex_id_zero_base: { Temp dst = get_ssa_temp(ctx, &instr->dest.ssa); bld.copy(Definition(dst), get_arg(ctx, ctx->args->ac.vertex_id)); @@ -8458,7 +8736,6 @@ void visit_tex(isel_context *ctx, nir_tex_instr *instr) tex->da = da; tex->definitions[0] = Definition(tmp_dst); tex->dim = dim; - tex->can_reorder = true; ctx->block->instructions.emplace_back(std::move(tex)); if (div_by_6) { @@ -8491,7 +8768,6 @@ void visit_tex(isel_context *ctx, nir_tex_instr *instr) tex->da = da; Temp size = bld.tmp(v2); tex->definitions[0] = Definition(size); - tex->can_reorder = true; ctx->block->instructions.emplace_back(std::move(tex)); emit_split_vector(ctx, size, size.size()); @@ -8593,7 +8869,6 @@ void visit_tex(isel_context *ctx, nir_tex_instr *instr) mubuf->operands[2] = Operand((uint32_t) 0); mubuf->definitions[0] = Definition(tmp_dst); mubuf->idxen = true; - mubuf->can_reorder = true; ctx->block->instructions.emplace_back(std::move(mubuf)); expand_vector(ctx, tmp_dst, dst, instr->dest.ssa.num_components, (1 << last_bit) - 1); @@ -8642,7 +8917,6 @@ void visit_tex(isel_context *ctx, nir_tex_instr *instr) tex->unrm = true; tex->da = da; tex->definitions[0] = Definition(tmp_dst); - tex->can_reorder = true; ctx->block->instructions.emplace_back(std::move(tex)); if (instr->op == nir_texop_samples_identical) { @@ -8736,14 +9010,34 @@ void visit_tex(isel_context *ctx, nir_tex_instr *instr) } if (instr->op == nir_texop_tg4) { - if (has_offset) { - opcode = aco_opcode::image_gather4_lz_o; - if (has_compare) + if (has_offset) { /* image_gather4_*_o */ + if (has_compare) { opcode = aco_opcode::image_gather4_c_lz_o; + if (has_lod) + opcode = aco_opcode::image_gather4_c_l_o; + if (has_bias) + opcode = aco_opcode::image_gather4_c_b_o; + } else { + opcode = aco_opcode::image_gather4_lz_o; + if (has_lod) + opcode = aco_opcode::image_gather4_l_o; + if (has_bias) + opcode = aco_opcode::image_gather4_b_o; + } } else { - opcode = aco_opcode::image_gather4_lz; - if (has_compare) + if (has_compare) { opcode = aco_opcode::image_gather4_c_lz; + if (has_lod) + opcode = aco_opcode::image_gather4_c_l; + if (has_bias) + opcode = aco_opcode::image_gather4_c_b; + } else { + opcode = aco_opcode::image_gather4_lz; + if (has_lod) + opcode = aco_opcode::image_gather4_l; + if (has_bias) + opcode = aco_opcode::image_gather4_b; + } } } else if (instr->op == nir_texop_lod) { opcode = aco_opcode::image_get_lod; @@ -8766,7 +9060,6 @@ void visit_tex(isel_context *ctx, nir_tex_instr *instr) tex->dmask = dmask; tex->da = da; tex->definitions[0] = Definition(tmp_dst); - tex->can_reorder = true; ctx->block->instructions.emplace_back(std::move(tex)); if (tg4_integer_cube_workaround) { @@ -8794,13 +9087,19 @@ void visit_tex(isel_context *ctx, nir_tex_instr *instr) } -Operand get_phi_operand(isel_context *ctx, nir_ssa_def *ssa) +Operand get_phi_operand(isel_context *ctx, nir_ssa_def *ssa, RegClass rc, bool logical) { Temp tmp = get_ssa_temp(ctx, ssa); - if (ssa->parent_instr->type == nir_instr_type_ssa_undef) - return Operand(tmp.regClass()); - else + if (ssa->parent_instr->type == nir_instr_type_ssa_undef) { + return Operand(rc); + } else if (logical && ssa->bit_size == 1 && ssa->parent_instr->type == nir_instr_type_load_const) { + if (ctx->program->wave_size == 64) + return Operand(nir_instr_as_load_const(ssa->parent_instr)->value[0].b ? UINT64_MAX : 0u); + else + return Operand(nir_instr_as_load_const(ssa->parent_instr)->value[0].b ? UINT32_MAX : 0u); + } else { return Operand(tmp); + } } void visit_phi(isel_context *ctx, nir_phi_instr *instr) @@ -8843,7 +9142,7 @@ void visit_phi(isel_context *ctx, nir_phi_instr *instr) if (!(ctx->block->kind & block_kind_loop_header) && cur_pred_idx >= preds.size()) continue; cur_pred_idx++; - Operand op = get_phi_operand(ctx, src.second); + Operand op = get_phi_operand(ctx, src.second, dst.regClass(), logical); operands[num_operands++] = op; num_defined += !op.isUndefined(); } @@ -9847,6 +10146,7 @@ static bool export_fs_mrt_color(isel_context *ctx, int slot) bool is_int8 = (ctx->options->key.fs.is_int8 >> slot) & 1; bool is_int10 = (ctx->options->key.fs.is_int10 >> slot) & 1; + bool is_16bit = values[0].regClass() == v2b; switch (col_format) { @@ -9877,16 +10177,38 @@ static bool export_fs_mrt_color(isel_context *ctx, int slot) case V_028714_SPI_SHADER_FP16_ABGR: enabled_channels = 0x5; compr_op = aco_opcode::v_cvt_pkrtz_f16_f32; + if (is_16bit) { + if (ctx->options->chip_class >= GFX9) { + /* Pack the FP16 values together instead of converting them to + * FP32 and back to FP16. + * TODO: use p_create_vector and let the compiler optimizes. + */ + compr_op = aco_opcode::v_pack_b32_f16; + } else { + for (unsigned i = 0; i < 4; i++) { + if ((write_mask >> i) & 1) + values[i] = bld.vop1(aco_opcode::v_cvt_f32_f16, bld.def(v1), values[i]); + } + } + } break; case V_028714_SPI_SHADER_UNORM16_ABGR: enabled_channels = 0x5; - compr_op = aco_opcode::v_cvt_pknorm_u16_f32; + if (is_16bit && ctx->options->chip_class >= GFX9) { + compr_op = aco_opcode::v_cvt_pknorm_u16_f16; + } else { + compr_op = aco_opcode::v_cvt_pknorm_u16_f32; + } break; case V_028714_SPI_SHADER_SNORM16_ABGR: enabled_channels = 0x5; - compr_op = aco_opcode::v_cvt_pknorm_i16_f32; + if (is_16bit && ctx->options->chip_class >= GFX9) { + compr_op = aco_opcode::v_cvt_pknorm_i16_f16; + } else { + compr_op = aco_opcode::v_cvt_pknorm_i16_f32; + } break; case V_028714_SPI_SHADER_UINT16_ABGR: { @@ -9904,6 +10226,13 @@ static bool export_fs_mrt_color(isel_context *ctx, int slot) values[i]); } } + } else if (is_16bit) { + for (unsigned i = 0; i < 4; i++) { + if ((write_mask >> i) & 1) { + Temp tmp = convert_int(ctx, bld, values[i].getTemp(), 16, 32, false); + values[i] = Operand(tmp); + } + } } break; } @@ -9928,6 +10257,13 @@ static bool export_fs_mrt_color(isel_context *ctx, int slot) values[i]); } } + } else if (is_16bit) { + for (unsigned i = 0; i < 4; i++) { + if ((write_mask >> i) & 1) { + Temp tmp = convert_int(ctx, bld, values[i].getTemp(), 16, 32, true); + values[i] = Operand(tmp); + } + } } break; @@ -9942,6 +10278,26 @@ static bool export_fs_mrt_color(isel_context *ctx, int slot) if (target == V_008DFC_SQ_EXP_NULL) return false; + /* Replace NaN by zero (only 32-bit) to fix game bugs if requested. */ + if (ctx->options->enable_mrt_output_nan_fixup && + !is_16bit && + (col_format == V_028714_SPI_SHADER_32_R || + col_format == V_028714_SPI_SHADER_32_GR || + col_format == V_028714_SPI_SHADER_32_AR || + col_format == V_028714_SPI_SHADER_32_ABGR || + col_format == V_028714_SPI_SHADER_FP16_ABGR)) { + for (int i = 0; i < 4; i++) { + if (!(write_mask & (1 << i))) + continue; + + Temp isnan = bld.vopc(aco_opcode::v_cmp_class_f32, + bld.hint_vcc(bld.def(bld.lm)), values[i], + bld.copy(bld.def(v1), Operand(3u))); + values[i] = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), values[i], + bld.copy(bld.def(v1), Operand(0u)), isnan); + } + } + if ((bool) compr_op) { for (int i = 0; i < 2; i++) { /* check if at least one of the values to be compressed is enabled */ @@ -9986,6 +10342,13 @@ static void create_fs_exports(isel_context *ctx) create_null_export(ctx); } +static void create_workgroup_barrier(Builder& bld) +{ + bld.barrier(aco_opcode::p_barrier, + memory_sync_info(storage_shared, semantic_acqrel, scope_workgroup), + scope_workgroup); +} + static void write_tcs_tess_factors(isel_context *ctx) { unsigned outer_comps; @@ -10010,9 +10373,7 @@ static void write_tcs_tess_factors(isel_context *ctx) Builder bld(ctx->program, ctx->block); - bld.barrier(aco_opcode::p_memory_barrier_shared); - if (unlikely(ctx->program->chip_class != GFX6 && ctx->program->workgroup_size > ctx->program->wave_size)) - bld.sopp(aco_opcode::s_barrier); + create_workgroup_barrier(bld); Temp tcs_rel_ids = get_arg(ctx, ctx->args->ac.tcs_rel_ids); Temp invocation_id = bld.vop3(aco_opcode::v_bfe_u32, bld.def(v1), tcs_rel_ids, Operand(8u), Operand(5u)); @@ -10062,8 +10423,8 @@ static void write_tcs_tess_factors(isel_context *ctx) Temp control_word = bld.copy(bld.def(v1), Operand(0x80000000u)); bld.mubuf(aco_opcode::buffer_store_dword, /* SRSRC */ hs_ring_tess_factor, /* VADDR */ Operand(v1), /* SOFFSET */ tf_base, /* VDATA */ control_word, - /* immediate OFFSET */ 0, /* OFFEN */ false, /* idxen*/ false, /* addr64 */ false, - /* disable_wqm */ false, /* glc */ true); + /* immediate OFFSET */ 0, /* OFFEN */ false, /* swizzled */ false, /* idxen*/ false, + /* addr64 */ false, /* disable_wqm */ false, /* glc */ true); tf_const_offset += 4; begin_divergent_if_else(ctx, &ic_rel_patch_id_is_zero); @@ -10073,7 +10434,7 @@ static void write_tcs_tess_factors(isel_context *ctx) assert(stride == 2 || stride == 4 || stride == 6); Temp tf_vec = create_vec_from_array(ctx, out, stride, RegType::vgpr, 4u); - store_vmem_mubuf(ctx, tf_vec, hs_ring_tess_factor, byte_offset, tf_base, tf_const_offset, 4, (1 << stride) - 1, true, false); + store_vmem_mubuf(ctx, tf_vec, hs_ring_tess_factor, byte_offset, tf_base, tf_const_offset, 4, (1 << stride) - 1, true, memory_sync_info()); /* Store to offchip for TES to read - only if TES reads them */ if (ctx->args->options->key.tcs.tes_reads_tess_factors) { @@ -10081,11 +10442,11 @@ static void write_tcs_tess_factors(isel_context *ctx) Temp oc_lds = get_arg(ctx, ctx->args->oc_lds); std::pair vmem_offs_outer = get_tcs_per_patch_output_vmem_offset(ctx, nullptr, ctx->tcs_tess_lvl_out_loc); - store_vmem_mubuf(ctx, tf_outer_vec, hs_ring_tess_offchip, vmem_offs_outer.first, oc_lds, vmem_offs_outer.second, 4, (1 << outer_comps) - 1, true, false); + store_vmem_mubuf(ctx, tf_outer_vec, hs_ring_tess_offchip, vmem_offs_outer.first, oc_lds, vmem_offs_outer.second, 4, (1 << outer_comps) - 1, true, memory_sync_info(storage_vmem_output)); if (likely(inner_comps)) { std::pair vmem_offs_inner = get_tcs_per_patch_output_vmem_offset(ctx, nullptr, ctx->tcs_tess_lvl_in_loc); - store_vmem_mubuf(ctx, tf_inner_vec, hs_ring_tess_offchip, vmem_offs_inner.first, oc_lds, vmem_offs_inner.second, 4, (1 << inner_comps) - 1, true, false); + store_vmem_mubuf(ctx, tf_inner_vec, hs_ring_tess_offchip, vmem_offs_inner.first, oc_lds, vmem_offs_inner.second, 4, (1 << inner_comps) - 1, true, memory_sync_info(storage_vmem_output)); } } @@ -10171,7 +10532,6 @@ static void emit_stream_output(isel_context *ctx, store->glc = true; store->dlc = false; store->slc = true; - store->can_reorder = true; ctx->block->instructions.emplace_back(std::move(store)); } } @@ -10353,7 +10713,8 @@ void setup_fp_mode(isel_context *ctx, nir_shader *shader) float_controls & (FLOAT_CONTROLS_ROUNDING_MODE_RTZ_FP16 | FLOAT_CONTROLS_ROUNDING_MODE_RTZ_FP64 | FLOAT_CONTROLS_ROUNDING_MODE_RTE_FP16 | FLOAT_CONTROLS_ROUNDING_MODE_RTE_FP64); - /* default to preserving fp16 and fp64 denorms, since it's free */ + /* default to preserving fp16 and fp64 denorms, since it's free for fp64 and + * the precision seems needed for Wolfenstein: Youngblood to render correctly */ if (program->next_fp_mode.must_flush_denorms16_64) program->next_fp_mode.denorm16_64 = 0; else @@ -10590,8 +10951,7 @@ void ngg_emit_nogs_output(isel_context *ctx) if (ctx->stage == ngg_vertex_gs) { /* Wait for GS threads to store primitive ID in LDS. */ - bld.barrier(aco_opcode::p_memory_barrier_shared); - bld.sopp(aco_opcode::s_barrier); + create_workgroup_barrier(bld); /* Calculate LDS address where the GS threads stored the primitive ID. */ Temp wave_id_in_tg = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc), @@ -10675,8 +11035,7 @@ void select_program(Program *program, if (i) { Builder bld(ctx.program, ctx.block); - bld.barrier(aco_opcode::p_memory_barrier_shared); - bld.sopp(aco_opcode::s_barrier); + create_workgroup_barrier(bld); if (ctx.stage == vertex_geometry_gs || ctx.stage == tess_eval_geometry_gs) { ctx.gs_wave_id = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1, m0), bld.def(s1, scc), get_arg(&ctx, args->merged_wave_info), Operand((8u << 16) | 16u)); @@ -10699,7 +11058,8 @@ void select_program(Program *program, ngg_emit_nogs_output(&ctx); } else if (nir->info.stage == MESA_SHADER_GEOMETRY) { Builder bld(ctx.program, ctx.block); - bld.barrier(aco_opcode::p_memory_barrier_gs_data); + bld.barrier(aco_opcode::p_barrier, + memory_sync_info(storage_vmem_output, semantic_release, scope_device)); bld.sopp(aco_opcode::s_sendmsg, bld.m0(ctx.gs_wave_id), -1, sendmsg_gs_done(false, false, 0)); } else if (nir->info.stage == MESA_SHADER_TESS_CTRL) { write_tcs_tess_factors(&ctx); @@ -10731,7 +11091,7 @@ void select_program(Program *program, ctx.block->kind |= block_kind_uniform; Builder bld(ctx.program, ctx.block); if (ctx.program->wb_smem_l1_on_end) - bld.smem(aco_opcode::s_dcache_wb, false); + bld.smem(aco_opcode::s_dcache_wb, memory_sync_info(storage_buffer, semantic_volatile)); bld.sopp(aco_opcode::s_endpgm); cleanup_cfg(program); @@ -10743,16 +11103,6 @@ void select_gs_copy_shader(Program *program, struct nir_shader *gs_shader, { isel_context ctx = setup_isel_context(program, 1, &gs_shader, config, args, true); - program->next_fp_mode.preserve_signed_zero_inf_nan32 = false; - program->next_fp_mode.preserve_signed_zero_inf_nan16_64 = false; - program->next_fp_mode.must_flush_denorms32 = false; - program->next_fp_mode.must_flush_denorms16_64 = false; - program->next_fp_mode.care_about_round32 = false; - program->next_fp_mode.care_about_round16_64 = false; - program->next_fp_mode.denorm16_64 = fp_denorm_keep; - program->next_fp_mode.denorm32 = 0; - program->next_fp_mode.round32 = fp_round_ne; - program->next_fp_mode.round16_64 = fp_round_ne; ctx.block->fp_mode = program->next_fp_mode; add_startpgm(&ctx); @@ -10826,8 +11176,6 @@ void select_gs_copy_shader(Program *program, struct nir_shader *gs_shader, mubuf->glc = true; mubuf->slc = true; mubuf->dlc = args->options->chip_class >= GFX10; - mubuf->barrier = barrier_none; - mubuf->can_reorder = true; ctx.outputs.mask[i] |= 1 << j; ctx.outputs.temps[i * 4u + j] = mubuf->definitions[0].getTemp();