From 32d3435a78675ff5ebf933d45b9b99fdc4dc7d82 Mon Sep 17 00:00:00 2001 From: Gert Wollny Date: Fri, 27 Dec 2019 17:49:27 +0100 Subject: [PATCH] r600/sfn: Add GDS instructions Signed-off-by: Gert Wollny Part-of: --- src/gallium/drivers/r600/meson.build | 2 + .../drivers/r600/sfn/sfn_instruction_gds.cpp | 145 +++++++++++++ .../drivers/r600/sfn/sfn_instruction_gds.h | 195 ++++++++++++++++++ .../drivers/r600/sfn/sfn_ir_to_assembly.cpp | 162 +++++++++++++++ 4 files changed, 504 insertions(+) create mode 100644 src/gallium/drivers/r600/sfn/sfn_instruction_gds.cpp create mode 100644 src/gallium/drivers/r600/sfn/sfn_instruction_gds.h diff --git a/src/gallium/drivers/r600/meson.build b/src/gallium/drivers/r600/meson.build index 3f85127d9cc..5080741c308 100644 --- a/src/gallium/drivers/r600/meson.build +++ b/src/gallium/drivers/r600/meson.build @@ -131,6 +131,8 @@ files_r600 = files( 'sfn/sfn_instruction_export.h', 'sfn/sfn_instruction_fetch.cpp', 'sfn/sfn_instruction_fetch.h', + 'sfn/sfn_instruction_gds.cpp', + 'sfn/sfn_instruction_gds.h', 'sfn/sfn_instruction_misc.cpp', 'sfn/sfn_instruction_misc.h', 'sfn/sfn_instruction_tex.cpp', diff --git a/src/gallium/drivers/r600/sfn/sfn_instruction_gds.cpp b/src/gallium/drivers/r600/sfn/sfn_instruction_gds.cpp new file mode 100644 index 00000000000..2fd8c7292a3 --- /dev/null +++ b/src/gallium/drivers/r600/sfn/sfn_instruction_gds.cpp @@ -0,0 +1,145 @@ +/* -*- mesa-c++ -*- + * + * Copyright (c) 2019 Collabora LTD + * + * Author: Gert Wollny + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * on the rights to use, copy, modify, merge, publish, distribute, sub + * license, and/or sell copies of the Software, and to permit persons to whom + * the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice (including the next + * paragraph) shall be included in all copies or substantial portions of the + * Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL + * THE AUTHOR(S) AND/OR THEIR SUPPLIERS BE LIABLE FOR ANY CLAIM, + * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE + * USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +#include "sfn_instruction_gds.h" +#include "sfn_liverange.h" + +namespace r600 { + +GDSInstr::GDSInstr(ESDOp op, const GPRVector& dest, const PValue& value, + const PValue& value2, const PValue& uav_id, int uav_base): + Instruction(gds), + m_op(op), + m_src(value), + m_src2(value2), + m_dest(dest), + m_dest_swizzle({PIPE_SWIZZLE_X,7,7,7}), + m_src_swizzle({PIPE_SWIZZLE_0, PIPE_SWIZZLE_X, 7}), + m_buffer_index_mode(bim_none), + m_uav_id(uav_id), + m_uav_base(uav_base), + m_flags(0) +{ + add_remappable_src_value(&m_src); + add_remappable_src_value(&m_uav_id); + add_remappable_dst_value(&m_dest); +} + +GDSInstr::GDSInstr(ESDOp op, const GPRVector& dest, const PValue& value, + const PValue& uav_id, int uav_base): + GDSInstr(op, dest, value, PValue(), uav_id, uav_base) +{ +} + +GDSInstr::GDSInstr(ESDOp op, const GPRVector& dest, + const PValue& uav_id, int uav_base): + GDSInstr(op, dest, PValue(), PValue(), uav_id, uav_base) +{ + m_src_swizzle[1] = PIPE_SWIZZLE_1; +} + +bool GDSInstr::is_equal_to(UNUSED const Instruction& lhs) const +{ + return false; +} + +void GDSInstr::do_print(std::ostream& os) const +{ + const char *swz = "xyzw01?_"; + os << lds_ops.at(m_op).name << " R" << m_dest.sel() << "."; + for (int i = 0; i < 4; ++i) { + os << swz[m_dest_swizzle[i]]; + } + if (m_src) + os << " " << *m_src; + + os << " UAV:" << *m_uav_id; +} + +RatInstruction::RatInstruction(ECFOpCode cf_opcode, ERatOp rat_op, + const GPRVector& data, const GPRVector& index, + int rat_id, const PValue& rat_id_offset, + int burst_count, int comp_mask, int element_size, bool ack): + Instruction(rat), + m_cf_opcode(cf_opcode), + m_rat_op(rat_op), + m_data(data), + m_index(index), + m_rat_id(rat_id), + m_rat_id_offset(rat_id_offset), + m_burst_count(burst_count), + m_comp_mask(comp_mask), + m_element_size(element_size), + m_need_ack(ack) +{ + add_remappable_src_value(&m_data); + add_remappable_src_value(&m_rat_id_offset); + add_remappable_src_value(&m_index); +} + +bool RatInstruction::is_equal_to(UNUSED const Instruction& lhs) const +{ + return false; +} + +void RatInstruction::do_print(std::ostream& os) const +{ + os << "MEM_RAT RAT(" << m_rat_id; + if (m_rat_id_offset) + os << "+" << *m_rat_id_offset; + os << ") @" << m_index << " OP:" << m_rat_op << " " << m_data; + os << " BC:" << m_burst_count + << " MASK:" << m_comp_mask + << " ES:" << m_element_size; + if (m_need_ack) + os << " ACK"; +} + +RatInstruction::ERatOp RatInstruction::opcode(nir_intrinsic_op opcode) +{ + switch (opcode) { + case nir_intrinsic_ssbo_atomic_add: + return ADD_RTN; + case nir_intrinsic_ssbo_atomic_and: + return AND_RTN; + case nir_intrinsic_ssbo_atomic_exchange: + return XCHG_RTN; + case nir_intrinsic_ssbo_atomic_umax: + return MAX_UINT_RTN; + case nir_intrinsic_ssbo_atomic_umin: + return MIN_UINT_RTN; + case nir_intrinsic_ssbo_atomic_imax: + return MAX_INT_RTN; + case nir_intrinsic_ssbo_atomic_imin: + return MIN_INT_RTN; + case nir_intrinsic_ssbo_atomic_xor: + return XOR_RTN; + default: + return UNSUPPORTED; + } +} + +} diff --git a/src/gallium/drivers/r600/sfn/sfn_instruction_gds.h b/src/gallium/drivers/r600/sfn/sfn_instruction_gds.h new file mode 100644 index 00000000000..1499d7fb736 --- /dev/null +++ b/src/gallium/drivers/r600/sfn/sfn_instruction_gds.h @@ -0,0 +1,195 @@ +/* -*- mesa-c++ -*- + * + * Copyright (c) 2018-2019 Collabora LTD + * + * Author: Gert Wollny + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * on the rights to use, copy, modify, merge, publish, distribute, sub + * license, and/or sell copies of the Software, and to permit persons to whom + * the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice (including the next + * paragraph) shall be included in all copies or substantial portions of the + * Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL + * THE AUTHOR(S) AND/OR THEIR SUPPLIERS BE LIABLE FOR ANY CLAIM, + * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE + * USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +#ifndef SFN_GDSINSTR_H +#define SFN_GDSINSTR_H + +#include "sfn_instruction_base.h" + +#include + +namespace r600 { + +class GDSInstr : public Instruction +{ +public: + GDSInstr(ESDOp op, const GPRVector& dest, const PValue& value, + const PValue &uav_id, int uav_base); + GDSInstr(ESDOp op, const GPRVector& dest, const PValue& value, + const PValue& value2, const PValue &uav_id, int uav_base); + GDSInstr(ESDOp op, const GPRVector& dest, const PValue &uav_id, int uav_base); + + ESDOp op() const {return m_op;} + + int src_sel() const { + if (!m_src) + return 0; + + assert(m_src->type() == Value::gpr); + return m_src->sel(); + } + + int src2_chan() const { + if (!m_src2) + return 0; + + assert(m_src->type() == Value::gpr); + return m_src->chan(); + } + + int src_swizzle(int idx) const {assert(idx < 3); return m_src_swizzle[idx];} + + int dest_sel() const { + return m_dest.sel(); + } + + int dest_swizzle(int i) const { + if (i < 4) + return m_dest_swizzle[i]; + return 7; + } + + void set_dest_swizzle(const std::array& swz) { + m_dest_swizzle = swz; + } + + PValue uav_id() const {return m_uav_id;} + int uav_base() const {return m_uav_base;} + +private: + + bool is_equal_to(const Instruction& lhs) const override; + void do_print(std::ostream& os) const override; + + ESDOp m_op; + + PValue m_src; + PValue m_src2; + GPRVector m_dest; + std::array m_dest_swizzle; + std::array m_src_swizzle; + + EBufferIndexMode m_buffer_index_mode; + PValue m_uav_id; + int m_uav_base; + std::bitset<8> m_flags; + +}; + +class RatInstruction : public Instruction { + +public: + enum ERatOp { + NOP, + STORE_TYPED, + STORE_RAW, + STORE_RAW_FDENORM, + CMPXCHG_INT, + CMPXCHG_FLT, + CMPXCHG_FDENORM, + ADD, + SUB, + RSUB, + MIN_INT, + MIN_UINT, + MAX_INT, + MAX_UINT, + AND, + OR, + XOR, + MSKOR, + INC_UINT, + DEC_UINT, + NOP_RTN = 32, + XCHG_RTN = 34, + XCHG_FDENORM_RTN, + CMPXCHG_INT_RTN, + CMPXCHG_FLT_RTN, + CMPXCHG_FDENORM_RTN, + ADD_RTN, + SUB_RTN, + RSUB_RTN, + MIN_INT_RTN, + MIN_UINT_RTN, + MAX_INT_RTN, + MAX_UINT_RTN, + AND_RTN, + OR_RTN, + XOR_RTN, + MSKOR_RTN, + UINT_RTN, + UNSUPPORTED + }; + + RatInstruction(ECFOpCode cf_opcode, ERatOp rat_op, + const GPRVector& data, const GPRVector& index, + int rat_id, const PValue& rat_id_offset, + int burst_count, int comp_mask, int element_size, + bool ack); + + PValue rat_id_offset() const { return m_rat_id_offset;} + + ERatOp rat_op() const {return m_rat_op;} + + int data_gpr() const {return m_data.sel();} + int index_gpr() const {return m_index.sel();} + int elm_size() const {return m_element_size;} + + int comp_mask() const {return m_comp_mask;} + + bool need_ack() const {return m_need_ack;} + int burst_count() const {return m_burst_count;} + + static ERatOp opcode(nir_intrinsic_op opcode); + + int data_swz(int chan) const {return m_data.chan_i(chan);} + +private: + + bool is_equal_to(const Instruction& lhs) const override; + void do_print(std::ostream& os) const override; + + ECFOpCode m_cf_opcode; + ERatOp m_rat_op; + + GPRVector m_data; + GPRVector m_index; + + int m_rat_id; + PValue m_rat_id_offset; + int m_burst_count; + int m_comp_mask; + int m_element_size; + + std::bitset<8> m_flags; + + bool m_need_ack; + +}; + +} + +#endif // SFN_GDSINSTR_H diff --git a/src/gallium/drivers/r600/sfn/sfn_ir_to_assembly.cpp b/src/gallium/drivers/r600/sfn/sfn_ir_to_assembly.cpp index 4909ad498d8..db75b908f5f 100644 --- a/src/gallium/drivers/r600/sfn/sfn_ir_to_assembly.cpp +++ b/src/gallium/drivers/r600/sfn/sfn_ir_to_assembly.cpp @@ -27,6 +27,7 @@ #include "sfn_ir_to_assembly.h" #include "sfn_conditionaljumptracker.h" #include "sfn_callstack.h" +#include "sfn_instruction_gds.h" #include "sfn_instruction_misc.h" #include "sfn_instruction_fetch.h" @@ -61,6 +62,8 @@ private: bool emit_loop_continue(const LoopContInstruction& instr); bool emit_wait_ack(const WaitAck& instr); bool emit_wr_scratch(const WriteScratchInstruction& instr); + bool emit_gds(const GDSInstr& instr); + bool emit_rat(const RatInstruction& instr); bool emit_load_addr(PValue addr); bool emit_fs_pixel_export(const ExportInstruction & exi); @@ -176,6 +179,10 @@ bool AssemblyFromShaderLegacyImpl::emit(const Instruction::Pointer i) return emit_wait_ack(static_cast(*i)); case Instruction::mem_wr_scratch: return emit_wr_scratch(static_cast(*i)); + case Instruction::gds: + return emit_gds(static_cast(*i)); + case Instruction::rat: + return emit_rat(static_cast(*i)); default: return false; } @@ -837,6 +844,161 @@ bool AssemblyFromShaderLegacyImpl::emit_wr_scratch(const WriteScratchInstruction extern const std::map ds_opcode_map; +bool AssemblyFromShaderLegacyImpl::emit_gds(const GDSInstr& instr) +{ + struct r600_bytecode_gds gds; + + int uav_idx = -1; + auto addr = instr.uav_id(); + if (addr->type() != Value::literal) { + if (!m_bc->index_loaded[1] || m_loop_nesting || + m_bc->index_reg[1] != addr->sel()) { + struct r600_bytecode_alu alu; + + memset(&alu, 0, sizeof(alu)); + alu.op = opcode_map.at(op2_lshr_int); + alu.dst.sel = addr->sel(); + alu.dst.chan = addr->chan(); + alu.src[0].sel = addr->sel(); + alu.src[0].chan = addr->chan(); + alu.src[1].sel = ALU_SRC_LITERAL; + alu.src[1].value = 2; + alu.last = 1; + alu.dst.write = 1; + int r = r600_bytecode_add_alu(m_bc, &alu); + if (r) + return false; + + memset(&alu, 0, sizeof(alu)); + alu.op = opcode_map.at(op1_mova_int); + alu.dst.chan = 0; + alu.src[0].sel = addr->sel(); + alu.src[0].chan = addr->chan(); + alu.last = 1; + r = r600_bytecode_add_alu(m_bc, &alu); + if (r) + return false; + + m_bc->ar_loaded = 0; + + alu.op = opcode_map.at(op1_set_cf_idx1); + alu.dst.chan = 0; + alu.src[0].sel = 0; + alu.src[0].chan = 0; + alu.last = 1; + + r = r600_bytecode_add_alu(m_bc, &alu); + if (r) + return false; + + m_bc->index_reg[1] = addr->sel(); + m_bc->index_loaded[1] = true; + } + } else { + const LiteralValue& addr_reg = dynamic_cast(*addr); + uav_idx = addr_reg.value() >> 2; + } + + memset(&gds, 0, sizeof(struct r600_bytecode_gds)); + + gds.op = ds_opcode_map.at(instr.op()); + gds.dst_gpr = instr.dest_sel(); + gds.uav_id = (uav_idx >= 0 ? uav_idx : 0) + instr.uav_base(); + gds.uav_index_mode = uav_idx >= 0 ? bim_none : bim_one; + gds.src_gpr = instr.src_sel(); + + if (instr.op() == DS_OP_CMP_XCHG_RET) { + gds.src_sel_z = 1; + } else { + gds.src_sel_z = 7; + } + + gds.src_sel_x = instr.src_swizzle(0); + gds.src_sel_y = instr.src_swizzle(1); + + gds.dst_sel_x = 0; + gds.dst_sel_y = 7; + gds.dst_sel_z = 7; + gds.dst_sel_w = 7; + gds.src_gpr2 = 0; + gds.alloc_consume = 1; // Not Cayman + + int r = r600_bytecode_add_gds(m_bc, &gds); + if (r) + return false; + m_bc->cf_last->vpm = 1; + return true; +} + + +bool AssemblyFromShaderLegacyImpl::emit_rat(const RatInstruction& instr) +{ + struct r600_bytecode_gds gds; + + int rat_idx = -1; + EBufferIndexMode rat_index_mode = bim_none; + auto addr = instr.rat_id_offset(); + + if (addr) { + if (addr->type() != Value::literal) { + rat_index_mode = bim_one; + if (!m_bc->index_loaded[1] || m_loop_nesting || m_bc->index_reg[1] != addr->sel()) { + struct r600_bytecode_alu alu; + + memset(&alu, 0, sizeof(alu)); + alu.op = opcode_map.at(op1_mova_int); + alu.dst.chan = 0; + alu.src[0].sel = addr->sel(); + alu.src[0].chan = addr->chan(); + alu.last = 1; + int r = r600_bytecode_add_alu(m_bc, &alu); + if (r) + return false; + + m_bc->ar_loaded = 0; + + alu.op = opcode_map.at(op1_set_cf_idx1); + alu.dst.chan = 0; + alu.src[0].sel = 0; + alu.src[0].chan = 0; + alu.last = 1; + + r = r600_bytecode_add_alu(m_bc, &alu); + if (r) + return false; + + m_bc->index_reg[1] = addr->sel(); + m_bc->index_loaded[1] = true; + + } + } else { + const LiteralValue& addr_reg = dynamic_cast(*addr); + rat_idx = addr_reg.value(); + } + } + memset(&gds, 0, sizeof(struct r600_bytecode_gds)); + + r600_bytecode_add_cfinst(m_bc, CF_OP_MEM_RAT); + auto cf = m_bc->cf_last; + cf->rat.id = rat_idx + m_shader->rat_base; + cf->rat.inst = instr.rat_op(); + cf->rat.index_mode = rat_index_mode; + cf->output.type = instr.need_ack() ? 3 : 1; + cf->output.gpr = instr.data_gpr(); + cf->output.index_gpr = instr.index_gpr(); + cf->output.comp_mask = instr.comp_mask(); + cf->output.burst_count = instr.burst_count(); + cf->output.swizzle_x = instr.data_swz(0); + cf->output.swizzle_y = instr.data_swz(1); + cf->output.swizzle_z = instr.data_swz(2); + cf->output.swizzle_w = instr.data_swz(3); + cf->vpm = 1; + cf->barrier = 1; + cf->mark = instr.need_ack(); + cf->output.elem_size = instr.elm_size(); + return true; +} + bool AssemblyFromShaderLegacyImpl::copy_dst(r600_bytecode_alu_dst& dst, const Value& d) { -- 2.30.2