src/gallium/drivers/r600/sfn/sfn_emitssboinstruction.cpp

   1 #include "sfn_emitssboinstruction.h"
   2
   3 #include "sfn_instruction_fetch.h"
   4 #include "sfn_instruction_gds.h"
   5 #include "sfn_instruction_misc.h"
   6 #include "../r600_pipe.h"
   7
   8 namespace r600 {
   9
  10 bool EmitSSBOInstruction::do_emit(nir_instr* instr)
  11 {
  12    const nir_intrinsic_instr *intr = nir_instr_as_intrinsic(instr);
  13    switch (intr->intrinsic) {
  14    case nir_intrinsic_atomic_counter_add:
  15    case nir_intrinsic_atomic_counter_and:
  16    case nir_intrinsic_atomic_counter_exchange:
  17    case nir_intrinsic_atomic_counter_max:
  18    case nir_intrinsic_atomic_counter_min:
  19    case nir_intrinsic_atomic_counter_or:
  20    case nir_intrinsic_atomic_counter_xor:
  21    case nir_intrinsic_atomic_counter_comp_swap:
  22       return emit_atomic(intr);
  23    case nir_intrinsic_atomic_counter_read:
  24    case nir_intrinsic_atomic_counter_post_dec:
  25       return emit_unary_atomic(intr);
  26    case nir_intrinsic_atomic_counter_inc:
  27       return emit_atomic_inc(intr);
  28    case nir_intrinsic_atomic_counter_pre_dec:
  29       return emit_atomic_pre_dec(intr);
  30    case nir_intrinsic_load_ssbo:
  31        return emit_load_ssbo(intr);
  32     case nir_intrinsic_store_ssbo:
  33       return emit_store_ssbo(intr);
  34    default:
  35       return false;
  36    }
  37 }
  38
  39 bool EmitSSBOInstruction::emit_atomic(const nir_intrinsic_instr* instr)
  40 {
  41    ESDOp op = get_opcode(instr->intrinsic);
  42
  43    if (DS_OP_INVALID == op)
  44       return false;
  45
  46    GPRVector dest = make_dest(instr);
  47
  48    int base = nir_intrinsic_base(instr);
  49
  50    PValue uav_id = from_nir(instr->src[0], 0);
  51
  52    PValue value = from_nir_with_fetch_constant(instr->src[1], 0);
  53
  54    GDSInstr *ir = nullptr;
  55    if (instr->intrinsic == nir_intrinsic_atomic_counter_comp_swap)  {
  56       PValue value2 = from_nir_with_fetch_constant(instr->src[1], 1);
  57       ir = new GDSInstr(op, dest, value, value2, uav_id, base);
  58    } else {
  59       ir = new GDSInstr(op, dest, value, uav_id, base);
  60    }
  61
  62    emit_instruction(ir);
  63    return true;
  64 }
  65
  66 bool EmitSSBOInstruction::emit_unary_atomic(const nir_intrinsic_instr* instr)
  67 {
  68    ESDOp op = get_opcode(instr->intrinsic);
  69
  70    if (DS_OP_INVALID == op)
  71       return false;
  72
  73    GPRVector dest = make_dest(instr);
  74
  75    PValue uav_id = from_nir(instr->src[0], 0);
  76
  77    auto ir = new GDSInstr(op, dest, uav_id, nir_intrinsic_base(instr));
  78
  79    emit_instruction(ir);
  80    return true;
  81 }
  82
  83 ESDOp EmitSSBOInstruction::get_opcode(const nir_intrinsic_op opcode)
  84 {
  85    switch (opcode) {
  86    case nir_intrinsic_atomic_counter_add:
  87       return DS_OP_ADD_RET;
  88    case nir_intrinsic_atomic_counter_and:
  89       return DS_OP_AND_RET;
  90    case nir_intrinsic_atomic_counter_exchange:
  91       return DS_OP_XCHG_RET;
  92    case nir_intrinsic_atomic_counter_inc:
  93       return DS_OP_INC_RET;
  94    case nir_intrinsic_atomic_counter_max:
  95       return DS_OP_MAX_UINT_RET;
  96    case nir_intrinsic_atomic_counter_min:
  97       return DS_OP_MIN_UINT_RET;
  98    case nir_intrinsic_atomic_counter_or:
  99       return DS_OP_OR_RET;
 100    case nir_intrinsic_atomic_counter_read:
 101       return DS_OP_READ_RET;
 102    case nir_intrinsic_atomic_counter_xor:
 103       return DS_OP_XOR_RET;
 104    case nir_intrinsic_atomic_counter_post_dec:
 105       return DS_OP_DEC_RET;
 106    case nir_intrinsic_atomic_counter_comp_swap:
 107       return DS_OP_CMP_XCHG_RET;
 108    case nir_intrinsic_atomic_counter_pre_dec:
 109    default:
 110       return DS_OP_INVALID;
 111    }
 112 }
 113
 114
 115 bool EmitSSBOInstruction::emit_atomic_add(const nir_intrinsic_instr* instr)
 116 {
 117    GPRVector dest = make_dest(instr);
 118
 119    PValue value = from_nir_with_fetch_constant(instr->src[1], 0);
 120
 121    PValue uav_id = from_nir(instr->src[0], 0);
 122
 123    auto ir = new GDSInstr(DS_OP_ADD_RET, dest, value, uav_id,
 124                           nir_intrinsic_base(instr));
 125
 126    emit_instruction(ir);
 127    return true;
 128 }
 129
 130 bool EmitSSBOInstruction::emit_atomic_inc(const nir_intrinsic_instr* instr)
 131 {
 132    GPRVector dest = make_dest(instr);
 133
 134    PValue uav_id = from_nir(instr->src[0], 0);
 135
 136
 137    if (!m_atomic_limit) {
 138       int one_tmp = allocate_temp_register();
 139       m_atomic_limit = PValue(new GPRValue(one_tmp, 0));
 140       emit_instruction(new AluInstruction(op1_mov, m_atomic_limit,
 141                        PValue(new LiteralValue(0xffffffff)),
 142                        {alu_write, alu_last_instr}));
 143    }
 144
 145    auto ir = new GDSInstr(DS_OP_INC_RET, dest, m_atomic_limit, uav_id,
 146                           nir_intrinsic_base(instr));
 147    emit_instruction(ir);
 148    return true;
 149 }
 150
 151 bool EmitSSBOInstruction::emit_atomic_pre_dec(const nir_intrinsic_instr *instr)
 152 {
 153    GPRVector dest = make_dest(instr);
 154
 155    PValue uav_id = from_nir(instr->src[0], 0);
 156
 157    int one_tmp = allocate_temp_register();
 158    PValue value(new GPRValue(one_tmp, 0));
 159    emit_instruction(new AluInstruction(op1_mov, value,  Value::one_i,
 160                     {alu_write, alu_last_instr}));
 161
 162    auto ir = new GDSInstr(DS_OP_SUB_RET, dest, value, uav_id,
 163                           nir_intrinsic_base(instr));
 164    emit_instruction(ir);
 165
 166    ir = new GDSInstr(DS_OP_READ_RET, dest, uav_id, nir_intrinsic_base(instr));
 167    emit_instruction(ir);
 168
 169    return true;
 170 }
 171
 172 bool EmitSSBOInstruction::emit_load_ssbo(const nir_intrinsic_instr* instr)
 173 {
 174    GPRVector dest = make_dest(instr);
 175
 176    /** src0 not used, should be some offset */
 177    auto addr = from_nir_with_fetch_constant(instr->src[1], 0);
 178    PValue addr_temp = create_register_from_nir_src(instr->src[1], 1);
 179
 180    /** Should be lowered in nir */
 181    emit_instruction(new AluInstruction(op2_lshr_int, addr_temp, {addr, PValue(new LiteralValue(2))},
 182                     {alu_write, alu_last_instr}));
 183
 184    const EVTXDataFormat formats[4] = {
 185       fmt_32,
 186       fmt_32_32,
 187       fmt_32_32_32,
 188       fmt_32_32_32_32
 189    };
 190
 191    const std::array<int,4> dest_swt[4] = {
 192       {0,7,7,7},
 193       {0,1,7,7},
 194       {0,1,2,7},
 195       {0,1,2,3}
 196    };
 197
 198    /* TODO fix resource index */
 199    auto ir = new FetchInstruction(dest, addr_temp,
 200                                   R600_IMAGE_REAL_RESOURCE_OFFSET, from_nir(instr->src[0], 0),
 201                                   formats[instr->num_components-1], vtx_nf_int);
 202    ir->set_dest_swizzle(dest_swt[instr->num_components - 1]);
 203    ir->set_flag(vtx_use_tc);
 204
 205    emit_instruction(ir);
 206    return true;
 207 }
 208
 209 bool EmitSSBOInstruction::emit_store_ssbo(const nir_intrinsic_instr* instr)
 210 {
 211
 212    GPRVector::Swizzle swz = {7,7,7,7};
 213    for (int i = 0; i <  instr->src[0].ssa->num_components; ++i)
 214       swz[i] = i;
 215
 216    auto orig_addr = from_nir(instr->src[2], 0);
 217
 218    int temp1 = allocate_temp_register();
 219    GPRVector addr_vec(temp1, {0,1,2,7});
 220
 221    auto rat_id = from_nir(instr->src[1], 0);
 222
 223    emit_instruction(new AluInstruction(op2_lshr_int, addr_vec.reg_i(0), orig_addr,
 224                                        PValue(new LiteralValue(2)), write));
 225    emit_instruction(new AluInstruction(op1_mov, addr_vec.reg_i(1), Value::zero, write));
 226    emit_instruction(new AluInstruction(op1_mov, addr_vec.reg_i(2), Value::zero, last_write));
 227
 228
 229 //#define WRITE_AS_VECTOR
 230 #ifdef WRITE_AS_VECTOR
 231    std::unique_ptr<GPRVector> value(vec_from_nir_with_fetch_constant(instr->src[0],
 232                                     (1 << instr->src[0].ssa->num_components) - 1, swz));
 233
 234    /* TODO fix resource index */
 235    int nelements = instr->src[0].ssa->num_components - 1;
 236    if (nelements == 2)
 237       nelements = 3;
 238    auto ir = new RatInstruction(cf_mem_rat, RatInstruction::STORE_TYPED,
 239                                 *value, addr_vec, 0, rat_id, 11,
 240                                 (1 << instr->src[0].ssa->num_components) - 1,
 241                                 0, false);
 242    emit_instruction(ir);
 243 #else
 244
 245    auto values = vec_from_nir_with_fetch_constant(instr->src[0],
 246          (1 << instr->src[0].ssa->num_components) - 1, {0,1,2,3}, true);
 247
 248    emit_instruction(new RatInstruction(cf_mem_rat, RatInstruction::STORE_TYPED,
 249                                        values, addr_vec, 0, rat_id, 1,
 250                                        1, 0, false));
 251    for (int i = 1; i < instr->src[0].ssa->num_components; ++i) {
 252       emit_instruction(new AluInstruction(op1_mov, values.reg_i(0), from_nir(instr->src[0], i), write));
 253       emit_instruction(new AluInstruction(op2_add_int, addr_vec.reg_i(0),
 254                                           {addr_vec.reg_i(0), Value::one_i}, last_write));
 255       emit_instruction(new RatInstruction(cf_mem_rat, RatInstruction::STORE_TYPED,
 256                                           values, addr_vec, 0, rat_id, 1,
 257                                           1, 0, false));
 258    }
 259 #endif
 260    return true;
 261 }
 262
 263 GPRVector EmitSSBOInstruction::make_dest(const nir_intrinsic_instr* ir)
 264 {
 265    GPRVector::Values v;
 266    int i;
 267    for (i = 0; i < 4; ++i)
 268       v[i] = from_nir(ir->dest, i);
 269    return GPRVector(v);
 270 }
 271
 272 }