4 * Copyright (c) 2019 Valve Corporation
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the "Software"),
8 * to deal in the Software without restriction, including without limitation
9 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
10 * and/or sell copies of the Software, and to permit persons to whom the
11 * Software is furnished to do so, subject to the following conditions:
13 * The above copyright notice and this permission notice (including the next
14 * paragraph) shall be included in all copies or substantial portions of the
17 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
18 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
19 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
20 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
21 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
22 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
25 * This file was generated by aco_builder_h.py
32 #include "util/u_math.h"
33 #include "util/bitscan.h"
37 _dpp_quad_perm = 0x000,
45 dpp_row_mirror = 0x140,
46 dpp_row_half_mirror = 0x141,
47 dpp_row_bcast15 = 0x142,
48 dpp_row_bcast31 = 0x143
52 dpp_quad_perm(unsigned lane0, unsigned lane1, unsigned lane2, unsigned lane3)
54 assert(lane0 < 4 && lane1 < 4 && lane2 < 4 && lane3 < 4);
55 return (dpp_ctrl)(lane0 | (lane1 << 2) | (lane2 << 4) | (lane3 << 6));
59 dpp_row_sl(unsigned amount)
61 assert(amount > 0 && amount < 16);
62 return (dpp_ctrl)(((unsigned) _dpp_row_sl) | amount);
66 dpp_row_sr(unsigned amount)
68 assert(amount > 0 && amount < 16);
69 return (dpp_ctrl)(((unsigned) _dpp_row_sr) | amount);
73 ds_pattern_bitmode(unsigned and_mask, unsigned or_mask, unsigned xor_mask)
75 assert(and_mask < 32 && or_mask < 32 && xor_mask < 32);
76 return and_mask | (or_mask << 5) | (xor_mask << 10);
79 aco_ptr<Instruction> create_s_mov(Definition dst, Operand src);
85 sendmsg_save_wave = 4,
86 sendmsg_stall_wave_gen = 5,
87 sendmsg_halt_waves = 6,
88 sendmsg_ordered_ps_done = 7,
89 sendmsg_early_prim_dealloc = 8,
90 sendmsg_gs_alloc_req = 9,
91 sendmsg_id_mask = 0xf,
95 sendmsg_gs(bool cut, bool emit, unsigned stream)
98 return (sendmsg)((unsigned)_sendmsg_gs | (cut << 4) | (emit << 5) | (stream << 8));
102 sendmsg_gs_done(bool cut, bool emit, unsigned stream)
105 return (sendmsg)((unsigned)_sendmsg_gs_done | (cut << 4) | (emit << 5) | (stream << 8));
113 Result(Instruction *instr) : instr(instr) {}
115 operator Instruction *() const {
119 operator Temp() const {
120 return instr->definitions[0].getTemp();
123 operator Operand() const {
124 return Operand((Temp)*this);
127 Definition& def(unsigned index) const {
128 return instr->definitions[index];
131 aco_ptr<Instruction> get_ptr() const {
132 return aco_ptr<Instruction>(instr);
138 Op(Temp tmp) : op(tmp) {}
139 Op(Operand op_) : op(op_) {}
140 Op(Result res) : op((Temp)res) {}
143 enum WaveSpecificOpcode {
144 s_cselect = (unsigned) aco_opcode::s_cselect_b64,
145 s_cmp_lg = (unsigned) aco_opcode::s_cmp_lg_u64,
146 s_and = (unsigned) aco_opcode::s_and_b64,
147 s_andn2 = (unsigned) aco_opcode::s_andn2_b64,
148 s_or = (unsigned) aco_opcode::s_or_b64,
149 s_orn2 = (unsigned) aco_opcode::s_orn2_b64,
150 s_not = (unsigned) aco_opcode::s_not_b64,
151 s_mov = (unsigned) aco_opcode::s_mov_b64,
152 s_wqm = (unsigned) aco_opcode::s_wqm_b64,
153 s_and_saveexec = (unsigned) aco_opcode::s_and_saveexec_b64,
154 s_or_saveexec = (unsigned) aco_opcode::s_or_saveexec_b64,
155 s_xnor = (unsigned) aco_opcode::s_xnor_b64,
156 s_xor = (unsigned) aco_opcode::s_xor_b64,
157 s_bcnt1_i32 = (unsigned) aco_opcode::s_bcnt1_i32_b64,
158 s_bitcmp1 = (unsigned) aco_opcode::s_bitcmp1_b64,
159 s_ff1_i32 = (unsigned) aco_opcode::s_ff1_i32_b64,
164 bool start; // only when use_iterator == false
167 std::vector<aco_ptr<Instruction>> *instructions;
168 std::vector<aco_ptr<Instruction>>::iterator it;
170 Builder(Program *pgm) : program(pgm), use_iterator(false), start(false), lm(pgm->lane_mask), instructions(NULL) {}
171 Builder(Program *pgm, Block *block) : program(pgm), use_iterator(false), start(false), lm(pgm ? pgm->lane_mask : s2), instructions(&block->instructions) {}
172 Builder(Program *pgm, std::vector<aco_ptr<Instruction>> *instrs) : program(pgm), use_iterator(false), start(false), lm(pgm ? pgm->lane_mask : s2), instructions(instrs) {}
174 void moveEnd(Block *block) {
175 instructions = &block->instructions;
179 use_iterator = false;
184 void reset(Block *block) {
185 use_iterator = false;
187 instructions = &block->instructions;
190 void reset(std::vector<aco_ptr<Instruction>> *instrs) {
191 use_iterator = false;
193 instructions = instrs;
196 void reset(std::vector<aco_ptr<Instruction>> *instrs, std::vector<aco_ptr<Instruction>>::iterator instr_it) {
199 instructions = instrs;
203 Result insert(aco_ptr<Instruction> instr) {
204 Instruction *instr_ptr = instr.get();
207 it = instructions->emplace(it, std::move(instr));
210 instructions->emplace_back(std::move(instr));
212 instructions->emplace(instructions->begin(), std::move(instr));
215 return Result(instr_ptr);
218 Result insert(Instruction* instr) {
221 it = instructions->emplace(it, aco_ptr<Instruction>(instr));
224 instructions->emplace_back(aco_ptr<Instruction>(instr));
226 instructions->emplace(instructions->begin(), aco_ptr<Instruction>(instr));
229 return Result(instr);
232 Temp tmp(RegClass rc) {
233 return (Temp){program->allocateId(), rc};
236 Temp tmp(RegType type, unsigned size) {
237 return (Temp){program->allocateId(), RegClass(type, size)};
240 Definition def(RegClass rc) {
241 return Definition((Temp){program->allocateId(), rc});
244 Definition def(RegType type, unsigned size) {
245 return Definition((Temp){program->allocateId(), RegClass(type, size)});
248 Definition def(RegClass rc, PhysReg reg) {
249 return Definition(program->allocateId(), reg, rc);
252 inline aco_opcode w64or32(WaveSpecificOpcode opcode) const {
253 if (program->wave_size == 64)
254 return (aco_opcode) opcode;
258 return aco_opcode::s_cselect_b32;
260 return aco_opcode::s_cmp_lg_u32;
262 return aco_opcode::s_and_b32;
264 return aco_opcode::s_andn2_b32;
266 return aco_opcode::s_or_b32;
268 return aco_opcode::s_orn2_b32;
270 return aco_opcode::s_not_b32;
272 return aco_opcode::s_mov_b32;
274 return aco_opcode::s_wqm_b32;
276 return aco_opcode::s_and_saveexec_b32;
278 return aco_opcode::s_or_saveexec_b32;
280 return aco_opcode::s_xnor_b32;
282 return aco_opcode::s_xor_b32;
284 return aco_opcode::s_bcnt1_i32_b32;
286 return aco_opcode::s_bitcmp1_b32;
288 return aco_opcode::s_ff1_i32_b32;
290 unreachable("Unsupported wave specific opcode.");
294 % for fixed in ['m0', 'vcc', 'exec', 'scc']:
295 Operand ${fixed}(Temp tmp) {
296 % if fixed == 'vcc' or fixed == 'exec':
297 assert(tmp.regClass() == lm);
300 op.setFixed(aco::${fixed});
304 Definition ${fixed}(Definition def) {
305 % if fixed == 'vcc' or fixed == 'exec':
306 assert(def.regClass() == lm);
308 def.setFixed(aco::${fixed});
312 Definition hint_${fixed}(Definition def) {
313 % if fixed == 'vcc' or fixed == 'exec':
314 assert(def.regClass() == lm);
316 def.setHint(aco::${fixed});
321 /* hand-written helpers */
322 Temp as_uniform(Op op)
324 assert(op.op.isTemp());
325 if (op.op.getTemp().type() == RegType::vgpr)
326 return pseudo(aco_opcode::p_as_uniform, def(RegType::sgpr, op.op.size()), op);
328 return op.op.getTemp();
331 Result v_mul_imm(Definition dst, Temp tmp, uint32_t imm, bool bits24=false)
333 assert(tmp.type() == RegType::vgpr);
335 return vop1(aco_opcode::v_mov_b32, dst, Operand(0u));
336 } else if (imm == 1) {
337 return copy(dst, Operand(tmp));
338 } else if (util_is_power_of_two_or_zero(imm)) {
339 return vop2(aco_opcode::v_lshlrev_b32, dst, Operand((uint32_t)ffs(imm) - 1u), tmp);
341 return vop2(aco_opcode::v_mul_u32_u24, dst, Operand(imm), tmp);
343 Temp imm_tmp = copy(def(v1), Operand(imm));
344 return vop3(aco_opcode::v_mul_lo_u32, dst, imm_tmp, tmp);
348 Result v_mul24_imm(Definition dst, Temp tmp, uint32_t imm)
350 return v_mul_imm(dst, tmp, imm, true);
353 Result copy(Definition dst, Op op_) {
355 if (dst.regClass() == s1 && op.size() == 1 && op.isLiteral()) {
356 uint32_t imm = op.constantValue();
357 if (imm == 0x3e22f983) {
358 if (program->chip_class >= GFX8)
359 op.setFixed(PhysReg{248}); /* it can be an inline constant on GFX8+ */
360 } else if (imm >= 0xffff8000 || imm <= 0x7fff) {
361 return sopk(aco_opcode::s_movk_i32, dst, imm & 0xFFFFu);
362 } else if (util_bitreverse(imm) <= 64 || util_bitreverse(imm) >= 0xFFFFFFF0) {
363 uint32_t rev = util_bitreverse(imm);
364 return dst.regClass() == v1 ?
365 vop1(aco_opcode::v_bfrev_b32, dst, Operand(rev)) :
366 sop1(aco_opcode::s_brev_b32, dst, Operand(rev));
367 } else if (imm != 0) {
368 unsigned start = (ffs(imm) - 1) & 0x1f;
369 unsigned size = util_bitcount(imm) & 0x1f;
370 if ((((1u << size) - 1u) << start) == imm)
371 return sop2(aco_opcode::s_bfm_b32, dst, Operand(size), Operand(start));
375 if (dst.regClass() == s2) {
376 return sop1(aco_opcode::s_mov_b64, dst, op);
377 } else if (op.size() > 1) {
378 return pseudo(aco_opcode::p_create_vector, dst, op);
379 } else if (dst.regClass() == v1 || dst.regClass() == v1.as_linear()) {
380 return vop1(aco_opcode::v_mov_b32, dst, op);
382 assert(dst.regClass() == s1);
383 return sop1(aco_opcode::s_mov_b32, dst, op);
387 Result vadd32(Definition dst, Op a, Op b, bool carry_out=false, Op carry_in=Op(Operand(s2)), bool post_ra=false) {
388 if (!b.op.isTemp() || b.op.regClass().type() != RegType::vgpr)
390 assert((post_ra || b.op.hasRegClass()) && b.op.regClass().type() == RegType::vgpr);
392 if (!carry_in.op.isUndefined())
393 return vop2(aco_opcode::v_addc_co_u32, Definition(dst), hint_vcc(def(lm)), a, b, carry_in);
394 else if (program->chip_class >= GFX10 && carry_out)
395 return vop3(aco_opcode::v_add_co_u32_e64, Definition(dst), def(s2), a, b);
396 else if (program->chip_class < GFX9 || carry_out)
397 return vop2(aco_opcode::v_add_co_u32, Definition(dst), hint_vcc(def(lm)), a, b);
399 return vop2(aco_opcode::v_add_u32, Definition(dst), a, b);
402 Result vsub32(Definition dst, Op a, Op b, bool carry_out=false, Op borrow=Op(Operand(s2)))
404 if (!borrow.op.isUndefined() || program->chip_class < GFX9)
407 bool reverse = !b.op.isTemp() || b.op.regClass().type() != RegType::vgpr;
410 assert(b.op.isTemp() && b.op.regClass().type() == RegType::vgpr);
416 if (borrow.op.isUndefined())
417 op = reverse ? aco_opcode::v_subrev_co_u32 : aco_opcode::v_sub_co_u32;
419 op = reverse ? aco_opcode::v_subbrev_co_u32 : aco_opcode::v_subb_co_u32;
421 op = reverse ? aco_opcode::v_subrev_u32 : aco_opcode::v_sub_u32;
424 if (program->chip_class >= GFX10 && op == aco_opcode::v_subrev_co_u32) {
426 op = aco_opcode::v_subrev_co_u32_e64;
427 } else if (program->chip_class >= GFX10 && op == aco_opcode::v_sub_co_u32) {
429 op = aco_opcode::v_sub_co_u32_e64;
432 int num_ops = borrow.op.isUndefined() ? 2 : 3;
433 int num_defs = carry_out ? 2 : 1;
434 aco_ptr<Instruction> sub;
436 sub.reset(create_instruction<VOP3A_instruction>(op, Format::VOP3B, num_ops, num_defs));
438 sub.reset(create_instruction<VOP2_instruction>(op, Format::VOP2, num_ops, num_defs));
439 sub->operands[0] = a.op;
440 sub->operands[1] = b.op;
441 if (!borrow.op.isUndefined())
442 sub->operands[2] = borrow.op;
443 sub->definitions[0] = dst;
445 sub->definitions[1] = Definition(carry);
446 sub->definitions[1].setHint(aco::vcc);
448 return insert(std::move(sub));
451 Result readlane(Definition dst, Op vsrc, Op lane)
453 if (program->chip_class >= GFX8)
454 return vop3(aco_opcode::v_readlane_b32_e64, dst, vsrc, lane);
456 return vop2(aco_opcode::v_readlane_b32, dst, vsrc, lane);
458 Result writelane(Definition dst, Op val, Op lane, Op vsrc) {
459 if (program->chip_class >= GFX8)
460 return vop3(aco_opcode::v_writelane_b32_e64, dst, val, lane, vsrc);
462 return vop2(aco_opcode::v_writelane_b32, dst, val, lane, vsrc);
466 formats = [("pseudo", [Format.PSEUDO], 'Pseudo_instruction', list(itertools.product(range(5), range(5))) + [(8, 1), (1, 8)]),
467 ("sop1", [Format.SOP1], 'SOP1_instruction', [(1, 1), (2, 1), (3, 2)]),
468 ("sop2", [Format.SOP2], 'SOP2_instruction', itertools.product([1, 2], [2, 3])),
469 ("sopk", [Format.SOPK], 'SOPK_instruction', itertools.product([0, 1, 2], [0, 1])),
470 ("sopp", [Format.SOPP], 'SOPP_instruction', [(0, 0), (0, 1)]),
471 ("sopc", [Format.SOPC], 'SOPC_instruction', [(1, 2)]),
472 ("smem", [Format.SMEM], 'SMEM_instruction', [(0, 4), (0, 3), (1, 0), (1, 3), (1, 2), (0, 0)]),
473 ("ds", [Format.DS], 'DS_instruction', [(1, 1), (1, 2), (0, 3), (0, 4)]),
474 ("mubuf", [Format.MUBUF], 'MUBUF_instruction', [(0, 4), (1, 3)]),
475 ("mtbuf", [Format.MTBUF], 'MTBUF_instruction', [(0, 4), (1, 3)]),
476 ("mimg", [Format.MIMG], 'MIMG_instruction', [(0, 3), (1, 3)]),
477 ("exp", [Format.EXP], 'Export_instruction', [(0, 4)]),
478 ("branch", [Format.PSEUDO_BRANCH], 'Pseudo_branch_instruction', itertools.product([0], [0, 1])),
479 ("barrier", [Format.PSEUDO_BARRIER], 'Pseudo_barrier_instruction', [(0, 0)]),
480 ("reduction", [Format.PSEUDO_REDUCTION], 'Pseudo_reduction_instruction', [(3, 2), (3, 4)]),
481 ("vop1", [Format.VOP1], 'VOP1_instruction', [(1, 1), (2, 2)]),
482 ("vop2", [Format.VOP2], 'VOP2_instruction', itertools.product([1, 2], [2, 3])),
483 ("vopc", [Format.VOPC], 'VOPC_instruction', itertools.product([1, 2], [2])),
484 ("vop3", [Format.VOP3A], 'VOP3A_instruction', [(1, 3), (1, 2), (1, 1), (2, 2)]),
485 ("vintrp", [Format.VINTRP], 'Interp_instruction', [(1, 2), (1, 3)]),
486 ("vop1_dpp", [Format.VOP1, Format.DPP], 'DPP_instruction', [(1, 1)]),
487 ("vop2_dpp", [Format.VOP2, Format.DPP], 'DPP_instruction', itertools.product([1, 2], [2, 3])),
488 ("vopc_dpp", [Format.VOPC, Format.DPP], 'DPP_instruction', itertools.product([1, 2], [2])),
489 ("vop1_e64", [Format.VOP1, Format.VOP3A], 'VOP3A_instruction', itertools.product([1], [1])),
490 ("vop2_e64", [Format.VOP2, Format.VOP3A], 'VOP3A_instruction', itertools.product([1, 2], [2, 3])),
491 ("vopc_e64", [Format.VOPC, Format.VOP3A], 'VOP3A_instruction', itertools.product([1, 2], [2])),
492 ("flat", [Format.FLAT], 'FLAT_instruction', [(0, 3), (1, 2)]),
493 ("global", [Format.GLOBAL], 'FLAT_instruction', [(0, 3), (1, 2)])]
495 % for name, formats, struct, shapes in formats:
496 % for num_definitions, num_operands in shapes:
498 args = ['aco_opcode opcode']
499 for i in range(num_definitions):
500 args.append('Definition def%d' % i)
501 for i in range(num_operands):
502 args.append('Op op%d' % i)
504 args += f.get_builder_field_decls()
507 Result ${name}(${', '.join(args)})
509 ${struct} *instr = create_instruction<${struct}>(opcode, (Format)(${'|'.join('(int)Format::%s' % f.name for f in formats)}), ${num_operands}, ${num_definitions});
510 % for i in range(num_definitions):
511 instr->definitions[${i}] = def${i};
513 % for i in range(num_operands):
514 instr->operands[${i}] = op${i}.op;
517 % for dest, field_name in zip(f.get_builder_field_dests(), f.get_builder_field_names()):
518 instr->${dest} = ${field_name};
521 return insert(instr);
524 % if name == 'sop1' or name == 'sop2' or name == 'sopc':
526 args[0] = 'WaveSpecificOpcode opcode'
528 for i in range(num_definitions):
529 params.append('def%d' % i)
530 for i in range(num_operands):
531 params.append('op%d' % i)
534 inline Result ${name}(${', '.join(args)})
536 return ${name}(w64or32(opcode), ${', '.join(params)});
545 #endif /* _ACO_BUILDER_ */"""
547 from aco_opcodes
import opcodes
, Format
548 from mako
.template
import Template
550 print(Template(template
).render(opcodes
=opcodes
, Format
=Format
))