aco: reserve 2 sgprs for each branch
[mesa.git] / src / amd / compiler / aco_assembler.cpp
1 #include <vector>
2 #include <algorithm>
3
4 #include "aco_ir.h"
5 #include "common/sid.h"
6 #include "ac_shader_util.h"
7 #include "util/u_math.h"
8
9 namespace aco {
10
11 struct asm_context {
12 Program *program;
13 enum chip_class chip_class;
14 std::vector<std::pair<int, SOPP_instruction*>> branches;
15 std::vector<unsigned> constaddrs;
16 const int16_t* opcode;
17 // TODO: keep track of branch instructions referring blocks
18 // and, when emitting the block, correct the offset in instr
19 asm_context(Program* program) : program(program), chip_class(program->chip_class) {
20 if (chip_class <= GFX7)
21 opcode = &instr_info.opcode_gfx7[0];
22 else if (chip_class <= GFX9)
23 opcode = &instr_info.opcode_gfx9[0];
24 else if (chip_class >= GFX10)
25 opcode = &instr_info.opcode_gfx10[0];
26 }
27
28 int subvector_begin_pos = -1;
29 };
30
31 static uint32_t get_sdwa_sel(unsigned sel, PhysReg reg)
32 {
33 if (sel & sdwa_isra) {
34 unsigned size = sdwa_rasize & sel;
35 if (size == 1)
36 return reg.byte();
37 else /* size == 2 */
38 return sdwa_isword | (reg.byte() >> 1);
39 }
40 return sel & sdwa_asuint;
41 }
42
43 void emit_instruction(asm_context& ctx, std::vector<uint32_t>& out, Instruction* instr)
44 {
45 /* lower remaining pseudo-instructions */
46 if (instr->opcode == aco_opcode::p_constaddr) {
47 unsigned dest = instr->definitions[0].physReg();
48 unsigned offset = instr->operands[0].constantValue();
49
50 /* s_getpc_b64 dest[0:1] */
51 uint32_t encoding = (0b101111101 << 23);
52 uint32_t opcode = ctx.opcode[(int)aco_opcode::s_getpc_b64];
53 if (opcode >= 55 && ctx.chip_class <= GFX9) {
54 assert(ctx.chip_class == GFX9 && opcode < 60);
55 opcode = opcode - 4;
56 }
57 encoding |= dest << 16;
58 encoding |= opcode << 8;
59 out.push_back(encoding);
60
61 /* s_add_u32 dest[0], dest[0], ... */
62 encoding = (0b10 << 30);
63 encoding |= ctx.opcode[(int)aco_opcode::s_add_u32] << 23;
64 encoding |= dest << 16;
65 encoding |= dest;
66 encoding |= 255 << 8;
67 out.push_back(encoding);
68 ctx.constaddrs.push_back(out.size());
69 out.push_back(offset);
70
71 /* s_addc_u32 dest[1], dest[1], 0 */
72 encoding = (0b10 << 30);
73 encoding |= ctx.opcode[(int)aco_opcode::s_addc_u32] << 23;
74 encoding |= (dest + 1) << 16;
75 encoding |= dest + 1;
76 encoding |= 128 << 8;
77 out.push_back(encoding);
78 return;
79 }
80
81 uint32_t opcode = ctx.opcode[(int)instr->opcode];
82 if (opcode == (uint32_t)-1) {
83 char *out;
84 size_t outsize;
85 FILE *memf = open_memstream(&out, &outsize);
86
87 fprintf(memf, "Unsupported opcode: ");
88 aco_print_instr(instr, memf);
89 fclose(memf);
90
91 aco_err(ctx.program, out);
92 free(out);
93
94 abort();
95 }
96
97 switch (instr->format) {
98 case Format::SOP2: {
99 uint32_t encoding = (0b10 << 30);
100 encoding |= opcode << 23;
101 encoding |= !instr->definitions.empty() ? instr->definitions[0].physReg() << 16 : 0;
102 encoding |= instr->operands.size() >= 2 ? instr->operands[1].physReg() << 8 : 0;
103 encoding |= !instr->operands.empty() ? instr->operands[0].physReg() : 0;
104 out.push_back(encoding);
105 break;
106 }
107 case Format::SOPK: {
108 SOPK_instruction *sopk = static_cast<SOPK_instruction*>(instr);
109
110 if (instr->opcode == aco_opcode::s_subvector_loop_begin) {
111 assert(ctx.chip_class >= GFX10);
112 assert(ctx.subvector_begin_pos == -1);
113 ctx.subvector_begin_pos = out.size();
114 } else if (instr->opcode == aco_opcode::s_subvector_loop_end) {
115 assert(ctx.chip_class >= GFX10);
116 assert(ctx.subvector_begin_pos != -1);
117 /* Adjust s_subvector_loop_begin instruction to the address after the end */
118 out[ctx.subvector_begin_pos] |= (out.size() - ctx.subvector_begin_pos);
119 /* Adjust s_subvector_loop_end instruction to the address after the beginning */
120 sopk->imm = (uint16_t)(ctx.subvector_begin_pos - (int)out.size());
121 ctx.subvector_begin_pos = -1;
122 }
123
124 uint32_t encoding = (0b1011 << 28);
125 encoding |= opcode << 23;
126 encoding |=
127 !instr->definitions.empty() && !(instr->definitions[0].physReg() == scc) ?
128 instr->definitions[0].physReg() << 16 :
129 !instr->operands.empty() && instr->operands[0].physReg() <= 127 ?
130 instr->operands[0].physReg() << 16 : 0;
131 encoding |= sopk->imm;
132 out.push_back(encoding);
133 break;
134 }
135 case Format::SOP1: {
136 uint32_t encoding = (0b101111101 << 23);
137 if (opcode >= 55 && ctx.chip_class <= GFX9) {
138 assert(ctx.chip_class == GFX9 && opcode < 60);
139 opcode = opcode - 4;
140 }
141 encoding |= !instr->definitions.empty() ? instr->definitions[0].physReg() << 16 : 0;
142 encoding |= opcode << 8;
143 encoding |= !instr->operands.empty() ? instr->operands[0].physReg() : 0;
144 out.push_back(encoding);
145 break;
146 }
147 case Format::SOPC: {
148 uint32_t encoding = (0b101111110 << 23);
149 encoding |= opcode << 16;
150 encoding |= instr->operands.size() == 2 ? instr->operands[1].physReg() << 8 : 0;
151 encoding |= !instr->operands.empty() ? instr->operands[0].physReg() : 0;
152 out.push_back(encoding);
153 break;
154 }
155 case Format::SOPP: {
156 SOPP_instruction* sopp = static_cast<SOPP_instruction*>(instr);
157 uint32_t encoding = (0b101111111 << 23);
158 encoding |= opcode << 16;
159 encoding |= (uint16_t) sopp->imm;
160 if (sopp->block != -1)
161 ctx.branches.emplace_back(out.size(), sopp);
162 out.push_back(encoding);
163 break;
164 }
165 case Format::SMEM: {
166 SMEM_instruction* smem = static_cast<SMEM_instruction*>(instr);
167 bool soe = instr->operands.size() >= (!instr->definitions.empty() ? 3 : 4);
168 bool is_load = !instr->definitions.empty();
169 uint32_t encoding = 0;
170
171 if (ctx.chip_class <= GFX7) {
172 encoding = (0b11000 << 27);
173 encoding |= opcode << 22;
174 encoding |= instr->definitions.size() ? instr->definitions[0].physReg() << 15 : 0;
175 encoding |= instr->operands.size() ? (instr->operands[0].physReg() >> 1) << 9 : 0;
176 if (instr->operands.size() >= 2) {
177 if (!instr->operands[1].isConstant() || instr->operands[1].constantValue() >= 1024) {
178 encoding |= instr->operands[1].physReg().reg();
179 } else {
180 encoding |= instr->operands[1].constantValue() >> 2;
181 encoding |= 1 << 8;
182 }
183 }
184 out.push_back(encoding);
185 /* SMRD instructions can take a literal on GFX6 & GFX7 */
186 if (instr->operands.size() >= 2 && instr->operands[1].isConstant() && instr->operands[1].constantValue() >= 1024)
187 out.push_back(instr->operands[1].constantValue() >> 2);
188 return;
189 }
190
191 if (ctx.chip_class <= GFX9) {
192 encoding = (0b110000 << 26);
193 assert(!smem->dlc); /* Device-level coherent is not supported on GFX9 and lower */
194 encoding |= smem->nv ? 1 << 15 : 0;
195 } else {
196 encoding = (0b111101 << 26);
197 assert(!smem->nv); /* Non-volatile is not supported on GFX10 */
198 encoding |= smem->dlc ? 1 << 14 : 0;
199 }
200
201 encoding |= opcode << 18;
202 encoding |= smem->glc ? 1 << 16 : 0;
203
204 if (ctx.chip_class <= GFX9) {
205 if (instr->operands.size() >= 2)
206 encoding |= instr->operands[1].isConstant() ? 1 << 17 : 0; /* IMM - immediate enable */
207 }
208 if (ctx.chip_class == GFX9) {
209 encoding |= soe ? 1 << 14 : 0;
210 }
211
212 if (is_load || instr->operands.size() >= 3) { /* SDATA */
213 encoding |= (is_load ? instr->definitions[0].physReg() : instr->operands[2].physReg()) << 6;
214 }
215 if (instr->operands.size() >= 1) { /* SBASE */
216 encoding |= instr->operands[0].physReg() >> 1;
217 }
218
219 out.push_back(encoding);
220 encoding = 0;
221
222 int32_t offset = 0;
223 uint32_t soffset = ctx.chip_class >= GFX10
224 ? sgpr_null /* On GFX10 this is disabled by specifying SGPR_NULL */
225 : 0; /* On GFX9, it is disabled by the SOE bit (and it's not present on GFX8 and below) */
226 if (instr->operands.size() >= 2) {
227 const Operand &op_off1 = instr->operands[1];
228 if (ctx.chip_class <= GFX9) {
229 offset = op_off1.isConstant() ? op_off1.constantValue() : op_off1.physReg();
230 } else {
231 /* GFX10 only supports constants in OFFSET, so put the operand in SOFFSET if it's an SGPR */
232 if (op_off1.isConstant()) {
233 offset = op_off1.constantValue();
234 } else {
235 soffset = op_off1.physReg();
236 assert(!soe); /* There is no place to put the other SGPR offset, if any */
237 }
238 }
239
240 if (soe) {
241 const Operand &op_off2 = instr->operands.back();
242 assert(ctx.chip_class >= GFX9); /* GFX8 and below don't support specifying a constant and an SGPR at the same time */
243 assert(!op_off2.isConstant());
244 soffset = op_off2.physReg();
245 }
246 }
247 encoding |= offset;
248 encoding |= soffset << 25;
249
250 out.push_back(encoding);
251 return;
252 }
253 case Format::VOP2: {
254 uint32_t encoding = 0;
255 encoding |= opcode << 25;
256 encoding |= (0xFF & instr->definitions[0].physReg()) << 17;
257 encoding |= (0xFF & instr->operands[1].physReg()) << 9;
258 encoding |= instr->operands[0].physReg();
259 out.push_back(encoding);
260 break;
261 }
262 case Format::VOP1: {
263 uint32_t encoding = (0b0111111 << 25);
264 if (!instr->definitions.empty())
265 encoding |= (0xFF & instr->definitions[0].physReg()) << 17;
266 encoding |= opcode << 9;
267 if (!instr->operands.empty())
268 encoding |= instr->operands[0].physReg();
269 out.push_back(encoding);
270 break;
271 }
272 case Format::VOPC: {
273 uint32_t encoding = (0b0111110 << 25);
274 encoding |= opcode << 17;
275 encoding |= (0xFF & instr->operands[1].physReg()) << 9;
276 encoding |= instr->operands[0].physReg();
277 out.push_back(encoding);
278 break;
279 }
280 case Format::VINTRP: {
281 Interp_instruction* interp = static_cast<Interp_instruction*>(instr);
282 uint32_t encoding = 0;
283
284 if (instr->opcode == aco_opcode::v_interp_p1ll_f16 ||
285 instr->opcode == aco_opcode::v_interp_p1lv_f16 ||
286 instr->opcode == aco_opcode::v_interp_p2_legacy_f16 ||
287 instr->opcode == aco_opcode::v_interp_p2_f16) {
288 if (ctx.chip_class == GFX8 || ctx.chip_class == GFX9) {
289 encoding = (0b110100 << 26);
290 } else if (ctx.chip_class >= GFX10) {
291 encoding = (0b110101 << 26);
292 } else {
293 unreachable("Unknown chip_class.");
294 }
295
296 encoding |= opcode << 16;
297 encoding |= (0xFF & instr->definitions[0].physReg());
298 out.push_back(encoding);
299
300 encoding = 0;
301 encoding |= interp->attribute;
302 encoding |= interp->component << 6;
303 encoding |= instr->operands[0].physReg() << 9;
304 if (instr->opcode == aco_opcode::v_interp_p2_f16 ||
305 instr->opcode == aco_opcode::v_interp_p2_legacy_f16 ||
306 instr->opcode == aco_opcode::v_interp_p1lv_f16) {
307 encoding |= instr->operands[2].physReg() << 18;
308 }
309 out.push_back(encoding);
310 } else {
311 if (ctx.chip_class == GFX8 || ctx.chip_class == GFX9) {
312 encoding = (0b110101 << 26); /* Vega ISA doc says 110010 but it's wrong */
313 } else {
314 encoding = (0b110010 << 26);
315 }
316
317 assert(encoding);
318 encoding |= (0xFF & instr->definitions[0].physReg()) << 18;
319 encoding |= opcode << 16;
320 encoding |= interp->attribute << 10;
321 encoding |= interp->component << 8;
322 if (instr->opcode == aco_opcode::v_interp_mov_f32)
323 encoding |= (0x3 & instr->operands[0].constantValue());
324 else
325 encoding |= (0xFF & instr->operands[0].physReg());
326 out.push_back(encoding);
327 }
328 break;
329 }
330 case Format::DS: {
331 DS_instruction* ds = static_cast<DS_instruction*>(instr);
332 uint32_t encoding = (0b110110 << 26);
333 if (ctx.chip_class == GFX8 || ctx.chip_class == GFX9) {
334 encoding |= opcode << 17;
335 encoding |= (ds->gds ? 1 : 0) << 16;
336 } else {
337 encoding |= opcode << 18;
338 encoding |= (ds->gds ? 1 : 0) << 17;
339 }
340 encoding |= ((0xFF & ds->offset1) << 8);
341 encoding |= (0xFFFF & ds->offset0);
342 out.push_back(encoding);
343 encoding = 0;
344 unsigned reg = !instr->definitions.empty() ? instr->definitions[0].physReg() : 0;
345 encoding |= (0xFF & reg) << 24;
346 reg = instr->operands.size() >= 3 && !(instr->operands[2].physReg() == m0) ? instr->operands[2].physReg() : 0;
347 encoding |= (0xFF & reg) << 16;
348 reg = instr->operands.size() >= 2 && !(instr->operands[1].physReg() == m0) ? instr->operands[1].physReg() : 0;
349 encoding |= (0xFF & reg) << 8;
350 encoding |= (0xFF & instr->operands[0].physReg());
351 out.push_back(encoding);
352 break;
353 }
354 case Format::MUBUF: {
355 MUBUF_instruction* mubuf = static_cast<MUBUF_instruction*>(instr);
356 uint32_t encoding = (0b111000 << 26);
357 encoding |= opcode << 18;
358 encoding |= (mubuf->lds ? 1 : 0) << 16;
359 encoding |= (mubuf->glc ? 1 : 0) << 14;
360 encoding |= (mubuf->idxen ? 1 : 0) << 13;
361 assert(!mubuf->addr64 || ctx.chip_class <= GFX7);
362 if (ctx.chip_class == GFX6 || ctx.chip_class == GFX7)
363 encoding |= (mubuf->addr64 ? 1 : 0) << 15;
364 encoding |= (mubuf->offen ? 1 : 0) << 12;
365 if (ctx.chip_class == GFX8 || ctx.chip_class == GFX9) {
366 assert(!mubuf->dlc); /* Device-level coherent is not supported on GFX9 and lower */
367 encoding |= (mubuf->slc ? 1 : 0) << 17;
368 } else if (ctx.chip_class >= GFX10) {
369 encoding |= (mubuf->dlc ? 1 : 0) << 15;
370 }
371 encoding |= 0x0FFF & mubuf->offset;
372 out.push_back(encoding);
373 encoding = 0;
374 if (ctx.chip_class <= GFX7 || ctx.chip_class >= GFX10) {
375 encoding |= (mubuf->slc ? 1 : 0) << 22;
376 }
377 encoding |= instr->operands[2].physReg() << 24;
378 encoding |= (mubuf->tfe ? 1 : 0) << 23;
379 encoding |= (instr->operands[0].physReg() >> 2) << 16;
380 unsigned reg = instr->operands.size() > 3 ? instr->operands[3].physReg() : instr->definitions[0].physReg();
381 encoding |= (0xFF & reg) << 8;
382 encoding |= (0xFF & instr->operands[1].physReg());
383 out.push_back(encoding);
384 break;
385 }
386 case Format::MTBUF: {
387 MTBUF_instruction* mtbuf = static_cast<MTBUF_instruction*>(instr);
388
389 uint32_t img_format = ac_get_tbuffer_format(ctx.chip_class, mtbuf->dfmt, mtbuf->nfmt);
390 uint32_t encoding = (0b111010 << 26);
391 assert(img_format <= 0x7F);
392 assert(!mtbuf->dlc || ctx.chip_class >= GFX10);
393 encoding |= (mtbuf->dlc ? 1 : 0) << 15; /* DLC bit replaces one bit of the OPCODE on GFX10 */
394 encoding |= (mtbuf->glc ? 1 : 0) << 14;
395 encoding |= (mtbuf->idxen ? 1 : 0) << 13;
396 encoding |= (mtbuf->offen ? 1 : 0) << 12;
397 encoding |= 0x0FFF & mtbuf->offset;
398 encoding |= (img_format << 19); /* Handles both the GFX10 FORMAT and the old NFMT+DFMT */
399
400 if (ctx.chip_class == GFX8 || ctx.chip_class == GFX9) {
401 encoding |= opcode << 15;
402 } else {
403 encoding |= (opcode & 0x07) << 16; /* 3 LSBs of 4-bit OPCODE */
404 }
405
406 out.push_back(encoding);
407 encoding = 0;
408
409 encoding |= instr->operands[2].physReg() << 24;
410 encoding |= (mtbuf->tfe ? 1 : 0) << 23;
411 encoding |= (mtbuf->slc ? 1 : 0) << 22;
412 encoding |= (instr->operands[0].physReg() >> 2) << 16;
413 unsigned reg = instr->operands.size() > 3 ? instr->operands[3].physReg() : instr->definitions[0].physReg();
414 encoding |= (0xFF & reg) << 8;
415 encoding |= (0xFF & instr->operands[1].physReg());
416
417 if (ctx.chip_class >= GFX10) {
418 encoding |= (((opcode & 0x08) >> 3) << 21); /* MSB of 4-bit OPCODE */
419 }
420
421 out.push_back(encoding);
422 break;
423 }
424 case Format::MIMG: {
425 MIMG_instruction* mimg = static_cast<MIMG_instruction*>(instr);
426 uint32_t encoding = (0b111100 << 26);
427 encoding |= mimg->slc ? 1 << 25 : 0;
428 encoding |= opcode << 18;
429 encoding |= mimg->lwe ? 1 << 17 : 0;
430 encoding |= mimg->tfe ? 1 << 16 : 0;
431 encoding |= mimg->glc ? 1 << 13 : 0;
432 encoding |= mimg->unrm ? 1 << 12 : 0;
433 if (ctx.chip_class <= GFX9) {
434 assert(!mimg->dlc); /* Device-level coherent is not supported on GFX9 and lower */
435 assert(!mimg->r128);
436 encoding |= mimg->a16 ? 1 << 15 : 0;
437 encoding |= mimg->da ? 1 << 14 : 0;
438 } else {
439 encoding |= mimg->r128 ? 1 << 15 : 0; /* GFX10: A16 moved to 2nd word, R128 replaces it in 1st word */
440 encoding |= mimg->dim << 3; /* GFX10: dimensionality instead of declare array */
441 encoding |= mimg->dlc ? 1 << 7 : 0;
442 }
443 encoding |= (0xF & mimg->dmask) << 8;
444 out.push_back(encoding);
445 encoding = (0xFF & instr->operands[2].physReg()); /* VADDR */
446 if (!instr->definitions.empty()) {
447 encoding |= (0xFF & instr->definitions[0].physReg()) << 8; /* VDATA */
448 } else if (instr->operands[1].regClass().type() == RegType::vgpr) {
449 encoding |= (0xFF & instr->operands[1].physReg()) << 8; /* VDATA */
450 }
451 encoding |= (0x1F & (instr->operands[0].physReg() >> 2)) << 16; /* T# (resource) */
452 if (instr->operands[1].regClass().type() == RegType::sgpr)
453 encoding |= (0x1F & (instr->operands[1].physReg() >> 2)) << 21; /* sampler */
454
455 assert(!mimg->d16 || ctx.chip_class >= GFX9);
456 encoding |= mimg->d16 ? 1 << 15 : 0;
457 if (ctx.chip_class >= GFX10) {
458 encoding |= mimg->a16 ? 1 << 14 : 0; /* GFX10: A16 still exists, but is in a different place */
459 }
460
461 out.push_back(encoding);
462 break;
463 }
464 case Format::FLAT:
465 case Format::SCRATCH:
466 case Format::GLOBAL: {
467 FLAT_instruction *flat = static_cast<FLAT_instruction*>(instr);
468 uint32_t encoding = (0b110111 << 26);
469 encoding |= opcode << 18;
470 if (ctx.chip_class <= GFX9) {
471 assert(flat->offset <= 0x1fff);
472 encoding |= flat->offset & 0x1fff;
473 } else if (instr->format == Format::FLAT) {
474 /* GFX10 has a 12-bit immediate OFFSET field,
475 * but it has a hw bug: it ignores the offset, called FlatSegmentOffsetBug
476 */
477 assert(flat->offset == 0);
478 } else {
479 assert(flat->offset <= 0xfff);
480 encoding |= flat->offset & 0xfff;
481 }
482 if (instr->format == Format::SCRATCH)
483 encoding |= 1 << 14;
484 else if (instr->format == Format::GLOBAL)
485 encoding |= 2 << 14;
486 encoding |= flat->lds ? 1 << 13 : 0;
487 encoding |= flat->glc ? 1 << 16 : 0;
488 encoding |= flat->slc ? 1 << 17 : 0;
489 if (ctx.chip_class >= GFX10) {
490 assert(!flat->nv);
491 encoding |= flat->dlc ? 1 << 12 : 0;
492 } else {
493 assert(!flat->dlc);
494 }
495 out.push_back(encoding);
496 encoding = (0xFF & instr->operands[0].physReg());
497 if (!instr->definitions.empty())
498 encoding |= (0xFF & instr->definitions[0].physReg()) << 24;
499 if (instr->operands.size() >= 3)
500 encoding |= (0xFF & instr->operands[2].physReg()) << 8;
501 if (!instr->operands[1].isUndefined()) {
502 assert(ctx.chip_class >= GFX10 || instr->operands[1].physReg() != 0x7F);
503 assert(instr->format != Format::FLAT);
504 encoding |= instr->operands[1].physReg() << 16;
505 } else if (instr->format != Format::FLAT || ctx.chip_class >= GFX10) { /* SADDR is actually used with FLAT on GFX10 */
506 if (ctx.chip_class <= GFX9)
507 encoding |= 0x7F << 16;
508 else
509 encoding |= sgpr_null << 16;
510 }
511 encoding |= flat->nv ? 1 << 23 : 0;
512 out.push_back(encoding);
513 break;
514 }
515 case Format::EXP: {
516 Export_instruction* exp = static_cast<Export_instruction*>(instr);
517 uint32_t encoding;
518 if (ctx.chip_class == GFX8 || ctx.chip_class == GFX9) {
519 encoding = (0b110001 << 26);
520 } else {
521 encoding = (0b111110 << 26);
522 }
523
524 encoding |= exp->valid_mask ? 0b1 << 12 : 0;
525 encoding |= exp->done ? 0b1 << 11 : 0;
526 encoding |= exp->compressed ? 0b1 << 10 : 0;
527 encoding |= exp->dest << 4;
528 encoding |= exp->enabled_mask;
529 out.push_back(encoding);
530 encoding = 0xFF & exp->operands[0].physReg();
531 encoding |= (0xFF & exp->operands[1].physReg()) << 8;
532 encoding |= (0xFF & exp->operands[2].physReg()) << 16;
533 encoding |= (0xFF & exp->operands[3].physReg()) << 24;
534 out.push_back(encoding);
535 break;
536 }
537 case Format::PSEUDO:
538 case Format::PSEUDO_BARRIER:
539 if (instr->opcode != aco_opcode::p_unit_test)
540 unreachable("Pseudo instructions should be lowered before assembly.");
541 break;
542 default:
543 if ((uint16_t) instr->format & (uint16_t) Format::VOP3A) {
544 VOP3A_instruction* vop3 = static_cast<VOP3A_instruction*>(instr);
545
546 if ((uint16_t) instr->format & (uint16_t) Format::VOP2) {
547 opcode = opcode + 0x100;
548 } else if ((uint16_t) instr->format & (uint16_t) Format::VOP1) {
549 if (ctx.chip_class == GFX8 || ctx.chip_class == GFX9)
550 opcode = opcode + 0x140;
551 else
552 opcode = opcode + 0x180;
553 } else if ((uint16_t) instr->format & (uint16_t) Format::VOPC) {
554 opcode = opcode + 0x0;
555 } else if ((uint16_t) instr->format & (uint16_t) Format::VINTRP) {
556 opcode = opcode + 0x270;
557 }
558
559 uint32_t encoding;
560 if (ctx.chip_class <= GFX9) {
561 encoding = (0b110100 << 26);
562 } else if (ctx.chip_class >= GFX10) {
563 encoding = (0b110101 << 26);
564 } else {
565 unreachable("Unknown chip_class.");
566 }
567
568 if (ctx.chip_class <= GFX7) {
569 encoding |= opcode << 17;
570 encoding |= (vop3->clamp ? 1 : 0) << 11;
571 } else {
572 encoding |= opcode << 16;
573 encoding |= (vop3->clamp ? 1 : 0) << 15;
574 }
575 encoding |= vop3->opsel << 11;
576 for (unsigned i = 0; i < 3; i++)
577 encoding |= vop3->abs[i] << (8+i);
578 if (instr->definitions.size() == 2)
579 encoding |= instr->definitions[1].physReg() << 8;
580 encoding |= (0xFF & instr->definitions[0].physReg());
581 out.push_back(encoding);
582 encoding = 0;
583 if (instr->opcode == aco_opcode::v_interp_mov_f32) {
584 encoding = 0x3 & instr->operands[0].constantValue();
585 } else {
586 for (unsigned i = 0; i < instr->operands.size(); i++)
587 encoding |= instr->operands[i].physReg() << (i * 9);
588 }
589 encoding |= vop3->omod << 27;
590 for (unsigned i = 0; i < 3; i++)
591 encoding |= vop3->neg[i] << (29+i);
592 out.push_back(encoding);
593
594 } else if (instr->format == Format::VOP3P) {
595 VOP3P_instruction* vop3 = static_cast<VOP3P_instruction*>(instr);
596
597 uint32_t encoding;
598 if (ctx.chip_class == GFX9) {
599 encoding = (0b110100111 << 23);
600 } else if (ctx.chip_class >= GFX10) {
601 encoding = (0b110011 << 26);
602 } else {
603 unreachable("Unknown chip_class.");
604 }
605
606 encoding |= opcode << 16;
607 encoding |= (vop3->clamp ? 1 : 0) << 15;
608 encoding |= vop3->opsel_lo << 11;
609 encoding |= (vop3->opsel_hi & 0x4) ? 1 : 0 << 14;
610 for (unsigned i = 0; i < 3; i++)
611 encoding |= vop3->neg_hi[i] << (8+i);
612 encoding |= (0xFF & instr->definitions[0].physReg());
613 out.push_back(encoding);
614 encoding = 0;
615 for (unsigned i = 0; i < instr->operands.size(); i++)
616 encoding |= instr->operands[i].physReg() << (i * 9);
617 encoding |= vop3->opsel_hi & 0x3 << 27;
618 for (unsigned i = 0; i < 3; i++)
619 encoding |= vop3->neg_lo[i] << (29+i);
620 out.push_back(encoding);
621
622 } else if (instr->isDPP()){
623 assert(ctx.chip_class >= GFX8);
624 /* first emit the instruction without the DPP operand */
625 Operand dpp_op = instr->operands[0];
626 instr->operands[0] = Operand(PhysReg{250}, v1);
627 instr->format = (Format) ((uint16_t) instr->format & ~(uint16_t)Format::DPP);
628 emit_instruction(ctx, out, instr);
629 DPP_instruction* dpp = static_cast<DPP_instruction*>(instr);
630 uint32_t encoding = (0xF & dpp->row_mask) << 28;
631 encoding |= (0xF & dpp->bank_mask) << 24;
632 encoding |= dpp->abs[1] << 23;
633 encoding |= dpp->neg[1] << 22;
634 encoding |= dpp->abs[0] << 21;
635 encoding |= dpp->neg[0] << 20;
636 if (ctx.chip_class >= GFX10)
637 encoding |= 1 << 18; /* set Fetch Inactive to match GFX9 behaviour */
638 encoding |= dpp->bound_ctrl << 19;
639 encoding |= dpp->dpp_ctrl << 8;
640 encoding |= (0xFF) & dpp_op.physReg();
641 out.push_back(encoding);
642 return;
643 } else if (instr->isSDWA()) {
644 /* first emit the instruction without the SDWA operand */
645 Operand sdwa_op = instr->operands[0];
646 instr->operands[0] = Operand(PhysReg{249}, v1);
647 instr->format = (Format) ((uint16_t) instr->format & ~(uint16_t)Format::SDWA);
648 emit_instruction(ctx, out, instr);
649
650 SDWA_instruction* sdwa = static_cast<SDWA_instruction*>(instr);
651 uint32_t encoding = 0;
652
653 if ((uint16_t)instr->format & (uint16_t)Format::VOPC) {
654 if (instr->definitions[0].physReg() != vcc) {
655 encoding |= instr->definitions[0].physReg() << 8;
656 encoding |= 1 << 15;
657 }
658 encoding |= (sdwa->clamp ? 1 : 0) << 13;
659 } else {
660 encoding |= get_sdwa_sel(sdwa->dst_sel, instr->definitions[0].physReg()) << 8;
661 uint32_t dst_u = sdwa->dst_sel & sdwa_sext ? 1 : 0;
662 if (sdwa->dst_preserve || (sdwa->dst_sel & sdwa_isra))
663 dst_u = 2;
664 encoding |= dst_u << 11;
665 encoding |= (sdwa->clamp ? 1 : 0) << 13;
666 encoding |= sdwa->omod << 14;
667 }
668
669 encoding |= get_sdwa_sel(sdwa->sel[0], sdwa_op.physReg()) << 16;
670 encoding |= sdwa->sel[0] & sdwa_sext ? 1 << 19 : 0;
671 encoding |= sdwa->abs[0] << 21;
672 encoding |= sdwa->neg[0] << 20;
673
674 if (instr->operands.size() >= 2) {
675 encoding |= get_sdwa_sel(sdwa->sel[1], instr->operands[1].physReg()) << 24;
676 encoding |= sdwa->sel[1] & sdwa_sext ? 1 << 27 : 0;
677 encoding |= sdwa->abs[1] << 29;
678 encoding |= sdwa->neg[1] << 28;
679 }
680
681 encoding |= 0xFF & sdwa_op.physReg();
682 encoding |= (sdwa_op.physReg() < 256) << 23;
683 if (instr->operands.size() >= 2)
684 encoding |= (instr->operands[1].physReg() < 256) << 31;
685 out.push_back(encoding);
686 } else {
687 unreachable("unimplemented instruction format");
688 }
689 break;
690 }
691
692 /* append literal dword */
693 for (const Operand& op : instr->operands) {
694 if (op.isLiteral()) {
695 out.push_back(op.constantValue());
696 break;
697 }
698 }
699 }
700
701 void emit_block(asm_context& ctx, std::vector<uint32_t>& out, Block& block)
702 {
703 for (aco_ptr<Instruction>& instr : block.instructions) {
704 #if 0
705 int start_idx = out.size();
706 std::cerr << "Encoding:\t" << std::endl;
707 aco_print_instr(&*instr, stderr);
708 std::cerr << std::endl;
709 #endif
710 emit_instruction(ctx, out, instr.get());
711 #if 0
712 for (int i = start_idx; i < out.size(); i++)
713 std::cerr << "encoding: " << "0x" << std::setfill('0') << std::setw(8) << std::hex << out[i] << std::endl;
714 #endif
715 }
716 }
717
718 void fix_exports(asm_context& ctx, std::vector<uint32_t>& out, Program* program)
719 {
720 bool exported = false;
721 for (Block& block : program->blocks) {
722 if (!(block.kind & block_kind_export_end))
723 continue;
724 std::vector<aco_ptr<Instruction>>::reverse_iterator it = block.instructions.rbegin();
725 while ( it != block.instructions.rend())
726 {
727 if ((*it)->format == Format::EXP) {
728 Export_instruction* exp = static_cast<Export_instruction*>((*it).get());
729 if (program->stage & (hw_vs | hw_ngg_gs)) {
730 if (exp->dest >= V_008DFC_SQ_EXP_POS && exp->dest <= (V_008DFC_SQ_EXP_POS + 3)) {
731 exp->done = true;
732 exported = true;
733 break;
734 }
735 } else {
736 exp->done = true;
737 exp->valid_mask = true;
738 exported = true;
739 break;
740 }
741 } else if ((*it)->definitions.size() && (*it)->definitions[0].physReg() == exec)
742 break;
743 ++it;
744 }
745 }
746
747 if (!exported) {
748 /* Abort in order to avoid a GPU hang. */
749 aco_err(program, "Missing export in %s shader:", (program->stage & hw_vs) ? "vertex" : "fragment");
750 aco_print_program(program, stderr);
751 abort();
752 }
753 }
754
755 static void fix_branches_gfx10(asm_context& ctx, std::vector<uint32_t>& out)
756 {
757 /* Branches with an offset of 0x3f are buggy on GFX10, we workaround by inserting NOPs if needed. */
758 bool gfx10_3f_bug = false;
759
760 do {
761 auto buggy_branch_it = std::find_if(ctx.branches.begin(), ctx.branches.end(), [&ctx](const auto &branch) -> bool {
762 return ((int)ctx.program->blocks[branch.second->block].offset - branch.first - 1) == 0x3f;
763 });
764
765 gfx10_3f_bug = buggy_branch_it != ctx.branches.end();
766
767 if (gfx10_3f_bug) {
768 /* Insert an s_nop after the branch */
769 constexpr uint32_t s_nop_0 = 0xbf800000u;
770 int s_nop_pos = buggy_branch_it->first + 1;
771 auto out_pos = std::next(out.begin(), s_nop_pos);
772 out.insert(out_pos, s_nop_0);
773
774 /* Update the offset of each affected block */
775 for (Block& block : ctx.program->blocks) {
776 if (block.offset > (unsigned)buggy_branch_it->first)
777 block.offset++;
778 }
779
780 /* Update the branches following the current one */
781 for (auto branch_it = std::next(buggy_branch_it); branch_it != ctx.branches.end(); ++branch_it)
782 branch_it->first++;
783
784 /* Find first constant address after the inserted instruction */
785 auto caddr_it = std::find_if(ctx.constaddrs.begin(), ctx.constaddrs.end(), [s_nop_pos](const int &caddr_pos) -> bool {
786 return caddr_pos >= s_nop_pos;
787 });
788
789 /* Update the locations of constant addresses */
790 for (; caddr_it != ctx.constaddrs.end(); ++caddr_it)
791 (*caddr_it)++;
792
793 }
794 } while (gfx10_3f_bug);
795 }
796
797 void fix_branches(asm_context& ctx, std::vector<uint32_t>& out)
798 {
799 if (ctx.chip_class == GFX10)
800 fix_branches_gfx10(ctx, out);
801
802 for (std::pair<int, SOPP_instruction*> &branch : ctx.branches) {
803 int offset = (int)ctx.program->blocks[branch.second->block].offset - branch.first - 1;
804 out[branch.first] |= (uint16_t) offset;
805 }
806 }
807
808 void fix_constaddrs(asm_context& ctx, std::vector<uint32_t>& out)
809 {
810 for (unsigned addr : ctx.constaddrs)
811 out[addr] += (out.size() - addr + 1u) * 4u;
812 }
813
814 unsigned emit_program(Program* program,
815 std::vector<uint32_t>& code)
816 {
817 asm_context ctx(program);
818
819 if (program->stage & (hw_vs | hw_fs | hw_ngg_gs))
820 fix_exports(ctx, code, program);
821
822 for (Block& block : program->blocks) {
823 block.offset = code.size();
824 emit_block(ctx, code, block);
825 }
826
827 fix_branches(ctx, code);
828
829 unsigned exec_size = code.size() * sizeof(uint32_t);
830
831 if (program->chip_class >= GFX10) {
832 /* Pad output with s_code_end so instruction prefetching doesn't cause
833 * page faults */
834 unsigned final_size = align(code.size() + 3 * 16, 16);
835 while (code.size() < final_size)
836 code.push_back(0xbf9f0000u);
837 }
838
839 fix_constaddrs(ctx, code);
840
841 while (program->constant_data.size() % 4u)
842 program->constant_data.push_back(0);
843 /* Copy constant data */
844 code.insert(code.end(), (uint32_t*)program->constant_data.data(),
845 (uint32_t*)(program->constant_data.data() + program->constant_data.size()));
846
847 return exec_size;
848 }
849
850 }