5 #include "aco_builder.h"
6 #include "common/sid.h"
7 #include "ac_shader_util.h"
8 #include "util/u_math.h"
14 enum chip_class chip_class
;
15 std::vector
<std::pair
<int, SOPP_instruction
*>> branches
;
16 std::vector
<unsigned> constaddrs
;
17 const int16_t* opcode
;
18 // TODO: keep track of branch instructions referring blocks
19 // and, when emitting the block, correct the offset in instr
20 asm_context(Program
* program
) : program(program
), chip_class(program
->chip_class
) {
21 if (chip_class
<= GFX7
)
22 opcode
= &instr_info
.opcode_gfx7
[0];
23 else if (chip_class
<= GFX9
)
24 opcode
= &instr_info
.opcode_gfx9
[0];
25 else if (chip_class
>= GFX10
)
26 opcode
= &instr_info
.opcode_gfx10
[0];
29 int subvector_begin_pos
= -1;
32 static uint32_t get_sdwa_sel(unsigned sel
, PhysReg reg
)
34 if (sel
& sdwa_isra
) {
35 unsigned size
= sdwa_rasize
& sel
;
39 return sdwa_isword
| (reg
.byte() >> 1);
41 return sel
& sdwa_asuint
;
44 void emit_instruction(asm_context
& ctx
, std::vector
<uint32_t>& out
, Instruction
* instr
)
46 /* lower remaining pseudo-instructions */
47 if (instr
->opcode
== aco_opcode::p_constaddr
) {
48 unsigned dest
= instr
->definitions
[0].physReg();
49 unsigned offset
= instr
->operands
[0].constantValue();
51 /* s_getpc_b64 dest[0:1] */
52 uint32_t encoding
= (0b101111101 << 23);
53 uint32_t opcode
= ctx
.opcode
[(int)aco_opcode::s_getpc_b64
];
54 if (opcode
>= 55 && ctx
.chip_class
<= GFX9
) {
55 assert(ctx
.chip_class
== GFX9
&& opcode
< 60);
58 encoding
|= dest
<< 16;
59 encoding
|= opcode
<< 8;
60 out
.push_back(encoding
);
62 /* s_add_u32 dest[0], dest[0], ... */
63 encoding
= (0b10 << 30);
64 encoding
|= ctx
.opcode
[(int)aco_opcode::s_add_u32
] << 23;
65 encoding
|= dest
<< 16;
68 out
.push_back(encoding
);
69 ctx
.constaddrs
.push_back(out
.size());
70 out
.push_back(offset
);
72 /* s_addc_u32 dest[1], dest[1], 0 */
73 encoding
= (0b10 << 30);
74 encoding
|= ctx
.opcode
[(int)aco_opcode::s_addc_u32
] << 23;
75 encoding
|= (dest
+ 1) << 16;
78 out
.push_back(encoding
);
82 uint32_t opcode
= ctx
.opcode
[(int)instr
->opcode
];
83 if (opcode
== (uint32_t)-1) {
86 FILE *memf
= open_memstream(&out
, &outsize
);
88 fprintf(memf
, "Unsupported opcode: ");
89 aco_print_instr(instr
, memf
);
92 aco_err(ctx
.program
, out
);
98 switch (instr
->format
) {
100 uint32_t encoding
= (0b10 << 30);
101 encoding
|= opcode
<< 23;
102 encoding
|= !instr
->definitions
.empty() ? instr
->definitions
[0].physReg() << 16 : 0;
103 encoding
|= instr
->operands
.size() >= 2 ? instr
->operands
[1].physReg() << 8 : 0;
104 encoding
|= !instr
->operands
.empty() ? instr
->operands
[0].physReg() : 0;
105 out
.push_back(encoding
);
109 SOPK_instruction
*sopk
= static_cast<SOPK_instruction
*>(instr
);
111 if (instr
->opcode
== aco_opcode::s_subvector_loop_begin
) {
112 assert(ctx
.chip_class
>= GFX10
);
113 assert(ctx
.subvector_begin_pos
== -1);
114 ctx
.subvector_begin_pos
= out
.size();
115 } else if (instr
->opcode
== aco_opcode::s_subvector_loop_end
) {
116 assert(ctx
.chip_class
>= GFX10
);
117 assert(ctx
.subvector_begin_pos
!= -1);
118 /* Adjust s_subvector_loop_begin instruction to the address after the end */
119 out
[ctx
.subvector_begin_pos
] |= (out
.size() - ctx
.subvector_begin_pos
);
120 /* Adjust s_subvector_loop_end instruction to the address after the beginning */
121 sopk
->imm
= (uint16_t)(ctx
.subvector_begin_pos
- (int)out
.size());
122 ctx
.subvector_begin_pos
= -1;
125 uint32_t encoding
= (0b1011 << 28);
126 encoding
|= opcode
<< 23;
128 !instr
->definitions
.empty() && !(instr
->definitions
[0].physReg() == scc
) ?
129 instr
->definitions
[0].physReg() << 16 :
130 !instr
->operands
.empty() && instr
->operands
[0].physReg() <= 127 ?
131 instr
->operands
[0].physReg() << 16 : 0;
132 encoding
|= sopk
->imm
;
133 out
.push_back(encoding
);
137 uint32_t encoding
= (0b101111101 << 23);
138 if (opcode
>= 55 && ctx
.chip_class
<= GFX9
) {
139 assert(ctx
.chip_class
== GFX9
&& opcode
< 60);
142 encoding
|= !instr
->definitions
.empty() ? instr
->definitions
[0].physReg() << 16 : 0;
143 encoding
|= opcode
<< 8;
144 encoding
|= !instr
->operands
.empty() ? instr
->operands
[0].physReg() : 0;
145 out
.push_back(encoding
);
149 uint32_t encoding
= (0b101111110 << 23);
150 encoding
|= opcode
<< 16;
151 encoding
|= instr
->operands
.size() == 2 ? instr
->operands
[1].physReg() << 8 : 0;
152 encoding
|= !instr
->operands
.empty() ? instr
->operands
[0].physReg() : 0;
153 out
.push_back(encoding
);
157 SOPP_instruction
* sopp
= static_cast<SOPP_instruction
*>(instr
);
158 uint32_t encoding
= (0b101111111 << 23);
159 encoding
|= opcode
<< 16;
160 encoding
|= (uint16_t) sopp
->imm
;
161 if (sopp
->block
!= -1) {
162 sopp
->pass_flags
= 0;
163 ctx
.branches
.emplace_back(out
.size(), sopp
);
165 out
.push_back(encoding
);
169 SMEM_instruction
* smem
= static_cast<SMEM_instruction
*>(instr
);
170 bool soe
= instr
->operands
.size() >= (!instr
->definitions
.empty() ? 3 : 4);
171 bool is_load
= !instr
->definitions
.empty();
172 uint32_t encoding
= 0;
174 if (ctx
.chip_class
<= GFX7
) {
175 encoding
= (0b11000 << 27);
176 encoding
|= opcode
<< 22;
177 encoding
|= instr
->definitions
.size() ? instr
->definitions
[0].physReg() << 15 : 0;
178 encoding
|= instr
->operands
.size() ? (instr
->operands
[0].physReg() >> 1) << 9 : 0;
179 if (instr
->operands
.size() >= 2) {
180 if (!instr
->operands
[1].isConstant() || instr
->operands
[1].constantValue() >= 1024) {
181 encoding
|= instr
->operands
[1].physReg().reg();
183 encoding
|= instr
->operands
[1].constantValue() >> 2;
187 out
.push_back(encoding
);
188 /* SMRD instructions can take a literal on GFX6 & GFX7 */
189 if (instr
->operands
.size() >= 2 && instr
->operands
[1].isConstant() && instr
->operands
[1].constantValue() >= 1024)
190 out
.push_back(instr
->operands
[1].constantValue() >> 2);
194 if (ctx
.chip_class
<= GFX9
) {
195 encoding
= (0b110000 << 26);
196 assert(!smem
->dlc
); /* Device-level coherent is not supported on GFX9 and lower */
197 encoding
|= smem
->nv
? 1 << 15 : 0;
199 encoding
= (0b111101 << 26);
200 assert(!smem
->nv
); /* Non-volatile is not supported on GFX10 */
201 encoding
|= smem
->dlc
? 1 << 14 : 0;
204 encoding
|= opcode
<< 18;
205 encoding
|= smem
->glc
? 1 << 16 : 0;
207 if (ctx
.chip_class
<= GFX9
) {
208 if (instr
->operands
.size() >= 2)
209 encoding
|= instr
->operands
[1].isConstant() ? 1 << 17 : 0; /* IMM - immediate enable */
211 if (ctx
.chip_class
== GFX9
) {
212 encoding
|= soe
? 1 << 14 : 0;
215 if (is_load
|| instr
->operands
.size() >= 3) { /* SDATA */
216 encoding
|= (is_load
? instr
->definitions
[0].physReg() : instr
->operands
[2].physReg()) << 6;
218 if (instr
->operands
.size() >= 1) { /* SBASE */
219 encoding
|= instr
->operands
[0].physReg() >> 1;
222 out
.push_back(encoding
);
226 uint32_t soffset
= ctx
.chip_class
>= GFX10
227 ? sgpr_null
/* On GFX10 this is disabled by specifying SGPR_NULL */
228 : 0; /* On GFX9, it is disabled by the SOE bit (and it's not present on GFX8 and below) */
229 if (instr
->operands
.size() >= 2) {
230 const Operand
&op_off1
= instr
->operands
[1];
231 if (ctx
.chip_class
<= GFX9
) {
232 offset
= op_off1
.isConstant() ? op_off1
.constantValue() : op_off1
.physReg();
234 /* GFX10 only supports constants in OFFSET, so put the operand in SOFFSET if it's an SGPR */
235 if (op_off1
.isConstant()) {
236 offset
= op_off1
.constantValue();
238 soffset
= op_off1
.physReg();
239 assert(!soe
); /* There is no place to put the other SGPR offset, if any */
244 const Operand
&op_off2
= instr
->operands
.back();
245 assert(ctx
.chip_class
>= GFX9
); /* GFX8 and below don't support specifying a constant and an SGPR at the same time */
246 assert(!op_off2
.isConstant());
247 soffset
= op_off2
.physReg();
251 encoding
|= soffset
<< 25;
253 out
.push_back(encoding
);
257 uint32_t encoding
= 0;
258 encoding
|= opcode
<< 25;
259 encoding
|= (0xFF & instr
->definitions
[0].physReg()) << 17;
260 encoding
|= (0xFF & instr
->operands
[1].physReg()) << 9;
261 encoding
|= instr
->operands
[0].physReg();
262 out
.push_back(encoding
);
266 uint32_t encoding
= (0b0111111 << 25);
267 if (!instr
->definitions
.empty())
268 encoding
|= (0xFF & instr
->definitions
[0].physReg()) << 17;
269 encoding
|= opcode
<< 9;
270 if (!instr
->operands
.empty())
271 encoding
|= instr
->operands
[0].physReg();
272 out
.push_back(encoding
);
276 uint32_t encoding
= (0b0111110 << 25);
277 encoding
|= opcode
<< 17;
278 encoding
|= (0xFF & instr
->operands
[1].physReg()) << 9;
279 encoding
|= instr
->operands
[0].physReg();
280 out
.push_back(encoding
);
283 case Format::VINTRP
: {
284 Interp_instruction
* interp
= static_cast<Interp_instruction
*>(instr
);
285 uint32_t encoding
= 0;
287 if (instr
->opcode
== aco_opcode::v_interp_p1ll_f16
||
288 instr
->opcode
== aco_opcode::v_interp_p1lv_f16
||
289 instr
->opcode
== aco_opcode::v_interp_p2_legacy_f16
||
290 instr
->opcode
== aco_opcode::v_interp_p2_f16
) {
291 if (ctx
.chip_class
== GFX8
|| ctx
.chip_class
== GFX9
) {
292 encoding
= (0b110100 << 26);
293 } else if (ctx
.chip_class
>= GFX10
) {
294 encoding
= (0b110101 << 26);
296 unreachable("Unknown chip_class.");
299 encoding
|= opcode
<< 16;
300 encoding
|= (0xFF & instr
->definitions
[0].physReg());
301 out
.push_back(encoding
);
304 encoding
|= interp
->attribute
;
305 encoding
|= interp
->component
<< 6;
306 encoding
|= instr
->operands
[0].physReg() << 9;
307 if (instr
->opcode
== aco_opcode::v_interp_p2_f16
||
308 instr
->opcode
== aco_opcode::v_interp_p2_legacy_f16
||
309 instr
->opcode
== aco_opcode::v_interp_p1lv_f16
) {
310 encoding
|= instr
->operands
[2].physReg() << 18;
312 out
.push_back(encoding
);
314 if (ctx
.chip_class
== GFX8
|| ctx
.chip_class
== GFX9
) {
315 encoding
= (0b110101 << 26); /* Vega ISA doc says 110010 but it's wrong */
317 encoding
= (0b110010 << 26);
321 encoding
|= (0xFF & instr
->definitions
[0].physReg()) << 18;
322 encoding
|= opcode
<< 16;
323 encoding
|= interp
->attribute
<< 10;
324 encoding
|= interp
->component
<< 8;
325 if (instr
->opcode
== aco_opcode::v_interp_mov_f32
)
326 encoding
|= (0x3 & instr
->operands
[0].constantValue());
328 encoding
|= (0xFF & instr
->operands
[0].physReg());
329 out
.push_back(encoding
);
334 DS_instruction
* ds
= static_cast<DS_instruction
*>(instr
);
335 uint32_t encoding
= (0b110110 << 26);
336 if (ctx
.chip_class
== GFX8
|| ctx
.chip_class
== GFX9
) {
337 encoding
|= opcode
<< 17;
338 encoding
|= (ds
->gds
? 1 : 0) << 16;
340 encoding
|= opcode
<< 18;
341 encoding
|= (ds
->gds
? 1 : 0) << 17;
343 encoding
|= ((0xFF & ds
->offset1
) << 8);
344 encoding
|= (0xFFFF & ds
->offset0
);
345 out
.push_back(encoding
);
347 unsigned reg
= !instr
->definitions
.empty() ? instr
->definitions
[0].physReg() : 0;
348 encoding
|= (0xFF & reg
) << 24;
349 reg
= instr
->operands
.size() >= 3 && !(instr
->operands
[2].physReg() == m0
) ? instr
->operands
[2].physReg() : 0;
350 encoding
|= (0xFF & reg
) << 16;
351 reg
= instr
->operands
.size() >= 2 && !(instr
->operands
[1].physReg() == m0
) ? instr
->operands
[1].physReg() : 0;
352 encoding
|= (0xFF & reg
) << 8;
353 encoding
|= (0xFF & instr
->operands
[0].physReg());
354 out
.push_back(encoding
);
357 case Format::MUBUF
: {
358 MUBUF_instruction
* mubuf
= static_cast<MUBUF_instruction
*>(instr
);
359 uint32_t encoding
= (0b111000 << 26);
360 encoding
|= opcode
<< 18;
361 encoding
|= (mubuf
->lds
? 1 : 0) << 16;
362 encoding
|= (mubuf
->glc
? 1 : 0) << 14;
363 encoding
|= (mubuf
->idxen
? 1 : 0) << 13;
364 assert(!mubuf
->addr64
|| ctx
.chip_class
<= GFX7
);
365 if (ctx
.chip_class
== GFX6
|| ctx
.chip_class
== GFX7
)
366 encoding
|= (mubuf
->addr64
? 1 : 0) << 15;
367 encoding
|= (mubuf
->offen
? 1 : 0) << 12;
368 if (ctx
.chip_class
== GFX8
|| ctx
.chip_class
== GFX9
) {
369 assert(!mubuf
->dlc
); /* Device-level coherent is not supported on GFX9 and lower */
370 encoding
|= (mubuf
->slc
? 1 : 0) << 17;
371 } else if (ctx
.chip_class
>= GFX10
) {
372 encoding
|= (mubuf
->dlc
? 1 : 0) << 15;
374 encoding
|= 0x0FFF & mubuf
->offset
;
375 out
.push_back(encoding
);
377 if (ctx
.chip_class
<= GFX7
|| ctx
.chip_class
>= GFX10
) {
378 encoding
|= (mubuf
->slc
? 1 : 0) << 22;
380 encoding
|= instr
->operands
[2].physReg() << 24;
381 encoding
|= (mubuf
->tfe
? 1 : 0) << 23;
382 encoding
|= (instr
->operands
[0].physReg() >> 2) << 16;
383 unsigned reg
= instr
->operands
.size() > 3 ? instr
->operands
[3].physReg() : instr
->definitions
[0].physReg();
384 encoding
|= (0xFF & reg
) << 8;
385 encoding
|= (0xFF & instr
->operands
[1].physReg());
386 out
.push_back(encoding
);
389 case Format::MTBUF
: {
390 MTBUF_instruction
* mtbuf
= static_cast<MTBUF_instruction
*>(instr
);
392 uint32_t img_format
= ac_get_tbuffer_format(ctx
.chip_class
, mtbuf
->dfmt
, mtbuf
->nfmt
);
393 uint32_t encoding
= (0b111010 << 26);
394 assert(img_format
<= 0x7F);
395 assert(!mtbuf
->dlc
|| ctx
.chip_class
>= GFX10
);
396 encoding
|= (mtbuf
->dlc
? 1 : 0) << 15; /* DLC bit replaces one bit of the OPCODE on GFX10 */
397 encoding
|= (mtbuf
->glc
? 1 : 0) << 14;
398 encoding
|= (mtbuf
->idxen
? 1 : 0) << 13;
399 encoding
|= (mtbuf
->offen
? 1 : 0) << 12;
400 encoding
|= 0x0FFF & mtbuf
->offset
;
401 encoding
|= (img_format
<< 19); /* Handles both the GFX10 FORMAT and the old NFMT+DFMT */
403 if (ctx
.chip_class
== GFX8
|| ctx
.chip_class
== GFX9
) {
404 encoding
|= opcode
<< 15;
406 encoding
|= (opcode
& 0x07) << 16; /* 3 LSBs of 4-bit OPCODE */
409 out
.push_back(encoding
);
412 encoding
|= instr
->operands
[2].physReg() << 24;
413 encoding
|= (mtbuf
->tfe
? 1 : 0) << 23;
414 encoding
|= (mtbuf
->slc
? 1 : 0) << 22;
415 encoding
|= (instr
->operands
[0].physReg() >> 2) << 16;
416 unsigned reg
= instr
->operands
.size() > 3 ? instr
->operands
[3].physReg() : instr
->definitions
[0].physReg();
417 encoding
|= (0xFF & reg
) << 8;
418 encoding
|= (0xFF & instr
->operands
[1].physReg());
420 if (ctx
.chip_class
>= GFX10
) {
421 encoding
|= (((opcode
& 0x08) >> 3) << 21); /* MSB of 4-bit OPCODE */
424 out
.push_back(encoding
);
428 MIMG_instruction
* mimg
= static_cast<MIMG_instruction
*>(instr
);
429 uint32_t encoding
= (0b111100 << 26);
430 encoding
|= mimg
->slc
? 1 << 25 : 0;
431 encoding
|= opcode
<< 18;
432 encoding
|= mimg
->lwe
? 1 << 17 : 0;
433 encoding
|= mimg
->tfe
? 1 << 16 : 0;
434 encoding
|= mimg
->glc
? 1 << 13 : 0;
435 encoding
|= mimg
->unrm
? 1 << 12 : 0;
436 if (ctx
.chip_class
<= GFX9
) {
437 assert(!mimg
->dlc
); /* Device-level coherent is not supported on GFX9 and lower */
439 encoding
|= mimg
->a16
? 1 << 15 : 0;
440 encoding
|= mimg
->da
? 1 << 14 : 0;
442 encoding
|= mimg
->r128
? 1 << 15 : 0; /* GFX10: A16 moved to 2nd word, R128 replaces it in 1st word */
443 encoding
|= mimg
->dim
<< 3; /* GFX10: dimensionality instead of declare array */
444 encoding
|= mimg
->dlc
? 1 << 7 : 0;
446 encoding
|= (0xF & mimg
->dmask
) << 8;
447 out
.push_back(encoding
);
448 encoding
= (0xFF & instr
->operands
[2].physReg()); /* VADDR */
449 if (!instr
->definitions
.empty()) {
450 encoding
|= (0xFF & instr
->definitions
[0].physReg()) << 8; /* VDATA */
451 } else if (instr
->operands
[1].regClass().type() == RegType::vgpr
) {
452 encoding
|= (0xFF & instr
->operands
[1].physReg()) << 8; /* VDATA */
454 encoding
|= (0x1F & (instr
->operands
[0].physReg() >> 2)) << 16; /* T# (resource) */
455 if (instr
->operands
[1].regClass().type() == RegType::sgpr
)
456 encoding
|= (0x1F & (instr
->operands
[1].physReg() >> 2)) << 21; /* sampler */
458 assert(!mimg
->d16
|| ctx
.chip_class
>= GFX9
);
459 encoding
|= mimg
->d16
? 1 << 15 : 0;
460 if (ctx
.chip_class
>= GFX10
) {
461 encoding
|= mimg
->a16
? 1 << 14 : 0; /* GFX10: A16 still exists, but is in a different place */
464 out
.push_back(encoding
);
468 case Format::SCRATCH
:
469 case Format::GLOBAL
: {
470 FLAT_instruction
*flat
= static_cast<FLAT_instruction
*>(instr
);
471 uint32_t encoding
= (0b110111 << 26);
472 encoding
|= opcode
<< 18;
473 if (ctx
.chip_class
<= GFX9
) {
474 assert(flat
->offset
<= 0x1fff);
475 encoding
|= flat
->offset
& 0x1fff;
476 } else if (instr
->format
== Format::FLAT
) {
477 /* GFX10 has a 12-bit immediate OFFSET field,
478 * but it has a hw bug: it ignores the offset, called FlatSegmentOffsetBug
480 assert(flat
->offset
== 0);
482 assert(flat
->offset
<= 0xfff);
483 encoding
|= flat
->offset
& 0xfff;
485 if (instr
->format
== Format::SCRATCH
)
487 else if (instr
->format
== Format::GLOBAL
)
489 encoding
|= flat
->lds
? 1 << 13 : 0;
490 encoding
|= flat
->glc
? 1 << 16 : 0;
491 encoding
|= flat
->slc
? 1 << 17 : 0;
492 if (ctx
.chip_class
>= GFX10
) {
494 encoding
|= flat
->dlc
? 1 << 12 : 0;
498 out
.push_back(encoding
);
499 encoding
= (0xFF & instr
->operands
[0].physReg());
500 if (!instr
->definitions
.empty())
501 encoding
|= (0xFF & instr
->definitions
[0].physReg()) << 24;
502 if (instr
->operands
.size() >= 3)
503 encoding
|= (0xFF & instr
->operands
[2].physReg()) << 8;
504 if (!instr
->operands
[1].isUndefined()) {
505 assert(ctx
.chip_class
>= GFX10
|| instr
->operands
[1].physReg() != 0x7F);
506 assert(instr
->format
!= Format::FLAT
);
507 encoding
|= instr
->operands
[1].physReg() << 16;
508 } else if (instr
->format
!= Format::FLAT
|| ctx
.chip_class
>= GFX10
) { /* SADDR is actually used with FLAT on GFX10 */
509 if (ctx
.chip_class
<= GFX9
)
510 encoding
|= 0x7F << 16;
512 encoding
|= sgpr_null
<< 16;
514 encoding
|= flat
->nv
? 1 << 23 : 0;
515 out
.push_back(encoding
);
519 Export_instruction
* exp
= static_cast<Export_instruction
*>(instr
);
521 if (ctx
.chip_class
== GFX8
|| ctx
.chip_class
== GFX9
) {
522 encoding
= (0b110001 << 26);
524 encoding
= (0b111110 << 26);
527 encoding
|= exp
->valid_mask
? 0b1 << 12 : 0;
528 encoding
|= exp
->done
? 0b1 << 11 : 0;
529 encoding
|= exp
->compressed
? 0b1 << 10 : 0;
530 encoding
|= exp
->dest
<< 4;
531 encoding
|= exp
->enabled_mask
;
532 out
.push_back(encoding
);
533 encoding
= 0xFF & exp
->operands
[0].physReg();
534 encoding
|= (0xFF & exp
->operands
[1].physReg()) << 8;
535 encoding
|= (0xFF & exp
->operands
[2].physReg()) << 16;
536 encoding
|= (0xFF & exp
->operands
[3].physReg()) << 24;
537 out
.push_back(encoding
);
541 case Format::PSEUDO_BARRIER
:
542 if (instr
->opcode
!= aco_opcode::p_unit_test
)
543 unreachable("Pseudo instructions should be lowered before assembly.");
546 if ((uint16_t) instr
->format
& (uint16_t) Format::VOP3A
) {
547 VOP3A_instruction
* vop3
= static_cast<VOP3A_instruction
*>(instr
);
549 if ((uint16_t) instr
->format
& (uint16_t) Format::VOP2
) {
550 opcode
= opcode
+ 0x100;
551 } else if ((uint16_t) instr
->format
& (uint16_t) Format::VOP1
) {
552 if (ctx
.chip_class
== GFX8
|| ctx
.chip_class
== GFX9
)
553 opcode
= opcode
+ 0x140;
555 opcode
= opcode
+ 0x180;
556 } else if ((uint16_t) instr
->format
& (uint16_t) Format::VOPC
) {
557 opcode
= opcode
+ 0x0;
558 } else if ((uint16_t) instr
->format
& (uint16_t) Format::VINTRP
) {
559 opcode
= opcode
+ 0x270;
563 if (ctx
.chip_class
<= GFX9
) {
564 encoding
= (0b110100 << 26);
565 } else if (ctx
.chip_class
>= GFX10
) {
566 encoding
= (0b110101 << 26);
568 unreachable("Unknown chip_class.");
571 if (ctx
.chip_class
<= GFX7
) {
572 encoding
|= opcode
<< 17;
573 encoding
|= (vop3
->clamp
? 1 : 0) << 11;
575 encoding
|= opcode
<< 16;
576 encoding
|= (vop3
->clamp
? 1 : 0) << 15;
578 encoding
|= vop3
->opsel
<< 11;
579 for (unsigned i
= 0; i
< 3; i
++)
580 encoding
|= vop3
->abs
[i
] << (8+i
);
581 if (instr
->definitions
.size() == 2)
582 encoding
|= instr
->definitions
[1].physReg() << 8;
583 encoding
|= (0xFF & instr
->definitions
[0].physReg());
584 out
.push_back(encoding
);
586 if (instr
->opcode
== aco_opcode::v_interp_mov_f32
) {
587 encoding
= 0x3 & instr
->operands
[0].constantValue();
589 for (unsigned i
= 0; i
< instr
->operands
.size(); i
++)
590 encoding
|= instr
->operands
[i
].physReg() << (i
* 9);
592 encoding
|= vop3
->omod
<< 27;
593 for (unsigned i
= 0; i
< 3; i
++)
594 encoding
|= vop3
->neg
[i
] << (29+i
);
595 out
.push_back(encoding
);
597 } else if (instr
->format
== Format::VOP3P
) {
598 VOP3P_instruction
* vop3
= static_cast<VOP3P_instruction
*>(instr
);
601 if (ctx
.chip_class
== GFX9
) {
602 encoding
= (0b110100111 << 23);
603 } else if (ctx
.chip_class
>= GFX10
) {
604 encoding
= (0b110011 << 26);
606 unreachable("Unknown chip_class.");
609 encoding
|= opcode
<< 16;
610 encoding
|= (vop3
->clamp
? 1 : 0) << 15;
611 encoding
|= vop3
->opsel_lo
<< 11;
612 encoding
|= (vop3
->opsel_hi
& 0x4) ? 1 : 0 << 14;
613 for (unsigned i
= 0; i
< 3; i
++)
614 encoding
|= vop3
->neg_hi
[i
] << (8+i
);
615 encoding
|= (0xFF & instr
->definitions
[0].physReg());
616 out
.push_back(encoding
);
618 for (unsigned i
= 0; i
< instr
->operands
.size(); i
++)
619 encoding
|= instr
->operands
[i
].physReg() << (i
* 9);
620 encoding
|= vop3
->opsel_hi
& 0x3 << 27;
621 for (unsigned i
= 0; i
< 3; i
++)
622 encoding
|= vop3
->neg_lo
[i
] << (29+i
);
623 out
.push_back(encoding
);
625 } else if (instr
->isDPP()){
626 assert(ctx
.chip_class
>= GFX8
);
627 /* first emit the instruction without the DPP operand */
628 Operand dpp_op
= instr
->operands
[0];
629 instr
->operands
[0] = Operand(PhysReg
{250}, v1
);
630 instr
->format
= (Format
) ((uint16_t) instr
->format
& ~(uint16_t)Format::DPP
);
631 emit_instruction(ctx
, out
, instr
);
632 DPP_instruction
* dpp
= static_cast<DPP_instruction
*>(instr
);
633 uint32_t encoding
= (0xF & dpp
->row_mask
) << 28;
634 encoding
|= (0xF & dpp
->bank_mask
) << 24;
635 encoding
|= dpp
->abs
[1] << 23;
636 encoding
|= dpp
->neg
[1] << 22;
637 encoding
|= dpp
->abs
[0] << 21;
638 encoding
|= dpp
->neg
[0] << 20;
639 if (ctx
.chip_class
>= GFX10
)
640 encoding
|= 1 << 18; /* set Fetch Inactive to match GFX9 behaviour */
641 encoding
|= dpp
->bound_ctrl
<< 19;
642 encoding
|= dpp
->dpp_ctrl
<< 8;
643 encoding
|= (0xFF) & dpp_op
.physReg();
644 out
.push_back(encoding
);
646 } else if (instr
->isSDWA()) {
647 /* first emit the instruction without the SDWA operand */
648 Operand sdwa_op
= instr
->operands
[0];
649 instr
->operands
[0] = Operand(PhysReg
{249}, v1
);
650 instr
->format
= (Format
) ((uint16_t) instr
->format
& ~(uint16_t)Format::SDWA
);
651 emit_instruction(ctx
, out
, instr
);
653 SDWA_instruction
* sdwa
= static_cast<SDWA_instruction
*>(instr
);
654 uint32_t encoding
= 0;
656 if ((uint16_t)instr
->format
& (uint16_t)Format::VOPC
) {
657 if (instr
->definitions
[0].physReg() != vcc
) {
658 encoding
|= instr
->definitions
[0].physReg() << 8;
661 encoding
|= (sdwa
->clamp
? 1 : 0) << 13;
663 encoding
|= get_sdwa_sel(sdwa
->dst_sel
, instr
->definitions
[0].physReg()) << 8;
664 uint32_t dst_u
= sdwa
->dst_sel
& sdwa_sext
? 1 : 0;
665 if (sdwa
->dst_preserve
|| (sdwa
->dst_sel
& sdwa_isra
))
667 encoding
|= dst_u
<< 11;
668 encoding
|= (sdwa
->clamp
? 1 : 0) << 13;
669 encoding
|= sdwa
->omod
<< 14;
672 encoding
|= get_sdwa_sel(sdwa
->sel
[0], sdwa_op
.physReg()) << 16;
673 encoding
|= sdwa
->sel
[0] & sdwa_sext
? 1 << 19 : 0;
674 encoding
|= sdwa
->abs
[0] << 21;
675 encoding
|= sdwa
->neg
[0] << 20;
677 if (instr
->operands
.size() >= 2) {
678 encoding
|= get_sdwa_sel(sdwa
->sel
[1], instr
->operands
[1].physReg()) << 24;
679 encoding
|= sdwa
->sel
[1] & sdwa_sext
? 1 << 27 : 0;
680 encoding
|= sdwa
->abs
[1] << 29;
681 encoding
|= sdwa
->neg
[1] << 28;
684 encoding
|= 0xFF & sdwa_op
.physReg();
685 encoding
|= (sdwa_op
.physReg() < 256) << 23;
686 if (instr
->operands
.size() >= 2)
687 encoding
|= (instr
->operands
[1].physReg() < 256) << 31;
688 out
.push_back(encoding
);
690 unreachable("unimplemented instruction format");
695 /* append literal dword */
696 for (const Operand
& op
: instr
->operands
) {
697 if (op
.isLiteral()) {
698 out
.push_back(op
.constantValue());
704 void emit_block(asm_context
& ctx
, std::vector
<uint32_t>& out
, Block
& block
)
706 for (aco_ptr
<Instruction
>& instr
: block
.instructions
) {
708 int start_idx
= out
.size();
709 std::cerr
<< "Encoding:\t" << std::endl
;
710 aco_print_instr(&*instr
, stderr
);
711 std::cerr
<< std::endl
;
713 emit_instruction(ctx
, out
, instr
.get());
715 for (int i
= start_idx
; i
< out
.size(); i
++)
716 std::cerr
<< "encoding: " << "0x" << std::setfill('0') << std::setw(8) << std::hex
<< out
[i
] << std::endl
;
721 void fix_exports(asm_context
& ctx
, std::vector
<uint32_t>& out
, Program
* program
)
723 bool exported
= false;
724 for (Block
& block
: program
->blocks
) {
725 if (!(block
.kind
& block_kind_export_end
))
727 std::vector
<aco_ptr
<Instruction
>>::reverse_iterator it
= block
.instructions
.rbegin();
728 while ( it
!= block
.instructions
.rend())
730 if ((*it
)->format
== Format::EXP
) {
731 Export_instruction
* exp
= static_cast<Export_instruction
*>((*it
).get());
732 if (program
->stage
& (hw_vs
| hw_ngg_gs
)) {
733 if (exp
->dest
>= V_008DFC_SQ_EXP_POS
&& exp
->dest
<= (V_008DFC_SQ_EXP_POS
+ 3)) {
740 exp
->valid_mask
= true;
744 } else if ((*it
)->definitions
.size() && (*it
)->definitions
[0].physReg() == exec
)
751 /* Abort in order to avoid a GPU hang. */
752 aco_err(program
, "Missing export in %s shader:", (program
->stage
& hw_vs
) ? "vertex" : "fragment");
753 aco_print_program(program
, stderr
);
758 static void insert_code(asm_context
& ctx
, std::vector
<uint32_t>& out
, unsigned insert_before
,
759 unsigned insert_count
, const uint32_t *insert_data
)
761 out
.insert(out
.begin() + insert_before
, insert_data
, insert_data
+ insert_count
);
763 /* Update the offset of each affected block */
764 for (Block
& block
: ctx
.program
->blocks
) {
765 if (block
.offset
>= insert_before
)
766 block
.offset
+= insert_count
;
769 /* Find first branch after the inserted code */
770 auto branch_it
= std::find_if(ctx
.branches
.begin(), ctx
.branches
.end(), [insert_before
](const auto &branch
) -> bool {
771 return (unsigned)branch
.first
>= insert_before
;
774 /* Update the locations of branches */
775 for (; branch_it
!= ctx
.branches
.end(); ++branch_it
)
776 branch_it
->first
+= insert_count
;
778 /* Find first constant address after the inserted code */
779 auto caddr_it
= std::find_if(ctx
.constaddrs
.begin(), ctx
.constaddrs
.end(), [insert_before
](const int &caddr_pos
) -> bool {
780 return (unsigned)caddr_pos
>= insert_before
;
783 /* Update the locations of constant addresses */
784 for (; caddr_it
!= ctx
.constaddrs
.end(); ++caddr_it
)
785 (*caddr_it
) += insert_count
;
788 static void fix_branches_gfx10(asm_context
& ctx
, std::vector
<uint32_t>& out
)
790 /* Branches with an offset of 0x3f are buggy on GFX10, we workaround by inserting NOPs if needed. */
791 bool gfx10_3f_bug
= false;
794 auto buggy_branch_it
= std::find_if(ctx
.branches
.begin(), ctx
.branches
.end(), [&ctx
](const auto &branch
) -> bool {
795 return ((int)ctx
.program
->blocks
[branch
.second
->block
].offset
- branch
.first
- 1) == 0x3f;
798 gfx10_3f_bug
= buggy_branch_it
!= ctx
.branches
.end();
801 /* Insert an s_nop after the branch */
802 constexpr uint32_t s_nop_0
= 0xbf800000u
;
803 insert_code(ctx
, out
, buggy_branch_it
->first
+ 1, 1, &s_nop_0
);
805 } while (gfx10_3f_bug
);
808 void emit_long_jump(asm_context
& ctx
, SOPP_instruction
*branch
, bool backwards
, std::vector
<uint32_t>& out
)
810 Builder
bld(ctx
.program
);
812 Definition
def_tmp_lo(branch
->definitions
[0].physReg(), s1
);
813 Operand
op_tmp_lo(branch
->definitions
[0].physReg(), s1
);
814 Definition
def_tmp_hi(branch
->definitions
[0].physReg().advance(4), s1
);
815 Operand
op_tmp_hi(branch
->definitions
[0].physReg().advance(4), s1
);
817 aco_ptr
<Instruction
> instr
;
819 if (branch
->opcode
!= aco_opcode::s_branch
) {
820 /* for conditional branches, skip the long jump if the condition is false */
822 switch (branch
->opcode
) {
823 case aco_opcode::s_cbranch_scc0
:
824 inv
= aco_opcode::s_cbranch_scc1
;
826 case aco_opcode::s_cbranch_scc1
:
827 inv
= aco_opcode::s_cbranch_scc0
;
829 case aco_opcode::s_cbranch_vccz
:
830 inv
= aco_opcode::s_cbranch_vccnz
;
832 case aco_opcode::s_cbranch_vccnz
:
833 inv
= aco_opcode::s_cbranch_vccz
;
835 case aco_opcode::s_cbranch_execz
:
836 inv
= aco_opcode::s_cbranch_execnz
;
838 case aco_opcode::s_cbranch_execnz
:
839 inv
= aco_opcode::s_cbranch_execz
;
842 unreachable("Unhandled long jump.");
844 instr
.reset(bld
.sopp(inv
, -1, 7));
845 emit_instruction(ctx
, out
, instr
.get());
848 /* create the new PC and stash SCC in the LSB */
849 instr
.reset(bld
.sop1(aco_opcode::s_getpc_b64
, branch
->definitions
[0]).instr
);
850 emit_instruction(ctx
, out
, instr
.get());
852 instr
.reset(bld
.sop2(aco_opcode::s_addc_u32
, def_tmp_lo
, op_tmp_lo
, Operand(0u)).instr
);
853 instr
->operands
[1].setFixed(PhysReg
{255}); /* this operand has to be a literal */
854 emit_instruction(ctx
, out
, instr
.get());
855 branch
->pass_flags
= out
.size();
857 instr
.reset(bld
.sop2(aco_opcode::s_addc_u32
, def_tmp_hi
, op_tmp_hi
, Operand(backwards
? UINT32_MAX
: 0u)).instr
);
858 emit_instruction(ctx
, out
, instr
.get());
860 /* restore SCC and clear the LSB of the new PC */
861 instr
.reset(bld
.sopc(aco_opcode::s_bitcmp1_b32
, def_tmp_lo
, op_tmp_lo
, Operand(0u)).instr
);
862 emit_instruction(ctx
, out
, instr
.get());
863 instr
.reset(bld
.sop1(aco_opcode::s_bitset0_b32
, def_tmp_lo
, Operand(0u)).instr
);
864 emit_instruction(ctx
, out
, instr
.get());
866 /* create the s_setpc_b64 to jump */
867 instr
.reset(bld
.sop1(aco_opcode::s_setpc_b64
, Operand(branch
->definitions
[0].physReg(), s2
)).instr
);
868 emit_instruction(ctx
, out
, instr
.get());
871 void fix_branches(asm_context
& ctx
, std::vector
<uint32_t>& out
)
877 if (ctx
.chip_class
== GFX10
)
878 fix_branches_gfx10(ctx
, out
);
880 for (std::pair
<int, SOPP_instruction
*> &branch
: ctx
.branches
) {
881 int offset
= (int)ctx
.program
->blocks
[branch
.second
->block
].offset
- branch
.first
- 1;
882 if ((offset
< INT16_MIN
|| offset
> INT16_MAX
) && !branch
.second
->pass_flags
) {
883 std::vector
<uint32_t> long_jump
;
884 bool backwards
= ctx
.program
->blocks
[branch
.second
->block
].offset
< (unsigned)branch
.first
;
885 emit_long_jump(ctx
, branch
.second
, backwards
, long_jump
);
887 out
[branch
.first
] = long_jump
[0];
888 insert_code(ctx
, out
, branch
.first
+ 1, long_jump
.size() - 1, long_jump
.data() + 1);
894 if (branch
.second
->pass_flags
) {
895 int after_getpc
= branch
.first
+ branch
.second
->pass_flags
- 2;
896 offset
= (int)ctx
.program
->blocks
[branch
.second
->block
].offset
- after_getpc
;
897 out
[branch
.first
+ branch
.second
->pass_flags
- 1] = offset
* 4;
899 out
[branch
.first
] &= 0xffff0000u
;
900 out
[branch
.first
] |= (uint16_t) offset
;
906 void fix_constaddrs(asm_context
& ctx
, std::vector
<uint32_t>& out
)
908 for (unsigned addr
: ctx
.constaddrs
)
909 out
[addr
] += (out
.size() - addr
+ 1u) * 4u;
912 unsigned emit_program(Program
* program
,
913 std::vector
<uint32_t>& code
)
915 asm_context
ctx(program
);
917 if (program
->stage
& (hw_vs
| hw_fs
| hw_ngg_gs
))
918 fix_exports(ctx
, code
, program
);
920 for (Block
& block
: program
->blocks
) {
921 block
.offset
= code
.size();
922 emit_block(ctx
, code
, block
);
925 fix_branches(ctx
, code
);
927 unsigned exec_size
= code
.size() * sizeof(uint32_t);
929 if (program
->chip_class
>= GFX10
) {
930 /* Pad output with s_code_end so instruction prefetching doesn't cause
932 unsigned final_size
= align(code
.size() + 3 * 16, 16);
933 while (code
.size() < final_size
)
934 code
.push_back(0xbf9f0000u
);
937 fix_constaddrs(ctx
, code
);
939 while (program
->constant_data
.size() % 4u)
940 program
->constant_data
.push_back(0);
941 /* Copy constant data */
942 code
.insert(code
.end(), (uint32_t*)program
->constant_data
.data(),
943 (uint32_t*)(program
->constant_data
.data() + program
->constant_data
.size()));