0d74458519c488dc15fb0b4f2c0cc5b53cb4a887
[mesa.git] / src / amd / compiler / aco_validate.cpp
1 /*
2 * Copyright © 2018 Valve Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 *
23 */
24
25 #include "aco_ir.h"
26
27 #include <array>
28 #include <map>
29
30 namespace aco {
31
32 static void aco_log(Program *program, enum radv_compiler_debug_level level,
33 const char *prefix, const char *file, unsigned line,
34 const char *fmt, va_list args)
35 {
36 char *msg;
37
38 msg = ralloc_strdup(NULL, prefix);
39
40 ralloc_asprintf_append(&msg, " In file %s:%u\n", file, line);
41 ralloc_asprintf_append(&msg, " ");
42 ralloc_vasprintf_append(&msg, fmt, args);
43
44 if (program->debug.func)
45 program->debug.func(program->debug.private_data, level, msg);
46
47 fprintf(stderr, "%s\n", msg);
48
49 ralloc_free(msg);
50 }
51
52 void _aco_perfwarn(Program *program, const char *file, unsigned line,
53 const char *fmt, ...)
54 {
55 va_list args;
56
57 va_start(args, fmt);
58 aco_log(program, RADV_COMPILER_DEBUG_LEVEL_PERFWARN,
59 "ACO PERFWARN:\n", file, line, fmt, args);
60 va_end(args);
61 }
62
63 void _aco_err(Program *program, const char *file, unsigned line,
64 const char *fmt, ...)
65 {
66 va_list args;
67
68 va_start(args, fmt);
69 aco_log(program, RADV_COMPILER_DEBUG_LEVEL_ERROR,
70 "ACO ERROR:\n", file, line, fmt, args);
71 va_end(args);
72 }
73
74 bool validate_ir(Program* program)
75 {
76 bool is_valid = true;
77 auto check = [&program, &is_valid](bool check, const char * msg, aco::Instruction * instr) -> void {
78 if (!check) {
79 char *out;
80 size_t outsize;
81 FILE *memf = open_memstream(&out, &outsize);
82
83 fprintf(memf, "%s: ", msg);
84 aco_print_instr(instr, memf);
85 fclose(memf);
86
87 aco_err(program, out);
88 free(out);
89
90 is_valid = false;
91 }
92 };
93
94 auto check_block = [&program, &is_valid](bool check, const char * msg, aco::Block * block) -> void {
95 if (!check) {
96 aco_err(program, "%s: BB%u", msg, block->index);
97 is_valid = false;
98 }
99 };
100
101 for (Block& block : program->blocks) {
102 for (aco_ptr<Instruction>& instr : block.instructions) {
103
104 /* check base format */
105 Format base_format = instr->format;
106 base_format = (Format)((uint32_t)base_format & ~(uint32_t)Format::SDWA);
107 base_format = (Format)((uint32_t)base_format & ~(uint32_t)Format::DPP);
108 if ((uint32_t)base_format & (uint32_t)Format::VOP1)
109 base_format = Format::VOP1;
110 else if ((uint32_t)base_format & (uint32_t)Format::VOP2)
111 base_format = Format::VOP2;
112 else if ((uint32_t)base_format & (uint32_t)Format::VOPC)
113 base_format = Format::VOPC;
114 else if ((uint32_t)base_format & (uint32_t)Format::VINTRP) {
115 if (instr->opcode == aco_opcode::v_interp_p1ll_f16 ||
116 instr->opcode == aco_opcode::v_interp_p1lv_f16 ||
117 instr->opcode == aco_opcode::v_interp_p2_legacy_f16 ||
118 instr->opcode == aco_opcode::v_interp_p2_f16) {
119 /* v_interp_*_fp16 are considered VINTRP by the compiler but
120 * they are emitted as VOP3.
121 */
122 base_format = Format::VOP3;
123 } else {
124 base_format = Format::VINTRP;
125 }
126 }
127 check(base_format == instr_info.format[(int)instr->opcode], "Wrong base format for instruction", instr.get());
128
129 /* check VOP3 modifiers */
130 if (((uint32_t)instr->format & (uint32_t)Format::VOP3) && instr->format != Format::VOP3) {
131 check(base_format == Format::VOP2 ||
132 base_format == Format::VOP1 ||
133 base_format == Format::VOPC ||
134 base_format == Format::VINTRP,
135 "Format cannot have VOP3A/VOP3B applied", instr.get());
136 }
137
138 /* check SDWA */
139 if (instr->isSDWA()) {
140 check(base_format == Format::VOP2 ||
141 base_format == Format::VOP1 ||
142 base_format == Format::VOPC,
143 "Format cannot have SDWA applied", instr.get());
144
145 check(program->chip_class >= GFX8, "SDWA is GFX8+ only", instr.get());
146
147 SDWA_instruction *sdwa = static_cast<SDWA_instruction*>(instr.get());
148 check(sdwa->omod == 0 || program->chip_class >= GFX9, "SDWA omod only supported on GFX9+", instr.get());
149 if (base_format == Format::VOPC) {
150 check(sdwa->clamp == false || program->chip_class == GFX8, "SDWA VOPC clamp only supported on GFX8", instr.get());
151 check((instr->definitions[0].isFixed() && instr->definitions[0].physReg() == vcc) ||
152 program->chip_class >= GFX9,
153 "SDWA+VOPC definition must be fixed to vcc on GFX8", instr.get());
154 }
155
156 if (instr->operands.size() >= 3) {
157 check(instr->operands[2].isFixed() && instr->operands[2].physReg() == vcc,
158 "3rd operand must be fixed to vcc with SDWA", instr.get());
159 }
160 if (instr->definitions.size() >= 2) {
161 check(instr->definitions[1].isFixed() && instr->definitions[1].physReg() == vcc,
162 "2nd definition must be fixed to vcc with SDWA", instr.get());
163 }
164
165 check(instr->opcode != aco_opcode::v_madmk_f32 &&
166 instr->opcode != aco_opcode::v_madak_f32 &&
167 instr->opcode != aco_opcode::v_madmk_f16 &&
168 instr->opcode != aco_opcode::v_madak_f16 &&
169 instr->opcode != aco_opcode::v_readfirstlane_b32 &&
170 instr->opcode != aco_opcode::v_clrexcp &&
171 instr->opcode != aco_opcode::v_swap_b32,
172 "SDWA can't be used with this opcode", instr.get());
173 if (program->chip_class != GFX8) {
174 check(instr->opcode != aco_opcode::v_mac_f32 &&
175 instr->opcode != aco_opcode::v_mac_f16 &&
176 instr->opcode != aco_opcode::v_fmac_f32 &&
177 instr->opcode != aco_opcode::v_fmac_f16,
178 "SDWA can't be used with this opcode", instr.get());
179 }
180
181 for (unsigned i = 0; i < MIN2(instr->operands.size(), 2); i++) {
182 if (instr->operands[i].hasRegClass() && instr->operands[i].regClass().is_subdword())
183 check((sdwa->sel[i] & sdwa_asuint) == (sdwa_isra | instr->operands[i].bytes()), "Unexpected SDWA sel for sub-dword operand", instr.get());
184 }
185 if (instr->definitions[0].regClass().is_subdword())
186 check((sdwa->dst_sel & sdwa_asuint) == (sdwa_isra | instr->definitions[0].bytes()), "Unexpected SDWA sel for sub-dword definition", instr.get());
187 }
188
189 /* check opsel */
190 if (instr->isVOP3()) {
191 VOP3A_instruction *vop3 = static_cast<VOP3A_instruction*>(instr.get());
192 check(vop3->opsel == 0 || program->chip_class >= GFX9, "Opsel is only supported on GFX9+", instr.get());
193
194 for (unsigned i = 0; i < 3; i++) {
195 if (i >= instr->operands.size() ||
196 (instr->operands[i].hasRegClass() && instr->operands[i].regClass().is_subdword() && !instr->operands[i].isFixed()))
197 check((vop3->opsel & (1 << i)) == 0, "Unexpected opsel for operand", instr.get());
198 }
199 if (instr->definitions[0].regClass().is_subdword() && !instr->definitions[0].isFixed())
200 check((vop3->opsel & (1 << 3)) == 0, "Unexpected opsel for sub-dword definition", instr.get());
201 }
202
203 /* check for undefs */
204 for (unsigned i = 0; i < instr->operands.size(); i++) {
205 if (instr->operands[i].isUndefined()) {
206 bool flat = instr->format == Format::FLAT || instr->format == Format::SCRATCH || instr->format == Format::GLOBAL;
207 bool can_be_undef = is_phi(instr) || instr->format == Format::EXP ||
208 instr->format == Format::PSEUDO_REDUCTION ||
209 instr->opcode == aco_opcode::p_create_vector ||
210 (flat && i == 1) || (instr->format == Format::MIMG && i == 1) ||
211 ((instr->format == Format::MUBUF || instr->format == Format::MTBUF) && i == 1);
212 check(can_be_undef, "Undefs can only be used in certain operands", instr.get());
213 } else {
214 check(instr->operands[i].isFixed() || instr->operands[i].isTemp() || instr->operands[i].isConstant(), "Uninitialized Operand", instr.get());
215 }
216 }
217
218 /* check subdword definitions */
219 for (unsigned i = 0; i < instr->definitions.size(); i++) {
220 if (instr->definitions[i].regClass().is_subdword())
221 check(instr->format == Format::PSEUDO || instr->definitions[i].bytes() <= 4, "Only Pseudo instructions can write subdword registers larger than 4 bytes", instr.get());
222 }
223
224 if (instr->isSALU() || instr->isVALU()) {
225 /* check literals */
226 Operand literal(s1);
227 for (unsigned i = 0; i < instr->operands.size(); i++)
228 {
229 Operand op = instr->operands[i];
230 if (!op.isLiteral())
231 continue;
232
233 check(instr->format == Format::SOP1 ||
234 instr->format == Format::SOP2 ||
235 instr->format == Format::SOPC ||
236 instr->format == Format::VOP1 ||
237 instr->format == Format::VOP2 ||
238 instr->format == Format::VOPC ||
239 (instr->isVOP3() && program->chip_class >= GFX10),
240 "Literal applied on wrong instruction format", instr.get());
241
242 check(literal.isUndefined() || (literal.size() == op.size() && literal.constantValue() == op.constantValue()), "Only 1 Literal allowed", instr.get());
243 literal = op;
244 check(!instr->isVALU() || instr->isVOP3() || i == 0 || i == 2, "Wrong source position for Literal argument", instr.get());
245 }
246
247 /* check num sgprs for VALU */
248 if (instr->isVALU()) {
249 bool is_shift64 = instr->opcode == aco_opcode::v_lshlrev_b64 ||
250 instr->opcode == aco_opcode::v_lshrrev_b64 ||
251 instr->opcode == aco_opcode::v_ashrrev_i64;
252 unsigned const_bus_limit = 1;
253 if (program->chip_class >= GFX10 && !is_shift64)
254 const_bus_limit = 2;
255
256 uint32_t scalar_mask = instr->isVOP3() ? 0x7 : 0x5;
257 if (instr->isSDWA())
258 scalar_mask = program->chip_class >= GFX9 ? 0x7 : 0x4;
259
260 if ((int) instr->format & (int) Format::VOPC ||
261 instr->opcode == aco_opcode::v_readfirstlane_b32 ||
262 instr->opcode == aco_opcode::v_readlane_b32 ||
263 instr->opcode == aco_opcode::v_readlane_b32_e64) {
264 check(instr->definitions[0].getTemp().type() == RegType::sgpr,
265 "Wrong Definition type for VALU instruction", instr.get());
266 } else {
267 check(instr->definitions[0].getTemp().type() == RegType::vgpr,
268 "Wrong Definition type for VALU instruction", instr.get());
269 }
270
271 unsigned num_sgprs = 0;
272 unsigned sgpr[] = {0, 0};
273 for (unsigned i = 0; i < instr->operands.size(); i++)
274 {
275 Operand op = instr->operands[i];
276 if (instr->opcode == aco_opcode::v_readfirstlane_b32 ||
277 instr->opcode == aco_opcode::v_readlane_b32 ||
278 instr->opcode == aco_opcode::v_readlane_b32_e64) {
279 check(i != 1 ||
280 (op.isTemp() && op.regClass().type() == RegType::sgpr) ||
281 op.isConstant(),
282 "Must be a SGPR or a constant", instr.get());
283 check(i == 1 ||
284 (op.isTemp() && op.regClass().type() == RegType::vgpr && op.bytes() <= 4),
285 "Wrong Operand type for VALU instruction", instr.get());
286 continue;
287 }
288
289 if (instr->opcode == aco_opcode::v_writelane_b32 ||
290 instr->opcode == aco_opcode::v_writelane_b32_e64) {
291 check(i != 2 ||
292 (op.isTemp() && op.regClass().type() == RegType::vgpr && op.bytes() <= 4),
293 "Wrong Operand type for VALU instruction", instr.get());
294 check(i == 2 ||
295 (op.isTemp() && op.regClass().type() == RegType::sgpr) ||
296 op.isConstant(),
297 "Must be a SGPR or a constant", instr.get());
298 continue;
299 }
300 if (op.isTemp() && instr->operands[i].regClass().type() == RegType::sgpr) {
301 check(scalar_mask & (1 << i), "Wrong source position for SGPR argument", instr.get());
302
303 if (op.tempId() != sgpr[0] && op.tempId() != sgpr[1]) {
304 if (num_sgprs < 2)
305 sgpr[num_sgprs++] = op.tempId();
306 }
307 }
308
309 if (op.isConstant() && !op.isLiteral())
310 check(scalar_mask & (1 << i), "Wrong source position for constant argument", instr.get());
311 }
312 check(num_sgprs + (literal.isUndefined() ? 0 : 1) <= const_bus_limit, "Too many SGPRs/literals", instr.get());
313 }
314
315 if (instr->format == Format::SOP1 || instr->format == Format::SOP2) {
316 check(instr->definitions[0].getTemp().type() == RegType::sgpr, "Wrong Definition type for SALU instruction", instr.get());
317 for (const Operand& op : instr->operands) {
318 check(op.isConstant() || op.regClass().type() <= RegType::sgpr,
319 "Wrong Operand type for SALU instruction", instr.get());
320 }
321 }
322 }
323
324 switch (instr->format) {
325 case Format::PSEUDO: {
326 bool is_subdword = false;
327 bool has_const_sgpr = false;
328 bool has_literal = false;
329 for (Definition def : instr->definitions)
330 is_subdword |= def.regClass().is_subdword();
331 for (unsigned i = 0; i < instr->operands.size(); i++) {
332 if (instr->opcode == aco_opcode::p_extract_vector && i == 1)
333 continue;
334 Operand op = instr->operands[i];
335 is_subdword |= op.hasRegClass() && op.regClass().is_subdword();
336 has_const_sgpr |= op.isConstant() || (op.hasRegClass() && op.regClass().type() == RegType::sgpr);
337 has_literal |= op.isLiteral();
338 }
339
340 check(!is_subdword || !has_const_sgpr || program->chip_class >= GFX9,
341 "Sub-dword pseudo instructions can only take constants or SGPRs on GFX9+", instr.get());
342 check(!is_subdword || !has_literal, "Sub-dword pseudo instructions cannot take literals", instr.get());
343
344 if (instr->opcode == aco_opcode::p_create_vector) {
345 unsigned size = 0;
346 for (const Operand& op : instr->operands) {
347 size += op.bytes();
348 }
349 check(size == instr->definitions[0].bytes(), "Definition size does not match operand sizes", instr.get());
350 if (instr->definitions[0].getTemp().type() == RegType::sgpr) {
351 for (const Operand& op : instr->operands) {
352 check(op.isConstant() || op.regClass().type() == RegType::sgpr,
353 "Wrong Operand type for scalar vector", instr.get());
354 }
355 }
356 } else if (instr->opcode == aco_opcode::p_extract_vector) {
357 check((instr->operands[0].isTemp()) && instr->operands[1].isConstant(), "Wrong Operand types", instr.get());
358 check((instr->operands[1].constantValue() + 1) * instr->definitions[0].bytes() <= instr->operands[0].bytes(), "Index out of range", instr.get());
359 check(instr->definitions[0].getTemp().type() == RegType::vgpr || instr->operands[0].regClass().type() == RegType::sgpr,
360 "Cannot extract SGPR value from VGPR vector", instr.get());
361 } else if (instr->opcode == aco_opcode::p_parallelcopy) {
362 check(instr->definitions.size() == instr->operands.size(), "Number of Operands does not match number of Definitions", instr.get());
363 for (unsigned i = 0; i < instr->operands.size(); i++) {
364 if (instr->operands[i].isTemp())
365 check((instr->definitions[i].getTemp().type() == instr->operands[i].regClass().type()) ||
366 (instr->definitions[i].getTemp().type() == RegType::vgpr && instr->operands[i].regClass().type() == RegType::sgpr),
367 "Operand and Definition types do not match", instr.get());
368 }
369 } else if (instr->opcode == aco_opcode::p_phi) {
370 check(instr->operands.size() == block.logical_preds.size(), "Number of Operands does not match number of predecessors", instr.get());
371 check(instr->definitions[0].getTemp().type() == RegType::vgpr, "Logical Phi Definition must be vgpr", instr.get());
372 } else if (instr->opcode == aco_opcode::p_linear_phi) {
373 for (const Operand& op : instr->operands)
374 check(!op.isTemp() || op.getTemp().is_linear(), "Wrong Operand type", instr.get());
375 check(instr->operands.size() == block.linear_preds.size(), "Number of Operands does not match number of predecessors", instr.get());
376 }
377 break;
378 }
379 case Format::SMEM: {
380 if (instr->operands.size() >= 1)
381 check(instr->operands[0].isTemp() && instr->operands[0].regClass().type() == RegType::sgpr, "SMEM operands must be sgpr", instr.get());
382 if (instr->operands.size() >= 2)
383 check(instr->operands[1].isConstant() || (instr->operands[1].isTemp() && instr->operands[1].regClass().type() == RegType::sgpr),
384 "SMEM offset must be constant or sgpr", instr.get());
385 if (!instr->definitions.empty())
386 check(instr->definitions[0].getTemp().type() == RegType::sgpr, "SMEM result must be sgpr", instr.get());
387 break;
388 }
389 case Format::MTBUF:
390 case Format::MUBUF: {
391 check(instr->operands.size() > 1, "VMEM instructions must have at least one operand", instr.get());
392 check(instr->operands[1].hasRegClass() && instr->operands[1].regClass().type() == RegType::vgpr,
393 "VADDR must be in vgpr for VMEM instructions", instr.get());
394 check(instr->operands[0].isTemp() && instr->operands[0].regClass().type() == RegType::sgpr, "VMEM resource constant must be sgpr", instr.get());
395 check(instr->operands.size() < 4 || (instr->operands[3].isTemp() && instr->operands[3].regClass().type() == RegType::vgpr), "VMEM write data must be vgpr", instr.get());
396 break;
397 }
398 case Format::MIMG: {
399 check(instr->operands.size() == 3, "MIMG instructions must have exactly 3 operands", instr.get());
400 check(instr->operands[0].hasRegClass() && (instr->operands[0].regClass() == s4 || instr->operands[0].regClass() == s8),
401 "MIMG operands[0] (resource constant) must be in 4 or 8 SGPRs", instr.get());
402 if (instr->operands[1].hasRegClass() && instr->operands[1].regClass().type() == RegType::sgpr)
403 check(instr->operands[1].regClass() == s4, "MIMG operands[1] (sampler constant) must be 4 SGPRs", instr.get());
404 else if (instr->operands[1].hasRegClass() && instr->operands[1].regClass().type() == RegType::vgpr)
405 check((instr->definitions.empty() || instr->definitions[0].regClass() == instr->operands[1].regClass() ||
406 instr->opcode == aco_opcode::image_atomic_cmpswap || instr->opcode == aco_opcode::image_atomic_fcmpswap),
407 "MIMG operands[1] (VDATA) must be the same as definitions[0] for atomics", instr.get());
408 check(instr->operands[2].hasRegClass() && instr->operands[2].regClass().type() == RegType::vgpr,
409 "MIMG operands[2] (VADDR) must be VGPR", instr.get());
410 check(instr->definitions.empty() || (instr->definitions[0].isTemp() && instr->definitions[0].regClass().type() == RegType::vgpr),
411 "MIMG definitions[0] (VDATA) must be VGPR", instr.get());
412 break;
413 }
414 case Format::DS: {
415 for (const Operand& op : instr->operands) {
416 check((op.isTemp() && op.regClass().type() == RegType::vgpr) || op.physReg() == m0,
417 "Only VGPRs are valid DS instruction operands", instr.get());
418 }
419 if (!instr->definitions.empty())
420 check(instr->definitions[0].getTemp().type() == RegType::vgpr, "DS instruction must return VGPR", instr.get());
421 break;
422 }
423 case Format::EXP: {
424 for (unsigned i = 0; i < 4; i++)
425 check(instr->operands[i].hasRegClass() && instr->operands[i].regClass().type() == RegType::vgpr,
426 "Only VGPRs are valid Export arguments", instr.get());
427 break;
428 }
429 case Format::FLAT:
430 check(instr->operands[1].isUndefined(), "Flat instructions don't support SADDR", instr.get());
431 /* fallthrough */
432 case Format::GLOBAL:
433 case Format::SCRATCH: {
434 check(instr->operands[0].isTemp() && instr->operands[0].regClass().type() == RegType::vgpr, "FLAT/GLOBAL/SCRATCH address must be vgpr", instr.get());
435 check(instr->operands[1].hasRegClass() && instr->operands[1].regClass().type() == RegType::sgpr,
436 "FLAT/GLOBAL/SCRATCH sgpr address must be undefined or sgpr", instr.get());
437 if (!instr->definitions.empty())
438 check(instr->definitions[0].getTemp().type() == RegType::vgpr, "FLAT/GLOBAL/SCRATCH result must be vgpr", instr.get());
439 else
440 check(instr->operands[2].regClass().type() == RegType::vgpr, "FLAT/GLOBAL/SCRATCH data must be vgpr", instr.get());
441 break;
442 }
443 default:
444 break;
445 }
446 }
447 }
448
449 /* validate CFG */
450 for (unsigned i = 0; i < program->blocks.size(); i++) {
451 Block& block = program->blocks[i];
452 check_block(block.index == i, "block.index must match actual index", &block);
453
454 /* predecessors/successors should be sorted */
455 for (unsigned j = 0; j + 1 < block.linear_preds.size(); j++)
456 check_block(block.linear_preds[j] < block.linear_preds[j + 1], "linear predecessors must be sorted", &block);
457 for (unsigned j = 0; j + 1 < block.logical_preds.size(); j++)
458 check_block(block.logical_preds[j] < block.logical_preds[j + 1], "logical predecessors must be sorted", &block);
459 for (unsigned j = 0; j + 1 < block.linear_succs.size(); j++)
460 check_block(block.linear_succs[j] < block.linear_succs[j + 1], "linear successors must be sorted", &block);
461 for (unsigned j = 0; j + 1 < block.logical_succs.size(); j++)
462 check_block(block.logical_succs[j] < block.logical_succs[j + 1], "logical successors must be sorted", &block);
463
464 /* critical edges are not allowed */
465 if (block.linear_preds.size() > 1) {
466 for (unsigned pred : block.linear_preds)
467 check_block(program->blocks[pred].linear_succs.size() == 1, "linear critical edges are not allowed", &program->blocks[pred]);
468 for (unsigned pred : block.logical_preds)
469 check_block(program->blocks[pred].logical_succs.size() == 1, "logical critical edges are not allowed", &program->blocks[pred]);
470 }
471 }
472
473 return is_valid;
474 }
475
476 /* RA validation */
477 namespace {
478
479 struct Location {
480 Location() : block(NULL), instr(NULL) {}
481
482 Block *block;
483 Instruction *instr; //NULL if it's the block's live-in
484 };
485
486 struct Assignment {
487 Location defloc;
488 Location firstloc;
489 PhysReg reg;
490 };
491
492 bool ra_fail(Program *program, Location loc, Location loc2, const char *fmt, ...) {
493 va_list args;
494 va_start(args, fmt);
495 char msg[1024];
496 vsprintf(msg, fmt, args);
497 va_end(args);
498
499 char *out;
500 size_t outsize;
501 FILE *memf = open_memstream(&out, &outsize);
502
503 fprintf(memf, "RA error found at instruction in BB%d:\n", loc.block->index);
504 if (loc.instr) {
505 aco_print_instr(loc.instr, memf);
506 fprintf(memf, "\n%s", msg);
507 } else {
508 fprintf(memf, "%s", msg);
509 }
510 if (loc2.block) {
511 fprintf(memf, " in BB%d:\n", loc2.block->index);
512 aco_print_instr(loc2.instr, memf);
513 }
514 fprintf(memf, "\n\n");
515 fclose(memf);
516
517 aco_err(program, out);
518 free(out);
519
520 return true;
521 }
522
523 bool validate_subdword_operand(chip_class chip, const aco_ptr<Instruction>& instr, unsigned index)
524 {
525 Operand op = instr->operands[index];
526 unsigned byte = op.physReg().byte();
527
528 if (instr->opcode == aco_opcode::p_as_uniform)
529 return byte == 0;
530 if (instr->format == Format::PSEUDO && chip >= GFX8)
531 return true;
532 if (instr->isSDWA() && (static_cast<SDWA_instruction *>(instr.get())->sel[index] & sdwa_asuint) == (sdwa_isra | op.bytes()))
533 return true;
534 if (byte == 2 && can_use_opsel(chip, instr->opcode, index, 1))
535 return true;
536
537 switch (instr->opcode) {
538 case aco_opcode::v_cvt_f32_ubyte1:
539 if (byte == 1)
540 return true;
541 break;
542 case aco_opcode::v_cvt_f32_ubyte2:
543 if (byte == 2)
544 return true;
545 break;
546 case aco_opcode::v_cvt_f32_ubyte3:
547 if (byte == 3)
548 return true;
549 break;
550 case aco_opcode::ds_write_b8_d16_hi:
551 case aco_opcode::ds_write_b16_d16_hi:
552 if (byte == 2 && index == 1)
553 return true;
554 break;
555 case aco_opcode::buffer_store_byte_d16_hi:
556 case aco_opcode::buffer_store_short_d16_hi:
557 if (byte == 2 && index == 3)
558 return true;
559 break;
560 case aco_opcode::flat_store_byte_d16_hi:
561 case aco_opcode::flat_store_short_d16_hi:
562 case aco_opcode::scratch_store_byte_d16_hi:
563 case aco_opcode::scratch_store_short_d16_hi:
564 case aco_opcode::global_store_byte_d16_hi:
565 case aco_opcode::global_store_short_d16_hi:
566 if (byte == 2 && index == 2)
567 return true;
568 default:
569 break;
570 }
571
572 return byte == 0;
573 }
574
575 bool validate_subdword_definition(chip_class chip, const aco_ptr<Instruction>& instr)
576 {
577 Definition def = instr->definitions[0];
578 unsigned byte = def.physReg().byte();
579
580 if (instr->format == Format::PSEUDO && chip >= GFX8)
581 return true;
582 if (instr->isSDWA() && static_cast<SDWA_instruction *>(instr.get())->dst_sel == (sdwa_isra | def.bytes()))
583 return true;
584 if (byte == 2 && can_use_opsel(chip, instr->opcode, -1, 1))
585 return true;
586
587 switch (instr->opcode) {
588 case aco_opcode::buffer_load_ubyte_d16_hi:
589 case aco_opcode::buffer_load_short_d16_hi:
590 case aco_opcode::flat_load_ubyte_d16_hi:
591 case aco_opcode::flat_load_short_d16_hi:
592 case aco_opcode::scratch_load_ubyte_d16_hi:
593 case aco_opcode::scratch_load_short_d16_hi:
594 case aco_opcode::global_load_ubyte_d16_hi:
595 case aco_opcode::global_load_short_d16_hi:
596 case aco_opcode::ds_read_u8_d16_hi:
597 case aco_opcode::ds_read_u16_d16_hi:
598 return byte == 2;
599 default:
600 break;
601 }
602
603 return byte == 0;
604 }
605
606 unsigned get_subdword_bytes_written(Program *program, const aco_ptr<Instruction>& instr, unsigned index)
607 {
608 chip_class chip = program->chip_class;
609 Definition def = instr->definitions[index];
610
611 if (instr->format == Format::PSEUDO)
612 return chip >= GFX8 ? def.bytes() : def.size() * 4u;
613 if (instr->isSDWA() && static_cast<SDWA_instruction *>(instr.get())->dst_sel == (sdwa_isra | def.bytes()))
614 return def.bytes();
615
616 switch (instr->opcode) {
617 case aco_opcode::buffer_load_ubyte_d16:
618 case aco_opcode::buffer_load_short_d16:
619 case aco_opcode::flat_load_ubyte_d16:
620 case aco_opcode::flat_load_short_d16:
621 case aco_opcode::scratch_load_ubyte_d16:
622 case aco_opcode::scratch_load_short_d16:
623 case aco_opcode::global_load_ubyte_d16:
624 case aco_opcode::global_load_short_d16:
625 case aco_opcode::ds_read_u8_d16:
626 case aco_opcode::ds_read_u16_d16:
627 case aco_opcode::buffer_load_ubyte_d16_hi:
628 case aco_opcode::buffer_load_short_d16_hi:
629 case aco_opcode::flat_load_ubyte_d16_hi:
630 case aco_opcode::flat_load_short_d16_hi:
631 case aco_opcode::scratch_load_ubyte_d16_hi:
632 case aco_opcode::scratch_load_short_d16_hi:
633 case aco_opcode::global_load_ubyte_d16_hi:
634 case aco_opcode::global_load_short_d16_hi:
635 case aco_opcode::ds_read_u8_d16_hi:
636 case aco_opcode::ds_read_u16_d16_hi:
637 return program->sram_ecc_enabled ? 4 : 2;
638 case aco_opcode::v_mad_f16:
639 case aco_opcode::v_mad_u16:
640 case aco_opcode::v_mad_i16:
641 case aco_opcode::v_fma_f16:
642 case aco_opcode::v_div_fixup_f16:
643 case aco_opcode::v_interp_p2_f16:
644 if (chip >= GFX9)
645 return 2;
646 default:
647 break;
648 }
649
650 return MAX2(chip >= GFX10 ? def.bytes() : 4, instr_info.definition_size[(int)instr->opcode] / 8u);
651 }
652
653 } /* end namespace */
654
655 bool validate_ra(Program *program, const struct radv_nir_compiler_options *options) {
656 if (!(debug_flags & DEBUG_VALIDATE_RA))
657 return false;
658
659 bool err = false;
660 aco::live live_vars = aco::live_var_analysis(program, options);
661 std::vector<std::vector<Temp>> phi_sgpr_ops(program->blocks.size());
662
663 std::map<unsigned, Assignment> assignments;
664 for (Block& block : program->blocks) {
665 Location loc;
666 loc.block = &block;
667 for (aco_ptr<Instruction>& instr : block.instructions) {
668 if (instr->opcode == aco_opcode::p_phi) {
669 for (unsigned i = 0; i < instr->operands.size(); i++) {
670 if (instr->operands[i].isTemp() &&
671 instr->operands[i].getTemp().type() == RegType::sgpr &&
672 instr->operands[i].isFirstKill())
673 phi_sgpr_ops[block.logical_preds[i]].emplace_back(instr->operands[i].getTemp());
674 }
675 }
676
677 loc.instr = instr.get();
678 for (unsigned i = 0; i < instr->operands.size(); i++) {
679 Operand& op = instr->operands[i];
680 if (!op.isTemp())
681 continue;
682 if (!op.isFixed())
683 err |= ra_fail(program, loc, Location(), "Operand %d is not assigned a register", i);
684 if (assignments.count(op.tempId()) && assignments[op.tempId()].reg != op.physReg())
685 err |= ra_fail(program, loc, assignments.at(op.tempId()).firstloc, "Operand %d has an inconsistent register assignment with instruction", i);
686 if ((op.getTemp().type() == RegType::vgpr && op.physReg().reg_b + op.bytes() > (256 + program->config->num_vgprs) * 4) ||
687 (op.getTemp().type() == RegType::sgpr && op.physReg() + op.size() > program->config->num_sgprs && op.physReg() < program->sgpr_limit))
688 err |= ra_fail(program, loc, assignments.at(op.tempId()).firstloc, "Operand %d has an out-of-bounds register assignment", i);
689 if (op.physReg() == vcc && !program->needs_vcc)
690 err |= ra_fail(program, loc, Location(), "Operand %d fixed to vcc but needs_vcc=false", i);
691 if (op.regClass().is_subdword() && !validate_subdword_operand(program->chip_class, instr, i))
692 err |= ra_fail(program, loc, Location(), "Operand %d not aligned correctly", i);
693 if (!assignments[op.tempId()].firstloc.block)
694 assignments[op.tempId()].firstloc = loc;
695 if (!assignments[op.tempId()].defloc.block)
696 assignments[op.tempId()].reg = op.physReg();
697 }
698
699 for (unsigned i = 0; i < instr->definitions.size(); i++) {
700 Definition& def = instr->definitions[i];
701 if (!def.isTemp())
702 continue;
703 if (!def.isFixed())
704 err |= ra_fail(program, loc, Location(), "Definition %d is not assigned a register", i);
705 if (assignments[def.tempId()].defloc.block)
706 err |= ra_fail(program, loc, assignments.at(def.tempId()).defloc, "Temporary %%%d also defined by instruction", def.tempId());
707 if ((def.getTemp().type() == RegType::vgpr && def.physReg().reg_b + def.bytes() > (256 + program->config->num_vgprs) * 4) ||
708 (def.getTemp().type() == RegType::sgpr && def.physReg() + def.size() > program->config->num_sgprs && def.physReg() < program->sgpr_limit))
709 err |= ra_fail(program, loc, assignments.at(def.tempId()).firstloc, "Definition %d has an out-of-bounds register assignment", i);
710 if (def.physReg() == vcc && !program->needs_vcc)
711 err |= ra_fail(program, loc, Location(), "Definition %d fixed to vcc but needs_vcc=false", i);
712 if (def.regClass().is_subdword() && !validate_subdword_definition(program->chip_class, instr))
713 err |= ra_fail(program, loc, Location(), "Definition %d not aligned correctly", i);
714 if (!assignments[def.tempId()].firstloc.block)
715 assignments[def.tempId()].firstloc = loc;
716 assignments[def.tempId()].defloc = loc;
717 assignments[def.tempId()].reg = def.physReg();
718 }
719 }
720 }
721
722 for (Block& block : program->blocks) {
723 Location loc;
724 loc.block = &block;
725
726 std::array<unsigned, 2048> regs; /* register file in bytes */
727 regs.fill(0);
728
729 std::set<Temp> live;
730 live.insert(live_vars.live_out[block.index].begin(), live_vars.live_out[block.index].end());
731 /* remove killed p_phi sgpr operands */
732 for (Temp tmp : phi_sgpr_ops[block.index])
733 live.erase(tmp);
734
735 /* check live out */
736 for (Temp tmp : live) {
737 PhysReg reg = assignments.at(tmp.id()).reg;
738 for (unsigned i = 0; i < tmp.bytes(); i++) {
739 if (regs[reg.reg_b + i]) {
740 err |= ra_fail(program, loc, Location(), "Assignment of element %d of %%%d already taken by %%%d in live-out", i, tmp.id(), regs[reg.reg_b + i]);
741 }
742 regs[reg.reg_b + i] = tmp.id();
743 }
744 }
745 regs.fill(0);
746
747 for (auto it = block.instructions.rbegin(); it != block.instructions.rend(); ++it) {
748 aco_ptr<Instruction>& instr = *it;
749
750 /* check killed p_phi sgpr operands */
751 if (instr->opcode == aco_opcode::p_logical_end) {
752 for (Temp tmp : phi_sgpr_ops[block.index]) {
753 PhysReg reg = assignments.at(tmp.id()).reg;
754 for (unsigned i = 0; i < tmp.bytes(); i++) {
755 if (regs[reg.reg_b + i])
756 err |= ra_fail(program, loc, Location(), "Assignment of element %d of %%%d already taken by %%%d in live-out", i, tmp.id(), regs[reg.reg_b + i]);
757 }
758 live.emplace(tmp);
759 }
760 }
761
762 for (const Definition& def : instr->definitions) {
763 if (!def.isTemp())
764 continue;
765 live.erase(def.getTemp());
766 }
767
768 /* don't count phi operands as live-in, since they are actually
769 * killed when they are copied at the predecessor */
770 if (instr->opcode != aco_opcode::p_phi && instr->opcode != aco_opcode::p_linear_phi) {
771 for (const Operand& op : instr->operands) {
772 if (!op.isTemp())
773 continue;
774 live.insert(op.getTemp());
775 }
776 }
777 }
778
779 for (Temp tmp : live) {
780 PhysReg reg = assignments.at(tmp.id()).reg;
781 for (unsigned i = 0; i < tmp.bytes(); i++)
782 regs[reg.reg_b + i] = tmp.id();
783 }
784
785 for (aco_ptr<Instruction>& instr : block.instructions) {
786 loc.instr = instr.get();
787
788 /* remove killed p_phi operands from regs */
789 if (instr->opcode == aco_opcode::p_logical_end) {
790 for (Temp tmp : phi_sgpr_ops[block.index]) {
791 PhysReg reg = assignments.at(tmp.id()).reg;
792 for (unsigned i = 0; i < tmp.bytes(); i++)
793 regs[reg.reg_b + i] = 0;
794 }
795 }
796
797 if (instr->opcode != aco_opcode::p_phi && instr->opcode != aco_opcode::p_linear_phi) {
798 for (const Operand& op : instr->operands) {
799 if (!op.isTemp())
800 continue;
801 if (op.isFirstKillBeforeDef()) {
802 for (unsigned j = 0; j < op.getTemp().bytes(); j++)
803 regs[op.physReg().reg_b + j] = 0;
804 }
805 }
806 }
807
808 for (unsigned i = 0; i < instr->definitions.size(); i++) {
809 Definition& def = instr->definitions[i];
810 if (!def.isTemp())
811 continue;
812 Temp tmp = def.getTemp();
813 PhysReg reg = assignments.at(tmp.id()).reg;
814 for (unsigned j = 0; j < tmp.bytes(); j++) {
815 if (regs[reg.reg_b + j])
816 err |= ra_fail(program, loc, assignments.at(regs[reg.reg_b + j]).defloc, "Assignment of element %d of %%%d already taken by %%%d from instruction", i, tmp.id(), regs[reg.reg_b + j]);
817 regs[reg.reg_b + j] = tmp.id();
818 }
819 if (def.regClass().is_subdword() && def.bytes() < 4) {
820 unsigned written = get_subdword_bytes_written(program, instr, i);
821 /* If written=4, the instruction still might write the upper half. In that case, it's the lower half that isn't preserved */
822 for (unsigned j = reg.byte() & ~(written - 1); j < written; j++) {
823 unsigned written_reg = reg.reg() * 4u + j;
824 if (regs[written_reg] && regs[written_reg] != def.tempId())
825 err |= ra_fail(program, loc, assignments.at(regs[written_reg]).defloc, "Assignment of element %d of %%%d overwrites the full register taken by %%%d from instruction", i, tmp.id(), regs[written_reg]);
826 }
827 }
828 }
829
830 for (const Definition& def : instr->definitions) {
831 if (!def.isTemp())
832 continue;
833 if (def.isKill()) {
834 for (unsigned j = 0; j < def.getTemp().bytes(); j++)
835 regs[def.physReg().reg_b + j] = 0;
836 }
837 }
838
839 if (instr->opcode != aco_opcode::p_phi && instr->opcode != aco_opcode::p_linear_phi) {
840 for (const Operand& op : instr->operands) {
841 if (!op.isTemp())
842 continue;
843 if (op.isLateKill() && op.isFirstKill()) {
844 for (unsigned j = 0; j < op.getTemp().bytes(); j++)
845 regs[op.physReg().reg_b + j] = 0;
846 }
847 }
848 }
849 }
850 }
851
852 return err;
853 }
854 }