nir/opt_large_constants: Fix a type/deref_type typo
[mesa.git] / src / amd / compiler / aco_validate.cpp
1 /*
2 * Copyright © 2018 Valve Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 *
23 */
24
25 #include "aco_ir.h"
26
27 #include <array>
28 #include <map>
29
30 namespace aco {
31
32 static void aco_log(Program *program, enum radv_compiler_debug_level level,
33 const char *prefix, const char *file, unsigned line,
34 const char *fmt, va_list args)
35 {
36 char *msg;
37
38 msg = ralloc_strdup(NULL, prefix);
39
40 ralloc_asprintf_append(&msg, " In file %s:%u\n", file, line);
41 ralloc_asprintf_append(&msg, " ");
42 ralloc_vasprintf_append(&msg, fmt, args);
43
44 if (program->debug.func)
45 program->debug.func(program->debug.private_data, level, msg);
46
47 fprintf(stderr, "%s\n", msg);
48
49 ralloc_free(msg);
50 }
51
52 void _aco_perfwarn(Program *program, const char *file, unsigned line,
53 const char *fmt, ...)
54 {
55 va_list args;
56
57 va_start(args, fmt);
58 aco_log(program, RADV_COMPILER_DEBUG_LEVEL_PERFWARN,
59 "ACO PERFWARN:\n", file, line, fmt, args);
60 va_end(args);
61 }
62
63 void _aco_err(Program *program, const char *file, unsigned line,
64 const char *fmt, ...)
65 {
66 va_list args;
67
68 va_start(args, fmt);
69 aco_log(program, RADV_COMPILER_DEBUG_LEVEL_ERROR,
70 "ACO ERROR:\n", file, line, fmt, args);
71 va_end(args);
72 }
73
74 bool validate_ir(Program* program)
75 {
76 bool is_valid = true;
77 auto check = [&program, &is_valid](bool check, const char * msg, aco::Instruction * instr) -> void {
78 if (!check) {
79 char *out;
80 size_t outsize;
81 FILE *memf = open_memstream(&out, &outsize);
82
83 fprintf(memf, "%s: ", msg);
84 aco_print_instr(instr, memf);
85 fclose(memf);
86
87 aco_err(program, out);
88 free(out);
89
90 is_valid = false;
91 }
92 };
93
94 auto check_block = [&program, &is_valid](bool check, const char * msg, aco::Block * block) -> void {
95 if (!check) {
96 aco_err(program, "%s: BB%u", msg, block->index);
97 is_valid = false;
98 }
99 };
100
101 for (Block& block : program->blocks) {
102 for (aco_ptr<Instruction>& instr : block.instructions) {
103
104 /* check base format */
105 Format base_format = instr->format;
106 base_format = (Format)((uint32_t)base_format & ~(uint32_t)Format::SDWA);
107 base_format = (Format)((uint32_t)base_format & ~(uint32_t)Format::DPP);
108 if ((uint32_t)base_format & (uint32_t)Format::VOP1)
109 base_format = Format::VOP1;
110 else if ((uint32_t)base_format & (uint32_t)Format::VOP2)
111 base_format = Format::VOP2;
112 else if ((uint32_t)base_format & (uint32_t)Format::VOPC)
113 base_format = Format::VOPC;
114 else if ((uint32_t)base_format & (uint32_t)Format::VINTRP) {
115 if (instr->opcode == aco_opcode::v_interp_p1ll_f16 ||
116 instr->opcode == aco_opcode::v_interp_p1lv_f16 ||
117 instr->opcode == aco_opcode::v_interp_p2_legacy_f16 ||
118 instr->opcode == aco_opcode::v_interp_p2_f16) {
119 /* v_interp_*_fp16 are considered VINTRP by the compiler but
120 * they are emitted as VOP3.
121 */
122 base_format = Format::VOP3;
123 } else {
124 base_format = Format::VINTRP;
125 }
126 }
127 check(base_format == instr_info.format[(int)instr->opcode], "Wrong base format for instruction", instr.get());
128
129 /* check VOP3 modifiers */
130 if (((uint32_t)instr->format & (uint32_t)Format::VOP3) && instr->format != Format::VOP3) {
131 check(base_format == Format::VOP2 ||
132 base_format == Format::VOP1 ||
133 base_format == Format::VOPC ||
134 base_format == Format::VINTRP,
135 "Format cannot have VOP3A/VOP3B applied", instr.get());
136 }
137
138 /* check SDWA */
139 if (instr->isSDWA()) {
140 check(base_format == Format::VOP2 ||
141 base_format == Format::VOP1 ||
142 base_format == Format::VOPC,
143 "Format cannot have SDWA applied", instr.get());
144
145 check(program->chip_class >= GFX8, "SDWA is GFX8+ only", instr.get());
146
147 SDWA_instruction *sdwa = static_cast<SDWA_instruction*>(instr.get());
148 check(sdwa->omod == 0 || program->chip_class >= GFX9, "SDWA omod only supported on GFX9+", instr.get());
149 if (base_format == Format::VOPC) {
150 check(sdwa->clamp == false || program->chip_class == GFX8, "SDWA VOPC clamp only supported on GFX8", instr.get());
151 check((instr->definitions[0].isFixed() && instr->definitions[0].physReg() == vcc) ||
152 program->chip_class >= GFX9,
153 "SDWA+VOPC definition must be fixed to vcc on GFX8", instr.get());
154 }
155
156 if (instr->operands.size() >= 3) {
157 check(instr->operands[2].isFixed() && instr->operands[2].physReg() == vcc,
158 "3rd operand must be fixed to vcc with SDWA", instr.get());
159 }
160 if (instr->definitions.size() >= 2) {
161 check(instr->definitions[1].isFixed() && instr->definitions[1].physReg() == vcc,
162 "2nd definition must be fixed to vcc with SDWA", instr.get());
163 }
164
165 check(instr->opcode != aco_opcode::v_madmk_f32 &&
166 instr->opcode != aco_opcode::v_madak_f32 &&
167 instr->opcode != aco_opcode::v_madmk_f16 &&
168 instr->opcode != aco_opcode::v_madak_f16 &&
169 instr->opcode != aco_opcode::v_readfirstlane_b32 &&
170 instr->opcode != aco_opcode::v_clrexcp &&
171 instr->opcode != aco_opcode::v_swap_b32,
172 "SDWA can't be used with this opcode", instr.get());
173 if (program->chip_class != GFX8) {
174 check(instr->opcode != aco_opcode::v_mac_f32 &&
175 instr->opcode != aco_opcode::v_mac_f16 &&
176 instr->opcode != aco_opcode::v_fmac_f32 &&
177 instr->opcode != aco_opcode::v_fmac_f16,
178 "SDWA can't be used with this opcode", instr.get());
179 }
180
181 for (unsigned i = 0; i < MIN2(instr->operands.size(), 2); i++) {
182 if (instr->operands[i].hasRegClass() && instr->operands[i].regClass().is_subdword())
183 check((sdwa->sel[i] & sdwa_asuint) == (sdwa_isra | instr->operands[i].bytes()), "Unexpected SDWA sel for sub-dword operand", instr.get());
184 }
185 if (instr->definitions[0].regClass().is_subdword())
186 check((sdwa->dst_sel & sdwa_asuint) == (sdwa_isra | instr->definitions[0].bytes()), "Unexpected SDWA sel for sub-dword definition", instr.get());
187 }
188
189 /* check opsel */
190 if (instr->isVOP3()) {
191 VOP3A_instruction *vop3 = static_cast<VOP3A_instruction*>(instr.get());
192 check(vop3->opsel == 0 || program->chip_class >= GFX9, "Opsel is only supported on GFX9+", instr.get());
193
194 for (unsigned i = 0; i < 3; i++) {
195 if (i >= instr->operands.size() ||
196 (instr->operands[i].hasRegClass() && instr->operands[i].regClass().is_subdword() && !instr->operands[i].isFixed()))
197 check((vop3->opsel & (1 << i)) == 0, "Unexpected opsel for operand", instr.get());
198 }
199 if (instr->definitions[0].regClass().is_subdword() && !instr->definitions[0].isFixed())
200 check((vop3->opsel & (1 << 3)) == 0, "Unexpected opsel for sub-dword definition", instr.get());
201 }
202
203 /* check for undefs */
204 for (unsigned i = 0; i < instr->operands.size(); i++) {
205 if (instr->operands[i].isUndefined()) {
206 bool flat = instr->format == Format::FLAT || instr->format == Format::SCRATCH || instr->format == Format::GLOBAL;
207 bool can_be_undef = is_phi(instr) || instr->format == Format::EXP ||
208 instr->format == Format::PSEUDO_REDUCTION ||
209 instr->opcode == aco_opcode::p_create_vector ||
210 (flat && i == 1) || (instr->format == Format::MIMG && i == 1) ||
211 ((instr->format == Format::MUBUF || instr->format == Format::MTBUF) && i == 1);
212 check(can_be_undef, "Undefs can only be used in certain operands", instr.get());
213 } else {
214 check(instr->operands[i].isFixed() || instr->operands[i].isTemp() || instr->operands[i].isConstant(), "Uninitialized Operand", instr.get());
215 }
216 }
217
218 /* check subdword definitions */
219 for (unsigned i = 0; i < instr->definitions.size(); i++) {
220 if (instr->definitions[i].regClass().is_subdword())
221 check(instr->format == Format::PSEUDO || instr->definitions[i].bytes() <= 4, "Only Pseudo instructions can write subdword registers larger than 4 bytes", instr.get());
222 }
223
224 if (instr->isSALU() || instr->isVALU()) {
225 /* check literals */
226 Operand literal(s1);
227 for (unsigned i = 0; i < instr->operands.size(); i++)
228 {
229 Operand op = instr->operands[i];
230 if (!op.isLiteral())
231 continue;
232
233 check(instr->format == Format::SOP1 ||
234 instr->format == Format::SOP2 ||
235 instr->format == Format::SOPC ||
236 instr->format == Format::VOP1 ||
237 instr->format == Format::VOP2 ||
238 instr->format == Format::VOPC ||
239 (instr->isVOP3() && program->chip_class >= GFX10),
240 "Literal applied on wrong instruction format", instr.get());
241
242 check(literal.isUndefined() || (literal.size() == op.size() && literal.constantValue() == op.constantValue()), "Only 1 Literal allowed", instr.get());
243 literal = op;
244 check(!instr->isVALU() || instr->isVOP3() || i == 0 || i == 2, "Wrong source position for Literal argument", instr.get());
245 }
246
247 /* check num sgprs for VALU */
248 if (instr->isVALU()) {
249 bool is_shift64 = instr->opcode == aco_opcode::v_lshlrev_b64 ||
250 instr->opcode == aco_opcode::v_lshrrev_b64 ||
251 instr->opcode == aco_opcode::v_ashrrev_i64;
252 unsigned const_bus_limit = 1;
253 if (program->chip_class >= GFX10 && !is_shift64)
254 const_bus_limit = 2;
255
256 uint32_t scalar_mask = instr->isVOP3() ? 0x7 : 0x5;
257 if (instr->isSDWA())
258 scalar_mask = program->chip_class >= GFX9 ? 0x7 : 0x4;
259
260 if ((int) instr->format & (int) Format::VOPC ||
261 instr->opcode == aco_opcode::v_readfirstlane_b32 ||
262 instr->opcode == aco_opcode::v_readlane_b32 ||
263 instr->opcode == aco_opcode::v_readlane_b32_e64) {
264 check(instr->definitions[0].getTemp().type() == RegType::sgpr,
265 "Wrong Definition type for VALU instruction", instr.get());
266 } else {
267 check(instr->definitions[0].getTemp().type() == RegType::vgpr,
268 "Wrong Definition type for VALU instruction", instr.get());
269 }
270
271 unsigned num_sgprs = 0;
272 unsigned sgpr[] = {0, 0};
273 for (unsigned i = 0; i < instr->operands.size(); i++)
274 {
275 Operand op = instr->operands[i];
276 if (instr->opcode == aco_opcode::v_readfirstlane_b32 ||
277 instr->opcode == aco_opcode::v_readlane_b32 ||
278 instr->opcode == aco_opcode::v_readlane_b32_e64) {
279 check(i != 1 ||
280 (op.isTemp() && op.regClass().type() == RegType::sgpr) ||
281 op.isConstant(),
282 "Must be a SGPR or a constant", instr.get());
283 check(i == 1 ||
284 (op.isTemp() && op.regClass().type() == RegType::vgpr && op.bytes() <= 4),
285 "Wrong Operand type for VALU instruction", instr.get());
286 continue;
287 }
288
289 if (instr->opcode == aco_opcode::v_writelane_b32 ||
290 instr->opcode == aco_opcode::v_writelane_b32_e64) {
291 check(i != 2 ||
292 (op.isTemp() && op.regClass().type() == RegType::vgpr && op.bytes() <= 4),
293 "Wrong Operand type for VALU instruction", instr.get());
294 check(i == 2 ||
295 (op.isTemp() && op.regClass().type() == RegType::sgpr) ||
296 op.isConstant(),
297 "Must be a SGPR or a constant", instr.get());
298 continue;
299 }
300 if (op.isTemp() && instr->operands[i].regClass().type() == RegType::sgpr) {
301 check(scalar_mask & (1 << i), "Wrong source position for SGPR argument", instr.get());
302
303 if (op.tempId() != sgpr[0] && op.tempId() != sgpr[1]) {
304 if (num_sgprs < 2)
305 sgpr[num_sgprs++] = op.tempId();
306 }
307 }
308
309 if (op.isConstant() && !op.isLiteral())
310 check(scalar_mask & (1 << i), "Wrong source position for constant argument", instr.get());
311 }
312 check(num_sgprs + (literal.isUndefined() ? 0 : 1) <= const_bus_limit, "Too many SGPRs/literals", instr.get());
313 }
314
315 if (instr->format == Format::SOP1 || instr->format == Format::SOP2) {
316 check(instr->definitions[0].getTemp().type() == RegType::sgpr, "Wrong Definition type for SALU instruction", instr.get());
317 for (const Operand& op : instr->operands) {
318 check(op.isConstant() || op.regClass().type() <= RegType::sgpr,
319 "Wrong Operand type for SALU instruction", instr.get());
320 }
321 }
322 }
323
324 switch (instr->format) {
325 case Format::PSEUDO: {
326 bool is_subdword = false;
327 bool has_const_sgpr = false;
328 bool has_literal = false;
329 for (Definition def : instr->definitions)
330 is_subdword |= def.regClass().is_subdword();
331 for (unsigned i = 0; i < instr->operands.size(); i++) {
332 if (instr->opcode == aco_opcode::p_extract_vector && i == 1)
333 continue;
334 Operand op = instr->operands[i];
335 is_subdword |= op.hasRegClass() && op.regClass().is_subdword();
336 has_const_sgpr |= op.isConstant() || (op.hasRegClass() && op.regClass().type() == RegType::sgpr);
337 has_literal |= op.isLiteral();
338 }
339
340 check(!is_subdword || !has_const_sgpr || program->chip_class >= GFX9,
341 "Sub-dword pseudo instructions can only take constants or SGPRs on GFX9+", instr.get());
342 check(!is_subdword || !has_literal, "Sub-dword pseudo instructions cannot take literals", instr.get());
343
344 if (instr->opcode == aco_opcode::p_create_vector) {
345 unsigned size = 0;
346 for (const Operand& op : instr->operands) {
347 size += op.bytes();
348 }
349 check(size == instr->definitions[0].bytes(), "Definition size does not match operand sizes", instr.get());
350 if (instr->definitions[0].getTemp().type() == RegType::sgpr) {
351 for (const Operand& op : instr->operands) {
352 check(op.isConstant() || op.regClass().type() == RegType::sgpr,
353 "Wrong Operand type for scalar vector", instr.get());
354 }
355 }
356 } else if (instr->opcode == aco_opcode::p_extract_vector) {
357 check((instr->operands[0].isTemp()) && instr->operands[1].isConstant(), "Wrong Operand types", instr.get());
358 check((instr->operands[1].constantValue() + 1) * instr->definitions[0].bytes() <= instr->operands[0].bytes(), "Index out of range", instr.get());
359 check(instr->definitions[0].getTemp().type() == RegType::vgpr || instr->operands[0].regClass().type() == RegType::sgpr,
360 "Cannot extract SGPR value from VGPR vector", instr.get());
361 } else if (instr->opcode == aco_opcode::p_parallelcopy) {
362 check(instr->definitions.size() == instr->operands.size(), "Number of Operands does not match number of Definitions", instr.get());
363 for (unsigned i = 0; i < instr->operands.size(); i++) {
364 if (instr->operands[i].isTemp())
365 check((instr->definitions[i].getTemp().type() == instr->operands[i].regClass().type()) ||
366 (instr->definitions[i].getTemp().type() == RegType::vgpr && instr->operands[i].regClass().type() == RegType::sgpr),
367 "Operand and Definition types do not match", instr.get());
368 }
369 } else if (instr->opcode == aco_opcode::p_phi) {
370 check(instr->operands.size() == block.logical_preds.size(), "Number of Operands does not match number of predecessors", instr.get());
371 check(instr->definitions[0].getTemp().type() == RegType::vgpr, "Logical Phi Definition must be vgpr", instr.get());
372 } else if (instr->opcode == aco_opcode::p_linear_phi) {
373 for (const Operand& op : instr->operands)
374 check(!op.isTemp() || op.getTemp().is_linear(), "Wrong Operand type", instr.get());
375 check(instr->operands.size() == block.linear_preds.size(), "Number of Operands does not match number of predecessors", instr.get());
376 }
377 break;
378 }
379 case Format::SMEM: {
380 if (instr->operands.size() >= 1)
381 check(instr->operands[0].isTemp() && instr->operands[0].regClass().type() == RegType::sgpr, "SMEM operands must be sgpr", instr.get());
382 if (instr->operands.size() >= 2)
383 check(instr->operands[1].isConstant() || (instr->operands[1].isTemp() && instr->operands[1].regClass().type() == RegType::sgpr),
384 "SMEM offset must be constant or sgpr", instr.get());
385 if (!instr->definitions.empty())
386 check(instr->definitions[0].getTemp().type() == RegType::sgpr, "SMEM result must be sgpr", instr.get());
387 break;
388 }
389 case Format::MTBUF:
390 case Format::MUBUF: {
391 check(instr->operands.size() > 1, "VMEM instructions must have at least one operand", instr.get());
392 check(instr->operands[1].hasRegClass() && instr->operands[1].regClass().type() == RegType::vgpr,
393 "VADDR must be in vgpr for VMEM instructions", instr.get());
394 check(instr->operands[0].isTemp() && instr->operands[0].regClass().type() == RegType::sgpr, "VMEM resource constant must be sgpr", instr.get());
395 check(instr->operands.size() < 4 || (instr->operands[3].isTemp() && instr->operands[3].regClass().type() == RegType::vgpr), "VMEM write data must be vgpr", instr.get());
396 break;
397 }
398 case Format::MIMG: {
399 check(instr->operands.size() == 3, "MIMG instructions must have exactly 3 operands", instr.get());
400 check(instr->operands[0].hasRegClass() && (instr->operands[0].regClass() == s4 || instr->operands[0].regClass() == s8),
401 "MIMG operands[0] (resource constant) must be in 4 or 8 SGPRs", instr.get());
402 if (instr->operands[1].hasRegClass() && instr->operands[1].regClass().type() == RegType::sgpr)
403 check(instr->operands[1].regClass() == s4, "MIMG operands[1] (sampler constant) must be 4 SGPRs", instr.get());
404 else if (instr->operands[1].hasRegClass() && instr->operands[1].regClass().type() == RegType::vgpr)
405 check((instr->definitions.empty() || instr->definitions[0].regClass() == instr->operands[1].regClass() ||
406 instr->opcode == aco_opcode::image_atomic_cmpswap || instr->opcode == aco_opcode::image_atomic_fcmpswap),
407 "MIMG operands[1] (VDATA) must be the same as definitions[0] for atomics", instr.get());
408 check(instr->operands[2].hasRegClass() && instr->operands[2].regClass().type() == RegType::vgpr,
409 "MIMG operands[2] (VADDR) must be VGPR", instr.get());
410 check(instr->definitions.empty() || (instr->definitions[0].isTemp() && instr->definitions[0].regClass().type() == RegType::vgpr),
411 "MIMG definitions[0] (VDATA) must be VGPR", instr.get());
412 break;
413 }
414 case Format::DS: {
415 for (const Operand& op : instr->operands) {
416 check((op.isTemp() && op.regClass().type() == RegType::vgpr) || op.physReg() == m0,
417 "Only VGPRs are valid DS instruction operands", instr.get());
418 }
419 if (!instr->definitions.empty())
420 check(instr->definitions[0].getTemp().type() == RegType::vgpr, "DS instruction must return VGPR", instr.get());
421 break;
422 }
423 case Format::EXP: {
424 for (unsigned i = 0; i < 4; i++)
425 check(instr->operands[i].hasRegClass() && instr->operands[i].regClass().type() == RegType::vgpr,
426 "Only VGPRs are valid Export arguments", instr.get());
427 break;
428 }
429 case Format::FLAT:
430 check(instr->operands[1].isUndefined(), "Flat instructions don't support SADDR", instr.get());
431 /* fallthrough */
432 case Format::GLOBAL:
433 case Format::SCRATCH: {
434 check(instr->operands[0].isTemp() && instr->operands[0].regClass().type() == RegType::vgpr, "FLAT/GLOBAL/SCRATCH address must be vgpr", instr.get());
435 check(instr->operands[1].hasRegClass() && instr->operands[1].regClass().type() == RegType::sgpr,
436 "FLAT/GLOBAL/SCRATCH sgpr address must be undefined or sgpr", instr.get());
437 if (!instr->definitions.empty())
438 check(instr->definitions[0].getTemp().type() == RegType::vgpr, "FLAT/GLOBAL/SCRATCH result must be vgpr", instr.get());
439 else
440 check(instr->operands[2].regClass().type() == RegType::vgpr, "FLAT/GLOBAL/SCRATCH data must be vgpr", instr.get());
441 break;
442 }
443 default:
444 break;
445 }
446 }
447 }
448
449 /* validate CFG */
450 for (unsigned i = 0; i < program->blocks.size(); i++) {
451 Block& block = program->blocks[i];
452 check_block(block.index == i, "block.index must match actual index", &block);
453
454 /* predecessors/successors should be sorted */
455 for (unsigned j = 0; j + 1 < block.linear_preds.size(); j++)
456 check_block(block.linear_preds[j] < block.linear_preds[j + 1], "linear predecessors must be sorted", &block);
457 for (unsigned j = 0; j + 1 < block.logical_preds.size(); j++)
458 check_block(block.logical_preds[j] < block.logical_preds[j + 1], "logical predecessors must be sorted", &block);
459 for (unsigned j = 0; j + 1 < block.linear_succs.size(); j++)
460 check_block(block.linear_succs[j] < block.linear_succs[j + 1], "linear successors must be sorted", &block);
461 for (unsigned j = 0; j + 1 < block.logical_succs.size(); j++)
462 check_block(block.logical_succs[j] < block.logical_succs[j + 1], "logical successors must be sorted", &block);
463
464 /* critical edges are not allowed */
465 if (block.linear_preds.size() > 1) {
466 for (unsigned pred : block.linear_preds)
467 check_block(program->blocks[pred].linear_succs.size() == 1, "linear critical edges are not allowed", &program->blocks[pred]);
468 for (unsigned pred : block.logical_preds)
469 check_block(program->blocks[pred].logical_succs.size() == 1, "logical critical edges are not allowed", &program->blocks[pred]);
470 }
471 }
472
473 return is_valid;
474 }
475
476 /* RA validation */
477 namespace {
478
479 struct Location {
480 Location() : block(NULL), instr(NULL) {}
481
482 Block *block;
483 Instruction *instr; //NULL if it's the block's live-in
484 };
485
486 struct Assignment {
487 Location defloc;
488 Location firstloc;
489 PhysReg reg;
490 };
491
492 bool ra_fail(Program *program, Location loc, Location loc2, const char *fmt, ...) {
493 va_list args;
494 va_start(args, fmt);
495 char msg[1024];
496 vsprintf(msg, fmt, args);
497 va_end(args);
498
499 char *out;
500 size_t outsize;
501 FILE *memf = open_memstream(&out, &outsize);
502
503 fprintf(memf, "RA error found at instruction in BB%d:\n", loc.block->index);
504 if (loc.instr) {
505 aco_print_instr(loc.instr, memf);
506 fprintf(memf, "\n%s", msg);
507 } else {
508 fprintf(memf, "%s", msg);
509 }
510 if (loc2.block) {
511 fprintf(memf, " in BB%d:\n", loc2.block->index);
512 aco_print_instr(loc2.instr, memf);
513 }
514 fprintf(memf, "\n\n");
515
516 aco_err(program, out);
517 free(out);
518
519 return true;
520 }
521
522 bool validate_subdword_operand(chip_class chip, const aco_ptr<Instruction>& instr, unsigned index)
523 {
524 Operand op = instr->operands[index];
525 unsigned byte = op.physReg().byte();
526
527 if (instr->opcode == aco_opcode::p_as_uniform)
528 return byte == 0;
529 if (instr->format == Format::PSEUDO && chip >= GFX8)
530 return true;
531 if (instr->isSDWA() && (static_cast<SDWA_instruction *>(instr.get())->sel[index] & sdwa_asuint) == (sdwa_isra | op.bytes()))
532 return true;
533 if (byte == 2 && can_use_opsel(chip, instr->opcode, index, 1))
534 return true;
535
536 switch (instr->opcode) {
537 case aco_opcode::v_cvt_f32_ubyte1:
538 if (byte == 1)
539 return true;
540 break;
541 case aco_opcode::v_cvt_f32_ubyte2:
542 if (byte == 2)
543 return true;
544 break;
545 case aco_opcode::v_cvt_f32_ubyte3:
546 if (byte == 3)
547 return true;
548 break;
549 case aco_opcode::ds_write_b8_d16_hi:
550 case aco_opcode::ds_write_b16_d16_hi:
551 if (byte == 2 && index == 1)
552 return true;
553 break;
554 case aco_opcode::buffer_store_byte_d16_hi:
555 case aco_opcode::buffer_store_short_d16_hi:
556 if (byte == 2 && index == 3)
557 return true;
558 break;
559 case aco_opcode::flat_store_byte_d16_hi:
560 case aco_opcode::flat_store_short_d16_hi:
561 case aco_opcode::scratch_store_byte_d16_hi:
562 case aco_opcode::scratch_store_short_d16_hi:
563 case aco_opcode::global_store_byte_d16_hi:
564 case aco_opcode::global_store_short_d16_hi:
565 if (byte == 2 && index == 2)
566 return true;
567 default:
568 break;
569 }
570
571 return byte == 0;
572 }
573
574 bool validate_subdword_definition(chip_class chip, const aco_ptr<Instruction>& instr)
575 {
576 Definition def = instr->definitions[0];
577 unsigned byte = def.physReg().byte();
578
579 if (instr->format == Format::PSEUDO && chip >= GFX8)
580 return true;
581 if (instr->isSDWA() && static_cast<SDWA_instruction *>(instr.get())->dst_sel == (sdwa_isra | def.bytes()))
582 return true;
583 if (byte == 2 && can_use_opsel(chip, instr->opcode, -1, 1))
584 return true;
585
586 switch (instr->opcode) {
587 case aco_opcode::buffer_load_ubyte_d16_hi:
588 case aco_opcode::buffer_load_short_d16_hi:
589 case aco_opcode::flat_load_ubyte_d16_hi:
590 case aco_opcode::flat_load_short_d16_hi:
591 case aco_opcode::scratch_load_ubyte_d16_hi:
592 case aco_opcode::scratch_load_short_d16_hi:
593 case aco_opcode::global_load_ubyte_d16_hi:
594 case aco_opcode::global_load_short_d16_hi:
595 case aco_opcode::ds_read_u8_d16_hi:
596 case aco_opcode::ds_read_u16_d16_hi:
597 return byte == 2;
598 default:
599 break;
600 }
601
602 return byte == 0;
603 }
604
605 unsigned get_subdword_bytes_written(Program *program, const aco_ptr<Instruction>& instr, unsigned index)
606 {
607 chip_class chip = program->chip_class;
608 Definition def = instr->definitions[index];
609
610 if (instr->format == Format::PSEUDO)
611 return chip >= GFX8 ? def.bytes() : def.size() * 4u;
612 if (instr->isSDWA() && static_cast<SDWA_instruction *>(instr.get())->dst_sel == (sdwa_isra | def.bytes()))
613 return def.bytes();
614
615 switch (instr->opcode) {
616 case aco_opcode::buffer_load_ubyte_d16:
617 case aco_opcode::buffer_load_short_d16:
618 case aco_opcode::flat_load_ubyte_d16:
619 case aco_opcode::flat_load_short_d16:
620 case aco_opcode::scratch_load_ubyte_d16:
621 case aco_opcode::scratch_load_short_d16:
622 case aco_opcode::global_load_ubyte_d16:
623 case aco_opcode::global_load_short_d16:
624 case aco_opcode::ds_read_u8_d16:
625 case aco_opcode::ds_read_u16_d16:
626 case aco_opcode::buffer_load_ubyte_d16_hi:
627 case aco_opcode::buffer_load_short_d16_hi:
628 case aco_opcode::flat_load_ubyte_d16_hi:
629 case aco_opcode::flat_load_short_d16_hi:
630 case aco_opcode::scratch_load_ubyte_d16_hi:
631 case aco_opcode::scratch_load_short_d16_hi:
632 case aco_opcode::global_load_ubyte_d16_hi:
633 case aco_opcode::global_load_short_d16_hi:
634 case aco_opcode::ds_read_u8_d16_hi:
635 case aco_opcode::ds_read_u16_d16_hi:
636 return program->sram_ecc_enabled ? 4 : 2;
637 case aco_opcode::v_mad_f16:
638 case aco_opcode::v_mad_u16:
639 case aco_opcode::v_mad_i16:
640 case aco_opcode::v_fma_f16:
641 case aco_opcode::v_div_fixup_f16:
642 case aco_opcode::v_interp_p2_f16:
643 if (chip >= GFX9)
644 return 2;
645 default:
646 break;
647 }
648
649 return MAX2(chip >= GFX10 ? def.bytes() : 4, instr_info.definition_size[(int)instr->opcode] / 8u);
650 }
651
652 } /* end namespace */
653
654 bool validate_ra(Program *program, const struct radv_nir_compiler_options *options) {
655 if (!(debug_flags & DEBUG_VALIDATE_RA))
656 return false;
657
658 bool err = false;
659 aco::live live_vars = aco::live_var_analysis(program, options);
660 std::vector<std::vector<Temp>> phi_sgpr_ops(program->blocks.size());
661
662 std::map<unsigned, Assignment> assignments;
663 for (Block& block : program->blocks) {
664 Location loc;
665 loc.block = &block;
666 for (aco_ptr<Instruction>& instr : block.instructions) {
667 if (instr->opcode == aco_opcode::p_phi) {
668 for (unsigned i = 0; i < instr->operands.size(); i++) {
669 if (instr->operands[i].isTemp() &&
670 instr->operands[i].getTemp().type() == RegType::sgpr &&
671 instr->operands[i].isFirstKill())
672 phi_sgpr_ops[block.logical_preds[i]].emplace_back(instr->operands[i].getTemp());
673 }
674 }
675
676 loc.instr = instr.get();
677 for (unsigned i = 0; i < instr->operands.size(); i++) {
678 Operand& op = instr->operands[i];
679 if (!op.isTemp())
680 continue;
681 if (!op.isFixed())
682 err |= ra_fail(program, loc, Location(), "Operand %d is not assigned a register", i);
683 if (assignments.count(op.tempId()) && assignments[op.tempId()].reg != op.physReg())
684 err |= ra_fail(program, loc, assignments.at(op.tempId()).firstloc, "Operand %d has an inconsistent register assignment with instruction", i);
685 if ((op.getTemp().type() == RegType::vgpr && op.physReg().reg_b + op.bytes() > (256 + program->config->num_vgprs) * 4) ||
686 (op.getTemp().type() == RegType::sgpr && op.physReg() + op.size() > program->config->num_sgprs && op.physReg() < program->sgpr_limit))
687 err |= ra_fail(program, loc, assignments.at(op.tempId()).firstloc, "Operand %d has an out-of-bounds register assignment", i);
688 if (op.physReg() == vcc && !program->needs_vcc)
689 err |= ra_fail(program, loc, Location(), "Operand %d fixed to vcc but needs_vcc=false", i);
690 if (op.regClass().is_subdword() && !validate_subdword_operand(program->chip_class, instr, i))
691 err |= ra_fail(program, loc, Location(), "Operand %d not aligned correctly", i);
692 if (!assignments[op.tempId()].firstloc.block)
693 assignments[op.tempId()].firstloc = loc;
694 if (!assignments[op.tempId()].defloc.block)
695 assignments[op.tempId()].reg = op.physReg();
696 }
697
698 for (unsigned i = 0; i < instr->definitions.size(); i++) {
699 Definition& def = instr->definitions[i];
700 if (!def.isTemp())
701 continue;
702 if (!def.isFixed())
703 err |= ra_fail(program, loc, Location(), "Definition %d is not assigned a register", i);
704 if (assignments[def.tempId()].defloc.block)
705 err |= ra_fail(program, loc, assignments.at(def.tempId()).defloc, "Temporary %%%d also defined by instruction", def.tempId());
706 if ((def.getTemp().type() == RegType::vgpr && def.physReg().reg_b + def.bytes() > (256 + program->config->num_vgprs) * 4) ||
707 (def.getTemp().type() == RegType::sgpr && def.physReg() + def.size() > program->config->num_sgprs && def.physReg() < program->sgpr_limit))
708 err |= ra_fail(program, loc, assignments.at(def.tempId()).firstloc, "Definition %d has an out-of-bounds register assignment", i);
709 if (def.physReg() == vcc && !program->needs_vcc)
710 err |= ra_fail(program, loc, Location(), "Definition %d fixed to vcc but needs_vcc=false", i);
711 if (def.regClass().is_subdword() && !validate_subdword_definition(program->chip_class, instr))
712 err |= ra_fail(program, loc, Location(), "Definition %d not aligned correctly", i);
713 if (!assignments[def.tempId()].firstloc.block)
714 assignments[def.tempId()].firstloc = loc;
715 assignments[def.tempId()].defloc = loc;
716 assignments[def.tempId()].reg = def.physReg();
717 }
718 }
719 }
720
721 for (Block& block : program->blocks) {
722 Location loc;
723 loc.block = &block;
724
725 std::array<unsigned, 2048> regs; /* register file in bytes */
726 regs.fill(0);
727
728 std::set<Temp> live;
729 live.insert(live_vars.live_out[block.index].begin(), live_vars.live_out[block.index].end());
730 /* remove killed p_phi sgpr operands */
731 for (Temp tmp : phi_sgpr_ops[block.index])
732 live.erase(tmp);
733
734 /* check live out */
735 for (Temp tmp : live) {
736 PhysReg reg = assignments.at(tmp.id()).reg;
737 for (unsigned i = 0; i < tmp.bytes(); i++) {
738 if (regs[reg.reg_b + i]) {
739 err |= ra_fail(program, loc, Location(), "Assignment of element %d of %%%d already taken by %%%d in live-out", i, tmp.id(), regs[reg.reg_b + i]);
740 }
741 regs[reg.reg_b + i] = tmp.id();
742 }
743 }
744 regs.fill(0);
745
746 for (auto it = block.instructions.rbegin(); it != block.instructions.rend(); ++it) {
747 aco_ptr<Instruction>& instr = *it;
748
749 /* check killed p_phi sgpr operands */
750 if (instr->opcode == aco_opcode::p_logical_end) {
751 for (Temp tmp : phi_sgpr_ops[block.index]) {
752 PhysReg reg = assignments.at(tmp.id()).reg;
753 for (unsigned i = 0; i < tmp.bytes(); i++) {
754 if (regs[reg.reg_b + i])
755 err |= ra_fail(program, loc, Location(), "Assignment of element %d of %%%d already taken by %%%d in live-out", i, tmp.id(), regs[reg.reg_b + i]);
756 }
757 live.emplace(tmp);
758 }
759 }
760
761 for (const Definition& def : instr->definitions) {
762 if (!def.isTemp())
763 continue;
764 live.erase(def.getTemp());
765 }
766
767 /* don't count phi operands as live-in, since they are actually
768 * killed when they are copied at the predecessor */
769 if (instr->opcode != aco_opcode::p_phi && instr->opcode != aco_opcode::p_linear_phi) {
770 for (const Operand& op : instr->operands) {
771 if (!op.isTemp())
772 continue;
773 live.insert(op.getTemp());
774 }
775 }
776 }
777
778 for (Temp tmp : live) {
779 PhysReg reg = assignments.at(tmp.id()).reg;
780 for (unsigned i = 0; i < tmp.bytes(); i++)
781 regs[reg.reg_b + i] = tmp.id();
782 }
783
784 for (aco_ptr<Instruction>& instr : block.instructions) {
785 loc.instr = instr.get();
786
787 /* remove killed p_phi operands from regs */
788 if (instr->opcode == aco_opcode::p_logical_end) {
789 for (Temp tmp : phi_sgpr_ops[block.index]) {
790 PhysReg reg = assignments.at(tmp.id()).reg;
791 for (unsigned i = 0; i < tmp.bytes(); i++)
792 regs[reg.reg_b + i] = 0;
793 }
794 }
795
796 if (instr->opcode != aco_opcode::p_phi && instr->opcode != aco_opcode::p_linear_phi) {
797 for (const Operand& op : instr->operands) {
798 if (!op.isTemp())
799 continue;
800 if (op.isFirstKillBeforeDef()) {
801 for (unsigned j = 0; j < op.getTemp().bytes(); j++)
802 regs[op.physReg().reg_b + j] = 0;
803 }
804 }
805 }
806
807 for (unsigned i = 0; i < instr->definitions.size(); i++) {
808 Definition& def = instr->definitions[i];
809 if (!def.isTemp())
810 continue;
811 Temp tmp = def.getTemp();
812 PhysReg reg = assignments.at(tmp.id()).reg;
813 for (unsigned j = 0; j < tmp.bytes(); j++) {
814 if (regs[reg.reg_b + j])
815 err |= ra_fail(program, loc, assignments.at(regs[reg.reg_b + j]).defloc, "Assignment of element %d of %%%d already taken by %%%d from instruction", i, tmp.id(), regs[reg.reg_b + j]);
816 regs[reg.reg_b + j] = tmp.id();
817 }
818 if (def.regClass().is_subdword() && def.bytes() < 4) {
819 unsigned written = get_subdword_bytes_written(program, instr, i);
820 /* If written=4, the instruction still might write the upper half. In that case, it's the lower half that isn't preserved */
821 for (unsigned j = reg.byte() & ~(written - 1); j < written; j++) {
822 unsigned written_reg = reg.reg() * 4u + j;
823 if (regs[written_reg] && regs[written_reg] != def.tempId())
824 err |= ra_fail(program, loc, assignments.at(regs[written_reg]).defloc, "Assignment of element %d of %%%d overwrites the full register taken by %%%d from instruction", i, tmp.id(), regs[written_reg]);
825 }
826 }
827 }
828
829 for (const Definition& def : instr->definitions) {
830 if (!def.isTemp())
831 continue;
832 if (def.isKill()) {
833 for (unsigned j = 0; j < def.getTemp().bytes(); j++)
834 regs[def.physReg().reg_b + j] = 0;
835 }
836 }
837
838 if (instr->opcode != aco_opcode::p_phi && instr->opcode != aco_opcode::p_linear_phi) {
839 for (const Operand& op : instr->operands) {
840 if (!op.isTemp())
841 continue;
842 if (op.isLateKill() && op.isFirstKill()) {
843 for (unsigned j = 0; j < op.getTemp().bytes(); j++)
844 regs[op.physReg().reg_b + j] = 0;
845 }
846 }
847 }
848 }
849 }
850
851 return err;
852 }
853 }