ac: add ac_build_readlane without optimization barrier
[mesa.git] / src / amd / compiler / aco_validate.cpp
1 /*
2 * Copyright © 2018 Valve Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 *
23 */
24
25 #include "aco_ir.h"
26
27 #include <array>
28 #include <map>
29
30 namespace aco {
31
32 #ifndef NDEBUG
33 void perfwarn(bool cond, const char *msg, Instruction *instr)
34 {
35 if (cond) {
36 fprintf(stderr, "ACO performance warning: %s\n", msg);
37 if (instr) {
38 fprintf(stderr, "instruction: ");
39 aco_print_instr(instr, stderr);
40 fprintf(stderr, "\n");
41 }
42
43 if (debug_flags & DEBUG_PERFWARN)
44 exit(1);
45 }
46 }
47 #endif
48
49 void validate(Program* program, FILE * output)
50 {
51 if (!(debug_flags & DEBUG_VALIDATE))
52 return;
53
54 bool is_valid = true;
55 auto check = [&output, &is_valid](bool check, const char * msg, aco::Instruction * instr) -> void {
56 if (!check) {
57 fprintf(output, "%s: ", msg);
58 aco_print_instr(instr, output);
59 fprintf(output, "\n");
60 is_valid = false;
61 }
62 };
63 auto check_block = [&output, &is_valid](bool check, const char * msg, aco::Block * block) -> void {
64 if (!check) {
65 fprintf(output, "%s: BB%u\n", msg, block->index);
66 is_valid = false;
67 }
68 };
69
70 for (Block& block : program->blocks) {
71 for (aco_ptr<Instruction>& instr : block.instructions) {
72
73 /* check base format */
74 Format base_format = instr->format;
75 base_format = (Format)((uint32_t)base_format & ~(uint32_t)Format::SDWA);
76 base_format = (Format)((uint32_t)base_format & ~(uint32_t)Format::DPP);
77 if ((uint32_t)base_format & (uint32_t)Format::VOP1)
78 base_format = Format::VOP1;
79 else if ((uint32_t)base_format & (uint32_t)Format::VOP2)
80 base_format = Format::VOP2;
81 else if ((uint32_t)base_format & (uint32_t)Format::VOPC)
82 base_format = Format::VOPC;
83 else if ((uint32_t)base_format & (uint32_t)Format::VINTRP)
84 base_format = Format::VINTRP;
85 check(base_format == instr_info.format[(int)instr->opcode], "Wrong base format for instruction", instr.get());
86
87 /* check VOP3 modifiers */
88 if (((uint32_t)instr->format & (uint32_t)Format::VOP3) && instr->format != Format::VOP3) {
89 check(base_format == Format::VOP2 ||
90 base_format == Format::VOP1 ||
91 base_format == Format::VOPC ||
92 base_format == Format::VINTRP,
93 "Format cannot have VOP3A/VOP3B applied", instr.get());
94 }
95
96 /* check for undefs */
97 for (unsigned i = 0; i < instr->operands.size(); i++) {
98 if (instr->operands[i].isUndefined()) {
99 bool flat = instr->format == Format::FLAT || instr->format == Format::SCRATCH || instr->format == Format::GLOBAL;
100 bool can_be_undef = is_phi(instr) || instr->format == Format::EXP ||
101 instr->format == Format::PSEUDO_REDUCTION ||
102 (flat && i == 1) || (instr->format == Format::MIMG && i == 2) ||
103 ((instr->format == Format::MUBUF || instr->format == Format::MTBUF) && i == 0);
104 check(can_be_undef, "Undefs can only be used in certain operands", instr.get());
105 }
106 }
107
108 if (instr->isSALU() || instr->isVALU()) {
109 /* check literals */
110 Operand literal(s1);
111 for (unsigned i = 0; i < instr->operands.size(); i++)
112 {
113 Operand op = instr->operands[i];
114 if (!op.isLiteral())
115 continue;
116
117 check(instr->format == Format::SOP1 ||
118 instr->format == Format::SOP2 ||
119 instr->format == Format::SOPC ||
120 instr->format == Format::VOP1 ||
121 instr->format == Format::VOP2 ||
122 instr->format == Format::VOPC ||
123 (instr->isVOP3() && program->chip_class >= GFX10),
124 "Literal applied on wrong instruction format", instr.get());
125
126 check(literal.isUndefined() || (literal.size() == op.size() && literal.constantValue() == op.constantValue()), "Only 1 Literal allowed", instr.get());
127 literal = op;
128 check(!instr->isVALU() || instr->isVOP3() || i == 0 || i == 2, "Wrong source position for Literal argument", instr.get());
129 }
130
131 /* check num sgprs for VALU */
132 if (instr->isVALU()) {
133 bool is_shift64 = instr->opcode == aco_opcode::v_lshlrev_b64 ||
134 instr->opcode == aco_opcode::v_lshrrev_b64 ||
135 instr->opcode == aco_opcode::v_ashrrev_i64;
136 unsigned const_bus_limit = 1;
137 if (program->chip_class >= GFX10 && !is_shift64)
138 const_bus_limit = 2;
139
140 check(instr->definitions[0].getTemp().type() == RegType::vgpr ||
141 (int) instr->format & (int) Format::VOPC ||
142 instr->opcode == aco_opcode::v_readfirstlane_b32 ||
143 instr->opcode == aco_opcode::v_readlane_b32 ||
144 instr->opcode == aco_opcode::v_readlane_b32_e64,
145 "Wrong Definition type for VALU instruction", instr.get());
146 unsigned num_sgprs = 0;
147 unsigned sgpr[] = {0, 0};
148 for (unsigned i = 0; i < instr->operands.size(); i++)
149 {
150 Operand op = instr->operands[i];
151 if (instr->opcode == aco_opcode::v_readfirstlane_b32 ||
152 instr->opcode == aco_opcode::v_readlane_b32 ||
153 instr->opcode == aco_opcode::v_readlane_b32_e64 ||
154 instr->opcode == aco_opcode::v_writelane_b32 ||
155 instr->opcode == aco_opcode::v_writelane_b32_e64) {
156 check(!op.isLiteral(), "No literal allowed on VALU instruction", instr.get());
157 check(i == 1 || (op.isTemp() && op.regClass() == v1), "Wrong Operand type for VALU instruction", instr.get());
158 continue;
159 }
160 if (op.isTemp() && instr->operands[i].regClass().type() == RegType::sgpr) {
161 check(i != 1 || instr->isVOP3(), "Wrong source position for SGPR argument", instr.get());
162
163 if (op.tempId() != sgpr[0] && op.tempId() != sgpr[1]) {
164 if (num_sgprs < 2)
165 sgpr[num_sgprs++] = op.tempId();
166 }
167 }
168
169 if (op.isConstant() && !op.isLiteral())
170 check(i == 0 || instr->isVOP3(), "Wrong source position for constant argument", instr.get());
171 }
172 check(num_sgprs + (literal.isUndefined() ? 0 : 1) <= const_bus_limit, "Too many SGPRs/literals", instr.get());
173 }
174
175 if (instr->format == Format::SOP1 || instr->format == Format::SOP2) {
176 check(instr->definitions[0].getTemp().type() == RegType::sgpr, "Wrong Definition type for SALU instruction", instr.get());
177 for (const Operand& op : instr->operands) {
178 check(op.isConstant() || op.regClass().type() <= RegType::sgpr,
179 "Wrong Operand type for SALU instruction", instr.get());
180 }
181 }
182 }
183
184 switch (instr->format) {
185 case Format::PSEUDO: {
186 if (instr->opcode == aco_opcode::p_create_vector) {
187 unsigned size = 0;
188 for (const Operand& op : instr->operands) {
189 size += op.size();
190 }
191 check(size == instr->definitions[0].size(), "Definition size does not match operand sizes", instr.get());
192 if (instr->definitions[0].getTemp().type() == RegType::sgpr) {
193 for (const Operand& op : instr->operands) {
194 check(op.isConstant() || op.regClass().type() == RegType::sgpr,
195 "Wrong Operand type for scalar vector", instr.get());
196 }
197 }
198 } else if (instr->opcode == aco_opcode::p_extract_vector) {
199 check((instr->operands[0].isTemp()) && instr->operands[1].isConstant(), "Wrong Operand types", instr.get());
200 check(instr->operands[1].constantValue() < instr->operands[0].size(), "Index out of range", instr.get());
201 check(instr->definitions[0].getTemp().type() == RegType::vgpr || instr->operands[0].regClass().type() == RegType::sgpr,
202 "Cannot extract SGPR value from VGPR vector", instr.get());
203 } else if (instr->opcode == aco_opcode::p_parallelcopy) {
204 check(instr->definitions.size() == instr->operands.size(), "Number of Operands does not match number of Definitions", instr.get());
205 for (unsigned i = 0; i < instr->operands.size(); i++) {
206 if (instr->operands[i].isTemp())
207 check((instr->definitions[i].getTemp().type() == instr->operands[i].regClass().type()) ||
208 (instr->definitions[i].getTemp().type() == RegType::vgpr && instr->operands[i].regClass().type() == RegType::sgpr),
209 "Operand and Definition types do not match", instr.get());
210 }
211 } else if (instr->opcode == aco_opcode::p_phi) {
212 check(instr->operands.size() == block.logical_preds.size(), "Number of Operands does not match number of predecessors", instr.get());
213 check(instr->definitions[0].getTemp().type() == RegType::vgpr || instr->definitions[0].getTemp().regClass() == program->lane_mask, "Logical Phi Definition must be vgpr or divergent boolean", instr.get());
214 } else if (instr->opcode == aco_opcode::p_linear_phi) {
215 for (const Operand& op : instr->operands)
216 check(!op.isTemp() || op.getTemp().is_linear(), "Wrong Operand type", instr.get());
217 check(instr->operands.size() == block.linear_preds.size(), "Number of Operands does not match number of predecessors", instr.get());
218 }
219 break;
220 }
221 case Format::SMEM: {
222 if (instr->operands.size() >= 1)
223 check(instr->operands[0].isTemp() && instr->operands[0].regClass().type() == RegType::sgpr, "SMEM operands must be sgpr", instr.get());
224 if (instr->operands.size() >= 2)
225 check(instr->operands[1].isConstant() || (instr->operands[1].isTemp() && instr->operands[1].regClass().type() == RegType::sgpr),
226 "SMEM offset must be constant or sgpr", instr.get());
227 if (!instr->definitions.empty())
228 check(instr->definitions[0].getTemp().type() == RegType::sgpr, "SMEM result must be sgpr", instr.get());
229 break;
230 }
231 case Format::MTBUF:
232 case Format::MUBUF:
233 case Format::MIMG: {
234 check(instr->operands.size() > 1, "VMEM instructions must have at least one operand", instr.get());
235 check(instr->operands[0].hasRegClass() && instr->operands[0].regClass().type() == RegType::vgpr,
236 "VADDR must be in vgpr for VMEM instructions", instr.get());
237 check(instr->operands[1].isTemp() && instr->operands[1].regClass().type() == RegType::sgpr, "VMEM resource constant must be sgpr", instr.get());
238 check(instr->operands.size() < 4 || (instr->operands[3].isTemp() && instr->operands[3].regClass().type() == RegType::vgpr), "VMEM write data must be vgpr", instr.get());
239 break;
240 }
241 case Format::DS: {
242 for (const Operand& op : instr->operands) {
243 check((op.isTemp() && op.regClass().type() == RegType::vgpr) || op.physReg() == m0,
244 "Only VGPRs are valid DS instruction operands", instr.get());
245 }
246 if (!instr->definitions.empty())
247 check(instr->definitions[0].getTemp().type() == RegType::vgpr, "DS instruction must return VGPR", instr.get());
248 break;
249 }
250 case Format::EXP: {
251 for (unsigned i = 0; i < 4; i++)
252 check(instr->operands[i].hasRegClass() && instr->operands[i].regClass().type() == RegType::vgpr,
253 "Only VGPRs are valid Export arguments", instr.get());
254 break;
255 }
256 case Format::FLAT:
257 check(instr->operands[1].isUndefined(), "Flat instructions don't support SADDR", instr.get());
258 /* fallthrough */
259 case Format::GLOBAL:
260 case Format::SCRATCH: {
261 check(instr->operands[0].isTemp() && instr->operands[0].regClass().type() == RegType::vgpr, "FLAT/GLOBAL/SCRATCH address must be vgpr", instr.get());
262 check(instr->operands[1].hasRegClass() && instr->operands[1].regClass().type() == RegType::sgpr,
263 "FLAT/GLOBAL/SCRATCH sgpr address must be undefined or sgpr", instr.get());
264 if (!instr->definitions.empty())
265 check(instr->definitions[0].getTemp().type() == RegType::vgpr, "FLAT/GLOBAL/SCRATCH result must be vgpr", instr.get());
266 else
267 check(instr->operands[2].regClass().type() == RegType::vgpr, "FLAT/GLOBAL/SCRATCH data must be vgpr", instr.get());
268 break;
269 }
270 default:
271 break;
272 }
273 }
274 }
275
276 /* validate CFG */
277 for (unsigned i = 0; i < program->blocks.size(); i++) {
278 Block& block = program->blocks[i];
279 check_block(block.index == i, "block.index must match actual index", &block);
280
281 /* predecessors/successors should be sorted */
282 for (unsigned j = 0; j + 1 < block.linear_preds.size(); j++)
283 check_block(block.linear_preds[j] < block.linear_preds[j + 1], "linear predecessors must be sorted", &block);
284 for (unsigned j = 0; j + 1 < block.logical_preds.size(); j++)
285 check_block(block.logical_preds[j] < block.logical_preds[j + 1], "logical predecessors must be sorted", &block);
286 for (unsigned j = 0; j + 1 < block.linear_succs.size(); j++)
287 check_block(block.linear_succs[j] < block.linear_succs[j + 1], "linear successors must be sorted", &block);
288 for (unsigned j = 0; j + 1 < block.logical_succs.size(); j++)
289 check_block(block.logical_succs[j] < block.logical_succs[j + 1], "logical successors must be sorted", &block);
290
291 /* critical edges are not allowed */
292 if (block.linear_preds.size() > 1) {
293 for (unsigned pred : block.linear_preds)
294 check_block(program->blocks[pred].linear_succs.size() == 1, "linear critical edges are not allowed", &program->blocks[pred]);
295 for (unsigned pred : block.logical_preds)
296 check_block(program->blocks[pred].logical_succs.size() == 1, "logical critical edges are not allowed", &program->blocks[pred]);
297 }
298 }
299
300 assert(is_valid);
301 }
302
303 /* RA validation */
304 namespace {
305
306 struct Location {
307 Location() : block(NULL), instr(NULL) {}
308
309 Block *block;
310 Instruction *instr; //NULL if it's the block's live-in
311 };
312
313 struct Assignment {
314 Location defloc;
315 Location firstloc;
316 PhysReg reg;
317 };
318
319 bool ra_fail(FILE *output, Location loc, Location loc2, const char *fmt, ...) {
320 va_list args;
321 va_start(args, fmt);
322 char msg[1024];
323 vsprintf(msg, fmt, args);
324 va_end(args);
325
326 fprintf(stderr, "RA error found at instruction in BB%d:\n", loc.block->index);
327 if (loc.instr) {
328 aco_print_instr(loc.instr, stderr);
329 fprintf(stderr, "\n%s", msg);
330 } else {
331 fprintf(stderr, "%s", msg);
332 }
333 if (loc2.block) {
334 fprintf(stderr, " in BB%d:\n", loc2.block->index);
335 aco_print_instr(loc2.instr, stderr);
336 }
337 fprintf(stderr, "\n\n");
338
339 return true;
340 }
341
342 } /* end namespace */
343
344 bool validate_ra(Program *program, const struct radv_nir_compiler_options *options, FILE *output) {
345 if (!(debug_flags & DEBUG_VALIDATE_RA))
346 return false;
347
348 bool err = false;
349 aco::live live_vars = aco::live_var_analysis(program, options);
350 std::vector<std::vector<Temp>> phi_sgpr_ops(program->blocks.size());
351
352 std::map<unsigned, Assignment> assignments;
353 for (Block& block : program->blocks) {
354 Location loc;
355 loc.block = &block;
356 for (aco_ptr<Instruction>& instr : block.instructions) {
357 if (instr->opcode == aco_opcode::p_phi) {
358 for (unsigned i = 0; i < instr->operands.size(); i++) {
359 if (instr->operands[i].isTemp() &&
360 instr->operands[i].getTemp().type() == RegType::sgpr &&
361 instr->operands[i].isFirstKill())
362 phi_sgpr_ops[block.logical_preds[i]].emplace_back(instr->operands[i].getTemp());
363 }
364 }
365
366 loc.instr = instr.get();
367 for (unsigned i = 0; i < instr->operands.size(); i++) {
368 Operand& op = instr->operands[i];
369 if (!op.isTemp())
370 continue;
371 if (!op.isFixed())
372 err |= ra_fail(output, loc, Location(), "Operand %d is not assigned a register", i);
373 if (assignments.count(op.tempId()) && assignments[op.tempId()].reg != op.physReg())
374 err |= ra_fail(output, loc, assignments.at(op.tempId()).firstloc, "Operand %d has an inconsistent register assignment with instruction", i);
375 if ((op.getTemp().type() == RegType::vgpr && op.physReg() + op.size() > 256 + program->config->num_vgprs) ||
376 (op.getTemp().type() == RegType::sgpr && op.physReg() + op.size() > program->config->num_sgprs && op.physReg() < program->sgpr_limit))
377 err |= ra_fail(output, loc, assignments.at(op.tempId()).firstloc, "Operand %d has an out-of-bounds register assignment", i);
378 if (!assignments[op.tempId()].firstloc.block)
379 assignments[op.tempId()].firstloc = loc;
380 if (!assignments[op.tempId()].defloc.block)
381 assignments[op.tempId()].reg = op.physReg();
382 }
383
384 for (unsigned i = 0; i < instr->definitions.size(); i++) {
385 Definition& def = instr->definitions[i];
386 if (!def.isTemp())
387 continue;
388 if (!def.isFixed())
389 err |= ra_fail(output, loc, Location(), "Definition %d is not assigned a register", i);
390 if (assignments[def.tempId()].defloc.block)
391 err |= ra_fail(output, loc, assignments.at(def.tempId()).defloc, "Temporary %%%d also defined by instruction", def.tempId());
392 if ((def.getTemp().type() == RegType::vgpr && def.physReg() + def.size() > 256 + program->config->num_vgprs) ||
393 (def.getTemp().type() == RegType::sgpr && def.physReg() + def.size() > program->config->num_sgprs && def.physReg() < program->sgpr_limit))
394 err |= ra_fail(output, loc, assignments.at(def.tempId()).firstloc, "Definition %d has an out-of-bounds register assignment", i);
395 if (!assignments[def.tempId()].firstloc.block)
396 assignments[def.tempId()].firstloc = loc;
397 assignments[def.tempId()].defloc = loc;
398 assignments[def.tempId()].reg = def.physReg();
399 }
400 }
401 }
402
403 for (Block& block : program->blocks) {
404 Location loc;
405 loc.block = &block;
406
407 std::array<unsigned, 512> regs;
408 regs.fill(0);
409
410 std::set<Temp> live;
411 live.insert(live_vars.live_out[block.index].begin(), live_vars.live_out[block.index].end());
412 /* remove killed p_phi sgpr operands */
413 for (Temp tmp : phi_sgpr_ops[block.index])
414 live.erase(tmp);
415
416 /* check live out */
417 for (Temp tmp : live) {
418 PhysReg reg = assignments.at(tmp.id()).reg;
419 for (unsigned i = 0; i < tmp.size(); i++) {
420 if (regs[reg + i]) {
421 err |= ra_fail(output, loc, Location(), "Assignment of element %d of %%%d already taken by %%%d in live-out", i, tmp.id(), regs[reg + i]);
422 }
423 regs[reg + i] = tmp.id();
424 }
425 }
426 regs.fill(0);
427
428 for (auto it = block.instructions.rbegin(); it != block.instructions.rend(); ++it) {
429 aco_ptr<Instruction>& instr = *it;
430
431 /* check killed p_phi sgpr operands */
432 if (instr->opcode == aco_opcode::p_logical_end) {
433 for (Temp tmp : phi_sgpr_ops[block.index]) {
434 PhysReg reg = assignments.at(tmp.id()).reg;
435 for (unsigned i = 0; i < tmp.size(); i++) {
436 if (regs[reg + i])
437 err |= ra_fail(output, loc, Location(), "Assignment of element %d of %%%d already taken by %%%d in live-out", i, tmp.id(), regs[reg + i]);
438 }
439 live.emplace(tmp);
440 }
441 }
442
443 for (const Definition& def : instr->definitions) {
444 if (!def.isTemp())
445 continue;
446 live.erase(def.getTemp());
447 }
448
449 /* don't count phi operands as live-in, since they are actually
450 * killed when they are copied at the predecessor */
451 if (instr->opcode != aco_opcode::p_phi && instr->opcode != aco_opcode::p_linear_phi) {
452 for (const Operand& op : instr->operands) {
453 if (!op.isTemp())
454 continue;
455 live.insert(op.getTemp());
456 }
457 }
458 }
459
460 for (Temp tmp : live) {
461 PhysReg reg = assignments.at(tmp.id()).reg;
462 for (unsigned i = 0; i < tmp.size(); i++)
463 regs[reg + i] = tmp.id();
464 }
465
466 for (aco_ptr<Instruction>& instr : block.instructions) {
467 loc.instr = instr.get();
468
469 /* remove killed p_phi operands from regs */
470 if (instr->opcode == aco_opcode::p_logical_end) {
471 for (Temp tmp : phi_sgpr_ops[block.index]) {
472 PhysReg reg = assignments.at(tmp.id()).reg;
473 regs[reg] = 0;
474 }
475 }
476
477 if (instr->opcode != aco_opcode::p_phi && instr->opcode != aco_opcode::p_linear_phi) {
478 for (const Operand& op : instr->operands) {
479 if (!op.isTemp())
480 continue;
481 if (op.isFirstKill()) {
482 for (unsigned j = 0; j < op.getTemp().size(); j++)
483 regs[op.physReg() + j] = 0;
484 }
485 }
486 }
487
488 for (unsigned i = 0; i < instr->definitions.size(); i++) {
489 Definition& def = instr->definitions[i];
490 if (!def.isTemp())
491 continue;
492 Temp tmp = def.getTemp();
493 PhysReg reg = assignments.at(tmp.id()).reg;
494 for (unsigned j = 0; j < tmp.size(); j++) {
495 if (regs[reg + j])
496 err |= ra_fail(output, loc, assignments.at(regs[reg + i]).defloc, "Assignment of element %d of %%%d already taken by %%%d from instruction", i, tmp.id(), regs[reg + j]);
497 regs[reg + j] = tmp.id();
498 }
499 }
500
501 for (const Definition& def : instr->definitions) {
502 if (!def.isTemp())
503 continue;
504 if (def.isKill()) {
505 for (unsigned j = 0; j < def.getTemp().size(); j++)
506 regs[def.physReg() + j] = 0;
507 }
508 }
509 }
510 }
511
512 return err;
513 }
514 }