projects / mesa.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
history | raw | HEAD
amd: Move all amd/common code that depends on LLVM to amd/llvm.
[mesa.git] / src / amd / compiler / aco_optimizer.cpp
1 /*
2 * Copyright © 2018 Valve Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 *
23 * Authors:
24 * Daniel Schürmann (daniel.schuermann@campus.tu-berlin.de)
25 *
26 */
27
28 #include <algorithm>
29 #include <math.h>
30
31 #include "aco_ir.h"
32 #include "util/half_float.h"
33 #include "util/u_math.h"
34
35 namespace aco {
36
37 /**
38 * The optimizer works in 4 phases:
39 * (1) The first pass collects information for each ssa-def,
40 * propagates reg->reg operands of the same type, inline constants
41 * and neg/abs input modifiers.
42 * (2) The second pass combines instructions like mad, omod, clamp and
43 * propagates sgpr's on VALU instructions.
44 * This pass depends on information collected in the first pass.
45 * (3) The third pass goes backwards, and selects instructions,
46 * i.e. decides if a mad instruction is profitable and eliminates dead code.
47 * (4) The fourth pass cleans up the sequence: literals get applied and dead
48 * instructions are removed from the sequence.
49 */
50
51
52 struct mad_info {
53 aco_ptr<Instruction> add_instr;
54 uint32_t mul_temp_id;
55 uint32_t literal_idx;
56 bool needs_vop3;
57 bool check_literal;
58
59 mad_info(aco_ptr<Instruction> instr, uint32_t id, bool vop3)
60 : add_instr(std::move(instr)), mul_temp_id(id), needs_vop3(vop3), check_literal(false) {}
61 };
62
63 enum Label {
64 label_vec = 1 << 0,
65 label_constant = 1 << 1,
66 label_abs = 1 << 2,
67 label_neg = 1 << 3,
68 label_mul = 1 << 4,
69 label_temp = 1 << 5,
70 label_literal = 1 << 6,
71 label_mad = 1 << 7,
72 label_omod2 = 1 << 8,
73 label_omod4 = 1 << 9,
74 label_omod5 = 1 << 10,
75 label_omod_success = 1 << 11,
76 label_clamp = 1 << 12,
77 label_clamp_success = 1 << 13,
78 label_undefined = 1 << 14,
79 label_vcc = 1 << 15,
80 label_b2f = 1 << 16,
81 label_add_sub = 1 << 17,
82 label_bitwise = 1 << 18,
83 label_minmax = 1 << 19,
84 label_fcmp = 1 << 20,
85 };
86
87 static constexpr uint32_t instr_labels = label_vec | label_mul | label_mad | label_omod_success | label_clamp_success | label_add_sub | label_bitwise | label_minmax | label_fcmp;
88 static constexpr uint32_t temp_labels = label_abs | label_neg | label_temp | label_vcc | label_b2f;
89 static constexpr uint32_t val_labels = label_constant | label_literal | label_mad;
90
91 struct ssa_info {
92 uint32_t val;
93 union {
94 Temp temp;
95 Instruction* instr;
96 };
97 uint32_t label;
98
99 void add_label(Label new_label)
100 {
101 /* Since all labels which use "instr" use it for the same thing
102 * (indicating the defining instruction), there is no need to clear
103 * any other instr labels. */
104 if (new_label & instr_labels)
105 label &= ~temp_labels; /* instr and temp alias */
106
107 if (new_label & temp_labels) {
108 label &= ~temp_labels;
109 label &= ~instr_labels; /* instr and temp alias */
110 }
111
112 if (new_label & val_labels)
113 label &= ~val_labels;
114
115 label |= new_label;
116 }
117
118 void set_vec(Instruction* vec)
119 {
120 add_label(label_vec);
121 instr = vec;
122 }
123
124 bool is_vec()
125 {
126 return label & label_vec;
127 }
128
129 void set_constant(uint32_t constant)
130 {
131 add_label(label_constant);
132 val = constant;
133 }
134
135 bool is_constant()
136 {
137 return label & label_constant;
138 }
139
140 void set_abs(Temp abs_temp)
141 {
142 add_label(label_abs);
143 temp = abs_temp;
144 }
145
146 bool is_abs()
147 {
148 return label & label_abs;
149 }
150
151 void set_neg(Temp neg_temp)
152 {
153 add_label(label_neg);
154 temp = neg_temp;
155 }
156
157 bool is_neg()
158 {
159 return label & label_neg;
160 }
161
162 void set_neg_abs(Temp neg_abs_temp)
163 {
164 add_label((Label)((uint32_t)label_abs | (uint32_t)label_neg));
165 temp = neg_abs_temp;
166 }
167
168 void set_mul(Instruction* mul)
169 {
170 add_label(label_mul);
171 instr = mul;
172 }
173
174 bool is_mul()
175 {
176 return label & label_mul;
177 }
178
179 void set_temp(Temp tmp)
180 {
181 add_label(label_temp);
182 temp = tmp;
183 }
184
185 bool is_temp()
186 {
187 return label & label_temp;
188 }
189
190 void set_literal(uint32_t lit)
191 {
192 add_label(label_literal);
193 val = lit;
194 }
195
196 bool is_literal()
197 {
198 return label & label_literal;
199 }
200
201 void set_mad(Instruction* mad, uint32_t mad_info_idx)
202 {
203 add_label(label_mad);
204 val = mad_info_idx;
205 instr = mad;
206 }
207
208 bool is_mad()
209 {
210 return label & label_mad;
211 }
212
213 void set_omod2()
214 {
215 add_label(label_omod2);
216 }
217
218 bool is_omod2()
219 {
220 return label & label_omod2;
221 }
222
223 void set_omod4()
224 {
225 add_label(label_omod4);
226 }
227
228 bool is_omod4()
229 {
230 return label & label_omod4;
231 }
232
233 void set_omod5()
234 {
235 add_label(label_omod5);
236 }
237
238 bool is_omod5()
239 {
240 return label & label_omod5;
241 }
242
243 void set_omod_success(Instruction* omod_instr)
244 {
245 add_label(label_omod_success);
246 instr = omod_instr;
247 }
248
249 bool is_omod_success()
250 {
251 return label & label_omod_success;
252 }
253
254 void set_clamp()
255 {
256 add_label(label_clamp);
257 }
258
259 bool is_clamp()
260 {
261 return label & label_clamp;
262 }
263
264 void set_clamp_success(Instruction* clamp_instr)
265 {
266 add_label(label_clamp_success);
267 instr = clamp_instr;
268 }
269
270 bool is_clamp_success()
271 {
272 return label & label_clamp_success;
273 }
274
275 void set_undefined()
276 {
277 add_label(label_undefined);
278 }
279
280 bool is_undefined()
281 {
282 return label & label_undefined;
283 }
284
285 void set_vcc(Temp vcc)
286 {
287 add_label(label_vcc);
288 temp = vcc;
289 }
290
291 bool is_vcc()
292 {
293 return label & label_vcc;
294 }
295
296 bool is_constant_or_literal()
297 {
298 return is_constant() || is_literal();
299 }
300
301 void set_b2f(Temp val)
302 {
303 add_label(label_b2f);
304 temp = val;
305 }
306
307 bool is_b2f()
308 {
309 return label & label_b2f;
310 }
311
312 void set_add_sub(Instruction *add_sub_instr)
313 {
314 add_label(label_add_sub);
315 instr = add_sub_instr;
316 }
317
318 bool is_add_sub()
319 {
320 return label & label_add_sub;
321 }
322
323 void set_bitwise(Instruction *bitwise_instr)
324 {
325 add_label(label_bitwise);
326 instr = bitwise_instr;
327 }
328
329 bool is_bitwise()
330 {
331 return label & label_bitwise;
332 }
333
334 void set_minmax(Instruction *minmax_instr)
335 {
336 add_label(label_minmax);
337 instr = minmax_instr;
338 }
339
340 bool is_minmax()
341 {
342 return label & label_minmax;
343 }
344
345 void set_fcmp(Instruction *fcmp_instr)
346 {
347 add_label(label_fcmp);
348 instr = fcmp_instr;
349 }
350
351 bool is_fcmp()
352 {
353 return label & label_fcmp;
354 }
355
356 };
357
358 struct opt_ctx {
359 Program* program;
360 std::vector<aco_ptr<Instruction>> instructions;
361 ssa_info* info;
362 std::pair<uint32_t,Temp> last_literal;
363 std::vector<mad_info> mad_infos;
364 std::vector<uint16_t> uses;
365 };
366
367 bool can_swap_operands(aco_ptr<Instruction>& instr)
368 {
369 if (instr->operands[0].isConstant() ||
370 (instr->operands[0].isTemp() && instr->operands[0].getTemp().type() == RegType::sgpr))
371 return false;
372
373 switch (instr->opcode) {
374 case aco_opcode::v_add_f32:
375 case aco_opcode::v_mul_f32:
376 case aco_opcode::v_or_b32:
377 case aco_opcode::v_and_b32:
378 case aco_opcode::v_xor_b32:
379 case aco_opcode::v_max_f32:
380 case aco_opcode::v_min_f32:
381 case aco_opcode::v_cmp_eq_f32:
382 case aco_opcode::v_cmp_lg_f32:
383 return true;
384 case aco_opcode::v_sub_f32:
385 instr->opcode = aco_opcode::v_subrev_f32;
386 return true;
387 case aco_opcode::v_cmp_lt_f32:
388 instr->opcode = aco_opcode::v_cmp_gt_f32;
389 return true;
390 case aco_opcode::v_cmp_ge_f32:
391 instr->opcode = aco_opcode::v_cmp_le_f32;
392 return true;
393 case aco_opcode::v_cmp_lt_i32:
394 instr->opcode = aco_opcode::v_cmp_gt_i32;
395 return true;
396 default:
397 return false;
398 }
399 }
400
401 bool can_use_VOP3(aco_ptr<Instruction>& instr)
402 {
403 if (instr->operands.size() && instr->operands[0].isLiteral())
404 return false;
405
406 if (instr->isDPP() || instr->isSDWA())
407 return false;
408
409 return instr->opcode != aco_opcode::v_madmk_f32 &&
410 instr->opcode != aco_opcode::v_madak_f32 &&
411 instr->opcode != aco_opcode::v_madmk_f16 &&
412 instr->opcode != aco_opcode::v_madak_f16;
413 }
414
415 void to_VOP3(opt_ctx& ctx, aco_ptr<Instruction>& instr)
416 {
417 if (instr->isVOP3())
418 return;
419
420 assert(!instr->operands[0].isLiteral());
421 aco_ptr<Instruction> tmp = std::move(instr);
422 Format format = asVOP3(tmp->format);
423 instr.reset(create_instruction<VOP3A_instruction>(tmp->opcode, format, tmp->operands.size(), tmp->definitions.size()));
424 std::copy(tmp->operands.cbegin(), tmp->operands.cend(), instr->operands.begin());
425 for (unsigned i = 0; i < instr->definitions.size(); i++) {
426 instr->definitions[i] = tmp->definitions[i];
427 if (instr->definitions[i].isTemp()) {
428 ssa_info& info = ctx.info[instr->definitions[i].tempId()];
429 if (info.label & instr_labels && info.instr == tmp.get())
430 info.instr = instr.get();
431 }
432 }
433 }
434
435 bool valu_can_accept_literal(opt_ctx& ctx, aco_ptr<Instruction>& instr)
436 {
437 // TODO: VOP3 can take a literal on GFX10
438 return !instr->isSDWA() && !instr->isDPP() && !instr->isVOP3();
439 }
440
441 /* only covers special cases */
442 bool can_accept_constant(aco_ptr<Instruction>& instr, unsigned operand)
443 {
444 switch (instr->opcode) {
445 case aco_opcode::v_interp_p2_f32:
446 case aco_opcode::v_mac_f32:
447 case aco_opcode::v_writelane_b32:
448 case aco_opcode::v_cndmask_b32:
449 return operand != 2;
450 case aco_opcode::s_addk_i32:
451 case aco_opcode::s_mulk_i32:
452 case aco_opcode::p_wqm:
453 case aco_opcode::p_extract_vector:
454 case aco_opcode::p_split_vector:
455 return operand != 0;
456 default:
457 if ((instr->format == Format::MUBUF ||
458 instr->format == Format::MIMG) &&
459 instr->definitions.size() == 1 &&
460 instr->operands.size() == 4) {
461 return operand != 3;
462 }
463 return true;
464 }
465 }
466
467 bool parse_base_offset(opt_ctx &ctx, Instruction* instr, unsigned op_index, Temp *base, uint32_t *offset)
468 {
469 Operand op = instr->operands[op_index];
470
471 if (!op.isTemp())
472 return false;
473 Temp tmp = op.getTemp();
474 if (!ctx.info[tmp.id()].is_add_sub())
475 return false;
476
477 Instruction *add_instr = ctx.info[tmp.id()].instr;
478
479 switch (add_instr->opcode) {
480 case aco_opcode::v_add_u32:
481 case aco_opcode::v_add_co_u32:
482 case aco_opcode::s_add_i32:
483 case aco_opcode::s_add_u32:
484 break;
485 default:
486 return false;
487 }
488
489 for (unsigned i = 0; i < 2; i++) {
490 if (add_instr->operands[i].isConstant()) {
491 *offset = add_instr->operands[i].constantValue();
492 } else if (add_instr->operands[i].isTemp() &&
493 ctx.info[add_instr->operands[i].tempId()].is_constant_or_literal()) {
494 *offset = ctx.info[add_instr->operands[i].tempId()].val;
495 } else {
496 continue;
497 }
498 if (!add_instr->operands[!i].isTemp())
499 continue;
500
501 uint32_t offset2 = 0;
502 if (parse_base_offset(ctx, add_instr, !i, base, &offset2)) {
503 *offset += offset2;
504 } else {
505 *base = add_instr->operands[!i].getTemp();
506 }
507 return true;
508 }
509
510 return false;
511 }
512
513 void label_instruction(opt_ctx &ctx, aco_ptr<Instruction>& instr)
514 {
515 if (instr->isSALU() || instr->isVALU() || instr->format == Format::PSEUDO) {
516 ASSERTED bool all_const = false;
517 for (Operand& op : instr->operands)
518 all_const = all_const && (!op.isTemp() || ctx.info[op.tempId()].is_constant_or_literal());
519 perfwarn(all_const, "All instruction operands are constant", instr.get());
520 }
521
522 for (unsigned i = 0; i < instr->operands.size(); i++)
523 {
524 if (!instr->operands[i].isTemp())
525 continue;
526
527 ssa_info info = ctx.info[instr->operands[i].tempId()];
528 /* propagate undef */
529 if (info.is_undefined() && is_phi(instr))
530 instr->operands[i] = Operand(instr->operands[i].regClass());
531 /* propagate reg->reg of same type */
532 if (info.is_temp() && info.temp.regClass() == instr->operands[i].getTemp().regClass()) {
533 instr->operands[i].setTemp(ctx.info[instr->operands[i].tempId()].temp);
534 info = ctx.info[info.temp.id()];
535 }
536
537 /* SALU / PSEUDO: propagate inline constants */
538 if (instr->isSALU() || instr->format == Format::PSEUDO) {
539 if (info.is_temp() && info.temp.type() == RegType::sgpr) {
540 instr->operands[i].setTemp(info.temp);
541 info = ctx.info[info.temp.id()];
542 } else if (info.is_temp() && info.temp.type() == RegType::vgpr) {
543 /* propagate vgpr if it can take it */
544 switch (instr->opcode) {
545 case aco_opcode::p_create_vector:
546 case aco_opcode::p_split_vector:
547 case aco_opcode::p_extract_vector:
548 case aco_opcode::p_phi: {
549 const bool all_vgpr = std::none_of(instr->definitions.begin(), instr->definitions.end(),
550 [] (const Definition& def) { return def.getTemp().type() != RegType::vgpr;});
551 if (all_vgpr) {
552 instr->operands[i] = Operand(info.temp);
553 info = ctx.info[info.temp.id()];
554 }
555 break;
556 }
557 default:
558 break;
559 }
560 }
561 if ((info.is_constant() || (info.is_literal() && instr->format == Format::PSEUDO)) && !instr->operands[i].isFixed() && can_accept_constant(instr, i)) {
562 instr->operands[i] = Operand(info.val);
563 continue;
564 }
565 }
566
567 /* VALU: propagate neg, abs & inline constants */
568 else if (instr->isVALU()) {
569 if (info.is_temp() && info.temp.type() == RegType::vgpr) {
570 instr->operands[i].setTemp(info.temp);
571 info = ctx.info[info.temp.id()];
572 }
573 if (info.is_abs() && (can_use_VOP3(instr) || instr->isDPP()) && instr_info.can_use_input_modifiers[(int)instr->opcode]) {
574 if (!instr->isDPP())
575 to_VOP3(ctx, instr);
576 instr->operands[i] = Operand(info.temp);
577 if (instr->isDPP())
578 static_cast<DPP_instruction*>(instr.get())->abs[i] = true;
579 else
580 static_cast<VOP3A_instruction*>(instr.get())->abs[i] = true;
581 }
582 if (info.is_neg() && instr->opcode == aco_opcode::v_add_f32) {
583 instr->opcode = i ? aco_opcode::v_sub_f32 : aco_opcode::v_subrev_f32;
584 instr->operands[i].setTemp(info.temp);
585 continue;
586 } else if (info.is_neg() && (can_use_VOP3(instr) || instr->isDPP()) && instr_info.can_use_input_modifiers[(int)instr->opcode]) {
587 if (!instr->isDPP())
588 to_VOP3(ctx, instr);
589 instr->operands[i].setTemp(info.temp);
590 if (instr->isDPP())
591 static_cast<DPP_instruction*>(instr.get())->neg[i] = true;
592 else
593 static_cast<VOP3A_instruction*>(instr.get())->neg[i] = true;
594 continue;
595 }
596 if (info.is_constant() && can_accept_constant(instr, i)) {
597 perfwarn(instr->opcode == aco_opcode::v_cndmask_b32 && i == 2, "v_cndmask_b32 with a constant selector", instr.get());
598 if (i == 0) {
599 instr->operands[i] = Operand(info.val);
600 continue;
601 } else if (!instr->isVOP3() && can_swap_operands(instr)) {
602 instr->operands[i] = instr->operands[0];
603 instr->operands[0] = Operand(info.val);
604 continue;
605 } else if (can_use_VOP3(instr)) {
606 to_VOP3(ctx, instr);
607 instr->operands[i] = Operand(info.val);
608 continue;
609 }
610 }
611 }
612
613 /* MUBUF: propagate constants and combine additions */
614 else if (instr->format == Format::MUBUF) {
615 MUBUF_instruction *mubuf = static_cast<MUBUF_instruction *>(instr.get());
616 Temp base;
617 uint32_t offset;
618 while (info.is_temp())
619 info = ctx.info[info.temp.id()];
620
621 if (mubuf->offen && i == 0 && info.is_constant_or_literal() && mubuf->offset + info.val < 4096) {
622 assert(!mubuf->idxen);
623 instr->operands[i] = Operand(v1);
624 mubuf->offset += info.val;
625 mubuf->offen = false;
626 continue;
627 } else if (i == 2 && info.is_constant_or_literal() && mubuf->offset + info.val < 4096) {
628 instr->operands[2] = Operand((uint32_t) 0);
629 mubuf->offset += info.val;
630 continue;
631 } else if (mubuf->offen && i == 0 && parse_base_offset(ctx, instr.get(), i, &base, &offset) && base.regClass() == v1 && mubuf->offset + offset < 4096) {
632 assert(!mubuf->idxen);
633 instr->operands[i].setTemp(base);
634 mubuf->offset += offset;
635 continue;
636 } else if (i == 2 && parse_base_offset(ctx, instr.get(), i, &base, &offset) && base.regClass() == s1 && mubuf->offset + offset < 4096) {
637 instr->operands[i].setTemp(base);
638 mubuf->offset += offset;
639 continue;
640 }
641 }
642
643 /* DS: combine additions */
644 else if (instr->format == Format::DS) {
645
646 DS_instruction *ds = static_cast<DS_instruction *>(instr.get());
647 Temp base;
648 uint32_t offset;
649 if (i == 0 && parse_base_offset(ctx, instr.get(), i, &base, &offset) && base.regClass() == instr->operands[i].regClass()) {
650 if (instr->opcode == aco_opcode::ds_write2_b32 || instr->opcode == aco_opcode::ds_read2_b32) {
651 if (offset % 4 == 0 &&
652 ds->offset0 + (offset >> 2) <= 255 &&
653 ds->offset1 + (offset >> 2) <= 255) {
654 instr->operands[i].setTemp(base);
655 ds->offset0 += offset >> 2;
656 ds->offset1 += offset >> 2;
657 }
658 } else {
659 if (ds->offset0 + offset <= 65535) {
660 instr->operands[i].setTemp(base);
661 ds->offset0 += offset;
662 }
663 }
664 }
665 }
666
667 /* SMEM: propagate constants and combine additions */
668 else if (instr->format == Format::SMEM) {
669
670 SMEM_instruction *smem = static_cast<SMEM_instruction *>(instr.get());
671 Temp base;
672 uint32_t offset;
673 if (i == 1 && info.is_constant_or_literal() && info.val <= 0xFFFFF) {
674 instr->operands[i] = Operand(info.val);
675 continue;
676 } else if (i == 1 && parse_base_offset(ctx, instr.get(), i, &base, &offset) && base.regClass() == s1 && offset <= 0xFFFFF && ctx.program->chip_class >= GFX9) {
677 bool soe = smem->operands.size() >= (!smem->definitions.empty() ? 3 : 4);
678 if (soe &&
679 (!ctx.info[smem->operands.back().tempId()].is_constant_or_literal() ||
680 ctx.info[smem->operands.back().tempId()].val != 0)) {
681 continue;
682 }
683 if (soe) {
684 smem->operands[1] = Operand(offset);
685 smem->operands.back() = Operand(base);
686 } else {
687 SMEM_instruction *new_instr = create_instruction<SMEM_instruction>(smem->opcode, Format::SMEM, smem->operands.size() + 1, smem->definitions.size());
688 new_instr->operands[0] = smem->operands[0];
689 new_instr->operands[1] = Operand(offset);
690 if (smem->definitions.empty())
691 new_instr->operands[2] = smem->operands[2];
692 new_instr->operands.back() = Operand(base);
693 if (!smem->definitions.empty())
694 new_instr->definitions[0] = smem->definitions[0];
695 instr.reset(new_instr);
696 smem = static_cast<SMEM_instruction *>(instr.get());
697 }
698 continue;
699 }
700 }
701 }
702
703 /* if this instruction doesn't define anything, return */
704 if (instr->definitions.empty())
705 return;
706
707 switch (instr->opcode) {
708 case aco_opcode::p_create_vector: {
709 unsigned num_ops = instr->operands.size();
710 for (const Operand& op : instr->operands) {
711 if (op.isTemp() && ctx.info[op.tempId()].is_vec())
712 num_ops += ctx.info[op.tempId()].instr->operands.size() - 1;
713 }
714 if (num_ops != instr->operands.size()) {
715 aco_ptr<Instruction> old_vec = std::move(instr);
716 instr.reset(create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, num_ops, 1));
717 instr->definitions[0] = old_vec->definitions[0];
718 unsigned k = 0;
719 for (Operand& old_op : old_vec->operands) {
720 if (old_op.isTemp() && ctx.info[old_op.tempId()].is_vec()) {
721 for (unsigned j = 0; j < ctx.info[old_op.tempId()].instr->operands.size(); j++)
722 instr->operands[k++] = ctx.info[old_op.tempId()].instr->operands[j];
723 } else {
724 instr->operands[k++] = old_op;
725 }
726 }
727 assert(k == num_ops);
728 }
729 if (instr->operands.size() == 1 && instr->operands[0].isTemp())
730 ctx.info[instr->definitions[0].tempId()].set_temp(instr->operands[0].getTemp());
731 else if (instr->definitions[0].getTemp().size() == instr->operands.size())
732 ctx.info[instr->definitions[0].tempId()].set_vec(instr.get());
733 break;
734 }
735 case aco_opcode::p_split_vector: {
736 if (!ctx.info[instr->operands[0].tempId()].is_vec())
737 break;
738 Instruction* vec = ctx.info[instr->operands[0].tempId()].instr;
739 assert(instr->definitions.size() == vec->operands.size());
740 for (unsigned i = 0; i < instr->definitions.size(); i++) {
741 Operand vec_op = vec->operands[i];
742 if (vec_op.isConstant()) {
743 if (vec_op.isLiteral())
744 ctx.info[instr->definitions[i].tempId()].set_literal(vec_op.constantValue());
745 else
746 ctx.info[instr->definitions[i].tempId()].set_constant(vec_op.constantValue());
747 } else {
748 assert(vec_op.isTemp());
749 ctx.info[instr->definitions[i].tempId()].set_temp(vec_op.getTemp());
750 }
751 }
752 break;
753 }
754 case aco_opcode::p_extract_vector: { /* mov */
755 if (!ctx.info[instr->operands[0].tempId()].is_vec())
756 break;
757 Instruction* vec = ctx.info[instr->operands[0].tempId()].instr;
758 if (vec->definitions[0].getTemp().size() == vec->operands.size()) { /* TODO: what about 64bit or other combinations? */
759
760 /* convert this extract into a mov instruction */
761 Operand vec_op = vec->operands[instr->operands[1].constantValue()];
762 bool is_vgpr = instr->definitions[0].getTemp().type() == RegType::vgpr;
763 aco_opcode opcode = is_vgpr ? aco_opcode::v_mov_b32 : aco_opcode::s_mov_b32;
764 Format format = is_vgpr ? Format::VOP1 : Format::SOP1;
765 instr->opcode = opcode;
766 instr->format = format;
767 instr->operands = {instr->operands.begin(), 1 };
768 instr->operands[0] = vec_op;
769
770 if (vec_op.isConstant()) {
771 if (vec_op.isLiteral())
772 ctx.info[instr->definitions[0].tempId()].set_literal(vec_op.constantValue());
773 else
774 ctx.info[instr->definitions[0].tempId()].set_constant(vec_op.constantValue());
775 } else {
776 assert(vec_op.isTemp());
777 ctx.info[instr->definitions[0].tempId()].set_temp(vec_op.getTemp());
778 }
779 }
780 break;
781 }
782 case aco_opcode::s_mov_b32: /* propagate */
783 case aco_opcode::s_mov_b64:
784 case aco_opcode::v_mov_b32:
785 case aco_opcode::p_as_uniform:
786 if (instr->definitions[0].isFixed()) {
787 /* don't copy-propagate copies into fixed registers */
788 } else if (instr->operands[0].isConstant()) {
789 if (instr->operands[0].isLiteral())
790 ctx.info[instr->definitions[0].tempId()].set_literal(instr->operands[0].constantValue());
791 else
792 ctx.info[instr->definitions[0].tempId()].set_constant(instr->operands[0].constantValue());
793 } else if (instr->isDPP()) {
794 // TODO
795 } else if (instr->operands[0].isTemp()) {
796 ctx.info[instr->definitions[0].tempId()].set_temp(instr->operands[0].getTemp());
797 } else {
798 assert(instr->operands[0].isFixed());
799 }
800 break;
801 case aco_opcode::p_is_helper:
802 if (!ctx.program->needs_wqm)
803 ctx.info[instr->definitions[0].tempId()].set_constant(0u);
804 break;
805 case aco_opcode::s_movk_i32: {
806 uint32_t v = static_cast<SOPK_instruction*>(instr.get())->imm;
807 v = v & 0x8000 ? (v | 0xffff0000) : v;
808 if (v <= 64 || v >= 0xfffffff0)
809 ctx.info[instr->definitions[0].tempId()].set_constant(v);
810 else
811 ctx.info[instr->definitions[0].tempId()].set_literal(v);
812 break;
813 }
814 case aco_opcode::v_bfrev_b32:
815 case aco_opcode::s_brev_b32: {
816 if (instr->operands[0].isConstant()) {
817 uint32_t v = util_bitreverse(instr->operands[0].constantValue());
818 if (v <= 64 || v >= 0xfffffff0)
819 ctx.info[instr->definitions[0].tempId()].set_constant(v);
820 else
821 ctx.info[instr->definitions[0].tempId()].set_literal(v);
822 }
823 break;
824 }
825 case aco_opcode::s_bfm_b32: {
826 if (instr->operands[0].isConstant() && instr->operands[1].isConstant()) {
827 unsigned size = instr->operands[0].constantValue() & 0x1f;
828 unsigned start = instr->operands[1].constantValue() & 0x1f;
829 uint32_t v = ((1u << size) - 1u) << start;
830 if (v <= 64 || v >= 0xfffffff0)
831 ctx.info[instr->definitions[0].tempId()].set_constant(v);
832 else
833 ctx.info[instr->definitions[0].tempId()].set_literal(v);
834 }
835 }
836 case aco_opcode::v_mul_f32: { /* omod */
837 /* TODO: try to move the negate/abs modifier to the consumer instead */
838 if (instr->isVOP3()) {
839 VOP3A_instruction *vop3 = static_cast<VOP3A_instruction*>(instr.get());
840 if (vop3->abs[0] || vop3->abs[1] || vop3->neg[0] || vop3->neg[1] || vop3->omod || vop3->clamp)
841 break;
842 }
843
844 for (unsigned i = 0; i < 2; i++) {
845 if (instr->operands[!i].isConstant() && instr->operands[i].isTemp()) {
846 if (instr->operands[!i].constantValue() == 0x40000000) { /* 2.0 */
847 ctx.info[instr->operands[i].tempId()].set_omod2();
848 } else if (instr->operands[!i].constantValue() == 0x40800000) { /* 4.0 */
849 ctx.info[instr->operands[i].tempId()].set_omod4();
850 } else if (instr->operands[!i].constantValue() == 0x3f000000) { /* 0.5 */
851 ctx.info[instr->operands[i].tempId()].set_omod5();
852 } else if (instr->operands[!i].constantValue() == 0x3f800000) { /* 1.0 */
853 ctx.info[instr->definitions[0].tempId()].set_temp(instr->operands[i].getTemp());
854 } else {
855 continue;
856 }
857 break;
858 }
859 }
860 break;
861 }
862 case aco_opcode::v_and_b32: /* abs */
863 if (instr->operands[0].constantEquals(0x7FFFFFFF) && instr->operands[1].isTemp())
864 ctx.info[instr->definitions[0].tempId()].set_abs(instr->operands[1].getTemp());
865 else
866 ctx.info[instr->definitions[0].tempId()].set_bitwise(instr.get());
867 break;
868 case aco_opcode::v_xor_b32: { /* neg */
869 if (instr->operands[0].constantEquals(0x80000000u) && instr->operands[1].isTemp()) {
870 if (ctx.info[instr->operands[1].tempId()].is_neg()) {
871 ctx.info[instr->definitions[0].tempId()].set_temp(ctx.info[instr->operands[1].tempId()].temp);
872 } else {
873 if (ctx.info[instr->operands[1].tempId()].is_abs()) { /* neg(abs(x)) */
874 instr->operands[1].setTemp(ctx.info[instr->operands[1].tempId()].temp);
875 instr->opcode = aco_opcode::v_or_b32;
876 ctx.info[instr->definitions[0].tempId()].set_neg_abs(instr->operands[1].getTemp());
877 } else {
878 ctx.info[instr->definitions[0].tempId()].set_neg(instr->operands[1].getTemp());
879 }
880 }
881 } else {
882 ctx.info[instr->definitions[0].tempId()].set_bitwise(instr.get());
883 }
884 break;
885 }
886 case aco_opcode::v_med3_f32: { /* clamp */
887 VOP3A_instruction* vop3 = static_cast<VOP3A_instruction*>(instr.get());
888 if (vop3->abs[0] || vop3->neg[0] || vop3->opsel[0] ||
889 vop3->abs[1] || vop3->neg[1] || vop3->opsel[1] ||
890 vop3->abs[2] || vop3->neg[2] || vop3->opsel[2] ||
891 vop3->omod != 0)
892 break;
893
894 unsigned idx = 0;
895 bool found_zero = false, found_one = false;
896 for (unsigned i = 0; i < 3; i++)
897 {
898 if (instr->operands[i].constantEquals(0))
899 found_zero = true;
900 else if (instr->operands[i].constantEquals(0x3f800000)) /* 1.0 */
901 found_one = true;
902 else
903 idx = i;
904 }
905 if (found_zero && found_one && instr->operands[idx].isTemp()) {
906 ctx.info[instr->operands[idx].tempId()].set_clamp();
907 }
908 break;
909 }
910 case aco_opcode::v_cndmask_b32:
911 if (instr->operands[0].constantEquals(0) &&
912 instr->operands[1].constantEquals(0xFFFFFFFF) &&
913 instr->operands[2].isTemp())
914 ctx.info[instr->definitions[0].tempId()].set_vcc(instr->operands[2].getTemp());
915 else if (instr->operands[0].constantEquals(0) &&
916 instr->operands[1].constantEquals(0x3f800000u) &&
917 instr->operands[2].isTemp())
918 ctx.info[instr->definitions[0].tempId()].set_b2f(instr->operands[2].getTemp());
919 break;
920 case aco_opcode::v_cmp_lg_u32:
921 if (instr->format == Format::VOPC && /* don't optimize VOP3 / SDWA / DPP */
922 instr->operands[0].constantEquals(0) &&
923 instr->operands[1].isTemp() && ctx.info[instr->operands[1].tempId()].is_vcc())
924 ctx.info[instr->definitions[0].tempId()].set_temp(ctx.info[instr->operands[1].tempId()].temp);
925 break;
926 case aco_opcode::p_phi:
927 case aco_opcode::p_linear_phi: {
928 /* lower_bool_phis() can create phis like this */
929 bool all_same_temp = instr->operands[0].isTemp();
930 /* this check is needed when moving uniform loop counters out of a divergent loop */
931 if (all_same_temp)
932 all_same_temp = instr->definitions[0].regClass() == instr->operands[0].regClass();
933 for (unsigned i = 1; all_same_temp && (i < instr->operands.size()); i++) {
934 if (!instr->operands[i].isTemp() || instr->operands[i].tempId() != instr->operands[0].tempId())
935 all_same_temp = false;
936 }
937 if (all_same_temp) {
938 ctx.info[instr->definitions[0].tempId()].set_temp(instr->operands[0].getTemp());
939 } else {
940 bool all_undef = instr->operands[0].isUndefined();
941 for (unsigned i = 1; all_undef && (i < instr->operands.size()); i++) {
942 if (!instr->operands[i].isUndefined())
943 all_undef = false;
944 }
945 if (all_undef)
946 ctx.info[instr->definitions[0].tempId()].set_undefined();
947 }
948 break;
949 }
950 case aco_opcode::v_add_u32:
951 case aco_opcode::v_add_co_u32:
952 case aco_opcode::s_add_i32:
953 case aco_opcode::s_add_u32:
954 ctx.info[instr->definitions[0].tempId()].set_add_sub(instr.get());
955 break;
956 case aco_opcode::s_not_b32:
957 case aco_opcode::s_not_b64:
958 case aco_opcode::s_and_b32:
959 case aco_opcode::s_and_b64:
960 case aco_opcode::s_or_b32:
961 case aco_opcode::s_or_b64:
962 case aco_opcode::s_xor_b32:
963 case aco_opcode::s_xor_b64:
964 case aco_opcode::s_lshl_b32:
965 case aco_opcode::v_or_b32:
966 case aco_opcode::v_lshlrev_b32:
967 ctx.info[instr->definitions[0].tempId()].set_bitwise(instr.get());
968 break;
969 case aco_opcode::v_min_f32:
970 case aco_opcode::v_min_f16:
971 case aco_opcode::v_min_u32:
972 case aco_opcode::v_min_i32:
973 case aco_opcode::v_min_u16:
974 case aco_opcode::v_min_i16:
975 case aco_opcode::v_max_f32:
976 case aco_opcode::v_max_f16:
977 case aco_opcode::v_max_u32:
978 case aco_opcode::v_max_i32:
979 case aco_opcode::v_max_u16:
980 case aco_opcode::v_max_i16:
981 ctx.info[instr->definitions[0].tempId()].set_minmax(instr.get());
982 break;
983 case aco_opcode::v_cmp_lt_f32:
984 case aco_opcode::v_cmp_eq_f32:
985 case aco_opcode::v_cmp_le_f32:
986 case aco_opcode::v_cmp_gt_f32:
987 case aco_opcode::v_cmp_lg_f32:
988 case aco_opcode::v_cmp_ge_f32:
989 case aco_opcode::v_cmp_o_f32:
990 case aco_opcode::v_cmp_u_f32:
991 case aco_opcode::v_cmp_nge_f32:
992 case aco_opcode::v_cmp_nlg_f32:
993 case aco_opcode::v_cmp_ngt_f32:
994 case aco_opcode::v_cmp_nle_f32:
995 case aco_opcode::v_cmp_neq_f32:
996 case aco_opcode::v_cmp_nlt_f32:
997 ctx.info[instr->definitions[0].tempId()].set_fcmp(instr.get());
998 break;
999 default:
1000 break;
1001 }
1002 }
1003
1004 ALWAYS_INLINE bool get_cmp_info(aco_opcode op, aco_opcode *ordered, aco_opcode *unordered, aco_opcode *inverse)
1005 {
1006 *ordered = *unordered = op;
1007 switch (op) {
1008 #define CMP(ord, unord) \
1009 case aco_opcode::v_cmp_##ord##_f32:\
1010 case aco_opcode::v_cmp_n##unord##_f32:\
1011 *ordered = aco_opcode::v_cmp_##ord##_f32;\
1012 *unordered = aco_opcode::v_cmp_n##unord##_f32;\
1013 *inverse = op == aco_opcode::v_cmp_n##unord##_f32 ? aco_opcode::v_cmp_##unord##_f32 : aco_opcode::v_cmp_n##ord##_f32;\
1014 return true;
1015 CMP(lt, /*n*/ge)
1016 CMP(eq, /*n*/lg)
1017 CMP(le, /*n*/gt)
1018 CMP(gt, /*n*/le)
1019 CMP(lg, /*n*/eq)
1020 CMP(ge, /*n*/lt)
1021 #undef CMP
1022 default:
1023 return false;
1024 }
1025 }
1026
1027 aco_opcode get_ordered(aco_opcode op)
1028 {
1029 aco_opcode ordered, unordered, inverse;
1030 return get_cmp_info(op, &ordered, &unordered, &inverse) ? ordered : aco_opcode::last_opcode;
1031 }
1032
1033 aco_opcode get_unordered(aco_opcode op)
1034 {
1035 aco_opcode ordered, unordered, inverse;
1036 return get_cmp_info(op, &ordered, &unordered, &inverse) ? unordered : aco_opcode::last_opcode;
1037 }
1038
1039 aco_opcode get_inverse(aco_opcode op)
1040 {
1041 aco_opcode ordered, unordered, inverse;
1042 return get_cmp_info(op, &ordered, &unordered, &inverse) ? inverse : aco_opcode::last_opcode;
1043 }
1044
1045 bool is_cmp(aco_opcode op)
1046 {
1047 aco_opcode ordered, unordered, inverse;
1048 return get_cmp_info(op, &ordered, &unordered, &inverse);
1049 }
1050
1051 unsigned original_temp_id(opt_ctx &ctx, Temp tmp)
1052 {
1053 if (ctx.info[tmp.id()].is_temp())
1054 return ctx.info[tmp.id()].temp.id();
1055 else
1056 return tmp.id();
1057 }
1058
1059 void decrease_uses(opt_ctx &ctx, Instruction* instr)
1060 {
1061 if (!--ctx.uses[instr->definitions[0].tempId()]) {
1062 for (const Operand& op : instr->operands) {
1063 if (op.isTemp())
1064 ctx.uses[op.tempId()]--;
1065 }
1066 }
1067 }
1068
1069 Instruction *follow_operand(opt_ctx &ctx, Operand op, bool ignore_uses=false)
1070 {
1071 if (!op.isTemp() || !(ctx.info[op.tempId()].label & instr_labels))
1072 return nullptr;
1073 if (!ignore_uses && ctx.uses[op.tempId()] > 1)
1074 return nullptr;
1075
1076 Instruction *instr = ctx.info[op.tempId()].instr;
1077
1078 if (instr->definitions.size() == 2) {
1079 assert(instr->definitions[0].isTemp() && instr->definitions[0].tempId() == op.tempId());
1080 if (instr->definitions[1].isTemp() && ctx.uses[instr->definitions[1].tempId()])
1081 return nullptr;
1082 }
1083
1084 return instr;
1085 }
1086
1087 /* s_or_b64(neq(a, a), neq(b, b)) -> v_cmp_u_f32(a, b)
1088 * s_and_b64(eq(a, a), eq(b, b)) -> v_cmp_o_f32(a, b) */
1089 bool combine_ordering_test(opt_ctx &ctx, aco_ptr<Instruction>& instr)
1090 {
1091 if (instr->opcode != aco_opcode::s_or_b64 && instr->opcode != aco_opcode::s_and_b64)
1092 return false;
1093 if (instr->definitions[1].isTemp() && ctx.uses[instr->definitions[1].tempId()])
1094 return false;
1095
1096 bool neg[2] = {false, false};
1097 bool abs[2] = {false, false};
1098 bool opsel[2] = {false, false};
1099 Instruction *op_instr[2];
1100 Temp op[2];
1101
1102 for (unsigned i = 0; i < 2; i++) {
1103 op_instr[i] = follow_operand(ctx, instr->operands[i], true);
1104 if (!op_instr[i])
1105 return false;
1106
1107 aco_opcode expected_cmp = instr->opcode == aco_opcode::s_or_b64 ?
1108 aco_opcode::v_cmp_neq_f32 : aco_opcode::v_cmp_eq_f32;
1109
1110 if (op_instr[i]->opcode != expected_cmp)
1111 return false;
1112 if (!op_instr[i]->operands[0].isTemp() || !op_instr[i]->operands[1].isTemp())
1113 return false;
1114
1115 if (op_instr[i]->isVOP3()) {
1116 VOP3A_instruction *vop3 = static_cast<VOP3A_instruction*>(op_instr[i]);
1117 if (vop3->neg[0] != vop3->neg[1] || vop3->abs[0] != vop3->abs[1] || vop3->opsel[0] != vop3->opsel[1])
1118 return false;
1119 neg[i] = vop3->neg[0];
1120 abs[i] = vop3->abs[0];
1121 opsel[i] = vop3->opsel[0];
1122 }
1123
1124 Temp op0 = op_instr[i]->operands[0].getTemp();
1125 Temp op1 = op_instr[i]->operands[1].getTemp();
1126 if (original_temp_id(ctx, op0) != original_temp_id(ctx, op1))
1127 return false;
1128 /* shouldn't happen yet, but best to be safe */
1129 if (op1.type() != RegType::vgpr)
1130 return false;
1131
1132 op[i] = op1;
1133 }
1134
1135 ctx.uses[op[0].id()]++;
1136 ctx.uses[op[1].id()]++;
1137 decrease_uses(ctx, op_instr[0]);
1138 decrease_uses(ctx, op_instr[1]);
1139
1140 aco_opcode new_op = instr->opcode == aco_opcode::s_or_b64 ?
1141 aco_opcode::v_cmp_u_f32 : aco_opcode::v_cmp_o_f32;
1142 Instruction *new_instr;
1143 if (neg[0] || neg[1] || abs[0] || abs[1] || opsel[0] || opsel[1]) {
1144 VOP3A_instruction *vop3 = create_instruction<VOP3A_instruction>(new_op, asVOP3(Format::VOPC), 2, 1);
1145 for (unsigned i = 0; i < 2; i++) {
1146 vop3->neg[i] = neg[i];
1147 vop3->abs[i] = abs[i];
1148 vop3->opsel[i] = opsel[i];
1149 }
1150 new_instr = static_cast<Instruction *>(vop3);
1151 } else {
1152 new_instr = create_instruction<VOPC_instruction>(new_op, Format::VOPC, 2, 1);
1153 }
1154 new_instr->operands[0] = Operand(op[0]);
1155 new_instr->operands[1] = Operand(op[1]);
1156 new_instr->definitions[0] = instr->definitions[0];
1157
1158 ctx.info[instr->definitions[0].tempId()].label = 0;
1159 ctx.info[instr->definitions[0].tempId()].set_fcmp(new_instr);
1160
1161 instr.reset(new_instr);
1162
1163 return true;
1164 }
1165
1166 /* s_or_b64(v_cmp_u_f32(a, b), cmp(a, b)) -> get_unordered(cmp)(a, b)
1167 * s_and_b64(v_cmp_o_f32(a, b), cmp(a, b)) -> get_ordered(cmp)(a, b) */
1168 bool combine_comparison_ordering(opt_ctx &ctx, aco_ptr<Instruction>& instr)
1169 {
1170 if (instr->opcode != aco_opcode::s_or_b64 && instr->opcode != aco_opcode::s_and_b64)
1171 return false;
1172 if (instr->definitions[1].isTemp() && ctx.uses[instr->definitions[1].tempId()])
1173 return false;
1174
1175 aco_opcode expected_nan_test = instr->opcode == aco_opcode::s_or_b64 ?
1176 aco_opcode::v_cmp_u_f32 : aco_opcode::v_cmp_o_f32;
1177
1178 Instruction *nan_test = follow_operand(ctx, instr->operands[0], true);
1179 Instruction *cmp = follow_operand(ctx, instr->operands[1], true);
1180 if (!nan_test || !cmp)
1181 return false;
1182
1183 if (cmp->opcode == expected_nan_test)
1184 std::swap(nan_test, cmp);
1185 else if (nan_test->opcode != expected_nan_test)
1186 return false;
1187
1188 if (!is_cmp(cmp->opcode))
1189 return false;
1190
1191 if (!nan_test->operands[0].isTemp() || !nan_test->operands[1].isTemp())
1192 return false;
1193 if (!cmp->operands[0].isTemp() || !cmp->operands[1].isTemp())
1194 return false;
1195
1196 unsigned prop_cmp0 = original_temp_id(ctx, cmp->operands[0].getTemp());
1197 unsigned prop_cmp1 = original_temp_id(ctx, cmp->operands[1].getTemp());
1198 unsigned prop_nan0 = original_temp_id(ctx, nan_test->operands[0].getTemp());
1199 unsigned prop_nan1 = original_temp_id(ctx, nan_test->operands[1].getTemp());
1200 if (prop_cmp0 != prop_nan0 && prop_cmp0 != prop_nan1)
1201 return false;
1202 if (prop_cmp1 != prop_nan0 && prop_cmp1 != prop_nan1)
1203 return false;
1204
1205 ctx.uses[cmp->operands[0].tempId()]++;
1206 ctx.uses[cmp->operands[1].tempId()]++;
1207 decrease_uses(ctx, nan_test);
1208 decrease_uses(ctx, cmp);
1209
1210 aco_opcode new_op = instr->opcode == aco_opcode::s_or_b64 ?
1211 get_unordered(cmp->opcode) : get_ordered(cmp->opcode);
1212 Instruction *new_instr;
1213 if (cmp->isVOP3()) {
1214 VOP3A_instruction *new_vop3 = create_instruction<VOP3A_instruction>(new_op, asVOP3(Format::VOPC), 2, 1);
1215 VOP3A_instruction *cmp_vop3 = static_cast<VOP3A_instruction*>(cmp);
1216 memcpy(new_vop3->abs, cmp_vop3->abs, sizeof(new_vop3->abs));
1217 memcpy(new_vop3->opsel, cmp_vop3->opsel, sizeof(new_vop3->opsel));
1218 memcpy(new_vop3->neg, cmp_vop3->neg, sizeof(new_vop3->neg));
1219 new_vop3->clamp = cmp_vop3->clamp;
1220 new_vop3->omod = cmp_vop3->omod;
1221 new_instr = new_vop3;
1222 } else {
1223 new_instr = create_instruction<VOPC_instruction>(new_op, Format::VOPC, 2, 1);
1224 }
1225 new_instr->operands[0] = cmp->operands[0];
1226 new_instr->operands[1] = cmp->operands[1];
1227 new_instr->definitions[0] = instr->definitions[0];
1228
1229 ctx.info[instr->definitions[0].tempId()].label = 0;
1230 ctx.info[instr->definitions[0].tempId()].set_fcmp(new_instr);
1231
1232 instr.reset(new_instr);
1233
1234 return true;
1235 }
1236
1237 /* s_or_b64(v_cmp_neq_f32(a, a), cmp(a, #b)) and b is not NaN -> get_unordered(cmp)(a, b)
1238 * s_and_b64(v_cmp_eq_f32(a, a), cmp(a, #b)) and b is not NaN -> get_ordered(cmp)(a, b) */
1239 bool combine_constant_comparison_ordering(opt_ctx &ctx, aco_ptr<Instruction>& instr)
1240 {
1241 if (instr->opcode != aco_opcode::s_or_b64 && instr->opcode != aco_opcode::s_and_b64)
1242 return false;
1243 if (instr->definitions[1].isTemp() && ctx.uses[instr->definitions[1].tempId()])
1244 return false;
1245
1246 Instruction *nan_test = follow_operand(ctx, instr->operands[0], true);
1247 Instruction *cmp = follow_operand(ctx, instr->operands[1], true);
1248
1249 if (!nan_test || !cmp)
1250 return false;
1251
1252 aco_opcode expected_nan_test = instr->opcode == aco_opcode::s_or_b64 ?
1253 aco_opcode::v_cmp_neq_f32 : aco_opcode::v_cmp_eq_f32;
1254 if (cmp->opcode == expected_nan_test)
1255 std::swap(nan_test, cmp);
1256 else if (nan_test->opcode != expected_nan_test)
1257 return false;
1258
1259 if (!is_cmp(cmp->opcode))
1260 return false;
1261
1262 if (!nan_test->operands[0].isTemp() || !nan_test->operands[1].isTemp())
1263 return false;
1264 if (!cmp->operands[0].isTemp() && !cmp->operands[1].isTemp())
1265 return false;
1266
1267 unsigned prop_nan0 = original_temp_id(ctx, nan_test->operands[0].getTemp());
1268 unsigned prop_nan1 = original_temp_id(ctx, nan_test->operands[1].getTemp());
1269 if (prop_nan0 != prop_nan1)
1270 return false;
1271
1272 int constant_operand = -1;
1273 for (unsigned i = 0; i < 2; i++) {
1274 if (cmp->operands[i].isTemp() && original_temp_id(ctx, cmp->operands[i].getTemp()) == prop_nan0) {
1275 constant_operand = !i;
1276 break;
1277 }
1278 }
1279 if (constant_operand == -1)
1280 return false;
1281
1282 uint32_t constant;
1283 if (cmp->operands[constant_operand].isConstant()) {
1284 constant = cmp->operands[constant_operand].constantValue();
1285 } else if (cmp->operands[constant_operand].isTemp()) {
1286 unsigned id = cmp->operands[constant_operand].tempId();
1287 if (!ctx.info[id].is_constant() && !ctx.info[id].is_literal())
1288 return false;
1289 constant = ctx.info[id].val;
1290 } else {
1291 return false;
1292 }
1293
1294 float constantf;
1295 memcpy(&constantf, &constant, 4);
1296 if (isnan(constantf))
1297 return false;
1298
1299 if (cmp->operands[0].isTemp())
1300 ctx.uses[cmp->operands[0].tempId()]++;
1301 if (cmp->operands[1].isTemp())
1302 ctx.uses[cmp->operands[1].tempId()]++;
1303 decrease_uses(ctx, nan_test);
1304 decrease_uses(ctx, cmp);
1305
1306 aco_opcode new_op = instr->opcode == aco_opcode::s_or_b64 ?
1307 get_unordered(cmp->opcode) : get_ordered(cmp->opcode);
1308 Instruction *new_instr;
1309 if (cmp->isVOP3()) {
1310 VOP3A_instruction *new_vop3 = create_instruction<VOP3A_instruction>(new_op, asVOP3(Format::VOPC), 2, 1);
1311 VOP3A_instruction *cmp_vop3 = static_cast<VOP3A_instruction*>(cmp);
1312 memcpy(new_vop3->abs, cmp_vop3->abs, sizeof(new_vop3->abs));
1313 memcpy(new_vop3->opsel, cmp_vop3->opsel, sizeof(new_vop3->opsel));
1314 memcpy(new_vop3->neg, cmp_vop3->neg, sizeof(new_vop3->neg));
1315 new_vop3->clamp = cmp_vop3->clamp;
1316 new_vop3->omod = cmp_vop3->omod;
1317 new_instr = new_vop3;
1318 } else {
1319 new_instr = create_instruction<VOPC_instruction>(new_op, Format::VOPC, 2, 1);
1320 }
1321 new_instr->operands[0] = cmp->operands[0];
1322 new_instr->operands[1] = cmp->operands[1];
1323 new_instr->definitions[0] = instr->definitions[0];
1324
1325 ctx.info[instr->definitions[0].tempId()].label = 0;
1326 ctx.info[instr->definitions[0].tempId()].set_fcmp(new_instr);
1327
1328 instr.reset(new_instr);
1329
1330 return true;
1331 }
1332
1333 /* s_not_b64(cmp(a, b) -> get_inverse(cmp)(a, b) */
1334 bool combine_inverse_comparison(opt_ctx &ctx, aco_ptr<Instruction>& instr)
1335 {
1336 if (instr->opcode != aco_opcode::s_not_b64)
1337 return false;
1338 if (instr->definitions[1].isTemp() && ctx.uses[instr->definitions[1].tempId()])
1339 return false;
1340 if (!instr->operands[0].isTemp())
1341 return false;
1342
1343 Instruction *cmp = follow_operand(ctx, instr->operands[0]);
1344 if (!cmp)
1345 return false;
1346
1347 aco_opcode new_opcode = get_inverse(cmp->opcode);
1348 if (new_opcode == aco_opcode::last_opcode)
1349 return false;
1350
1351 if (cmp->operands[0].isTemp())
1352 ctx.uses[cmp->operands[0].tempId()]++;
1353 if (cmp->operands[1].isTemp())
1354 ctx.uses[cmp->operands[1].tempId()]++;
1355 decrease_uses(ctx, cmp);
1356
1357 Instruction *new_instr;
1358 if (cmp->isVOP3()) {
1359 VOP3A_instruction *new_vop3 = create_instruction<VOP3A_instruction>(new_opcode, asVOP3(Format::VOPC), 2, 1);
1360 VOP3A_instruction *cmp_vop3 = static_cast<VOP3A_instruction*>(cmp);
1361 memcpy(new_vop3->abs, cmp_vop3->abs, sizeof(new_vop3->abs));
1362 memcpy(new_vop3->opsel, cmp_vop3->opsel, sizeof(new_vop3->opsel));
1363 memcpy(new_vop3->neg, cmp_vop3->neg, sizeof(new_vop3->neg));
1364 new_vop3->clamp = cmp_vop3->clamp;
1365 new_vop3->omod = cmp_vop3->omod;
1366 new_instr = new_vop3;
1367 } else {
1368 new_instr = create_instruction<VOPC_instruction>(new_opcode, Format::VOPC, 2, 1);
1369 }
1370 new_instr->operands[0] = cmp->operands[0];
1371 new_instr->operands[1] = cmp->operands[1];
1372 new_instr->definitions[0] = instr->definitions[0];
1373
1374 ctx.info[instr->definitions[0].tempId()].label = 0;
1375 ctx.info[instr->definitions[0].tempId()].set_fcmp(new_instr);
1376
1377 instr.reset(new_instr);
1378
1379 return true;
1380 }
1381
1382 /* op1(op2(1, 2), 0) if swap = false
1383 * op1(0, op2(1, 2)) if swap = true */
1384 bool match_op3_for_vop3(opt_ctx &ctx, aco_opcode op1, aco_opcode op2,
1385 Instruction* op1_instr, bool swap, const char *shuffle_str,
1386 Operand operands[3], bool neg[3], bool abs[3], bool opsel[3],
1387 bool *op1_clamp, unsigned *op1_omod,
1388 bool *inbetween_neg, bool *inbetween_abs, bool *inbetween_opsel)
1389 {
1390 /* checks */
1391 if (op1_instr->opcode != op1)
1392 return false;
1393
1394 Instruction *op2_instr = follow_operand(ctx, op1_instr->operands[swap]);
1395 if (!op2_instr || op2_instr->opcode != op2)
1396 return false;
1397
1398 VOP3A_instruction *op1_vop3 = op1_instr->isVOP3() ? static_cast<VOP3A_instruction *>(op1_instr) : NULL;
1399 VOP3A_instruction *op2_vop3 = op2_instr->isVOP3() ? static_cast<VOP3A_instruction *>(op2_instr) : NULL;
1400
1401 /* don't support inbetween clamp/omod */
1402 if (op2_vop3 && (op2_vop3->clamp || op2_vop3->omod))
1403 return false;
1404
1405 /* get operands and modifiers and check inbetween modifiers */
1406 *op1_clamp = op1_vop3 ? op1_vop3->clamp : false;
1407 *op1_omod = op1_vop3 ? op1_vop3->omod : 0u;
1408
1409 if (inbetween_neg)
1410 *inbetween_neg = op1_vop3 ? op1_vop3->neg[swap] : false;
1411 else if (op1_vop3 && op1_vop3->neg[swap])
1412 return false;
1413
1414 if (inbetween_abs)
1415 *inbetween_abs = op1_vop3 ? op1_vop3->abs[swap] : false;
1416 else if (op1_vop3 && op1_vop3->abs[swap])
1417 return false;
1418
1419 if (inbetween_opsel)
1420 *inbetween_opsel = op1_vop3 ? op1_vop3->opsel[swap] : false;
1421 else if (op1_vop3 && op1_vop3->opsel[swap])
1422 return false;
1423
1424 int shuffle[3];
1425 shuffle[shuffle_str[0] - '0'] = 0;
1426 shuffle[shuffle_str[1] - '0'] = 1;
1427 shuffle[shuffle_str[2] - '0'] = 2;
1428
1429 operands[shuffle[0]] = op1_instr->operands[!swap];
1430 neg[shuffle[0]] = op1_vop3 ? op1_vop3->neg[!swap] : false;
1431 abs[shuffle[0]] = op1_vop3 ? op1_vop3->abs[!swap] : false;
1432 opsel[shuffle[0]] = op1_vop3 ? op1_vop3->opsel[!swap] : false;
1433
1434 for (unsigned i = 0; i < 2; i++) {
1435 operands[shuffle[i + 1]] = op2_instr->operands[i];
1436 neg[shuffle[i + 1]] = op2_vop3 ? op2_vop3->neg[i] : false;
1437 abs[shuffle[i + 1]] = op2_vop3 ? op2_vop3->abs[i] : false;
1438 opsel[shuffle[i + 1]] = op2_vop3 ? op2_vop3->opsel[i] : false;
1439 }
1440
1441 /* check operands */
1442 unsigned sgpr_id = 0;
1443 for (unsigned i = 0; i < 3; i++) {
1444 Operand op = operands[i];
1445 if (op.isLiteral()) {
1446 return false;
1447 } else if (op.isTemp() && op.getTemp().type() == RegType::sgpr) {
1448 if (sgpr_id && sgpr_id != op.tempId())
1449 return false;
1450 sgpr_id = op.tempId();
1451 }
1452 }
1453
1454 return true;
1455 }
1456
1457 void create_vop3_for_op3(opt_ctx& ctx, aco_opcode opcode, aco_ptr<Instruction>& instr,
1458 Operand operands[3], bool neg[3], bool abs[3], bool opsel[3],
1459 bool clamp, unsigned omod)
1460 {
1461 VOP3A_instruction *new_instr = create_instruction<VOP3A_instruction>(opcode, Format::VOP3A, 3, 1);
1462 memcpy(new_instr->abs, abs, sizeof(bool[3]));
1463 memcpy(new_instr->opsel, opsel, sizeof(bool[3]));
1464 memcpy(new_instr->neg, neg, sizeof(bool[3]));
1465 new_instr->clamp = clamp;
1466 new_instr->omod = omod;
1467 new_instr->operands[0] = operands[0];
1468 new_instr->operands[1] = operands[1];
1469 new_instr->operands[2] = operands[2];
1470 new_instr->definitions[0] = instr->definitions[0];
1471 ctx.info[instr->definitions[0].tempId()].label = 0;
1472
1473 instr.reset(new_instr);
1474 }
1475
1476 bool combine_minmax3(opt_ctx& ctx, aco_ptr<Instruction>& instr, aco_opcode new_op)
1477 {
1478 uint32_t omod_clamp = ctx.info[instr->definitions[0].tempId()].label &
1479 (label_omod_success | label_clamp_success);
1480
1481 for (unsigned swap = 0; swap < 2; swap++) {
1482 Operand operands[3];
1483 bool neg[3], abs[3], opsel[3], clamp, inbetween_neg, inbetween_abs;
1484 unsigned omod;
1485 if (match_op3_for_vop3(ctx, instr->opcode, instr->opcode, instr.get(), swap,
1486 "012", operands, neg, abs, opsel,
1487 &clamp, &omod, &inbetween_neg, &inbetween_abs, NULL)) {
1488 ctx.uses[instr->operands[swap].tempId()]--;
1489 neg[1] ^= inbetween_neg;
1490 neg[2] ^= inbetween_neg;
1491 abs[1] |= inbetween_abs;
1492 abs[2] |= inbetween_abs;
1493 create_vop3_for_op3(ctx, new_op, instr, operands, neg, abs, opsel, clamp, omod);
1494 if (omod_clamp & label_omod_success)
1495 ctx.info[instr->definitions[0].tempId()].set_omod_success(instr.get());
1496 if (omod_clamp & label_clamp_success)
1497 ctx.info[instr->definitions[0].tempId()].set_clamp_success(instr.get());
1498 return true;
1499 }
1500 }
1501 return false;
1502 }
1503
1504 bool combine_three_valu_op(opt_ctx& ctx, aco_ptr<Instruction>& instr, aco_opcode op2, aco_opcode new_op, const char *shuffle, uint8_t ops)
1505 {
1506 uint32_t omod_clamp = ctx.info[instr->definitions[0].tempId()].label &
1507 (label_omod_success | label_clamp_success);
1508
1509 for (unsigned swap = 0; swap < 2; swap++) {
1510 if (!((1 << swap) & ops))
1511 continue;
1512
1513 Operand operands[3];
1514 bool neg[3], abs[3], opsel[3], clamp;
1515 unsigned omod;
1516 if (match_op3_for_vop3(ctx, instr->opcode, op2,
1517 instr.get(), swap, shuffle,
1518 operands, neg, abs, opsel,
1519 &clamp, &omod, NULL, NULL, NULL)) {
1520 ctx.uses[instr->operands[swap].tempId()]--;
1521 create_vop3_for_op3(ctx, new_op, instr, operands, neg, abs, opsel, clamp, omod);
1522 if (omod_clamp & label_omod_success)
1523 ctx.info[instr->definitions[0].tempId()].set_omod_success(instr.get());
1524 if (omod_clamp & label_clamp_success)
1525 ctx.info[instr->definitions[0].tempId()].set_clamp_success(instr.get());
1526 return true;
1527 }
1528 }
1529 return false;
1530 }
1531
1532 /* s_not_b32(s_and_b32(a, b)) -> s_nand_b32(a, b)
1533 * s_not_b32(s_or_b32(a, b)) -> s_nor_b32(a, b)
1534 * s_not_b32(s_xor_b32(a, b)) -> s_xnor_b32(a, b)
1535 * s_not_b64(s_and_b64(a, b)) -> s_nand_b64(a, b)
1536 * s_not_b64(s_or_b64(a, b)) -> s_nor_b64(a, b)
1537 * s_not_b64(s_xor_b64(a, b)) -> s_xnor_b64(a, b) */
1538 bool combine_salu_not_bitwise(opt_ctx& ctx, aco_ptr<Instruction>& instr)
1539 {
1540 /* checks */
1541 if (!instr->operands[0].isTemp())
1542 return false;
1543 if (instr->definitions[1].isTemp() && ctx.uses[instr->definitions[1].tempId()])
1544 return false;
1545
1546 Instruction *op2_instr = follow_operand(ctx, instr->operands[0]);
1547 if (!op2_instr)
1548 return false;
1549 switch (op2_instr->opcode) {
1550 case aco_opcode::s_and_b32:
1551 case aco_opcode::s_or_b32:
1552 case aco_opcode::s_xor_b32:
1553 case aco_opcode::s_and_b64:
1554 case aco_opcode::s_or_b64:
1555 case aco_opcode::s_xor_b64:
1556 break;
1557 default:
1558 return false;
1559 }
1560
1561 /* create instruction */
1562 std::swap(instr->definitions[0], op2_instr->definitions[0]);
1563 ctx.uses[instr->operands[0].tempId()]--;
1564 ctx.info[op2_instr->definitions[0].tempId()].label = 0;
1565
1566 switch (op2_instr->opcode) {
1567 case aco_opcode::s_and_b32:
1568 op2_instr->opcode = aco_opcode::s_nand_b32;
1569 break;
1570 case aco_opcode::s_or_b32:
1571 op2_instr->opcode = aco_opcode::s_nor_b32;
1572 break;
1573 case aco_opcode::s_xor_b32:
1574 op2_instr->opcode = aco_opcode::s_xnor_b32;
1575 break;
1576 case aco_opcode::s_and_b64:
1577 op2_instr->opcode = aco_opcode::s_nand_b64;
1578 break;
1579 case aco_opcode::s_or_b64:
1580 op2_instr->opcode = aco_opcode::s_nor_b64;
1581 break;
1582 case aco_opcode::s_xor_b64:
1583 op2_instr->opcode = aco_opcode::s_xnor_b64;
1584 break;
1585 default:
1586 break;
1587 }
1588
1589 return true;
1590 }
1591
1592 /* s_and_b32(a, s_not_b32(b)) -> s_andn2_b32(a, b)
1593 * s_or_b32(a, s_not_b32(b)) -> s_orn2_b32(a, b)
1594 * s_and_b64(a, s_not_b64(b)) -> s_andn2_b64(a, b)
1595 * s_or_b64(a, s_not_b64(b)) -> s_orn2_b64(a, b) */
1596 bool combine_salu_n2(opt_ctx& ctx, aco_ptr<Instruction>& instr)
1597 {
1598 if (instr->definitions[1].isTemp() && ctx.uses[instr->definitions[1].tempId()])
1599 return false;
1600
1601 for (unsigned i = 0; i < 2; i++) {
1602 Instruction *op2_instr = follow_operand(ctx, instr->operands[i]);
1603 if (!op2_instr || (op2_instr->opcode != aco_opcode::s_not_b32 && op2_instr->opcode != aco_opcode::s_not_b64))
1604 continue;
1605
1606 ctx.uses[instr->operands[i].tempId()]--;
1607 instr->operands[0] = instr->operands[!i];
1608 instr->operands[1] = op2_instr->operands[0];
1609 ctx.info[instr->definitions[0].tempId()].label = 0;
1610
1611 switch (instr->opcode) {
1612 case aco_opcode::s_and_b32:
1613 instr->opcode = aco_opcode::s_andn2_b32;
1614 break;
1615 case aco_opcode::s_or_b32:
1616 instr->opcode = aco_opcode::s_orn2_b32;
1617 break;
1618 case aco_opcode::s_and_b64:
1619 instr->opcode = aco_opcode::s_andn2_b64;
1620 break;
1621 case aco_opcode::s_or_b64:
1622 instr->opcode = aco_opcode::s_orn2_b64;
1623 break;
1624 default:
1625 break;
1626 }
1627
1628 return true;
1629 }
1630 return false;
1631 }
1632
1633 /* s_add_{i32,u32}(a, s_lshl_b32(b, <n>)) -> s_lshl<n>_add_u32(a, b) */
1634 bool combine_salu_lshl_add(opt_ctx& ctx, aco_ptr<Instruction>& instr)
1635 {
1636 if (instr->definitions[1].isTemp() && ctx.uses[instr->definitions[1].tempId()])
1637 return false;
1638
1639 for (unsigned i = 0; i < 2; i++) {
1640 Instruction *op2_instr = follow_operand(ctx, instr->operands[i]);
1641 if (!op2_instr || op2_instr->opcode != aco_opcode::s_lshl_b32 || !op2_instr->operands[1].isConstant())
1642 continue;
1643
1644 uint32_t shift = op2_instr->operands[1].constantValue();
1645 if (shift < 1 || shift > 4)
1646 continue;
1647
1648 ctx.uses[instr->operands[i].tempId()]--;
1649 instr->operands[1] = instr->operands[!i];
1650 instr->operands[0] = op2_instr->operands[0];
1651 ctx.info[instr->definitions[0].tempId()].label = 0;
1652
1653 instr->opcode = ((aco_opcode[]){aco_opcode::s_lshl1_add_u32,
1654 aco_opcode::s_lshl2_add_u32,
1655 aco_opcode::s_lshl3_add_u32,
1656 aco_opcode::s_lshl4_add_u32})[shift - 1];
1657
1658 return true;
1659 }
1660 return false;
1661 }
1662
1663 bool get_minmax_info(aco_opcode op, aco_opcode *min, aco_opcode *max, aco_opcode *min3, aco_opcode *max3, aco_opcode *med3, bool *some_gfx9_only)
1664 {
1665 switch (op) {
1666 #define MINMAX(type, gfx9) \
1667 case aco_opcode::v_min_##type:\
1668 case aco_opcode::v_max_##type:\
1669 case aco_opcode::v_med3_##type:\
1670 *min = aco_opcode::v_min_##type;\
1671 *max = aco_opcode::v_max_##type;\
1672 *med3 = aco_opcode::v_med3_##type;\
1673 *min3 = aco_opcode::v_min3_##type;\
1674 *max3 = aco_opcode::v_max3_##type;\
1675 *some_gfx9_only = gfx9;\
1676 return true;
1677 MINMAX(f32, false)
1678 MINMAX(u32, false)
1679 MINMAX(i32, false)
1680 MINMAX(f16, true)
1681 MINMAX(u16, true)
1682 MINMAX(i16, true)
1683 #undef MINMAX
1684 default:
1685 return false;
1686 }
1687 }
1688
1689 /* v_min_{f,u,i}{16,32}(v_max_{f,u,i}{16,32}(a, lb), ub) -> v_med3_{f,u,i}{16,32}(a, lb, ub) when ub > lb
1690 * v_max_{f,u,i}{16,32}(v_min_{f,u,i}{16,32}(a, ub), lb) -> v_med3_{f,u,i}{16,32}(a, lb, ub) when ub > lb */
1691 bool combine_clamp(opt_ctx& ctx, aco_ptr<Instruction>& instr,
1692 aco_opcode min, aco_opcode max, aco_opcode med)
1693 {
1694 aco_opcode other_op;
1695 if (instr->opcode == min)
1696 other_op = max;
1697 else if (instr->opcode == max)
1698 other_op = min;
1699 else
1700 return false;
1701
1702 uint32_t omod_clamp = ctx.info[instr->definitions[0].tempId()].label &
1703 (label_omod_success | label_clamp_success);
1704
1705 for (unsigned swap = 0; swap < 2; swap++) {
1706 Operand operands[3];
1707 bool neg[3], abs[3], opsel[3], clamp, inbetween_neg, inbetween_abs;
1708 unsigned omod;
1709 if (match_op3_for_vop3(ctx, instr->opcode, other_op, instr.get(), swap,
1710 "012", operands, neg, abs, opsel,
1711 &clamp, &omod, &inbetween_neg, &inbetween_abs, NULL)) {
1712 int const0_idx = -1, const1_idx = -1;
1713 uint32_t const0 = 0, const1 = 0;
1714 for (int i = 0; i < 3; i++) {
1715 uint32_t val;
1716 if (operands[i].isConstant()) {
1717 val = operands[i].constantValue();
1718 } else if (operands[i].isTemp() && ctx.uses[operands[i].tempId()] == 1 &&
1719 ctx.info[operands[i].tempId()].is_constant_or_literal()) {
1720 val = ctx.info[operands[i].tempId()].val;
1721 } else {
1722 continue;
1723 }
1724 if (const0_idx >= 0) {
1725 const1_idx = i;
1726 const1 = val;
1727 } else {
1728 const0_idx = i;
1729 const0 = val;
1730 }
1731 }
1732 if (const0_idx < 0 || const1_idx < 0)
1733 continue;
1734
1735 if (opsel[const0_idx])
1736 const0 >>= 16;
1737 if (opsel[const1_idx])
1738 const1 >>= 16;
1739
1740 int lower_idx = const0_idx;
1741 switch (min) {
1742 case aco_opcode::v_min_f32:
1743 case aco_opcode::v_min_f16: {
1744 float const0_f, const1_f;
1745 if (min == aco_opcode::v_min_f32) {
1746 memcpy(&const0_f, &const0, 4);
1747 memcpy(&const1_f, &const1, 4);
1748 } else {
1749 const0_f = _mesa_half_to_float(const0);
1750 const1_f = _mesa_half_to_float(const1);
1751 }
1752 if (abs[const0_idx]) const0_f = fabsf(const0_f);
1753 if (abs[const1_idx]) const1_f = fabsf(const1_f);
1754 if (neg[const0_idx]) const0_f = -const0_f;
1755 if (neg[const1_idx]) const1_f = -const1_f;
1756 lower_idx = const0_f < const1_f ? const0_idx : const1_idx;
1757 break;
1758 }
1759 case aco_opcode::v_min_u32: {
1760 lower_idx = const0 < const1 ? const0_idx : const1_idx;
1761 break;
1762 }
1763 case aco_opcode::v_min_u16: {
1764 lower_idx = (uint16_t)const0 < (uint16_t)const1 ? const0_idx : const1_idx;
1765 break;
1766 }
1767 case aco_opcode::v_min_i32: {
1768 int32_t const0_i = const0 & 0x80000000u ? -2147483648 + (int32_t)(const0 & 0x7fffffffu) : const0;
1769 int32_t const1_i = const1 & 0x80000000u ? -2147483648 + (int32_t)(const1 & 0x7fffffffu) : const1;
1770 lower_idx = const0_i < const1_i ? const0_idx : const1_idx;
1771 break;
1772 }
1773 case aco_opcode::v_min_i16: {
1774 int16_t const0_i = const0 & 0x8000u ? -32768 + (int16_t)(const0 & 0x7fffu) : const0;
1775 int16_t const1_i = const1 & 0x8000u ? -32768 + (int16_t)(const1 & 0x7fffu) : const1;
1776 lower_idx = const0_i < const1_i ? const0_idx : const1_idx;
1777 break;
1778 }
1779 default:
1780 break;
1781 }
1782 int upper_idx = lower_idx == const0_idx ? const1_idx : const0_idx;
1783
1784 if (instr->opcode == min) {
1785 if (upper_idx != 0 || lower_idx == 0)
1786 return false;
1787 } else {
1788 if (upper_idx == 0 || lower_idx != 0)
1789 return false;
1790 }
1791
1792 neg[1] ^= inbetween_neg;
1793 neg[2] ^= inbetween_neg;
1794 abs[1] |= inbetween_abs;
1795 abs[2] |= inbetween_abs;
1796
1797 ctx.uses[instr->operands[swap].tempId()]--;
1798 create_vop3_for_op3(ctx, med, instr, operands, neg, abs, opsel, clamp, omod);
1799 if (omod_clamp & label_omod_success)
1800 ctx.info[instr->definitions[0].tempId()].set_omod_success(instr.get());
1801 if (omod_clamp & label_clamp_success)
1802 ctx.info[instr->definitions[0].tempId()].set_clamp_success(instr.get());
1803
1804 return true;
1805 }
1806 }
1807
1808 return false;
1809 }
1810
1811
1812 void apply_sgprs(opt_ctx &ctx, aco_ptr<Instruction>& instr)
1813 {
1814 /* apply sgprs */
1815 uint32_t sgpr_idx = 0;
1816 uint32_t sgpr_info_id = 0;
1817 bool has_sgpr = false;
1818 uint32_t sgpr_ssa_id = 0;
1819 /* find 'best' possible sgpr */
1820 for (unsigned i = 0; i < instr->operands.size(); i++)
1821 {
1822 if (instr->operands[i].isLiteral()) {
1823 has_sgpr = true;
1824 break;
1825 }
1826 if (!instr->operands[i].isTemp())
1827 continue;
1828 if (instr->operands[i].getTemp().type() == RegType::sgpr) {
1829 has_sgpr = true;
1830 sgpr_ssa_id = instr->operands[i].tempId();
1831 continue;
1832 }
1833 ssa_info& info = ctx.info[instr->operands[i].tempId()];
1834 if (info.is_temp() && info.temp.type() == RegType::sgpr) {
1835 uint16_t uses = ctx.uses[instr->operands[i].tempId()];
1836 if (sgpr_info_id == 0 || uses < ctx.uses[sgpr_info_id]) {
1837 sgpr_idx = i;
1838 sgpr_info_id = instr->operands[i].tempId();
1839 }
1840 }
1841 }
1842 if (!has_sgpr && sgpr_info_id != 0) {
1843 ssa_info& info = ctx.info[sgpr_info_id];
1844 if (sgpr_idx == 0 || instr->isVOP3()) {
1845 instr->operands[sgpr_idx] = Operand(info.temp);
1846 ctx.uses[sgpr_info_id]--;
1847 ctx.uses[info.temp.id()]++;
1848 } else if (can_swap_operands(instr)) {
1849 instr->operands[sgpr_idx] = instr->operands[0];
1850 instr->operands[0] = Operand(info.temp);
1851 ctx.uses[sgpr_info_id]--;
1852 ctx.uses[info.temp.id()]++;
1853 } else if (can_use_VOP3(instr)) {
1854 to_VOP3(ctx, instr);
1855 instr->operands[sgpr_idx] = Operand(info.temp);
1856 ctx.uses[sgpr_info_id]--;
1857 ctx.uses[info.temp.id()]++;
1858 }
1859
1860 /* we can have two sgprs on one instruction if it is the same sgpr! */
1861 } else if (sgpr_info_id != 0 &&
1862 sgpr_ssa_id == sgpr_info_id &&
1863 ctx.uses[sgpr_info_id] == 1 &&
1864 can_use_VOP3(instr)) {
1865 to_VOP3(ctx, instr);
1866 instr->operands[sgpr_idx] = Operand(ctx.info[sgpr_info_id].temp);
1867 ctx.uses[sgpr_info_id]--;
1868 ctx.uses[ctx.info[sgpr_info_id].temp.id()]++;
1869 }
1870 }
1871
1872 bool apply_omod_clamp(opt_ctx &ctx, aco_ptr<Instruction>& instr)
1873 {
1874 /* check if we could apply omod on predecessor */
1875 if (instr->opcode == aco_opcode::v_mul_f32) {
1876 if (instr->operands[1].isTemp() && ctx.info[instr->operands[1].tempId()].is_omod_success()) {
1877
1878 /* omod was successfully applied */
1879 /* if the omod instruction is v_mad, we also have to change the original add */
1880 if (ctx.info[instr->operands[1].tempId()].is_mad()) {
1881 Instruction* add_instr = ctx.mad_infos[ctx.info[instr->operands[1].tempId()].val].add_instr.get();
1882 if (ctx.info[instr->definitions[0].tempId()].is_clamp())
1883 static_cast<VOP3A_instruction*>(add_instr)->clamp = true;
1884 add_instr->definitions[0] = instr->definitions[0];
1885 }
1886
1887 Instruction* omod_instr = ctx.info[instr->operands[1].tempId()].instr;
1888 /* check if we have an additional clamp modifier */
1889 if (ctx.info[instr->definitions[0].tempId()].is_clamp() && ctx.uses[instr->definitions[0].tempId()] == 1) {
1890 static_cast<VOP3A_instruction*>(omod_instr)->clamp = true;
1891 ctx.info[instr->definitions[0].tempId()].set_clamp_success(omod_instr);
1892 }
1893 /* change definition ssa-id of modified instruction */
1894 omod_instr->definitions[0] = instr->definitions[0];
1895
1896 /* change the definition of instr to something unused, e.g. the original omod def */
1897 instr->definitions[0] = Definition(instr->operands[1].getTemp());
1898 ctx.uses[instr->definitions[0].tempId()] = 0;
1899 return true;
1900 }
1901 if (!ctx.info[instr->definitions[0].tempId()].label) {
1902 /* in all other cases, label this instruction as option for multiply-add */
1903 ctx.info[instr->definitions[0].tempId()].set_mul(instr.get());
1904 }
1905 }
1906
1907 /* check if we could apply clamp on predecessor */
1908 if (instr->opcode == aco_opcode::v_med3_f32) {
1909 unsigned idx = 0;
1910 bool found_zero = false, found_one = false;
1911 for (unsigned i = 0; i < 3; i++)
1912 {
1913 if (instr->operands[i].constantEquals(0))
1914 found_zero = true;
1915 else if (instr->operands[i].constantEquals(0x3f800000)) /* 1.0 */
1916 found_one = true;
1917 else
1918 idx = i;
1919 }
1920 if (found_zero && found_one && instr->operands[idx].isTemp() &&
1921 ctx.info[instr->operands[idx].tempId()].is_clamp_success()) {
1922 /* clamp was successfully applied */
1923 /* if the clamp instruction is v_mad, we also have to change the original add */
1924 if (ctx.info[instr->operands[idx].tempId()].is_mad()) {
1925 Instruction* add_instr = ctx.mad_infos[ctx.info[instr->operands[idx].tempId()].val].add_instr.get();
1926 add_instr->definitions[0] = instr->definitions[0];
1927 }
1928 Instruction* clamp_instr = ctx.info[instr->operands[idx].tempId()].instr;
1929 /* change definition ssa-id of modified instruction */
1930 clamp_instr->definitions[0] = instr->definitions[0];
1931
1932 /* change the definition of instr to something unused, e.g. the original omod def */
1933 instr->definitions[0] = Definition(instr->operands[idx].getTemp());
1934 ctx.uses[instr->definitions[0].tempId()] = 0;
1935 return true;
1936 }
1937 }
1938
1939 /* apply omod / clamp modifiers if the def is used only once and the instruction can have modifiers */
1940 if (!instr->definitions.empty() && ctx.uses[instr->definitions[0].tempId()] == 1 &&
1941 can_use_VOP3(instr) && instr_info.can_use_output_modifiers[(int)instr->opcode]) {
1942 if(ctx.info[instr->definitions[0].tempId()].is_omod2()) {
1943 to_VOP3(ctx, instr);
1944 static_cast<VOP3A_instruction*>(instr.get())->omod = 1;
1945 ctx.info[instr->definitions[0].tempId()].set_omod_success(instr.get());
1946 } else if (ctx.info[instr->definitions[0].tempId()].is_omod4()) {
1947 to_VOP3(ctx, instr);
1948 static_cast<VOP3A_instruction*>(instr.get())->omod = 2;
1949 ctx.info[instr->definitions[0].tempId()].set_omod_success(instr.get());
1950 } else if (ctx.info[instr->definitions[0].tempId()].is_omod5()) {
1951 to_VOP3(ctx, instr);
1952 static_cast<VOP3A_instruction*>(instr.get())->omod = 3;
1953 ctx.info[instr->definitions[0].tempId()].set_omod_success(instr.get());
1954 } else if (ctx.info[instr->definitions[0].tempId()].is_clamp()) {
1955 to_VOP3(ctx, instr);
1956 static_cast<VOP3A_instruction*>(instr.get())->clamp = true;
1957 ctx.info[instr->definitions[0].tempId()].set_clamp_success(instr.get());
1958 }
1959 }
1960
1961 return false;
1962 }
1963
1964 // TODO: we could possibly move the whole label_instruction pass to combine_instruction:
1965 // this would mean that we'd have to fix the instruction uses while value propagation
1966
1967 void combine_instruction(opt_ctx &ctx, aco_ptr<Instruction>& instr)
1968 {
1969 if (instr->definitions.empty() || !ctx.uses[instr->definitions[0].tempId()])
1970 return;
1971
1972 if (instr->isVALU()) {
1973 apply_sgprs(ctx, instr);
1974 if (apply_omod_clamp(ctx, instr))
1975 return;
1976 }
1977
1978 /* TODO: There are still some peephole optimizations that could be done:
1979 * - abs(a - b) -> s_absdiff_i32
1980 * - various patterns for s_bitcmp{0,1}_b32 and s_bitset{0,1}_b32
1981 * - patterns for v_alignbit_b32 and v_alignbyte_b32
1982 * These aren't probably too interesting though.
1983 * There are also patterns for v_cmp_class_f{16,32,64}. This is difficult but
1984 * probably more useful than the previously mentioned optimizations.
1985 * The various comparison optimizations also currently only work with 32-bit
1986 * floats. */
1987
1988 /* neg(mul(a, b)) -> mul(neg(a), b) */
1989 if (ctx.info[instr->definitions[0].tempId()].is_neg() && ctx.uses[instr->operands[1].tempId()] == 1) {
1990 Temp val = ctx.info[instr->definitions[0].tempId()].temp;
1991
1992 if (!ctx.info[val.id()].is_mul())
1993 return;
1994
1995 Instruction* mul_instr = ctx.info[val.id()].instr;
1996
1997 if (mul_instr->operands[0].isLiteral())
1998 return;
1999 if (mul_instr->isVOP3() && static_cast<VOP3A_instruction*>(mul_instr)->clamp)
2000 return;
2001
2002 /* convert to mul(neg(a), b) */
2003 ctx.uses[mul_instr->definitions[0].tempId()]--;
2004 Definition def = instr->definitions[0];
2005 /* neg(abs(mul(a, b))) -> mul(neg(abs(a)), abs(b)) */
2006 bool is_abs = ctx.info[instr->definitions[0].tempId()].is_abs();
2007 instr.reset(create_instruction<VOP3A_instruction>(aco_opcode::v_mul_f32, asVOP3(Format::VOP2), 2, 1));
2008 instr->operands[0] = mul_instr->operands[0];
2009 instr->operands[1] = mul_instr->operands[1];
2010 instr->definitions[0] = def;
2011 VOP3A_instruction* new_mul = static_cast<VOP3A_instruction*>(instr.get());
2012 if (mul_instr->isVOP3()) {
2013 VOP3A_instruction* mul = static_cast<VOP3A_instruction*>(mul_instr);
2014 new_mul->neg[0] = mul->neg[0] && !is_abs;
2015 new_mul->neg[1] = mul->neg[1] && !is_abs;
2016 new_mul->abs[0] = mul->abs[0] || is_abs;
2017 new_mul->abs[1] = mul->abs[1] || is_abs;
2018 new_mul->omod = mul->omod;
2019 }
2020 new_mul->neg[0] ^= true;
2021 new_mul->clamp = false;
2022
2023 ctx.info[instr->definitions[0].tempId()].set_mul(instr.get());
2024 return;
2025 }
2026 /* combine mul+add -> mad */
2027 else if (instr->opcode == aco_opcode::v_add_f32 ||
2028 instr->opcode == aco_opcode::v_sub_f32 ||
2029 instr->opcode == aco_opcode::v_subrev_f32) {
2030
2031 uint32_t uses_src0 = UINT32_MAX;
2032 uint32_t uses_src1 = UINT32_MAX;
2033 Instruction* mul_instr = nullptr;
2034 unsigned add_op_idx;
2035 /* check if any of the operands is a multiplication */
2036 if (instr->operands[0].isTemp() && ctx.info[instr->operands[0].tempId()].is_mul())
2037 uses_src0 = ctx.uses[instr->operands[0].tempId()];
2038 if (instr->operands[1].isTemp() && ctx.info[instr->operands[1].tempId()].is_mul())
2039 uses_src1 = ctx.uses[instr->operands[1].tempId()];
2040
2041 /* find the 'best' mul instruction to combine with the add */
2042 if (uses_src0 < uses_src1) {
2043 mul_instr = ctx.info[instr->operands[0].tempId()].instr;
2044 add_op_idx = 1;
2045 } else if (uses_src1 < uses_src0) {
2046 mul_instr = ctx.info[instr->operands[1].tempId()].instr;
2047 add_op_idx = 0;
2048 } else if (uses_src0 != UINT32_MAX) {
2049 /* tiebreaker: quite random what to pick */
2050 if (ctx.info[instr->operands[0].tempId()].instr->operands[0].isLiteral()) {
2051 mul_instr = ctx.info[instr->operands[1].tempId()].instr;
2052 add_op_idx = 0;
2053 } else {
2054 mul_instr = ctx.info[instr->operands[0].tempId()].instr;
2055 add_op_idx = 1;
2056 }
2057 }
2058 if (mul_instr) {
2059 Operand op[3] = {Operand(v1), Operand(v1), Operand(v1)};
2060 bool neg[3] = {false, false, false};
2061 bool abs[3] = {false, false, false};
2062 unsigned omod = 0;
2063 bool clamp = false;
2064 bool need_vop3 = false;
2065 int num_sgpr = 0;
2066 op[0] = mul_instr->operands[0];
2067 op[1] = mul_instr->operands[1];
2068 op[2] = instr->operands[add_op_idx];
2069 for (unsigned i = 0; i < 3; i++)
2070 {
2071 if (op[i].isLiteral())
2072 return;
2073 if (op[i].isTemp() && op[i].getTemp().type() == RegType::sgpr)
2074 num_sgpr++;
2075 if (!(i == 0 || (op[i].isTemp() && op[i].getTemp().type() == RegType::vgpr)))
2076 need_vop3 = true;
2077 }
2078 // TODO: would be better to check this before selecting a mul instr?
2079 if (num_sgpr > 1)
2080 return;
2081
2082 if (mul_instr->isVOP3()) {
2083 VOP3A_instruction* vop3 = static_cast<VOP3A_instruction*> (mul_instr);
2084 neg[0] = vop3->neg[0];
2085 neg[1] = vop3->neg[1];
2086 abs[0] = vop3->abs[0];
2087 abs[1] = vop3->abs[1];
2088 need_vop3 = true;
2089 /* we cannot use these modifiers between mul and add */
2090 if (vop3->clamp || vop3->omod)
2091 return;
2092 }
2093
2094 /* convert to mad */
2095 ctx.uses[mul_instr->definitions[0].tempId()]--;
2096 if (ctx.uses[mul_instr->definitions[0].tempId()]) {
2097 if (op[0].isTemp())
2098 ctx.uses[op[0].tempId()]++;
2099 if (op[1].isTemp())
2100 ctx.uses[op[1].tempId()]++;
2101 }
2102
2103 if (instr->isVOP3()) {
2104 VOP3A_instruction* vop3 = static_cast<VOP3A_instruction*> (instr.get());
2105 neg[2] = vop3->neg[add_op_idx];
2106 abs[2] = vop3->abs[add_op_idx];
2107 omod = vop3->omod;
2108 clamp = vop3->clamp;
2109 /* abs of the multiplication result */
2110 if (vop3->abs[1 - add_op_idx]) {
2111 neg[0] = false;
2112 neg[1] = false;
2113 abs[0] = true;
2114 abs[1] = true;
2115 }
2116 /* neg of the multiplication result */
2117 neg[1] = neg[1] ^ vop3->neg[1 - add_op_idx];
2118 need_vop3 = true;
2119 }
2120 if (instr->opcode == aco_opcode::v_sub_f32) {
2121 neg[1 + add_op_idx] = neg[1 + add_op_idx] ^ true;
2122 need_vop3 = true;
2123 } else if (instr->opcode == aco_opcode::v_subrev_f32) {
2124 neg[2 - add_op_idx] = neg[2 - add_op_idx] ^ true;
2125 need_vop3 = true;
2126 }
2127
2128 aco_ptr<VOP3A_instruction> mad{create_instruction<VOP3A_instruction>(aco_opcode::v_mad_f32, Format::VOP3A, 3, 1)};
2129 for (unsigned i = 0; i < 3; i++)
2130 {
2131 mad->operands[i] = op[i];
2132 mad->neg[i] = neg[i];
2133 mad->abs[i] = abs[i];
2134 }
2135 mad->omod = omod;
2136 mad->clamp = clamp;
2137 mad->definitions[0] = instr->definitions[0];
2138
2139 /* mark this ssa_def to be re-checked for profitability and literals */
2140 ctx.mad_infos.emplace_back(std::move(instr), mul_instr->definitions[0].tempId(), need_vop3);
2141 ctx.info[mad->definitions[0].tempId()].set_mad(mad.get(), ctx.mad_infos.size() - 1);
2142 instr.reset(mad.release());
2143 return;
2144 }
2145 }
2146 /* v_mul_f32(v_cndmask_b32(0, 1.0, cond), a) -> v_cndmask_b32(0, a, cond) */
2147 else if (instr->opcode == aco_opcode::v_mul_f32 && !instr->isVOP3()) {
2148 for (unsigned i = 0; i < 2; i++) {
2149 if (instr->operands[i].isTemp() && ctx.info[instr->operands[i].tempId()].is_b2f() &&
2150 ctx.uses[instr->operands[i].tempId()] == 1 &&
2151 instr->operands[!i].isTemp() && instr->operands[!i].getTemp().type() == RegType::vgpr) {
2152 ctx.uses[instr->operands[i].tempId()]--;
2153 ctx.uses[ctx.info[instr->operands[i].tempId()].temp.id()]++;
2154
2155 aco_ptr<VOP2_instruction> new_instr{create_instruction<VOP2_instruction>(aco_opcode::v_cndmask_b32, Format::VOP2, 3, 1)};
2156 new_instr->operands[0] = Operand(0u);
2157 new_instr->operands[1] = instr->operands[!i];
2158 new_instr->operands[2] = Operand(ctx.info[instr->operands[i].tempId()].temp);
2159 new_instr->definitions[0] = instr->definitions[0];
2160 instr.reset(new_instr.release());
2161 ctx.info[instr->definitions[0].tempId()].label = 0;
2162 return;
2163 }
2164 }
2165 } else if (instr->opcode == aco_opcode::v_or_b32 && ctx.program->chip_class >= GFX9) {
2166 if (combine_three_valu_op(ctx, instr, aco_opcode::v_or_b32, aco_opcode::v_or3_b32, "012", 1 | 2)) ;
2167 else if (combine_three_valu_op(ctx, instr, aco_opcode::v_and_b32, aco_opcode::v_and_or_b32, "120", 1 | 2)) ;
2168 else combine_three_valu_op(ctx, instr, aco_opcode::v_lshlrev_b32, aco_opcode::v_lshl_or_b32, "210", 1 | 2);
2169 } else if (instr->opcode == aco_opcode::v_add_u32 && ctx.program->chip_class >= GFX9) {
2170 if (combine_three_valu_op(ctx, instr, aco_opcode::v_xor_b32, aco_opcode::v_xad_u32, "120", 1 | 2)) ;
2171 else if (combine_three_valu_op(ctx, instr, aco_opcode::v_add_u32, aco_opcode::v_add3_u32, "012", 1 | 2)) ;
2172 else combine_three_valu_op(ctx, instr, aco_opcode::v_lshlrev_b32, aco_opcode::v_lshl_add_u32, "210", 1 | 2);
2173 } else if (instr->opcode == aco_opcode::v_lshlrev_b32 && ctx.program->chip_class >= GFX9) {
2174 combine_three_valu_op(ctx, instr, aco_opcode::v_add_u32, aco_opcode::v_add_lshl_u32, "120", 2);
2175 } else if ((instr->opcode == aco_opcode::s_add_u32 || instr->opcode == aco_opcode::s_add_i32) && ctx.program->chip_class >= GFX9) {
2176 combine_salu_lshl_add(ctx, instr);
2177 } else if (instr->opcode == aco_opcode::s_not_b32) {
2178 combine_salu_not_bitwise(ctx, instr);
2179 } else if (instr->opcode == aco_opcode::s_not_b64) {
2180 if (combine_inverse_comparison(ctx, instr)) ;
2181 else combine_salu_not_bitwise(ctx, instr);
2182 } else if (instr->opcode == aco_opcode::s_and_b32 || instr->opcode == aco_opcode::s_or_b32) {
2183 combine_salu_n2(ctx, instr);
2184 } else if (instr->opcode == aco_opcode::s_and_b64 || instr->opcode == aco_opcode::s_or_b64) {
2185 if (combine_ordering_test(ctx, instr)) ;
2186 else if (combine_comparison_ordering(ctx, instr)) ;
2187 else if (combine_constant_comparison_ordering(ctx, instr)) ;
2188 else combine_salu_n2(ctx, instr);
2189 } else {
2190 aco_opcode min, max, min3, max3, med3;
2191 bool some_gfx9_only;
2192 if (get_minmax_info(instr->opcode, &min, &max, &min3, &max3, &med3, &some_gfx9_only) &&
2193 (!some_gfx9_only || ctx.program->chip_class >= GFX9)) {
2194 if (combine_minmax3(ctx, instr, instr->opcode == min ? min3 : max3)) ;
2195 else combine_clamp(ctx, instr, min, max, med3);
2196 }
2197 }
2198 }
2199
2200
2201 void select_instruction(opt_ctx &ctx, aco_ptr<Instruction>& instr)
2202 {
2203 const uint32_t threshold = 4;
2204
2205 /* Dead Code Elimination:
2206 * We remove instructions if they define temporaries which all are unused */
2207 const bool is_used = instr->definitions.empty() ||
2208 std::any_of(instr->definitions.begin(), instr->definitions.end(),
2209 [&ctx](const Definition& def) { return ctx.uses[def.tempId()]; });
2210 if (!is_used) {
2211 instr.reset();
2212 return;
2213 }
2214
2215 /* convert split_vector into extract_vector if only one definition is ever used */
2216 if (instr->opcode == aco_opcode::p_split_vector) {
2217 unsigned num_used = 0;
2218 unsigned idx = 0;
2219 for (unsigned i = 0; i < instr->definitions.size(); i++) {
2220 if (ctx.uses[instr->definitions[i].tempId()]) {
2221 num_used++;
2222 idx = i;
2223 }
2224 }
2225 if (num_used == 1) {
2226 aco_ptr<Pseudo_instruction> extract{create_instruction<Pseudo_instruction>(aco_opcode::p_extract_vector, Format::PSEUDO, 2, 1)};
2227 extract->operands[0] = instr->operands[0];
2228 extract->operands[1] = Operand((uint32_t) idx);
2229 extract->definitions[0] = instr->definitions[idx];
2230 instr.reset(extract.release());
2231 }
2232 }
2233
2234 /* re-check mad instructions */
2235 if (instr->opcode == aco_opcode::v_mad_f32 && ctx.info[instr->definitions[0].tempId()].is_mad()) {
2236 mad_info* info = &ctx.mad_infos[ctx.info[instr->definitions[0].tempId()].val];
2237 /* first, check profitability */
2238 if (ctx.uses[info->mul_temp_id]) {
2239 ctx.uses[info->mul_temp_id]++;
2240 instr.swap(info->add_instr);
2241
2242 /* second, check possible literals */
2243 } else if (!info->needs_vop3) {
2244 uint32_t literal_idx = 0;
2245 uint32_t literal_uses = UINT32_MAX;
2246 for (unsigned i = 0; i < instr->operands.size(); i++)
2247 {
2248 if (!instr->operands[i].isTemp())
2249 continue;
2250 /* if one of the operands is sgpr, we cannot add a literal somewhere else */
2251 if (instr->operands[i].getTemp().type() == RegType::sgpr) {
2252 if (ctx.info[instr->operands[i].tempId()].is_literal()) {
2253 literal_uses = ctx.uses[instr->operands[i].tempId()];
2254 literal_idx = i;
2255 } else {
2256 literal_uses = UINT32_MAX;
2257 }
2258 break;
2259 }
2260 else if (ctx.info[instr->operands[i].tempId()].is_literal() &&
2261 ctx.uses[instr->operands[i].tempId()] < literal_uses) {
2262 literal_uses = ctx.uses[instr->operands[i].tempId()];
2263 literal_idx = i;
2264 }
2265 }
2266 if (literal_uses < threshold) {
2267 ctx.uses[instr->operands[literal_idx].tempId()]--;
2268 info->check_literal = true;
2269 info->literal_idx = literal_idx;
2270 }
2271 }
2272 return;
2273 }
2274
2275 /* check for literals */
2276 /* we do not apply the literals yet as we don't know if it is profitable */
2277 if (instr->isSALU()) {
2278 uint32_t literal_idx = 0;
2279 uint32_t literal_uses = UINT32_MAX;
2280 bool has_literal = false;
2281 for (unsigned i = 0; i < instr->operands.size(); i++)
2282 {
2283 if (instr->operands[i].isLiteral()) {
2284 has_literal = true;
2285 break;
2286 }
2287 if (!instr->operands[i].isTemp())
2288 continue;
2289 if (ctx.info[instr->operands[i].tempId()].is_literal() &&
2290 ctx.uses[instr->operands[i].tempId()] < literal_uses) {
2291 literal_uses = ctx.uses[instr->operands[i].tempId()];
2292 literal_idx = i;
2293 }
2294 }
2295 if (!has_literal && literal_uses < threshold) {
2296 ctx.uses[instr->operands[literal_idx].tempId()]--;
2297 if (ctx.uses[instr->operands[literal_idx].tempId()] == 0)
2298 instr->operands[literal_idx] = Operand(ctx.info[instr->operands[literal_idx].tempId()].val);
2299 }
2300 } else if (instr->isVALU() && valu_can_accept_literal(ctx, instr) &&
2301 instr->operands[0].isTemp() &&
2302 ctx.info[instr->operands[0].tempId()].is_literal() &&
2303 ctx.uses[instr->operands[0].tempId()] < threshold) {
2304 ctx.uses[instr->operands[0].tempId()]--;
2305 if (ctx.uses[instr->operands[0].tempId()] == 0)
2306 instr->operands[0] = Operand(ctx.info[instr->operands[0].tempId()].val);
2307 }
2308
2309 }
2310
2311
2312 void apply_literals(opt_ctx &ctx, aco_ptr<Instruction>& instr)
2313 {
2314 /* Cleanup Dead Instructions */
2315 if (!instr)
2316 return;
2317
2318 /* apply literals on SALU */
2319 if (instr->isSALU()) {
2320 for (Operand& op : instr->operands) {
2321 if (!op.isTemp())
2322 continue;
2323 if (op.isLiteral())
2324 break;
2325 if (ctx.info[op.tempId()].is_literal() &&
2326 ctx.uses[op.tempId()] == 0)
2327 op = Operand(ctx.info[op.tempId()].val);
2328 }
2329 }
2330
2331 /* apply literals on VALU */
2332 else if (instr->isVALU() && !instr->isVOP3() &&
2333 instr->operands[0].isTemp() &&
2334 ctx.info[instr->operands[0].tempId()].is_literal() &&
2335 ctx.uses[instr->operands[0].tempId()] == 0) {
2336 instr->operands[0] = Operand(ctx.info[instr->operands[0].tempId()].val);
2337 }
2338
2339 /* apply literals on MAD */
2340 else if (instr->opcode == aco_opcode::v_mad_f32 && ctx.info[instr->definitions[0].tempId()].is_mad()) {
2341 mad_info* info = &ctx.mad_infos[ctx.info[instr->definitions[0].tempId()].val];
2342 aco_ptr<Instruction> new_mad;
2343 if (info->check_literal && ctx.uses[instr->operands[info->literal_idx].tempId()] == 0) {
2344 if (info->literal_idx == 2) { /* add literal -> madak */
2345 new_mad.reset(create_instruction<VOP2_instruction>(aco_opcode::v_madak_f32, Format::VOP2, 3, 1));
2346 new_mad->operands[0] = instr->operands[0];
2347 new_mad->operands[1] = instr->operands[1];
2348 } else { /* mul literal -> madmk */
2349 new_mad.reset(create_instruction<VOP2_instruction>(aco_opcode::v_madmk_f32, Format::VOP2, 3, 1));
2350 new_mad->operands[0] = instr->operands[1 - info->literal_idx];
2351 new_mad->operands[1] = instr->operands[2];
2352 }
2353 new_mad->operands[2] = Operand(ctx.info[instr->operands[info->literal_idx].tempId()].val);
2354 new_mad->definitions[0] = instr->definitions[0];
2355 instr.swap(new_mad);
2356 }
2357 }
2358
2359 ctx.instructions.emplace_back(std::move(instr));
2360 }
2361
2362
2363 void optimize(Program* program)
2364 {
2365 opt_ctx ctx;
2366 ctx.program = program;
2367 std::vector<ssa_info> info(program->peekAllocationId());
2368 ctx.info = info.data();
2369
2370 /* 1. Bottom-Up DAG pass (forward) to label all ssa-defs */
2371 for (Block& block : program->blocks) {
2372 for (aco_ptr<Instruction>& instr : block.instructions)
2373 label_instruction(ctx, instr);
2374 }
2375
2376 ctx.uses = std::move(dead_code_analysis(program));
2377
2378 /* 2. Combine v_mad, omod, clamp and propagate sgpr on VALU instructions */
2379 for (Block& block : program->blocks) {
2380 for (aco_ptr<Instruction>& instr : block.instructions)
2381 combine_instruction(ctx, instr);
2382 }
2383
2384 /* 3. Top-Down DAG pass (backward) to select instructions (includes DCE) */
2385 for (std::vector<Block>::reverse_iterator it = program->blocks.rbegin(); it != program->blocks.rend(); ++it) {
2386 Block* block = &(*it);
2387 for (std::vector<aco_ptr<Instruction>>::reverse_iterator it = block->instructions.rbegin(); it != block->instructions.rend(); ++it)
2388 select_instruction(ctx, *it);
2389 }
2390
2391 /* 4. Add literals to instructions */
2392 for (Block& block : program->blocks) {
2393 ctx.instructions.clear();
2394 for (aco_ptr<Instruction>& instr : block.instructions)
2395 apply_literals(ctx, instr);
2396 block.instructions.swap(ctx.instructions);
2397 }
2398
2399 }
2400
2401 }