projects / mesa.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
history | raw | HEAD
pan/midgard: De-special-case branching
[mesa.git] / src / panfrost / midgard / midgard_schedule.c
1 /*
2 * Copyright (C) 2018-2019 Alyssa Rosenzweig <alyssa@rosenzweig.io>
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21 * SOFTWARE.
22 */
23
24 #include "compiler.h"
25 #include "midgard_ops.h"
26 #include "util/u_memory.h"
27 #include "util/register_allocate.h"
28
29 /* Create a mask of accessed components from a swizzle to figure out vector
30 * dependencies */
31
32 static unsigned
33 swizzle_to_access_mask(unsigned swizzle)
34 {
35 unsigned component_mask = 0;
36
37 for (int i = 0; i < 4; ++i) {
38 unsigned c = (swizzle >> (2 * i)) & 3;
39 component_mask |= (1 << c);
40 }
41
42 return component_mask;
43 }
44
45 /* Does the mask cover more than a scalar? */
46
47 static bool
48 is_single_component_mask(unsigned mask)
49 {
50 int components = 0;
51
52 for (int c = 0; c < 8; ++c) {
53 if (mask & (1 << c))
54 components++;
55 }
56
57 return components == 1;
58 }
59
60 /* Checks for an SSA data hazard between two adjacent instructions, keeping in
61 * mind that we are a vector architecture and we can write to different
62 * components simultaneously */
63
64 static bool
65 can_run_concurrent_ssa(midgard_instruction *first, midgard_instruction *second)
66 {
67 /* Each instruction reads some registers and writes to a register. See
68 * where the first writes */
69
70 /* Figure out where exactly we wrote to */
71 int source = first->ssa_args.dest;
72 int source_mask = first->mask;
73
74 /* As long as the second doesn't read from the first, we're okay */
75 if (second->ssa_args.src0 == source) {
76 if (first->type == TAG_ALU_4) {
77 /* Figure out which components we just read from */
78
79 int q = second->alu.src1;
80 midgard_vector_alu_src *m = (midgard_vector_alu_src *) &q;
81
82 /* Check if there are components in common, and fail if so */
83 if (swizzle_to_access_mask(m->swizzle) & source_mask)
84 return false;
85 } else
86 return false;
87
88 }
89
90 if (second->ssa_args.src1 == source)
91 return false;
92
93 /* Otherwise, it's safe in that regard. Another data hazard is both
94 * writing to the same place, of course */
95
96 if (second->ssa_args.dest == source) {
97 /* ...but only if the components overlap */
98
99 if (second->mask & source_mask)
100 return false;
101 }
102
103 /* ...That's it */
104 return true;
105 }
106
107 static bool
108 midgard_has_hazard(
109 midgard_instruction **segment, unsigned segment_size,
110 midgard_instruction *ains)
111 {
112 for (int s = 0; s < segment_size; ++s)
113 if (!can_run_concurrent_ssa(segment[s], ains))
114 return true;
115
116 return false;
117
118
119 }
120
121 /* Fragment writeout (of r0) is allowed when:
122 *
123 * - All components of r0 are written in the bundle
124 * - No components of r0 are written in VLUT
125 * - Non-pipelined dependencies of r0 are not written in the bundle
126 *
127 * This function checks if these requirements are satisfied given the content
128 * of a scheduled bundle.
129 */
130
131 static bool
132 can_writeout_fragment(compiler_context *ctx, midgard_instruction **bundle, unsigned count, unsigned node_count)
133 {
134 /* First scan for which components of r0 are written out. Initially
135 * none are written */
136
137 uint8_t r0_written_mask = 0x0;
138
139 /* Simultaneously we scan for the set of dependencies */
140 BITSET_WORD *dependencies = calloc(sizeof(BITSET_WORD), BITSET_WORDS(node_count));
141
142 for (unsigned i = 0; i < count; ++i) {
143 midgard_instruction *ins = bundle[i];
144
145 if (ins->ssa_args.dest != SSA_FIXED_REGISTER(0))
146 continue;
147
148 /* Record written out mask */
149 r0_written_mask |= ins->mask;
150
151 /* Record dependencies, but only if they won't become pipeline
152 * registers. We know we can't be live after this, because
153 * we're writeout at the very end of the shader. So check if
154 * they were written before us. */
155
156 unsigned src0 = ins->ssa_args.src0;
157 unsigned src1 = ins->ssa_args.src1;
158
159 if (!mir_is_written_before(ctx, bundle[0], src0))
160 src0 = -1;
161
162 if (!mir_is_written_before(ctx, bundle[0], src1))
163 src1 = -1;
164
165 if ((src0 > 0) && (src0 < node_count))
166 BITSET_SET(dependencies, src0);
167
168 if ((src1 > 0) && (src1 < node_count))
169 BITSET_SET(dependencies, src1);
170
171 /* Requirement 2 */
172 if (ins->unit == UNIT_VLUT)
173 return false;
174 }
175
176 /* Requirement 1 */
177 if ((r0_written_mask & 0xF) != 0xF)
178 return false;
179
180 /* Requirement 3 */
181
182 for (unsigned i = 0; i < count; ++i) {
183 unsigned dest = bundle[i]->ssa_args.dest;
184
185 if (dest < node_count && BITSET_TEST(dependencies, dest))
186 return false;
187 }
188
189 /* Otherwise, we're good to go */
190 return true;
191 }
192
193 /* Schedules, but does not emit, a single basic block. After scheduling, the
194 * final tag and size of the block are known, which are necessary for branching
195 * */
196
197 static midgard_bundle
198 schedule_bundle(compiler_context *ctx, midgard_block *block, midgard_instruction *ins, int *skip)
199 {
200 int instructions_emitted = 0, packed_idx = 0;
201 midgard_bundle bundle = { 0 };
202
203 midgard_instruction *scheduled[5] = { NULL };
204
205 uint8_t tag = ins->type;
206
207 /* Default to the instruction's tag */
208 bundle.tag = tag;
209
210 switch (ins->type) {
211 case TAG_ALU_4: {
212 uint32_t control = 0;
213 size_t bytes_emitted = sizeof(control);
214
215 /* TODO: Constant combining */
216 int index = 0, last_unit = 0;
217
218 /* Previous instructions, for the purpose of parallelism */
219 midgard_instruction *segment[4] = {0};
220 int segment_size = 0;
221
222 instructions_emitted = -1;
223 midgard_instruction *pins = ins;
224
225 unsigned constant_count = 0;
226
227 for (;;) {
228 midgard_instruction *ains = pins;
229
230 /* Advance instruction pointer */
231 if (index) {
232 ains = mir_next_op(pins);
233 pins = ains;
234 }
235
236 /* Out-of-work condition */
237 if ((struct list_head *) ains == &block->instructions)
238 break;
239
240 /* Ensure that the chain can continue */
241 if (ains->type != TAG_ALU_4) break;
242
243 /* If there's already something in the bundle and we
244 * have weird scheduler constraints, break now */
245 if (ains->precede_break && index) break;
246
247 /* According to the presentation "The ARM
248 * Mali-T880 Mobile GPU" from HotChips 27,
249 * there are two pipeline stages. Branching
250 * position determined experimentally. Lines
251 * are executed in parallel:
252 *
253 * [ VMUL ] [ SADD ]
254 * [ VADD ] [ SMUL ] [ LUT ] [ BRANCH ]
255 *
256 * Verify that there are no ordering dependencies here.
257 *
258 * TODO: Allow for parallelism!!!
259 */
260
261 /* Pick a unit for it if it doesn't force a particular unit */
262
263 int unit = ains->unit;
264
265 if (!unit) {
266 int op = ains->alu.op;
267 int units = alu_opcode_props[op].props;
268
269 bool scalarable = units & UNITS_SCALAR;
270 bool could_scalar = is_single_component_mask(ains->mask);
271
272 /* Only 16/32-bit can run on a scalar unit */
273 could_scalar &= ains->alu.reg_mode != midgard_reg_mode_8;
274 could_scalar &= ains->alu.reg_mode != midgard_reg_mode_64;
275 could_scalar &= ains->alu.dest_override == midgard_dest_override_none;
276
277 if (ains->alu.reg_mode == midgard_reg_mode_16) {
278 /* If we're running in 16-bit mode, we
279 * can't have any 8-bit sources on the
280 * scalar unit (since the scalar unit
281 * doesn't understand 8-bit) */
282
283 midgard_vector_alu_src s1 =
284 vector_alu_from_unsigned(ains->alu.src1);
285
286 could_scalar &= !s1.half;
287
288 if (!ains->ssa_args.inline_constant) {
289 midgard_vector_alu_src s2 =
290 vector_alu_from_unsigned(ains->alu.src2);
291
292 could_scalar &= !s2.half;
293 }
294
295 }
296
297 bool scalar = could_scalar && scalarable;
298
299 /* TODO: Check ahead-of-time for other scalar
300 * hazards that otherwise get aborted out */
301
302 if (scalar)
303 assert(units & UNITS_SCALAR);
304
305 if (!scalar) {
306 if (last_unit >= UNIT_VADD) {
307 if (units & UNIT_VLUT)
308 unit = UNIT_VLUT;
309 else
310 break;
311 } else {
312 if ((units & UNIT_VMUL) && last_unit < UNIT_VMUL)
313 unit = UNIT_VMUL;
314 else if ((units & UNIT_VADD) && !(control & UNIT_VADD))
315 unit = UNIT_VADD;
316 else if (units & UNIT_VLUT)
317 unit = UNIT_VLUT;
318 else
319 break;
320 }
321 } else {
322 if (last_unit >= UNIT_VADD) {
323 if ((units & UNIT_SMUL) && !(control & UNIT_SMUL))
324 unit = UNIT_SMUL;
325 else if (units & UNIT_VLUT)
326 unit = UNIT_VLUT;
327 else
328 break;
329 } else {
330 if ((units & UNIT_VMUL) && (last_unit < UNIT_VMUL))
331 unit = UNIT_VMUL;
332 else if ((units & UNIT_SADD) && !(control & UNIT_SADD) && !midgard_has_hazard(segment, segment_size, ains))
333 unit = UNIT_SADD;
334 else if (units & UNIT_VADD)
335 unit = UNIT_VADD;
336 else if (units & UNIT_SMUL)
337 unit = UNIT_SMUL;
338 else if (units & UNIT_VLUT)
339 unit = UNIT_VLUT;
340 else
341 break;
342 }
343 }
344
345 assert(unit & units);
346 }
347
348 /* Late unit check, this time for encoding (not parallelism) */
349 if (unit <= last_unit) break;
350
351 /* Clear the segment */
352 if (last_unit < UNIT_VADD && unit >= UNIT_VADD)
353 segment_size = 0;
354
355 if (midgard_has_hazard(segment, segment_size, ains))
356 break;
357
358 /* We're good to go -- emit the instruction */
359 ains->unit = unit;
360
361 segment[segment_size++] = ains;
362
363 /* We try to reuse constants if possible, by adjusting
364 * the swizzle */
365
366 if (ains->has_blend_constant) {
367 /* Everything conflicts with the blend constant */
368 if (bundle.has_embedded_constants)
369 break;
370
371 bundle.has_blend_constant = 1;
372 bundle.has_embedded_constants = 1;
373 } else if (ains->has_constants && ains->alu.reg_mode == midgard_reg_mode_16) {
374 /* TODO: DRY with the analysis pass */
375
376 if (bundle.has_blend_constant)
377 break;
378
379 if (constant_count)
380 break;
381
382 /* TODO: Fix packing XXX */
383 uint16_t *bundles = (uint16_t *) bundle.constants;
384 uint32_t *constants = (uint32_t *) ains->constants;
385
386 /* Copy them wholesale */
387 for (unsigned i = 0; i < 4; ++i)
388 bundles[i] = constants[i];
389
390 bundle.has_embedded_constants = true;
391 constant_count = 4;
392 } else if (ains->has_constants) {
393 /* By definition, blend constants conflict with
394 * everything, so if there are already
395 * constants we break the bundle *now* */
396
397 if (bundle.has_blend_constant)
398 break;
399
400 /* For anything but blend constants, we can do
401 * proper analysis, however */
402
403 /* TODO: Mask by which are used */
404 uint32_t *constants = (uint32_t *) ains->constants;
405 uint32_t *bundles = (uint32_t *) bundle.constants;
406
407 uint32_t indices[4] = { 0 };
408 bool break_bundle = false;
409
410 for (unsigned i = 0; i < 4; ++i) {
411 uint32_t cons = constants[i];
412 bool constant_found = false;
413
414 /* Search for the constant */
415 for (unsigned j = 0; j < constant_count; ++j) {
416 if (bundles[j] != cons)
417 continue;
418
419 /* We found it, reuse */
420 indices[i] = j;
421 constant_found = true;
422 break;
423 }
424
425 if (constant_found)
426 continue;
427
428 /* We didn't find it, so allocate it */
429 unsigned idx = constant_count++;
430
431 if (idx >= 4) {
432 /* Uh-oh, out of space */
433 break_bundle = true;
434 break;
435 }
436
437 /* We have space, copy it in! */
438 bundles[idx] = cons;
439 indices[i] = idx;
440 }
441
442 if (break_bundle)
443 break;
444
445 /* Cool, we have it in. So use indices as a
446 * swizzle */
447
448 unsigned swizzle = SWIZZLE_FROM_ARRAY(indices);
449 unsigned r_constant = SSA_FIXED_REGISTER(REGISTER_CONSTANT);
450
451 if (ains->ssa_args.src0 == r_constant)
452 ains->alu.src1 = vector_alu_apply_swizzle(ains->alu.src1, swizzle);
453
454 if (ains->ssa_args.src1 == r_constant)
455 ains->alu.src2 = vector_alu_apply_swizzle(ains->alu.src2, swizzle);
456
457 bundle.has_embedded_constants = true;
458 }
459
460 if (ains->unit & UNITS_ANY_VECTOR) {
461 bytes_emitted += sizeof(midgard_reg_info);
462 bytes_emitted += sizeof(midgard_vector_alu);
463 } else if (ains->compact_branch) {
464 /* All of r0 has to be written out along with
465 * the branch writeout */
466
467 if (ains->writeout && !can_writeout_fragment(ctx, scheduled, index, ctx->temp_count)) {
468 /* We only work on full moves
469 * at the beginning. We could
470 * probably do better */
471 if (index != 0)
472 break;
473
474 /* Inject a move */
475 midgard_instruction ins = v_mov(0, blank_alu_src, SSA_FIXED_REGISTER(0));
476 ins.unit = UNIT_VMUL;
477 control |= ins.unit;
478
479 /* TODO don't leak */
480 midgard_instruction *move =
481 mem_dup(&ins, sizeof(midgard_instruction));
482 bytes_emitted += sizeof(midgard_reg_info);
483 bytes_emitted += sizeof(midgard_vector_alu);
484 bundle.instructions[packed_idx++] = move;
485 }
486
487 if (ains->unit == ALU_ENAB_BRANCH) {
488 bytes_emitted += sizeof(midgard_branch_extended);
489 } else {
490 bytes_emitted += sizeof(ains->br_compact);
491 }
492 } else {
493 bytes_emitted += sizeof(midgard_reg_info);
494 bytes_emitted += sizeof(midgard_scalar_alu);
495 }
496
497 /* Defer marking until after writing to allow for break */
498 scheduled[index] = ains;
499 control |= ains->unit;
500 last_unit = ains->unit;
501 ++instructions_emitted;
502 ++index;
503 }
504
505 int padding = 0;
506
507 /* Pad ALU op to nearest word */
508
509 if (bytes_emitted & 15) {
510 padding = 16 - (bytes_emitted & 15);
511 bytes_emitted += padding;
512 }
513
514 /* Constants must always be quadwords */
515 if (bundle.has_embedded_constants)
516 bytes_emitted += 16;
517
518 /* Size ALU instruction for tag */
519 bundle.tag = (TAG_ALU_4) + (bytes_emitted / 16) - 1;
520 bundle.padding = padding;
521 bundle.control = bundle.tag | control;
522
523 break;
524 }
525
526 case TAG_LOAD_STORE_4: {
527 /* Load store instructions have two words at once. If
528 * we only have one queued up, we need to NOP pad.
529 * Otherwise, we store both in succession to save space
530 * and cycles -- letting them go in parallel -- skip
531 * the next. The usefulness of this optimisation is
532 * greatly dependent on the quality of the instruction
533 * scheduler.
534 */
535
536 midgard_instruction *next_op = mir_next_op(ins);
537
538 if ((struct list_head *) next_op != &block->instructions && next_op->type == TAG_LOAD_STORE_4) {
539 /* TODO: Concurrency check */
540 instructions_emitted++;
541 }
542
543 break;
544 }
545
546 case TAG_TEXTURE_4: {
547 /* Which tag we use depends on the shader stage */
548 bool in_frag = ctx->stage == MESA_SHADER_FRAGMENT;
549 bundle.tag = in_frag ? TAG_TEXTURE_4 : TAG_TEXTURE_4_VTX;
550 break;
551 }
552
553 default:
554 unreachable("Unknown tag");
555 break;
556 }
557
558 /* Copy the instructions into the bundle */
559 bundle.instruction_count = instructions_emitted + 1 + packed_idx;
560
561 midgard_instruction *uins = ins;
562 for (; packed_idx < bundle.instruction_count; ++packed_idx) {
563 bundle.instructions[packed_idx] = uins;
564 uins = mir_next_op(uins);
565 }
566
567 *skip = instructions_emitted;
568
569 return bundle;
570 }
571
572 /* Schedule a single block by iterating its instruction to create bundles.
573 * While we go, tally about the bundle sizes to compute the block size. */
574
575 static void
576 schedule_block(compiler_context *ctx, midgard_block *block)
577 {
578 util_dynarray_init(&block->bundles, NULL);
579
580 block->quadword_count = 0;
581
582 mir_foreach_instr_in_block(block, ins) {
583 int skip;
584 midgard_bundle bundle = schedule_bundle(ctx, block, ins, &skip);
585 util_dynarray_append(&block->bundles, midgard_bundle, bundle);
586
587 if (bundle.has_blend_constant) {
588 /* TODO: Multiblock? */
589 int quadwords_within_block = block->quadword_count + quadword_size(bundle.tag) - 1;
590 ctx->blend_constant_offset = quadwords_within_block * 0x10;
591 }
592
593 while(skip--)
594 ins = mir_next_op(ins);
595
596 block->quadword_count += quadword_size(bundle.tag);
597 }
598
599 block->is_scheduled = true;
600 }
601
602 /* The following passes reorder MIR instructions to enable better scheduling */
603
604 static void
605 midgard_pair_load_store(compiler_context *ctx, midgard_block *block)
606 {
607 mir_foreach_instr_in_block_safe(block, ins) {
608 if (ins->type != TAG_LOAD_STORE_4) continue;
609
610 /* We've found a load/store op. Check if next is also load/store. */
611 midgard_instruction *next_op = mir_next_op(ins);
612 if (&next_op->link != &block->instructions) {
613 if (next_op->type == TAG_LOAD_STORE_4) {
614 /* If so, we're done since we're a pair */
615 ins = mir_next_op(ins);
616 continue;
617 }
618
619 /* Maximum search distance to pair, to avoid register pressure disasters */
620 int search_distance = 8;
621
622 /* Otherwise, we have an orphaned load/store -- search for another load */
623 mir_foreach_instr_in_block_from(block, c, mir_next_op(ins)) {
624 /* Terminate search if necessary */
625 if (!(search_distance--)) break;
626
627 if (c->type != TAG_LOAD_STORE_4) continue;
628
629 /* Stores cannot be reordered, since they have
630 * dependencies. For the same reason, indirect
631 * loads cannot be reordered as their index is
632 * loaded in r27.w */
633
634 if (OP_IS_STORE(c->load_store.op)) continue;
635
636 /* It appears the 0x800 bit is set whenever a
637 * load is direct, unset when it is indirect.
638 * Skip indirect loads. */
639
640 if (!(c->load_store.unknown & 0x800)) continue;
641
642 /* We found one! Move it up to pair and remove it from the old location */
643
644 mir_insert_instruction_before(ins, *c);
645 mir_remove_instruction(c);
646
647 break;
648 }
649 }
650 }
651 }
652
653 /* When we're 'squeezing down' the values in the IR, we maintain a hash
654 * as such */
655
656 static unsigned
657 find_or_allocate_temp(compiler_context *ctx, unsigned hash)
658 {
659 if ((hash < 0) || (hash >= SSA_FIXED_MINIMUM))
660 return hash;
661
662 unsigned temp = (uintptr_t) _mesa_hash_table_u64_search(
663 ctx->hash_to_temp, hash + 1);
664
665 if (temp)
666 return temp - 1;
667
668 /* If no temp is find, allocate one */
669 temp = ctx->temp_count++;
670 ctx->max_hash = MAX2(ctx->max_hash, hash);
671
672 _mesa_hash_table_u64_insert(ctx->hash_to_temp,
673 hash + 1, (void *) ((uintptr_t) temp + 1));
674
675 return temp;
676 }
677
678 /* Reassigns numbering to get rid of gaps in the indices */
679
680 static void
681 mir_squeeze_index(compiler_context *ctx)
682 {
683 /* Reset */
684 ctx->temp_count = 0;
685 /* TODO don't leak old hash_to_temp */
686 ctx->hash_to_temp = _mesa_hash_table_u64_create(NULL);
687
688 mir_foreach_instr_global(ctx, ins) {
689 ins->ssa_args.dest = find_or_allocate_temp(ctx, ins->ssa_args.dest);
690 ins->ssa_args.src0 = find_or_allocate_temp(ctx, ins->ssa_args.src0);
691
692 if (!ins->ssa_args.inline_constant)
693 ins->ssa_args.src1 = find_or_allocate_temp(ctx, ins->ssa_args.src1);
694
695 }
696 }
697
698 static midgard_instruction
699 v_load_store_scratch(
700 unsigned srcdest,
701 unsigned index,
702 bool is_store,
703 unsigned mask)
704 {
705 /* We index by 32-bit vec4s */
706 unsigned byte = (index * 4 * 4);
707
708 midgard_instruction ins = {
709 .type = TAG_LOAD_STORE_4,
710 .mask = mask,
711 .ssa_args = {
712 .dest = -1,
713 .src0 = -1,
714 .src1 = -1
715 },
716 .load_store = {
717 .op = is_store ? midgard_op_st_int4 : midgard_op_ld_int4,
718 .swizzle = SWIZZLE_XYZW,
719
720 /* For register spilling - to thread local storage */
721 .unknown = 0x1EEA,
722
723 /* Splattered across, TODO combine logically */
724 .varying_parameters = (byte & 0x1FF) << 1,
725 .address = (byte >> 9)
726 }
727 };
728
729 if (is_store) {
730 /* r0 = r26, r1 = r27 */
731 assert(srcdest == SSA_FIXED_REGISTER(26) || srcdest == SSA_FIXED_REGISTER(27));
732 ins.ssa_args.src0 = (srcdest == SSA_FIXED_REGISTER(27)) ? SSA_FIXED_REGISTER(1) : SSA_FIXED_REGISTER(0);
733 } else {
734 ins.ssa_args.dest = srcdest;
735 }
736
737 return ins;
738 }
739
740 void
741 schedule_program(compiler_context *ctx)
742 {
743 struct ra_graph *g = NULL;
744 bool spilled = false;
745 int iter_count = 1000; /* max iterations */
746
747 /* Number of 128-bit slots in memory we've spilled into */
748 unsigned spill_count = 0;
749
750 midgard_promote_uniforms(ctx, 8);
751
752 mir_foreach_block(ctx, block) {
753 midgard_pair_load_store(ctx, block);
754 }
755
756 /* Must be lowered right before RA */
757 mir_squeeze_index(ctx);
758 mir_lower_special_reads(ctx);
759
760 /* Lowering can introduce some dead moves */
761
762 mir_foreach_block(ctx, block) {
763 midgard_opt_dead_move_eliminate(ctx, block);
764 }
765
766 do {
767 /* If we spill, find the best spill node and spill it */
768
769 unsigned spill_index = ctx->temp_count;
770 if (g && spilled) {
771 /* All nodes are equal in spill cost, but we can't
772 * spill nodes written to from an unspill */
773
774 for (unsigned i = 0; i < ctx->temp_count; ++i) {
775 ra_set_node_spill_cost(g, i, 1.0);
776 }
777
778 mir_foreach_instr_global(ctx, ins) {
779 if (ins->type != TAG_LOAD_STORE_4) continue;
780 if (ins->load_store.op != midgard_op_ld_int4) continue;
781 if (ins->load_store.unknown != 0x1EEA) continue;
782 ra_set_node_spill_cost(g, ins->ssa_args.dest, -1.0);
783 }
784
785 int spill_node = ra_get_best_spill_node(g);
786
787 if (spill_node < 0) {
788 mir_print_shader(ctx);
789 assert(0);
790 }
791
792 /* Check the class. Work registers legitimately spill
793 * to TLS, but special registers just spill to work
794 * registers */
795 unsigned class = ra_get_node_class(g, spill_node);
796 bool is_special = (class >> 2) != REG_CLASS_WORK;
797 bool is_special_w = (class >> 2) == REG_CLASS_TEXW;
798
799 /* Allocate TLS slot (maybe) */
800 unsigned spill_slot = !is_special ? spill_count++ : 0;
801 midgard_instruction *spill_move = NULL;
802
803 /* For TLS, replace all stores to the spilled node. For
804 * special reads, just keep as-is; the class will be demoted
805 * implicitly. For special writes, spill to a work register */
806
807 if (!is_special || is_special_w) {
808 mir_foreach_instr_global_safe(ctx, ins) {
809 if (ins->ssa_args.dest != spill_node) continue;
810
811 midgard_instruction st;
812
813 if (is_special_w) {
814 spill_slot = spill_index++;
815 st = v_mov(spill_node, blank_alu_src, spill_slot);
816 } else {
817 ins->ssa_args.dest = SSA_FIXED_REGISTER(26);
818 st = v_load_store_scratch(ins->ssa_args.dest, spill_slot, true, ins->mask);
819 }
820
821 spill_move = mir_insert_instruction_before(mir_next_op(ins), st);
822
823 if (!is_special)
824 ctx->spills++;
825 }
826 }
827
828 /* Insert a load from TLS before the first consecutive
829 * use of the node, rewriting to use spilled indices to
830 * break up the live range. Or, for special, insert a
831 * move. Ironically the latter *increases* register
832 * pressure, but the two uses of the spilling mechanism
833 * are somewhat orthogonal. (special spilling is to use
834 * work registers to back special registers; TLS
835 * spilling is to use memory to back work registers) */
836
837 mir_foreach_block(ctx, block) {
838
839 bool consecutive_skip = false;
840 unsigned consecutive_index = 0;
841
842 mir_foreach_instr_in_block(block, ins) {
843 /* We can't rewrite the move used to spill in the first place */
844 if (ins == spill_move) continue;
845
846 if (!mir_has_arg(ins, spill_node)) {
847 consecutive_skip = false;
848 continue;
849 }
850
851 if (consecutive_skip) {
852 /* Rewrite */
853 mir_rewrite_index_src_single(ins, spill_node, consecutive_index);
854 continue;
855 }
856
857 if (!is_special_w) {
858 consecutive_index = ++spill_index;
859
860 midgard_instruction *before = ins;
861
862 /* For a csel, go back one more not to break up the bundle */
863 if (ins->type == TAG_ALU_4 && OP_IS_CSEL(ins->alu.op))
864 before = mir_prev_op(before);
865
866 midgard_instruction st;
867
868 if (is_special) {
869 /* Move */
870 st = v_mov(spill_node, blank_alu_src, consecutive_index);
871 } else {
872 /* TLS load */
873 st = v_load_store_scratch(consecutive_index, spill_slot, false, 0xF);
874 }
875
876 mir_insert_instruction_before(before, st);
877 // consecutive_skip = true;
878 } else {
879 /* Special writes already have their move spilled in */
880 consecutive_index = spill_slot;
881 }
882
883
884 /* Rewrite to use */
885 mir_rewrite_index_src_single(ins, spill_node, consecutive_index);
886
887 if (!is_special)
888 ctx->fills++;
889 }
890 }
891 }
892
893 mir_squeeze_index(ctx);
894
895 g = NULL;
896 g = allocate_registers(ctx, &spilled);
897 } while(spilled && ((iter_count--) > 0));
898
899 /* We can simplify a bit after RA */
900
901 mir_foreach_block(ctx, block) {
902 midgard_opt_post_move_eliminate(ctx, block, g);
903 }
904
905 /* After RA finishes, we schedule all at once */
906
907 mir_foreach_block(ctx, block) {
908 schedule_block(ctx, block);
909 }
910
911 /* Finally, we create pipeline registers as a peephole pass after
912 * scheduling. This isn't totally optimal, since there are cases where
913 * the usage of pipeline registers can eliminate spills, but it does
914 * save some power */
915
916 mir_create_pipeline_registers(ctx);
917
918 if (iter_count <= 0) {
919 fprintf(stderr, "panfrost: Gave up allocating registers, rendering will be incomplete\n");
920 assert(0);
921 }
922
923 /* Report spilling information. spill_count is in 128-bit slots (vec4 x
924 * fp32), but tls_size is in bytes, so multiply by 16 */
925
926 ctx->tls_size = spill_count * 16;
927
928 install_registers(ctx, g);
929 }