2 * Copyright (C) 2018-2019 Alyssa Rosenzweig <alyssa@rosenzweig.io>
3 * Copyright (C) 2019 Collabora, Ltd.
5 * Permission is hereby granted, free of charge, to any person obtaining a
6 * copy of this software and associated documentation files (the "Software"),
7 * to deal in the Software without restriction, including without limitation
8 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
9 * and/or sell copies of the Software, and to permit persons to whom the
10 * Software is furnished to do so, subject to the following conditions:
12 * The above copyright notice and this permission notice (including the next
13 * paragraph) shall be included in all copies or substantial portions of the
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
26 #include "midgard_ops.h"
27 #include "util/u_math.h"
28 #include "util/u_memory.h"
29 #include "midgard_quirks.h"
32 /* Physical register: 0-31 */
35 /* Byte offset into the physical register: 0-15 */
38 /* log2(bytes per component) for fast mul/div */
42 /* Shift up by reg_offset and horizontally by dst_offset. */
45 offset_swizzle(unsigned *swizzle
, unsigned reg_offset
, unsigned srcshift
, unsigned dstshift
, unsigned dst_offset
)
47 unsigned out
[MIR_VEC_COMPONENTS
];
49 signed reg_comp
= reg_offset
>> srcshift
;
50 signed dst_comp
= dst_offset
>> dstshift
;
52 unsigned max_component
= (16 >> srcshift
) - 1;
54 assert(reg_comp
<< srcshift
== reg_offset
);
55 assert(dst_comp
<< dstshift
== dst_offset
);
57 for (signed c
= 0; c
< MIR_VEC_COMPONENTS
; ++c
) {
58 signed comp
= MAX2(c
- dst_comp
, 0);
59 out
[c
] = MIN2(swizzle
[comp
] + reg_comp
, max_component
);
62 memcpy(swizzle
, out
, sizeof(out
));
65 /* Helper to return the default phys_reg for a given register */
67 static struct phys_reg
68 default_phys_reg(int reg
, unsigned shift
)
79 /* Determine which physical register, swizzle, and mask a virtual
80 * register corresponds to */
82 static struct phys_reg
83 index_to_reg(compiler_context
*ctx
, struct lcra_state
*l
, unsigned reg
, unsigned shift
)
85 /* Check for special cases */
87 return default_phys_reg(REGISTER_UNUSED
, shift
);
88 else if (reg
>= SSA_FIXED_MINIMUM
)
89 return default_phys_reg(SSA_REG_FROM_FIXED(reg
), shift
);
91 return default_phys_reg(REGISTER_UNUSED
, shift
);
94 .reg
= l
->solutions
[reg
] / 16,
95 .offset
= l
->solutions
[reg
] & 0xF,
99 /* Report that we actually use this register, and return it */
102 ctx
->work_registers
= MAX2(ctx
->work_registers
, r
.reg
);
108 set_class(unsigned *classes
, unsigned node
, unsigned class)
110 if (node
< SSA_FIXED_MINIMUM
&& class != classes
[node
]) {
111 assert(classes
[node
] == REG_CLASS_WORK
);
112 classes
[node
] = class;
116 /* Special register classes impose special constraints on who can read their
117 * values, so check that */
120 check_read_class(unsigned *classes
, unsigned tag
, unsigned node
)
122 /* Non-nodes are implicitly ok */
123 if (node
>= SSA_FIXED_MINIMUM
)
126 switch (classes
[node
]) {
128 return (tag
== TAG_LOAD_STORE_4
);
130 return (tag
== TAG_TEXTURE_4
);
132 return (tag
!= TAG_LOAD_STORE_4
);
136 unreachable("Invalid class");
141 check_write_class(unsigned *classes
, unsigned tag
, unsigned node
)
143 /* Non-nodes are implicitly ok */
144 if (node
>= SSA_FIXED_MINIMUM
)
147 switch (classes
[node
]) {
151 return (tag
== TAG_TEXTURE_4
);
154 return IS_ALU(tag
) || (tag
== TAG_LOAD_STORE_4
);
156 unreachable("Invalid class");
160 /* Prepass before RA to ensure special class restrictions are met. The idea is
161 * to create a bit field of types of instructions that read a particular index.
162 * Later, we'll add moves as appropriate and rewrite to specialize by type. */
165 mark_node_class (unsigned *bitfield
, unsigned node
)
167 if (node
< SSA_FIXED_MINIMUM
)
168 BITSET_SET(bitfield
, node
);
172 mir_lower_special_reads(compiler_context
*ctx
)
174 size_t sz
= BITSET_WORDS(ctx
->temp_count
) * sizeof(BITSET_WORD
);
176 /* Bitfields for the various types of registers we could have. aluw can
177 * be written by either ALU or load/store */
179 unsigned *alur
= calloc(sz
, 1);
180 unsigned *aluw
= calloc(sz
, 1);
181 unsigned *brar
= calloc(sz
, 1);
182 unsigned *ldst
= calloc(sz
, 1);
183 unsigned *texr
= calloc(sz
, 1);
184 unsigned *texw
= calloc(sz
, 1);
186 /* Pass #1 is analysis, a linear scan to fill out the bitfields */
188 mir_foreach_instr_global(ctx
, ins
) {
191 mark_node_class(aluw
, ins
->dest
);
192 mark_node_class(alur
, ins
->src
[0]);
193 mark_node_class(alur
, ins
->src
[1]);
194 mark_node_class(alur
, ins
->src
[2]);
196 if (ins
->compact_branch
&& ins
->writeout
)
197 mark_node_class(brar
, ins
->src
[0]);
201 case TAG_LOAD_STORE_4
:
202 mark_node_class(aluw
, ins
->dest
);
203 mark_node_class(ldst
, ins
->src
[0]);
204 mark_node_class(ldst
, ins
->src
[1]);
205 mark_node_class(ldst
, ins
->src
[2]);
209 mark_node_class(texr
, ins
->src
[0]);
210 mark_node_class(texr
, ins
->src
[1]);
211 mark_node_class(texr
, ins
->src
[2]);
212 mark_node_class(texw
, ins
->dest
);
217 /* Pass #2 is lowering now that we've analyzed all the classes.
218 * Conceptually, if an index is only marked for a single type of use,
219 * there is nothing to lower. If it is marked for different uses, we
220 * split up based on the number of types of uses. To do so, we divide
221 * into N distinct classes of use (where N>1 by definition), emit N-1
222 * moves from the index to copies of the index, and finally rewrite N-1
223 * of the types of uses to use the corresponding move */
225 unsigned spill_idx
= ctx
->temp_count
;
227 for (unsigned i
= 0; i
< ctx
->temp_count
; ++i
) {
228 bool is_alur
= BITSET_TEST(alur
, i
);
229 bool is_aluw
= BITSET_TEST(aluw
, i
);
230 bool is_brar
= BITSET_TEST(brar
, i
);
231 bool is_ldst
= BITSET_TEST(ldst
, i
);
232 bool is_texr
= BITSET_TEST(texr
, i
);
233 bool is_texw
= BITSET_TEST(texw
, i
);
235 /* Analyse to check how many distinct uses there are. ALU ops
236 * (alur) can read the results of the texture pipeline (texw)
237 * but not ldst or texr. Load/store ops (ldst) cannot read
238 * anything but load/store inputs. Texture pipeline cannot read
239 * anything but texture inputs. TODO: Simplify. */
242 (is_alur
&& (is_ldst
|| is_texr
)) ||
243 (is_ldst
&& (is_alur
|| is_texr
|| is_texw
)) ||
244 (is_texr
&& (is_alur
|| is_ldst
|| is_texw
)) ||
245 (is_texw
&& (is_aluw
|| is_ldst
|| is_texr
)) ||
246 (is_brar
&& is_texw
);
251 /* Use the index as-is as the work copy. Emit copies for
254 unsigned classes
[] = { TAG_LOAD_STORE_4
, TAG_TEXTURE_4
, TAG_TEXTURE_4
, TAG_ALU_4
};
255 bool collisions
[] = { is_ldst
, is_texr
, is_texw
&& is_aluw
, is_brar
};
257 for (unsigned j
= 0; j
< ARRAY_SIZE(collisions
); ++j
) {
258 if (!collisions
[j
]) continue;
260 /* When the hazard is from reading, we move and rewrite
261 * sources (typical case). When it's from writing, we
262 * flip the move and rewrite destinations (obscure,
263 * only from control flow -- impossible in SSA) */
265 bool hazard_write
= (j
== 2);
267 unsigned idx
= spill_idx
++;
269 /* Insert move before each read/write, depending on the
270 * hazard we're trying to account for */
272 mir_foreach_instr_global_safe(ctx
, pre_use
) {
273 if (pre_use
->type
!= classes
[j
])
277 if (pre_use
->dest
!= i
)
280 midgard_instruction m
= v_mov(idx
, i
);
281 m
.dest_type
= pre_use
->dest_type
;
282 m
.src_types
[1] = m
.dest_type
;
283 m
.mask
= pre_use
->mask
;
285 midgard_instruction
*use
= mir_next_op(pre_use
);
287 mir_insert_instruction_before(ctx
, use
, m
);
288 mir_rewrite_index_dst_single(pre_use
, i
, idx
);
290 if (!mir_has_arg(pre_use
, i
))
295 midgard_instruction m
= v_mov(i
, idx
);
296 m
.mask
= mir_from_bytemask(mir_round_bytemask_up(
297 mir_bytemask_of_read_components(pre_use
, i
), 32), 32);
298 mir_insert_instruction_before(ctx
, pre_use
, m
);
299 mir_rewrite_index_src_single(pre_use
, i
, idx
);
314 mir_compute_interference(
315 compiler_context
*ctx
,
316 struct lcra_state
*l
)
318 /* First, we need liveness information to be computed per block */
319 mir_compute_liveness(ctx
);
321 /* We need to force r1.w live throughout a blend shader */
326 mir_foreach_block(ctx
, _block
) {
327 midgard_block
*block
= (midgard_block
*) _block
;
328 mir_foreach_instr_in_block_rev(block
, ins
) {
337 mir_foreach_instr_global(ctx
, ins
) {
338 if (ins
->dest
< ctx
->temp_count
)
339 lcra_add_node_interference(l
, ins
->dest
, mir_bytemask(ins
), r1w
, 0xF);
343 /* Now that every block has live_in/live_out computed, we can determine
344 * interference by walking each block linearly. Take live_out at the
345 * end of each block and walk the block backwards. */
347 mir_foreach_block(ctx
, _blk
) {
348 midgard_block
*blk
= (midgard_block
*) _blk
;
349 uint16_t *live
= mem_dup(_blk
->live_out
, ctx
->temp_count
* sizeof(uint16_t));
351 mir_foreach_instr_in_block_rev(blk
, ins
) {
352 /* Mark all registers live after the instruction as
353 * interfering with the destination */
355 unsigned dest
= ins
->dest
;
357 if (dest
< ctx
->temp_count
) {
358 for (unsigned i
= 0; i
< ctx
->temp_count
; ++i
)
360 unsigned mask
= mir_bytemask(ins
);
361 lcra_add_node_interference(l
, dest
, mask
, i
, live
[i
]);
366 mir_liveness_ins_update(live
, ins
, ctx
->temp_count
);
374 mir_is_64(midgard_instruction
*ins
)
376 if (nir_alu_type_get_type_size(ins
->dest_type
) == 64)
379 mir_foreach_src(ins
, v
) {
380 if (nir_alu_type_get_type_size(ins
->src_types
[v
]) == 64)
387 /* This routine performs the actual register allocation. It should be succeeded
388 * by install_registers */
390 static struct lcra_state
*
391 allocate_registers(compiler_context
*ctx
, bool *spilled
)
393 /* The number of vec4 work registers available depends on when the
394 * uniforms start and the shader stage. By ABI we limit blend shaders
395 * to 8 registers, should be lower XXX */
396 int work_count
= ctx
->is_blend
? 8 :
397 16 - MAX2((ctx
->uniform_cutoff
- 8), 0);
399 /* No register allocation to do with no SSA */
401 if (!ctx
->temp_count
)
404 /* Initialize LCRA. Allocate an extra node at the end for a precoloured
405 * r1 for interference */
407 struct lcra_state
*l
= lcra_alloc_equations(ctx
->temp_count
+ 1, 5);
408 unsigned node_r1
= ctx
->temp_count
;
410 /* Starts of classes, in bytes */
411 l
->class_start
[REG_CLASS_WORK
] = 16 * 0;
412 l
->class_start
[REG_CLASS_LDST
] = 16 * 26;
413 l
->class_start
[REG_CLASS_TEXR
] = 16 * 28;
414 l
->class_start
[REG_CLASS_TEXW
] = 16 * 28;
416 l
->class_size
[REG_CLASS_WORK
] = 16 * work_count
;
417 l
->class_size
[REG_CLASS_LDST
] = 16 * 2;
418 l
->class_size
[REG_CLASS_TEXR
] = 16 * 2;
419 l
->class_size
[REG_CLASS_TEXW
] = 16 * 2;
421 lcra_set_disjoint_class(l
, REG_CLASS_TEXR
, REG_CLASS_TEXW
);
423 /* To save space on T*20, we don't have real texture registers.
424 * Instead, tex inputs reuse the load/store pipeline registers, and
425 * tex outputs use work r0/r1. Note we still use TEXR/TEXW classes,
426 * noting that this handles interferences and sizes correctly. */
428 if (ctx
->quirks
& MIDGARD_INTERPIPE_REG_ALIASING
) {
429 l
->class_start
[REG_CLASS_TEXR
] = l
->class_start
[REG_CLASS_LDST
];
430 l
->class_start
[REG_CLASS_TEXW
] = l
->class_start
[REG_CLASS_WORK
];
433 unsigned *found_class
= calloc(sizeof(unsigned), ctx
->temp_count
);
434 unsigned *min_alignment
= calloc(sizeof(unsigned), ctx
->temp_count
);
435 unsigned *min_bound
= calloc(sizeof(unsigned), ctx
->temp_count
);
437 mir_foreach_instr_global(ctx
, ins
) {
438 /* Swizzles of 32-bit sources on 64-bit instructions need to be
439 * aligned to either bottom (xy) or top (zw). More general
440 * swizzle lowering should happen prior to scheduling (TODO),
441 * but once we get RA we shouldn't disrupt this further. Align
442 * sources of 64-bit instructions. */
444 if (ins
->type
== TAG_ALU_4
&& mir_is_64(ins
)) {
445 mir_foreach_src(ins
, v
) {
446 unsigned s
= ins
->src
[v
];
448 if (s
< ctx
->temp_count
)
449 min_alignment
[s
] = 3;
453 if (ins
->type
== TAG_LOAD_STORE_4
&& OP_HAS_ADDRESS(ins
->load_store
.op
)) {
454 mir_foreach_src(ins
, v
) {
455 unsigned s
= ins
->src
[v
];
456 unsigned size
= nir_alu_type_get_type_size(ins
->src_types
[v
]);
458 if (s
< ctx
->temp_count
)
459 min_alignment
[s
] = (size
== 64) ? 3 : 2;
463 if (ins
->dest
>= SSA_FIXED_MINIMUM
) continue;
465 unsigned size
= nir_alu_type_get_type_size(ins
->dest_type
);
470 /* 0 for x, 1 for xy, 2 for xyz, 3 for xyzw */
471 int comps1
= util_logbase2(ins
->mask
);
473 int bytes
= (comps1
+ 1) * (size
/ 8);
475 /* Use the largest class if there's ambiguity, this
476 * handles partial writes */
478 int dest
= ins
->dest
;
479 found_class
[dest
] = MAX2(found_class
[dest
], bytes
);
481 min_alignment
[dest
] =
482 (size
== 16) ? 1 : /* (1 << 1) = 2-byte */
483 (size
== 32) ? 2 : /* (1 << 2) = 4-byte */
484 (size
== 64) ? 3 : /* (1 << 3) = 8-byte */
487 /* We can't cross xy/zw boundaries. TODO: vec8 can */
491 /* We don't have a swizzle for the conditional and we don't
492 * want to muck with the conditional itself, so just force
493 * alignment for now */
495 if (ins
->type
== TAG_ALU_4
&& OP_IS_CSEL_V(ins
->alu
.op
)) {
496 min_alignment
[dest
] = 4; /* 1 << 4= 16-byte = vec4 */
498 /* LCRA assumes bound >= alignment */
499 min_bound
[dest
] = 16;
502 /* Since ld/st swizzles and masks are 32-bit only, we need them
503 * aligned to enable final packing */
504 if (ins
->type
== TAG_LOAD_STORE_4
)
505 min_alignment
[dest
] = MAX2(min_alignment
[dest
], 2);
508 for (unsigned i
= 0; i
< ctx
->temp_count
; ++i
) {
509 lcra_set_alignment(l
, i
, min_alignment
[i
] ? min_alignment
[i
] : 2,
510 min_bound
[i
] ? min_bound
[i
] : 16);
511 lcra_restrict_range(l
, i
, found_class
[i
]);
518 /* Next, we'll determine semantic class. We default to zero (work).
519 * But, if we're used with a special operation, that will force us to a
520 * particular class. Each node must be assigned to exactly one class; a
521 * prepass before RA should have lowered what-would-have-been
522 * multiclass nodes into a series of moves to break it up into multiple
525 mir_foreach_instr_global(ctx
, ins
) {
526 /* Check if this operation imposes any classes */
528 if (ins
->type
== TAG_LOAD_STORE_4
) {
529 set_class(l
->class, ins
->src
[0], REG_CLASS_LDST
);
530 set_class(l
->class, ins
->src
[1], REG_CLASS_LDST
);
531 set_class(l
->class, ins
->src
[2], REG_CLASS_LDST
);
533 if (OP_IS_VEC4_ONLY(ins
->load_store
.op
)) {
534 lcra_restrict_range(l
, ins
->dest
, 16);
535 lcra_restrict_range(l
, ins
->src
[0], 16);
536 lcra_restrict_range(l
, ins
->src
[1], 16);
537 lcra_restrict_range(l
, ins
->src
[2], 16);
539 } else if (ins
->type
== TAG_TEXTURE_4
) {
540 set_class(l
->class, ins
->dest
, REG_CLASS_TEXW
);
541 set_class(l
->class, ins
->src
[0], REG_CLASS_TEXR
);
542 set_class(l
->class, ins
->src
[1], REG_CLASS_TEXR
);
543 set_class(l
->class, ins
->src
[2], REG_CLASS_TEXR
);
544 set_class(l
->class, ins
->src
[3], REG_CLASS_TEXR
);
548 /* Check that the semantics of the class are respected */
549 mir_foreach_instr_global(ctx
, ins
) {
550 assert(check_write_class(l
->class, ins
->type
, ins
->dest
));
551 assert(check_read_class(l
->class, ins
->type
, ins
->src
[0]));
552 assert(check_read_class(l
->class, ins
->type
, ins
->src
[1]));
553 assert(check_read_class(l
->class, ins
->type
, ins
->src
[2]));
556 /* Mark writeout to r0, depth to r1.x, stencil to r1.y,
557 * render target to r1.z, unknown to r1.w */
558 mir_foreach_instr_global(ctx
, ins
) {
559 if (!(ins
->compact_branch
&& ins
->writeout
)) continue;
561 if (ins
->src
[0] < ctx
->temp_count
)
562 l
->solutions
[ins
->src
[0]] = 0;
564 if (ins
->src
[2] < ctx
->temp_count
)
565 l
->solutions
[ins
->src
[2]] = (16 * 1) + COMPONENT_X
* 4;
567 if (ins
->src
[3] < ctx
->temp_count
)
568 l
->solutions
[ins
->src
[3]] = (16 * 1) + COMPONENT_Y
* 4;
570 if (ins
->src
[1] < ctx
->temp_count
)
571 l
->solutions
[ins
->src
[1]] = (16 * 1) + COMPONENT_Z
* 4;
573 if (ins
->dest
< ctx
->temp_count
)
574 l
->solutions
[ins
->dest
] = (16 * 1) + COMPONENT_W
* 4;
577 /* Destinations of instructions in a writeout block cannot be assigned
578 * to r1 unless they are actually used as r1 from the writeout itself,
579 * since the writes to r1 are special. A code sequence like:
581 * sadd.fmov r1.x, [...]
582 * vadd.fadd r0, r1, r2
585 * will misbehave since the r1.x write will be interpreted as a
586 * gl_FragDepth write so it won't show up correctly when r1 is read in
587 * the following segment. We model this as interference.
590 l
->solutions
[node_r1
] = (16 * 1);
592 mir_foreach_block(ctx
, _blk
) {
593 midgard_block
*blk
= (midgard_block
*) _blk
;
595 mir_foreach_bundle_in_block(blk
, v
) {
596 /* We need at least a writeout and nonwriteout instruction */
597 if (v
->instruction_count
< 2)
600 /* Branches always come at the end */
601 midgard_instruction
*br
= v
->instructions
[v
->instruction_count
- 1];
606 for (signed i
= v
->instruction_count
- 2; i
>= 0; --i
) {
607 midgard_instruction
*ins
= v
->instructions
[i
];
609 if (ins
->dest
>= ctx
->temp_count
)
612 bool used_as_r1
= (br
->dest
== ins
->dest
);
614 mir_foreach_src(br
, s
)
615 used_as_r1
|= (s
> 0) && (br
->src
[s
] == ins
->dest
);
618 lcra_add_node_interference(l
, ins
->dest
, mir_bytemask(ins
), node_r1
, 0xFFFF);
623 /* Precolour blend input to r0. Note writeout is necessarily at the end
624 * and blend shaders are single-RT only so there is only a single
625 * writeout block, so this cannot conflict with the writeout r0 (there
626 * is no need to have an intermediate move) */
628 if (ctx
->blend_input
!= ~0) {
629 assert(ctx
->blend_input
< ctx
->temp_count
);
630 l
->solutions
[ctx
->blend_input
] = 0;
633 mir_compute_interference(ctx
, l
);
635 *spilled
= !lcra_solve(l
);
640 /* Once registers have been decided via register allocation
641 * (allocate_registers), we need to rewrite the MIR to use registers instead of
645 install_registers_instr(
646 compiler_context
*ctx
,
647 struct lcra_state
*l
,
648 midgard_instruction
*ins
)
650 unsigned src_shift
[MIR_SRC_COUNT
];
652 for (unsigned i
= 0; i
< MIR_SRC_COUNT
; ++i
) {
654 util_logbase2(nir_alu_type_get_type_size(ins
->src_types
[i
]) / 8);
657 unsigned dest_shift
=
658 util_logbase2(nir_alu_type_get_type_size(ins
->dest_type
) / 8);
665 if (ins
->compact_branch
)
668 struct phys_reg src1
= index_to_reg(ctx
, l
, ins
->src
[0], src_shift
[0]);
669 struct phys_reg src2
= index_to_reg(ctx
, l
, ins
->src
[1], src_shift
[1]);
670 struct phys_reg dest
= index_to_reg(ctx
, l
, ins
->dest
, dest_shift
);
672 mir_set_bytemask(ins
, mir_bytemask(ins
) << dest
.offset
);
674 unsigned dest_offset
=
675 GET_CHANNEL_COUNT(alu_opcode_props
[ins
->alu
.op
].props
) ? 0 :
678 offset_swizzle(ins
->swizzle
[0], src1
.offset
, src1
.shift
, dest
.shift
, dest_offset
);
680 ins
->registers
.src1_reg
= src1
.reg
;
682 ins
->registers
.src2_imm
= ins
->has_inline_constant
;
684 if (ins
->has_inline_constant
) {
685 /* Encode inline 16-bit constant. See disassembler for
686 * where the algorithm is from */
688 ins
->registers
.src2_reg
= ins
->inline_constant
>> 11;
690 int lower_11
= ins
->inline_constant
& ((1 << 12) - 1);
691 uint16_t imm
= ((lower_11
>> 8) & 0x7) |
692 ((lower_11
& 0xFF) << 3);
694 ins
->alu
.src2
= imm
<< 2;
696 offset_swizzle(ins
->swizzle
[1], src2
.offset
, src2
.shift
, dest
.shift
, dest_offset
);
698 ins
->registers
.src2_reg
= src2
.reg
;
701 ins
->registers
.out_reg
= dest
.reg
;
705 case TAG_LOAD_STORE_4
: {
706 /* Which physical register we read off depends on
707 * whether we are loading or storing -- think about the
708 * logical dataflow */
710 bool encodes_src
= OP_IS_STORE(ins
->load_store
.op
);
713 struct phys_reg src
= index_to_reg(ctx
, l
, ins
->src
[0], src_shift
[0]);
714 assert(src
.reg
== 26 || src
.reg
== 27);
716 ins
->load_store
.reg
= src
.reg
- 26;
717 offset_swizzle(ins
->swizzle
[0], src
.offset
, src
.shift
, 0, 0);
719 struct phys_reg dst
= index_to_reg(ctx
, l
, ins
->dest
, dest_shift
);
721 ins
->load_store
.reg
= dst
.reg
;
722 offset_swizzle(ins
->swizzle
[0], 0, 2, 2, dst
.offset
);
723 mir_set_bytemask(ins
, mir_bytemask(ins
) << dst
.offset
);
726 /* We also follow up by actual arguments */
728 unsigned src2
= ins
->src
[1];
729 unsigned src3
= ins
->src
[2];
732 struct phys_reg src
= index_to_reg(ctx
, l
, src2
, 2);
733 unsigned component
= src
.offset
>> src
.shift
;
734 assert(component
<< src
.shift
== src
.offset
);
735 ins
->load_store
.arg_1
|= midgard_ldst_reg(src
.reg
, component
);
739 struct phys_reg src
= index_to_reg(ctx
, l
, src3
, 2);
740 unsigned component
= src
.offset
>> src
.shift
;
741 assert(component
<< src
.shift
== src
.offset
);
742 ins
->load_store
.arg_2
|= midgard_ldst_reg(src
.reg
, component
);
748 case TAG_TEXTURE_4
: {
749 if (ins
->texture
.op
== TEXTURE_OP_BARRIER
)
752 /* Grab RA results */
753 struct phys_reg dest
= index_to_reg(ctx
, l
, ins
->dest
, dest_shift
);
754 struct phys_reg coord
= index_to_reg(ctx
, l
, ins
->src
[1], src_shift
[1]);
755 struct phys_reg lod
= index_to_reg(ctx
, l
, ins
->src
[2], src_shift
[2]);
756 struct phys_reg offset
= index_to_reg(ctx
, l
, ins
->src
[3], src_shift
[3]);
758 /* First, install the texture coordinate */
759 ins
->texture
.in_reg_select
= coord
.reg
& 1;
760 offset_swizzle(ins
->swizzle
[1], coord
.offset
, coord
.shift
, dest
.shift
, 0);
762 /* Next, install the destination */
763 ins
->texture
.out_reg_select
= dest
.reg
& 1;
764 offset_swizzle(ins
->swizzle
[0], 0, 2, dest
.shift
,
765 dest_shift
== 1 ? dest
.offset
% 8 :
767 mir_set_bytemask(ins
, mir_bytemask(ins
) << dest
.offset
);
769 /* If there is a register LOD/bias, use it */
770 if (ins
->src
[2] != ~0) {
771 assert(!(lod
.offset
& 3));
772 midgard_tex_register_select sel
= {
773 .select
= lod
.reg
& 1,
775 .component
= lod
.offset
/ 4
779 memcpy(&packed
, &sel
, sizeof(packed
));
780 ins
->texture
.bias
= packed
;
783 /* If there is an offset register, install it */
784 if (ins
->src
[3] != ~0) {
785 unsigned x
= offset
.offset
/ 4;
789 /* Check range, TODO: half-registers */
792 ins
->texture
.offset
=
794 (offset
.reg
& 1) << 1 | /* select */
795 (0 << 2) | /* upper */
796 (x
<< 3) | /* swizzle */
797 (y
<< 5) | /* swizzle */
798 (z
<< 7); /* swizzle */
810 install_registers(compiler_context
*ctx
, struct lcra_state
*l
)
812 mir_foreach_instr_global(ctx
, ins
)
813 install_registers_instr(ctx
, l
, ins
);
817 /* If register allocation fails, find the best spill node */
820 mir_choose_spill_node(
821 compiler_context
*ctx
,
822 struct lcra_state
*l
)
824 /* We can't spill a previously spilled value or an unspill */
826 mir_foreach_instr_global(ctx
, ins
) {
827 if (ins
->no_spill
& (1 << l
->spill_class
)) {
828 lcra_set_node_spill_cost(l
, ins
->dest
, -1);
830 if (l
->spill_class
!= REG_CLASS_WORK
) {
831 mir_foreach_src(ins
, s
)
832 lcra_set_node_spill_cost(l
, ins
->src
[s
], -1);
837 return lcra_get_best_spill_node(l
);
840 /* Once we've chosen a spill node, spill it */
844 compiler_context
*ctx
,
846 unsigned spill_class
,
847 unsigned *spill_count
)
849 if (spill_class
== REG_CLASS_WORK
&& ctx
->is_blend
)
850 unreachable("Blend shader spilling is currently unimplemented");
852 unsigned spill_index
= ctx
->temp_count
;
854 /* We have a spill node, so check the class. Work registers
855 * legitimately spill to TLS, but special registers just spill to work
858 bool is_special
= spill_class
!= REG_CLASS_WORK
;
859 bool is_special_w
= spill_class
== REG_CLASS_TEXW
;
861 /* Allocate TLS slot (maybe) */
862 unsigned spill_slot
= !is_special
? (*spill_count
)++ : 0;
864 /* For TLS, replace all stores to the spilled node. For
865 * special reads, just keep as-is; the class will be demoted
866 * implicitly. For special writes, spill to a work register */
868 if (!is_special
|| is_special_w
) {
870 spill_slot
= spill_index
++;
872 mir_foreach_block(ctx
, _block
) {
873 midgard_block
*block
= (midgard_block
*) _block
;
874 mir_foreach_instr_in_block_safe(block
, ins
) {
875 if (ins
->dest
!= spill_node
) continue;
877 midgard_instruction st
;
880 st
= v_mov(spill_node
, spill_slot
);
881 st
.no_spill
|= (1 << spill_class
);
883 ins
->dest
= spill_index
++;
884 ins
->no_spill
|= (1 << spill_class
);
885 st
= v_load_store_scratch(ins
->dest
, spill_slot
, true, ins
->mask
);
888 /* Hint: don't rewrite this node */
891 mir_insert_instruction_after_scheduled(ctx
, block
, ins
, st
);
899 /* For special reads, figure out how many bytes we need */
900 unsigned read_bytemask
= 0;
902 mir_foreach_instr_global_safe(ctx
, ins
) {
903 read_bytemask
|= mir_bytemask_of_read_components(ins
, spill_node
);
906 /* Insert a load from TLS before the first consecutive
907 * use of the node, rewriting to use spilled indices to
908 * break up the live range. Or, for special, insert a
909 * move. Ironically the latter *increases* register
910 * pressure, but the two uses of the spilling mechanism
911 * are somewhat orthogonal. (special spilling is to use
912 * work registers to back special registers; TLS
913 * spilling is to use memory to back work registers) */
915 mir_foreach_block(ctx
, _block
) {
916 midgard_block
*block
= (midgard_block
*) _block
;
917 mir_foreach_instr_in_block(block
, ins
) {
918 /* We can't rewrite the moves used to spill in the
919 * first place. These moves are hinted. */
920 if (ins
->hint
) continue;
922 /* If we don't use the spilled value, nothing to do */
923 if (!mir_has_arg(ins
, spill_node
)) continue;
928 index
= ++spill_index
;
930 midgard_instruction
*before
= ins
;
931 midgard_instruction st
;
935 st
= v_mov(spill_node
, index
);
936 st
.no_spill
|= (1 << spill_class
);
939 st
= v_load_store_scratch(index
, spill_slot
, false, 0xF);
942 /* Mask the load based on the component count
943 * actually needed to prevent RA loops */
945 st
.mask
= mir_from_bytemask(mir_round_bytemask_up(
946 read_bytemask
, 32), 32);
948 mir_insert_instruction_before_scheduled(ctx
, block
, before
, st
);
950 /* Special writes already have their move spilled in */
956 mir_rewrite_index_src_single(ins
, spill_node
, index
);
965 mir_foreach_instr_global(ctx
, ins
) {
970 /* Run register allocation in a loop, spilling until we succeed */
973 mir_ra(compiler_context
*ctx
)
975 struct lcra_state
*l
= NULL
;
976 bool spilled
= false;
977 int iter_count
= 1000; /* max iterations */
979 /* Number of 128-bit slots in memory we've spilled into */
980 unsigned spill_count
= 0;
983 mir_create_pipeline_registers(ctx
);
987 signed spill_node
= mir_choose_spill_node(ctx
, l
);
989 if (spill_node
== -1) {
990 fprintf(stderr
, "ERROR: Failed to choose spill node\n");
994 mir_spill_register(ctx
, spill_node
, l
->spill_class
, &spill_count
);
997 mir_squeeze_index(ctx
);
998 mir_invalidate_liveness(ctx
);
1005 l
= allocate_registers(ctx
, &spilled
);
1006 } while(spilled
&& ((iter_count
--) > 0));
1008 if (iter_count
<= 0) {
1009 fprintf(stderr
, "panfrost: Gave up allocating registers, rendering will be incomplete\n");
1013 /* Report spilling information. spill_count is in 128-bit slots (vec4 x
1014 * fp32), but tls_size is in bytes, so multiply by 16 */
1016 ctx
->tls_size
= spill_count
* 16;
1018 install_registers(ctx
, l
);