2 * Copyright (C) 2019 Alyssa Rosenzweig <alyssa@rosenzweig.io>
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
25 #include "midgard_ops.h"
27 void mir_rewrite_index_src_single(midgard_instruction
*ins
, unsigned old
, unsigned new)
29 for (unsigned i
= 0; i
< ARRAY_SIZE(ins
->src
); ++i
) {
30 if (ins
->src
[i
] == old
)
35 void mir_rewrite_index_dst_single(midgard_instruction
*ins
, unsigned old
, unsigned new)
41 static midgard_vector_alu_src
42 mir_get_alu_src(midgard_instruction
*ins
, unsigned idx
)
44 unsigned b
= (idx
== 0) ? ins
->alu
.src1
: ins
->alu
.src2
;
45 return vector_alu_from_unsigned(b
);
49 mir_get_swizzle(midgard_instruction
*ins
, unsigned idx
)
51 if (ins
->type
== TAG_ALU_4
) {
52 if (idx
== 2 || ins
->compact_branch
)
53 return ins
->cond_swizzle
;
55 return (mir_get_alu_src(ins
, idx
)).swizzle
;
56 } else if (ins
->type
== TAG_LOAD_STORE_4
) {
57 /* Main swizzle of a load is on the destination */
58 if (!OP_IS_STORE(ins
->load_store
.op
))
63 return ins
->load_store
.swizzle
;
67 (idx
== 2) ? ins
->load_store
.arg_2
: ins
->load_store
.arg_1
;
69 /* TODO: Integrate component count with properties */
70 unsigned components
= 1;
71 switch (ins
->load_store
.op
) {
72 case midgard_op_ld_int4
:
73 components
= (idx
== 0) ? 2 : 1;
75 case midgard_op_st_int4
:
76 components
= (idx
== 1) ? 2 : 1;
78 case midgard_op_ld_cubemap_coords
:
81 case midgard_op_ldst_perspective_division_z
:
84 case midgard_op_ldst_perspective_division_w
:
92 return component_to_swizzle(midgard_ldst_select(raw
).component
, components
);
95 unreachable("Unknown load/store source");
97 } else if (ins
->type
== TAG_TEXTURE_4
) {
100 return ins
->texture
.in_reg_swizzle
;
102 /* Swizzle on bias doesn't make sense */
105 unreachable("Unknown texture source");
108 unreachable("Unknown type");
113 mir_set_swizzle(midgard_instruction
*ins
, unsigned idx
, unsigned new)
115 if (ins
->type
== TAG_ALU_4
) {
116 if (idx
== 2 || ins
->compact_branch
) {
117 ins
->cond_swizzle
= new;
121 unsigned b
= (idx
== 0) ? ins
->alu
.src1
: ins
->alu
.src2
;
123 midgard_vector_alu_src s
=
124 vector_alu_from_unsigned(b
);
127 unsigned pack
= vector_alu_srco_unsigned(s
);
130 ins
->alu
.src1
= pack
;
132 ins
->alu
.src2
= pack
;
133 } else if (ins
->type
== TAG_LOAD_STORE_4
) {
134 /* Main swizzle of a load is on the destination */
135 if (!OP_IS_STORE(ins
->load_store
.op
))
140 ins
->load_store
.swizzle
= new;
145 (idx
== 2) ? ins
->load_store
.arg_2
: ins
->load_store
.arg_1
;
147 midgard_ldst_register_select sel
148 = midgard_ldst_select(raw
);
149 sel
.component
= swizzle_to_component(new);
150 uint8_t packed
= midgard_ldst_pack(sel
);
153 ins
->load_store
.arg_2
= packed
;
155 ins
->load_store
.arg_1
= packed
;
163 } else if (ins
->type
== TAG_TEXTURE_4
) {
166 ins
->texture
.in_reg_swizzle
= new;
173 unreachable("Unknown type");
178 mir_rewrite_index_src_single_swizzle(midgard_instruction
*ins
, unsigned old
, unsigned new, unsigned swizzle
)
180 for (unsigned i
= 0; i
< ARRAY_SIZE(ins
->src
); ++i
) {
181 if (ins
->src
[i
] != old
) continue;
185 mir_set_swizzle(ins
, i
,
186 pan_compose_swizzle(mir_get_swizzle(ins
, i
), swizzle
));
191 mir_rewrite_index_src(compiler_context
*ctx
, unsigned old
, unsigned new)
193 mir_foreach_instr_global(ctx
, ins
) {
194 mir_rewrite_index_src_single(ins
, old
, new);
199 mir_rewrite_index_src_swizzle(compiler_context
*ctx
, unsigned old
, unsigned new, unsigned swizzle
)
201 mir_foreach_instr_global(ctx
, ins
) {
202 mir_rewrite_index_src_single_swizzle(ins
, old
, new, swizzle
);
207 mir_rewrite_index_dst(compiler_context
*ctx
, unsigned old
, unsigned new)
209 mir_foreach_instr_global(ctx
, ins
) {
210 mir_rewrite_index_dst_single(ins
, old
, new);
215 mir_rewrite_index(compiler_context
*ctx
, unsigned old
, unsigned new)
217 mir_rewrite_index_src(ctx
, old
, new);
218 mir_rewrite_index_dst(ctx
, old
, new);
222 mir_use_count(compiler_context
*ctx
, unsigned value
)
224 unsigned used_count
= 0;
226 mir_foreach_instr_global(ctx
, ins
) {
227 if (mir_has_arg(ins
, value
))
234 /* Checks if a value is used only once (or totally dead), which is an important
235 * heuristic to figure out if certain optimizations are Worth It (TM) */
238 mir_single_use(compiler_context
*ctx
, unsigned value
)
240 /* We can replicate constants in places so who cares */
241 if (value
== SSA_FIXED_REGISTER(REGISTER_CONSTANT
))
244 return mir_use_count(ctx
, value
) <= 1;
248 mir_nontrivial_raw_mod(midgard_vector_alu_src src
, bool is_int
)
251 return src
.mod
== midgard_int_shift
;
257 mir_nontrivial_mod(midgard_vector_alu_src src
, bool is_int
, unsigned mask
)
259 if (mir_nontrivial_raw_mod(src
, is_int
)) return true;
261 /* size-conversion */
262 if (src
.half
) return true;
265 for (unsigned c
= 0; c
< 4; ++c
) {
266 if (!(mask
& (1 << c
))) continue;
267 if (((src
.swizzle
>> (2*c
)) & 3) != c
) return true;
274 mir_nontrivial_source2_mod(midgard_instruction
*ins
)
276 bool is_int
= midgard_is_integer_op(ins
->alu
.op
);
278 midgard_vector_alu_src src2
=
279 vector_alu_from_unsigned(ins
->alu
.src2
);
281 return mir_nontrivial_mod(src2
, is_int
, ins
->mask
);
285 mir_nontrivial_source2_mod_simple(midgard_instruction
*ins
)
287 bool is_int
= midgard_is_integer_op(ins
->alu
.op
);
289 midgard_vector_alu_src src2
=
290 vector_alu_from_unsigned(ins
->alu
.src2
);
292 return mir_nontrivial_raw_mod(src2
, is_int
) || src2
.half
;
296 mir_nontrivial_outmod(midgard_instruction
*ins
)
298 bool is_int
= midgard_is_integer_op(ins
->alu
.op
);
299 unsigned mod
= ins
->alu
.outmod
;
305 /* Type conversion is a sort of outmod */
306 if (ins
->alu
.dest_override
!= midgard_dest_override_none
)
310 return mod
!= midgard_outmod_int_wrap
;
312 return mod
!= midgard_outmod_none
;
315 /* Checks if an index will be used as a special register -- basically, if we're
316 * used as the input to a non-ALU op */
319 mir_special_index(compiler_context
*ctx
, unsigned idx
)
321 mir_foreach_instr_global(ctx
, ins
) {
322 bool is_ldst
= ins
->type
== TAG_LOAD_STORE_4
;
323 bool is_tex
= ins
->type
== TAG_TEXTURE_4
;
324 bool is_writeout
= ins
->compact_branch
&& ins
->writeout
;
326 if (!(is_ldst
|| is_tex
|| is_writeout
))
329 if (mir_has_arg(ins
, idx
))
336 /* Is a node written before a given instruction? */
339 mir_is_written_before(compiler_context
*ctx
, midgard_instruction
*ins
, unsigned node
)
341 if (node
>= SSA_FIXED_MINIMUM
)
344 mir_foreach_instr_global(ctx
, q
) {
355 /* Creates a mask of the components of a node read by an instruction, by
356 * analyzing the swizzle with respect to the instruction's mask. E.g.:
358 * fadd r0.xz, r1.yyyy, r2.zwyx
360 * will return a mask of Z/Y for r2
364 mir_mask_of_read_components_single(unsigned swizzle
, unsigned outmask
)
368 for (unsigned c
= 0; c
< 4; ++c
) {
369 if (!(outmask
& (1 << c
))) continue;
371 unsigned comp
= (swizzle
>> (2*c
)) & 3;
379 mir_source_count(midgard_instruction
*ins
)
381 if (ins
->type
== TAG_ALU_4
) {
382 /* ALU is always binary, except csel */
383 return OP_IS_CSEL(ins
->alu
.op
) ? 3 : 2;
384 } else if (ins
->type
== TAG_LOAD_STORE_4
) {
385 bool load
= !OP_IS_STORE(ins
->load_store
.op
);
386 return (load
? 2 : 3);
387 } else if (ins
->type
== TAG_TEXTURE_4
) {
388 /* Coords, bias.. TODO: Offsets? */
391 unreachable("Invalid instruction type");
396 mir_mask_of_read_components(midgard_instruction
*ins
, unsigned node
)
400 for (unsigned i
= 0; i
< mir_source_count(ins
); ++i
) {
401 if (ins
->src
[i
] != node
) continue;
403 /* Branch writeout uses all components */
404 if (ins
->compact_branch
&& ins
->writeout
&& (i
== 0))
407 /* Conditional branches read one component (TODO: multi branch??) */
408 if (ins
->compact_branch
&& !ins
->prepacked_branch
&& ins
->branch
.conditional
&& (i
== 0))
411 /* ALU ops act componentwise so we need to pay attention to
412 * their mask. Texture/ldst does not so we don't clamp source
413 * readmasks based on the writemask */
414 unsigned qmask
= (ins
->type
== TAG_ALU_4
) ? ins
->mask
: 0xF;
416 /* Handle dot products and things */
417 if (ins
->type
== TAG_ALU_4
&& !ins
->compact_branch
) {
418 unsigned channel_override
=
419 GET_CHANNEL_COUNT(alu_opcode_props
[ins
->alu
.op
].props
);
421 if (channel_override
)
422 qmask
= mask_of(channel_override
);
425 unsigned swizzle
= mir_get_swizzle(ins
, i
);
426 unsigned m
= mir_mask_of_read_components_single(swizzle
, qmask
);
435 mir_ubo_shift(midgard_load_store_op op
)
438 case midgard_op_ld_ubo_char
:
440 case midgard_op_ld_ubo_char2
:
442 case midgard_op_ld_ubo_char4
:
444 case midgard_op_ld_ubo_short4
:
446 case midgard_op_ld_ubo_int4
:
449 unreachable("Invalid op");
453 /* Register allocation occurs after instruction scheduling, which is fine until
454 * we start needing to spill registers and therefore insert instructions into
455 * an already-scheduled program. We don't have to be terribly efficient about
456 * this, since spilling is already slow. So just semantically we need to insert
457 * the instruction into a new bundle before/after the bundle of the instruction
460 static midgard_bundle
461 mir_bundle_for_op(compiler_context
*ctx
, midgard_instruction ins
)
463 midgard_instruction
*u
= mir_upload_ins(ctx
, ins
);
465 midgard_bundle bundle
= {
467 .instruction_count
= 1,
468 .instructions
= { u
},
471 if (bundle
.tag
== TAG_ALU_4
) {
472 assert(OP_IS_MOVE(u
->alu
.op
));
475 size_t bytes_emitted
= sizeof(uint32_t) + sizeof(midgard_reg_info
) + sizeof(midgard_vector_alu
);
476 bundle
.padding
= ~(bytes_emitted
- 1) & 0xF;
477 bundle
.control
= ins
.type
| u
->unit
;
484 mir_bundle_idx_for_ins(midgard_instruction
*tag
, midgard_block
*block
)
486 midgard_bundle
*bundles
=
487 (midgard_bundle
*) block
->bundles
.data
;
489 size_t count
= (block
->bundles
.size
/ sizeof(midgard_bundle
));
491 for (unsigned i
= 0; i
< count
; ++i
) {
492 for (unsigned j
= 0; j
< bundles
[i
].instruction_count
; ++j
) {
493 if (bundles
[i
].instructions
[j
] == tag
)
498 mir_print_instruction(tag
);
499 unreachable("Instruction not scheduled in block");
503 mir_insert_instruction_before_scheduled(
504 compiler_context
*ctx
,
505 midgard_block
*block
,
506 midgard_instruction
*tag
,
507 midgard_instruction ins
)
509 unsigned before
= mir_bundle_idx_for_ins(tag
, block
);
510 size_t count
= util_dynarray_num_elements(&block
->bundles
, midgard_bundle
);
511 UNUSED
void *unused
= util_dynarray_grow(&block
->bundles
, midgard_bundle
, 1);
513 midgard_bundle
*bundles
= (midgard_bundle
*) block
->bundles
.data
;
514 memmove(bundles
+ before
+ 1, bundles
+ before
, (count
- before
) * sizeof(midgard_bundle
));
515 midgard_bundle
*before_bundle
= bundles
+ before
+ 1;
517 midgard_bundle
new = mir_bundle_for_op(ctx
, ins
);
518 memcpy(bundles
+ before
, &new, sizeof(new));
520 list_addtail(&new.instructions
[0]->link
, &before_bundle
->instructions
[0]->link
);
524 mir_insert_instruction_after_scheduled(
525 compiler_context
*ctx
,
526 midgard_block
*block
,
527 midgard_instruction
*tag
,
528 midgard_instruction ins
)
530 /* We need to grow the bundles array to add our new bundle */
531 size_t count
= util_dynarray_num_elements(&block
->bundles
, midgard_bundle
);
532 UNUSED
void *unused
= util_dynarray_grow(&block
->bundles
, midgard_bundle
, 1);
534 /* Find the bundle that we want to insert after */
535 unsigned after
= mir_bundle_idx_for_ins(tag
, block
);
537 /* All the bundles after that one, we move ahead by one */
538 midgard_bundle
*bundles
= (midgard_bundle
*) block
->bundles
.data
;
539 memmove(bundles
+ after
+ 2, bundles
+ after
+ 1, (count
- after
- 1) * sizeof(midgard_bundle
));
540 midgard_bundle
*after_bundle
= bundles
+ after
;
542 midgard_bundle
new = mir_bundle_for_op(ctx
, ins
);
543 memcpy(bundles
+ after
+ 1, &new, sizeof(new));
544 list_add(&new.instructions
[0]->link
, &after_bundle
->instructions
[after_bundle
->instruction_count
- 1]->link
);
547 /* Flip the first-two arguments of a (binary) op. Currently ALU
548 * only, no known uses for ldst/tex */
551 mir_flip(midgard_instruction
*ins
)
553 unsigned temp
= ins
->src
[0];
554 ins
->src
[0] = ins
->src
[1];
557 assert(ins
->type
== TAG_ALU_4
);
559 temp
= ins
->alu
.src1
;
560 ins
->alu
.src1
= ins
->alu
.src2
;
561 ins
->alu
.src2
= temp
;
564 /* Before squashing, calculate ctx->temp_count just by observing the MIR */
567 mir_compute_temp_count(compiler_context
*ctx
)
572 unsigned max_dest
= 0;
574 mir_foreach_instr_global(ctx
, ins
) {
575 if (ins
->dest
< SSA_FIXED_MINIMUM
)
576 max_dest
= MAX2(max_dest
, ins
->dest
+ 1);
579 ctx
->temp_count
= max_dest
;