2 * Copyright (C) 2018-2019 Alyssa Rosenzweig <alyssa@rosenzweig.io>
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
25 #include "midgard_ops.h"
27 static midgard_int_mod
28 mir_get_imod(bool shift
, nir_alu_type T
, bool half
, bool scalar
)
32 /* Sign-extension, really... */
33 return scalar
? 0 : midgard_int_normal
;
37 return midgard_int_shift
;
39 if (nir_alu_type_get_base_type(T
) == nir_type_int
)
40 return midgard_int_sign_extend
;
42 return midgard_int_zero_extend
;
46 mir_pack_mod(midgard_instruction
*ins
, unsigned i
, bool scalar
)
48 bool integer
= midgard_is_integer_op(ins
->alu
.op
);
49 unsigned base_size
= (8 << ins
->alu
.reg_mode
);
50 unsigned sz
= nir_alu_type_get_type_size(ins
->src_types
[i
]);
51 bool half
= (sz
== (base_size
>> 1));
54 mir_get_imod(ins
->src_shift
[i
], ins
->src_types
[i
], half
, scalar
) :
55 ((ins
->src_abs
[i
] << 0) |
56 ((ins
->src_neg
[i
] << 1)));
59 /* Midgard IR only knows vector ALU types, but we sometimes need to actually
60 * use scalar ALU instructions, for functional or performance reasons. To do
61 * this, we just demote vector ALU payloads to scalar. */
64 component_from_mask(unsigned mask
)
66 for (int c
= 0; c
< 8; ++c
) {
76 mir_pack_scalar_source(unsigned mod
, bool is_full
, unsigned component
)
78 midgard_scalar_alu_src s
= {
81 .component
= component
<< (is_full
? 1 : 0)
85 memcpy(&o
, &s
, sizeof(s
));
87 return o
& ((1 << 6) - 1);
90 static midgard_scalar_alu
91 vector_to_scalar_alu(midgard_vector_alu v
, midgard_instruction
*ins
)
93 bool is_full
= nir_alu_type_get_type_size(ins
->dest_type
) == 32;
95 bool half_0
= nir_alu_type_get_type_size(ins
->src_types
[0]) == 16;
96 bool half_1
= nir_alu_type_get_type_size(ins
->src_types
[1]) == 16;
97 unsigned comp
= component_from_mask(ins
->mask
);
99 unsigned packed_src
[2] = {
100 mir_pack_scalar_source(mir_pack_mod(ins
, 0, true), !half_0
, ins
->swizzle
[0][comp
]),
101 mir_pack_scalar_source(mir_pack_mod(ins
, 1, true), !half_1
, ins
->swizzle
[1][comp
])
104 /* The output component is from the mask */
105 midgard_scalar_alu s
= {
107 .src1
= packed_src
[0],
108 .src2
= packed_src
[1],
111 .output_full
= is_full
,
112 .output_component
= comp
115 /* Full components are physically spaced out */
117 assert(s
.output_component
< 4);
118 s
.output_component
<<= 1;
121 /* Inline constant is passed along rather than trying to extract it
124 if (ins
->has_inline_constant
) {
126 int lower_11
= ins
->inline_constant
& ((1 << 12) - 1);
127 imm
|= (lower_11
>> 9) & 3;
128 imm
|= (lower_11
>> 6) & 4;
129 imm
|= (lower_11
>> 2) & 0x38;
130 imm
|= (lower_11
& 63) << 6;
138 /* 64-bit swizzles are super easy since there are 2 components of 2 components
139 * in an 8-bit field ... lots of duplication to go around!
141 * Swizzles of 32-bit vectors accessed from 64-bit instructions are a little
142 * funny -- pack them *as if* they were native 64-bit, using rep_* flags to
143 * flag upper. For instance, xy would become 64-bit XY but that's just xyzw
144 * native. Likewise, zz would become 64-bit XX with rep* so it would be xyxy
145 * with rep. Pretty nifty, huh? */
148 mir_pack_swizzle_64(unsigned *swizzle
, unsigned max_component
)
152 for (unsigned i
= 0; i
< 2; ++i
) {
153 assert(swizzle
[i
] <= max_component
);
155 unsigned a
= (swizzle
[i
] & 1) ?
156 (COMPONENT_W
<< 2) | COMPONENT_Z
:
157 (COMPONENT_Y
<< 2) | COMPONENT_X
;
159 packed
|= a
<< (i
* 4);
166 mir_pack_mask_alu(midgard_instruction
*ins
)
168 unsigned effective
= ins
->mask
;
170 /* If we have a destination override, we need to figure out whether to
171 * override to the lower or upper half, shifting the effective mask in
172 * the latter, so AAAA.... becomes AAAA */
174 unsigned inst_size
= 8 << ins
->alu
.reg_mode
;
175 signed upper_shift
= mir_upper_override(ins
, inst_size
);
177 if (upper_shift
>= 0) {
178 effective
>>= upper_shift
;
179 ins
->alu
.dest_override
= upper_shift
?
180 midgard_dest_override_upper
:
181 midgard_dest_override_lower
;
183 ins
->alu
.dest_override
= midgard_dest_override_none
;
186 if (ins
->alu
.reg_mode
== midgard_reg_mode_32
)
187 ins
->alu
.mask
= expand_writemask(effective
, 2);
188 else if (ins
->alu
.reg_mode
== midgard_reg_mode_64
)
189 ins
->alu
.mask
= expand_writemask(effective
, 1);
191 ins
->alu
.mask
= effective
;
195 mir_pack_swizzle(unsigned mask
, unsigned *swizzle
,
196 nir_alu_type T
, midgard_reg_mode reg_mode
,
197 bool op_channeled
, bool *rep_low
, bool *rep_high
)
200 unsigned sz
= nir_alu_type_get_type_size(T
);
202 if (reg_mode
== midgard_reg_mode_64
) {
203 assert(sz
== 64 || sz
== 32);
204 unsigned components
= (sz
== 32) ? 4 : 2;
206 packed
= mir_pack_swizzle_64(swizzle
, components
);
209 bool lo
= swizzle
[0] >= COMPONENT_Z
;
210 bool hi
= swizzle
[1] >= COMPONENT_Z
;
213 /* We can't mix halves... */
221 } else if (sz
< 32) {
222 unreachable("Cannot encode 8/16 swizzle in 64-bit");
225 /* For 32-bit, swizzle packing is stupid-simple. For 16-bit,
226 * the strategy is to check whether the nibble we're on is
227 * upper or lower. We need all components to be on the same
228 * "side"; that much is enforced by the ISA and should have
229 * been lowered. TODO: 8-bit packing. TODO: vec8 */
231 unsigned first
= mask
? ffs(mask
) - 1 : 0;
232 bool upper
= swizzle
[first
] > 3;
237 bool dest_up
= !op_channeled
&& (first
>= 4);
239 for (unsigned c
= (dest_up
? 4 : 0); c
< (dest_up
? 8 : 4); ++c
) {
240 unsigned v
= swizzle
[c
];
242 bool t_upper
= v
> 3;
244 /* Ensure we're doing something sane */
246 if (mask
& (1 << c
)) {
247 assert(t_upper
== upper
);
251 /* Use the non upper part */
254 packed
|= v
<< (2 * (c
% 4));
258 /* Replicate for now.. should really pick a side for
261 if (reg_mode
== midgard_reg_mode_16
&& sz
== 16) {
264 } else if (reg_mode
== midgard_reg_mode_16
&& sz
== 8) {
267 } else if (reg_mode
== midgard_reg_mode_32
) {
270 unreachable("Unhandled reg mode");
278 mir_pack_vector_srcs(midgard_instruction
*ins
)
280 bool channeled
= GET_CHANNEL_COUNT(alu_opcode_props
[ins
->alu
.op
].props
);
282 midgard_reg_mode mode
= ins
->alu
.reg_mode
;
283 unsigned base_size
= (8 << mode
);
285 for (unsigned i
= 0; i
< 2; ++i
) {
286 if (ins
->has_inline_constant
&& (i
== 1))
289 if (ins
->src
[i
] == ~0)
292 bool rep_lo
= false, rep_hi
= false;
293 unsigned sz
= nir_alu_type_get_type_size(ins
->src_types
[i
]);
294 bool half
= (sz
== (base_size
>> 1));
296 assert((sz
== base_size
) || half
);
298 unsigned swizzle
= mir_pack_swizzle(ins
->mask
, ins
->swizzle
[i
],
299 ins
->src_types
[i
], ins
->alu
.reg_mode
,
300 channeled
, &rep_lo
, &rep_hi
);
302 midgard_vector_alu_src pack
= {
303 .mod
= mir_pack_mod(ins
, i
, false),
310 unsigned p
= vector_alu_srco_unsigned(pack
);
320 mir_pack_swizzle_ldst(midgard_instruction
*ins
)
322 /* TODO: non-32-bit, non-vec4 */
323 for (unsigned c
= 0; c
< 4; ++c
) {
324 unsigned v
= ins
->swizzle
[0][c
];
329 ins
->load_store
.swizzle
|= v
<< (2 * c
);
336 mir_pack_swizzle_tex(midgard_instruction
*ins
)
338 for (unsigned i
= 0; i
< 2; ++i
) {
341 for (unsigned c
= 0; c
< 4; ++c
) {
342 unsigned v
= ins
->swizzle
[i
][c
];
347 packed
|= v
<< (2 * c
);
351 ins
->texture
.swizzle
= packed
;
353 ins
->texture
.in_reg_swizzle
= packed
;
356 /* TODO: bias component */
359 /* Load store masks are 4-bits. Load/store ops pack for that. vec4 is the
360 * natural mask width; vec8 is constrained to be in pairs, vec2 is duplicated. TODO: 8-bit?
364 mir_pack_ldst_mask(midgard_instruction
*ins
)
366 unsigned sz
= nir_alu_type_get_type_size(ins
->dest_type
);
367 unsigned packed
= ins
->mask
;
370 packed
= ((ins
->mask
& 0x2) ? (0x8 | 0x4) : 0) |
371 ((ins
->mask
& 0x1) ? (0x2 | 0x1) : 0);
372 } else if (sz
== 16) {
375 for (unsigned i
= 0; i
< 4; ++i
) {
376 /* Make sure we're duplicated */
377 bool u
= (ins
->mask
& (1 << (2*i
+ 0))) != 0;
378 bool v
= (ins
->mask
& (1 << (2*i
+ 1))) != 0;
387 ins
->load_store
.mask
= packed
;
391 mir_lower_inverts(midgard_instruction
*ins
)
399 switch (ins
->alu
.op
) {
400 case midgard_alu_op_iand
:
401 /* a & ~b = iandnot(a, b) */
402 /* ~a & ~b = ~(a | b) = inor(a, b) */
404 if (inv
[0] && inv
[1])
405 ins
->alu
.op
= midgard_alu_op_inor
;
407 ins
->alu
.op
= midgard_alu_op_iandnot
;
410 case midgard_alu_op_ior
:
411 /* a | ~b = iornot(a, b) */
412 /* ~a | ~b = ~(a & b) = inand(a, b) */
414 if (inv
[0] && inv
[1])
415 ins
->alu
.op
= midgard_alu_op_inand
;
417 ins
->alu
.op
= midgard_alu_op_iornot
;
421 case midgard_alu_op_ixor
:
422 /* ~a ^ b = a ^ ~b = ~(a ^ b) = inxor(a, b) */
423 /* ~a ^ ~b = a ^ b */
426 ins
->alu
.op
= midgard_alu_op_inxor
;
436 emit_alu_bundle(compiler_context
*ctx
,
437 midgard_bundle
*bundle
,
438 struct util_dynarray
*emission
,
441 /* Emit the control word */
442 util_dynarray_append(emission
, uint32_t, bundle
->control
| lookahead
);
444 /* Next up, emit register words */
445 for (unsigned i
= 0; i
< bundle
->instruction_count
; ++i
) {
446 midgard_instruction
*ins
= bundle
->instructions
[i
];
448 /* Check if this instruction has registers */
449 if (ins
->compact_branch
) continue;
451 /* Otherwise, just emit the registers */
452 uint16_t reg_word
= 0;
453 memcpy(®_word
, &ins
->registers
, sizeof(uint16_t));
454 util_dynarray_append(emission
, uint16_t, reg_word
);
457 /* Now, we emit the body itself */
458 for (unsigned i
= 0; i
< bundle
->instruction_count
; ++i
) {
459 midgard_instruction
*ins
= bundle
->instructions
[i
];
461 /* Where is this body */
465 /* In case we demote to a scalar */
466 midgard_scalar_alu scalarized
;
468 if (!ins
->compact_branch
)
469 mir_lower_inverts(ins
);
471 if (ins
->unit
& UNITS_ANY_VECTOR
) {
472 mir_pack_mask_alu(ins
);
473 mir_pack_vector_srcs(ins
);
474 size
= sizeof(midgard_vector_alu
);
476 } else if (ins
->unit
== ALU_ENAB_BR_COMPACT
) {
477 size
= sizeof(midgard_branch_cond
);
478 source
= &ins
->br_compact
;
479 } else if (ins
->compact_branch
) { /* misnomer */
480 size
= sizeof(midgard_branch_extended
);
481 source
= &ins
->branch_extended
;
483 size
= sizeof(midgard_scalar_alu
);
484 scalarized
= vector_to_scalar_alu(ins
->alu
, ins
);
485 source
= &scalarized
;
488 memcpy(util_dynarray_grow_bytes(emission
, size
, 1), source
, size
);
491 /* Emit padding (all zero) */
492 memset(util_dynarray_grow_bytes(emission
, bundle
->padding
, 1), 0, bundle
->padding
);
494 /* Tack on constants */
496 if (bundle
->has_embedded_constants
)
497 util_dynarray_append(emission
, midgard_constants
, bundle
->constants
);
500 /* Shift applied to the immediate used as an offset. Probably this is papering
501 * over some other semantic distinction else well, but it unifies things in the
502 * compiler so I don't mind. */
505 mir_ldst_imm_shift(midgard_load_store_op op
)
507 if (OP_IS_UBO_READ(op
))
513 static enum mali_sampler_type
514 midgard_sampler_type(nir_alu_type t
) {
515 switch (nir_alu_type_get_base_type(t
))
518 return MALI_SAMPLER_FLOAT
;
520 return MALI_SAMPLER_SIGNED
;
522 return MALI_SAMPLER_UNSIGNED
;
524 unreachable("Unknown sampler type");
528 /* After everything is scheduled, emit whole bundles at a time */
531 emit_binary_bundle(compiler_context
*ctx
,
532 midgard_bundle
*bundle
,
533 struct util_dynarray
*emission
,
536 int lookahead
= next_tag
<< 4;
538 switch (bundle
->tag
) {
547 emit_alu_bundle(ctx
, bundle
, emission
, lookahead
);
550 case TAG_LOAD_STORE_4
: {
551 /* One or two composing instructions */
553 uint64_t current64
, next64
= LDST_NOP
;
557 for (unsigned i
= 0; i
< bundle
->instruction_count
; ++i
) {
558 mir_pack_ldst_mask(bundle
->instructions
[i
]);
560 mir_pack_swizzle_ldst(bundle
->instructions
[i
]);
562 /* Apply a constant offset */
563 unsigned offset
= bundle
->instructions
[i
]->constants
.u32
[0];
566 unsigned shift
= mir_ldst_imm_shift(bundle
->instructions
[i
]->load_store
.op
);
567 unsigned upper_shift
= 10 - shift
;
569 bundle
->instructions
[i
]->load_store
.varying_parameters
|= (offset
& ((1 << upper_shift
) - 1)) << shift
;
570 bundle
->instructions
[i
]->load_store
.address
|= (offset
>> upper_shift
);
574 memcpy(¤t64
, &bundle
->instructions
[0]->load_store
, sizeof(current64
));
576 if (bundle
->instruction_count
== 2)
577 memcpy(&next64
, &bundle
->instructions
[1]->load_store
, sizeof(next64
));
579 midgard_load_store instruction
= {
581 .next_type
= next_tag
,
586 util_dynarray_append(emission
, midgard_load_store
, instruction
);
592 case TAG_TEXTURE_4_VTX
:
593 case TAG_TEXTURE_4_BARRIER
: {
594 /* Texture instructions are easy, since there is no pipelining
595 * nor VLIW to worry about. We may need to set .cont/.last
598 midgard_instruction
*ins
= bundle
->instructions
[0];
600 ins
->texture
.type
= bundle
->tag
;
601 ins
->texture
.next_type
= next_tag
;
603 /* Nothing else to pack for barriers */
604 if (ins
->texture
.op
== TEXTURE_OP_BARRIER
) {
605 ins
->texture
.cont
= ins
->texture
.last
= 1;
606 util_dynarray_append(emission
, midgard_texture_word
, ins
->texture
);
610 signed override
= mir_upper_override(ins
, 32);
612 ins
->texture
.mask
= override
> 0 ?
613 ins
->mask
>> override
:
616 mir_pack_swizzle_tex(ins
);
618 unsigned osz
= nir_alu_type_get_type_size(ins
->dest_type
);
619 unsigned isz
= nir_alu_type_get_type_size(ins
->src_types
[1]);
621 assert(osz
== 32 || osz
== 16);
622 assert(isz
== 32 || isz
== 16);
624 ins
->texture
.out_full
= (osz
== 32);
625 ins
->texture
.out_upper
= override
> 0;
626 ins
->texture
.in_reg_full
= (isz
== 32);
627 ins
->texture
.sampler_type
= midgard_sampler_type(ins
->dest_type
);
629 if (mir_op_computes_derivatives(ctx
->stage
, ins
->texture
.op
)) {
630 ins
->texture
.cont
= !ins
->helper_terminate
;
631 ins
->texture
.last
= ins
->helper_terminate
|| ins
->helper_execute
;
633 ins
->texture
.cont
= ins
->texture
.last
= 1;
636 util_dynarray_append(emission
, midgard_texture_word
, ins
->texture
);
641 unreachable("Unknown midgard instruction type\n");