2 * Copyright (C) 2019 Collabora, Ltd.
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
23 * Authors (Collabora):
24 * Alyssa Rosenzweig <alyssa.rosenzweig@collabora.com>
29 /* Midgard's generic load/store instructions, particularly to implement SSBOs
30 * and globals, have support for address arithmetic natively. In particularly,
31 * they take two indirect arguments A, B and two immediates #s, #c, calculating
34 * A + (zext?(B) << #s) + #c
36 * This allows for fast indexing into arrays. This file tries to pattern match the offset in NIR with this form to reduce pressure on the ALU pipe.
49 mir_args_ssa(nir_ssa_scalar s
, unsigned count
)
51 nir_alu_instr
*alu
= nir_instr_as_alu(s
.def
->parent_instr
);
52 assert(count
<= nir_op_infos
[alu
->op
].num_inputs
);
54 for (unsigned i
= 0; i
< count
; ++i
) {
55 if (!alu
->src
[i
].src
.is_ssa
)
62 /* Matches a constant in either slot and moves it to the bias */
65 mir_match_constant(struct mir_address
*address
)
67 if (address
->A
.def
&& nir_ssa_scalar_is_const(address
->A
)) {
68 address
->bias
+= nir_ssa_scalar_as_uint(address
->A
);
69 address
->A
.def
= NULL
;
72 if (address
->B
.def
&& nir_ssa_scalar_is_const(address
->B
)) {
73 address
->bias
+= nir_ssa_scalar_as_uint(address
->B
);
74 address
->B
.def
= NULL
;
78 /* Matches an iadd when there is a free slot or constant */
81 mir_match_iadd(struct mir_address
*address
, bool first_free
)
83 if (!address
->B
.def
|| !nir_ssa_scalar_is_alu(address
->B
))
86 if (!mir_args_ssa(address
->B
, 2))
89 nir_op op
= nir_ssa_scalar_alu_op(address
->B
);
91 if (op
!= nir_op_iadd
) return;
93 nir_ssa_scalar op1
= nir_ssa_scalar_chase_alu_src(address
->B
, 0);
94 nir_ssa_scalar op2
= nir_ssa_scalar_chase_alu_src(address
->B
, 1);
96 if (nir_ssa_scalar_is_const(op1
)) {
97 address
->bias
+= nir_ssa_scalar_as_uint(op1
);
99 } else if (nir_ssa_scalar_is_const(op2
)) {
100 address
->bias
+= nir_ssa_scalar_as_uint(op2
);
102 } else if (first_free
&& !address
->A
.def
) {
108 /* Matches u2u64 and sets zext */
111 mir_match_u2u64(struct mir_address
*address
)
113 if (!address
->B
.def
|| !nir_ssa_scalar_is_alu(address
->B
))
116 if (!mir_args_ssa(address
->B
, 1))
119 nir_op op
= nir_ssa_scalar_alu_op(address
->B
);
120 if (op
!= nir_op_u2u64
) return;
121 nir_ssa_scalar arg
= nir_ssa_scalar_chase_alu_src(address
->B
, 0);
124 address
->zext
= true;
127 /* Matches ishl to shift */
130 mir_match_ishl(struct mir_address
*address
)
132 if (!address
->B
.def
|| !nir_ssa_scalar_is_alu(address
->B
))
135 if (!mir_args_ssa(address
->B
, 2))
138 nir_op op
= nir_ssa_scalar_alu_op(address
->B
);
139 if (op
!= nir_op_ishl
) return;
140 nir_ssa_scalar op1
= nir_ssa_scalar_chase_alu_src(address
->B
, 0);
141 nir_ssa_scalar op2
= nir_ssa_scalar_chase_alu_src(address
->B
, 1);
143 if (!nir_ssa_scalar_is_const(op2
)) return;
145 unsigned shift
= nir_ssa_scalar_as_uint(op2
);
146 if (shift
> 0x7) return;
149 address
->shift
= shift
;
152 /* Strings through mov which can happen from NIR vectorization */
155 mir_match_mov(struct mir_address
*address
)
157 if (address
->A
.def
&& nir_ssa_scalar_is_alu(address
->A
)) {
158 nir_op op
= nir_ssa_scalar_alu_op(address
->A
);
160 if (op
== nir_op_mov
&& mir_args_ssa(address
->A
, 1))
161 address
->A
= nir_ssa_scalar_chase_alu_src(address
->A
, 0);
164 if (address
->B
.def
&& nir_ssa_scalar_is_alu(address
->B
)) {
165 nir_op op
= nir_ssa_scalar_alu_op(address
->B
);
167 if (op
== nir_op_mov
&& mir_args_ssa(address
->B
, 1))
168 address
->B
= nir_ssa_scalar_chase_alu_src(address
->B
, 0);
172 /* Tries to pattern match into mir_address */
174 static struct mir_address
175 mir_match_offset(nir_ssa_def
*offset
, bool first_free
)
177 struct mir_address address
= {
178 .B
= { .def
= offset
}
181 mir_match_mov(&address
);
182 mir_match_constant(&address
);
183 mir_match_mov(&address
);
184 mir_match_iadd(&address
, first_free
);
185 mir_match_mov(&address
);
186 mir_match_u2u64(&address
);
187 mir_match_mov(&address
);
188 mir_match_ishl(&address
);
194 mir_set_offset(compiler_context
*ctx
, midgard_instruction
*ins
, nir_src
*offset
, bool is_shared
)
196 for(unsigned i
= 0; i
< 16; ++i
) {
197 ins
->swizzle
[1][i
] = 0;
198 ins
->swizzle
[2][i
] = 0;
201 bool force_zext
= (nir_src_bit_size(*offset
) < 64);
203 if (!offset
->is_ssa
) {
204 ins
->load_store
.arg_1
|= is_shared
? 0x6E : 0x7E;
205 ins
->src
[2] = nir_src_index(ctx
, offset
);
206 ins
->src_types
[2] = nir_type_uint
| nir_src_bit_size(*offset
);
209 ins
->load_store
.arg_1
|= 0x80;
214 struct mir_address match
= mir_match_offset(offset
->ssa
, !is_shared
);
217 ins
->src
[1] = nir_ssa_index(match
.A
.def
);
218 ins
->swizzle
[1][0] = match
.A
.comp
;
219 ins
->src_types
[1] = nir_type_uint
| match
.A
.def
->bit_size
;
221 ins
->load_store
.arg_1
|= is_shared
? 0x6E : 0x7E;
224 ins
->src
[2] = nir_ssa_index(match
.B
.def
);
225 ins
->swizzle
[2][0] = match
.B
.comp
;
226 ins
->src_types
[2] = nir_type_uint
| match
.B
.def
->bit_size
;
228 ins
->load_store
.arg_2
= 0x1E;
230 if (match
.zext
|| force_zext
)
231 ins
->load_store
.arg_1
|= 0x80;
233 assert(match
.shift
<= 7);
234 ins
->load_store
.arg_2
|= (match
.shift
) << 5;
236 ins
->constants
.u32
[0] = match
.bias
;