2 * Copyright (C) 2019 Collabora, Ltd.
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
23 * Authors (Collabora):
24 * Alyssa Rosenzweig <alyssa.rosenzweig@collabora.com>
29 /* Midgard's generic load/store instructions, particularly to implement SSBOs
30 * and globals, have support for address arithmetic natively. In particularly,
31 * they take two indirect arguments A, B and two immediates #s, #c, calculating
34 * A + (zext?(B) << #s) + #c
36 * This allows for fast indexing into arrays. This file tries to pattern match the offset in NIR with this form to reduce pressure on the ALU pipe.
48 /* Matches a constant in either slot and moves it to the bias */
51 mir_match_constant(struct mir_address
*address
)
53 if (address
->A
.def
&& nir_ssa_scalar_is_const(address
->A
)) {
54 address
->bias
+= nir_ssa_scalar_as_uint(address
->A
);
55 address
->A
.def
= NULL
;
58 if (address
->B
.def
&& nir_ssa_scalar_is_const(address
->B
)) {
59 address
->bias
+= nir_ssa_scalar_as_uint(address
->B
);
60 address
->B
.def
= NULL
;
64 /* Matches an iadd when there is a free slot or constant */
67 mir_match_iadd(struct mir_address
*address
, bool first_free
)
69 if (!address
->B
.def
|| !nir_ssa_scalar_is_alu(address
->B
))
72 nir_op op
= nir_ssa_scalar_alu_op(address
->B
);
74 if (op
!= nir_op_iadd
) return;
76 nir_ssa_scalar op1
= nir_ssa_scalar_chase_alu_src(address
->B
, 0);
77 nir_ssa_scalar op2
= nir_ssa_scalar_chase_alu_src(address
->B
, 1);
79 if (nir_ssa_scalar_is_const(op1
)) {
80 address
->bias
+= nir_ssa_scalar_as_uint(op1
);
82 } else if (nir_ssa_scalar_is_const(op2
)) {
83 address
->bias
+= nir_ssa_scalar_as_uint(op2
);
85 } else if (first_free
&& !address
->A
.def
) {
91 /* Matches u2u64 and sets zext */
94 mir_match_u2u64(struct mir_address
*address
)
96 if (!address
->B
.def
|| !nir_ssa_scalar_is_alu(address
->B
))
99 nir_op op
= nir_ssa_scalar_alu_op(address
->B
);
100 if (op
!= nir_op_u2u64
) return;
101 nir_ssa_scalar arg
= nir_ssa_scalar_chase_alu_src(address
->B
, 0);
104 address
->zext
= true;
107 /* Matches ishl to shift */
110 mir_match_ishl(struct mir_address
*address
)
112 if (!address
->B
.def
|| !nir_ssa_scalar_is_alu(address
->B
))
115 nir_op op
= nir_ssa_scalar_alu_op(address
->B
);
116 if (op
!= nir_op_ishl
) return;
117 nir_ssa_scalar op1
= nir_ssa_scalar_chase_alu_src(address
->B
, 0);
118 nir_ssa_scalar op2
= nir_ssa_scalar_chase_alu_src(address
->B
, 1);
120 if (!nir_ssa_scalar_is_const(op2
)) return;
122 unsigned shift
= nir_ssa_scalar_as_uint(op2
);
123 if (shift
> 0x7) return;
126 address
->shift
= shift
;
129 /* Strings through mov which can happen from NIR vectorization */
132 mir_match_mov(struct mir_address
*address
)
134 if (address
->A
.def
&& nir_ssa_scalar_is_alu(address
->A
)) {
135 nir_op op
= nir_ssa_scalar_alu_op(address
->A
);
137 if (op
== nir_op_mov
)
138 address
->A
= nir_ssa_scalar_chase_alu_src(address
->A
, 0);
141 if (address
->B
.def
&& nir_ssa_scalar_is_alu(address
->B
)) {
142 nir_op op
= nir_ssa_scalar_alu_op(address
->B
);
144 if (op
== nir_op_mov
)
145 address
->B
= nir_ssa_scalar_chase_alu_src(address
->B
, 0);
149 /* Tries to pattern match into mir_address */
151 static struct mir_address
152 mir_match_offset(nir_ssa_def
*offset
, bool first_free
)
154 struct mir_address address
= {
155 .B
= { .def
= offset
}
158 mir_match_mov(&address
);
159 mir_match_constant(&address
);
160 mir_match_mov(&address
);
161 mir_match_iadd(&address
, first_free
);
162 mir_match_mov(&address
);
163 mir_match_u2u64(&address
);
164 mir_match_mov(&address
);
165 mir_match_ishl(&address
);
171 mir_set_offset(compiler_context
*ctx
, midgard_instruction
*ins
, nir_src
*offset
, bool is_shared
)
173 for(unsigned i
= 0; i
< 16; ++i
) {
174 ins
->swizzle
[1][i
] = 0;
175 ins
->swizzle
[2][i
] = 0;
178 if (!offset
->is_ssa
) {
179 ins
->load_store
.arg_1
|= is_shared
? 0x6E : 0x7E;
180 ins
->src
[2] = nir_src_index(ctx
, offset
);
184 struct mir_address match
= mir_match_offset(offset
->ssa
, !is_shared
);
187 ins
->src
[1] = nir_ssa_index(match
.A
.def
);
188 ins
->swizzle
[1][0] = match
.A
.comp
;
190 ins
->load_store
.arg_1
|= is_shared
? 0x6E : 0x7E;
193 ins
->src
[2] = nir_ssa_index(match
.B
.def
);
194 ins
->swizzle
[2][0] = match
.B
.comp
;
196 ins
->load_store
.arg_2
= 0x1E;
199 ins
->load_store
.arg_1
|= 0x80;
201 assert(match
.shift
<= 7);
202 ins
->load_store
.arg_2
|= (match
.shift
) << 5;
204 ins
->constants
.u32
[0] = match
.bias
;