2 * Copyright (C) 2020 Collabora, Ltd.
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
26 /* This file contains the final passes of the compiler. Running after
27 * scheduling and RA, the IR is now finalized, so we need to emit it to actual
28 * bits on the wire (as well as fixup branches) */
31 bi_pack_header(bi_clause
*clause
, bi_clause
*next
)
33 struct bifrost_header header
= {
35 .no_end_of_shader
= (next
!= NULL
),
39 memcpy(&u
, &header
, sizeof(header
));
43 /* Represents the assignment of ports for a given bundle */
46 /* Register to assign to each port */
49 /* Read ports can be disabled */
52 /* Should we write FMA? what about ADD? If only a single port is
53 * enabled it is in port 2, else ADD/FMA is 2/3 respectively */
54 bool write_fma
, write_add
;
56 /* Should we read with port 3? */
59 /* Packed uniform/constant */
60 unsigned uniform_constant
;
62 /* Whether writes are actually for the last instruction */
63 bool first_instruction
;
66 /* Assigns a port for reading, before anything is written */
69 bi_assign_port_read(struct bi_registers
*regs
, unsigned src
)
71 /* We only assign for registers */
72 if (!(src
& BIR_INDEX_REGISTER
))
75 unsigned reg
= src
& ~BIR_INDEX_REGISTER
;
77 /* Check if we already assigned the port */
78 for (unsigned i
= 0; i
<= 1; ++i
) {
79 if (regs
->port
[i
] == reg
&& regs
->enabled
[i
])
83 if (regs
->port
[3] == reg
&& regs
->read_port3
)
88 for (unsigned i
= 0; i
<= 1; ++i
) {
89 if (!regs
->enabled
[i
]) {
91 regs
->enabled
[i
] = true;
96 if (!regs
->read_port3
) {
98 regs
->read_port3
= true;
102 static struct bi_registers
103 bi_assign_ports(bi_bundle now
, bi_bundle prev
)
105 struct bi_registers regs
= { 0 };
107 /* First, assign reads */
110 bi_foreach_src(now
.fma
, src
)
111 bi_assign_port_read(®s
, now
.fma
->src
[src
]);
114 bi_foreach_src(now
.add
, src
)
115 bi_assign_port_read(®s
, now
.add
->src
[src
]);
117 /* Next, assign writes */
119 if (prev
.fma
&& prev
.fma
->dest
& BIR_INDEX_REGISTER
) {
120 regs
.port
[2] = prev
.fma
->dest
& ~BIR_INDEX_REGISTER
;
121 regs
.write_fma
= true;
124 if (prev
.add
&& prev
.add
->dest
& BIR_INDEX_REGISTER
) {
125 unsigned r
= prev
.add
->dest
& ~BIR_INDEX_REGISTER
;
127 if (regs
.write_fma
) {
128 /* Scheduler constraint: cannot read 3 and write 2 */
129 assert(!regs
.read_port3
);
135 regs
.write_add
= true;
138 /* Finally, ensure port 1 > port 0 for the 63-x trick to function */
140 if (regs
.enabled
[0] && regs
.enabled
[1] && regs
.port
[1] < regs
.port
[0]) {
141 unsigned temp
= regs
.port
[0];
142 regs
.port
[0] = regs
.port
[1];
149 /* Determines the register control field, ignoring the first? flag */
151 static enum bifrost_reg_control
152 bi_pack_register_ctrl_lo(struct bi_registers r
)
156 assert(!r
.read_port3
);
157 return BIFROST_WRITE_ADD_P2_FMA_P3
;
160 return BIFROST_WRITE_FMA_P2_READ_P3
;
162 return BIFROST_WRITE_FMA_P2
;
164 } else if (r
.write_add
) {
166 return BIFROST_WRITE_ADD_P2_READ_P3
;
168 return BIFROST_WRITE_ADD_P2
;
169 } else if (r
.read_port3
)
170 return BIFROST_READ_P3
;
172 return BIFROST_REG_NONE
;
175 /* Ditto but account for the first? flag this time */
177 static enum bifrost_reg_control
178 bi_pack_register_ctrl(struct bi_registers r
)
180 enum bifrost_reg_control ctrl
= bi_pack_register_ctrl_lo(r
);
182 if (r
.first_instruction
) {
183 if (ctrl
== BIFROST_REG_NONE
)
184 ctrl
= BIFROST_FIRST_NONE
;
186 ctrl
|= BIFROST_FIRST_NONE
;
193 bi_pack_registers(struct bi_registers regs
)
195 enum bifrost_reg_control ctrl
= bi_pack_register_ctrl(regs
);
196 struct bifrost_regs s
;
199 if (regs
.enabled
[1]) {
200 /* Gotta save that bit!~ Required by the 63-x trick */
201 assert(regs
.port
[1] > regs
.port
[0]);
202 assert(regs
.enabled
[0]);
204 /* Do the 63-x trick, see docs/disasm */
205 if (regs
.port
[0] > 31) {
206 regs
.port
[0] = 63 - regs
.port
[0];
207 regs
.port
[1] = 63 - regs
.port
[1];
210 assert(regs
.port
[0] <= 31);
211 assert(regs
.port
[1] <= 63);
214 s
.reg1
= regs
.port
[1];
215 s
.reg0
= regs
.port
[0];
217 /* Port 1 disabled, so set to zero and use port 1 for ctrl */
220 if (regs
.enabled
[0]) {
221 /* Bit 0 upper bit of port 0 */
222 s
.reg1
|= (regs
.port
[0] >> 5);
224 /* Rest of port 0 in usual spot */
225 s
.reg0
= (regs
.port
[0] & 0b11111);
227 /* Bit 1 set if port 0 also disabled */
232 s
.reg3
= regs
.port
[3];
233 s
.reg2
= regs
.port
[2];
234 s
.uniform_const
= regs
.uniform_constant
;
236 memcpy(&packed
, &s
, sizeof(s
));
241 bi_pack_fma(bi_clause
*clause
, bi_bundle bundle
)
244 return BIFROST_FMA_NOP
;
248 bi_pack_add(bi_clause
*clause
, bi_bundle bundle
)
251 return BIFROST_ADD_NOP
;
254 struct bi_packed_bundle
{
259 static struct bi_packed_bundle
260 bi_pack_bundle(bi_clause
*clause
, bi_bundle bundle
, bi_bundle prev
)
262 struct bi_registers regs
= bi_assign_ports(bundle
, prev
);
264 uint64_t reg
= bi_pack_registers(regs
);
265 uint64_t fma
= bi_pack_fma(clause
, bundle
);
266 uint64_t add
= bi_pack_add(clause
, bundle
);
268 struct bi_packed_bundle packed
= {
269 .lo
= reg
| (fma
<< 35) | ((add
& 0b111111) << 58),
277 bi_pack_clause(bi_context
*ctx
, bi_clause
*clause
, bi_clause
*next
,
278 struct util_dynarray
*emission
)
280 struct bi_packed_bundle ins_1
= bi_pack_bundle(clause
, clause
->bundles
[0], clause
->bundles
[0]);
281 assert(clause
->bundle_count
== 1);
283 struct bifrost_fmt1 quad_1
= {
284 .tag
= BIFROST_FMT1_FINAL
,
285 .header
= bi_pack_header(clause
, next
),
287 .ins_2
= ins_1
.hi
& ((1 << 11) - 1),
288 .ins_0
= (ins_1
.hi
>> 11) & 0b111,
291 util_dynarray_append(emission
, struct bifrost_fmt1
, quad_1
);
295 bi_next_clause(bi_context
*ctx
, pan_block
*block
, bi_clause
*clause
)
297 /* Try the next clause in this block */
298 if (clause
->link
.next
!= &((bi_block
*) block
)->clauses
)
299 return list_first_entry(&(clause
->link
), bi_clause
, link
);
301 /* Try the next block, or the one after that if it's empty, etc .*/
302 pan_block
*next_block
= pan_next_block(block
);
304 bi_foreach_block_from(ctx
, next_block
, block
) {
305 bi_block
*blk
= (bi_block
*) block
;
307 if (!list_is_empty(&blk
->clauses
))
308 return list_first_entry(&(blk
->clauses
), bi_clause
, link
);
315 bi_pack(bi_context
*ctx
, struct util_dynarray
*emission
)
317 util_dynarray_init(emission
, NULL
);
319 bi_foreach_block(ctx
, _block
) {
320 bi_block
*block
= (bi_block
*) _block
;
322 bi_foreach_clause_in_block(block
, clause
) {
323 bi_clause
*next
= bi_next_clause(ctx
, _block
, clause
);
324 bi_pack_clause(ctx
, clause
, next
, emission
);