1 /* -*- mode: C; c-file-style: "k&r"; tab-width 4; indent-tabs-mode: t; -*- */
4 * Copyright (C) 2014 Rob Clark <robclark@freedesktop.org>
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the "Software"),
8 * to deal in the Software without restriction, including without limitation
9 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
10 * and/or sell copies of the Software, and to permit persons to whom the
11 * Software is furnished to do so, subject to the following conditions:
13 * The above copyright notice and this permission notice (including the next
14 * paragraph) shall be included in all copies or substantial portions of the
17 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
18 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
19 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
20 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
21 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
22 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
26 * Rob Clark <robclark@freedesktop.org>
29 #include "pipe/p_shader_tokens.h"
30 #include "util/u_math.h"
35 * Register Assignment:
37 * NOTE: currently only works on a single basic block.. need to think
38 * about how multiple basic blocks are going to get scheduled. But
39 * I think I want to re-arrange how blocks work, ie. get rid of the
40 * block nesting thing..
42 * NOTE: we could do register coalescing (eliminate moves) as part of
43 * the RA step.. OTOH I think we need to do scheduling before register
44 * assignment. And if we remove a mov that effects scheduling (unless
45 * we leave a placeholder nop, which seems lame), so I'm not really
46 * sure how practical this is to do both in a single stage. But OTOH
47 * I'm not really sure a sane way for the CP stage to realize when it
48 * cannot remove a mov due to multi-register constraints..
53 struct ir3_block
*block
;
62 # include "freedreno_util.h"
63 # define ra_debug (fd_mesa_debug & FD_DBG_OPTMSGS)
68 #define ra_dump_list(msg, n) do { \
70 debug_printf("-- " msg); \
71 ir3_dump_instr_list(n); \
75 #define ra_dump_instr(msg, n) do { \
77 debug_printf(">> " msg); \
78 ir3_dump_instr_single(n); \
82 #define ra_assert(ctx, x) do { \
85 debug_printf("RA: failed assert: %s\n", #x); \
86 (ctx)->error = true; \
91 /* sorta ugly way to retrofit half-precision support.. rather than
92 * passing extra param around, just OR in a high bit. All the low
93 * value arithmetic (ie. +/- offset within a contiguous vec4, etc)
94 * will continue to work as long as you don't underflow (and that
95 * would go badly anyways).
97 #define REG_HALF 0x8000
99 #define REG(n, wm, f) (struct ir3_register){ \
102 .wrmask = TGSI_WRITEMASK_ ## wm, \
105 /* check that the register exists, is a GPR and is not special (a0/p0) */
106 static struct ir3_register
* reg_check(struct ir3_instruction
*instr
, unsigned n
)
108 if ((n
< instr
->regs_count
) && reg_gpr(instr
->regs
[n
]) &&
109 !(instr
->regs
[n
]->flags
& IR3_REG_SSA
))
110 return instr
->regs
[n
];
114 /* figure out if an unassigned src register points back to the instr we
117 static bool instr_used_by(struct ir3_instruction
*instr
,
118 struct ir3_register
*src
)
120 struct ir3_instruction
*src_instr
= ssa(src
);
122 if (instr
== src_instr
)
124 if (src_instr
&& is_meta(src_instr
))
125 for (i
= 1; i
< src_instr
->regs_count
; i
++)
126 if (instr_used_by(instr
, src_instr
->regs
[i
]))
132 static bool instr_is_output(struct ir3_instruction
*instr
)
134 struct ir3_block
*block
= instr
->block
;
137 for (i
= 0; i
< block
->noutputs
; i
++)
138 if (instr
== block
->outputs
[i
])
144 /* live means read before written */
145 static void compute_liveregs(struct ir3_ra_ctx
*ctx
,
146 struct ir3_instruction
*instr
, regmask_t
*liveregs
)
148 struct ir3_block
*block
= instr
->block
;
149 struct ir3_instruction
*n
;
153 regmask_init(&written
);
155 for (n
= instr
->next
; n
; n
= n
->next
) {
156 struct ir3_register
*r
;
161 /* check first src's read: */
162 for (i
= 1; i
< n
->regs_count
; i
++) {
165 regmask_set_if_not(liveregs
, r
, &written
);
167 /* if any src points back to the instruction(s) in
168 * the block of neighbors that we are assigning then
169 * mark any written (clobbered) registers as live:
171 if (instr_used_by(instr
, n
->regs
[i
]))
172 regmask_or(liveregs
, liveregs
, &written
);
175 /* meta-instructions don't actually get scheduled,
176 * so don't let it's write confuse us.. what we
177 * really care about is when the src to the meta
183 /* then dst written (if assigned already): */
186 /* if an instruction *is* an output, then it is live */
187 if (!instr_is_output(n
))
188 regmask_set(&written
, r
);
193 /* be sure to account for output registers too: */
194 for (i
= 0; i
< block
->noutputs
; i
++) {
195 struct ir3_register
*r
;
196 if (!block
->outputs
[i
])
198 r
= reg_check(block
->outputs
[i
], 0);
200 regmask_set_if_not(liveregs
, r
, &written
);
203 /* if instruction is output, we need a reg that isn't written
204 * before the end.. equiv to the instr_used_by() check above
206 * TODO maybe should follow fanin/fanout?
208 if (instr_is_output(instr
))
209 regmask_or(liveregs
, liveregs
, &written
);
212 static int find_available(regmask_t
*liveregs
, int size
, bool half
)
215 unsigned f
= half
? IR3_REG_HALF
: 0;
216 for (i
= 0; i
< MAX_REG
- size
; i
++) {
217 if (!regmask_get(liveregs
, ®(i
, X
, f
))) {
218 unsigned start
= i
++;
219 for (; (i
< MAX_REG
) && ((i
- start
) < size
); i
++)
220 if (regmask_get(liveregs
, ®(i
, X
, f
)))
222 if ((i
- start
) >= size
)
230 static int alloc_block(struct ir3_ra_ctx
*ctx
,
231 struct ir3_instruction
*instr
, int size
)
233 struct ir3_register
*dst
= instr
->regs
[0];
234 struct ir3_instruction
*n
;
238 /* should only ever be called w/ head of neighbor list: */
239 debug_assert(!instr
->cp
.left
);
241 regmask_init(&liveregs
);
243 for (n
= instr
; n
; n
= n
->cp
.right
)
244 compute_liveregs(ctx
, n
, &liveregs
);
246 /* because we do assignment on fanout nodes for wrmask!=0x1, we
247 * need to handle this special case, where the fanout nodes all
248 * appear after one or more of the consumers of the src node:
250 * 0098:009: sam _, r2.x
251 * 0028:010: mul.f r3.z, r4.x, c13.x
252 * ; we start assigning here for '0098:009: sam'.. but
253 * ; would miss the usage at '0028:010: mul.f'
254 * 0101:009: _meta:fo _, _[0098:009: sam], off=2
256 if (is_meta(instr
) && (instr
->opc
== OPC_META_FO
))
257 compute_liveregs(ctx
, instr
->regs
[1]->instr
, &liveregs
);
259 name
= find_available(&liveregs
, size
,
260 !!(dst
->flags
& IR3_REG_HALF
));
262 if (dst
->flags
& IR3_REG_HALF
)
268 static type_t
half_type(type_t type
)
271 case TYPE_F32
: return TYPE_F16
;
272 case TYPE_U32
: return TYPE_U16
;
273 case TYPE_S32
: return TYPE_S16
;
274 /* instructions may already be fixed up: */
285 /* some instructions need fix-up if dst register is half precision: */
286 static void fixup_half_instr_dst(struct ir3_instruction
*instr
)
288 switch (instr
->category
) {
289 case 1: /* move instructions */
290 instr
->cat1
.dst_type
= half_type(instr
->cat1
.dst_type
);
293 switch (instr
->opc
) {
295 instr
->opc
= OPC_MAD_F16
;
298 instr
->opc
= OPC_SEL_B16
;
301 instr
->opc
= OPC_SEL_S16
;
304 instr
->opc
= OPC_SEL_F16
;
307 instr
->opc
= OPC_SAD_S16
;
309 /* instructions may already be fixed up: */
322 instr
->cat5
.type
= half_type(instr
->cat5
.type
);
326 /* some instructions need fix-up if src register is half precision: */
327 static void fixup_half_instr_src(struct ir3_instruction
*instr
)
329 switch (instr
->category
) {
330 case 1: /* move instructions */
331 instr
->cat1
.src_type
= half_type(instr
->cat1
.src_type
);
336 static void reg_assign(struct ir3_instruction
*instr
,
337 unsigned r
, unsigned name
)
339 struct ir3_register
*reg
= instr
->regs
[r
];
341 reg
->flags
&= ~IR3_REG_SSA
;
342 reg
->num
= name
& ~REG_HALF
;
344 if (name
& REG_HALF
) {
345 reg
->flags
|= IR3_REG_HALF
;
346 /* if dst reg being assigned, patch up the instr: */
347 if (reg
== instr
->regs
[0])
348 fixup_half_instr_dst(instr
);
350 fixup_half_instr_src(instr
);
354 static void instr_assign(struct ir3_ra_ctx
*ctx
,
355 struct ir3_instruction
*instr
, unsigned name
);
357 static void instr_assign_src(struct ir3_ra_ctx
*ctx
,
358 struct ir3_instruction
*instr
, unsigned r
, unsigned name
)
360 reg_assign(instr
, r
, name
);
362 if (is_meta(instr
)) {
363 switch (instr
->opc
) {
365 /* shader-input does not have a src, only block input: */
366 debug_assert(instr
->regs_count
== 2);
367 instr_assign(ctx
, instr
, name
);
370 instr_assign(ctx
, instr
, name
+ instr
->fo
.off
);
373 instr_assign(ctx
, instr
, name
- (r
- 1));
376 /* first arg of meta:deref is the addr reg (do not
377 * propagate), 2nd is actual src (fanin) which does
381 instr_assign(ctx
, instr
, name
+ instr
->deref
.off
);
389 static void instr_assign(struct ir3_ra_ctx
*ctx
,
390 struct ir3_instruction
*instr
, unsigned name
)
392 struct ir3_instruction
*n
;
393 struct ir3_register
*reg
= instr
->regs
[0];
395 /* check if already assigned: */
396 if (!(reg
->flags
& IR3_REG_SSA
)) {
397 /* ... and if so, sanity check: */
398 ra_assert(ctx
, reg
->num
== (name
& ~REG_HALF
));
402 /* rename this instructions dst register: */
403 reg_assign(instr
, 0, name
);
405 /* and rename any subsequent use of result of this instr: */
406 for (n
= instr
->next
; n
&& !ctx
->error
; n
= n
->next
) {
409 for (i
= 1; i
< n
->regs_count
; i
++) {
411 if ((reg
->flags
& IR3_REG_SSA
) && (reg
->instr
== instr
))
412 instr_assign_src(ctx
, n
, i
, name
);
416 /* To simplify the neighbor logic, and to "avoid" dealing with
417 * instructions which write more than one output, we actually
418 * do register assignment for instructions that produce multiple
419 * outputs on the fanout nodes and propagate up the assignment
420 * to the actual instruction:
422 if (is_meta(instr
) && (instr
->opc
== OPC_META_FO
)) {
423 struct ir3_instruction
*src
= ssa(instr
->regs
[1]);
424 debug_assert(name
>= instr
->fo
.off
);
426 instr_assign(ctx
, src
, name
- instr
->fo
.off
);
430 /* check neighbor list to see if it is already partially (or completely)
431 * assigned, in which case register block is already allocated and we
432 * just need to complete the assignment:
434 static int check_partial_assignment(struct ir3_ra_ctx
*ctx
,
435 struct ir3_instruction
*instr
)
437 struct ir3_instruction
*n
;
440 debug_assert(!instr
->cp
.left
);
442 for (n
= instr
; n
; n
= n
->cp
.right
) {
443 struct ir3_register
*dst
= n
->regs
[0];
444 if (!(dst
->flags
& IR3_REG_SSA
)) {
445 int name
= dst
->num
- off
;
446 debug_assert(name
>= 0);
455 /* allocate register name(s) for a list of neighboring instructions;
456 * instr should point to leftmost neighbor (head of list)
458 static void instr_alloc_and_assign(struct ir3_ra_ctx
*ctx
,
459 struct ir3_instruction
*instr
)
461 struct ir3_instruction
*n
;
462 struct ir3_register
*dst
;
465 debug_assert(!instr
->cp
.left
);
467 if (instr
->regs_count
== 0)
470 dst
= instr
->regs
[0];
472 /* for instructions w/ fanouts, do the actual register assignment
473 * on the group of fanout neighbor nodes and propagate the reg
474 * name back up to the texture instruction.
476 if (dst
->wrmask
!= 0x1)
479 name
= check_partial_assignment(ctx
, instr
);
481 /* allocate register(s): */
483 /* already partially assigned, just finish the job */
484 } else if (is_addr(instr
)) {
485 debug_assert(!instr
->cp
.right
);
486 name
= instr
->regs
[2]->num
+ instr
->deref
.off
;
487 } else if (reg_gpr(dst
)) {
489 /* number of consecutive registers to assign: */
490 size
= ir3_neighbor_count(instr
);
491 if (dst
->wrmask
!= 0x1)
492 size
= MAX2(size
, ffs(~dst
->wrmask
) - 1);
493 name
= alloc_block(ctx
, instr
, size
);
494 } else if (dst
->flags
& IR3_REG_ADDR
) {
495 debug_assert(!instr
->cp
.right
);
496 dst
->flags
&= ~IR3_REG_ADDR
;
497 name
= regid(REG_A0
, 0) | REG_HALF
;
499 debug_assert(!instr
->cp
.right
);
500 /* predicate register (p0).. etc */
501 name
= regid(REG_P0
, 0);
502 debug_assert(dst
->num
== name
);
505 ra_assert(ctx
, name
>= 0);
507 for (n
= instr
; n
&& !ctx
->error
; n
= n
->cp
.right
) {
508 instr_assign(ctx
, n
, name
);
513 static int block_ra(struct ir3_ra_ctx
*ctx
, struct ir3_block
*block
)
515 struct ir3_instruction
*n
;
517 /* frag shader inputs get pre-assigned, since we have some
518 * constraints/unknowns about setup for some of these regs:
520 if ((ctx
->type
== SHADER_FRAGMENT
) && !block
->parent
) {
522 if (ctx
->frag_face
&& (i
< block
->ninputs
) && block
->inputs
[i
]) {
523 /* if we have frag_face, it gets hr0.x */
524 instr_assign(ctx
, block
->inputs
[i
], REG_HALF
| 0);
527 for (j
= 0; i
< block
->ninputs
; i
++, j
++)
528 if (block
->inputs
[i
])
529 instr_assign(ctx
, block
->inputs
[i
], j
);
532 ra_dump_list("-------\n", block
->head
);
534 for (n
= block
->head
; n
&& !ctx
->error
; n
= n
->next
) {
535 ra_dump_instr("ASSIGN: ", n
);
536 instr_alloc_and_assign(ctx
, ir3_neighbor_first(n
));
537 ra_dump_list("-------\n", block
->head
);
540 return ctx
->error
? -1 : 0;
543 int ir3_block_ra(struct ir3_block
*block
, enum shader_t type
,
544 bool frag_coord
, bool frag_face
)
546 struct ir3_instruction
*n
;
547 struct ir3_ra_ctx ctx
= {
550 .frag_coord
= frag_coord
,
551 .frag_face
= frag_face
,
555 /* mark dst registers w/ SSA flag so we can see which
556 * have been assigned so far:
557 * NOTE: we really should set SSA flag consistently on
558 * every dst register in the frontend.
560 for (n
= block
->head
; n
; n
= n
->next
)
561 if (n
->regs_count
> 0)
562 n
->regs
[0]->flags
|= IR3_REG_SSA
;
564 ir3_clear_mark(block
->shader
);
565 ret
= block_ra(&ctx
, block
);