2 * Copyright © 2012 Intel Corporation
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
24 * Eric Anholt <eric@anholt.net>
29 #include "brw_shader.h"
33 * Walks the shader instructions generated and creates a set of basic
34 * blocks with successor/predecessor edges connecting them.
40 pop_stack(exec_list
*list
)
42 bblock_link
*link
= (bblock_link
*)list
->get_tail();
43 bblock_t
*block
= link
->block
;
50 link(void *mem_ctx
, bblock_t
*block
, enum bblock_link_kind kind
)
52 bblock_link
*l
= new(mem_ctx
) bblock_link(block
, kind
);
57 push_stack(exec_list
*list
, void *mem_ctx
, bblock_t
*block
)
59 /* The kind of the link is immaterial, but we need to provide one since
60 * this is (ab)using the edge data structure in order to implement a stack.
62 list
->push_tail(link(mem_ctx
, block
, bblock_link_logical
));
65 bblock_t::bblock_t(cfg_t
*cfg
) :
66 cfg(cfg
), start_ip(0), end_ip(0), num(0)
68 instructions
.make_empty();
70 children
.make_empty();
74 bblock_t::add_successor(void *mem_ctx
, bblock_t
*successor
,
75 enum bblock_link_kind kind
)
77 successor
->parents
.push_tail(::link(mem_ctx
, this, kind
));
78 children
.push_tail(::link(mem_ctx
, successor
, kind
));
82 bblock_t::is_predecessor_of(const bblock_t
*block
,
83 enum bblock_link_kind kind
) const
85 foreach_list_typed_safe (bblock_link
, parent
, link
, &block
->parents
) {
86 if (parent
->block
== this && parent
->kind
<= kind
) {
95 bblock_t::is_successor_of(const bblock_t
*block
,
96 enum bblock_link_kind kind
) const
98 foreach_list_typed_safe (bblock_link
, child
, link
, &block
->children
) {
99 if (child
->block
== this && child
->kind
<= kind
) {
108 ends_block(const backend_instruction
*inst
)
110 enum opcode op
= inst
->opcode
;
112 return op
== BRW_OPCODE_IF
||
113 op
== BRW_OPCODE_ELSE
||
114 op
== BRW_OPCODE_CONTINUE
||
115 op
== BRW_OPCODE_BREAK
||
116 op
== BRW_OPCODE_DO
||
117 op
== BRW_OPCODE_WHILE
;
121 starts_block(const backend_instruction
*inst
)
123 enum opcode op
= inst
->opcode
;
125 return op
== BRW_OPCODE_DO
||
126 op
== BRW_OPCODE_ENDIF
;
130 bblock_t::can_combine_with(const bblock_t
*that
) const
132 if ((const bblock_t
*)this->link
.next
!= that
)
135 if (ends_block(this->end()) ||
136 starts_block(that
->start()))
143 bblock_t::combine_with(bblock_t
*that
)
145 assert(this->can_combine_with(that
));
146 foreach_list_typed (bblock_link
, link
, link
, &that
->parents
) {
147 assert(link
->block
== this);
150 this->end_ip
= that
->end_ip
;
151 this->instructions
.append_list(&that
->instructions
);
153 this->cfg
->remove_block(that
);
157 bblock_t::dump() const
159 const backend_shader
*s
= this->cfg
->s
;
161 int ip
= this->start_ip
;
162 foreach_inst_in_block(backend_instruction
, inst
, this) {
163 fprintf(stderr
, "%5d: ", ip
);
164 s
->dump_instruction(inst
);
169 cfg_t::cfg_t(const backend_shader
*s
, exec_list
*instructions
) :
172 mem_ctx
= ralloc_context(NULL
);
173 block_list
.make_empty();
177 bblock_t
*cur
= NULL
;
180 bblock_t
*entry
= new_block();
181 bblock_t
*cur_if
= NULL
; /**< BB ending with IF. */
182 bblock_t
*cur_else
= NULL
; /**< BB ending with ELSE. */
183 bblock_t
*cur_endif
= NULL
; /**< BB starting with ENDIF. */
184 bblock_t
*cur_do
= NULL
; /**< BB starting with DO. */
185 bblock_t
*cur_while
= NULL
; /**< BB immediately following WHILE. */
186 exec_list if_stack
, else_stack
, do_stack
, while_stack
;
189 set_next_block(&cur
, entry
, ip
);
191 foreach_in_list_safe(backend_instruction
, inst
, instructions
) {
192 /* set_next_block wants the post-incremented ip */
195 inst
->exec_node::remove();
197 switch (inst
->opcode
) {
199 cur
->instructions
.push_tail(inst
);
201 /* Push our information onto a stack so we can recover from
204 push_stack(&if_stack
, mem_ctx
, cur_if
);
205 push_stack(&else_stack
, mem_ctx
, cur_else
);
211 /* Set up our immediately following block, full of "then"
215 cur_if
->add_successor(mem_ctx
, next
, bblock_link_logical
);
217 set_next_block(&cur
, next
, ip
);
220 case BRW_OPCODE_ELSE
:
221 cur
->instructions
.push_tail(inst
);
226 assert(cur_if
!= NULL
);
227 cur_if
->add_successor(mem_ctx
, next
, bblock_link_logical
);
228 cur_else
->add_successor(mem_ctx
, next
, bblock_link_physical
);
230 set_next_block(&cur
, next
, ip
);
233 case BRW_OPCODE_ENDIF
: {
234 if (cur
->instructions
.is_empty()) {
235 /* New block was just created; use it. */
238 cur_endif
= new_block();
240 cur
->add_successor(mem_ctx
, cur_endif
, bblock_link_logical
);
242 set_next_block(&cur
, cur_endif
, ip
- 1);
245 cur
->instructions
.push_tail(inst
);
248 cur_else
->add_successor(mem_ctx
, cur_endif
, bblock_link_logical
);
250 assert(cur_if
!= NULL
);
251 cur_if
->add_successor(mem_ctx
, cur_endif
, bblock_link_logical
);
254 assert(cur_if
->end()->opcode
== BRW_OPCODE_IF
);
255 assert(!cur_else
|| cur_else
->end()->opcode
== BRW_OPCODE_ELSE
);
257 /* Pop the stack so we're in the previous if/else/endif */
258 cur_if
= pop_stack(&if_stack
);
259 cur_else
= pop_stack(&else_stack
);
263 /* Push our information onto a stack so we can recover from
266 push_stack(&do_stack
, mem_ctx
, cur_do
);
267 push_stack(&while_stack
, mem_ctx
, cur_while
);
269 /* Set up the block just after the while. Don't know when exactly
270 * it will start, yet.
272 cur_while
= new_block();
274 if (cur
->instructions
.is_empty()) {
275 /* New block was just created; use it. */
278 cur_do
= new_block();
280 cur
->add_successor(mem_ctx
, cur_do
, bblock_link_logical
);
282 set_next_block(&cur
, cur_do
, ip
- 1);
285 cur
->instructions
.push_tail(inst
);
287 /* Represent divergent execution of the loop as a pair of alternative
288 * edges coming out of the DO instruction: For any physical iteration
289 * of the loop a given logical thread can either start off enabled
290 * (which is represented as the "next" successor), or disabled (if it
291 * has reached a non-uniform exit of the loop during a previous
292 * iteration, which is represented as the "cur_while" successor).
294 * The disabled edge will be taken by the logical thread anytime we
295 * arrive at the DO instruction through a back-edge coming from a
296 * conditional exit of the loop where divergent control flow started.
298 * This guarantees that there is a control-flow path from any
299 * divergence point of the loop into the convergence point
300 * (immediately past the WHILE instruction) such that it overlaps the
301 * whole IP region of divergent control flow (potentially the whole
302 * loop) *and* doesn't imply the execution of any instructions part
303 * of the loop (since the corresponding execution mask bit will be
304 * disabled for a diverging thread).
306 * This way we make sure that any variables that are live throughout
307 * the region of divergence for an inactive logical thread are also
308 * considered to interfere with any other variables assigned by
309 * active logical threads within the same physical region of the
310 * program, since otherwise we would risk cross-channel data
314 cur
->add_successor(mem_ctx
, next
, bblock_link_logical
);
315 cur
->add_successor(mem_ctx
, cur_while
, bblock_link_physical
);
316 set_next_block(&cur
, next
, ip
);
319 case BRW_OPCODE_CONTINUE
:
320 cur
->instructions
.push_tail(inst
);
322 /* A conditional CONTINUE may start a region of divergent control
323 * flow until the start of the next loop iteration (*not* until the
324 * end of the loop which is why the successor is not the top-level
325 * divergence point at cur_do). The live interval of any variable
326 * extending through a CONTINUE edge is guaranteed to overlap the
327 * whole region of divergent execution, because any variable live-out
328 * at the CONTINUE instruction will also be live-in at the top of the
329 * loop, and therefore also live-out at the bottom-most point of the
330 * loop which is reachable from the top (since a control flow path
331 * exists from a definition of the variable through this CONTINUE
332 * instruction, the top of the loop, the (reachable) bottom of the
333 * loop, the top of the loop again, into a use of the variable).
335 assert(cur_do
!= NULL
);
336 cur
->add_successor(mem_ctx
, cur_do
->next(), bblock_link_logical
);
340 cur
->add_successor(mem_ctx
, next
, bblock_link_logical
);
342 cur
->add_successor(mem_ctx
, next
, bblock_link_physical
);
344 set_next_block(&cur
, next
, ip
);
347 case BRW_OPCODE_BREAK
:
348 cur
->instructions
.push_tail(inst
);
350 /* A conditional BREAK instruction may start a region of divergent
351 * control flow until the end of the loop if the condition is
352 * non-uniform, in which case the loop will execute additional
353 * iterations with the present channel disabled. We model this as a
354 * control flow path from the divergence point to the convergence
355 * point that overlaps the whole IP range of the loop and skips over
356 * the execution of any other instructions part of the loop.
358 * See the DO case for additional explanation.
360 assert(cur_do
!= NULL
);
361 cur
->add_successor(mem_ctx
, cur_do
, bblock_link_physical
);
362 cur
->add_successor(mem_ctx
, cur_while
, bblock_link_logical
);
366 cur
->add_successor(mem_ctx
, next
, bblock_link_logical
);
368 set_next_block(&cur
, next
, ip
);
371 case BRW_OPCODE_WHILE
:
372 cur
->instructions
.push_tail(inst
);
374 assert(cur_do
!= NULL
&& cur_while
!= NULL
);
376 /* A conditional WHILE instruction may start a region of divergent
377 * control flow until the end of the loop, just like the BREAK
378 * instruction. See the BREAK case for more details. OTOH an
379 * unconditional WHILE instruction is non-divergent (just like an
380 * unconditional CONTINUE), and will necessarily lead to the
381 * execution of an additional iteration of the loop for all enabled
382 * channels, so we may skip over the divergence point at the top of
383 * the loop to keep the CFG as unambiguous as possible.
385 if (inst
->predicate
) {
386 cur
->add_successor(mem_ctx
, cur_do
, bblock_link_logical
);
388 cur
->add_successor(mem_ctx
, cur_do
->next(), bblock_link_logical
);
391 set_next_block(&cur
, cur_while
, ip
);
393 /* Pop the stack so we're in the previous loop */
394 cur_do
= pop_stack(&do_stack
);
395 cur_while
= pop_stack(&while_stack
);
399 cur
->instructions
.push_tail(inst
);
404 cur
->end_ip
= ip
- 1;
411 ralloc_free(mem_ctx
);
415 cfg_t::remove_block(bblock_t
*block
)
417 foreach_list_typed_safe (bblock_link
, predecessor
, link
, &block
->parents
) {
418 /* Remove block from all of its predecessors' successor lists. */
419 foreach_list_typed_safe (bblock_link
, successor
, link
,
420 &predecessor
->block
->children
) {
421 if (block
== successor
->block
) {
422 successor
->link
.remove();
423 ralloc_free(successor
);
427 /* Add removed-block's successors to its predecessors' successor lists. */
428 foreach_list_typed (bblock_link
, successor
, link
, &block
->children
) {
429 if (!successor
->block
->is_successor_of(predecessor
->block
,
431 predecessor
->block
->children
.push_tail(link(mem_ctx
,
438 foreach_list_typed_safe (bblock_link
, successor
, link
, &block
->children
) {
439 /* Remove block from all of its childrens' parents lists. */
440 foreach_list_typed_safe (bblock_link
, predecessor
, link
,
441 &successor
->block
->parents
) {
442 if (block
== predecessor
->block
) {
443 predecessor
->link
.remove();
444 ralloc_free(predecessor
);
448 /* Add removed-block's predecessors to its successors' predecessor lists. */
449 foreach_list_typed (bblock_link
, predecessor
, link
, &block
->parents
) {
450 if (!predecessor
->block
->is_predecessor_of(successor
->block
,
451 predecessor
->kind
)) {
452 successor
->block
->parents
.push_tail(link(mem_ctx
,
459 block
->link
.remove();
461 for (int b
= block
->num
; b
< this->num_blocks
- 1; b
++) {
462 this->blocks
[b
] = this->blocks
[b
+ 1];
463 this->blocks
[b
]->num
= b
;
466 this->blocks
[this->num_blocks
- 1]->num
= this->num_blocks
- 2;
473 bblock_t
*block
= new(mem_ctx
) bblock_t(this);
479 cfg_t::set_next_block(bblock_t
**cur
, bblock_t
*block
, int ip
)
482 (*cur
)->end_ip
= ip
- 1;
485 block
->start_ip
= ip
;
486 block
->num
= num_blocks
++;
487 block_list
.push_tail(&block
->link
);
492 cfg_t::make_block_array()
494 blocks
= ralloc_array(mem_ctx
, bblock_t
*, num_blocks
);
497 foreach_block (block
, this) {
500 assert(i
== num_blocks
);
506 const idom_tree
*idom
= (s
? &s
->idom_analysis
.require() : NULL
);
508 foreach_block (block
, this) {
509 if (idom
&& idom
->parent(block
))
510 fprintf(stderr
, "START B%d IDOM(B%d)", block
->num
,
511 idom
->parent(block
)->num
);
513 fprintf(stderr
, "START B%d IDOM(none)", block
->num
);
515 foreach_list_typed(bblock_link
, link
, link
, &block
->parents
) {
516 fprintf(stderr
, " <%cB%d",
517 link
->kind
== bblock_link_logical
? '-' : '~',
520 fprintf(stderr
, "\n");
523 fprintf(stderr
, "END B%d", block
->num
);
524 foreach_list_typed(bblock_link
, link
, link
, &block
->children
) {
525 fprintf(stderr
, " %c>B%d",
526 link
->kind
== bblock_link_logical
? '-' : '~',
529 fprintf(stderr
, "\n");
533 /* Calculates the immediate dominator of each block, according to "A Simple,
534 * Fast Dominance Algorithm" by Keith D. Cooper, Timothy J. Harvey, and Ken
537 * The authors claim that for control flow graphs of sizes normally encountered
538 * (less than 1000 nodes) that this algorithm is significantly faster than
539 * others like Lengauer-Tarjan.
541 idom_tree::idom_tree(const backend_shader
*s
) :
542 num_parents(s
->cfg
->num_blocks
),
543 parents(new bblock_t
*[num_parents
]())
547 parents
[0] = s
->cfg
->blocks
[0];
552 foreach_block(block
, s
->cfg
) {
556 bblock_t
*new_idom
= NULL
;
557 foreach_list_typed(bblock_link
, parent_link
, link
, &block
->parents
) {
558 if (parent(parent_link
->block
)) {
559 new_idom
= (new_idom
? intersect(new_idom
, parent_link
->block
) :
564 if (parent(block
) != new_idom
) {
565 parents
[block
->num
] = new_idom
;
572 idom_tree::~idom_tree()
578 idom_tree::intersect(bblock_t
*b1
, bblock_t
*b2
) const
580 /* Note, the comparisons here are the opposite of what the paper says
581 * because we index blocks from beginning -> end (i.e. reverse post-order)
582 * instead of post-order like they assume.
584 while (b1
->num
!= b2
->num
) {
585 while (b1
->num
> b2
->num
)
587 while (b2
->num
> b1
->num
)
595 idom_tree::dump() const
597 printf("digraph DominanceTree {\n");
598 for (unsigned i
= 0; i
< num_parents
; i
++)
599 printf("\t%d -> %d\n", parents
[i
]->num
, i
);
606 printf("digraph CFG {\n");
607 for (int b
= 0; b
< num_blocks
; b
++) {
608 bblock_t
*block
= this->blocks
[b
];
610 foreach_list_typed_safe (bblock_link
, child
, link
, &block
->children
) {
611 printf("\t%d -> %d\n", b
, child
->block
->num
);