projects / mesa.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
history | raw | HEAD
r600g/sb: Support gs5 sampler indexing (v2)
[mesa.git] / src / gallium / drivers / r600 / sb / sb_gcm.cpp
1 /*
2 * Copyright 2013 Vadim Girlin <vadimgirlin@gmail.com>
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * on the rights to use, copy, modify, merge, publish, distribute, sub
8 * license, and/or sell copies of the Software, and to permit persons to whom
9 * the Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHOR(S) AND/OR THEIR SUPPLIERS BE LIABLE FOR ANY CLAIM,
19 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
20 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
21 * USE OR OTHER DEALINGS IN THE SOFTWARE.
22 *
23 * Authors:
24 * Vadim Girlin
25 */
26
27 #define GCM_DEBUG 0
28
29 #if GCM_DEBUG
30 #define GCM_DUMP(a) do { a } while(0);
31 #else
32 #define GCM_DUMP(a)
33 #endif
34
35 #include <map>
36
37 #include "sb_bc.h"
38 #include "sb_shader.h"
39 #include "sb_pass.h"
40 #include "eg_sq.h" // V_SQ_CF_INDEX_NONE
41
42 namespace r600_sb {
43
44 int gcm::run() {
45
46 GCM_DUMP( sblog << "==== GCM ==== \n"; sh.dump_ir(); );
47
48 collect_instructions(sh.root, true);
49
50 init_def_count(uses, pending);
51
52 for (node_iterator N, I = pending.begin(), E = pending.end();
53 I != E; I = N) {
54 N = I;
55 ++N;
56 node *o = *I;
57
58 GCM_DUMP(
59 sblog << "pending : ";
60 dump::dump_op(o);
61 sblog << "\n";
62 );
63
64 if (td_is_ready(o)) {
65
66 GCM_DUMP(
67 sblog << " ready: ";
68 dump::dump_op(o);
69 sblog << "\n";
70 );
71 pending.remove_node(o);
72 ready.push_back(o);
73 } else {
74 }
75 }
76
77 sched_early(sh.root);
78
79 if (!pending.empty()) {
80 sblog << "##### gcm_sched_early_pass: unscheduled ops:\n";
81 dump::dump_op(pending.front());
82 }
83
84 assert(pending.empty());
85
86 GCM_DUMP( sh.dump_ir(); );
87
88 GCM_DUMP( sblog << "\n\n ############## gcm late\n\n"; );
89
90 collect_instructions(sh.root, false);
91
92 init_use_count(uses, pending);
93
94 sched_late(sh.root);
95 if (!pending.empty()) {
96 sblog << "##### gcm_sched_late_pass: unscheduled ops:\n";
97 dump::dump_op(pending.front());
98 }
99
100 assert(ucs_level == 0);
101 assert(pending.empty());
102
103 return 0;
104 }
105
106
107 void gcm::collect_instructions(container_node *c, bool early_pass) {
108 if (c->is_bb()) {
109
110 if (early_pass) {
111 for (node_iterator I = c->begin(), E = c->end(); I != E; ++I) {
112 node *n = *I;
113 if (n->flags & NF_DONT_MOVE) {
114 op_info &o = op_map[n];
115 o.top_bb = o.bottom_bb = static_cast<bb_node*>(c);
116 }
117 }
118 }
119
120 pending.append_from(c);
121 return;
122 }
123
124 for (node_iterator I = c->begin(), E = c->end(); I != E; ++I) {
125 if (I->is_container()) {
126 collect_instructions(static_cast<container_node*>(*I), early_pass);
127 }
128 }
129 }
130
131 void gcm::sched_early(container_node *n) {
132
133 region_node *r =
134 (n->type == NT_REGION) ? static_cast<region_node*>(n) : NULL;
135
136 if (r && r->loop_phi) {
137 sched_early(r->loop_phi);
138 }
139
140 for (node_iterator I = n->begin(), E = n->end(); I != E; ++I) {
141 if (I->type == NT_OP) {
142 node *op = *I;
143 if (op->subtype == NST_PHI) {
144 td_release_uses(op->dst);
145 }
146 } else if (I->is_container()) {
147 if (I->subtype == NST_BB) {
148 bb_node* bb = static_cast<bb_node*>(*I);
149 td_sched_bb(bb);
150 } else {
151 sched_early(static_cast<container_node*>(*I));
152 }
153 }
154 }
155
156 if (r && r->phi) {
157 sched_early(r->phi);
158 }
159 }
160
161 void gcm::td_schedule(bb_node *bb, node *n) {
162 GCM_DUMP(
163 sblog << "scheduling : ";
164 dump::dump_op(n);
165 sblog << "\n";
166 );
167 td_release_uses(n->dst);
168
169 bb->push_back(n);
170
171 op_map[n].top_bb = bb;
172
173 }
174
175 void gcm::td_sched_bb(bb_node* bb) {
176 GCM_DUMP(
177 sblog << "td scheduling BB_" << bb->id << "\n";
178 );
179
180 while (!ready.empty()) {
181 for (sq_iterator N, I = ready.begin(), E = ready.end(); I != E;
182 I = N) {
183 N = I; ++N;
184 td_schedule(bb, *I);
185 ready.erase(I);
186 }
187 }
188 }
189
190 bool gcm::td_is_ready(node* n) {
191 return uses[n] == 0;
192 }
193
194 void gcm::td_release_val(value *v) {
195
196 GCM_DUMP(
197 sblog << "td checking uses: ";
198 dump::dump_val(v);
199 sblog << "\n";
200 );
201
202 use_info *u = v->uses;
203 while (u) {
204 if (u->op->parent != &pending) {
205 u = u->next;
206 continue;
207 }
208
209 GCM_DUMP(
210 sblog << "td used in ";
211 dump::dump_op(u->op);
212 sblog << "\n";
213 );
214
215 if (--uses[u->op] == 0) {
216 GCM_DUMP(
217 sblog << "td released : ";
218 dump::dump_op(u->op);
219 sblog << "\n";
220 );
221
222 pending.remove_node(u->op);
223 ready.push_back(u->op);
224 }
225 u = u->next;
226 }
227
228 }
229
230 void gcm::td_release_uses(vvec& v) {
231 for (vvec::iterator I = v.begin(), E = v.end(); I != E; ++I) {
232 value *v = *I;
233 if (!v)
234 continue;
235
236 if (v->is_rel())
237 td_release_uses(v->mdef);
238 else
239 td_release_val(v);
240 }
241 }
242
243 void gcm::sched_late(container_node *n) {
244
245 bool stack_pushed = false;
246
247 if (n->is_depart()) {
248 depart_node *d = static_cast<depart_node*>(n);
249 push_uc_stack();
250 stack_pushed = true;
251 bu_release_phi_defs(d->target->phi, d->dep_id);
252 } else if (n->is_repeat()) {
253 repeat_node *r = static_cast<repeat_node*>(n);
254 assert(r->target->loop_phi);
255 push_uc_stack();
256 stack_pushed = true;
257 bu_release_phi_defs(r->target->loop_phi, r->rep_id);
258 }
259
260 for (node_riterator I = n->rbegin(), E = n->rend(); I != E; ++I) {
261 if (I->is_container()) {
262 if (I->subtype == NST_BB) {
263 bb_node* bb = static_cast<bb_node*>(*I);
264 bu_sched_bb(bb);
265 } else {
266 sched_late(static_cast<container_node*>(*I));
267 }
268 }
269 }
270
271 if (n->type == NT_IF) {
272 if_node *f = static_cast<if_node*>(n);
273 if (f->cond)
274 pending_defs.push_back(f->cond);
275 } else if (n->type == NT_REGION) {
276 region_node *r = static_cast<region_node*>(n);
277 if (r->loop_phi)
278 bu_release_phi_defs(r->loop_phi, 0);
279 }
280
281 if (stack_pushed)
282 pop_uc_stack();
283
284 }
285
286 void gcm::bu_sched_bb(bb_node* bb) {
287 GCM_DUMP(
288 sblog << "bu scheduling BB_" << bb->id << "\n";
289 );
290
291 bu_bb = bb;
292
293 if (!pending_nodes.empty()) {
294 GCM_DUMP(
295 sblog << "pending nodes:\n";
296 );
297
298 // TODO consider sorting the exports by array_base,
299 // possibly it can improve performance
300
301 for (node_list::iterator I = pending_nodes.begin(),
302 E = pending_nodes.end(); I != E; ++I) {
303 bu_release_op(*I);
304 }
305 pending_nodes.clear();
306 GCM_DUMP(
307 sblog << "pending nodes processed...\n";
308 );
309 }
310
311
312 if (!pending_defs.empty()) {
313 for (vvec::iterator I = pending_defs.begin(), E = pending_defs.end();
314 I != E; ++I) {
315 bu_release_val(*I);
316 }
317 pending_defs.clear();
318 }
319
320 for (sched_queue::iterator N, I = ready_above.begin(), E = ready_above.end();
321 I != E; I = N) {
322 N = I;
323 ++N;
324 node *n = *I;
325 if (op_map[n].bottom_bb == bb) {
326 add_ready(*I);
327 ready_above.erase(I);
328 }
329 }
330
331 unsigned cnt_ready[SQ_NUM];
332
333 container_node *clause = NULL;
334 unsigned last_inst_type = ~0;
335 unsigned last_count = 0;
336
337 bool s = true;
338 while (s) {
339 node *n;
340
341 s = false;
342
343 unsigned ready_mask = 0;
344
345 for (unsigned sq = SQ_CF; sq < SQ_NUM; ++sq) {
346 if (!bu_ready[sq].empty() || !bu_ready_next[sq].empty())
347 ready_mask |= (1 << sq);
348 }
349
350 if (!ready_mask) {
351 for (unsigned sq = SQ_CF; sq < SQ_NUM; ++sq) {
352 if (!bu_ready_early[sq].empty()) {
353 node *n = bu_ready_early[sq].front();
354 bu_ready_early[sq].pop_front();
355 bu_ready[sq].push_back(n);
356 break;
357 }
358 }
359 }
360
361 for (unsigned sq = SQ_CF; sq < SQ_NUM; ++sq) {
362
363 if (sq == SQ_CF && pending_exec_mask_update) {
364 pending_exec_mask_update = false;
365 sq = SQ_ALU;
366 --sq;
367 continue;
368 }
369
370 if (!bu_ready_next[sq].empty())
371 bu_ready[sq].splice(bu_ready[sq].end(), bu_ready_next[sq]);
372
373 cnt_ready[sq] = bu_ready[sq].size();
374
375 if ((sq == SQ_TEX || sq == SQ_VTX) && live_count <= rp_threshold &&
376 cnt_ready[sq] < ctx.max_fetch/2 &&
377 !bu_ready_next[SQ_ALU].empty()) {
378 sq = SQ_ALU;
379 --sq;
380 continue;
381 }
382
383 while (!bu_ready[sq].empty()) {
384
385 if (last_inst_type != sq) {
386 clause = NULL;
387 last_count = 0;
388 last_inst_type = sq;
389 }
390
391 // simple heuristic to limit register pressure,
392 if (sq == SQ_ALU && live_count > rp_threshold &&
393 (!bu_ready[SQ_TEX].empty() ||
394 !bu_ready[SQ_VTX].empty() ||
395 !bu_ready_next[SQ_TEX].empty() ||
396 !bu_ready_next[SQ_VTX].empty())) {
397 GCM_DUMP( sblog << "switching to fetch (regpressure)\n"; );
398 break;
399 }
400
401 n = bu_ready[sq].front();
402
403 // real count (e.g. SAMPLE_G will be expanded to 3 instructions,
404 // 2 SET_GRAD_ + 1 SAMPLE_G
405 unsigned ncnt = 1;
406 if (n->is_fetch_inst() && n->src.size() == 12) {
407 ncnt = 3;
408 }
409
410 bool sampler_indexing = false;
411 if (n->is_fetch_inst() &&
412 static_cast<fetch_node *>(n)->bc.sampler_index_mode != V_SQ_CF_INDEX_NONE)
413 {
414 sampler_indexing = true; // Give sampler indexed ops get their own clause
415 ncnt = sh.get_ctx().is_cayman() ? 2 : 3; // MOVA + SET_CF_IDX0/1
416 }
417
418 if ((sq == SQ_TEX || sq == SQ_VTX) &&
419 ((last_count >= ctx.max_fetch/2 &&
420 check_alu_ready_count(24)) ||
421 last_count + ncnt > ctx.max_fetch))
422 break;
423 else if (sq == SQ_CF && last_count > 4 &&
424 check_alu_ready_count(24))
425 break;
426
427 bu_ready[sq].pop_front();
428
429 if (sq != SQ_CF) {
430 if (!clause || sampler_indexing) {
431 clause = sh.create_clause(sq == SQ_ALU ?
432 NST_ALU_CLAUSE :
433 sq == SQ_TEX ? NST_TEX_CLAUSE :
434 NST_VTX_CLAUSE);
435 bb->push_front(clause);
436 }
437 } else {
438 clause = bb;
439 }
440
441 bu_schedule(clause, n);
442 s = true;
443 last_count += ncnt;
444 }
445 }
446 }
447
448 bu_bb = NULL;
449
450 GCM_DUMP(
451 sblog << "bu finished scheduling BB_" << bb->id << "\n";
452 );
453 }
454
455 void gcm::bu_release_defs(vvec& v, bool src) {
456 for (vvec::reverse_iterator I = v.rbegin(), E = v.rend(); I != E; ++I) {
457 value *v = *I;
458 if (!v || v->is_readonly())
459 continue;
460
461 if (v->is_rel()) {
462 if (!v->rel->is_readonly())
463 bu_release_val(v->rel);
464 bu_release_defs(v->muse, true);
465 } else if (src)
466 bu_release_val(v);
467 else {
468 if (live.remove_val(v)) {
469 --live_count;
470 }
471 }
472 }
473 }
474
475 void gcm::push_uc_stack() {
476 GCM_DUMP(
477 sblog << "pushing use count stack prev_level " << ucs_level
478 << " new level " << (ucs_level + 1) << "\n";
479 );
480 ++ucs_level;
481 if (ucs_level == nuc_stk.size()) {
482 nuc_stk.resize(ucs_level + 1);
483 }
484 else {
485 nuc_stk[ucs_level].clear();
486 }
487 }
488
489 bool gcm::bu_is_ready(node* n) {
490 nuc_map &cm = nuc_stk[ucs_level];
491 nuc_map::iterator F = cm.find(n);
492 unsigned uc = (F == cm.end() ? 0 : F->second);
493 return uc == uses[n];
494 }
495
496 void gcm::bu_schedule(container_node* c, node* n) {
497 GCM_DUMP(
498 sblog << "bu scheduling : ";
499 dump::dump_op(n);
500 sblog << "\n";
501 );
502
503 assert(op_map[n].bottom_bb == bu_bb);
504
505 bu_release_defs(n->src, true);
506 bu_release_defs(n->dst, false);
507
508 c->push_front(n);
509 }
510
511 void gcm::dump_uc_stack() {
512 sblog << "##### uc_stk start ####\n";
513 for (unsigned l = 0; l <= ucs_level; ++l) {
514 nuc_map &m = nuc_stk[l];
515
516 sblog << "nuc_stk[" << l << "] : @" << &m << "\n";
517
518 for (nuc_map::iterator I = m.begin(), E = m.end(); I != E; ++I) {
519 sblog << " uc " << I->second << " for ";
520 dump::dump_op(I->first);
521 sblog << "\n";
522 }
523 }
524 sblog << "##### uc_stk end ####\n";
525 }
526
527 void gcm::pop_uc_stack() {
528 nuc_map &pm = nuc_stk[ucs_level];
529 --ucs_level;
530 nuc_map &cm = nuc_stk[ucs_level];
531
532 GCM_DUMP(
533 sblog << "merging use stack from level " << (ucs_level+1)
534 << " to " << ucs_level << "\n";
535 );
536
537 for (nuc_map::iterator N, I = pm.begin(), E = pm.end(); I != E; ++I) {
538 node *n = I->first;
539
540 GCM_DUMP(
541 sblog << " " << cm[n] << " += " << I->second << " for ";
542 dump::dump_op(n);
543 sblog << "\n";
544 );
545
546 unsigned uc = cm[n] += I->second;
547
548 if (n->parent == &pending && uc == uses[n]) {
549 cm.erase(n);
550 pending_nodes.push_back(n);
551 GCM_DUMP(
552 sblog << "pushed pending_node due to stack pop ";
553 dump::dump_op(n);
554 sblog << "\n";
555 );
556 }
557 }
558 }
559
560 void gcm::bu_find_best_bb(node *n, op_info &oi) {
561
562 GCM_DUMP(
563 sblog << " find best bb : ";
564 dump::dump_op(n);
565 sblog << "\n";
566 );
567
568 if (oi.bottom_bb)
569 return;
570
571 // don't hoist generated copies
572 if (n->flags & NF_DONT_HOIST) {
573 oi.bottom_bb = bu_bb;
574 return;
575 }
576
577 bb_node* best_bb = bu_bb;
578 bb_node* top_bb = oi.top_bb;
579 assert(oi.top_bb && !oi.bottom_bb);
580
581 node *c = best_bb;
582
583 // FIXME top_bb may be located inside the loop so we'll never enter it
584 // in the loop below, and the instruction will be incorrectly placed at the
585 // beginning of the shader.
586 // For now just check if top_bb's loop_level is higher than of
587 // current bb and abort the search for better bb in such case,
588 // but this problem may require more complete (and more expensive) fix
589 if (top_bb->loop_level <= best_bb->loop_level) {
590 while (c && c != top_bb) {
591
592 if (c->prev) {
593 c = c->prev;
594 } else {
595 c = c->parent;
596 if (!c)
597 break;
598 continue;
599 }
600
601 if (c->subtype == NST_BB) {
602 bb_node *bb = static_cast<bb_node*>(c);
603 if (bb->loop_level < best_bb->loop_level)
604 best_bb = bb;
605 }
606 }
607 }
608
609 oi.bottom_bb = best_bb;
610 }
611
612 void gcm::add_ready(node *n) {
613 sched_queue_id sq = sh.get_queue_id(n);
614 if (n->flags & NF_SCHEDULE_EARLY)
615 bu_ready_early[sq].push_back(n);
616 else if (sq == SQ_ALU && n->is_copy_mov())
617 bu_ready[sq].push_front(n);
618 else if (n->is_alu_inst()) {
619 alu_node *a = static_cast<alu_node*>(n);
620 if (a->bc.op_ptr->flags & AF_PRED && a->dst[2]) {
621 // PRED_SET instruction that updates exec mask
622 pending_exec_mask_update = true;
623 }
624 bu_ready_next[sq].push_back(n);
625 } else
626 bu_ready_next[sq].push_back(n);
627 }
628
629 void gcm::bu_release_op(node * n) {
630 op_info &oi = op_map[n];
631
632 GCM_DUMP(
633 sblog << " bu release op ";
634 dump::dump_op(n);
635 );
636
637 nuc_stk[ucs_level].erase(n);
638 pending.remove_node(n);
639
640 bu_find_best_bb(n, oi);
641
642 if (oi.bottom_bb == bu_bb) {
643 GCM_DUMP( sblog << " ready\n";);
644 add_ready(n);
645 } else {
646 GCM_DUMP( sblog << " ready_above\n";);
647 ready_above.push_back(n);
648 }
649 }
650
651 void gcm::bu_release_phi_defs(container_node* p, unsigned op)
652 {
653 for (node_riterator I = p->rbegin(), E = p->rend(); I != E; ++I) {
654 node *o = *I;
655 value *v = o->src[op];
656 if (v && !v->is_readonly())
657 pending_defs.push_back(o->src[op]);
658
659 }
660 }
661
662 unsigned gcm::get_uc_vec(vvec &vv) {
663 unsigned c = 0;
664 for (vvec::iterator I = vv.begin(), E = vv.end(); I != E; ++I) {
665 value *v = *I;
666 if (!v)
667 continue;
668
669 if (v->is_rel())
670 c += get_uc_vec(v->mdef);
671 else
672 c += v->use_count();
673 }
674 return c;
675 }
676
677 void gcm::init_use_count(nuc_map& m, container_node &s) {
678 m.clear();
679 for (node_iterator I = s.begin(), E = s.end(); I != E; ++I) {
680 node *n = *I;
681 unsigned uc = get_uc_vec(n->dst);
682 GCM_DUMP(
683 sblog << "uc " << uc << " ";
684 dump::dump_op(n);
685 sblog << "\n";
686 );
687 if (!uc) {
688 pending_nodes.push_back(n);
689 GCM_DUMP(
690 sblog << "pushed pending_node in init ";
691 dump::dump_op(n);
692 sblog << "\n";
693 );
694
695 } else
696 m[n] = uc;
697 }
698 }
699
700 void gcm::bu_release_val(value* v) {
701 node *n = v->any_def();
702
703 if (n && n->parent == &pending) {
704 nuc_map &m = nuc_stk[ucs_level];
705 unsigned uc = ++m[n];
706 unsigned uc2 = uses[n];
707
708 if (live.add_val(v)) {
709 ++live_count;
710 GCM_DUMP ( sblog << "live_count: " << live_count << "\n"; );
711 }
712
713 GCM_DUMP(
714 sblog << "release val ";
715 dump::dump_val(v);
716 sblog << " for node ";
717 dump::dump_op(n);
718 sblog << " new uc=" << uc << ", total " << uc2 << "\n";
719 );
720
721 if (uc == uc2)
722 bu_release_op(n);
723 }
724
725 }
726
727 void gcm::init_def_count(nuc_map& m, container_node& s) {
728 m.clear();
729 for (node_iterator I = s.begin(), E = s.end(); I != E; ++I) {
730 node *n = *I;
731 unsigned dc = get_dc_vec(n->src, true) + get_dc_vec(n->dst, false);
732 m[n] = dc;
733
734 GCM_DUMP(
735 sblog << "dc " << dc << " ";
736 dump::dump_op(n);
737 sblog << "\n";
738 );
739 }
740 }
741
742 unsigned gcm::get_dc_vec(vvec& vv, bool src) {
743 unsigned c = 0;
744 for (vvec::iterator I = vv.begin(), E = vv.end(); I != E; ++I) {
745 value *v = *I;
746 if (!v || v->is_readonly())
747 continue;
748
749 if (v->is_rel()) {
750 c += v->rel->def != NULL;
751 c += get_dc_vec(v->muse, true);
752 }
753 else if (src) {
754 c += v->def != NULL;
755 c += v->adef != NULL;
756 }
757 }
758 return c;
759 }
760
761 unsigned gcm::real_alu_count(sched_queue& q, unsigned max) {
762 sq_iterator I(q.begin()), E(q.end());
763 unsigned c = 0;
764
765 while (I != E && c < max) {
766 node *n = *I;
767 if (n->is_alu_inst()) {
768 if (!n->is_copy_mov() || !n->src[0]->is_any_gpr())
769 ++c;
770 } else if (n->is_alu_packed()) {
771 c += static_cast<container_node*>(n)->count();
772 }
773 ++I;
774 }
775
776 return c;
777 }
778
779 bool gcm::check_alu_ready_count(unsigned threshold) {
780 unsigned r = real_alu_count(bu_ready[SQ_ALU], threshold);
781 if (r >= threshold)
782 return true;
783 r += real_alu_count(bu_ready_next[SQ_ALU], threshold - r);
784 return r >= threshold;
785 }
786
787 } // namespace r600_sb