src/gallium/drivers/r600/sb/sb_pass.h

   1 /*
   2  * Copyright 2013 Vadim Girlin <vadimgirlin@gmail.com>
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * on the rights to use, copy, modify, merge, publish, distribute, sub
   8  * license, and/or sell copies of the Software, and to permit persons to whom
   9  * the Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
  18  * THE AUTHOR(S) AND/OR THEIR SUPPLIERS BE LIABLE FOR ANY CLAIM,
  19  * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
  20  * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
  21  * USE OR OTHER DEALINGS IN THE SOFTWARE.
  22  *
  23  * Authors:
  24  *      Vadim Girlin
  25  */
  26
  27 #ifndef SB_PASS_H_
  28 #define SB_PASS_H_
  29
  30 #include <stack>
  31
  32 namespace r600_sb {
  33
  34 class pass {
  35 protected:
  36         sb_context &ctx;
  37         shader &sh;
  38
  39 public:
  40         pass(shader &s);
  41
  42         virtual int run();
  43
  44         virtual ~pass() {}
  45 };
  46
  47 class vpass : public pass {
  48
  49 public:
  50
  51         vpass(shader &s) : pass(s) {}
  52
  53         virtual int init();
  54         virtual int done();
  55
  56         virtual int run();
  57         virtual void run_on(container_node &n);
  58
  59         virtual bool visit(node &n, bool enter);
  60         virtual bool visit(container_node &n, bool enter);
  61         virtual bool visit(alu_group_node &n, bool enter);
  62         virtual bool visit(cf_node &n, bool enter);
  63         virtual bool visit(alu_node &n, bool enter);
  64         virtual bool visit(alu_packed_node &n, bool enter);
  65         virtual bool visit(fetch_node &n, bool enter);
  66         virtual bool visit(region_node &n, bool enter);
  67         virtual bool visit(repeat_node &n, bool enter);
  68         virtual bool visit(depart_node &n, bool enter);
  69         virtual bool visit(if_node &n, bool enter);
  70         virtual bool visit(bb_node &n, bool enter);
  71
  72 };
  73
  74 class rev_vpass : public vpass {
  75
  76 public:
  77         rev_vpass(shader &s) : vpass(s) {}
  78
  79         virtual void run_on(container_node &n);
  80 };
  81
  82
  83 // =================== PASSES
  84
  85 class bytecode;
  86
  87 class bc_dump : public vpass {
  88         using vpass::visit;
  89
  90         uint32_t *bc_data;
  91         unsigned ndw;
  92
  93         unsigned id;
  94
  95         unsigned new_group, group_index;
  96
  97 public:
  98
  99         bc_dump(shader &s, bytecode *bc = NULL);
 100
 101         bc_dump(shader &s, uint32_t *bc_ptr, unsigned ndw) :
 102                 vpass(s), bc_data(bc_ptr), ndw(ndw), id(), new_group(), group_index() {}
 103
 104         virtual int init();
 105         virtual int done();
 106
 107         virtual bool visit(cf_node &n, bool enter);
 108         virtual bool visit(alu_node &n, bool enter);
 109         virtual bool visit(fetch_node &n, bool enter);
 110
 111         void dump_dw(unsigned dw_id, unsigned count = 2);
 112
 113         void dump(cf_node& n);
 114         void dump(alu_node& n);
 115         void dump(fetch_node& n);
 116 };
 117
 118
 119 class dce_cleanup : public vpass {
 120         using vpass::visit;
 121
 122         bool remove_unused;
 123
 124 public:
 125
 126         dce_cleanup(shader &s) : vpass(s),
 127                 remove_unused(s.dce_flags & DF_REMOVE_UNUSED), nodes_changed(false) {}
 128
 129         virtual int run();
 130
 131         virtual bool visit(node &n, bool enter);
 132         virtual bool visit(alu_group_node &n, bool enter);
 133         virtual bool visit(cf_node &n, bool enter);
 134         virtual bool visit(alu_node &n, bool enter);
 135         virtual bool visit(alu_packed_node &n, bool enter);
 136         virtual bool visit(fetch_node &n, bool enter);
 137         virtual bool visit(region_node &n, bool enter);
 138         virtual bool visit(container_node &n, bool enter);
 139
 140 private:
 141
 142         void cleanup_dst(node &n);
 143         bool cleanup_dst_vec(vvec &vv);
 144
 145         // Did we alter/remove nodes during a single pass?
 146         bool nodes_changed;
 147 };
 148
 149
 150 class def_use : public pass {
 151
 152 public:
 153
 154         def_use(shader &sh) : pass(sh) {}
 155
 156         virtual int run();
 157         void run_on(node *n, bool defs);
 158
 159 private:
 160
 161         void process_uses(node *n);
 162         void process_defs(node *n, vvec &vv, bool arr_def);
 163         void process_phi(container_node *c, bool defs, bool uses);
 164 };
 165
 166
 167
 168 class dump : public vpass {
 169         using vpass::visit;
 170
 171         int level;
 172
 173 public:
 174
 175         dump(shader &s) : vpass(s), level(0) {}
 176
 177         virtual bool visit(node &n, bool enter);
 178         virtual bool visit(container_node &n, bool enter);
 179         virtual bool visit(alu_group_node &n, bool enter);
 180         virtual bool visit(cf_node &n, bool enter);
 181         virtual bool visit(alu_node &n, bool enter);
 182         virtual bool visit(alu_packed_node &n, bool enter);
 183         virtual bool visit(fetch_node &n, bool enter);
 184         virtual bool visit(region_node &n, bool enter);
 185         virtual bool visit(repeat_node &n, bool enter);
 186         virtual bool visit(depart_node &n, bool enter);
 187         virtual bool visit(if_node &n, bool enter);
 188         virtual bool visit(bb_node &n, bool enter);
 189
 190
 191         static void dump_op(node &n, const char *name);
 192         static void dump_vec(const vvec & vv);
 193         static void dump_set(shader &sh, val_set & v);
 194
 195         static void dump_rels(vvec & vv);
 196
 197         static void dump_val(value *v);
 198         static void dump_op(node *n);
 199
 200         static void dump_op_list(container_node *c);
 201         static void dump_queue(sched_queue &q);
 202
 203         static void dump_alu(alu_node *n);
 204
 205 private:
 206
 207         void indent();
 208
 209         void dump_common(node &n);
 210         void dump_flags(node &n);
 211
 212         void dump_live_values(container_node &n, bool before);
 213 };
 214
 215
 216 // Global Code Motion
 217
 218 class gcm : public pass {
 219
 220         sched_queue bu_ready[SQ_NUM];
 221         sched_queue bu_ready_next[SQ_NUM];
 222         sched_queue bu_ready_early[SQ_NUM];
 223         sched_queue ready;
 224         sched_queue ready_above;
 225
 226         container_node pending;
 227
 228         struct op_info {
 229                 bb_node* top_bb;
 230                 bb_node* bottom_bb;
 231                 op_info() : top_bb(), bottom_bb() {}
 232         };
 233
 234         typedef std::map<node*, op_info> op_info_map;
 235
 236         typedef std::map<node*, unsigned> nuc_map;
 237
 238         op_info_map op_map;
 239         nuc_map uses;
 240
 241         typedef std::vector<nuc_map> nuc_stack;
 242
 243         nuc_stack nuc_stk;
 244         unsigned ucs_level;
 245
 246         bb_node * bu_bb;
 247
 248         vvec pending_defs;
 249
 250         node_list pending_nodes;
 251
 252         unsigned cur_sq;
 253
 254         // for register pressure tracking in bottom-up pass
 255         val_set live;
 256         int live_count;
 257
 258         static const int rp_threshold = 100;
 259
 260         bool pending_exec_mask_update;
 261
 262 public:
 263
 264         gcm(shader &sh) : pass(sh),
 265                 bu_ready(), bu_ready_next(), bu_ready_early(),
 266                 ready(), op_map(), uses(), nuc_stk(1), ucs_level(),
 267                 bu_bb(), pending_defs(), pending_nodes(), cur_sq(),
 268                 live(), live_count(), pending_exec_mask_update() {}
 269
 270         virtual int run();
 271
 272 private:
 273
 274         void collect_instructions(container_node *c, bool early_pass);
 275
 276         void sched_early(container_node *n);
 277         void td_sched_bb(bb_node *bb);
 278         bool td_is_ready(node *n);
 279         void td_release_uses(vvec &v);
 280         void td_release_val(value *v);
 281         void td_schedule(bb_node *bb, node *n);
 282
 283         void sched_late(container_node *n);
 284         void bu_sched_bb(bb_node *bb);
 285         void bu_release_defs(vvec &v, bool src);
 286         void bu_release_phi_defs(container_node *p, unsigned op);
 287         bool bu_is_ready(node *n);
 288         void bu_release_val(value *v);
 289         void bu_release_op(node * n);
 290         void bu_find_best_bb(node *n, op_info &oi);
 291         void bu_schedule(container_node *bb, node *n);
 292
 293         void push_uc_stack();
 294         void pop_uc_stack();
 295
 296         void init_def_count(nuc_map &m, container_node &s);
 297         void init_use_count(nuc_map &m, container_node &s);
 298         unsigned get_uc_vec(vvec &vv);
 299         unsigned get_dc_vec(vvec &vv, bool src);
 300
 301         void add_ready(node *n);
 302
 303         void dump_uc_stack();
 304
 305         unsigned real_alu_count(sched_queue &q, unsigned max);
 306
 307         // check if we have not less than threshold ready alu instructions
 308         bool check_alu_ready_count(unsigned threshold);
 309 };
 310
 311
 312 class gvn : public vpass {
 313         using vpass::visit;
 314
 315 public:
 316
 317         gvn(shader &sh) : vpass(sh) {}
 318
 319         virtual bool visit(node &n, bool enter);
 320         virtual bool visit(cf_node &n, bool enter);
 321         virtual bool visit(alu_node &n, bool enter);
 322         virtual bool visit(alu_packed_node &n, bool enter);
 323         virtual bool visit(fetch_node &n, bool enter);
 324         virtual bool visit(region_node &n, bool enter);
 325
 326 private:
 327
 328         void process_op(node &n, bool rewrite = true);
 329
 330         // returns true if the value was rewritten
 331         bool process_src(value* &v, bool rewrite);
 332
 333
 334         void process_alu_src_constants(node &n, value* &v);
 335 };
 336
 337
 338 class if_conversion : public pass {
 339
 340 public:
 341
 342         if_conversion(shader &sh) : pass(sh) {}
 343
 344         virtual int run();
 345
 346         bool run_on(region_node *r);
 347
 348         void convert_kill_instructions(region_node *r, value *em, bool branch,
 349                                        container_node *c);
 350
 351         bool check_and_convert(region_node *r);
 352
 353         alu_node* convert_phi(value *select, node *phi);
 354
 355 };
 356
 357
 358 class liveness : public rev_vpass {
 359         using vpass::visit;
 360
 361         val_set live;
 362         bool live_changed;
 363
 364 public:
 365
 366         liveness(shader &s) : rev_vpass(s), live_changed(false) {}
 367
 368         virtual int init();
 369
 370         virtual bool visit(node &n, bool enter);
 371         virtual bool visit(bb_node &n, bool enter);
 372         virtual bool visit(container_node &n, bool enter);
 373         virtual bool visit(alu_group_node &n, bool enter);
 374         virtual bool visit(cf_node &n, bool enter);
 375         virtual bool visit(alu_node &n, bool enter);
 376         virtual bool visit(alu_packed_node &n, bool enter);
 377         virtual bool visit(fetch_node &n, bool enter);
 378         virtual bool visit(region_node &n, bool enter);
 379         virtual bool visit(repeat_node &n, bool enter);
 380         virtual bool visit(depart_node &n, bool enter);
 381         virtual bool visit(if_node &n, bool enter);
 382
 383 private:
 384
 385         void update_interferences();
 386         void process_op(node &n);
 387
 388         bool remove_val(value *v);
 389         bool remove_vec(vvec &v);
 390         bool process_outs(node& n);
 391         void process_ins(node& n);
 392
 393         void process_phi_outs(container_node *phi);
 394         void process_phi_branch(container_node *phi, unsigned id);
 395
 396         bool process_maydef(value *v);
 397
 398         bool add_vec(vvec &vv, bool src);
 399
 400         void update_src_vec(vvec &vv, bool src);
 401 };
 402
 403
 404 struct bool_op_info {
 405         bool invert;
 406         unsigned int_cvt;
 407
 408         alu_node *n;
 409 };
 410
 411 class peephole : public pass {
 412
 413 public:
 414
 415         peephole(shader &sh) : pass(sh) {}
 416
 417         virtual int run();
 418
 419         void run_on(container_node *c);
 420
 421         void optimize_cc_op(alu_node *a);
 422
 423         void optimize_cc_op2(alu_node *a);
 424         void optimize_CNDcc_op(alu_node *a);
 425
 426         bool get_bool_op_info(value *b, bool_op_info& bop);
 427         bool get_bool_flt_to_int_source(alu_node* &a);
 428         void convert_float_setcc(alu_node *f2i, alu_node *s);
 429 };
 430
 431
 432 class psi_ops : public rev_vpass {
 433         using rev_vpass::visit;
 434
 435 public:
 436
 437         psi_ops(shader &s) : rev_vpass(s) {}
 438
 439         virtual bool visit(node &n, bool enter);
 440         virtual bool visit(alu_node &n, bool enter);
 441
 442         bool try_inline(node &n);
 443         bool try_reduce(node &n);
 444         bool eliminate(node &n);
 445
 446         void unpredicate(node *n);
 447 };
 448
 449
 450 // check correctness of the generated code, e.g.:
 451 // - expected source operand value is the last value written to its gpr,
 452 // - all arguments of phi node should be allocated to the same gpr,
 453 // TODO other tests
 454 class ra_checker : public pass {
 455
 456         typedef std::map<sel_chan, value *> reg_value_map;
 457
 458         typedef std::vector<reg_value_map> regmap_stack;
 459
 460         regmap_stack rm_stack;
 461         unsigned rm_stk_level;
 462
 463         value* prev_dst[5];
 464
 465 public:
 466
 467         ra_checker(shader &sh) : pass(sh), rm_stk_level(0), prev_dst() {}
 468
 469         virtual int run();
 470
 471         void run_on(container_node *c);
 472
 473         void dump_error(const error_info &e);
 474         void dump_all_errors();
 475
 476 private:
 477
 478         reg_value_map& rmap() { return rm_stack[rm_stk_level]; }
 479
 480         void push_stack();
 481         void pop_stack();
 482
 483         // when going out of the alu clause, values in the clause temporary gprs,
 484         // AR, predicate values, PS/PV are destroyed
 485         void kill_alu_only_regs();
 486         void error(node *n, unsigned id, std::string msg);
 487
 488         void check_phi_src(container_node *p, unsigned id);
 489         void process_phi_dst(container_node *p);
 490         void check_alu_group(alu_group_node *g);
 491         void process_op_dst(node *n);
 492         void check_op_src(node *n);
 493         void check_src_vec(node *n, unsigned id, vvec &vv, bool src);
 494         void check_value_gpr(node *n, unsigned id, value *v);
 495 };
 496
 497 // =======================================
 498
 499
 500 class ra_coalesce : public pass {
 501
 502 public:
 503
 504         ra_coalesce(shader &sh) : pass(sh) {}
 505
 506         virtual int run();
 507 };
 508
 509
 510
 511 // =======================================
 512
 513 class ra_init : public pass {
 514
 515 public:
 516
 517         ra_init(shader &sh) : pass(sh), prev_chans() {
 518
 519                 // The parameter below affects register channels distribution.
 520                 // For cayman (VLIW-4) we're trying to distribute the channels
 521                 // uniformly, this means significantly better alu slots utilization
 522                 // at the expense of higher gpr usage. Hopefully this will improve
 523                 // performance, though it has to be proven with real benchmarks yet.
 524                 // For VLIW-5 this method could also slightly improve slots
 525                 // utilization, but increased register pressure seems more significant
 526                 // and overall performance effect is negative according to some
 527                 // benchmarks, so it's not used currently. Basically, VLIW-5 doesn't
 528                 // really need it because trans slot (unrestricted by register write
 529                 // channel) allows to consume most deviations from uniform channel
 530                 // distribution.
 531                 // Value 3 means that for new allocation we'll use channel that differs
 532                 // from 3 last used channels. 0 for VLIW-5 effectively turns this off.
 533
 534                 ra_tune = sh.get_ctx().is_cayman() ? 3 : 0;
 535         }
 536
 537         virtual int run();
 538
 539 private:
 540
 541         unsigned prev_chans;
 542         unsigned ra_tune;
 543
 544         void add_prev_chan(unsigned chan);
 545         unsigned get_preferable_chan_mask();
 546
 547         void ra_node(container_node *c);
 548         void process_op(node *n);
 549
 550         void color(value *v);
 551
 552         void color_bs_constraint(ra_constraint *c);
 553
 554         void assign_color(value *v, sel_chan c);
 555         void alloc_arrays();
 556 };
 557
 558 // =======================================
 559
 560 class ra_split : public pass {
 561
 562 public:
 563
 564         ra_split(shader &sh) : pass(sh) {}
 565
 566         virtual int run();
 567
 568         void split(container_node *n);
 569         void split_op(node *n);
 570         void split_alu_packed(alu_packed_node *n);
 571         void split_vector_inst(node *n);
 572
 573         void split_packed_ins(alu_packed_node *n);
 574
 575 #if 0
 576         void split_pinned_outs(node *n);
 577 #endif
 578
 579         void split_vec(vvec &vv, vvec &v1, vvec &v2, bool allow_swz);
 580
 581         void split_phi_src(container_node *loc, container_node *c, unsigned id,
 582                            bool loop);
 583         void split_phi_dst(node *loc, container_node *c, bool loop);
 584         void init_phi_constraints(container_node *c);
 585 };
 586
 587
 588
 589 class ssa_prepare : public vpass {
 590         using vpass::visit;
 591
 592         typedef std::vector<val_set> vd_stk;
 593         vd_stk stk;
 594
 595         unsigned level;
 596
 597 public:
 598         ssa_prepare(shader &s) : vpass(s), level(0) {}
 599
 600         virtual bool visit(cf_node &n, bool enter);
 601         virtual bool visit(alu_node &n, bool enter);
 602         virtual bool visit(fetch_node &n, bool enter);
 603         virtual bool visit(region_node &n, bool enter);
 604         virtual bool visit(repeat_node &n, bool enter);
 605         virtual bool visit(depart_node &n, bool enter);
 606
 607 private:
 608
 609         void push_stk() {
 610                 ++level;
 611                 if (level + 1 > stk.size())
 612                         stk.resize(level+1);
 613                 else
 614                         stk[level].clear();
 615         }
 616         void pop_stk() {
 617                 assert(level);
 618                 --level;
 619                 stk[level].add_set(stk[level + 1]);
 620         }
 621
 622         void add_defs(node &n);
 623
 624         val_set & cur_set() { return stk[level]; }
 625
 626         container_node* create_phi_nodes(int count);
 627 };
 628
 629 class ssa_rename : public vpass {
 630         using vpass::visit;
 631
 632         typedef sb_map<value*, unsigned> def_map;
 633
 634         def_map def_count;
 635         std::stack<def_map> rename_stack;
 636
 637         typedef std::map<uint32_t, value*> val_map;
 638         val_map values;
 639
 640 public:
 641
 642         ssa_rename(shader &s) : vpass(s) {}
 643
 644         virtual int init();
 645
 646         virtual bool visit(container_node &n, bool enter);
 647         virtual bool visit(node &n, bool enter);
 648         virtual bool visit(alu_group_node &n, bool enter);
 649         virtual bool visit(cf_node &n, bool enter);
 650         virtual bool visit(alu_node &n, bool enter);
 651         virtual bool visit(alu_packed_node &n, bool enter);
 652         virtual bool visit(fetch_node &n, bool enter);
 653         virtual bool visit(region_node &n, bool enter);
 654         virtual bool visit(repeat_node &n, bool enter);
 655         virtual bool visit(depart_node &n, bool enter);
 656         virtual bool visit(if_node &n, bool enter);
 657
 658 private:
 659
 660         void push(node *phi);
 661         void pop();
 662
 663         unsigned get_index(def_map& m, value* v);
 664         void set_index(def_map& m, value* v, unsigned index);
 665         unsigned new_index(def_map& m, value* v);
 666
 667         value* rename_use(node *n, value* v);
 668         value* rename_def(node *def, value* v);
 669
 670         void rename_src_vec(node *n, vvec &vv, bool src);
 671         void rename_dst_vec(node *def, vvec &vv, bool set_def);
 672
 673         void rename_src(node *n);
 674         void rename_dst(node *n);
 675
 676         void rename_phi_args(container_node *phi, unsigned op, bool def);
 677
 678         void rename_virt(node *n);
 679         void rename_virt_val(node *n, value *v);
 680 };
 681
 682 class bc_finalizer : public pass {
 683
 684         cf_node *last_export[EXP_TYPE_COUNT];
 685         cf_node *last_cf;
 686
 687         unsigned ngpr;
 688         unsigned nstack;
 689
 690 public:
 691
 692         bc_finalizer(shader &sh) : pass(sh), last_export(), last_cf(), ngpr(),
 693                 nstack() {}
 694
 695         virtual int run();
 696
 697         void finalize_loop(region_node *r);
 698         void finalize_if(region_node *r);
 699
 700         void run_on(container_node *c);
 701
 702         void insert_rv6xx_load_ar_workaround(alu_group_node *b4);
 703         void finalize_alu_group(alu_group_node *g, node *prev_node);
 704         bool finalize_alu_src(alu_group_node *g, alu_node *a, alu_group_node *prev_node);
 705
 706         void emit_set_grad(fetch_node* f);
 707         void finalize_fetch(fetch_node *f);
 708
 709         void finalize_cf(cf_node *c);
 710
 711         sel_chan translate_kcache(cf_node *alu, value *v);
 712
 713         void update_ngpr(unsigned gpr);
 714         void update_nstack(region_node *r, unsigned add = 0);
 715
 716         unsigned get_stack_depth(node *n, unsigned &loops, unsigned &ifs,
 717                                  unsigned add = 0);
 718
 719         void cf_peephole();
 720
 721 private:
 722         void copy_fetch_src(fetch_node &dst, fetch_node &src, unsigned arg_start);
 723         void emit_set_texture_offsets(fetch_node &f);
 724 };
 725
 726
 727 } // namespace r600_sb
 728
 729 #endif /* SB_PASS_H_ */