src/gallium/drivers/r600/sb/sb_ra_init.cpp

   1 /*
   2  * Copyright 2013 Vadim Girlin <vadimgirlin@gmail.com>
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * on the rights to use, copy, modify, merge, publish, distribute, sub
   8  * license, and/or sell copies of the Software, and to permit persons to whom
   9  * the Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
  18  * THE AUTHOR(S) AND/OR THEIR SUPPLIERS BE LIABLE FOR ANY CLAIM,
  19  * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
  20  * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
  21  * USE OR OTHER DEALINGS IN THE SOFTWARE.
  22  *
  23  * Authors:
  24  *      Vadim Girlin
  25  */
  26
  27 #define RA_DEBUG 0
  28
  29 #if RA_DEBUG
  30 #define RA_DUMP(q) do { q } while (0)
  31 #else
  32 #define RA_DUMP(q)
  33 #endif
  34
  35 #include <cstring>
  36 #include <iostream>
  37 #include <iomanip>
  38
  39 #include "sb_bc.h"
  40 #include "sb_shader.h"
  41
  42 #include "sb_pass.h"
  43
  44 namespace r600_sb {
  45
  46 using std::cerr;
  47
  48 class regbits {
  49         typedef uint32_t basetype;
  50         static const unsigned bt_bytes = sizeof(basetype);
  51         static const unsigned bt_index_shift = 5;
  52         static const unsigned bt_index_mask = (1u << bt_index_shift) - 1;
  53         static const unsigned bt_bits = bt_bytes << 3;
  54         static const unsigned size = MAX_GPR * 4 / bt_bits;
  55
  56         basetype dta[size];
  57
  58         unsigned num_temps;
  59
  60 public:
  61
  62         regbits(unsigned num_temps) : dta(), num_temps(num_temps) {}
  63         regbits(unsigned num_temps, unsigned value)     : num_temps(num_temps)
  64         { set_all(value); }
  65
  66         regbits(shader &sh, val_set &vs) : num_temps(sh.get_ctx().alu_temp_gprs)
  67         { set_all(1); from_val_set(sh, vs); }
  68
  69         void set_all(unsigned val);
  70         void from_val_set(shader &sh, val_set &vs);
  71
  72         void set(unsigned index);
  73         void clear(unsigned index);
  74         bool get(unsigned index);
  75
  76         void set(unsigned index, unsigned val);
  77
  78         sel_chan find_free_bit();
  79         sel_chan find_free_chans(unsigned mask);
  80         sel_chan find_free_array(unsigned size, unsigned mask);
  81
  82         void dump();
  83 };
  84
  85 // =======================================
  86
  87 void regbits::dump() {
  88         for (unsigned i = 0; i < size * bt_bits; ++i) {
  89
  90                 if (!(i & 31))
  91                         cerr << "\n";
  92
  93                 if (!(i & 3))
  94                         cerr << "    " << std::setw(3) << (i / 4) << " ";
  95
  96                 cerr << (get(i) ? 1 : 0);
  97         }
  98 }
  99
 100
 101 void regbits::set_all(unsigned v) {
 102         memset(&dta, v ? 0xFF : 0x00, size * bt_bytes);
 103 }
 104
 105 void regbits::from_val_set(shader &sh, val_set& vs) {
 106         val_set &s = vs;
 107         unsigned g;
 108         for (val_set::iterator I = s.begin(sh), E = s.end(sh); I != E; ++I) {
 109                 value *v = *I;
 110                 if (v->is_any_gpr()) {
 111                         g = v->get_final_gpr();
 112                         if (!g)
 113                                 continue;
 114                 } else
 115                         continue;
 116
 117                 assert(g);
 118                 --g;
 119                 assert(g < 512);
 120                 clear(g);
 121         }
 122 }
 123
 124 void regbits::set(unsigned index) {
 125         unsigned ih = index >> bt_index_shift;
 126         unsigned il = index & bt_index_mask;
 127         dta[ih] |= ((basetype)1u << il);
 128 }
 129
 130 void regbits::clear(unsigned index) {
 131         unsigned ih = index >> bt_index_shift;
 132         unsigned il = index & bt_index_mask;
 133         assert(ih < size);
 134         dta[ih] &= ~((basetype)1u << il);
 135 }
 136
 137 bool regbits::get(unsigned index) {
 138         unsigned ih = index >> bt_index_shift;
 139         unsigned il = index & bt_index_mask;
 140         return dta[ih] & ((basetype)1u << il);
 141 }
 142
 143 void regbits::set(unsigned index, unsigned val) {
 144         unsigned ih = index >> bt_index_shift;
 145         unsigned il = index & bt_index_mask;
 146         basetype bm = 1u << il;
 147         dta[ih] = (dta[ih] & ~bm) | (val << il);
 148 }
 149
 150 // free register for ra means the bit is set
 151 sel_chan regbits::find_free_bit() {
 152         unsigned elt = 0;
 153         unsigned bit = 0;
 154
 155         while (elt < size && !dta[elt])
 156                 ++elt;
 157
 158         if (elt >= size)
 159                 return 0;
 160
 161         bit = __builtin_ctz(dta[elt]) + (elt << bt_index_shift);
 162
 163         assert(bit < MAX_GPR - num_temps);
 164
 165         return bit + 1;
 166 }
 167
 168 // find free gpr component to use as indirectly addressable array
 169 sel_chan regbits::find_free_array(unsigned length, unsigned mask) {
 170         unsigned cc[4] = {};
 171
 172         // FIXME optimize this. though hopefully we won't have a lot of arrays
 173         for (unsigned a = 0; a < MAX_GPR - num_temps; ++a) {
 174                 for(unsigned c = 0; c < MAX_CHAN; ++c) {
 175                         if (mask & (1 << c)) {
 176                                 if (get((a << 2) | c)) {
 177                                         if (++cc[c] == length)
 178                                                 return sel_chan(a - length + 1, c);
 179                                 } else {
 180                                         cc[c] = 0;
 181                                 }
 182                         }
 183                 }
 184         }
 185         return 0;
 186 }
 187
 188 sel_chan regbits::find_free_chans(unsigned mask) {
 189         unsigned elt = 0;
 190         unsigned bit = 0;
 191
 192         basetype cd = dta[elt] >> bit;
 193
 194         do {
 195
 196                 if (!cd) {
 197                         if (++elt < size)
 198                                 cd = dta[elt];
 199                         else
 200                                 return 0;
 201
 202                         bit = 0;
 203                 }
 204
 205                 unsigned p = __builtin_ctz(cd) & ~(basetype)3u;
 206
 207                 if (p > bt_bits - bit) {
 208                         if (++elt < size)
 209                                 cd = dta[elt];
 210                         else
 211                                 return 0;
 212                         bit = 0;
 213                 }
 214
 215                 bit += p;
 216                 cd >>= p;
 217
 218                 if ((cd & mask) == mask) {
 219                         return ((elt << bt_index_shift) | bit) + 1;
 220                 }
 221
 222                 bit += 4;
 223                 cd >>= 4;
 224
 225         } while (1);
 226
 227         return 0;
 228 }
 229
 230 // ================================
 231
 232 void ra_init::alloc_arrays() {
 233
 234         gpr_array_vec &ga = sh.arrays();
 235
 236         for(gpr_array_vec::iterator I = ga.begin(), E = ga.end(); I != E; ++I) {
 237                 gpr_array *a = *I;
 238
 239                 RA_DUMP(
 240                         cerr << "array [" << a->array_size << "] at " << a->base_gpr << "\n";
 241                         cerr << "\n";
 242                 );
 243
 244                 // skip preallocated arrays (e.g. with preloaded inputs)
 245                 if (a->gpr) {
 246                         RA_DUMP( cerr << "   FIXED at " << a->gpr << "\n"; );
 247                         continue;
 248                 }
 249
 250                 bool dead = a->is_dead();
 251
 252                 if (dead) {
 253                         RA_DUMP( cerr << "   DEAD\n"; );
 254                         continue;
 255                 }
 256
 257                 val_set &s = a->interferences;
 258
 259
 260                 for (val_set::iterator I = s.begin(sh), E = s.end(sh); I != E; ++I) {
 261                         value *v = *I;
 262                         if (v->array == a)
 263                                 s.remove_val(v);
 264                 }
 265
 266                 RA_DUMP(
 267                         cerr << "  interf: ";
 268                         dump::dump_set(sh, s);
 269                         cerr << "\n";
 270                 );
 271
 272                 regbits rb(sh, s);
 273
 274                 sel_chan base = rb.find_free_array(a->array_size,
 275                                                    (1 << a->base_gpr.chan()));
 276
 277                 RA_DUMP( cerr << "  found base: " << base << "\n"; );
 278
 279                 a->gpr = base;
 280         }
 281 }
 282
 283
 284 int ra_init::run() {
 285
 286         alloc_arrays();
 287
 288         ra_node(sh.root);
 289         return 0;
 290 }
 291
 292 void ra_init::ra_node(container_node* c) {
 293
 294         for (node_iterator I = c->begin(), E = c->end(); I != E; ++I) {
 295                 node *n = *I;
 296                 if (n->type == NT_OP) {
 297                         process_op(n);
 298                 }
 299                 if (n->is_container() && !n->is_alu_packed()) {
 300                         ra_node(static_cast<container_node*>(n));
 301                 }
 302         }
 303 }
 304
 305 void ra_init::process_op(node* n) {
 306
 307         bool copy = n->is_copy_mov();
 308
 309         RA_DUMP(
 310                 cerr << "ra_init: process_op : ";
 311                 dump::dump_op(n);
 312                 cerr << "\n";
 313         );
 314
 315         if (n->is_alu_packed()) {
 316                 for (vvec::iterator I = n->src.begin(), E = n->src.end(); I != E; ++I) {
 317                         value *v = *I;
 318                         if (v && v->is_sgpr() && v->constraint &&
 319                                         v->constraint->kind == CK_PACKED_BS) {
 320                                 color_bs_constraint(v->constraint);
 321                                 break;
 322                         }
 323                 }
 324         }
 325
 326         if (n->is_fetch_inst() || n->is_cf_inst()) {
 327                 for (vvec::iterator I = n->src.begin(), E = n->src.end(); I != E; ++I) {
 328                         value *v = *I;
 329                         if (v && v->is_sgpr())
 330                                 color(v);
 331                 }
 332         }
 333
 334         for (vvec::iterator I = n->dst.begin(), E = n->dst.end(); I != E; ++I) {
 335                 value *v = *I;
 336                 if (!v)
 337                         continue;
 338                 if (v->is_sgpr()) {
 339                         if (!v->gpr) {
 340                                 if (copy && !v->constraint) {
 341                                         value *s = *(n->src.begin() + (I - n->dst.begin()));
 342                                         assert(s);
 343                                         if (s->is_sgpr()) {
 344                                                 assign_color(v, s->gpr);
 345                                         }
 346                                 } else
 347                                         color(v);
 348                         }
 349                 }
 350         }
 351 }
 352
 353 void ra_init::color_bs_constraint(ra_constraint* c) {
 354         vvec &vv = c->values;
 355         assert(vv.size() <= 8);
 356
 357         RA_DUMP(
 358                 cerr << "color_bs_constraint: ";
 359                 dump::dump_vec(vv);
 360                 cerr << "\n";
 361         );
 362
 363         regbits rb(ctx.alu_temp_gprs);
 364
 365         unsigned chan_count[4] = {};
 366         unsigned allowed_chans = 0x0F;
 367
 368         for (vvec::iterator I = vv.begin(), E = vv.end(); I != E; ++I) {
 369                 value *v = *I;
 370                 sel_chan gpr = v->get_final_gpr();
 371
 372                 if (!v || v->is_dead())
 373                         continue;
 374
 375                 val_set interf;
 376
 377                 if (v->chunk)
 378                         sh.coal.get_chunk_interferences(v->chunk, interf);
 379                 else
 380                         interf = v->interferences;
 381
 382                 RA_DUMP(
 383                         cerr << "   processing " << *v << "  interferences : ";
 384                         dump::dump_set(sh, interf);
 385                         cerr << "\n";
 386                 );
 387
 388                 if (gpr) {
 389                         unsigned chan = gpr.chan();
 390                         if (chan_count[chan] < 3) {
 391                                 ++chan_count[chan];
 392                                 continue;
 393                         } else {
 394                                 v->flags &= ~VLF_FIXED;
 395                                 allowed_chans &= ~(1 << chan);
 396                                 assert(allowed_chans);
 397                         }
 398                 }
 399
 400                 v->gpr = 0;
 401
 402                 gpr = 1;
 403                 rb.set_all(1);
 404
 405
 406                 rb.from_val_set(sh, interf);
 407
 408                 RA_DUMP(
 409                         cerr << "   regbits : ";
 410                         rb.dump();
 411                         cerr << "\n";
 412                 );
 413
 414                 while (allowed_chans && gpr.sel() < sh.num_nontemp_gpr()) {
 415
 416                         while (rb.get(gpr - 1) == 0)
 417                                 gpr = gpr + 1;
 418
 419                         RA_DUMP(
 420                                 cerr << "    trying " << gpr << "\n";
 421                         );
 422
 423                         unsigned chan = gpr.chan();
 424                         if (chan_count[chan] < 3) {
 425                                 ++chan_count[chan];
 426
 427                                 if (v->chunk) {
 428                                         vvec::iterator F = std::find(v->chunk->values.begin(),
 429                                                                      v->chunk->values.end(),
 430                                                                      v);
 431                                         v->chunk->values.erase(F);
 432                                         v->chunk = NULL;
 433                                 }
 434
 435                                 assign_color(v, gpr);
 436                                 break;
 437                         } else {
 438                                 allowed_chans &= ~(1 << chan);
 439                         }
 440                         gpr = gpr + 1;
 441                 }
 442
 443                 if (!gpr) {
 444                         cerr << "color_bs_constraint: failed...\n";
 445                         assert(!"coloring failed");
 446                 }
 447         }
 448 }
 449
 450 void ra_init::color(value* v) {
 451
 452         if (v->constraint && v->constraint->kind == CK_PACKED_BS) {
 453                 color_bs_constraint(v->constraint);
 454                 return;
 455         }
 456
 457         if (v->chunk && v->chunk->is_fixed())
 458                 return;
 459
 460         RA_DUMP(
 461                 cerr << "coloring ";
 462                 dump::dump_val(v);
 463                 cerr << "   interferences ";
 464                 dump::dump_set(sh, v->interferences);
 465                 cerr << "\n";
 466         );
 467
 468         if (v->is_reg_pinned()) {
 469                 assert(v->is_chan_pinned());
 470                 assign_color(v, v->pin_gpr);
 471                 return;
 472         }
 473
 474         regbits rb(sh, v->interferences);
 475         sel_chan c;
 476
 477         if (v->is_chan_pinned()) {
 478                 RA_DUMP( cerr << "chan_pinned = " << v->pin_gpr.chan() << "  "; );
 479                 unsigned mask = 1 << v->pin_gpr.chan();
 480                 c = rb.find_free_chans(mask) + v->pin_gpr.chan();
 481         } else {
 482                 c = rb.find_free_bit();
 483         }
 484
 485         assert(c && c.sel() < 128 - ctx.alu_temp_gprs && "color failed");
 486         assign_color(v, c);
 487 }
 488
 489 void ra_init::assign_color(value* v, sel_chan c) {
 490         v->gpr = c;
 491         RA_DUMP(
 492                 cerr << "colored ";
 493                 dump::dump_val(v);
 494                 cerr << " to " << c << "\n";
 495         );
 496 }
 497
 498 // ===================================================
 499
 500 int ra_split::run() {
 501         split(sh.root);
 502         return 0;
 503 }
 504
 505 void ra_split::split_phi_src(container_node *loc, container_node *c,
 506                              unsigned id, bool loop) {
 507         for (node_iterator I = c->begin(), E = c->end(); I != E; ++I) {
 508                 node *p = *I;
 509                 value* &v = p->src[id], *d = p->dst[0];
 510                 assert(v);
 511
 512                 if (!d->is_sgpr() || v->is_undef())
 513                         continue;
 514
 515                 value *t = sh.create_temp_value();
 516                 if (loop && id == 0)
 517                         loc->insert_before(sh.create_copy_mov(t, v));
 518                 else
 519                         loc->push_back(sh.create_copy_mov(t, v));
 520                 v = t;
 521
 522                 sh.coal.add_edge(v, d, coalescer::phi_cost);
 523         }
 524 }
 525
 526 void ra_split::split_phi_dst(node* loc, container_node *c, bool loop) {
 527         for (node_iterator I = c->begin(), E = c->end(); I != E; ++I) {
 528                 node *p = *I;
 529                 value* &v = p->dst[0];
 530                 assert(v);
 531
 532                 if (!v->is_sgpr())
 533                         continue;
 534
 535                 value *t = sh.create_temp_value();
 536                 node *cp = sh.create_copy_mov(v, t);
 537                 if (loop)
 538                         static_cast<container_node*>(loc)->push_front(cp);
 539                 else
 540                         loc->insert_after(cp);
 541                 v = t;
 542         }
 543 }
 544
 545
 546 void ra_split::init_phi_constraints(container_node *c) {
 547         for (node_iterator I = c->begin(), E = c->end(); I != E; ++I) {
 548                 node *p = *I;
 549                 ra_constraint *cc = sh.coal.create_constraint(CK_PHI);
 550                 cc->values.push_back(p->dst[0]);
 551
 552                 for (vvec::iterator I = p->src.begin(), E = p->src.end(); I != E; ++I) {
 553                         value *v = *I;
 554                         if (v->is_sgpr())
 555                                 cc->values.push_back(v);
 556                 }
 557
 558                 cc->update_values();
 559         }
 560 }
 561
 562 void ra_split::split(container_node* n) {
 563
 564         if (n->type == NT_DEPART) {
 565                 depart_node *d = static_cast<depart_node*>(n);
 566                 if (d->target->phi)
 567                         split_phi_src(d, d->target->phi, d->dep_id, false);
 568         } else if (n->type == NT_REPEAT) {
 569                 repeat_node *r = static_cast<repeat_node*>(n);
 570                 if (r->target->loop_phi)
 571                         split_phi_src(r, r->target->loop_phi, r->rep_id, true);
 572         } else if (n->type == NT_REGION) {
 573                 region_node *r = static_cast<region_node*>(n);
 574                 if (r->phi) {
 575                         split_phi_dst(r, r->phi, false);
 576                 }
 577                 if (r->loop_phi) {
 578                         split_phi_dst(r->get_entry_code_location(), r->loop_phi,
 579                                         true);
 580                         split_phi_src(r, r->loop_phi, 0, true);
 581                 }
 582         }
 583
 584         for (node_riterator N, I = n->rbegin(), E = n->rend(); I != E; I = N) {
 585                 N = I;
 586                 ++N;
 587                 node *o = *I;
 588                 if (o->type == NT_OP) {
 589                         split_op(o);
 590                 } else if (o->is_container()) {
 591                         split(static_cast<container_node*>(o));
 592                 }
 593         }
 594
 595         if (n->type == NT_REGION) {
 596                 region_node *r = static_cast<region_node*>(n);
 597                 if (r->phi)
 598                         init_phi_constraints(r->phi);
 599                 if (r->loop_phi)
 600                         init_phi_constraints(r->loop_phi);
 601         }
 602 }
 603
 604 void ra_split::split_op(node* n) {
 605         switch(n->subtype) {
 606                 case NST_ALU_PACKED_INST:
 607                         split_alu_packed(static_cast<alu_packed_node*>(n));
 608                         break;
 609                 case NST_FETCH_INST:
 610                 case NST_CF_INST:
 611                         split_vector_inst(n);
 612                 default:
 613                         break;
 614         }
 615 }
 616
 617 void ra_split::split_packed_ins(alu_packed_node *n) {
 618         vvec vv = n->src;
 619         vvec sv, dv;
 620
 621         for (vvec::iterator I = vv.begin(), E = vv.end(); I != E; ++I) {
 622
 623                 value *&v = *I;
 624
 625                 if (v && v->is_any_gpr() && !v->is_undef()) {
 626
 627                         vvec::iterator F = std::find(sv.begin(), sv.end(), v);
 628                         value *t;
 629
 630                         if (F != sv.end()) {
 631                                 t = *(dv.begin() + (F - sv.begin()));
 632                         } else {
 633                                 t = sh.create_temp_value();
 634                                 sv.push_back(v);
 635                                 dv.push_back(t);
 636                         }
 637                         v = t;
 638                 }
 639         }
 640
 641         unsigned cnt = sv.size();
 642
 643         if (cnt > 0) {
 644                 n->src = vv;
 645                 for (vvec::iterator SI = sv.begin(), DI = dv.begin(), SE = sv.end();
 646                                 SI != SE; ++SI, ++DI) {
 647                         n->insert_before(sh.create_copy_mov(*DI, *SI));
 648                 }
 649
 650                 ra_constraint *c = sh.coal.create_constraint(CK_PACKED_BS);
 651                 c->values = dv;
 652                 c->update_values();
 653         }
 654 }
 655
 656 // TODO handle other packed ops for cayman
 657 void ra_split::split_alu_packed(alu_packed_node* n) {
 658         switch (n->op()) {
 659                 case ALU_OP2_DOT4:
 660                 case ALU_OP2_CUBE:
 661                         split_packed_ins(n);
 662                         break;
 663                 default:
 664                         break;
 665         }
 666 }
 667
 668 void ra_split::split_vec(vvec &vv, vvec &v1, vvec &v2, bool allow_swz) {
 669         unsigned ch = 0;
 670         for (vvec::iterator I = vv.begin(), E = vv.end(); I != E; ++I, ++ch) {
 671
 672                 value* &o = *I;
 673
 674                 if (o) {
 675
 676                         assert(!o->is_dead());
 677
 678                         if (o->is_undef())
 679                                 continue;
 680
 681                         if (allow_swz && o->is_float_0_or_1())
 682                                 continue;
 683
 684                         value *t;
 685                         vvec::iterator F =
 686                                         allow_swz ? find(v2.begin(), v2.end(), o) : v2.end();
 687
 688                         if (F != v2.end()) {
 689                                 t = *(v1.begin() + (F - v2.begin()));
 690                         } else {
 691                                 t = sh.create_temp_value();
 692
 693                                 if (!allow_swz) {
 694                                         t->flags |= VLF_PIN_CHAN;
 695                                         t->pin_gpr = sel_chan(0, ch);
 696                                 }
 697
 698                                 v2.push_back(o);
 699                                 v1.push_back(t);
 700                         }
 701                         o = t;
 702                 }
 703         }
 704 }
 705
 706 void ra_split::split_vector_inst(node* n) {
 707         ra_constraint *c;
 708
 709         bool call_fs = n->is_cf_op(CF_OP_CALL_FS);
 710         bool no_src_swizzle = n->is_cf_inst() && (n->cf_op_flags() & CF_MEM);
 711
 712         no_src_swizzle |= n->is_fetch_op(FETCH_OP_VFETCH) ||
 713                         n->is_fetch_op(FETCH_OP_SEMFETCH);
 714
 715         if (!n->src.empty() && !call_fs) {
 716
 717                 // we may have more than one source vector -
 718                 // fetch instructions with FF_USEGRAD have gradient values in
 719                 // src vectors 1 (src[4-7] and 2 (src[8-11])
 720
 721                 unsigned nvec = n->src.size() >> 2;
 722                 assert(nvec << 2 == n->src.size());
 723
 724                 for (unsigned nv = 0; nv < nvec; ++nv) {
 725                         vvec sv, tv, nsrc(4);
 726                         unsigned arg_start = nv << 2;
 727
 728                         std::copy(n->src.begin() + arg_start,
 729                                   n->src.begin() + arg_start + 4,
 730                                   nsrc.begin());
 731
 732                         split_vec(nsrc, tv, sv, !no_src_swizzle);
 733
 734                         unsigned cnt = sv.size();
 735
 736                         if (no_src_swizzle || cnt) {
 737
 738                                 std::copy(nsrc.begin(), nsrc.end(), n->src.begin() + arg_start);
 739
 740                                 for(unsigned i = 0, s = tv.size(); i < s; ++i) {
 741                                         n->insert_before(sh.create_copy_mov(tv[i], sv[i]));
 742                                 }
 743
 744                                 c = sh.coal.create_constraint(CK_SAME_REG);
 745                                 c->values = tv;
 746                                 c->update_values();
 747                         }
 748                 }
 749         }
 750
 751         if (!n->dst.empty()) {
 752                 vvec sv, tv, ndst = n->dst;
 753
 754                 split_vec(ndst, tv, sv, true);
 755
 756                 if (sv.size()) {
 757                         n->dst = ndst;
 758
 759                         node *lp = n;
 760                         for(unsigned i = 0, s = tv.size(); i < s; ++i) {
 761                                 lp->insert_after(sh.create_copy_mov(sv[i], tv[i]));
 762                                 lp = lp->next;
 763                         }
 764
 765                         if (call_fs) {
 766                                 for (unsigned i = 0, cnt = tv.size(); i < cnt; ++i) {
 767                                         value *v = tv[i];
 768                                         value *s = sv[i];
 769                                         if (!v)
 770                                                 continue;
 771
 772                                         v->flags |= VLF_PIN_REG | VLF_PIN_CHAN;
 773                                         s->flags &= ~(VLF_PIN_REG | VLF_PIN_CHAN);
 774                                         sel_chan sel;
 775
 776                                         if (s->is_rel()) {
 777                                                 assert(s->rel->is_const());
 778                                                 sel = sel_chan(s->select.sel() +
 779                                                                                  s->rel->get_const_value().u,
 780                                                              s->select.chan());
 781                                         } else
 782                                                 sel = s->select;
 783
 784                                         v->gpr = v->pin_gpr = sel;
 785                                         v->fix();
 786                                 }
 787                         } else {
 788                                 c = sh.coal.create_constraint(CK_SAME_REG);
 789                                 c->values = tv;
 790                                 c->update_values();
 791                         }
 792                 }
 793         }
 794 }
 795
 796 } // namespace r600_sb