src/mesa/drivers/dri/i965/brw_fs_reg_allocate.cpp

   1 /*
   2  * Copyright © 2010 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  *
  23  * Authors:
  24  *    Eric Anholt <eric@anholt.net>
  25  *
  26  */
  27
  28 #include "brw_fs.h"
  29 #include "glsl/glsl_types.h"
  30 #include "glsl/ir_optimization.h"
  31 #include "glsl/ir_print_visitor.h"
  32
  33 static void
  34 assign_reg(int *reg_hw_locations, fs_reg *reg, int reg_width)
  35 {
  36    if (reg->file == GRF) {
  37       assert(reg->reg_offset >= 0);
  38       reg->reg = reg_hw_locations[reg->reg] + reg->reg_offset * reg_width;
  39       reg->reg_offset = 0;
  40    }
  41 }
  42
  43 void
  44 fs_visitor::assign_regs_trivial()
  45 {
  46    int hw_reg_mapping[this->virtual_grf_count + 1];
  47    int i;
  48    int reg_width = c->dispatch_width / 8;
  49
  50    /* Note that compressed instructions require alignment to 2 registers. */
  51    hw_reg_mapping[0] = ALIGN(this->first_non_payload_grf, reg_width);
  52    for (i = 1; i <= this->virtual_grf_count; i++) {
  53       hw_reg_mapping[i] = (hw_reg_mapping[i - 1] +
  54                            this->virtual_grf_sizes[i - 1] * reg_width);
  55    }
  56    this->grf_used = hw_reg_mapping[this->virtual_grf_count];
  57
  58    foreach_list(node, &this->instructions) {
  59       fs_inst *inst = (fs_inst *)node;
  60
  61       assign_reg(hw_reg_mapping, &inst->dst, reg_width);
  62       assign_reg(hw_reg_mapping, &inst->src[0], reg_width);
  63       assign_reg(hw_reg_mapping, &inst->src[1], reg_width);
  64       assign_reg(hw_reg_mapping, &inst->src[2], reg_width);
  65    }
  66
  67    if (this->grf_used >= max_grf) {
  68       fail("Ran out of regs on trivial allocator (%d/%d)\n",
  69            this->grf_used, max_grf);
  70    }
  71
  72 }
  73
  74 static void
  75 brw_alloc_reg_set_for_classes(struct brw_context *brw,
  76                               int *class_sizes,
  77                               int class_count,
  78                               int reg_width,
  79                               int base_reg_count)
  80 {
  81    struct intel_context *intel = &brw->intel;
  82
  83    /* Compute the total number of registers across all classes. */
  84    int ra_reg_count = 0;
  85    for (int i = 0; i < class_count; i++) {
  86       ra_reg_count += base_reg_count - (class_sizes[i] - 1);
  87    }
  88
  89    ralloc_free(brw->wm.ra_reg_to_grf);
  90    brw->wm.ra_reg_to_grf = ralloc_array(brw, uint8_t, ra_reg_count);
  91    ralloc_free(brw->wm.regs);
  92    brw->wm.regs = ra_alloc_reg_set(brw, ra_reg_count);
  93    ralloc_free(brw->wm.classes);
  94    brw->wm.classes = ralloc_array(brw, int, class_count + 1);
  95
  96    brw->wm.aligned_pairs_class = -1;
  97
  98    /* Now, add the registers to their classes, and add the conflicts
  99     * between them and the base GRF registers (and also each other).
 100     */
 101    int reg = 0;
 102    int pairs_base_reg = 0;
 103    int pairs_reg_count = 0;
 104    for (int i = 0; i < class_count; i++) {
 105       int class_reg_count = base_reg_count - (class_sizes[i] - 1);
 106       brw->wm.classes[i] = ra_alloc_reg_class(brw->wm.regs);
 107
 108       /* Save this off for the aligned pair class at the end. */
 109       if (class_sizes[i] == 2) {
 110          pairs_base_reg = reg;
 111          pairs_reg_count = class_reg_count;
 112       }
 113
 114       for (int j = 0; j < class_reg_count; j++) {
 115          ra_class_add_reg(brw->wm.regs, brw->wm.classes[i], reg);
 116
 117          brw->wm.ra_reg_to_grf[reg] = j;
 118
 119          for (int base_reg = j;
 120               base_reg < j + class_sizes[i];
 121               base_reg++) {
 122             ra_add_transitive_reg_conflict(brw->wm.regs, base_reg, reg);
 123          }
 124
 125          reg++;
 126       }
 127    }
 128    assert(reg == ra_reg_count);
 129
 130    /* Add a special class for aligned pairs, which we'll put delta_x/y
 131     * in on gen5 so that we can do PLN.
 132     */
 133    if (brw->has_pln && reg_width == 1 && intel->gen < 6) {
 134       brw->wm.aligned_pairs_class = ra_alloc_reg_class(brw->wm.regs);
 135
 136       for (int i = 0; i < pairs_reg_count; i++) {
 137          if ((brw->wm.ra_reg_to_grf[pairs_base_reg + i] & 1) == 0) {
 138             ra_class_add_reg(brw->wm.regs, brw->wm.aligned_pairs_class,
 139                              pairs_base_reg + i);
 140          }
 141       }
 142       class_count++;
 143    }
 144
 145    ra_set_finalize(brw->wm.regs);
 146 }
 147
 148 bool
 149 fs_visitor::assign_regs()
 150 {
 151    /* Most of this allocation was written for a reg_width of 1
 152     * (dispatch_width == 8).  In extending to 16-wide, the code was
 153     * left in place and it was converted to have the hardware
 154     * registers it's allocating be contiguous physical pairs of regs
 155     * for reg_width == 2.
 156     */
 157    int reg_width = c->dispatch_width / 8;
 158    int hw_reg_mapping[this->virtual_grf_count];
 159    int first_assigned_grf = ALIGN(this->first_non_payload_grf, reg_width);
 160    int base_reg_count = (max_grf - first_assigned_grf) / reg_width;
 161    int class_sizes[base_reg_count];
 162    int class_count = 0;
 163
 164    calculate_live_intervals();
 165
 166    /* Set up the register classes.
 167     *
 168     * The base registers store a scalar value.  For texture samples,
 169     * we get virtual GRFs composed of 4 contiguous hw register.  For
 170     * structures and arrays, we store them as contiguous larger things
 171     * than that, though we should be able to do better most of the
 172     * time.
 173     */
 174    class_sizes[class_count++] = 1;
 175    if (brw->has_pln && intel->gen < 6) {
 176       /* Always set up the (unaligned) pairs for gen5, so we can find
 177        * them for making the aligned pair class.
 178        */
 179       class_sizes[class_count++] = 2;
 180    }
 181    for (int r = 0; r < this->virtual_grf_count; r++) {
 182       int i;
 183
 184       for (i = 0; i < class_count; i++) {
 185          if (class_sizes[i] == this->virtual_grf_sizes[r])
 186             break;
 187       }
 188       if (i == class_count) {
 189          if (this->virtual_grf_sizes[r] >= base_reg_count) {
 190             fail("Object too large to register allocate.\n");
 191          }
 192
 193          class_sizes[class_count++] = this->virtual_grf_sizes[r];
 194       }
 195    }
 196
 197    brw_alloc_reg_set_for_classes(brw, class_sizes, class_count,
 198                                  reg_width, base_reg_count);
 199
 200    struct ra_graph *g = ra_alloc_interference_graph(brw->wm.regs,
 201                                                     this->virtual_grf_count);
 202
 203    for (int i = 0; i < this->virtual_grf_count; i++) {
 204       for (int c = 0; c < class_count; c++) {
 205          if (class_sizes[c] == this->virtual_grf_sizes[i]) {
 206             /* Special case: on pre-GEN6 hardware that supports PLN, the
 207              * second operand of a PLN instruction needs to be an
 208              * even-numbered register, so we have a special register class
 209              * wm_aligned_pairs_class to handle this case.  pre-GEN6 always
 210              * uses this->delta_x[BRW_WM_PERSPECTIVE_PIXEL_BARYCENTRIC] as the
 211              * second operand of a PLN instruction (since it doesn't support
 212              * any other interpolation modes).  So all we need to do is find
 213              * that register and set it to the appropriate class.
 214              */
 215             if (brw->wm.aligned_pairs_class >= 0 &&
 216                 this->delta_x[BRW_WM_PERSPECTIVE_PIXEL_BARYCENTRIC].reg == i) {
 217                ra_set_node_class(g, i, brw->wm.aligned_pairs_class);
 218             } else {
 219                ra_set_node_class(g, i, brw->wm.classes[c]);
 220             }
 221             break;
 222          }
 223       }
 224
 225       for (int j = 0; j < i; j++) {
 226          if (virtual_grf_interferes(i, j)) {
 227             ra_add_node_interference(g, i, j);
 228          }
 229       }
 230    }
 231
 232    if (!ra_allocate_no_spills(g)) {
 233       /* Failed to allocate registers.  Spill a reg, and the caller will
 234        * loop back into here to try again.
 235        */
 236       int reg = choose_spill_reg(g);
 237
 238       if (reg == -1) {
 239          fail("no register to spill\n");
 240       } else if (c->dispatch_width == 16) {
 241          fail("no spilling support on 16-wide yet\n");
 242       } else {
 243          spill_reg(reg);
 244       }
 245
 246
 247       ralloc_free(g);
 248
 249       return false;
 250    }
 251
 252    /* Get the chosen virtual registers for each node, and map virtual
 253     * regs in the register classes back down to real hardware reg
 254     * numbers.
 255     */
 256    this->grf_used = first_assigned_grf;
 257    for (int i = 0; i < this->virtual_grf_count; i++) {
 258       int reg = ra_get_node_reg(g, i);
 259
 260       hw_reg_mapping[i] = (first_assigned_grf +
 261                            brw->wm.ra_reg_to_grf[reg] * reg_width);
 262       this->grf_used = MAX2(this->grf_used,
 263                             hw_reg_mapping[i] + this->virtual_grf_sizes[i] *
 264                             reg_width);
 265    }
 266
 267    foreach_list(node, &this->instructions) {
 268       fs_inst *inst = (fs_inst *)node;
 269
 270       assign_reg(hw_reg_mapping, &inst->dst, reg_width);
 271       assign_reg(hw_reg_mapping, &inst->src[0], reg_width);
 272       assign_reg(hw_reg_mapping, &inst->src[1], reg_width);
 273       assign_reg(hw_reg_mapping, &inst->src[2], reg_width);
 274    }
 275
 276    ralloc_free(g);
 277
 278    return true;
 279 }
 280
 281 void
 282 fs_visitor::emit_unspill(fs_inst *inst, fs_reg dst, uint32_t spill_offset)
 283 {
 284    fs_inst *unspill_inst = new(mem_ctx) fs_inst(FS_OPCODE_UNSPILL, dst);
 285    unspill_inst->offset = spill_offset;
 286    unspill_inst->ir = inst->ir;
 287    unspill_inst->annotation = inst->annotation;
 288
 289    /* Choose a MRF that won't conflict with an MRF that's live across the
 290     * spill.  Nothing else will make it up to MRF 14/15.
 291     */
 292    unspill_inst->base_mrf = 14;
 293    unspill_inst->mlen = 1; /* header contains offset */
 294    inst->insert_before(unspill_inst);
 295 }
 296
 297 int
 298 fs_visitor::choose_spill_reg(struct ra_graph *g)
 299 {
 300    float loop_scale = 1.0;
 301    float spill_costs[this->virtual_grf_count];
 302    bool no_spill[this->virtual_grf_count];
 303
 304    for (int i = 0; i < this->virtual_grf_count; i++) {
 305       spill_costs[i] = 0.0;
 306       no_spill[i] = false;
 307    }
 308
 309    /* Calculate costs for spilling nodes.  Call it a cost of 1 per
 310     * spill/unspill we'll have to do, and guess that the insides of
 311     * loops run 10 times.
 312     */
 313    foreach_list(node, &this->instructions) {
 314       fs_inst *inst = (fs_inst *)node;
 315
 316       for (unsigned int i = 0; i < 3; i++) {
 317          if (inst->src[i].file == GRF) {
 318             spill_costs[inst->src[i].reg] += loop_scale;
 319          }
 320       }
 321
 322       if (inst->dst.file == GRF) {
 323          spill_costs[inst->dst.reg] += inst->regs_written() * loop_scale;
 324       }
 325
 326       switch (inst->opcode) {
 327
 328       case BRW_OPCODE_DO:
 329          loop_scale *= 10;
 330          break;
 331
 332       case BRW_OPCODE_WHILE:
 333          loop_scale /= 10;
 334          break;
 335
 336       case FS_OPCODE_SPILL:
 337          if (inst->src[0].file == GRF)
 338             no_spill[inst->src[0].reg] = true;
 339          break;
 340
 341       case FS_OPCODE_UNSPILL:
 342          if (inst->dst.file == GRF)
 343             no_spill[inst->dst.reg] = true;
 344          break;
 345
 346       default:
 347          break;
 348       }
 349    }
 350
 351    for (int i = 0; i < this->virtual_grf_count; i++) {
 352       if (!no_spill[i])
 353          ra_set_node_spill_cost(g, i, spill_costs[i]);
 354    }
 355
 356    return ra_get_best_spill_node(g);
 357 }
 358
 359 void
 360 fs_visitor::spill_reg(int spill_reg)
 361 {
 362    int size = virtual_grf_sizes[spill_reg];
 363    unsigned int spill_offset = c->last_scratch;
 364    assert(ALIGN(spill_offset, 16) == spill_offset); /* oword read/write req. */
 365    c->last_scratch += size * REG_SIZE;
 366
 367    /* Generate spill/unspill instructions for the objects being
 368     * spilled.  Right now, we spill or unspill the whole thing to a
 369     * virtual grf of the same size.  For most instructions, though, we
 370     * could just spill/unspill the GRF being accessed.
 371     */
 372    foreach_list(node, &this->instructions) {
 373       fs_inst *inst = (fs_inst *)node;
 374
 375       for (unsigned int i = 0; i < 3; i++) {
 376          if (inst->src[i].file == GRF &&
 377              inst->src[i].reg == spill_reg) {
 378             inst->src[i].reg = virtual_grf_alloc(1);
 379             emit_unspill(inst, inst->src[i],
 380                          spill_offset + REG_SIZE * inst->src[i].reg_offset);
 381          }
 382       }
 383
 384       if (inst->dst.file == GRF &&
 385           inst->dst.reg == spill_reg) {
 386          int subset_spill_offset = (spill_offset +
 387                                     REG_SIZE * inst->dst.reg_offset);
 388          inst->dst.reg = virtual_grf_alloc(inst->regs_written());
 389          inst->dst.reg_offset = 0;
 390
 391          /* If our write is going to affect just part of the
 392           * inst->regs_written(), then we need to unspill the destination
 393           * since we write back out all of the regs_written().
 394           */
 395          if (inst->predicated || inst->force_uncompressed || inst->force_sechalf) {
 396             fs_reg unspill_reg = inst->dst;
 397             for (int chan = 0; chan < inst->regs_written(); chan++) {
 398                emit_unspill(inst, unspill_reg,
 399                             subset_spill_offset + REG_SIZE * chan);
 400                unspill_reg.reg_offset++;
 401             }
 402          }
 403
 404          fs_reg spill_src = inst->dst;
 405          spill_src.reg_offset = 0;
 406          spill_src.abs = false;
 407          spill_src.negate = false;
 408          spill_src.smear = -1;
 409
 410          for (int chan = 0; chan < inst->regs_written(); chan++) {
 411             fs_inst *spill_inst = new(mem_ctx) fs_inst(FS_OPCODE_SPILL,
 412                                                        reg_null_f, spill_src);
 413             spill_src.reg_offset++;
 414             spill_inst->offset = subset_spill_offset + chan * REG_SIZE;
 415             spill_inst->ir = inst->ir;
 416             spill_inst->annotation = inst->annotation;
 417             spill_inst->base_mrf = 14;
 418             spill_inst->mlen = 2; /* header, value */
 419             inst->insert_after(spill_inst);
 420          }
 421       }
 422    }
 423
 424    this->live_intervals_valid = false;
 425 }