src/mesa/drivers/dri/i965/brw_fs_reg_allocate.cpp

   1 /*
   2  * Copyright © 2010 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  *
  23  * Authors:
  24  *    Eric Anholt <eric@anholt.net>
  25  *
  26  */
  27
  28 #include "brw_fs.h"
  29 #include "glsl/glsl_types.h"
  30 #include "glsl/ir_optimization.h"
  31 #include "glsl/ir_print_visitor.h"
  32
  33 static void
  34 assign_reg(int *reg_hw_locations, fs_reg *reg, int reg_width)
  35 {
  36    if (reg->file == GRF) {
  37       assert(reg->reg_offset >= 0);
  38       reg->reg = reg_hw_locations[reg->reg] + reg->reg_offset * reg_width;
  39       reg->reg_offset = 0;
  40    }
  41 }
  42
  43 void
  44 fs_visitor::assign_regs_trivial()
  45 {
  46    int hw_reg_mapping[this->virtual_grf_next + 1];
  47    int i;
  48    int reg_width = c->dispatch_width / 8;
  49
  50    /* Note that compressed instructions require alignment to 2 registers. */
  51    hw_reg_mapping[0] = ALIGN(this->first_non_payload_grf, reg_width);
  52    for (i = 1; i <= this->virtual_grf_next; i++) {
  53       hw_reg_mapping[i] = (hw_reg_mapping[i - 1] +
  54                            this->virtual_grf_sizes[i - 1] * reg_width);
  55    }
  56    this->grf_used = hw_reg_mapping[this->virtual_grf_next];
  57
  58    foreach_list(node, &this->instructions) {
  59       fs_inst *inst = (fs_inst *)node;
  60
  61       assign_reg(hw_reg_mapping, &inst->dst, reg_width);
  62       assign_reg(hw_reg_mapping, &inst->src[0], reg_width);
  63       assign_reg(hw_reg_mapping, &inst->src[1], reg_width);
  64    }
  65
  66    if (this->grf_used >= BRW_MAX_GRF) {
  67       fail("Ran out of regs on trivial allocator (%d/%d)\n",
  68            this->grf_used, BRW_MAX_GRF);
  69    }
  70
  71 }
  72
  73 static void
  74 brw_alloc_reg_set_for_classes(struct brw_context *brw,
  75                               int *class_sizes,
  76                               int class_count,
  77                               int reg_width,
  78                               int base_reg_count)
  79 {
  80    struct intel_context *intel = &brw->intel;
  81
  82    /* Compute the total number of registers across all classes. */
  83    int ra_reg_count = 0;
  84    for (int i = 0; i < class_count; i++) {
  85       ra_reg_count += base_reg_count - (class_sizes[i] - 1);
  86    }
  87
  88    ralloc_free(brw->wm.ra_reg_to_grf);
  89    brw->wm.ra_reg_to_grf = ralloc_array(brw, uint8_t, ra_reg_count);
  90    ralloc_free(brw->wm.regs);
  91    brw->wm.regs = ra_alloc_reg_set(ra_reg_count);
  92    ralloc_free(brw->wm.classes);
  93    brw->wm.classes = ralloc_array(brw, int, class_count + 1);
  94
  95    brw->wm.aligned_pairs_class = -1;
  96
  97    /* Now, add the registers to their classes, and add the conflicts
  98     * between them and the base GRF registers (and also each other).
  99     */
 100    int reg = 0;
 101    int pairs_base_reg = 0;
 102    int pairs_reg_count = 0;
 103    for (int i = 0; i < class_count; i++) {
 104       int class_reg_count = base_reg_count - (class_sizes[i] - 1);
 105       brw->wm.classes[i] = ra_alloc_reg_class(brw->wm.regs);
 106
 107       /* Save this off for the aligned pair class at the end. */
 108       if (class_sizes[i] == 2) {
 109          pairs_base_reg = reg;
 110          pairs_reg_count = class_reg_count;
 111       }
 112
 113       for (int j = 0; j < class_reg_count; j++) {
 114          ra_class_add_reg(brw->wm.regs, brw->wm.classes[i], reg);
 115
 116          brw->wm.ra_reg_to_grf[reg] = j;
 117
 118          for (int base_reg = j;
 119               base_reg < j + class_sizes[i];
 120               base_reg++) {
 121             ra_add_transitive_reg_conflict(brw->wm.regs, base_reg, reg);
 122          }
 123
 124          reg++;
 125       }
 126    }
 127    assert(reg == ra_reg_count);
 128
 129    /* Add a special class for aligned pairs, which we'll put delta_x/y
 130     * in on gen5 so that we can do PLN.
 131     */
 132    if (brw->has_pln && reg_width == 1 && intel->gen < 6) {
 133       brw->wm.aligned_pairs_class = ra_alloc_reg_class(brw->wm.regs);
 134
 135       for (int i = 0; i < pairs_reg_count; i++) {
 136          if ((brw->wm.ra_reg_to_grf[pairs_base_reg + i] & 1) == 0) {
 137             ra_class_add_reg(brw->wm.regs, brw->wm.aligned_pairs_class,
 138                              pairs_base_reg + i);
 139          }
 140       }
 141       class_count++;
 142    }
 143
 144    ra_set_finalize(brw->wm.regs);
 145 }
 146
 147 bool
 148 fs_visitor::assign_regs()
 149 {
 150    /* Most of this allocation was written for a reg_width of 1
 151     * (dispatch_width == 8).  In extending to 16-wide, the code was
 152     * left in place and it was converted to have the hardware
 153     * registers it's allocating be contiguous physical pairs of regs
 154     * for reg_width == 2.
 155     */
 156    int reg_width = c->dispatch_width / 8;
 157    int hw_reg_mapping[this->virtual_grf_next];
 158    int first_assigned_grf = ALIGN(this->first_non_payload_grf, reg_width);
 159    int base_reg_count = (BRW_MAX_GRF - first_assigned_grf) / reg_width;
 160    int class_sizes[base_reg_count];
 161    int class_count = 0;
 162
 163    calculate_live_intervals();
 164
 165    /* Set up the register classes.
 166     *
 167     * The base registers store a scalar value.  For texture samples,
 168     * we get virtual GRFs composed of 4 contiguous hw register.  For
 169     * structures and arrays, we store them as contiguous larger things
 170     * than that, though we should be able to do better most of the
 171     * time.
 172     */
 173    class_sizes[class_count++] = 1;
 174    if (brw->has_pln && intel->gen < 6) {
 175       /* Always set up the (unaligned) pairs for gen5, so we can find
 176        * them for making the aligned pair class.
 177        */
 178       class_sizes[class_count++] = 2;
 179    }
 180    for (int r = 0; r < this->virtual_grf_next; r++) {
 181       int i;
 182
 183       for (i = 0; i < class_count; i++) {
 184          if (class_sizes[i] == this->virtual_grf_sizes[r])
 185             break;
 186       }
 187       if (i == class_count) {
 188          if (this->virtual_grf_sizes[r] >= base_reg_count) {
 189             fail("Object too large to register allocate.\n");
 190          }
 191
 192          class_sizes[class_count++] = this->virtual_grf_sizes[r];
 193       }
 194    }
 195
 196    brw_alloc_reg_set_for_classes(brw, class_sizes, class_count,
 197                                  reg_width, base_reg_count);
 198
 199    struct ra_graph *g = ra_alloc_interference_graph(brw->wm.regs,
 200                                                     this->virtual_grf_next);
 201
 202    for (int i = 0; i < this->virtual_grf_next; i++) {
 203       for (int c = 0; c < class_count; c++) {
 204          if (class_sizes[c] == this->virtual_grf_sizes[i]) {
 205             if (brw->wm.aligned_pairs_class >= 0 &&
 206                 this->delta_x.reg == i) {
 207                ra_set_node_class(g, i, brw->wm.aligned_pairs_class);
 208             } else {
 209                ra_set_node_class(g, i, brw->wm.classes[c]);
 210             }
 211             break;
 212          }
 213       }
 214
 215       for (int j = 0; j < i; j++) {
 216          if (virtual_grf_interferes(i, j)) {
 217             ra_add_node_interference(g, i, j);
 218          }
 219       }
 220    }
 221
 222    if (!ra_allocate_no_spills(g)) {
 223       /* Failed to allocate registers.  Spill a reg, and the caller will
 224        * loop back into here to try again.
 225        */
 226       int reg = choose_spill_reg(g);
 227
 228       if (reg == -1) {
 229          fail("no register to spill\n");
 230       } else if (intel->gen >= 7) {
 231          fail("no spilling support on gen7 yet\n");
 232       } else if (c->dispatch_width == 16) {
 233          fail("no spilling support on 16-wide yet\n");
 234       } else {
 235          spill_reg(reg);
 236       }
 237
 238
 239       ralloc_free(g);
 240
 241       return false;
 242    }
 243
 244    /* Get the chosen virtual registers for each node, and map virtual
 245     * regs in the register classes back down to real hardware reg
 246     * numbers.
 247     */
 248    this->grf_used = first_assigned_grf;
 249    for (int i = 0; i < this->virtual_grf_next; i++) {
 250       int reg = ra_get_node_reg(g, i);
 251
 252       hw_reg_mapping[i] = (first_assigned_grf +
 253                            brw->wm.ra_reg_to_grf[reg] * reg_width);
 254       this->grf_used = MAX2(this->grf_used,
 255                             hw_reg_mapping[i] + this->virtual_grf_sizes[i] *
 256                             reg_width);
 257    }
 258
 259    foreach_list(node, &this->instructions) {
 260       fs_inst *inst = (fs_inst *)node;
 261
 262       assign_reg(hw_reg_mapping, &inst->dst, reg_width);
 263       assign_reg(hw_reg_mapping, &inst->src[0], reg_width);
 264       assign_reg(hw_reg_mapping, &inst->src[1], reg_width);
 265    }
 266
 267    ralloc_free(g);
 268
 269    return true;
 270 }
 271
 272 void
 273 fs_visitor::emit_unspill(fs_inst *inst, fs_reg dst, uint32_t spill_offset)
 274 {
 275    int size = virtual_grf_sizes[dst.reg];
 276    dst.reg_offset = 0;
 277
 278    for (int chan = 0; chan < size; chan++) {
 279       fs_inst *unspill_inst = new(mem_ctx) fs_inst(FS_OPCODE_UNSPILL,
 280                                                    dst);
 281       dst.reg_offset++;
 282       unspill_inst->offset = spill_offset + chan * REG_SIZE;
 283       unspill_inst->ir = inst->ir;
 284       unspill_inst->annotation = inst->annotation;
 285
 286       /* Choose a MRF that won't conflict with an MRF that's live across the
 287        * spill.  Nothing else will make it up to MRF 14/15.
 288        */
 289       unspill_inst->base_mrf = 14;
 290       unspill_inst->mlen = 1; /* header contains offset */
 291       inst->insert_before(unspill_inst);
 292    }
 293 }
 294
 295 int
 296 fs_visitor::choose_spill_reg(struct ra_graph *g)
 297 {
 298    float loop_scale = 1.0;
 299    float spill_costs[this->virtual_grf_next];
 300    bool no_spill[this->virtual_grf_next];
 301
 302    for (int i = 0; i < this->virtual_grf_next; i++) {
 303       spill_costs[i] = 0.0;
 304       no_spill[i] = false;
 305    }
 306
 307    /* Calculate costs for spilling nodes.  Call it a cost of 1 per
 308     * spill/unspill we'll have to do, and guess that the insides of
 309     * loops run 10 times.
 310     */
 311    foreach_list(node, &this->instructions) {
 312       fs_inst *inst = (fs_inst *)node;
 313
 314       for (unsigned int i = 0; i < 3; i++) {
 315          if (inst->src[i].file == GRF) {
 316             int size = virtual_grf_sizes[inst->src[i].reg];
 317             spill_costs[inst->src[i].reg] += size * loop_scale;
 318          }
 319       }
 320
 321       if (inst->dst.file == GRF) {
 322          int size = virtual_grf_sizes[inst->dst.reg];
 323          spill_costs[inst->dst.reg] += size * loop_scale;
 324       }
 325
 326       switch (inst->opcode) {
 327
 328       case BRW_OPCODE_DO:
 329          loop_scale *= 10;
 330          break;
 331
 332       case BRW_OPCODE_WHILE:
 333          loop_scale /= 10;
 334          break;
 335
 336       case FS_OPCODE_SPILL:
 337          if (inst->src[0].file == GRF)
 338             no_spill[inst->src[0].reg] = true;
 339          break;
 340
 341       case FS_OPCODE_UNSPILL:
 342          if (inst->dst.file == GRF)
 343             no_spill[inst->dst.reg] = true;
 344          break;
 345
 346       default:
 347          break;
 348       }
 349    }
 350
 351    for (int i = 0; i < this->virtual_grf_next; i++) {
 352       if (!no_spill[i])
 353          ra_set_node_spill_cost(g, i, spill_costs[i]);
 354    }
 355
 356    return ra_get_best_spill_node(g);
 357 }
 358
 359 void
 360 fs_visitor::spill_reg(int spill_reg)
 361 {
 362    int size = virtual_grf_sizes[spill_reg];
 363    unsigned int spill_offset = c->last_scratch;
 364    assert(ALIGN(spill_offset, 16) == spill_offset); /* oword read/write req. */
 365    c->last_scratch += size * REG_SIZE;
 366
 367    /* Generate spill/unspill instructions for the objects being
 368     * spilled.  Right now, we spill or unspill the whole thing to a
 369     * virtual grf of the same size.  For most instructions, though, we
 370     * could just spill/unspill the GRF being accessed.
 371     */
 372    foreach_list(node, &this->instructions) {
 373       fs_inst *inst = (fs_inst *)node;
 374
 375       for (unsigned int i = 0; i < 3; i++) {
 376          if (inst->src[i].file == GRF &&
 377              inst->src[i].reg == spill_reg) {
 378             inst->src[i].reg = virtual_grf_alloc(size);
 379             emit_unspill(inst, inst->src[i], spill_offset);
 380          }
 381       }
 382
 383       if (inst->dst.file == GRF &&
 384           inst->dst.reg == spill_reg) {
 385          inst->dst.reg = virtual_grf_alloc(size);
 386
 387          /* Since we spill/unspill the whole thing even if we access
 388           * just a component, we may need to unspill before the
 389           * instruction we're spilling for.
 390           */
 391          if (size != 1 || inst->predicated) {
 392             emit_unspill(inst, inst->dst, spill_offset);
 393          }
 394
 395          fs_reg spill_src = inst->dst;
 396          spill_src.reg_offset = 0;
 397          spill_src.abs = false;
 398          spill_src.negate = false;
 399          spill_src.smear = -1;
 400
 401          for (int chan = 0; chan < size; chan++) {
 402             fs_inst *spill_inst = new(mem_ctx) fs_inst(FS_OPCODE_SPILL,
 403                                                        reg_null_f, spill_src);
 404             spill_src.reg_offset++;
 405             spill_inst->offset = spill_offset + chan * REG_SIZE;
 406             spill_inst->ir = inst->ir;
 407             spill_inst->annotation = inst->annotation;
 408             spill_inst->base_mrf = 14;
 409             spill_inst->mlen = 2; /* header, value */
 410             inst->insert_after(spill_inst);
 411          }
 412       }
 413    }
 414
 415    this->live_intervals_valid = false;
 416 }