src/mesa/drivers/dri/i965/brw_fs_combine_constants.cpp

   1 /*
   2  * Copyright © 2014 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  */
  23
  24 /** @file brw_fs_combine_constants.cpp
  25  *
  26  * This file contains the opt_combine_constants() pass that runs after the
  27  * regular optimization loop. It passes over the instruction list and
  28  * selectively promotes immediate values to registers by emitting a mov(1)
  29  * instruction.
  30  *
  31  * This is useful on Gen 7 particularly, because a few instructions can be
  32  * coissued (i.e., issued in the same cycle as another thread on the same EU
  33  * issues an instruction) under some circumstances, one of which is that they
  34  * cannot use immediate values.
  35  */
  36
  37 #include "brw_fs.h"
  38 #include "brw_fs_live_variables.h"
  39 #include "brw_cfg.h"
  40
  41 /* Returns whether an instruction could co-issue if its immediate source were
  42  * replaced with a GRF source.
  43  */
  44 static bool
  45 could_coissue(const struct brw_device_info *devinfo, const fs_inst *inst)
  46 {
  47    if (devinfo->gen != 7)
  48       return false;
  49
  50    switch (inst->opcode) {
  51    case BRW_OPCODE_MOV:
  52    case BRW_OPCODE_CMP:
  53    case BRW_OPCODE_ADD:
  54    case BRW_OPCODE_MUL:
  55       return true;
  56    default:
  57       return false;
  58    }
  59 }
  60
  61 /**
  62  * Returns true for instructions that don't support immediate sources.
  63  */
  64 static bool
  65 must_promote_imm(const struct brw_device_info *devinfo, const fs_inst *inst)
  66 {
  67    switch (inst->opcode) {
  68    case SHADER_OPCODE_POW:
  69       return devinfo->gen < 8;
  70    case BRW_OPCODE_MAD:
  71    case BRW_OPCODE_LRP:
  72       return true;
  73    default:
  74       return false;
  75    }
  76 }
  77
  78 /** A box for putting fs_regs in a linked list. */
  79 struct reg_link {
  80    DECLARE_RALLOC_CXX_OPERATORS(reg_link)
  81
  82    reg_link(fs_reg *reg) : reg(reg) {}
  83
  84    struct exec_node link;
  85    fs_reg *reg;
  86 };
  87
  88 static struct exec_node *
  89 link(void *mem_ctx, fs_reg *reg)
  90 {
  91    reg_link *l = new(mem_ctx) reg_link(reg);
  92    return &l->link;
  93 }
  94
  95 /**
  96  * Information about an immediate value.
  97  */
  98 struct imm {
  99    /** The common ancestor of all blocks using this immediate value. */
 100    bblock_t *block;
 101
 102    /**
 103     * The instruction generating the immediate value, if all uses are contained
 104     * within a single basic block. Otherwise, NULL.
 105     */
 106    fs_inst *inst;
 107
 108    /**
 109     * A list of fs_regs that refer to this immediate.  If we promote it, we'll
 110     * have to patch these up to refer to the new GRF.
 111     */
 112    exec_list *uses;
 113
 114    /** The immediate value.  We currently only handle floats. */
 115    float val;
 116
 117    /**
 118     * The GRF register and subregister number where we've decided to store the
 119     * constant value.
 120     */
 121    uint8_t subreg_offset;
 122    uint16_t reg;
 123
 124    /** The number of coissuable instructions using this immediate. */
 125    uint16_t uses_by_coissue;
 126
 127    /**
 128     * Whether this constant is used by an instruction that can't handle an
 129     * immediate source (and already has to be promoted to a GRF).
 130     */
 131    bool must_promote;
 132
 133    uint16_t first_use_ip;
 134    uint16_t last_use_ip;
 135 };
 136
 137 /** The working set of information about immediates. */
 138 struct table {
 139    struct imm *imm;
 140    int size;
 141    int len;
 142 };
 143
 144 static struct imm *
 145 find_imm(struct table *table, float val)
 146 {
 147    assert(signbit(val) == 0);
 148
 149    for (int i = 0; i < table->len; i++) {
 150       if (table->imm[i].val == val) {
 151          return &table->imm[i];
 152       }
 153    }
 154    return NULL;
 155 }
 156
 157 static struct imm *
 158 new_imm(struct table *table, void *mem_ctx)
 159 {
 160    if (table->len == table->size) {
 161       table->size *= 2;
 162       table->imm = reralloc(mem_ctx, table->imm, struct imm, table->size);
 163    }
 164    return &table->imm[table->len++];
 165 }
 166
 167 /**
 168  * Comparator used for sorting an array of imm structures.
 169  *
 170  * We sort by basic block number, then last use IP, then first use IP (least
 171  * to greatest). This sorting causes immediates live in the same area to be
 172  * allocated to the same register in the hopes that all values will be dead
 173  * about the same time and the register can be reused.
 174  */
 175 static int
 176 compare(const void *_a, const void *_b)
 177 {
 178    const struct imm *a = (const struct imm *)_a,
 179                     *b = (const struct imm *)_b;
 180
 181    int block_diff = a->block->num - b->block->num;
 182    if (block_diff)
 183       return block_diff;
 184
 185    int end_diff = a->last_use_ip - b->last_use_ip;
 186    if (end_diff)
 187       return end_diff;
 188
 189    return a->first_use_ip - b->first_use_ip;
 190 }
 191
 192 bool
 193 fs_visitor::opt_combine_constants()
 194 {
 195    void *const_ctx = ralloc_context(NULL);
 196
 197    struct table table;
 198    table.size = 8;
 199    table.len = 0;
 200    table.imm = ralloc_array(const_ctx, struct imm, table.size);
 201
 202    cfg->calculate_idom();
 203    unsigned ip = -1;
 204
 205    /* Make a pass through all instructions and count the number of times each
 206     * constant is used by coissueable instructions or instructions that cannot
 207     * take immediate arguments.
 208     */
 209    foreach_block_and_inst(block, fs_inst, inst, cfg) {
 210       ip++;
 211
 212       if (!could_coissue(devinfo, inst) && !must_promote_imm(devinfo, inst))
 213          continue;
 214
 215       for (int i = 0; i < inst->sources; i++) {
 216          if (inst->src[i].file != IMM ||
 217              inst->src[i].type != BRW_REGISTER_TYPE_F)
 218             continue;
 219
 220          float val = fabsf(inst->src[i].fixed_hw_reg.dw1.f);
 221          struct imm *imm = find_imm(&table, val);
 222
 223          if (imm) {
 224             bblock_t *intersection = cfg_t::intersect(block, imm->block);
 225             if (intersection != imm->block)
 226                imm->inst = NULL;
 227             imm->block = intersection;
 228             imm->uses->push_tail(link(const_ctx, &inst->src[i]));
 229             imm->uses_by_coissue += could_coissue(devinfo, inst);
 230             imm->must_promote = imm->must_promote || must_promote_imm(devinfo, inst);
 231             imm->last_use_ip = ip;
 232          } else {
 233             imm = new_imm(&table, const_ctx);
 234             imm->block = block;
 235             imm->inst = inst;
 236             imm->uses = new(const_ctx) exec_list();
 237             imm->uses->push_tail(link(const_ctx, &inst->src[i]));
 238             imm->val = val;
 239             imm->uses_by_coissue = could_coissue(devinfo, inst);
 240             imm->must_promote = must_promote_imm(devinfo, inst);
 241             imm->first_use_ip = ip;
 242             imm->last_use_ip = ip;
 243          }
 244       }
 245    }
 246
 247    /* Remove constants from the table that don't have enough uses to make them
 248     * profitable to store in a register.
 249     */
 250    for (int i = 0; i < table.len;) {
 251       struct imm *imm = &table.imm[i];
 252
 253       if (!imm->must_promote && imm->uses_by_coissue < 4) {
 254          table.imm[i] = table.imm[table.len - 1];
 255          table.len--;
 256          continue;
 257       }
 258       i++;
 259    }
 260    if (table.len == 0) {
 261       ralloc_free(const_ctx);
 262       return false;
 263    }
 264    if (cfg->num_blocks != 1)
 265       qsort(table.imm, table.len, sizeof(struct imm), compare);
 266
 267
 268    /* Insert MOVs to load the constant values into GRFs. */
 269    fs_reg reg(GRF, alloc.allocate(dispatch_width / 8));
 270    reg.stride = 0;
 271    for (int i = 0; i < table.len; i++) {
 272       struct imm *imm = &table.imm[i];
 273
 274       fs_inst *mov = MOV(reg, fs_reg(imm->val));
 275       mov->force_writemask_all = true;
 276       if (imm->inst) {
 277          imm->inst->insert_before(imm->block, mov);
 278       } else {
 279          backend_instruction *inst = imm->block->last_non_control_flow_inst();
 280          inst->insert_after(imm->block, mov);
 281       }
 282       imm->reg = reg.reg;
 283       imm->subreg_offset = reg.subreg_offset;
 284
 285       reg.subreg_offset += sizeof(float);
 286       if ((unsigned)reg.subreg_offset == dispatch_width * sizeof(float)) {
 287          reg.reg = alloc.allocate(dispatch_width / 8);
 288          reg.subreg_offset = 0;
 289       }
 290    }
 291    promoted_constants = table.len;
 292
 293    /* Rewrite the immediate sources to refer to the new GRFs. */
 294    for (int i = 0; i < table.len; i++) {
 295       foreach_list_typed(reg_link, link, link, table.imm[i].uses) {
 296          fs_reg *reg = link->reg;
 297          reg->file = GRF;
 298          reg->reg = table.imm[i].reg;
 299          reg->subreg_offset = table.imm[i].subreg_offset;
 300          reg->stride = 0;
 301          reg->negate = signbit(reg->fixed_hw_reg.dw1.f) !=
 302                                signbit(table.imm[i].val);
 303          assert(fabsf(reg->fixed_hw_reg.dw1.f) == table.imm[i].val);
 304       }
 305    }
 306
 307    ralloc_free(const_ctx);
 308    invalidate_live_intervals();
 309
 310    return true;
 311 }