src/mesa/drivers/dri/i965/brw_vec4_copy_propagation.cpp

   1 /*
   2  * Copyright © 2011 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  */
  23
  24 /**
  25  * @file brw_vec4_copy_propagation.cpp
  26  *
  27  * Implements tracking of values copied between registers, and
  28  * optimizations based on that: copy propagation and constant
  29  * propagation.
  30  */
  31
  32 #include "brw_vec4.h"
  33 #include "brw_cfg.h"
  34 #include "brw_eu.h"
  35
  36 namespace brw {
  37
  38 struct copy_entry {
  39    src_reg *value[4];
  40    int saturatemask;
  41 };
  42
  43 static bool
  44 is_direct_copy(vec4_instruction *inst)
  45 {
  46    return (inst->opcode == BRW_OPCODE_MOV &&
  47            !inst->predicate &&
  48            inst->dst.file == VGRF &&
  49            !inst->dst.reladdr &&
  50            !inst->src[0].reladdr &&
  51            (inst->dst.type == inst->src[0].type ||
  52             (inst->dst.type == BRW_REGISTER_TYPE_F &&
  53              inst->src[0].type == BRW_REGISTER_TYPE_VF)));
  54 }
  55
  56 static bool
  57 is_dominated_by_previous_instruction(vec4_instruction *inst)
  58 {
  59    return (inst->opcode != BRW_OPCODE_DO &&
  60            inst->opcode != BRW_OPCODE_WHILE &&
  61            inst->opcode != BRW_OPCODE_ELSE &&
  62            inst->opcode != BRW_OPCODE_ENDIF);
  63 }
  64
  65 static bool
  66 is_channel_updated(vec4_instruction *inst, src_reg *values[4], int ch)
  67 {
  68    const src_reg *src = values[ch];
  69
  70    /* consider GRF only */
  71    assert(inst->dst.file == VGRF);
  72    if (!src || src->file != VGRF)
  73       return false;
  74
  75    return (src->in_range(inst->dst, inst->regs_written) &&
  76            inst->dst.writemask & (1 << BRW_GET_SWZ(src->swizzle, ch)));
  77 }
  78
  79 static unsigned
  80 swizzle_vf_imm(unsigned vf4, unsigned swizzle)
  81 {
  82    union {
  83       unsigned vf4;
  84       uint8_t vf[4];
  85    } v = { vf4 }, ret;
  86
  87    ret.vf[0] = v.vf[BRW_GET_SWZ(swizzle, 0)];
  88    ret.vf[1] = v.vf[BRW_GET_SWZ(swizzle, 1)];
  89    ret.vf[2] = v.vf[BRW_GET_SWZ(swizzle, 2)];
  90    ret.vf[3] = v.vf[BRW_GET_SWZ(swizzle, 3)];
  91
  92    return ret.vf4;
  93 }
  94
  95 static bool
  96 is_logic_op(enum opcode opcode)
  97 {
  98    return (opcode == BRW_OPCODE_AND ||
  99            opcode == BRW_OPCODE_OR  ||
 100            opcode == BRW_OPCODE_XOR ||
 101            opcode == BRW_OPCODE_NOT);
 102 }
 103
 104 static bool
 105 try_constant_propagate(const struct brw_device_info *devinfo,
 106                        vec4_instruction *inst,
 107                        int arg, struct copy_entry *entry)
 108 {
 109    /* For constant propagation, we only handle the same constant
 110     * across all 4 channels.  Some day, we should handle the 8-bit
 111     * float vector format, which would let us constant propagate
 112     * vectors better.
 113     */
 114    src_reg value = *entry->value[0];
 115    for (int i = 1; i < 4; i++) {
 116       if (!value.equals(*entry->value[i]))
 117          return false;
 118    }
 119
 120    if (value.file != IMM)
 121       return false;
 122
 123    if (value.type == BRW_REGISTER_TYPE_VF) {
 124       /* The result of bit-casting the component values of a vector float
 125        * cannot in general be represented as an immediate.
 126        */
 127       if (inst->src[arg].type != BRW_REGISTER_TYPE_F)
 128          return false;
 129    } else {
 130       value.type = inst->src[arg].type;
 131    }
 132
 133    if (inst->src[arg].abs) {
 134       if ((devinfo->gen >= 8 && is_logic_op(inst->opcode)) ||
 135           !brw_abs_immediate(value.type, &value.as_brw_reg())) {
 136          return false;
 137       }
 138    }
 139
 140    if (inst->src[arg].negate) {
 141       if ((devinfo->gen >= 8 && is_logic_op(inst->opcode)) ||
 142           !brw_negate_immediate(value.type, &value.as_brw_reg())) {
 143          return false;
 144       }
 145    }
 146
 147    if (value.type == BRW_REGISTER_TYPE_VF)
 148       value.ud = swizzle_vf_imm(value.ud, inst->src[arg].swizzle);
 149
 150    switch (inst->opcode) {
 151    case BRW_OPCODE_MOV:
 152    case SHADER_OPCODE_BROADCAST:
 153       inst->src[arg] = value;
 154       return true;
 155
 156    case SHADER_OPCODE_POW:
 157    case SHADER_OPCODE_INT_QUOTIENT:
 158    case SHADER_OPCODE_INT_REMAINDER:
 159       if (devinfo->gen < 8)
 160          break;
 161       /* fallthrough */
 162    case BRW_OPCODE_DP2:
 163    case BRW_OPCODE_DP3:
 164    case BRW_OPCODE_DP4:
 165    case BRW_OPCODE_DPH:
 166    case BRW_OPCODE_BFI1:
 167    case BRW_OPCODE_ASR:
 168    case BRW_OPCODE_SHL:
 169    case BRW_OPCODE_SHR:
 170    case BRW_OPCODE_SUBB:
 171       if (arg == 1) {
 172          inst->src[arg] = value;
 173          return true;
 174       }
 175       break;
 176
 177    case BRW_OPCODE_MACH:
 178    case BRW_OPCODE_MUL:
 179    case SHADER_OPCODE_MULH:
 180    case BRW_OPCODE_ADD:
 181    case BRW_OPCODE_OR:
 182    case BRW_OPCODE_AND:
 183    case BRW_OPCODE_XOR:
 184    case BRW_OPCODE_ADDC:
 185       if (arg == 1) {
 186          inst->src[arg] = value;
 187          return true;
 188       } else if (arg == 0 && inst->src[1].file != IMM) {
 189          /* Fit this constant in by commuting the operands.  Exception: we
 190           * can't do this for 32-bit integer MUL/MACH because it's asymmetric.
 191           */
 192          if ((inst->opcode == BRW_OPCODE_MUL ||
 193               inst->opcode == BRW_OPCODE_MACH) &&
 194              (inst->src[1].type == BRW_REGISTER_TYPE_D ||
 195               inst->src[1].type == BRW_REGISTER_TYPE_UD))
 196             break;
 197          inst->src[0] = inst->src[1];
 198          inst->src[1] = value;
 199          return true;
 200       }
 201       break;
 202    case GS_OPCODE_SET_WRITE_OFFSET:
 203       /* This is just a multiply by a constant with special strides.
 204        * The generator will handle immediates in both arguments (generating
 205        * a single MOV of the product).  So feel free to propagate in src0.
 206        */
 207       inst->src[arg] = value;
 208       return true;
 209
 210    case BRW_OPCODE_CMP:
 211       if (arg == 1) {
 212          inst->src[arg] = value;
 213          return true;
 214       } else if (arg == 0 && inst->src[1].file != IMM) {
 215          enum brw_conditional_mod new_cmod;
 216
 217          new_cmod = brw_swap_cmod(inst->conditional_mod);
 218          if (new_cmod != BRW_CONDITIONAL_NONE) {
 219             /* Fit this constant in by swapping the operands and
 220              * flipping the test.
 221              */
 222             inst->src[0] = inst->src[1];
 223             inst->src[1] = value;
 224             inst->conditional_mod = new_cmod;
 225             return true;
 226          }
 227       }
 228       break;
 229
 230    case BRW_OPCODE_SEL:
 231       if (arg == 1) {
 232          inst->src[arg] = value;
 233          return true;
 234       } else if (arg == 0 && inst->src[1].file != IMM) {
 235          inst->src[0] = inst->src[1];
 236          inst->src[1] = value;
 237
 238          /* If this was predicated, flipping operands means
 239           * we also need to flip the predicate.
 240           */
 241          if (inst->conditional_mod == BRW_CONDITIONAL_NONE) {
 242             inst->predicate_inverse = !inst->predicate_inverse;
 243          }
 244          return true;
 245       }
 246       break;
 247
 248    default:
 249       break;
 250    }
 251
 252    return false;
 253 }
 254
 255 static bool
 256 try_copy_propagate(const struct brw_device_info *devinfo,
 257                    vec4_instruction *inst,
 258                    int arg, struct copy_entry *entry)
 259 {
 260    /* Build up the value we are propagating as if it were the source of a
 261     * single MOV
 262     */
 263    /* For constant propagation, we only handle the same constant
 264     * across all 4 channels.  Some day, we should handle the 8-bit
 265     * float vector format, which would let us constant propagate
 266     * vectors better.
 267     */
 268    src_reg value = *entry->value[0];
 269    for (int i = 1; i < 4; i++) {
 270       /* This is equals() except we don't care about the swizzle. */
 271       if (value.file != entry->value[i]->file ||
 272           value.nr != entry->value[i]->nr ||
 273           value.reg_offset != entry->value[i]->reg_offset ||
 274           value.type != entry->value[i]->type ||
 275           value.negate != entry->value[i]->negate ||
 276           value.abs != entry->value[i]->abs) {
 277          return false;
 278       }
 279    }
 280
 281    /* Compute the swizzle of the original register by swizzling the
 282     * component loaded from each value according to the swizzle of
 283     * operand we're going to change.
 284     */
 285    int s[4];
 286    for (int i = 0; i < 4; i++) {
 287       s[i] = BRW_GET_SWZ(entry->value[i]->swizzle, i);
 288    }
 289    value.swizzle = BRW_SWIZZLE4(s[0], s[1], s[2], s[3]);
 290
 291    /* Check that we can propagate that value */
 292    if (value.file != UNIFORM &&
 293        value.file != VGRF &&
 294        value.file != ATTR)
 295       return false;
 296
 297    if (devinfo->gen >= 8 && (value.negate || value.abs) &&
 298        is_logic_op(inst->opcode)) {
 299       return false;
 300    }
 301
 302    bool has_source_modifiers = value.negate || value.abs;
 303
 304    /* gen6 math and gen7+ SENDs from GRFs ignore source modifiers on
 305     * instructions.
 306     */
 307    if ((has_source_modifiers || value.file == UNIFORM ||
 308         value.swizzle != BRW_SWIZZLE_XYZW) && !inst->can_do_source_mods(devinfo))
 309       return false;
 310
 311    if (has_source_modifiers &&
 312        value.type != inst->src[arg].type &&
 313        !inst->can_change_types())
 314       return false;
 315
 316    if (has_source_modifiers &&
 317        inst->opcode == SHADER_OPCODE_GEN4_SCRATCH_WRITE)
 318       return false;
 319
 320    unsigned composed_swizzle = brw_compose_swizzle(inst->src[arg].swizzle,
 321                                                    value.swizzle);
 322    if (inst->is_3src() &&
 323        value.file == UNIFORM &&
 324        !brw_is_single_value_swizzle(composed_swizzle))
 325       return false;
 326
 327    if (inst->is_send_from_grf())
 328       return false;
 329
 330    /* we can't generally copy-propagate UD negations becuse we
 331     * end up accessing the resulting values as signed integers
 332     * instead. See also resolve_ud_negate().
 333     */
 334    if (value.negate &&
 335        value.type == BRW_REGISTER_TYPE_UD)
 336       return false;
 337
 338    /* Don't report progress if this is a noop. */
 339    if (value.equals(inst->src[arg]))
 340       return false;
 341
 342    const unsigned dst_saturate_mask = inst->dst.writemask &
 343       brw_apply_swizzle_to_mask(inst->src[arg].swizzle, entry->saturatemask);
 344
 345    if (dst_saturate_mask) {
 346       /* We either saturate all or nothing. */
 347       if (dst_saturate_mask != inst->dst.writemask)
 348          return false;
 349
 350       /* Limit saturate propagation only to SEL with src1 bounded within 0.0
 351        * and 1.0, otherwise skip copy propagate altogether.
 352        */
 353       switch(inst->opcode) {
 354       case BRW_OPCODE_SEL:
 355          if (arg != 0 ||
 356              inst->src[0].type != BRW_REGISTER_TYPE_F ||
 357              inst->src[1].file != IMM ||
 358              inst->src[1].type != BRW_REGISTER_TYPE_F ||
 359              inst->src[1].f < 0.0 ||
 360              inst->src[1].f > 1.0) {
 361             return false;
 362          }
 363          if (!inst->saturate)
 364             inst->saturate = true;
 365          break;
 366       default:
 367          return false;
 368       }
 369    }
 370
 371    /* Build the final value */
 372    if (inst->src[arg].abs) {
 373       value.negate = false;
 374       value.abs = true;
 375    }
 376    if (inst->src[arg].negate)
 377       value.negate = !value.negate;
 378
 379    value.swizzle = composed_swizzle;
 380    if (has_source_modifiers &&
 381        value.type != inst->src[arg].type) {
 382       assert(inst->can_change_types());
 383       for (int i = 0; i < 3; i++) {
 384          inst->src[i].type = value.type;
 385       }
 386       inst->dst.type = value.type;
 387    } else {
 388       value.type = inst->src[arg].type;
 389    }
 390
 391    inst->src[arg] = value;
 392    return true;
 393 }
 394
 395 bool
 396 vec4_visitor::opt_copy_propagation(bool do_constant_prop)
 397 {
 398    bool progress = false;
 399    struct copy_entry entries[alloc.total_size];
 400
 401    memset(&entries, 0, sizeof(entries));
 402
 403    foreach_block_and_inst(block, vec4_instruction, inst, cfg) {
 404       /* This pass only works on basic blocks.  If there's flow
 405        * control, throw out all our information and start from
 406        * scratch.
 407        *
 408        * This should really be fixed by using a structure like in
 409        * src/glsl/opt_copy_propagation.cpp to track available copies.
 410        */
 411       if (!is_dominated_by_previous_instruction(inst)) {
 412          memset(&entries, 0, sizeof(entries));
 413          continue;
 414       }
 415
 416       /* For each source arg, see if each component comes from a copy
 417        * from the same type file (IMM, VGRF, UNIFORM), and try
 418        * optimizing out access to the copy result
 419        */
 420       for (int i = 2; i >= 0; i--) {
 421          /* Copied values end up in GRFs, and we don't track reladdr
 422           * accesses.
 423           */
 424          if (inst->src[i].file != VGRF ||
 425              inst->src[i].reladdr)
 426             continue;
 427
 428          /* We only handle single-register copies. */
 429          if (inst->regs_read(i) != 1)
 430             continue;
 431
 432          int reg = (alloc.offsets[inst->src[i].nr] +
 433                     inst->src[i].reg_offset);
 434
 435          /* Find the regs that each swizzle component came from.
 436           */
 437          struct copy_entry entry;
 438          memset(&entry, 0, sizeof(copy_entry));
 439          int c;
 440          for (c = 0; c < 4; c++) {
 441             int channel = BRW_GET_SWZ(inst->src[i].swizzle, c);
 442             entry.value[c] = entries[reg].value[channel];
 443
 444             /* If there's no available copy for this channel, bail.
 445              * We could be more aggressive here -- some channels might
 446              * not get used based on the destination writemask.
 447              */
 448             if (!entry.value[c])
 449                break;
 450
 451             entry.saturatemask |=
 452                (entries[reg].saturatemask & (1 << channel) ? 1 : 0) << c;
 453
 454             /* We'll only be able to copy propagate if the sources are
 455              * all from the same file -- there's no ability to swizzle
 456              * 0 or 1 constants in with source registers like in i915.
 457              */
 458             if (c > 0 && entry.value[c - 1]->file != entry.value[c]->file)
 459                break;
 460          }
 461
 462          if (c != 4)
 463             continue;
 464
 465          if (do_constant_prop && try_constant_propagate(devinfo, inst, i, &entry))
 466             progress = true;
 467
 468          if (try_copy_propagate(devinfo, inst, i, &entry))
 469             progress = true;
 470       }
 471
 472       /* Track available source registers. */
 473       if (inst->dst.file == VGRF) {
 474          const int reg =
 475             alloc.offsets[inst->dst.nr] + inst->dst.reg_offset;
 476
 477          /* Update our destination's current channel values.  For a direct copy,
 478           * the value is the newly propagated source.  Otherwise, we don't know
 479           * the new value, so clear it.
 480           */
 481          bool direct_copy = is_direct_copy(inst);
 482          entries[reg].saturatemask &= ~inst->dst.writemask;
 483          for (int i = 0; i < 4; i++) {
 484             if (inst->dst.writemask & (1 << i)) {
 485                entries[reg].value[i] = direct_copy ? &inst->src[0] : NULL;
 486                entries[reg].saturatemask |=
 487                   inst->saturate && direct_copy ? 1 << i : 0;
 488             }
 489          }
 490
 491          /* Clear the records for any registers whose current value came from
 492           * our destination's updated channels, as the two are no longer equal.
 493           */
 494          if (inst->dst.reladdr)
 495             memset(&entries, 0, sizeof(entries));
 496          else {
 497             for (unsigned i = 0; i < alloc.total_size; i++) {
 498                for (int j = 0; j < 4; j++) {
 499                   if (is_channel_updated(inst, entries[i].value, j)) {
 500                      entries[i].value[j] = NULL;
 501                      entries[i].saturatemask &= ~(1 << j);
 502                   }
 503                }
 504             }
 505          }
 506       }
 507    }
 508
 509    if (progress)
 510       invalidate_live_intervals();
 511
 512    return progress;
 513 }
 514
 515 } /* namespace brw */