src/mesa/drivers/dri/i965/brw_vec4_copy_propagation.cpp

   1 /*
   2  * Copyright © 2011 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  */
  23
  24 /**
  25  * @file brw_vec4_copy_propagation.cpp
  26  *
  27  * Implements tracking of values copied between registers, and
  28  * optimizations based on that: copy propagation and constant
  29  * propagation.
  30  */
  31
  32 #include "brw_vec4.h"
  33 #include "brw_cfg.h"
  34 #include "brw_eu.h"
  35
  36 namespace brw {
  37
  38 struct copy_entry {
  39    src_reg *value[4];
  40    int saturatemask;
  41 };
  42
  43 static bool
  44 is_direct_copy(vec4_instruction *inst)
  45 {
  46    return (inst->opcode == BRW_OPCODE_MOV &&
  47            !inst->predicate &&
  48            inst->dst.file == VGRF &&
  49            !inst->dst.reladdr &&
  50            !inst->src[0].reladdr &&
  51            (inst->dst.type == inst->src[0].type ||
  52             (inst->dst.type == BRW_REGISTER_TYPE_F &&
  53              inst->src[0].type == BRW_REGISTER_TYPE_VF)));
  54 }
  55
  56 static bool
  57 is_dominated_by_previous_instruction(vec4_instruction *inst)
  58 {
  59    return (inst->opcode != BRW_OPCODE_DO &&
  60            inst->opcode != BRW_OPCODE_WHILE &&
  61            inst->opcode != BRW_OPCODE_ELSE &&
  62            inst->opcode != BRW_OPCODE_ENDIF);
  63 }
  64
  65 static bool
  66 is_channel_updated(vec4_instruction *inst, src_reg *values[4], int ch)
  67 {
  68    const src_reg *src = values[ch];
  69
  70    /* consider GRF only */
  71    assert(inst->dst.file == VGRF);
  72    if (!src || src->file != VGRF)
  73       return false;
  74
  75    return (src->in_range(inst->dst, inst->regs_written) &&
  76            inst->dst.writemask & (1 << BRW_GET_SWZ(src->swizzle, ch)));
  77 }
  78
  79 static unsigned
  80 swizzle_vf_imm(unsigned vf4, unsigned swizzle)
  81 {
  82    union {
  83       unsigned vf4;
  84       uint8_t vf[4];
  85    } v = { vf4 }, ret;
  86
  87    ret.vf[0] = v.vf[BRW_GET_SWZ(swizzle, 0)];
  88    ret.vf[1] = v.vf[BRW_GET_SWZ(swizzle, 1)];
  89    ret.vf[2] = v.vf[BRW_GET_SWZ(swizzle, 2)];
  90    ret.vf[3] = v.vf[BRW_GET_SWZ(swizzle, 3)];
  91
  92    return ret.vf4;
  93 }
  94
  95 static bool
  96 is_logic_op(enum opcode opcode)
  97 {
  98    return (opcode == BRW_OPCODE_AND ||
  99            opcode == BRW_OPCODE_OR  ||
 100            opcode == BRW_OPCODE_XOR ||
 101            opcode == BRW_OPCODE_NOT);
 102 }
 103
 104 static bool
 105 try_constant_propagate(const struct brw_device_info *devinfo,
 106                        vec4_instruction *inst,
 107                        int arg, struct copy_entry *entry)
 108 {
 109    /* For constant propagation, we only handle the same constant
 110     * across all 4 channels.  Some day, we should handle the 8-bit
 111     * float vector format, which would let us constant propagate
 112     * vectors better.
 113     */
 114    src_reg value = *entry->value[0];
 115    for (int i = 1; i < 4; i++) {
 116       if (!value.equals(*entry->value[i]))
 117          return false;
 118    }
 119
 120    if (value.file != IMM)
 121       return false;
 122
 123    if (value.type == BRW_REGISTER_TYPE_VF) {
 124       /* The result of bit-casting the component values of a vector float
 125        * cannot in general be represented as an immediate.
 126        */
 127       if (inst->src[arg].type != BRW_REGISTER_TYPE_F)
 128          return false;
 129    } else {
 130       value.type = inst->src[arg].type;
 131    }
 132
 133    if (inst->src[arg].abs) {
 134       if ((devinfo->gen >= 8 && is_logic_op(inst->opcode)) ||
 135           !brw_abs_immediate(value.type, &value.as_brw_reg())) {
 136          return false;
 137       }
 138    }
 139
 140    if (inst->src[arg].negate) {
 141       if ((devinfo->gen >= 8 && is_logic_op(inst->opcode)) ||
 142           !brw_negate_immediate(value.type, &value.as_brw_reg())) {
 143          return false;
 144       }
 145    }
 146
 147    if (value.type == BRW_REGISTER_TYPE_VF)
 148       value.ud = swizzle_vf_imm(value.ud, inst->src[arg].swizzle);
 149
 150    switch (inst->opcode) {
 151    case BRW_OPCODE_MOV:
 152    case SHADER_OPCODE_BROADCAST:
 153       inst->src[arg] = value;
 154       return true;
 155
 156    case SHADER_OPCODE_POW:
 157    case SHADER_OPCODE_INT_QUOTIENT:
 158    case SHADER_OPCODE_INT_REMAINDER:
 159       if (devinfo->gen < 8)
 160          break;
 161       /* fallthrough */
 162    case BRW_OPCODE_DP2:
 163    case BRW_OPCODE_DP3:
 164    case BRW_OPCODE_DP4:
 165    case BRW_OPCODE_DPH:
 166    case BRW_OPCODE_BFI1:
 167    case BRW_OPCODE_ASR:
 168    case BRW_OPCODE_SHL:
 169    case BRW_OPCODE_SHR:
 170    case BRW_OPCODE_SUBB:
 171       if (arg == 1) {
 172          inst->src[arg] = value;
 173          return true;
 174       }
 175       break;
 176
 177    case BRW_OPCODE_MACH:
 178    case BRW_OPCODE_MUL:
 179    case SHADER_OPCODE_MULH:
 180    case BRW_OPCODE_ADD:
 181    case BRW_OPCODE_OR:
 182    case BRW_OPCODE_AND:
 183    case BRW_OPCODE_XOR:
 184    case BRW_OPCODE_ADDC:
 185       if (arg == 1) {
 186          inst->src[arg] = value;
 187          return true;
 188       } else if (arg == 0 && inst->src[1].file != IMM) {
 189          /* Fit this constant in by commuting the operands.  Exception: we
 190           * can't do this for 32-bit integer MUL/MACH because it's asymmetric.
 191           */
 192          if ((inst->opcode == BRW_OPCODE_MUL ||
 193               inst->opcode == BRW_OPCODE_MACH) &&
 194              (inst->src[1].type == BRW_REGISTER_TYPE_D ||
 195               inst->src[1].type == BRW_REGISTER_TYPE_UD))
 196             break;
 197          inst->src[0] = inst->src[1];
 198          inst->src[1] = value;
 199          return true;
 200       }
 201       break;
 202    case GS_OPCODE_SET_WRITE_OFFSET:
 203       /* This is just a multiply by a constant with special strides.
 204        * The generator will handle immediates in both arguments (generating
 205        * a single MOV of the product).  So feel free to propagate in src0.
 206        */
 207       inst->src[arg] = value;
 208       return true;
 209
 210    case BRW_OPCODE_CMP:
 211       if (arg == 1) {
 212          inst->src[arg] = value;
 213          return true;
 214       } else if (arg == 0 && inst->src[1].file != IMM) {
 215          enum brw_conditional_mod new_cmod;
 216
 217          new_cmod = brw_swap_cmod(inst->conditional_mod);
 218          if (new_cmod != BRW_CONDITIONAL_NONE) {
 219             /* Fit this constant in by swapping the operands and
 220              * flipping the test.
 221              */
 222             inst->src[0] = inst->src[1];
 223             inst->src[1] = value;
 224             inst->conditional_mod = new_cmod;
 225             return true;
 226          }
 227       }
 228       break;
 229
 230    case BRW_OPCODE_SEL:
 231       if (arg == 1) {
 232          inst->src[arg] = value;
 233          return true;
 234       } else if (arg == 0 && inst->src[1].file != IMM) {
 235          inst->src[0] = inst->src[1];
 236          inst->src[1] = value;
 237
 238          /* If this was predicated, flipping operands means
 239           * we also need to flip the predicate.
 240           */
 241          if (inst->conditional_mod == BRW_CONDITIONAL_NONE) {
 242             inst->predicate_inverse = !inst->predicate_inverse;
 243          }
 244          return true;
 245       }
 246       break;
 247
 248    default:
 249       break;
 250    }
 251
 252    return false;
 253 }
 254
 255 static bool
 256 try_copy_propagate(const struct brw_device_info *devinfo,
 257                    vec4_instruction *inst, int arg,
 258                    struct copy_entry *entry, int attributes_per_reg)
 259 {
 260    /* Build up the value we are propagating as if it were the source of a
 261     * single MOV
 262     */
 263    /* For constant propagation, we only handle the same constant
 264     * across all 4 channels.  Some day, we should handle the 8-bit
 265     * float vector format, which would let us constant propagate
 266     * vectors better.
 267     */
 268    src_reg value = *entry->value[0];
 269    for (int i = 1; i < 4; i++) {
 270       /* This is equals() except we don't care about the swizzle. */
 271       if (value.file != entry->value[i]->file ||
 272           value.nr != entry->value[i]->nr ||
 273           value.reg_offset != entry->value[i]->reg_offset ||
 274           value.type != entry->value[i]->type ||
 275           value.negate != entry->value[i]->negate ||
 276           value.abs != entry->value[i]->abs) {
 277          return false;
 278       }
 279    }
 280
 281    /* Compute the swizzle of the original register by swizzling the
 282     * component loaded from each value according to the swizzle of
 283     * operand we're going to change.
 284     */
 285    int s[4];
 286    for (int i = 0; i < 4; i++) {
 287       s[i] = BRW_GET_SWZ(entry->value[i]->swizzle, i);
 288    }
 289    value.swizzle = BRW_SWIZZLE4(s[0], s[1], s[2], s[3]);
 290
 291    /* Check that we can propagate that value */
 292    if (value.file != UNIFORM &&
 293        value.file != VGRF &&
 294        value.file != ATTR)
 295       return false;
 296
 297    if (devinfo->gen >= 8 && (value.negate || value.abs) &&
 298        is_logic_op(inst->opcode)) {
 299       return false;
 300    }
 301
 302    bool has_source_modifiers = value.negate || value.abs;
 303
 304    /* gen6 math and gen7+ SENDs from GRFs ignore source modifiers on
 305     * instructions.
 306     */
 307    if ((has_source_modifiers || value.file == UNIFORM ||
 308         value.swizzle != BRW_SWIZZLE_XYZW) && !inst->can_do_source_mods(devinfo))
 309       return false;
 310
 311    if (has_source_modifiers &&
 312        value.type != inst->src[arg].type &&
 313        !inst->can_change_types())
 314       return false;
 315
 316    if (has_source_modifiers &&
 317        inst->opcode == SHADER_OPCODE_GEN4_SCRATCH_WRITE)
 318       return false;
 319
 320    unsigned composed_swizzle = brw_compose_swizzle(inst->src[arg].swizzle,
 321                                                    value.swizzle);
 322    if (inst->is_3src() &&
 323        (value.file == UNIFORM ||
 324         (value.file == ATTR && attributes_per_reg != 1)) &&
 325        !brw_is_single_value_swizzle(composed_swizzle))
 326       return false;
 327
 328    if (inst->is_send_from_grf())
 329       return false;
 330
 331    /* we can't generally copy-propagate UD negations becuse we
 332     * end up accessing the resulting values as signed integers
 333     * instead. See also resolve_ud_negate().
 334     */
 335    if (value.negate &&
 336        value.type == BRW_REGISTER_TYPE_UD)
 337       return false;
 338
 339    /* Don't report progress if this is a noop. */
 340    if (value.equals(inst->src[arg]))
 341       return false;
 342
 343    const unsigned dst_saturate_mask = inst->dst.writemask &
 344       brw_apply_swizzle_to_mask(inst->src[arg].swizzle, entry->saturatemask);
 345
 346    if (dst_saturate_mask) {
 347       /* We either saturate all or nothing. */
 348       if (dst_saturate_mask != inst->dst.writemask)
 349          return false;
 350
 351       /* Limit saturate propagation only to SEL with src1 bounded within 0.0
 352        * and 1.0, otherwise skip copy propagate altogether.
 353        */
 354       switch(inst->opcode) {
 355       case BRW_OPCODE_SEL:
 356          if (arg != 0 ||
 357              inst->src[0].type != BRW_REGISTER_TYPE_F ||
 358              inst->src[1].file != IMM ||
 359              inst->src[1].type != BRW_REGISTER_TYPE_F ||
 360              inst->src[1].f < 0.0 ||
 361              inst->src[1].f > 1.0) {
 362             return false;
 363          }
 364          if (!inst->saturate)
 365             inst->saturate = true;
 366          break;
 367       default:
 368          return false;
 369       }
 370    }
 371
 372    /* Build the final value */
 373    if (inst->src[arg].abs) {
 374       value.negate = false;
 375       value.abs = true;
 376    }
 377    if (inst->src[arg].negate)
 378       value.negate = !value.negate;
 379
 380    value.swizzle = composed_swizzle;
 381    if (has_source_modifiers &&
 382        value.type != inst->src[arg].type) {
 383       assert(inst->can_change_types());
 384       for (int i = 0; i < 3; i++) {
 385          inst->src[i].type = value.type;
 386       }
 387       inst->dst.type = value.type;
 388    } else {
 389       value.type = inst->src[arg].type;
 390    }
 391
 392    inst->src[arg] = value;
 393    return true;
 394 }
 395
 396 bool
 397 vec4_visitor::opt_copy_propagation(bool do_constant_prop)
 398 {
 399    /* If we are in dual instanced or single mode, then attributes are going
 400     * to be interleaved, so one register contains two attribute slots.
 401     */
 402    const int attributes_per_reg =
 403       prog_data->dispatch_mode == DISPATCH_MODE_4X2_DUAL_OBJECT ? 1 : 2;
 404    bool progress = false;
 405    struct copy_entry entries[alloc.total_size];
 406
 407    memset(&entries, 0, sizeof(entries));
 408
 409    foreach_block_and_inst(block, vec4_instruction, inst, cfg) {
 410       /* This pass only works on basic blocks.  If there's flow
 411        * control, throw out all our information and start from
 412        * scratch.
 413        *
 414        * This should really be fixed by using a structure like in
 415        * src/glsl/opt_copy_propagation.cpp to track available copies.
 416        */
 417       if (!is_dominated_by_previous_instruction(inst)) {
 418          memset(&entries, 0, sizeof(entries));
 419          continue;
 420       }
 421
 422       /* For each source arg, see if each component comes from a copy
 423        * from the same type file (IMM, VGRF, UNIFORM), and try
 424        * optimizing out access to the copy result
 425        */
 426       for (int i = 2; i >= 0; i--) {
 427          /* Copied values end up in GRFs, and we don't track reladdr
 428           * accesses.
 429           */
 430          if (inst->src[i].file != VGRF ||
 431              inst->src[i].reladdr)
 432             continue;
 433
 434          /* We only handle single-register copies. */
 435          if (inst->regs_read(i) != 1)
 436             continue;
 437
 438          int reg = (alloc.offsets[inst->src[i].nr] +
 439                     inst->src[i].reg_offset);
 440
 441          /* Find the regs that each swizzle component came from.
 442           */
 443          struct copy_entry entry;
 444          memset(&entry, 0, sizeof(copy_entry));
 445          int c;
 446          for (c = 0; c < 4; c++) {
 447             int channel = BRW_GET_SWZ(inst->src[i].swizzle, c);
 448             entry.value[c] = entries[reg].value[channel];
 449
 450             /* If there's no available copy for this channel, bail.
 451              * We could be more aggressive here -- some channels might
 452              * not get used based on the destination writemask.
 453              */
 454             if (!entry.value[c])
 455                break;
 456
 457             entry.saturatemask |=
 458                (entries[reg].saturatemask & (1 << channel) ? 1 : 0) << c;
 459
 460             /* We'll only be able to copy propagate if the sources are
 461              * all from the same file -- there's no ability to swizzle
 462              * 0 or 1 constants in with source registers like in i915.
 463              */
 464             if (c > 0 && entry.value[c - 1]->file != entry.value[c]->file)
 465                break;
 466          }
 467
 468          if (c != 4)
 469             continue;
 470
 471          if (do_constant_prop && try_constant_propagate(devinfo, inst, i, &entry))
 472             progress = true;
 473
 474          if (try_copy_propagate(devinfo, inst, i, &entry, attributes_per_reg))
 475             progress = true;
 476       }
 477
 478       /* Track available source registers. */
 479       if (inst->dst.file == VGRF) {
 480          const int reg =
 481             alloc.offsets[inst->dst.nr] + inst->dst.reg_offset;
 482
 483          /* Update our destination's current channel values.  For a direct copy,
 484           * the value is the newly propagated source.  Otherwise, we don't know
 485           * the new value, so clear it.
 486           */
 487          bool direct_copy = is_direct_copy(inst);
 488          entries[reg].saturatemask &= ~inst->dst.writemask;
 489          for (int i = 0; i < 4; i++) {
 490             if (inst->dst.writemask & (1 << i)) {
 491                entries[reg].value[i] = direct_copy ? &inst->src[0] : NULL;
 492                entries[reg].saturatemask |=
 493                   inst->saturate && direct_copy ? 1 << i : 0;
 494             }
 495          }
 496
 497          /* Clear the records for any registers whose current value came from
 498           * our destination's updated channels, as the two are no longer equal.
 499           */
 500          if (inst->dst.reladdr)
 501             memset(&entries, 0, sizeof(entries));
 502          else {
 503             for (unsigned i = 0; i < alloc.total_size; i++) {
 504                for (int j = 0; j < 4; j++) {
 505                   if (is_channel_updated(inst, entries[i].value, j)) {
 506                      entries[i].value[j] = NULL;
 507                      entries[i].saturatemask &= ~(1 << j);
 508                   }
 509                }
 510             }
 511          }
 512       }
 513    }
 514
 515    if (progress)
 516       invalidate_live_intervals();
 517
 518    return progress;
 519 }
 520
 521 } /* namespace brw */