i965: Remove the create_raw_surface vtbl hook.
[mesa.git] / src / mesa / drivers / dri / i965 / brw_fs_copy_propagation.cpp
index 2816d3c14ee4adf87fa988b76171cfab35a2e469..764741d369ef9a82c96aa668f24febd9562e0baa 100644 (file)
@@ -34,7 +34,7 @@
 
 #define ACP_HASH_SIZE 16
 
-#include "main/bitset.h"
+#include "util/bitset.h"
 #include "brw_fs.h"
 #include "brw_cfg.h"
 
@@ -42,6 +42,9 @@ namespace { /* avoid conflict with opt_copy_propagation_elements */
 struct acp_entry : public exec_node {
    fs_reg dst;
    fs_reg src;
+   uint8_t regs_written;
+   enum opcode opcode;
+   bool saturate;
 };
 
 struct block_data {
@@ -103,11 +106,9 @@ fs_copy_prop_dataflow::fs_copy_prop_dataflow(void *mem_ctx, cfg_t *cfg,
    bd = rzalloc_array(mem_ctx, struct block_data, cfg->num_blocks);
 
    num_acp = 0;
-   for (int b = 0; b < cfg->num_blocks; b++) {
+   foreach_block (block, cfg) {
       for (int i = 0; i < ACP_HASH_SIZE; i++) {
-         foreach_list(entry_node, &out_acp[b][i]) {
-            num_acp++;
-         }
+         num_acp += out_acp[block->num][i].length();
       }
    }
 
@@ -116,23 +117,21 @@ fs_copy_prop_dataflow::fs_copy_prop_dataflow(void *mem_ctx, cfg_t *cfg,
    bitset_words = BITSET_WORDS(num_acp);
 
    int next_acp = 0;
-   for (int b = 0; b < cfg->num_blocks; b++) {
-      bd[b].livein = rzalloc_array(bd, BITSET_WORD, bitset_words);
-      bd[b].liveout = rzalloc_array(bd, BITSET_WORD, bitset_words);
-      bd[b].copy = rzalloc_array(bd, BITSET_WORD, bitset_words);
-      bd[b].kill = rzalloc_array(bd, BITSET_WORD, bitset_words);
+   foreach_block (block, cfg) {
+      bd[block->num].livein = rzalloc_array(bd, BITSET_WORD, bitset_words);
+      bd[block->num].liveout = rzalloc_array(bd, BITSET_WORD, bitset_words);
+      bd[block->num].copy = rzalloc_array(bd, BITSET_WORD, bitset_words);
+      bd[block->num].kill = rzalloc_array(bd, BITSET_WORD, bitset_words);
 
       for (int i = 0; i < ACP_HASH_SIZE; i++) {
-         foreach_list(entry_node, &out_acp[b][i]) {
-            acp_entry *entry = (acp_entry *)entry_node;
-
+         foreach_in_list(acp_entry, entry, &out_acp[block->num][i]) {
             acp[next_acp] = entry;
 
             /* opt_copy_propagate_local populates out_acp with copies created
              * in a block which are still live at the end of the block.  This
              * is exactly what we want in the COPY set.
              */
-            BITSET_SET(bd[b].copy, next_acp);
+            BITSET_SET(bd[block->num].copy, next_acp);
 
             next_acp++;
          }
@@ -153,12 +152,8 @@ void
 fs_copy_prop_dataflow::setup_initial_values()
 {
    /* Initialize the COPY and KILL sets. */
-   for (int b = 0; b < cfg->num_blocks; b++) {
-      bblock_t *block = cfg->blocks[b];
-
-      for (fs_inst *inst = (fs_inst *)block->start;
-           inst != block->end->next;
-           inst = (fs_inst *)inst->next) {
+   foreach_block (block, cfg) {
+      foreach_inst_in_block(fs_inst, inst, block) {
          if (inst->dst.file != GRF)
             continue;
 
@@ -166,7 +161,7 @@ fs_copy_prop_dataflow::setup_initial_values()
          for (int i = 0; i < num_acp; i++) {
             if (inst->overwrites_reg(acp[i]->dst) ||
                 inst->overwrites_reg(acp[i]->src)) {
-               BITSET_SET(bd[b].kill, i);
+               BITSET_SET(bd[block->num].kill, i);
             }
          }
       }
@@ -177,17 +172,16 @@ fs_copy_prop_dataflow::setup_initial_values()
     * For the others, set liveout to 0 (the empty set) and livein to ~0
     * (the universal set).
     */
-   for (int b = 0; b < cfg->num_blocks; b++) {
-      bblock_t *block = cfg->blocks[b];
+   foreach_block (block, cfg) {
       if (block->parents.is_empty()) {
          for (int i = 0; i < bitset_words; i++) {
-            bd[b].livein[i] = 0u;
-            bd[b].liveout[i] = bd[b].copy[i];
+            bd[block->num].livein[i] = 0u;
+            bd[block->num].liveout[i] = bd[block->num].copy[i];
          }
       } else {
          for (int i = 0; i < bitset_words; i++) {
-            bd[b].liveout[i] = 0u;
-            bd[b].livein[i] = ~0u;
+            bd[block->num].liveout[i] = 0u;
+            bd[block->num].livein[i] = ~0u;
          }
       }
    }
@@ -206,17 +200,18 @@ fs_copy_prop_dataflow::run()
       progress = false;
 
       /* Update liveout for all blocks. */
-      for (int b = 0; b < cfg->num_blocks; b++) {
-         if (cfg->blocks[b]->parents.is_empty())
+      foreach_block (block, cfg) {
+         if (block->parents.is_empty())
             continue;
 
          for (int i = 0; i < bitset_words; i++) {
-            const BITSET_WORD old_liveout = bd[b].liveout[i];
+            const BITSET_WORD old_liveout = bd[block->num].liveout[i];
 
-            bd[b].liveout[i] =
-               bd[b].copy[i] | (bd[b].livein[i] & ~bd[b].kill[i]);
+            bd[block->num].liveout[i] =
+               bd[block->num].copy[i] | (bd[block->num].livein[i] &
+                                         ~bd[block->num].kill[i]);
 
-            if (old_liveout != bd[b].liveout[i])
+            if (old_liveout != bd[block->num].liveout[i])
                progress = true;
          }
       }
@@ -224,21 +219,20 @@ fs_copy_prop_dataflow::run()
       /* Update livein for all blocks.  If a copy is live out of all parent
        * blocks, it's live coming in to this block.
        */
-      for (int b = 0; b < cfg->num_blocks; b++) {
-         if (cfg->blocks[b]->parents.is_empty())
+      foreach_block (block, cfg) {
+         if (block->parents.is_empty())
             continue;
 
          for (int i = 0; i < bitset_words; i++) {
-            const BITSET_WORD old_livein = bd[b].livein[i];
+            const BITSET_WORD old_livein = bd[block->num].livein[i];
 
-            bd[b].livein[i] = ~0u;
-            foreach_list(block_node, &cfg->blocks[b]->parents) {
-               bblock_link *link = (bblock_link *)block_node;
-               bblock_t *block = link->block;
-               bd[b].livein[i] &= bd[block->block_num].liveout[i];
+            bd[block->num].livein[i] = ~0u;
+            foreach_list_typed(bblock_link, parent_link, link, &block->parents) {
+               bblock_t *parent = parent_link->block;
+               bd[block->num].livein[i] &= bd[parent->num].liveout[i];
             }
 
-            if (old_livein != bd[b].livein[i])
+            if (old_livein != bd[block->num].livein[i])
                progress = true;
          }
       }
@@ -248,52 +242,72 @@ fs_copy_prop_dataflow::run()
 void
 fs_copy_prop_dataflow::dump_block_data() const
 {
-   for (int b = 0; b < cfg->num_blocks; b++) {
-      bblock_t *block = cfg->blocks[b];
-      fprintf(stderr, "Block %d [%d, %d] (parents ", block->block_num,
+   foreach_block (block, cfg) {
+      fprintf(stderr, "Block %d [%d, %d] (parents ", block->num,
              block->start_ip, block->end_ip);
-      foreach_list(block_node, &block->parents) {
-         bblock_t *parent = ((bblock_link *) block_node)->block;
-         fprintf(stderr, "%d ", parent->block_num);
+      foreach_list_typed(bblock_link, link, link, &block->parents) {
+         bblock_t *parent = link->block;
+         fprintf(stderr, "%d ", parent->num);
       }
       fprintf(stderr, "):\n");
       fprintf(stderr, "       livein = 0x");
       for (int i = 0; i < bitset_words; i++)
-         fprintf(stderr, "%08x", bd[b].livein[i]);
+         fprintf(stderr, "%08x", bd[block->num].livein[i]);
       fprintf(stderr, ", liveout = 0x");
       for (int i = 0; i < bitset_words; i++)
-         fprintf(stderr, "%08x", bd[b].liveout[i]);
+         fprintf(stderr, "%08x", bd[block->num].liveout[i]);
       fprintf(stderr, ",\n       copy   = 0x");
       for (int i = 0; i < bitset_words; i++)
-         fprintf(stderr, "%08x", bd[b].copy[i]);
+         fprintf(stderr, "%08x", bd[block->num].copy[i]);
       fprintf(stderr, ", kill    = 0x");
       for (int i = 0; i < bitset_words; i++)
-         fprintf(stderr, "%08x", bd[b].kill[i]);
+         fprintf(stderr, "%08x", bd[block->num].kill[i]);
       fprintf(stderr, "\n");
    }
 }
 
+static bool
+is_logic_op(enum opcode opcode)
+{
+   return (opcode == BRW_OPCODE_AND ||
+           opcode == BRW_OPCODE_OR  ||
+           opcode == BRW_OPCODE_XOR ||
+           opcode == BRW_OPCODE_NOT);
+}
+
 bool
 fs_visitor::try_copy_propagate(fs_inst *inst, int arg, acp_entry *entry)
 {
+   if (inst->src[arg].file != GRF)
+      return false;
+
    if (entry->src.file == IMM)
       return false;
+   assert(entry->src.file == GRF || entry->src.file == UNIFORM);
 
-   /* Bail if inst is reading more than entry is writing. */
-   if ((inst->regs_read(this, arg) * inst->src[arg].stride *
-        type_sz(inst->src[arg].type)) > type_sz(entry->dst.type))
+   if (entry->opcode == SHADER_OPCODE_LOAD_PAYLOAD &&
+       inst->opcode == SHADER_OPCODE_LOAD_PAYLOAD)
       return false;
 
-   if (inst->src[arg].file != entry->dst.file ||
-       inst->src[arg].reg != entry->dst.reg ||
-       inst->src[arg].reg_offset != entry->dst.reg_offset ||
-       inst->src[arg].subreg_offset != entry->dst.subreg_offset) {
+   assert(entry->dst.file == GRF);
+   if (inst->src[arg].reg != entry->dst.reg)
       return false;
-   }
 
-   /* See resolve_ud_negate() and comment in brw_fs_emit.cpp. */
-   if (inst->conditional_mod &&
-       inst->src[arg].type == BRW_REGISTER_TYPE_UD &&
+   /* Bail if inst is reading a range that isn't contained in the range
+    * that entry is writing.
+    */
+   if (inst->src[arg].reg_offset < entry->dst.reg_offset ||
+       (inst->src[arg].reg_offset * 32 + inst->src[arg].subreg_offset +
+        inst->regs_read(arg) * inst->src[arg].stride * 32) >
+       (entry->dst.reg_offset + entry->regs_written) * 32)
+      return false;
+
+   /* we can't generally copy-propagate UD negations because we
+    * can end up accessing the resulting values as signed integers
+    * instead. See also resolve_ud_negate() and comment in
+    * fs_generator::generate_code.
+    */
+   if (inst->src[arg].type == BRW_REGISTER_TYPE_UD &&
        entry->src.negate)
       return false;
 
@@ -301,7 +315,11 @@ fs_visitor::try_copy_propagate(fs_inst *inst, int arg, acp_entry *entry)
 
    if ((has_source_modifiers || entry->src.file == UNIFORM ||
         !entry->src.is_contiguous()) &&
-       !can_do_source_mods(inst))
+       !inst->can_do_source_mods(brw))
+      return false;
+
+   if (has_source_modifiers &&
+       inst->opcode == SHADER_OPCODE_GEN4_SCRATCH_WRITE)
       return false;
 
    /* Bail if the result of composing both strides would exceed the
@@ -331,11 +349,68 @@ fs_visitor::try_copy_propagate(fs_inst *inst, int arg, acp_entry *entry)
    if (has_source_modifiers && entry->dst.type != inst->src[arg].type)
       return false;
 
+   if (brw->gen >= 8 && (entry->src.negate || entry->src.abs) &&
+       is_logic_op(inst->opcode)) {
+      return false;
+   }
+
+   if (entry->saturate) {
+      switch(inst->opcode) {
+      case BRW_OPCODE_SEL:
+         if (inst->src[1].file != IMM ||
+             inst->src[1].fixed_hw_reg.dw1.f < 0.0 ||
+             inst->src[1].fixed_hw_reg.dw1.f > 1.0) {
+            return false;
+         }
+         break;
+      default:
+         return false;
+      }
+   }
+
    inst->src[arg].file = entry->src.file;
    inst->src[arg].reg = entry->src.reg;
-   inst->src[arg].reg_offset = entry->src.reg_offset;
-   inst->src[arg].subreg_offset = entry->src.subreg_offset;
    inst->src[arg].stride *= entry->src.stride;
+   inst->saturate = inst->saturate || entry->saturate;
+
+   switch (entry->src.file) {
+   case UNIFORM:
+      assert(entry->src.width == 1);
+   case BAD_FILE:
+   case HW_REG:
+      inst->src[arg].width = entry->src.width;
+      inst->src[arg].reg_offset = entry->src.reg_offset;
+      inst->src[arg].subreg_offset = entry->src.subreg_offset;
+      break;
+   case GRF:
+      {
+         assert(entry->src.width % inst->src[arg].width == 0);
+         /* In this case, we'll just leave the width alone.  The source
+          * register could have different widths depending on how it is
+          * being used.  For instance, if only half of the register was
+          * used then we want to preserve that and continue to only use
+          * half.
+          *
+          * Also, we have to deal with mapping parts of vgrfs to other
+          * parts of vgrfs so we have to do some reg_offset magic.
+          */
+
+         /* Compute the offset of inst->src[arg] relative to inst->dst */
+         assert(entry->dst.subreg_offset == 0);
+         int rel_offset = inst->src[arg].reg_offset - entry->dst.reg_offset;
+         int rel_suboffset = inst->src[arg].subreg_offset;
+
+         /* Compute the final register offset (in bytes) */
+         int offset = entry->src.reg_offset * 32 + entry->src.subreg_offset;
+         offset += rel_offset * 32 + rel_suboffset;
+         inst->src[arg].reg_offset = offset / 32;
+         inst->src[arg].subreg_offset = offset % 32;
+      }
+      break;
+   default:
+      unreachable("Invalid register file");
+      break;
+   }
 
    if (!inst->src[arg].abs) {
       inst->src[arg].abs = entry->src.abs;
@@ -353,35 +428,64 @@ fs_visitor::try_constant_propagate(fs_inst *inst, acp_entry *entry)
 
    if (entry->src.file != IMM)
       return false;
+   if (entry->saturate)
+      return false;
 
-   for (int i = 2; i >= 0; i--) {
-      if (inst->src[i].file != entry->dst.file ||
-          inst->src[i].reg != entry->dst.reg ||
-          inst->src[i].reg_offset != entry->dst.reg_offset ||
-          inst->src[i].subreg_offset != entry->dst.subreg_offset ||
-          inst->src[i].type != entry->dst.type ||
-          inst->src[i].stride > 1)
+   for (int i = inst->sources - 1; i >= 0; i--) {
+      if (inst->src[i].file != GRF)
          continue;
 
-      /* Don't bother with cases that should have been taken care of by the
-       * GLSL compiler's constant folding pass.
+      assert(entry->dst.file == GRF);
+      if (inst->src[i].reg != entry->dst.reg)
+         continue;
+
+      /* Bail if inst is reading a range that isn't contained in the range
+       * that entry is writing.
        */
-      if (inst->src[i].negate || inst->src[i].abs)
+      if (inst->src[i].reg_offset < entry->dst.reg_offset ||
+          (inst->src[i].reg_offset * 32 + inst->src[i].subreg_offset +
+           inst->regs_read(i) * inst->src[i].stride * 32) >
+          (entry->dst.reg_offset + entry->regs_written) * 32)
          continue;
 
+      fs_reg val = entry->src;
+      val.effective_width = inst->src[i].effective_width;
+      val.type = inst->src[i].type;
+
+      if (inst->src[i].abs) {
+         if ((brw->gen >= 8 && is_logic_op(inst->opcode)) ||
+             !brw_abs_immediate(val.type, &val.fixed_hw_reg)) {
+            continue;
+         }
+      }
+
+      if (inst->src[i].negate) {
+         if ((brw->gen >= 8 && is_logic_op(inst->opcode)) ||
+             !brw_negate_immediate(val.type, &val.fixed_hw_reg)) {
+            continue;
+         }
+      }
+
       switch (inst->opcode) {
       case BRW_OPCODE_MOV:
-         inst->src[i] = entry->src;
+      case SHADER_OPCODE_LOAD_PAYLOAD:
+         inst->src[i] = val;
          progress = true;
          break;
 
+      case SHADER_OPCODE_POW:
+      case SHADER_OPCODE_INT_QUOTIENT:
+      case SHADER_OPCODE_INT_REMAINDER:
+         if (brw->gen < 8)
+            break;
+         /* fallthrough */
       case BRW_OPCODE_BFI1:
       case BRW_OPCODE_ASR:
       case BRW_OPCODE_SHL:
       case BRW_OPCODE_SHR:
       case BRW_OPCODE_SUBB:
          if (i == 1) {
-            inst->src[i] = entry->src;
+            inst->src[i] = val;
             progress = true;
          }
          break;
@@ -394,7 +498,7 @@ fs_visitor::try_constant_propagate(fs_inst *inst, acp_entry *entry)
       case BRW_OPCODE_XOR:
       case BRW_OPCODE_ADDC:
          if (i == 1) {
-            inst->src[i] = entry->src;
+            inst->src[i] = val;
             progress = true;
          } else if (i == 0 && inst->src[1].file != IMM) {
             /* Fit this constant in by commuting the operands.
@@ -407,7 +511,7 @@ fs_visitor::try_constant_propagate(fs_inst *inst, acp_entry *entry)
                  inst->src[1].type == BRW_REGISTER_TYPE_UD))
                break;
             inst->src[0] = inst->src[1];
-            inst->src[1] = entry->src;
+            inst->src[1] = val;
             progress = true;
          }
          break;
@@ -415,18 +519,18 @@ fs_visitor::try_constant_propagate(fs_inst *inst, acp_entry *entry)
       case BRW_OPCODE_CMP:
       case BRW_OPCODE_IF:
          if (i == 1) {
-            inst->src[i] = entry->src;
+            inst->src[i] = val;
             progress = true;
          } else if (i == 0 && inst->src[1].file != IMM) {
-            uint32_t new_cmod;
+            enum brw_conditional_mod new_cmod;
 
             new_cmod = brw_swap_cmod(inst->conditional_mod);
-            if (new_cmod != ~0u) {
+            if (new_cmod != BRW_CONDITIONAL_NONE) {
                /* Fit this constant in by swapping the operands and
                 * flipping the test
                 */
                inst->src[0] = inst->src[1];
-               inst->src[1] = entry->src;
+               inst->src[1] = val;
                inst->conditional_mod = new_cmod;
                progress = true;
             }
@@ -435,11 +539,11 @@ fs_visitor::try_constant_propagate(fs_inst *inst, acp_entry *entry)
 
       case BRW_OPCODE_SEL:
          if (i == 1) {
-            inst->src[i] = entry->src;
+            inst->src[i] = val;
             progress = true;
          } else if (i == 0 && inst->src[1].file != IMM) {
             inst->src[0] = inst->src[1];
-            inst->src[1] = entry->src;
+            inst->src[1] = val;
 
             /* If this was predicated, flipping operands means
              * we also need to flip the predicate.
@@ -459,16 +563,22 @@ fs_visitor::try_constant_propagate(fs_inst *inst, acp_entry *entry)
           * anyway.
           */
          assert(i == 0);
-         if (inst->src[0].imm.f != 0.0f) {
+         if (inst->src[0].fixed_hw_reg.dw1.f != 0.0f) {
             inst->opcode = BRW_OPCODE_MOV;
-            inst->src[0] = entry->src;
-            inst->src[0].imm.f = 1.0f / inst->src[0].imm.f;
+            inst->src[0] = val;
+            inst->src[0].fixed_hw_reg.dw1.f = 1.0f / inst->src[0].fixed_hw_reg.dw1.f;
             progress = true;
          }
          break;
 
       case FS_OPCODE_UNIFORM_PULL_CONSTANT_LOAD:
-         inst->src[i] = entry->src;
+         inst->src[i] = val;
+         progress = true;
+         break;
+
+      case BRW_OPCODE_MAD:
+      case BRW_OPCODE_LRP:
+         inst->src[i] = val;
          progress = true;
          break;
 
@@ -479,27 +589,37 @@ fs_visitor::try_constant_propagate(fs_inst *inst, acp_entry *entry)
 
    return progress;
 }
+
+static bool
+can_propagate_from(fs_inst *inst)
+{
+   return (inst->opcode == BRW_OPCODE_MOV &&
+           inst->dst.file == GRF &&
+           ((inst->src[0].file == GRF &&
+             (inst->src[0].reg != inst->dst.reg ||
+              inst->src[0].reg_offset != inst->dst.reg_offset)) ||
+            inst->src[0].file == UNIFORM ||
+            inst->src[0].file == IMM) &&
+           inst->src[0].type == inst->dst.type &&
+           !inst->is_partial_write());
+}
+
 /* Walks a basic block and does copy propagation on it using the acp
  * list.
  */
 bool
-fs_visitor::opt_copy_propagate_local(void *mem_ctx, bblock_t *block,
+fs_visitor::opt_copy_propagate_local(void *copy_prop_ctx, bblock_t *block,
                                      exec_list *acp)
 {
    bool progress = false;
 
-   for (fs_inst *inst = (fs_inst *)block->start;
-       inst != block->end->next;
-       inst = (fs_inst *)inst->next) {
-
+   foreach_inst_in_block(fs_inst, inst, block) {
       /* Try propagating into this instruction. */
-      for (int i = 0; i < 3; i++) {
+      for (int i = 0; i < inst->sources; i++) {
          if (inst->src[i].file != GRF)
             continue;
 
-         foreach_list(entry_node, &acp[inst->src[i].reg % ACP_HASH_SIZE]) {
-            acp_entry *entry = (acp_entry *)entry_node;
-
+         foreach_in_list(acp_entry, entry, &acp[inst->src[i].reg % ACP_HASH_SIZE]) {
             if (try_constant_propagate(inst, entry))
                progress = true;
 
@@ -510,9 +630,7 @@ fs_visitor::opt_copy_propagate_local(void *mem_ctx, bblock_t *block,
 
       /* kill the destination from the ACP */
       if (inst->dst.file == GRF) {
-        foreach_list_safe(entry_node, &acp[inst->dst.reg % ACP_HASH_SIZE]) {
-           acp_entry *entry = (acp_entry *)entry_node;
-
+        foreach_in_list_safe(acp_entry, entry, &acp[inst->dst.reg % ACP_HASH_SIZE]) {
            if (inst->overwrites_reg(entry->dst)) {
               entry->remove();
            }
@@ -522,8 +640,7 @@ fs_visitor::opt_copy_propagate_local(void *mem_ctx, bblock_t *block,
           * the source, so walk across the entire table.
           */
          for (int i = 0; i < ACP_HASH_SIZE; i++) {
-            foreach_list_safe(entry_node, &acp[i]) {
-               acp_entry *entry = (acp_entry *)entry_node;
+            foreach_in_list_safe(acp_entry, entry, &acp[i]) {
                if (inst->overwrites_reg(entry->src))
                   entry->remove();
             }
@@ -533,20 +650,36 @@ fs_visitor::opt_copy_propagate_local(void *mem_ctx, bblock_t *block,
       /* If this instruction's source could potentially be folded into the
        * operand of another instruction, add it to the ACP.
        */
-      if (inst->opcode == BRW_OPCODE_MOV &&
-         inst->dst.file == GRF &&
-         ((inst->src[0].file == GRF &&
-           (inst->src[0].reg != inst->dst.reg ||
-            inst->src[0].reg_offset != inst->dst.reg_offset)) ||
-           inst->src[0].file == UNIFORM ||
-           inst->src[0].file == IMM) &&
-         inst->src[0].type == inst->dst.type &&
-         !inst->saturate &&
-         !inst->is_partial_write()) {
-        acp_entry *entry = ralloc(mem_ctx, acp_entry);
+      if (can_propagate_from(inst)) {
+        acp_entry *entry = ralloc(copy_prop_ctx, acp_entry);
         entry->dst = inst->dst;
         entry->src = inst->src[0];
+         entry->regs_written = inst->regs_written;
+         entry->opcode = inst->opcode;
+         entry->saturate = inst->saturate;
         acp[entry->dst.reg % ACP_HASH_SIZE].push_tail(entry);
+      } else if (inst->opcode == SHADER_OPCODE_LOAD_PAYLOAD &&
+                 inst->dst.file == GRF) {
+         int offset = 0;
+         for (int i = 0; i < inst->sources; i++) {
+            int regs_written = ((inst->src[i].effective_width *
+                                 type_sz(inst->src[i].type)) + 31) / 32;
+            if (inst->src[i].file == GRF) {
+               acp_entry *entry = ralloc(copy_prop_ctx, acp_entry);
+               entry->dst = inst->dst;
+               entry->dst.reg_offset = offset;
+               entry->dst.width = inst->src[i].effective_width;
+               entry->src = inst->src[i];
+               entry->regs_written = regs_written;
+               entry->opcode = inst->opcode;
+               if (!entry->dst.equals(inst->src[i])) {
+                  acp[entry->dst.reg % ACP_HASH_SIZE].push_tail(entry);
+               } else {
+                  ralloc_free(entry);
+               }
+            }
+            offset += regs_written;
+         }
       }
    }
 
@@ -557,45 +690,42 @@ bool
 fs_visitor::opt_copy_propagate()
 {
    bool progress = false;
-   void *mem_ctx = ralloc_context(this->mem_ctx);
-   cfg_t cfg(&instructions);
-   exec_list *out_acp[cfg.num_blocks];
-   for (int i = 0; i < cfg.num_blocks; i++)
+   void *copy_prop_ctx = ralloc_context(NULL);
+   exec_list *out_acp[cfg->num_blocks];
+
+   for (int i = 0; i < cfg->num_blocks; i++)
       out_acp[i] = new exec_list [ACP_HASH_SIZE];
 
    /* First, walk through each block doing local copy propagation and getting
     * the set of copies available at the end of the block.
     */
-   for (int b = 0; b < cfg.num_blocks; b++) {
-      bblock_t *block = cfg.blocks[b];
-
-      progress = opt_copy_propagate_local(mem_ctx, block,
-                                          out_acp[b]) || progress;
+   foreach_block (block, cfg) {
+      progress = opt_copy_propagate_local(copy_prop_ctx, block,
+                                          out_acp[block->num]) || progress;
    }
 
    /* Do dataflow analysis for those available copies. */
-   fs_copy_prop_dataflow dataflow(mem_ctx, &cfg, out_acp);
+   fs_copy_prop_dataflow dataflow(copy_prop_ctx, cfg, out_acp);
 
    /* Next, re-run local copy propagation, this time with the set of copies
     * provided by the dataflow analysis available at the start of a block.
     */
-   for (int b = 0; b < cfg.num_blocks; b++) {
-      bblock_t *block = cfg.blocks[b];
+   foreach_block (block, cfg) {
       exec_list in_acp[ACP_HASH_SIZE];
 
       for (int i = 0; i < dataflow.num_acp; i++) {
-         if (BITSET_TEST(dataflow.bd[b].livein, i)) {
+         if (BITSET_TEST(dataflow.bd[block->num].livein, i)) {
             struct acp_entry *entry = dataflow.acp[i];
             in_acp[entry->dst.reg % ACP_HASH_SIZE].push_tail(entry);
          }
       }
 
-      progress = opt_copy_propagate_local(mem_ctx, block, in_acp) || progress;
+      progress = opt_copy_propagate_local(copy_prop_ctx, block, in_acp) || progress;
    }
 
-   for (int i = 0; i < cfg.num_blocks; i++)
+   for (int i = 0; i < cfg->num_blocks; i++)
       delete [] out_acp[i];
-   ralloc_free(mem_ctx);
+   ralloc_free(copy_prop_ctx);
 
    if (progress)
       invalidate_live_intervals();