i965: Add _CACHE_ in brw_cache_id enum names.
[mesa.git] / src / mesa / drivers / dri / i965 / brw_eu_compact.c
index 8c30809f1d580a59af454c61b38df078e61bf82d..7117890197a2bb250d129132c3c9188338c506a4 100644 (file)
 
 /** @file brw_eu_compact.c
  *
- * Instruction compaction is a feature of gm45 and newer hardware that allows
+ * Instruction compaction is a feature of G45 and newer hardware that allows
  * for a smaller instruction encoding.
  *
  * The instruction cache is on the order of 32KB, and many programs generate
  * far more instructions than that.  The instruction cache is built to barely
- * keep up with instruction dispatch abaility in cache hit cases -- L1
+ * keep up with instruction dispatch ability in cache hit cases -- L1
  * instruction cache misses that still hit in the next level could limit
  * throughput by around 50%.
  *
  * The idea of instruction compaction is that most instructions use a tiny
  * subset of the GPU functionality, so we can encode what would be a 16 byte
  * instruction in 8 bytes using some lookup tables for various fields.
+ *
+ *
+ * Instruction compaction capabilities vary subtly by generation.
+ *
+ * G45's support for instruction compaction is very limited. Jump counts on
+ * this generation are in units of 16-byte uncompacted instructions. As such,
+ * all jump targets must be 16-byte aligned. Also, all instructions must be
+ * naturally aligned, i.e. uncompacted instructions must be 16-byte aligned.
+ * A G45-only instruction, NENOP, must be used to provide padding to align
+ * uncompacted instructions.
+ *
+ * Gen5 removes these restrictions and changes jump counts to be in units of
+ * 8-byte compacted instructions, allowing jump targets to be only 8-byte
+ * aligned. Uncompacted instructions can also be placed on 8-byte boundaries.
+ *
+ * Gen6 adds the ability to compact instructions with a limited range of
+ * immediate values. Compactable immediates have 12 unrestricted bits, and a
+ * 13th bit that's replicated through the high 20 bits, to create the 32-bit
+ * value of DW3 in the uncompacted instruction word.
+ *
+ * On Gen7 we can compact some control flow instructions with a small positive
+ * immediate in the low bits of DW3, like ENDIF with the JIP field. Other
+ * control flow instructions with UIP cannot be compacted, because of the
+ * replicated 13th bit. No control flow instructions can be compacted on Gen6
+ * since the jump count field is not in DW3.
+ *
+ *    break    JIP/UIP
+ *    cont     JIP/UIP
+ *    halt     JIP/UIP
+ *    if       JIP/UIP
+ *    else     JIP (plus UIP on BDW+)
+ *    endif    JIP
+ *    while    JIP (must be negative)
+ *
+ * Gen 8 adds support for compacting 3-src instructions.
  */
 
 #include "brw_context.h"
 #include "brw_eu.h"
 #include "intel_asm_annotation.h"
 
+static const uint32_t g45_control_index_table[32] = {
+   0b00000000000000000,
+   0b01000000000000000,
+   0b00110000000000000,
+   0b00000000000000010,
+   0b00100000000000000,
+   0b00010000000000000,
+   0b01000000000100000,
+   0b01000000100000000,
+   0b01010000000100000,
+   0b00000000100000010,
+   0b11000000000000000,
+   0b00001000100000010,
+   0b01001000100000000,
+   0b00000000100000000,
+   0b11000000000100000,
+   0b00001000100000000,
+   0b10110000000000000,
+   0b11010000000100000,
+   0b00110000100000000,
+   0b00100000100000000,
+   0b01000000000001000,
+   0b01000000000000100,
+   0b00111100000000000,
+   0b00101011000000000,
+   0b00110000000010000,
+   0b00010000100000000,
+   0b01000000000100100,
+   0b01000000000101000,
+   0b00110000000000110,
+   0b00000000000001010,
+   0b01010000000101000,
+   0b01010000000100100
+};
+
+static const uint32_t g45_datatype_table[32] = {
+   0b001000000000100001,
+   0b001011010110101101,
+   0b001000001000110001,
+   0b001111011110111101,
+   0b001011010110101100,
+   0b001000000110101101,
+   0b001000000000100000,
+   0b010100010110110001,
+   0b001100011000101101,
+   0b001000000000100010,
+   0b001000001000110110,
+   0b010000001000110001,
+   0b001000001000110010,
+   0b011000001000110010,
+   0b001111011110111100,
+   0b001000000100101000,
+   0b010100011000110001,
+   0b001010010100101001,
+   0b001000001000101001,
+   0b010000001000110110,
+   0b101000001000110001,
+   0b001011011000101101,
+   0b001000000100001001,
+   0b001011011000101100,
+   0b110100011000110001,
+   0b001000001110111101,
+   0b110000001000110001,
+   0b011000000100101010,
+   0b101000001000101001,
+   0b001011010110001100,
+   0b001000000110100001,
+   0b001010010100001000
+};
+
+static const uint16_t g45_subreg_table[32] = {
+   0b000000000000000,
+   0b000000010000000,
+   0b000001000000000,
+   0b000100000000000,
+   0b000000000100000,
+   0b100000000000000,
+   0b000000000010000,
+   0b001100000000000,
+   0b001010000000000,
+   0b000000100000000,
+   0b001000000000000,
+   0b000000000001000,
+   0b000000001000000,
+   0b000000000000001,
+   0b000010000000000,
+   0b000000010100000,
+   0b000000000000111,
+   0b000001000100000,
+   0b011000000000000,
+   0b000000110000000,
+   0b000000000000010,
+   0b000000000000100,
+   0b000000001100000,
+   0b000100000000010,
+   0b001110011000110,
+   0b001110100001000,
+   0b000110011000110,
+   0b000001000011000,
+   0b000110010000100,
+   0b001100000000110,
+   0b000000010000110,
+   0b000001000110000
+};
+
+static const uint16_t g45_src_index_table[32] = {
+   0b000000000000,
+   0b010001101000,
+   0b010110001000,
+   0b011010010000,
+   0b001101001000,
+   0b010110001010,
+   0b010101110000,
+   0b011001111000,
+   0b001000101000,
+   0b000000101000,
+   0b010001010000,
+   0b111101101100,
+   0b010110001100,
+   0b010001101100,
+   0b011010010100,
+   0b010001001100,
+   0b001100101000,
+   0b000000000010,
+   0b111101001100,
+   0b011001101000,
+   0b010101001000,
+   0b000000000100,
+   0b000000101100,
+   0b010001101010,
+   0b000000111000,
+   0b010101011000,
+   0b000100100000,
+   0b010110000000,
+   0b010000000100,
+   0b010000111000,
+   0b000101100000,
+   0b111101110100
+};
+
 static const uint32_t gen6_control_index_table[32] = {
    0b00000000000000000,
    0b01000000000000000,
@@ -496,7 +671,7 @@ static const uint16_t *src_index_table;
 static bool
 set_control_index(struct brw_context *brw, brw_compact_inst *dst, brw_inst *src)
 {
-   uint32_t uncompacted = brw->gen >= 8      /* 17b/SNB; 19b/IVB+ */
+   uint32_t uncompacted = brw->gen >= 8      /* 17b/G45; 19b/IVB+ */
       ? (brw_inst_bits(src, 33, 31) << 16) | /*  3b */
         (brw_inst_bits(src, 23, 12) <<  4) | /* 12b */
         (brw_inst_bits(src, 10,  9) <<  2) | /*  2b */
@@ -525,7 +700,7 @@ static bool
 set_datatype_index(struct brw_context *brw, brw_compact_inst *dst,
                    brw_inst *src)
 {
-   uint32_t uncompacted = brw->gen >= 8      /* 18b/SNB+; 21b/BDW+ */
+   uint32_t uncompacted = brw->gen >= 8      /* 18b/G45+; 21b/BDW+ */
       ? (brw_inst_bits(src, 63, 61) << 18) | /*  3b */
         (brw_inst_bits(src, 94, 89) << 12) | /*  6b */
         (brw_inst_bits(src, 46, 35))         /* 12b */
@@ -620,7 +795,7 @@ set_3src_control_index(struct brw_context *brw, brw_compact_inst *dst, brw_inst
       (brw_inst_bits(src, 34, 32) << 21) | /*  3b */
       (brw_inst_bits(src, 28,  8));        /* 21b */
 
-   if (brw->is_cherryview)
+   if (brw->gen >= 9 || brw->is_cherryview)
       uncompacted |= brw_inst_bits(src, 36, 35) << 24; /* 2b */
 
    for (int i = 0; i < ARRAY_SIZE(gen8_3src_control_index_table); i++) {
@@ -645,7 +820,7 @@ set_3src_source_index(struct brw_context *brw, brw_compact_inst *dst, brw_inst *
       (brw_inst_bits(src,  72,  65) << 19) | /*  8b */
       (brw_inst_bits(src,  55,  37));        /* 19b */
 
-   if (brw->is_cherryview) {
+   if (brw->gen >= 9 || brw->is_cherryview) {
       uncompacted |=
          (brw_inst_bits(src, 126, 125) << 47) | /* 2b */
          (brw_inst_bits(src, 105, 104) << 45) | /* 2b */
@@ -737,17 +912,7 @@ brw_try_compact_instruction(struct brw_context *brw, brw_compact_inst *dst,
 {
    brw_compact_inst temp;
 
-   if (brw_inst_opcode(brw, src) == BRW_OPCODE_IF ||
-       brw_inst_opcode(brw, src) == BRW_OPCODE_ELSE ||
-       brw_inst_opcode(brw, src) == BRW_OPCODE_ENDIF ||
-       brw_inst_opcode(brw, src) == BRW_OPCODE_HALT ||
-       brw_inst_opcode(brw, src) == BRW_OPCODE_DO ||
-       brw_inst_opcode(brw, src) == BRW_OPCODE_WHILE) {
-      /* FINISHME: The fixup code below, and brw_set_uip_jip and friends, needs
-       * to be able to handle compacted flow control instructions..
-       */
-      return false;
-   }
+   assert(brw_inst_cmpt_control(brw, src) == 0);
 
    if (is_3src(brw_inst_opcode(brw, src))) {
       if (brw->gen >= 8) {
@@ -766,7 +931,8 @@ brw_try_compact_instruction(struct brw_context *brw, brw_compact_inst *dst,
    bool is_immediate =
       brw_inst_src0_reg_file(brw, src) == BRW_IMMEDIATE_VALUE ||
       brw_inst_src1_reg_file(brw, src) == BRW_IMMEDIATE_VALUE;
-   if (is_immediate && !is_compactable_immediate(brw_inst_imm_ud(brw, src))) {
+   if (is_immediate &&
+       (brw->gen < 6 || !is_compactable_immediate(brw_inst_imm_ud(brw, src)))) {
       return false;
    }
 
@@ -891,7 +1057,7 @@ set_uncompacted_3src_control_index(struct brw_context *brw, brw_inst *dst,
    brw_inst_set_bits(dst, 34, 32, (uncompacted >> 21) & 0x7);
    brw_inst_set_bits(dst, 28,  8, (uncompacted >>  0) & 0x1fffff);
 
-   if (brw->is_cherryview)
+   if (brw->gen >= 9 || brw->is_cherryview)
       brw_inst_set_bits(dst, 36, 35, (uncompacted >> 24) & 0x3);
 }
 
@@ -910,7 +1076,7 @@ set_uncompacted_3src_source_index(struct brw_context *brw, brw_inst *dst,
    brw_inst_set_bits(dst,  72,  65, (uncompacted >> 19) & 0xff);
    brw_inst_set_bits(dst,  55,  37, (uncompacted >>  0) & 0x7ffff);
 
-   if (brw->is_cherryview) {
+   if (brw->gen >= 9 || brw->is_cherryview) {
       brw_inst_set_bits(dst, 126, 125, (uncompacted >> 47) & 0x3);
       brw_inst_set_bits(dst, 105, 104, (uncompacted >> 45) & 0x3);
       brw_inst_set_bits(dst,  84,  84, (uncompacted >> 44) & 0x1);
@@ -1031,24 +1197,60 @@ static void
 update_uip_jip(struct brw_context *brw, brw_inst *insn,
                int this_old_ip, int *compacted_counts)
 {
-   int scale = brw->gen >= 8 ? sizeof(brw_compact_inst) : 1;
+   /* JIP and UIP are in units of:
+    *    - bytes on Gen8+; and
+    *    - compacted instructions on Gen6+.
+    */
+   int shift = brw->gen >= 8 ? 3 : 0;
 
-   int32_t jip = brw_inst_jip(brw, insn) / scale;
-   jip -= compacted_between(this_old_ip, this_old_ip + jip, compacted_counts);
-   brw_inst_set_jip(brw, insn, jip * scale);
+   int32_t jip_compacted = brw_inst_jip(brw, insn) >> shift;
+   jip_compacted -= compacted_between(this_old_ip,
+                                      this_old_ip + (jip_compacted / 2),
+                                      compacted_counts);
+   brw_inst_set_jip(brw, insn, jip_compacted << shift);
 
    if (brw_inst_opcode(brw, insn) == BRW_OPCODE_ENDIF ||
-       brw_inst_opcode(brw, insn) == BRW_OPCODE_WHILE)
+       brw_inst_opcode(brw, insn) == BRW_OPCODE_WHILE ||
+       (brw_inst_opcode(brw, insn) == BRW_OPCODE_ELSE && brw->gen <= 7))
       return;
 
-   int32_t uip = brw_inst_uip(brw, insn) / scale;
-   uip -= compacted_between(this_old_ip, this_old_ip + uip, compacted_counts);
-   brw_inst_set_uip(brw, insn, uip * scale);
+   int32_t uip_compacted = brw_inst_uip(brw, insn) >> shift;
+   uip_compacted -= compacted_between(this_old_ip,
+                                      this_old_ip + (uip_compacted / 2),
+                                      compacted_counts);
+   brw_inst_set_uip(brw, insn, uip_compacted << shift);
+}
+
+static void
+update_gen4_jump_count(struct brw_context *brw, brw_inst *insn,
+                       int this_old_ip, int *compacted_counts)
+{
+   assert(brw->gen == 5 || brw->is_g4x);
+
+   /* Jump Count is in units of:
+    *    - uncompacted instructions on G45; and
+    *    - compacted instructions on Gen5.
+    */
+   int shift = brw->is_g4x ? 1 : 0;
+
+   int jump_count_compacted = brw_inst_gen4_jump_count(brw, insn) << shift;
+
+   int target_old_ip = this_old_ip + (jump_count_compacted / 2);
+
+   int this_compacted_count = compacted_counts[this_old_ip];
+   int target_compacted_count = compacted_counts[target_old_ip];
+
+   jump_count_compacted -= (target_compacted_count - this_compacted_count);
+   brw_inst_set_gen4_jump_count(brw, insn, jump_count_compacted >> shift);
 }
 
 void
 brw_init_compaction_tables(struct brw_context *brw)
 {
+   assert(g45_control_index_table[ARRAY_SIZE(g45_control_index_table) - 1] != 0);
+   assert(g45_datatype_table[ARRAY_SIZE(g45_datatype_table) - 1] != 0);
+   assert(g45_subreg_table[ARRAY_SIZE(g45_subreg_table) - 1] != 0);
+   assert(g45_src_index_table[ARRAY_SIZE(g45_src_index_table) - 1] != 0);
    assert(gen6_control_index_table[ARRAY_SIZE(gen6_control_index_table) - 1] != 0);
    assert(gen6_datatype_table[ARRAY_SIZE(gen6_datatype_table) - 1] != 0);
    assert(gen6_subreg_table[ARRAY_SIZE(gen6_subreg_table) - 1] != 0);
@@ -1063,6 +1265,7 @@ brw_init_compaction_tables(struct brw_context *brw)
    assert(gen8_src_index_table[ARRAY_SIZE(gen8_src_index_table) - 1] != 0);
 
    switch (brw->gen) {
+   case 9:
    case 8:
       control_index_table = gen8_control_index_table;
       datatype_table = gen8_datatype_table;
@@ -1081,8 +1284,15 @@ brw_init_compaction_tables(struct brw_context *brw)
       subreg_table = gen6_subreg_table;
       src_index_table = gen6_src_index_table;
       break;
+   case 5:
+   case 4:
+      control_index_table = g45_control_index_table;
+      datatype_table = g45_datatype_table;
+      subreg_table = g45_subreg_table;
+      src_index_table = g45_src_index_table;
+      break;
    default:
-      return;
+      unreachable("unknown generation");
    }
 }
 
@@ -1092,32 +1302,32 @@ brw_compact_instructions(struct brw_compile *p, int start_offset,
 {
    struct brw_context *brw = p->brw;
    void *store = p->store + start_offset / 16;
-   /* For an instruction at byte offset 8*i before compaction, this is the number
-    * of compacted instructions that preceded it.
+   /* For an instruction at byte offset 16*i before compaction, this is the
+    * number of compacted instructions minus the number of padding NOP/NENOPs
+    * that preceded it.
     */
-   int compacted_counts[(p->next_insn_offset - start_offset) / 8];
-   /* For an instruction at byte offset 8*i after compaction, this is the
-    * 8-byte offset it was at before compaction.
+   int compacted_counts[(p->next_insn_offset - start_offset) / sizeof(brw_inst)];
+   /* For an instruction at byte offset 8*i after compaction, this was its IP
+    * (in 16-byte units) before compaction.
     */
-   int old_ip[(p->next_insn_offset - start_offset) / 8];
+   int old_ip[(p->next_insn_offset - start_offset) / sizeof(brw_compact_inst)];
 
-   if (brw->gen < 6)
+   if (brw->gen == 4 && !brw->is_g4x)
       return;
 
-   int src_offset;
    int offset = 0;
    int compacted_count = 0;
-   for (src_offset = 0; src_offset < p->next_insn_offset - start_offset;) {
+   for (int src_offset = 0; src_offset < p->next_insn_offset - start_offset;
+        src_offset += sizeof(brw_inst)) {
       brw_inst *src = store + src_offset;
       void *dst = store + offset;
 
-      old_ip[offset / 8] = src_offset / 8;
-      compacted_counts[src_offset / 8] = compacted_count;
+      old_ip[offset / sizeof(brw_compact_inst)] = src_offset / sizeof(brw_inst);
+      compacted_counts[src_offset / sizeof(brw_inst)] = compacted_count;
 
       brw_inst saved = *src;
 
-      if (!brw_inst_cmpt_control(brw, src) &&
-          brw_try_compact_instruction(brw, dst, src)) {
+      if (brw_try_compact_instruction(brw, dst, src)) {
          compacted_count++;
 
          if (INTEL_DEBUG) {
@@ -1128,24 +1338,27 @@ brw_compact_instructions(struct brw_compile *p, int start_offset,
             }
          }
 
-         offset += 8;
-         src_offset += 16;
+         offset += sizeof(brw_compact_inst);
       } else {
-         int size = brw_inst_cmpt_control(brw, src) ? 8 : 16;
-
          /* It appears that the end of thread SEND instruction needs to be
-          * aligned, or the GPU hangs.
+          * aligned, or the GPU hangs. All uncompacted instructions need to be
+          * aligned on G45.
           */
-         if ((brw_inst_opcode(brw, src) == BRW_OPCODE_SEND ||
-              brw_inst_opcode(brw, src) == BRW_OPCODE_SENDC) &&
-             brw_inst_eot(brw, src) &&
-             (offset & 8) != 0) {
+         if ((offset & sizeof(brw_compact_inst)) != 0 &&
+             (((brw_inst_opcode(brw, src) == BRW_OPCODE_SEND ||
+                brw_inst_opcode(brw, src) == BRW_OPCODE_SENDC) &&
+               brw_inst_eot(brw, src)) ||
+              brw->is_g4x)) {
             brw_compact_inst *align = store + offset;
             memset(align, 0, sizeof(*align));
-            brw_compact_inst_set_opcode(align, BRW_OPCODE_NOP);
+            brw_compact_inst_set_opcode(align, brw->is_g4x ? BRW_OPCODE_NENOP :
+                                                             BRW_OPCODE_NOP);
             brw_compact_inst_set_cmpt_control(align, true);
-            offset += 8;
-            old_ip[offset / 8] = src_offset / 8;
+            offset += sizeof(brw_compact_inst);
+            compacted_count--;
+            compacted_counts[src_offset / sizeof(brw_inst)] = compacted_count;
+            old_ip[offset / sizeof(brw_compact_inst)] = src_offset / sizeof(brw_inst);
+
             dst = store + offset;
          }
 
@@ -1153,45 +1366,88 @@ brw_compact_instructions(struct brw_compile *p, int start_offset,
           * place.
           */
          if (offset != src_offset) {
-            memmove(dst, src, size);
+            memmove(dst, src, sizeof(brw_inst));
          }
-         offset += size;
-         src_offset += size;
+         offset += sizeof(brw_inst);
       }
    }
 
    /* Fix up control flow offsets. */
    p->next_insn_offset = start_offset + offset;
-   for (offset = 0; offset < p->next_insn_offset - start_offset;) {
+   for (offset = 0; offset < p->next_insn_offset - start_offset;
+        offset = next_offset(brw, store, offset)) {
       brw_inst *insn = store + offset;
-      int this_old_ip = old_ip[offset / 8];
+      int this_old_ip = old_ip[offset / sizeof(brw_compact_inst)];
       int this_compacted_count = compacted_counts[this_old_ip];
-      int target_old_ip, target_compacted_count;
 
       switch (brw_inst_opcode(brw, insn)) {
       case BRW_OPCODE_BREAK:
       case BRW_OPCODE_CONTINUE:
       case BRW_OPCODE_HALT:
-         update_uip_jip(brw, insn, this_old_ip, compacted_counts);
+         if (brw->gen >= 6) {
+            update_uip_jip(brw, insn, this_old_ip, compacted_counts);
+         } else {
+            update_gen4_jump_count(brw, insn, this_old_ip, compacted_counts);
+         }
          break;
 
       case BRW_OPCODE_IF:
+      case BRW_OPCODE_IFF:
       case BRW_OPCODE_ELSE:
       case BRW_OPCODE_ENDIF:
       case BRW_OPCODE_WHILE:
          if (brw->gen >= 7) {
-            update_uip_jip(brw, insn, this_old_ip, compacted_counts);
+            if (brw_inst_cmpt_control(brw, insn)) {
+               brw_inst uncompacted;
+               brw_uncompact_instruction(brw, &uncompacted,
+                                         (brw_compact_inst *)insn);
+
+               update_uip_jip(brw, &uncompacted, this_old_ip, compacted_counts);
+
+               bool ret = brw_try_compact_instruction(brw,
+                                                      (brw_compact_inst *)insn,
+                                                      &uncompacted);
+               assert(ret); (void)ret;
+            } else {
+               update_uip_jip(brw, insn, this_old_ip, compacted_counts);
+            }
          } else if (brw->gen == 6) {
-            int gen6_jump_count = brw_inst_gen6_jump_count(brw, insn);
-            target_old_ip = this_old_ip + gen6_jump_count;
-            target_compacted_count = compacted_counts[target_old_ip];
-            gen6_jump_count -= (target_compacted_count - this_compacted_count);
-            brw_inst_set_gen6_jump_count(brw, insn, gen6_jump_count);
+            assert(!brw_inst_cmpt_control(brw, insn));
+
+            /* Jump Count is in units of compacted instructions on Gen6. */
+            int jump_count_compacted = brw_inst_gen6_jump_count(brw, insn);
+
+            int target_old_ip = this_old_ip + (jump_count_compacted / 2);
+            int target_compacted_count = compacted_counts[target_old_ip];
+            jump_count_compacted -= (target_compacted_count - this_compacted_count);
+            brw_inst_set_gen6_jump_count(brw, insn, jump_count_compacted);
+         } else {
+            update_gen4_jump_count(brw, insn, this_old_ip, compacted_counts);
          }
          break;
-      }
 
-      offset = next_offset(brw, store, offset);
+      case BRW_OPCODE_ADD:
+         /* Add instructions modifying the IP register use an immediate src1,
+          * and Gens that use this cannot compact instructions with immediate
+          * operands.
+          */
+         if (brw_inst_cmpt_control(brw, insn))
+            break;
+
+         if (brw_inst_dst_reg_file(brw, insn) == BRW_ARCHITECTURE_REGISTER_FILE &&
+             brw_inst_dst_da_reg_nr(brw, insn) == BRW_ARF_IP) {
+            assert(brw_inst_src1_reg_file(brw, insn) == BRW_IMMEDIATE_VALUE);
+
+            int shift = 3;
+            int jump_compacted = brw_inst_imm_d(brw, insn) >> shift;
+
+            int target_old_ip = this_old_ip + (jump_compacted / 2);
+            int target_compacted_count = compacted_counts[target_old_ip];
+            jump_compacted -= (target_compacted_count - this_compacted_count);
+            brw_inst_set_imm_ud(brw, insn, jump_compacted << shift);
+         }
+         break;
+      }
    }
 
    /* p->nr_insn is counting the number of uncompacted instructions still, so
@@ -1199,21 +1455,22 @@ brw_compact_instructions(struct brw_compile *p, int start_offset,
     * alignment padding, so that the next compression pass (for the FS 8/16
     * compile passes) parses correctly.
     */
-   if (p->next_insn_offset & 8) {
+   if (p->next_insn_offset & sizeof(brw_compact_inst)) {
       brw_compact_inst *align = store + offset;
       memset(align, 0, sizeof(*align));
       brw_compact_inst_set_opcode(align, BRW_OPCODE_NOP);
       brw_compact_inst_set_cmpt_control(align, true);
-      p->next_insn_offset += 8;
+      p->next_insn_offset += sizeof(brw_compact_inst);
    }
-   p->nr_insn = p->next_insn_offset / 16;
+   p->nr_insn = p->next_insn_offset / sizeof(brw_inst);
 
    /* Update the instruction offsets for each annotation. */
    if (annotation) {
       for (int offset = 0, i = 0; i < num_annotations; i++) {
-         while (start_offset + old_ip[offset / 8] * 8 != annotation[i].offset) {
-            assert(start_offset + old_ip[offset / 8] * 8 <
-                   annotation[i].offset);
+         while (start_offset + old_ip[offset / sizeof(brw_compact_inst)] *
+                sizeof(brw_inst) != annotation[i].offset) {
+            assert(start_offset + old_ip[offset / sizeof(brw_compact_inst)] *
+                   sizeof(brw_inst) < annotation[i].offset);
             offset = next_offset(brw, store, offset);
          }