intel/compiler: Don't left-shift by >= the number of bits of the type
[mesa.git] / src / intel / compiler / brw_disasm.c
index 6da70605174790069dce48169509cb81e2be3eca..7d821dbc1faa5ae6a6a29798056f07e20bcb1abf 100644 (file)
@@ -30,6 +30,7 @@
 #include "brw_reg.h"
 #include "brw_inst.h"
 #include "brw_eu.h"
+#include "util/half_float.h"
 
 static bool
 has_jip(const struct gen_device_info *devinfo, enum opcode opcode)
@@ -79,6 +80,25 @@ is_logic_instruction(unsigned opcode)
           opcode == BRW_OPCODE_XOR;
 }
 
+static bool
+is_send(unsigned opcode)
+{
+   return opcode == BRW_OPCODE_SEND ||
+          opcode == BRW_OPCODE_SENDC ||
+          opcode == BRW_OPCODE_SENDS ||
+          opcode == BRW_OPCODE_SENDSC;
+}
+
+static bool
+is_split_send(UNUSED const struct gen_device_info *devinfo, unsigned opcode)
+{
+   if (devinfo->gen >= 12)
+      return is_send(opcode);
+   else
+      return opcode == BRW_OPCODE_SENDS ||
+             opcode == BRW_OPCODE_SENDSC;
+}
+
 const char *const conditional_modifier[16] = {
    [BRW_CONDITIONAL_NONE] = "",
    [BRW_CONDITIONAL_Z]    = ".z",
@@ -237,28 +257,6 @@ static const char *const access_mode[2] = {
    [1] = "align16",
 };
 
-static const char * const reg_encoding[] = {
-   [BRW_HW_REG_TYPE_UD]  = "UD",
-   [BRW_HW_REG_TYPE_D]   = "D",
-   [BRW_HW_REG_TYPE_UW]  = "UW",
-   [BRW_HW_REG_TYPE_W]   = "W",
-   [BRW_HW_REG_TYPE_F]   = "F",
-   [GEN8_HW_REG_TYPE_UQ] = "UQ",
-   [GEN8_HW_REG_TYPE_Q]  = "Q",
-
-   [BRW_HW_REG_TYPE_UB]  = "UB",
-   [BRW_HW_REG_TYPE_B]   = "B",
-   [GEN7_HW_REG_TYPE_DF] = "DF",
-   [GEN8_HW_REG_TYPE_HF] = "HF",
-};
-
-static const char *const three_source_reg_encoding[] = {
-   [BRW_3SRC_TYPE_F]  = "F",
-   [BRW_3SRC_TYPE_D]  = "D",
-   [BRW_3SRC_TYPE_UD] = "UD",
-   [BRW_3SRC_TYPE_DF] = "DF",
-};
-
 static const char *const reg_file[4] = {
    [0] = "A",
    [1] = "g",
@@ -310,7 +308,7 @@ static const char *const gen6_sfid[16] = {
    [BRW_SFID_MESSAGE_GATEWAY]          = "gateway",
    [BRW_SFID_URB]                      = "urb",
    [BRW_SFID_THREAD_SPAWNER]           = "thread_spawner",
-   [GEN6_SFID_DATAPORT_SAMPLER_CACHE]  = "sampler",
+   [GEN6_SFID_DATAPORT_SAMPLER_CACHE]  = "dp_sampler",
    [GEN6_SFID_DATAPORT_RENDER_CACHE]   = "render",
    [GEN6_SFID_DATAPORT_CONSTANT_CACHE] = "const",
    [GEN7_SFID_DATAPORT_DATA_CACHE]     = "data",
@@ -427,7 +425,7 @@ static const char *const dp_dc0_msg_type_gen7[16] = {
    [GEN7_DATAPORT_DC_UNTYPED_SURFACE_WRITE] = "DC untyped surface write",
 };
 
-static const char *const dp_dc1_msg_type_hsw[16] = {
+static const char *const dp_dc1_msg_type_hsw[32] = {
    [HSW_DATAPORT_DC_PORT1_UNTYPED_SURFACE_READ] = "untyped surface read",
    [HSW_DATAPORT_DC_PORT1_UNTYPED_ATOMIC_OP] = "DC untyped atomic op",
    [HSW_DATAPORT_DC_PORT1_UNTYPED_ATOMIC_OP_SIMD4X2] =
@@ -442,6 +440,15 @@ static const char *const dp_dc1_msg_type_hsw[16] = {
    [HSW_DATAPORT_DC_PORT1_ATOMIC_COUNTER_OP_SIMD4X2] =
       "DC 4x2 atomic counter op",
    [HSW_DATAPORT_DC_PORT1_TYPED_SURFACE_WRITE] = "DC typed surface write",
+   [GEN9_DATAPORT_DC_PORT1_A64_SCATTERED_READ] = "DC A64 scattered read",
+   [GEN8_DATAPORT_DC_PORT1_A64_UNTYPED_SURFACE_READ] = "DC A64 untyped surface read",
+   [GEN8_DATAPORT_DC_PORT1_A64_UNTYPED_ATOMIC_OP] = "DC A64 untyped atomic op",
+   [GEN8_DATAPORT_DC_PORT1_A64_UNTYPED_SURFACE_WRITE] = "DC A64 untyped surface write",
+   [GEN8_DATAPORT_DC_PORT1_A64_SCATTERED_WRITE] = "DC A64 scattered write",
+   [GEN9_DATAPORT_DC_PORT1_UNTYPED_ATOMIC_FLOAT_OP] =
+      "DC untyped atomic float op",
+   [GEN9_DATAPORT_DC_PORT1_A64_UNTYPED_ATOMIC_FLOAT_OP] =
+      "DC A64 untyped atomic float op",
 };
 
 static const char *const aop[16] = {
@@ -462,6 +469,12 @@ static const char *const aop[16] = {
    [BRW_AOP_PREDEC] = "predec",
 };
 
+static const char *const aop_float[4] = {
+   [BRW_AOP_FMAX]   = "fmax",
+   [BRW_AOP_FMIN]   = "fmin",
+   [BRW_AOP_FCMPWR] = "fcmpwr",
+};
+
 static const char * const pixel_interpolator_msg_types[4] = {
     [GEN7_PIXEL_INTERPOLATOR_LOC_SHARED_OFFSET] = "per_message_offset",
     [GEN7_PIXEL_INTERPOLATOR_LOC_SAMPLE] = "sample_position",
@@ -487,6 +500,14 @@ static const char *const math_function[16] = {
    [GEN8_MATH_FUNCTION_RSQRTM] = "rsqrtm",
 };
 
+static const char *const sync_function[16] = {
+   [TGL_SYNC_NOP] = "nop",
+   [TGL_SYNC_ALLRD] = "allrd",
+   [TGL_SYNC_ALLWR] = "allwr",
+   [TGL_SYNC_BAR] = "bar",
+   [TGL_SYNC_HOST] = "host",
+};
+
 static const char *const math_saturate[2] = {
    [0] = "",
    [1] = "sat"
@@ -722,10 +743,36 @@ reg(FILE *file, unsigned _reg_file, unsigned _reg_nr)
 static int
 dest(FILE *file, const struct gen_device_info *devinfo, const brw_inst *inst)
 {
-   unsigned elem_size = brw_element_size(devinfo, inst, dst);
+   enum brw_reg_type type = brw_inst_dst_type(devinfo, inst);
+   unsigned elem_size = brw_reg_type_to_size(type);
    int err = 0;
 
-   if (brw_inst_access_mode(devinfo, inst) == BRW_ALIGN_1) {
+   if (is_split_send(devinfo, brw_inst_opcode(devinfo, inst))) {
+      /* These are fixed for split sends */
+      type = BRW_REGISTER_TYPE_UD;
+      elem_size = 4;
+      if (devinfo->gen >= 12) {
+         err |= reg(file, brw_inst_send_dst_reg_file(devinfo, inst),
+                    brw_inst_dst_da_reg_nr(devinfo, inst));
+         string(file, brw_reg_type_to_letters(type));
+      } else if (brw_inst_dst_address_mode(devinfo, inst) == BRW_ADDRESS_DIRECT) {
+         err |= reg(file, brw_inst_send_dst_reg_file(devinfo, inst),
+                    brw_inst_dst_da_reg_nr(devinfo, inst));
+         unsigned subreg_nr = brw_inst_dst_da16_subreg_nr(devinfo, inst);
+         if (subreg_nr)
+            format(file, ".%u", subreg_nr);
+         string(file, brw_reg_type_to_letters(type));
+      } else {
+         string(file, "g[a0");
+         if (brw_inst_dst_ia_subreg_nr(devinfo, inst))
+            format(file, ".%"PRIu64, brw_inst_dst_ia_subreg_nr(devinfo, inst) /
+                   elem_size);
+         if (brw_inst_send_dst_ia16_addr_imm(devinfo, inst))
+            format(file, " %d", brw_inst_send_dst_ia16_addr_imm(devinfo, inst));
+         string(file, "]<");
+         string(file, brw_reg_type_to_letters(type));
+      }
+   } else if (brw_inst_access_mode(devinfo, inst) == BRW_ALIGN_1) {
       if (brw_inst_dst_address_mode(devinfo, inst) == BRW_ADDRESS_DIRECT) {
          err |= reg(file, brw_inst_dst_reg_file(devinfo, inst),
                     brw_inst_dst_da_reg_nr(devinfo, inst));
@@ -738,8 +785,7 @@ dest(FILE *file, const struct gen_device_info *devinfo, const brw_inst *inst)
          err |= control(file, "horiz stride", horiz_stride,
                         brw_inst_dst_hstride(devinfo, inst), NULL);
          string(file, ">");
-         err |= control(file, "dest reg encoding", reg_encoding,
-                        brw_inst_dst_reg_type(devinfo, inst), NULL);
+         string(file, brw_reg_type_to_letters(type));
       } else {
          string(file, "g[a0");
          if (brw_inst_dst_ia_subreg_nr(devinfo, inst))
@@ -751,8 +797,7 @@ dest(FILE *file, const struct gen_device_info *devinfo, const brw_inst *inst)
          err |= control(file, "horiz stride", horiz_stride,
                         brw_inst_dst_hstride(devinfo, inst), NULL);
          string(file, ">");
-         err |= control(file, "dest reg encoding", reg_encoding,
-                        brw_inst_dst_reg_type(devinfo, inst), NULL);
+         string(file, brw_reg_type_to_letters(type));
       }
    } else {
       if (brw_inst_dst_address_mode(devinfo, inst) == BRW_ADDRESS_DIRECT) {
@@ -765,8 +810,7 @@ dest(FILE *file, const struct gen_device_info *devinfo, const brw_inst *inst)
          string(file, "<1>");
          err |= control(file, "writemask", writemask,
                         brw_inst_da16_writemask(devinfo, inst), NULL);
-         err |= control(file, "dest reg encoding", reg_encoding,
-                        brw_inst_dst_reg_type(devinfo, inst), NULL);
+         string(file, brw_reg_type_to_letters(type));
       } else {
          err = 1;
          string(file, "Indirect align16 address mode not supported");
@@ -779,24 +823,43 @@ dest(FILE *file, const struct gen_device_info *devinfo, const brw_inst *inst)
 static int
 dest_3src(FILE *file, const struct gen_device_info *devinfo, const brw_inst *inst)
 {
+   bool is_align1 = brw_inst_3src_access_mode(devinfo, inst) == BRW_ALIGN_1;
    int err = 0;
    uint32_t reg_file;
+   unsigned subreg_nr;
+   enum brw_reg_type type;
 
-   if (devinfo->gen == 6 && brw_inst_3src_dst_reg_file(devinfo, inst))
+   if (devinfo->gen == 6 && brw_inst_3src_a16_dst_reg_file(devinfo, inst))
       reg_file = BRW_MESSAGE_REGISTER_FILE;
+   else if (devinfo->gen >= 12)
+      reg_file = brw_inst_3src_a1_dst_reg_file(devinfo, inst);
+   else if (is_align1 && brw_inst_3src_a1_dst_reg_file(devinfo, inst))
+      reg_file = BRW_ARCHITECTURE_REGISTER_FILE;
    else
       reg_file = BRW_GENERAL_REGISTER_FILE;
 
    err |= reg(file, reg_file, brw_inst_3src_dst_reg_nr(devinfo, inst));
    if (err == -1)
       return 0;
-   if (brw_inst_3src_dst_subreg_nr(devinfo, inst))
-      format(file, ".%"PRIu64, brw_inst_3src_dst_subreg_nr(devinfo, inst));
+
+   if (is_align1) {
+      type = brw_inst_3src_a1_dst_type(devinfo, inst);
+      subreg_nr = brw_inst_3src_a1_dst_subreg_nr(devinfo, inst);
+   } else {
+      type = brw_inst_3src_a16_dst_type(devinfo, inst);
+      subreg_nr = brw_inst_3src_a16_dst_subreg_nr(devinfo, inst) * 4;
+   }
+   subreg_nr /= brw_reg_type_to_size(type);
+
+   if (subreg_nr)
+      format(file, ".%u", subreg_nr);
    string(file, "<1>");
-   err |= control(file, "writemask", writemask,
-                  brw_inst_3src_dst_writemask(devinfo, inst), NULL);
-   err |= control(file, "dest reg encoding", three_source_reg_encoding,
-                  brw_inst_3src_dst_type(devinfo, inst), NULL);
+
+   if (!is_align1) {
+      err |= control(file, "writemask", writemask,
+                     brw_inst_3src_a16_dst_writemask(devinfo, inst), NULL);
+   }
+   string(file, brw_reg_type_to_letters(type));
 
    return 0;
 }
@@ -821,7 +884,7 @@ static int
 src_da1(FILE *file,
         const struct gen_device_info *devinfo,
         unsigned opcode,
-        unsigned type, unsigned _reg_file,
+        enum brw_reg_type type, unsigned _reg_file,
         unsigned _vert_stride, unsigned _width, unsigned _horiz_stride,
         unsigned reg_num, unsigned sub_reg_num, unsigned __abs,
         unsigned _negate)
@@ -839,11 +902,11 @@ src_da1(FILE *file,
    if (err == -1)
       return 0;
    if (sub_reg_num) {
-      unsigned elem_size = brw_hw_reg_type_to_size(devinfo, type, _reg_file);
+      unsigned elem_size = brw_reg_type_to_size(type);
       format(file, ".%d", sub_reg_num / elem_size);   /* use formal style like spec */
    }
    src_align1_region(file, _vert_stride, _width, _horiz_stride);
-   err |= control(file, "src reg encoding", reg_encoding, type, NULL);
+   string(file, brw_reg_type_to_letters(type));
    return err;
 }
 
@@ -851,8 +914,7 @@ static int
 src_ia1(FILE *file,
         const struct gen_device_info *devinfo,
         unsigned opcode,
-        unsigned type,
-        unsigned _reg_file,
+        enum brw_reg_type type,
         int _addr_imm,
         unsigned _addr_subreg_nr,
         unsigned _negate,
@@ -875,7 +937,7 @@ src_ia1(FILE *file,
       format(file, " %d", _addr_imm);
    string(file, "]");
    src_align1_region(file, _vert_stride, _width, _horiz_stride);
-   err |= control(file, "src reg encoding", reg_encoding, type, NULL);
+   string(file, brw_reg_type_to_letters(type));
    return err;
 }
 
@@ -905,7 +967,7 @@ static int
 src_da16(FILE *file,
          const struct gen_device_info *devinfo,
          unsigned opcode,
-         unsigned _reg_type,
+         enum brw_reg_type type,
          unsigned _reg_file,
          unsigned _vert_stride,
          unsigned _reg_nr,
@@ -927,8 +989,7 @@ src_da16(FILE *file,
    if (err == -1)
       return 0;
    if (_subreg_nr) {
-      unsigned elem_size =
-         brw_hw_reg_type_to_size(devinfo, _reg_type, _reg_file);
+      unsigned elem_size = brw_reg_type_to_size(type);
 
       /* bit4 for subreg number byte addressing. Make this same meaning as
          in da1 case, so output looks consistent. */
@@ -938,34 +999,175 @@ src_da16(FILE *file,
    err |= control(file, "vert stride", vert_stride, _vert_stride, NULL);
    string(file, ">");
    err |= src_swizzle(file, BRW_SWIZZLE4(swz_x, swz_y, swz_z, swz_w));
-   err |= control(file, "src da16 reg type", reg_encoding, _reg_type, NULL);
+   string(file, brw_reg_type_to_letters(type));
    return err;
 }
 
+static enum brw_vertical_stride
+vstride_from_align1_3src_vstride(const struct gen_device_info *devinfo,
+                                 enum gen10_align1_3src_vertical_stride vstride)
+{
+   switch (vstride) {
+   case BRW_ALIGN1_3SRC_VERTICAL_STRIDE_0: return BRW_VERTICAL_STRIDE_0;
+   case BRW_ALIGN1_3SRC_VERTICAL_STRIDE_2:
+      if (devinfo->gen >= 12)
+         return BRW_VERTICAL_STRIDE_1;
+      else
+         return BRW_VERTICAL_STRIDE_2;
+   case BRW_ALIGN1_3SRC_VERTICAL_STRIDE_4: return BRW_VERTICAL_STRIDE_4;
+   case BRW_ALIGN1_3SRC_VERTICAL_STRIDE_8: return BRW_VERTICAL_STRIDE_8;
+   default:
+      unreachable("not reached");
+   }
+}
+
+static enum brw_horizontal_stride
+hstride_from_align1_3src_hstride(enum gen10_align1_3src_src_horizontal_stride hstride)
+{
+   switch (hstride) {
+   case BRW_ALIGN1_3SRC_SRC_HORIZONTAL_STRIDE_0: return BRW_HORIZONTAL_STRIDE_0;
+   case BRW_ALIGN1_3SRC_SRC_HORIZONTAL_STRIDE_1: return BRW_HORIZONTAL_STRIDE_1;
+   case BRW_ALIGN1_3SRC_SRC_HORIZONTAL_STRIDE_2: return BRW_HORIZONTAL_STRIDE_2;
+   case BRW_ALIGN1_3SRC_SRC_HORIZONTAL_STRIDE_4: return BRW_HORIZONTAL_STRIDE_4;
+   default:
+      unreachable("not reached");
+   }
+}
+
+static enum brw_vertical_stride
+vstride_from_align1_3src_hstride(enum gen10_align1_3src_src_horizontal_stride hstride)
+{
+   switch (hstride) {
+   case BRW_ALIGN1_3SRC_SRC_HORIZONTAL_STRIDE_0: return BRW_VERTICAL_STRIDE_0;
+   case BRW_ALIGN1_3SRC_SRC_HORIZONTAL_STRIDE_1: return BRW_VERTICAL_STRIDE_1;
+   case BRW_ALIGN1_3SRC_SRC_HORIZONTAL_STRIDE_2: return BRW_VERTICAL_STRIDE_2;
+   case BRW_ALIGN1_3SRC_SRC_HORIZONTAL_STRIDE_4: return BRW_VERTICAL_STRIDE_4;
+   default:
+      unreachable("not reached");
+   }
+}
+
+/* From "GEN10 Regioning Rules for Align1 Ternary Operations" in the
+ * "Register Region Restrictions" documentation
+ */
+static enum brw_width
+implied_width(enum brw_vertical_stride _vert_stride,
+              enum brw_horizontal_stride _horiz_stride)
+{
+   /* "1. Width is 1 when Vertical and Horizontal Strides are both zero." */
+   if (_vert_stride == BRW_VERTICAL_STRIDE_0 &&
+       _horiz_stride == BRW_HORIZONTAL_STRIDE_0) {
+      return BRW_WIDTH_1;
+
+   /* "2. Width is equal to vertical stride when Horizontal Stride is zero." */
+   } else if (_horiz_stride == BRW_HORIZONTAL_STRIDE_0) {
+      switch (_vert_stride) {
+      case BRW_VERTICAL_STRIDE_2: return BRW_WIDTH_2;
+      case BRW_VERTICAL_STRIDE_4: return BRW_WIDTH_4;
+      case BRW_VERTICAL_STRIDE_8: return BRW_WIDTH_8;
+      case BRW_VERTICAL_STRIDE_0:
+      default:
+         unreachable("not reached");
+      }
+
+   } else {
+      /* FINISHME: Implement these: */
+
+      /* "3. Width is equal to Vertical Stride/Horizontal Stride when both
+       *     Strides are non-zero.
+       *
+       *  4. Vertical Stride must not be zero if Horizontal Stride is non-zero.
+       *     This implies Vertical Stride is always greater than Horizontal
+       *     Stride."
+       *
+       * Given these statements and the knowledge that the stride and width
+       * values are encoded in logarithmic form, we can perform the division
+       * by just subtracting.
+       */
+      return _vert_stride - _horiz_stride;
+   }
+}
+
 static int
 src0_3src(FILE *file, const struct gen_device_info *devinfo, const brw_inst *inst)
 {
    int err = 0;
-   unsigned src0_subreg_nr = brw_inst_3src_src0_subreg_nr(devinfo, inst);
+   unsigned reg_nr, subreg_nr;
+   enum brw_reg_file _file;
+   enum brw_reg_type type;
+   enum brw_vertical_stride _vert_stride;
+   enum brw_width _width;
+   enum brw_horizontal_stride _horiz_stride;
+   bool is_scalar_region;
+   bool is_align1 = brw_inst_3src_access_mode(devinfo, inst) == BRW_ALIGN_1;
+
+   if (is_align1) {
+      if (devinfo->gen >= 12 && !brw_inst_3src_a1_src0_is_imm(devinfo, inst)) {
+         _file = brw_inst_3src_a1_src0_reg_file(devinfo, inst);
+      } else if (brw_inst_3src_a1_src0_reg_file(devinfo, inst) ==
+                 BRW_ALIGN1_3SRC_GENERAL_REGISTER_FILE) {
+         _file = BRW_GENERAL_REGISTER_FILE;
+      } else if (brw_inst_3src_a1_src0_type(devinfo, inst) ==
+                 BRW_REGISTER_TYPE_NF) {
+         _file = BRW_ARCHITECTURE_REGISTER_FILE;
+      } else {
+         _file = BRW_IMMEDIATE_VALUE;
+         uint16_t imm_val = brw_inst_3src_a1_src0_imm(devinfo, inst);
+         enum brw_reg_type type = brw_inst_3src_a1_src0_type(devinfo, inst);
+
+         if (type == BRW_REGISTER_TYPE_W) {
+            format(file, "%dW", imm_val);
+         } else if (type == BRW_REGISTER_TYPE_UW) {
+            format(file, "0x%04xUW", imm_val);
+         } else if (type == BRW_REGISTER_TYPE_HF) {
+            format(file, "0x%04xHF", imm_val);
+         }
+         return 0;
+      }
+
+      reg_nr = brw_inst_3src_src0_reg_nr(devinfo, inst);
+      subreg_nr = brw_inst_3src_a1_src0_subreg_nr(devinfo, inst);
+      type = brw_inst_3src_a1_src0_type(devinfo, inst);
+      _vert_stride = vstride_from_align1_3src_vstride(
+         devinfo, brw_inst_3src_a1_src0_vstride(devinfo, inst));
+      _horiz_stride = hstride_from_align1_3src_hstride(
+                         brw_inst_3src_a1_src0_hstride(devinfo, inst));
+      _width = implied_width(_vert_stride, _horiz_stride);
+   } else {
+      _file = BRW_GENERAL_REGISTER_FILE;
+      reg_nr = brw_inst_3src_src0_reg_nr(devinfo, inst);
+      subreg_nr = brw_inst_3src_a16_src0_subreg_nr(devinfo, inst) * 4;
+      type = brw_inst_3src_a16_src_type(devinfo, inst);
+
+      if (brw_inst_3src_a16_src0_rep_ctrl(devinfo, inst)) {
+         _vert_stride = BRW_VERTICAL_STRIDE_0;
+         _width = BRW_WIDTH_1;
+         _horiz_stride = BRW_HORIZONTAL_STRIDE_0;
+      } else {
+         _vert_stride = BRW_VERTICAL_STRIDE_4;
+         _width = BRW_WIDTH_4;
+         _horiz_stride = BRW_HORIZONTAL_STRIDE_1;
+      }
+   }
+   is_scalar_region = _vert_stride == BRW_VERTICAL_STRIDE_0 &&
+                      _width == BRW_WIDTH_1 &&
+                      _horiz_stride == BRW_HORIZONTAL_STRIDE_0;
+
+   subreg_nr /= brw_reg_type_to_size(type);
 
    err |= control(file, "negate", m_negate,
                   brw_inst_3src_src0_negate(devinfo, inst), NULL);
    err |= control(file, "abs", _abs, brw_inst_3src_src0_abs(devinfo, inst), NULL);
 
-   err |= reg(file, BRW_GENERAL_REGISTER_FILE,
-              brw_inst_3src_src0_reg_nr(devinfo, inst));
+   err |= reg(file, _file, reg_nr);
    if (err == -1)
       return 0;
-   if (src0_subreg_nr || brw_inst_3src_src0_rep_ctrl(devinfo, inst))
-      format(file, ".%d", src0_subreg_nr);
-   if (brw_inst_3src_src0_rep_ctrl(devinfo, inst))
-      string(file, "<0,1,0>");
-   else {
-      string(file, "<4,4,1>");
-      err |= src_swizzle(file, brw_inst_3src_src0_swizzle(devinfo, inst));
-   }
-   err |= control(file, "src da16 reg type", three_source_reg_encoding,
-                  brw_inst_3src_src_type(devinfo, inst), NULL);
+   if (subreg_nr || is_scalar_region)
+      format(file, ".%d", subreg_nr);
+   src_align1_region(file, _vert_stride, _width, _horiz_stride);
+   if (!is_scalar_region && !is_align1)
+      err |= src_swizzle(file, brw_inst_3src_a16_src0_swizzle(devinfo, inst));
+   string(file, brw_reg_type_to_letters(type));
    return err;
 }
 
@@ -973,117 +1175,295 @@ static int
 src1_3src(FILE *file, const struct gen_device_info *devinfo, const brw_inst *inst)
 {
    int err = 0;
-   unsigned src1_subreg_nr = brw_inst_3src_src1_subreg_nr(devinfo, inst);
+   unsigned reg_nr, subreg_nr;
+   enum brw_reg_file _file;
+   enum brw_reg_type type;
+   enum brw_vertical_stride _vert_stride;
+   enum brw_width _width;
+   enum brw_horizontal_stride _horiz_stride;
+   bool is_scalar_region;
+   bool is_align1 = brw_inst_3src_access_mode(devinfo, inst) == BRW_ALIGN_1;
+
+   if (is_align1) {
+      if (devinfo->gen >= 12) {
+         _file = brw_inst_3src_a1_src1_reg_file(devinfo, inst);
+      } else if (brw_inst_3src_a1_src1_reg_file(devinfo, inst) ==
+                 BRW_ALIGN1_3SRC_GENERAL_REGISTER_FILE) {
+         _file = BRW_GENERAL_REGISTER_FILE;
+      } else {
+         _file = BRW_ARCHITECTURE_REGISTER_FILE;
+      }
+
+      reg_nr = brw_inst_3src_src1_reg_nr(devinfo, inst);
+      subreg_nr = brw_inst_3src_a1_src1_subreg_nr(devinfo, inst);
+      type = brw_inst_3src_a1_src1_type(devinfo, inst);
+
+      _vert_stride = vstride_from_align1_3src_vstride(
+         devinfo, brw_inst_3src_a1_src1_vstride(devinfo, inst));
+      _horiz_stride = hstride_from_align1_3src_hstride(
+                         brw_inst_3src_a1_src1_hstride(devinfo, inst));
+      _width = implied_width(_vert_stride, _horiz_stride);
+   } else {
+      _file = BRW_GENERAL_REGISTER_FILE;
+      reg_nr = brw_inst_3src_src1_reg_nr(devinfo, inst);
+      subreg_nr = brw_inst_3src_a16_src1_subreg_nr(devinfo, inst) * 4;
+      type = brw_inst_3src_a16_src_type(devinfo, inst);
+
+      if (brw_inst_3src_a16_src1_rep_ctrl(devinfo, inst)) {
+         _vert_stride = BRW_VERTICAL_STRIDE_0;
+         _width = BRW_WIDTH_1;
+         _horiz_stride = BRW_HORIZONTAL_STRIDE_0;
+      } else {
+         _vert_stride = BRW_VERTICAL_STRIDE_4;
+         _width = BRW_WIDTH_4;
+         _horiz_stride = BRW_HORIZONTAL_STRIDE_1;
+      }
+   }
+   is_scalar_region = _vert_stride == BRW_VERTICAL_STRIDE_0 &&
+                      _width == BRW_WIDTH_1 &&
+                      _horiz_stride == BRW_HORIZONTAL_STRIDE_0;
+
+   subreg_nr /= brw_reg_type_to_size(type);
 
    err |= control(file, "negate", m_negate,
                   brw_inst_3src_src1_negate(devinfo, inst), NULL);
    err |= control(file, "abs", _abs, brw_inst_3src_src1_abs(devinfo, inst), NULL);
 
-   err |= reg(file, BRW_GENERAL_REGISTER_FILE,
-              brw_inst_3src_src1_reg_nr(devinfo, inst));
+   err |= reg(file, _file, reg_nr);
    if (err == -1)
       return 0;
-   if (src1_subreg_nr || brw_inst_3src_src1_rep_ctrl(devinfo, inst))
-      format(file, ".%d", src1_subreg_nr);
-   if (brw_inst_3src_src1_rep_ctrl(devinfo, inst))
-      string(file, "<0,1,0>");
-   else {
-      string(file, "<4,4,1>");
-      err |= src_swizzle(file, brw_inst_3src_src1_swizzle(devinfo, inst));
-   }
-   err |= control(file, "src da16 reg type", three_source_reg_encoding,
-                  brw_inst_3src_src_type(devinfo, inst), NULL);
+   if (subreg_nr || is_scalar_region)
+      format(file, ".%d", subreg_nr);
+   src_align1_region(file, _vert_stride, _width, _horiz_stride);
+   if (!is_scalar_region && !is_align1)
+      err |= src_swizzle(file, brw_inst_3src_a16_src1_swizzle(devinfo, inst));
+   string(file, brw_reg_type_to_letters(type));
    return err;
 }
 
-
 static int
 src2_3src(FILE *file, const struct gen_device_info *devinfo, const brw_inst *inst)
 {
    int err = 0;
-   unsigned src2_subreg_nr = brw_inst_3src_src2_subreg_nr(devinfo, inst);
+   unsigned reg_nr, subreg_nr;
+   enum brw_reg_file _file;
+   enum brw_reg_type type;
+   enum brw_vertical_stride _vert_stride;
+   enum brw_width _width;
+   enum brw_horizontal_stride _horiz_stride;
+   bool is_scalar_region;
+   bool is_align1 = brw_inst_3src_access_mode(devinfo, inst) == BRW_ALIGN_1;
+
+   if (is_align1) {
+      if (devinfo->gen >= 12 && !brw_inst_3src_a1_src2_is_imm(devinfo, inst)) {
+         _file = brw_inst_3src_a1_src2_reg_file(devinfo, inst);
+      } else if (brw_inst_3src_a1_src2_reg_file(devinfo, inst) ==
+                 BRW_ALIGN1_3SRC_GENERAL_REGISTER_FILE) {
+         _file = BRW_GENERAL_REGISTER_FILE;
+      } else {
+         _file = BRW_IMMEDIATE_VALUE;
+         uint16_t imm_val = brw_inst_3src_a1_src2_imm(devinfo, inst);
+         enum brw_reg_type type = brw_inst_3src_a1_src2_type(devinfo, inst);
+
+         if (type == BRW_REGISTER_TYPE_W) {
+            format(file, "%dW", imm_val);
+         } else if (type == BRW_REGISTER_TYPE_UW) {
+            format(file, "0x%04xUW", imm_val);
+         } else if (type == BRW_REGISTER_TYPE_HF) {
+            format(file, "0x%04xHF", imm_val);
+         }
+         return 0;
+      }
+
+      reg_nr = brw_inst_3src_src2_reg_nr(devinfo, inst);
+      subreg_nr = brw_inst_3src_a1_src2_subreg_nr(devinfo, inst);
+      type = brw_inst_3src_a1_src2_type(devinfo, inst);
+      /* FINISHME: No vertical stride on src2. Is using the hstride in place
+       *           correct? Doesn't seem like it, since there's hstride=1 but
+       *           no vstride=1.
+       */
+      _vert_stride = vstride_from_align1_3src_hstride(
+                        brw_inst_3src_a1_src2_hstride(devinfo, inst));
+      _horiz_stride = hstride_from_align1_3src_hstride(
+                         brw_inst_3src_a1_src2_hstride(devinfo, inst));
+      _width = implied_width(_vert_stride, _horiz_stride);
+   } else {
+      _file = BRW_GENERAL_REGISTER_FILE;
+      reg_nr = brw_inst_3src_src2_reg_nr(devinfo, inst);
+      subreg_nr = brw_inst_3src_a16_src2_subreg_nr(devinfo, inst) * 4;
+      type = brw_inst_3src_a16_src_type(devinfo, inst);
+
+      if (brw_inst_3src_a16_src2_rep_ctrl(devinfo, inst)) {
+         _vert_stride = BRW_VERTICAL_STRIDE_0;
+         _width = BRW_WIDTH_1;
+         _horiz_stride = BRW_HORIZONTAL_STRIDE_0;
+      } else {
+         _vert_stride = BRW_VERTICAL_STRIDE_4;
+         _width = BRW_WIDTH_4;
+         _horiz_stride = BRW_HORIZONTAL_STRIDE_1;
+      }
+   }
+   is_scalar_region = _vert_stride == BRW_VERTICAL_STRIDE_0 &&
+                      _width == BRW_WIDTH_1 &&
+                      _horiz_stride == BRW_HORIZONTAL_STRIDE_0;
+
+   subreg_nr /= brw_reg_type_to_size(type);
 
    err |= control(file, "negate", m_negate,
                   brw_inst_3src_src2_negate(devinfo, inst), NULL);
    err |= control(file, "abs", _abs, brw_inst_3src_src2_abs(devinfo, inst), NULL);
 
-   err |= reg(file, BRW_GENERAL_REGISTER_FILE,
-              brw_inst_3src_src2_reg_nr(devinfo, inst));
+   err |= reg(file, _file, reg_nr);
    if (err == -1)
       return 0;
-   if (src2_subreg_nr || brw_inst_3src_src2_rep_ctrl(devinfo, inst))
-      format(file, ".%d", src2_subreg_nr);
-   if (brw_inst_3src_src2_rep_ctrl(devinfo, inst))
-      string(file, "<0,1,0>");
-   else {
-      string(file, "<4,4,1>");
-      err |= src_swizzle(file, brw_inst_3src_src2_swizzle(devinfo, inst));
-   }
-   err |= control(file, "src da16 reg type", three_source_reg_encoding,
-                  brw_inst_3src_src_type(devinfo, inst), NULL);
+   if (subreg_nr || is_scalar_region)
+      format(file, ".%d", subreg_nr);
+   src_align1_region(file, _vert_stride, _width, _horiz_stride);
+   if (!is_scalar_region && !is_align1)
+      err |= src_swizzle(file, brw_inst_3src_a16_src2_swizzle(devinfo, inst));
+   string(file, brw_reg_type_to_letters(type));
    return err;
 }
 
 static int
-imm(FILE *file, const struct gen_device_info *devinfo, enum hw_imm_type type,
+imm(FILE *file, const struct gen_device_info *devinfo, enum brw_reg_type type,
     const brw_inst *inst)
 {
    switch (type) {
-   case GEN8_HW_IMM_TYPE_UQ:
-      format(file, "0x%16lxUD", brw_inst_imm_uq(devinfo, inst));
+   case BRW_REGISTER_TYPE_UQ:
+      format(file, "0x%016"PRIx64"UQ", brw_inst_imm_uq(devinfo, inst));
       break;
-   case GEN8_HW_IMM_TYPE_Q:
-      format(file, "%ldD", brw_inst_imm_uq(devinfo, inst));
+   case BRW_REGISTER_TYPE_Q:
+      format(file, "0x%016"PRIx64"Q", brw_inst_imm_uq(devinfo, inst));
       break;
-   case BRW_HW_IMM_TYPE_UD:
+   case BRW_REGISTER_TYPE_UD:
       format(file, "0x%08xUD", brw_inst_imm_ud(devinfo, inst));
       break;
-   case BRW_HW_IMM_TYPE_D:
+   case BRW_REGISTER_TYPE_D:
       format(file, "%dD", brw_inst_imm_d(devinfo, inst));
       break;
-   case BRW_HW_IMM_TYPE_UW:
+   case BRW_REGISTER_TYPE_UW:
       format(file, "0x%04xUW", (uint16_t) brw_inst_imm_ud(devinfo, inst));
       break;
-   case BRW_HW_IMM_TYPE_W:
+   case BRW_REGISTER_TYPE_W:
       format(file, "%dW", (int16_t) brw_inst_imm_d(devinfo, inst));
       break;
-   case BRW_HW_IMM_TYPE_UV:
+   case BRW_REGISTER_TYPE_UV:
       format(file, "0x%08xUV", brw_inst_imm_ud(devinfo, inst));
       break;
-   case BRW_HW_IMM_TYPE_VF:
-      format(file, "[%-gF, %-gF, %-gF, %-gF]VF",
+   case BRW_REGISTER_TYPE_VF:
+      format(file, "0x%"PRIx64"VF", brw_inst_bits(inst, 127, 96));
+      pad(file, 48);
+      format(file, "/* [%-gF, %-gF, %-gF, %-gF]VF */",
              brw_vf_to_float(brw_inst_imm_ud(devinfo, inst)),
              brw_vf_to_float(brw_inst_imm_ud(devinfo, inst) >> 8),
              brw_vf_to_float(brw_inst_imm_ud(devinfo, inst) >> 16),
              brw_vf_to_float(brw_inst_imm_ud(devinfo, inst) >> 24));
       break;
-   case BRW_HW_IMM_TYPE_V:
+   case BRW_REGISTER_TYPE_V:
       format(file, "0x%08xV", brw_inst_imm_ud(devinfo, inst));
       break;
-   case BRW_HW_IMM_TYPE_F:
-      format(file, "%-gF", brw_inst_imm_f(devinfo, inst));
+   case BRW_REGISTER_TYPE_F:
+      /* The DIM instruction's src0 uses an F type but contains a
+       * 64-bit immediate
+       */
+      if (brw_inst_opcode(devinfo, inst) == BRW_OPCODE_DIM) {
+         format(file, "0x%"PRIx64"F", brw_inst_bits(inst, 127, 64));
+         pad(file, 48);
+         format(file, "/* %-gF */", brw_inst_imm_df(devinfo, inst));
+      } else {
+         format(file, "0x%"PRIx64"F", brw_inst_bits(inst, 127, 96));
+         pad(file, 48);
+         format(file, " /* %-gF */", brw_inst_imm_f(devinfo, inst));
+      }
       break;
-   case GEN8_HW_IMM_TYPE_DF:
-      format(file, "%-gDF", brw_inst_imm_df(devinfo, inst));
+   case BRW_REGISTER_TYPE_DF:
+      format(file, "0x%016"PRIx64"DF", brw_inst_bits(inst, 127, 64));
+      pad(file, 48);
+      format(file, "/* %-gDF */", brw_inst_imm_df(devinfo, inst));
       break;
-   case GEN8_HW_IMM_TYPE_HF:
+   case BRW_REGISTER_TYPE_HF:
       string(file, "Half Float IMM");
       break;
+   case BRW_REGISTER_TYPE_NF:
+   case BRW_REGISTER_TYPE_UB:
+   case BRW_REGISTER_TYPE_B:
+      format(file, "*** invalid immediate type %d ", type);
    }
    return 0;
 }
 
+static int
+src_sends_da(FILE *file,
+             const struct gen_device_info *devinfo,
+             enum brw_reg_type type,
+             enum brw_reg_file _reg_file,
+             unsigned _reg_nr,
+             unsigned _reg_subnr)
+{
+   int err = 0;
+
+   err |= reg(file, _reg_file, _reg_nr);
+   if (err == -1)
+      return 0;
+   if (_reg_subnr)
+      format(file, ".1");
+   string(file, brw_reg_type_to_letters(type));
+
+   return err;
+}
+
+static int
+src_sends_ia(FILE *file,
+             const struct gen_device_info *devinfo,
+             enum brw_reg_type type,
+             int _addr_imm,
+             unsigned _addr_subreg_nr)
+{
+   string(file, "g[a0");
+   if (_addr_subreg_nr)
+      format(file, ".1");
+   if (_addr_imm)
+      format(file, " %d", _addr_imm);
+   string(file, "]");
+   string(file, brw_reg_type_to_letters(type));
+
+   return 0;
+}
+
 static int
 src0(FILE *file, const struct gen_device_info *devinfo, const brw_inst *inst)
 {
-   if (brw_inst_src0_reg_file(devinfo, inst) == BRW_IMMEDIATE_VALUE) {
-      return imm(file, devinfo, brw_inst_src0_reg_type(devinfo, inst), inst);
+   if (is_split_send(devinfo, brw_inst_opcode(devinfo, inst))) {
+      if (devinfo->gen >= 12) {
+         return src_sends_da(file,
+                             devinfo,
+                             BRW_REGISTER_TYPE_UD,
+                             brw_inst_send_src0_reg_file(devinfo, inst),
+                             brw_inst_src0_da_reg_nr(devinfo, inst),
+                             0);
+      } else if (brw_inst_send_src0_address_mode(devinfo, inst) == BRW_ADDRESS_DIRECT) {
+         return src_sends_da(file,
+                             devinfo,
+                             BRW_REGISTER_TYPE_UD,
+                             brw_inst_send_src0_reg_file(devinfo, inst),
+                             brw_inst_src0_da_reg_nr(devinfo, inst),
+                             brw_inst_src0_da16_subreg_nr(devinfo, inst));
+      } else {
+         return src_sends_ia(file,
+                             devinfo,
+                             BRW_REGISTER_TYPE_UD,
+                             brw_inst_send_src0_ia16_addr_imm(devinfo, inst),
+                             brw_inst_src0_ia_subreg_nr(devinfo, inst));
+      }
+   } else if (brw_inst_src0_reg_file(devinfo, inst) == BRW_IMMEDIATE_VALUE) {
+      return imm(file, devinfo, brw_inst_src0_type(devinfo, inst), inst);
    } else if (brw_inst_access_mode(devinfo, inst) == BRW_ALIGN_1) {
       if (brw_inst_src0_address_mode(devinfo, inst) == BRW_ADDRESS_DIRECT) {
          return src_da1(file,
                         devinfo,
                         brw_inst_opcode(devinfo, inst),
-                        brw_inst_src0_reg_type(devinfo, inst),
+                        brw_inst_src0_type(devinfo, inst),
                         brw_inst_src0_reg_file(devinfo, inst),
                         brw_inst_src0_vstride(devinfo, inst),
                         brw_inst_src0_width(devinfo, inst),
@@ -1096,8 +1476,7 @@ src0(FILE *file, const struct gen_device_info *devinfo, const brw_inst *inst)
          return src_ia1(file,
                         devinfo,
                         brw_inst_opcode(devinfo, inst),
-                        brw_inst_src0_reg_type(devinfo, inst),
-                        brw_inst_src0_reg_file(devinfo, inst),
+                        brw_inst_src0_type(devinfo, inst),
                         brw_inst_src0_ia1_addr_imm(devinfo, inst),
                         brw_inst_src0_ia_subreg_nr(devinfo, inst),
                         brw_inst_src0_negate(devinfo, inst),
@@ -1111,7 +1490,7 @@ src0(FILE *file, const struct gen_device_info *devinfo, const brw_inst *inst)
          return src_da16(file,
                          devinfo,
                          brw_inst_opcode(devinfo, inst),
-                         brw_inst_src0_reg_type(devinfo, inst),
+                         brw_inst_src0_type(devinfo, inst),
                          brw_inst_src0_reg_file(devinfo, inst),
                          brw_inst_src0_vstride(devinfo, inst),
                          brw_inst_src0_da_reg_nr(devinfo, inst),
@@ -1132,14 +1511,21 @@ src0(FILE *file, const struct gen_device_info *devinfo, const brw_inst *inst)
 static int
 src1(FILE *file, const struct gen_device_info *devinfo, const brw_inst *inst)
 {
-   if (brw_inst_src1_reg_file(devinfo, inst) == BRW_IMMEDIATE_VALUE) {
-      return imm(file, devinfo, brw_inst_src1_reg_type(devinfo, inst), inst);
+   if (is_split_send(devinfo, brw_inst_opcode(devinfo, inst))) {
+      return src_sends_da(file,
+                          devinfo,
+                          BRW_REGISTER_TYPE_UD,
+                          brw_inst_send_src1_reg_file(devinfo, inst),
+                          brw_inst_send_src1_reg_nr(devinfo, inst),
+                          0 /* subreg_nr */);
+   } else if (brw_inst_src1_reg_file(devinfo, inst) == BRW_IMMEDIATE_VALUE) {
+      return imm(file, devinfo, brw_inst_src1_type(devinfo, inst), inst);
    } else if (brw_inst_access_mode(devinfo, inst) == BRW_ALIGN_1) {
       if (brw_inst_src1_address_mode(devinfo, inst) == BRW_ADDRESS_DIRECT) {
          return src_da1(file,
                         devinfo,
                         brw_inst_opcode(devinfo, inst),
-                        brw_inst_src1_reg_type(devinfo, inst),
+                        brw_inst_src1_type(devinfo, inst),
                         brw_inst_src1_reg_file(devinfo, inst),
                         brw_inst_src1_vstride(devinfo, inst),
                         brw_inst_src1_width(devinfo, inst),
@@ -1152,8 +1538,7 @@ src1(FILE *file, const struct gen_device_info *devinfo, const brw_inst *inst)
          return src_ia1(file,
                         devinfo,
                         brw_inst_opcode(devinfo, inst),
-                        brw_inst_src1_reg_type(devinfo, inst),
-                        brw_inst_src1_reg_file(devinfo, inst),
+                        brw_inst_src1_type(devinfo, inst),
                         brw_inst_src1_ia1_addr_imm(devinfo, inst),
                         brw_inst_src1_ia_subreg_nr(devinfo, inst),
                         brw_inst_src1_negate(devinfo, inst),
@@ -1167,7 +1552,7 @@ src1(FILE *file, const struct gen_device_info *devinfo, const brw_inst *inst)
          return src_da16(file,
                          devinfo,
                          brw_inst_opcode(devinfo, inst),
-                         brw_inst_src1_reg_type(devinfo, inst),
+                         brw_inst_src1_type(devinfo, inst),
                          brw_inst_src1_reg_file(devinfo, inst),
                          brw_inst_src1_vstride(devinfo, inst),
                          brw_inst_src1_da_reg_nr(devinfo, inst),
@@ -1219,6 +1604,19 @@ qtr_ctrl(FILE *file, const struct gen_device_info *devinfo, const brw_inst *inst
    return 0;
 }
 
+static int
+swsb(FILE *file, const struct gen_device_info *devinfo, const brw_inst *inst)
+{
+   const struct tgl_swsb swsb = tgl_swsb_decode(brw_inst_swsb(devinfo, inst));
+   if (swsb.regdist)
+      format(file, " @%d", swsb.regdist);
+   if (swsb.mode)
+      format(file, " $%d%s", swsb.sbid,
+             (swsb.mode & TGL_SBID_SET ? "" :
+              swsb.mode & TGL_SBID_DST ? ".dst" : ".src"));
+   return 0;
+}
+
 #ifdef DEBUG
 static __attribute__((__unused__)) int
 brw_disassemble_imm(const struct gen_device_info *devinfo,
@@ -1245,9 +1643,9 @@ brw_disassemble_inst(FILE *file, const struct gen_device_info *devinfo,
       string(file, "(");
       err |= control(file, "predicate inverse", pred_inv,
                      brw_inst_pred_inv(devinfo, inst), NULL);
-      format(file, "f%"PRIu64, devinfo->gen >= 7 ? brw_inst_flag_reg_nr(devinfo, inst) : 0);
-      if (brw_inst_flag_subreg_nr(devinfo, inst))
-         format(file, ".%"PRIu64, brw_inst_flag_subreg_nr(devinfo, inst));
+      format(file, "f%"PRIu64".%"PRIu64,
+             devinfo->gen >= 7 ? brw_inst_flag_reg_nr(devinfo, inst) : 0,
+             brw_inst_flag_subreg_nr(devinfo, inst));
       if (brw_inst_access_mode(devinfo, inst) == BRW_ALIGN_1) {
          err |= control(file, "predicate control align1", pred_ctrl_align1,
                         brw_inst_pred_control(devinfo, inst), NULL);
@@ -1259,8 +1657,10 @@ brw_disassemble_inst(FILE *file, const struct gen_device_info *devinfo,
    }
 
    err |= print_opcode(file, devinfo, opcode);
-   err |= control(file, "saturate", saturate, brw_inst_saturate(devinfo, inst),
-                  NULL);
+
+   if (!is_send(opcode))
+      err |= control(file, "saturate", saturate, brw_inst_saturate(devinfo, inst),
+                     NULL);
 
    err |= control(file, "debug control", debug_ctrl,
                   brw_inst_debug_control(devinfo, inst), NULL);
@@ -1269,7 +1669,13 @@ brw_disassemble_inst(FILE *file, const struct gen_device_info *devinfo,
       string(file, " ");
       err |= control(file, "function", math_function,
                      brw_inst_math_function(devinfo, inst), NULL);
-   } else if (opcode != BRW_OPCODE_SEND && opcode != BRW_OPCODE_SENDC) {
+
+   } else if (opcode == BRW_OPCODE_SYNC) {
+      string(file, " ");
+      err |= control(file, "function", sync_function,
+                     brw_inst_cond_modifier(devinfo, inst), NULL);
+
+   } else if (!is_send(opcode)) {
       err |= control(file, "conditional modifier", conditional_modifier,
                      brw_inst_cond_modifier(devinfo, inst), NULL);
 
@@ -1279,12 +1685,12 @@ brw_disassemble_inst(FILE *file, const struct gen_device_info *devinfo,
        */
       if (brw_inst_cond_modifier(devinfo, inst) &&
           (devinfo->gen < 6 || (opcode != BRW_OPCODE_SEL &&
+                                opcode != BRW_OPCODE_CSEL &&
                                 opcode != BRW_OPCODE_IF &&
                                 opcode != BRW_OPCODE_WHILE))) {
-         format(file, ".f%"PRIu64,
-                devinfo->gen >= 7 ? brw_inst_flag_reg_nr(devinfo, inst) : 0);
-         if (brw_inst_flag_subreg_nr(devinfo, inst))
-            format(file, ".%"PRIu64, brw_inst_flag_subreg_nr(devinfo, inst));
+         format(file, ".f%"PRIu64".%"PRIu64,
+                devinfo->gen >= 7 ? brw_inst_flag_reg_nr(devinfo, inst) : 0,
+                brw_inst_flag_subreg_nr(devinfo, inst));
       }
    }
 
@@ -1320,7 +1726,8 @@ brw_disassemble_inst(FILE *file, const struct gen_device_info *devinfo,
       format(file, "Pop: %"PRIu64, brw_inst_gen4_pop_count(devinfo, inst));
    } else if (devinfo->gen < 6 && (opcode == BRW_OPCODE_IF ||
                                    opcode == BRW_OPCODE_IFF ||
-                                   opcode == BRW_OPCODE_HALT)) {
+                                   opcode == BRW_OPCODE_HALT ||
+                                   opcode == BRW_OPCODE_WHILE)) {
       pad(file, 16);
       format(file, "Jump: %d", brw_inst_gen4_jump_count(devinfo, inst));
    } else if (devinfo->gen < 6 && opcode == BRW_OPCODE_ENDIF) {
@@ -1358,13 +1765,46 @@ brw_disassemble_inst(FILE *file, const struct gen_device_info *devinfo,
       }
    }
 
-   if (opcode == BRW_OPCODE_SEND || opcode == BRW_OPCODE_SENDC) {
+   if (is_send(opcode)) {
       enum brw_message_target sfid = brw_inst_sfid(devinfo, inst);
 
-      if (brw_inst_src1_reg_file(devinfo, inst) != BRW_IMMEDIATE_VALUE) {
-         /* show the indirect descriptor source */
-         pad(file, 48);
-         err |= src1(file, devinfo, inst);
+      bool has_imm_desc = false, has_imm_ex_desc = false;
+      uint32_t imm_desc = 0, imm_ex_desc = 0;
+      if (is_split_send(devinfo, opcode)) {
+         pad(file, 64);
+         if (brw_inst_send_sel_reg32_desc(devinfo, inst)) {
+            /* show the indirect descriptor source */
+            err |= src_sends_ia(file, devinfo, BRW_REGISTER_TYPE_UD, 0, 0);
+         } else {
+            has_imm_desc = true;
+            imm_desc = brw_inst_send_desc(devinfo, inst);
+            fprintf(file, "0x%08"PRIx32, imm_desc);
+         }
+
+         pad(file, 80);
+         if (brw_inst_send_sel_reg32_ex_desc(devinfo, inst)) {
+            /* show the indirect descriptor source */
+            err |= src_sends_ia(file, devinfo, BRW_REGISTER_TYPE_UD, 0,
+                                brw_inst_send_ex_desc_ia_subreg_nr(devinfo, inst));
+         } else {
+            has_imm_ex_desc = true;
+            imm_ex_desc = brw_inst_sends_ex_desc(devinfo, inst);
+            fprintf(file, "0x%08"PRIx32, imm_ex_desc);
+         }
+      } else {
+         if (brw_inst_src1_reg_file(devinfo, inst) != BRW_IMMEDIATE_VALUE) {
+            /* show the indirect descriptor source */
+            pad(file, 48);
+            err |= src1(file, devinfo, inst);
+            pad(file, 64);
+         } else {
+            has_imm_desc = true;
+            imm_desc = brw_inst_send_desc(devinfo, inst);
+            pad(file, 48);
+         }
+
+         /* Print message descriptor as immediate source */
+         fprintf(file, "0x%08"PRIx64, inst->data[1] >> 32);
       }
 
       newline(file);
@@ -1374,9 +1814,9 @@ brw_disassemble_inst(FILE *file, const struct gen_device_info *devinfo,
       fprintf(file, "            ");
       err |= control(file, "SFID", devinfo->gen >= 6 ? gen6_sfid : gen4_sfid,
                      sfid, &space);
+      string(file, " MsgDesc:");
 
-
-      if (brw_inst_src1_reg_file(devinfo, inst) != BRW_IMMEDIATE_VALUE) {
+      if (!has_imm_desc) {
          format(file, " indirect");
       } else {
          switch (sfid) {
@@ -1395,21 +1835,24 @@ brw_disassemble_inst(FILE *file, const struct gen_device_info *devinfo,
          case BRW_SFID_SAMPLER:
             if (devinfo->gen >= 5) {
                err |= control(file, "sampler message", gen5_sampler_msg_type,
-                              brw_inst_sampler_msg_type(devinfo, inst), &space);
+                              brw_sampler_desc_msg_type(devinfo, imm_desc),
+                              &space);
                err |= control(file, "sampler simd mode", gen5_sampler_simd_mode,
-                              brw_inst_sampler_simd_mode(devinfo, inst), &space);
-               format(file, " Surface = %"PRIu64" Sampler = %"PRIu64,
-                      brw_inst_binding_table_index(devinfo, inst),
-                      brw_inst_sampler(devinfo, inst));
+                              brw_sampler_desc_simd_mode(devinfo, imm_desc),
+                              &space);
+               format(file, " Surface = %u Sampler = %u",
+                      brw_sampler_desc_binding_table_index(devinfo, imm_desc),
+                      brw_sampler_desc_sampler(devinfo, imm_desc));
             } else {
-               format(file, " (%"PRIu64", %"PRIu64", %"PRIu64", ",
-                      brw_inst_binding_table_index(devinfo, inst),
-                      brw_inst_sampler(devinfo, inst),
-                      brw_inst_sampler_msg_type(devinfo, inst));
+               format(file, " (%u, %u, %u, ",
+                      brw_sampler_desc_binding_table_index(devinfo, imm_desc),
+                      brw_sampler_desc_sampler(devinfo, imm_desc),
+                      brw_sampler_desc_msg_type(devinfo, imm_desc));
                if (!devinfo->is_g4x) {
                   err |= control(file, "sampler target format",
                                  sampler_target_format,
-                                 brw_inst_sampler_return_format(devinfo, inst), NULL);
+                                 brw_sampler_desc_return_format(devinfo, imm_desc),
+                                 NULL);
                }
                string(file, ")");
             }
@@ -1418,29 +1861,31 @@ brw_disassemble_inst(FILE *file, const struct gen_device_info *devinfo,
          case GEN6_SFID_DATAPORT_CONSTANT_CACHE:
             /* aka BRW_SFID_DATAPORT_READ on Gen4-5 */
             if (devinfo->gen >= 6) {
-               format(file, " (%"PRIu64", %"PRIu64", %"PRIu64", %"PRIu64")",
-                      brw_inst_binding_table_index(devinfo, inst),
-                      brw_inst_dp_msg_control(devinfo, inst),
-                      brw_inst_dp_msg_type(devinfo, inst),
-                      devinfo->gen >= 7 ? 0 : brw_inst_dp_write_commit(devinfo, inst));
+               format(file, " (%u, %u, %u, %u)",
+                      brw_dp_desc_binding_table_index(devinfo, imm_desc),
+                      brw_dp_desc_msg_control(devinfo, imm_desc),
+                      brw_dp_desc_msg_type(devinfo, imm_desc),
+                      devinfo->gen >= 7 ? 0u :
+                      brw_dp_write_desc_write_commit(devinfo, imm_desc));
             } else {
                bool is_965 = devinfo->gen == 4 && !devinfo->is_g4x;
                err |= control(file, "DP read message type",
                               is_965 ? gen4_dp_read_port_msg_type :
                                        g45_dp_read_port_msg_type,
-                              brw_inst_dp_read_msg_type(devinfo, inst),
+                              brw_dp_read_desc_msg_type(devinfo, imm_desc),
                               &space);
 
-               format(file, " MsgCtrl = 0x%"PRIx64,
-                      brw_inst_dp_read_msg_control(devinfo, inst));
+               format(file, " MsgCtrl = 0x%u",
+                      brw_dp_read_desc_msg_control(devinfo, imm_desc));
 
-               format(file, " Surface = %"PRIu64, brw_inst_binding_table_index(devinfo, inst));
+               format(file, " Surface = %u",
+                      brw_dp_desc_binding_table_index(devinfo, imm_desc));
             }
             break;
 
          case GEN6_SFID_DATAPORT_RENDER_CACHE: {
             /* aka BRW_SFID_DATAPORT_WRITE on Gen4-5 */
-            unsigned msg_type = brw_inst_dp_write_msg_type(devinfo, inst);
+            unsigned msg_type = brw_dp_write_desc_msg_type(devinfo, imm_desc);
 
             err |= control(file, "DP rc message type",
                            dp_rc_msg_type(devinfo), msg_type, &space);
@@ -1454,16 +1899,18 @@ brw_disassemble_inst(FILE *file, const struct gen_device_info *devinfo,
                               brw_inst_rt_message_type(devinfo, inst), &space);
                if (devinfo->gen >= 6 && brw_inst_rt_slot_group(devinfo, inst))
                   string(file, " Hi");
-               if (brw_inst_rt_last(devinfo, inst))
+               if (brw_dp_write_desc_last_render_target(devinfo, imm_desc))
                   string(file, " LastRT");
-               if (devinfo->gen < 7 && brw_inst_dp_write_commit(devinfo, inst))
+               if (devinfo->gen < 7 &&
+                   brw_dp_write_desc_write_commit(devinfo, imm_desc))
                   string(file, " WriteCommit");
             } else {
-               format(file, " MsgCtrl = 0x%"PRIx64,
-                      brw_inst_dp_write_msg_control(devinfo, inst));
+               format(file, " MsgCtrl = 0x%u",
+                      brw_dp_write_desc_msg_control(devinfo, imm_desc));
             }
 
-            format(file, " Surface = %"PRIu64, brw_inst_binding_table_index(devinfo, inst));
+            format(file, " Surface = %u",
+                   brw_dp_desc_binding_table_index(devinfo, imm_desc));
             break;
          }
 
@@ -1520,17 +1967,20 @@ brw_disassemble_inst(FILE *file, const struct gen_device_info *devinfo,
 
                err |= control(file, "DP DC0 message type",
                               dp_dc0_msg_type_gen7,
-                              brw_inst_dp_msg_type(devinfo, inst), &space);
+                              brw_dp_desc_msg_type(devinfo, imm_desc), &space);
 
-               format(file, ", %"PRIu64", ", brw_inst_binding_table_index(devinfo, inst));
+               format(file, ", %u, ",
+                      brw_dp_desc_binding_table_index(devinfo, imm_desc));
 
                switch (brw_inst_dp_msg_type(devinfo, inst)) {
                case GEN7_DATAPORT_DC_UNTYPED_ATOMIC_OP:
                   control(file, "atomic op", aop,
-                          brw_inst_imm_ud(devinfo, inst) >> 8 & 0xf, &space);
+                          brw_dp_desc_msg_control(devinfo, imm_desc) & 0xf,
+                          &space);
                   break;
                default:
-                  format(file, "%"PRIu64, brw_inst_dp_msg_control(devinfo, inst));
+                  format(file, "%u",
+                         brw_dp_desc_msg_control(devinfo, imm_desc));
                }
                format(file, ")");
                break;
@@ -1541,14 +1991,14 @@ brw_disassemble_inst(FILE *file, const struct gen_device_info *devinfo,
             if (devinfo->gen >= 7) {
                format(file, " (");
 
-               unsigned msg_ctrl = brw_inst_dp_msg_control(devinfo, inst);
+               unsigned msg_ctrl = brw_dp_desc_msg_control(devinfo, imm_desc);
 
                err |= control(file, "DP DC1 message type",
                               dp_dc1_msg_type_hsw,
-                              brw_inst_dp_msg_type(devinfo, inst), &space);
+                              brw_dp_desc_msg_type(devinfo, imm_desc), &space);
 
-               format(file, ", Surface = %"PRIu64", ",
-                      brw_inst_binding_table_index(devinfo, inst));
+               format(file, ", Surface = %u, ",
+                      brw_dp_desc_binding_table_index(devinfo, imm_desc));
 
                switch (brw_inst_dp_msg_type(devinfo, inst)) {
                case HSW_DATAPORT_DC_PORT1_UNTYPED_ATOMIC_OP:
@@ -1559,17 +2009,26 @@ brw_disassemble_inst(FILE *file, const struct gen_device_info *devinfo,
                case HSW_DATAPORT_DC_PORT1_UNTYPED_ATOMIC_OP_SIMD4X2:
                case HSW_DATAPORT_DC_PORT1_TYPED_ATOMIC_OP_SIMD4X2:
                case HSW_DATAPORT_DC_PORT1_ATOMIC_COUNTER_OP_SIMD4X2:
+               case GEN8_DATAPORT_DC_PORT1_A64_UNTYPED_ATOMIC_OP:
                   control(file, "atomic op", aop, msg_ctrl & 0xf, &space);
                   break;
                case HSW_DATAPORT_DC_PORT1_UNTYPED_SURFACE_READ:
                case HSW_DATAPORT_DC_PORT1_UNTYPED_SURFACE_WRITE:
                case HSW_DATAPORT_DC_PORT1_TYPED_SURFACE_READ:
-               case HSW_DATAPORT_DC_PORT1_TYPED_SURFACE_WRITE: {
+               case HSW_DATAPORT_DC_PORT1_TYPED_SURFACE_WRITE:
+               case GEN8_DATAPORT_DC_PORT1_A64_UNTYPED_SURFACE_WRITE:
+               case GEN8_DATAPORT_DC_PORT1_A64_UNTYPED_SURFACE_READ: {
                   static const char *simd_modes[] = { "4x2", "16", "8" };
                   format(file, "SIMD%s, Mask = 0x%x",
                          simd_modes[msg_ctrl >> 4], msg_ctrl & 0xf);
                   break;
                }
+               case GEN9_DATAPORT_DC_PORT1_UNTYPED_ATOMIC_FLOAT_OP:
+               case GEN9_DATAPORT_DC_PORT1_A64_UNTYPED_ATOMIC_FLOAT_OP:
+                  format(file, "SIMD%d,", (msg_ctrl & (1 << 4)) ? 8 : 16);
+                  control(file, "atomic float op", aop_float, msg_ctrl & 0xf,
+                          &space);
+                  break;
                default:
                   format(file, "0x%x", msg_ctrl);
                }
@@ -1596,9 +2055,15 @@ brw_disassemble_inst(FILE *file, const struct gen_device_info *devinfo,
 
          if (space)
             string(file, " ");
-         format(file, "mlen %"PRIu64, brw_inst_mlen(devinfo, inst));
-         format(file, " rlen %"PRIu64, brw_inst_rlen(devinfo, inst));
       }
+      if (has_imm_desc)
+         format(file, "mlen %u", brw_message_desc_mlen(devinfo, imm_desc));
+      if (has_imm_ex_desc) {
+         format(file, " ex_mlen %u",
+                brw_message_ex_desc_ex_mlen(devinfo, imm_ex_desc));
+      }
+      if (has_imm_desc)
+         format(file, " rlen %u", brw_message_desc_rlen(devinfo, imm_desc));
    }
    pad(file, 64);
    if (opcode != BRW_OPCODE_NOP && opcode != BRW_OPCODE_NENOP) {
@@ -1613,9 +2078,12 @@ brw_disassemble_inst(FILE *file, const struct gen_device_info *devinfo,
          err |= control(file, "mask control", mask_ctrl,
                         brw_inst_mask_control(devinfo, inst), &space);
       }
-      err |= control(file, "dependency control", dep_ctrl,
-                     ((brw_inst_no_dd_check(devinfo, inst) << 1) |
-                      brw_inst_no_dd_clear(devinfo, inst)), &space);
+
+      if (devinfo->gen < 12) {
+         err |= control(file, "dependency control", dep_ctrl,
+                        ((brw_inst_no_dd_check(devinfo, inst) << 1) |
+                         brw_inst_no_dd_clear(devinfo, inst)), &space);
+      }
 
       if (devinfo->gen >= 6)
          err |= qtr_ctrl(file, devinfo, inst);
@@ -1631,9 +2099,14 @@ brw_disassemble_inst(FILE *file, const struct gen_device_info *devinfo,
          }
       }
 
+      if (devinfo->gen >= 12)
+         err |= swsb(file, devinfo, inst);
+
       err |= control(file, "compaction", cmpt_ctrl, is_compacted, &space);
       err |= control(file, "thread control", thread_ctrl,
-                     brw_inst_thread_control(devinfo, inst), &space);
+                     (devinfo->gen >= 12 ? brw_inst_atomic_control(devinfo, inst) :
+                                           brw_inst_thread_control(devinfo, inst)),
+                     &space);
       if (has_branch_ctrl(devinfo, opcode)) {
          err |= control(file, "branch ctrl", branch_ctrl,
                         brw_inst_branch_control(devinfo, inst), &space);
@@ -1641,7 +2114,7 @@ brw_disassemble_inst(FILE *file, const struct gen_device_info *devinfo,
          err |= control(file, "acc write control", accwr,
                         brw_inst_acc_wr_control(devinfo, inst), &space);
       }
-      if (opcode == BRW_OPCODE_SEND || opcode == BRW_OPCODE_SENDC)
+      if (is_send(opcode))
          err |= control(file, "end of thread", end_of_thread,
                         brw_inst_eot(devinfo, inst), &space);
       if (space)