unsigned num_channels,
bool header_present);
-void
-brw_typed_atomic(struct brw_codegen *p,
- struct brw_reg dst,
- struct brw_reg payload,
- struct brw_reg surface,
- unsigned atomic_op,
- unsigned msg_length,
- bool response_expected,
- bool header_present);
-
-void
-brw_typed_surface_read(struct brw_codegen *p,
- struct brw_reg dst,
- struct brw_reg payload,
- struct brw_reg surface,
- unsigned msg_length,
- unsigned num_channels,
- bool header_present);
-
-void
-brw_typed_surface_write(struct brw_codegen *p,
- struct brw_reg payload,
- struct brw_reg surface,
- unsigned msg_length,
- unsigned num_channels,
- bool header_present);
-
void
brw_memory_fence(struct brw_codegen *p,
struct brw_reg dst,
payload, surface, desc);
}
-void
-brw_typed_atomic(struct brw_codegen *p,
- struct brw_reg dst,
- struct brw_reg payload,
- struct brw_reg surface,
- unsigned atomic_op,
- unsigned msg_length,
- bool response_expected,
- bool header_present) {
- const struct gen_device_info *devinfo = p->devinfo;
- const unsigned sfid = (devinfo->gen >= 8 || devinfo->is_haswell ?
- HSW_SFID_DATAPORT_DATA_CACHE_1 :
- GEN6_SFID_DATAPORT_RENDER_CACHE);
- const bool align1 = brw_get_default_access_mode(p) == BRW_ALIGN_1;
- /* SIMD4x2 typed atomic instructions only exist on HSW+ */
- const bool has_simd4x2 = devinfo->gen >= 8 || devinfo->is_haswell;
- const unsigned exec_size = align1 ? 1 << brw_get_default_exec_size(p) :
- has_simd4x2 ? 0 : 8;
- /* Typed atomics don't support SIMD16 */
- assert(exec_size <= 8);
- const unsigned response_length =
- brw_surface_payload_size(p, response_expected, exec_size);
- const unsigned desc =
- brw_message_desc(devinfo, msg_length, response_length, header_present) |
- brw_dp_typed_atomic_desc(devinfo, exec_size, brw_get_default_group(p),
- atomic_op, response_expected);
- /* Mask out unused components -- See comment in brw_untyped_atomic(). */
- const unsigned mask = align1 ? WRITEMASK_XYZW : WRITEMASK_X;
-
- brw_send_indirect_surface_message(p, sfid, brw_writemask(dst, mask),
- payload, surface, desc);
-}
-
-void
-brw_typed_surface_read(struct brw_codegen *p,
- struct brw_reg dst,
- struct brw_reg payload,
- struct brw_reg surface,
- unsigned msg_length,
- unsigned num_channels,
- bool header_present)
-{
- const struct gen_device_info *devinfo = p->devinfo;
- const unsigned sfid = (devinfo->gen >= 8 || devinfo->is_haswell ?
- HSW_SFID_DATAPORT_DATA_CACHE_1 :
- GEN6_SFID_DATAPORT_RENDER_CACHE);
- const bool align1 = brw_get_default_access_mode(p) == BRW_ALIGN_1;
- /* SIMD4x2 typed read instructions only exist on HSW+ */
- const bool has_simd4x2 = devinfo->gen >= 8 || devinfo->is_haswell;
- const unsigned exec_size = align1 ? 1 << brw_get_default_exec_size(p) :
- has_simd4x2 ? 0 : 8;
- const unsigned response_length =
- brw_surface_payload_size(p, num_channels, exec_size);
- const unsigned desc =
- brw_message_desc(devinfo, msg_length, response_length, header_present) |
- brw_dp_typed_surface_rw_desc(devinfo, exec_size, brw_get_default_group(p),
- num_channels, false);
-
- brw_send_indirect_surface_message(p, sfid, dst, payload, surface, desc);
-}
-
-void
-brw_typed_surface_write(struct brw_codegen *p,
- struct brw_reg payload,
- struct brw_reg surface,
- unsigned msg_length,
- unsigned num_channels,
- bool header_present)
-{
- const struct gen_device_info *devinfo = p->devinfo;
- const unsigned sfid = (devinfo->gen >= 8 || devinfo->is_haswell ?
- HSW_SFID_DATAPORT_DATA_CACHE_1 :
- GEN6_SFID_DATAPORT_RENDER_CACHE);
- const bool align1 = brw_get_default_access_mode(p) == BRW_ALIGN_1;
- /* SIMD4x2 typed read instructions only exist on HSW+ */
- const bool has_simd4x2 = devinfo->gen >= 8 || devinfo->is_haswell;
- const unsigned exec_size = align1 ? 1 << brw_get_default_exec_size(p) :
- has_simd4x2 ? 0 : 8;
- const unsigned desc =
- brw_message_desc(devinfo, msg_length, 0, header_present) |
- brw_dp_typed_surface_rw_desc(devinfo, exec_size, brw_get_default_group(p),
- num_channels, true);
- /* Mask out unused components -- See comment in brw_untyped_atomic(). */
- const unsigned mask = !has_simd4x2 && !align1 ? WRITEMASK_X : WRITEMASK_XYZW;
-
- brw_send_indirect_surface_message(p, sfid, brw_writemask(brw_null_reg(), mask),
- payload, surface, desc);
-}
-
static void
brw_set_memory_fence_message(struct brw_codegen *p,
struct brw_inst *insn,
src[2].ud, inst->header_size);
break;
- case SHADER_OPCODE_TYPED_ATOMIC:
- assert(src[2].file == BRW_IMMEDIATE_VALUE);
- brw_typed_atomic(p, dst, src[0], src[1], src[2].ud, inst->mlen,
- !inst->dst.is_null(), inst->header_size);
- break;
-
- case SHADER_OPCODE_TYPED_SURFACE_READ:
- assert(src[2].file == BRW_IMMEDIATE_VALUE);
- brw_typed_surface_read(p, dst, src[0], src[1], inst->mlen,
- src[2].ud, inst->header_size);
- break;
-
- case SHADER_OPCODE_TYPED_SURFACE_WRITE:
- assert(src[2].file == BRW_IMMEDIATE_VALUE);
- brw_typed_surface_write(p, src[0], src[1], inst->mlen,
- src[2].ud, inst->header_size);
- break;
-
case SHADER_OPCODE_MEMORY_FENCE:
brw_memory_fence(p, dst, BRW_OPCODE_SEND);
break;
return emit_stride(bld, src_reg(tmp), n, has_simd4x2 ? 1 : 4, 1);
}
}
-
- /**
- * Convert an array of registers back into a VEC4 according to the
- * layout expected from some shared unit. If \p has_simd4x2 is true the
- * argument is left unmodified in SIMD4x2 form, otherwise it will be
- * rearranged from SIMD8 form.
- */
- static src_reg
- emit_extract(const vec4_builder &bld, const src_reg src,
- unsigned n, bool has_simd4x2)
- {
- if (src.file == BAD_FILE || n == 0) {
- return src_reg();
-
- } else {
- return emit_stride(bld, src, n, 1, has_simd4x2 ? 1 : 4);
- }
- }
}
}
has_simd4x2 && size ? 1 : size,
surface, op, rsize, pred);
}
-
- namespace {
- /**
- * Initialize the header present in typed surface messages.
- */
- src_reg
- emit_typed_message_header(const vec4_builder &bld)
- {
- const vec4_builder ubld = bld.exec_all();
- const dst_reg dst = bld.vgrf(BRW_REGISTER_TYPE_UD);
-
- ubld.MOV(dst, brw_imm_d(0));
-
- if (bld.shader->devinfo->gen == 7 &&
- !bld.shader->devinfo->is_haswell) {
- /* The sample mask is used on IVB for the SIMD8 messages that
- * have no SIMD4x2 variant. We only use the two X channels
- * in that case, mask everything else out.
- */
- ubld.MOV(writemask(dst, WRITEMASK_W), brw_imm_d(0x11));
- }
-
- return src_reg(dst);
- }
- }
-
- /**
- * Emit a typed surface read opcode. \p dims determines the number of
- * components of the address and \p size the number of components of the
- * returned value.
- */
- src_reg
- emit_typed_read(const vec4_builder &bld, const src_reg &surface,
- const src_reg &addr, unsigned dims, unsigned size)
- {
- const bool has_simd4x2 = (bld.shader->devinfo->gen >= 8 ||
- bld.shader->devinfo->is_haswell);
- const src_reg tmp =
- emit_send(bld, SHADER_OPCODE_TYPED_SURFACE_READ,
- emit_typed_message_header(bld),
- emit_insert(bld, addr, dims, has_simd4x2),
- has_simd4x2 ? 1 : dims,
- src_reg(), 0,
- surface, size,
- has_simd4x2 ? 1 : size);
-
- return emit_extract(bld, tmp, size, has_simd4x2);
- }
-
- /**
- * Emit a typed surface write opcode. \p dims determines the number of
- * components of the address and \p size the number of components of the
- * argument.
- */
- void
- emit_typed_write(const vec4_builder &bld, const src_reg &surface,
- const src_reg &addr, const src_reg &src,
- unsigned dims, unsigned size)
- {
- const bool has_simd4x2 = (bld.shader->devinfo->gen >= 8 ||
- bld.shader->devinfo->is_haswell);
- emit_send(bld, SHADER_OPCODE_TYPED_SURFACE_WRITE,
- emit_typed_message_header(bld),
- emit_insert(bld, addr, dims, has_simd4x2),
- has_simd4x2 ? 1 : dims,
- emit_insert(bld, src, size, has_simd4x2),
- has_simd4x2 ? 1 : size,
- surface, size, 0);
- }
-
- /**
- * Emit a typed surface atomic opcode. \p dims determines the number of
- * components of the address and \p rsize the number of components of
- * the returned value (either zero or one).
- */
- src_reg
- emit_typed_atomic(const vec4_builder &bld,
- const src_reg &surface, const src_reg &addr,
- const src_reg &src0, const src_reg &src1,
- unsigned dims, unsigned rsize, unsigned op,
- brw_predicate pred)
- {
- const bool has_simd4x2 = (bld.shader->devinfo->gen >= 8 ||
- bld.shader->devinfo->is_haswell);
-
- /* Zip the components of both sources, they are represented as the X
- * and Y components of the same vector.
- */
- const unsigned size = (src0.file != BAD_FILE) + (src1.file != BAD_FILE);
- const dst_reg srcs = bld.vgrf(BRW_REGISTER_TYPE_UD);
-
- if (size >= 1)
- bld.MOV(writemask(srcs, WRITEMASK_X), src0);
- if (size >= 2)
- bld.MOV(writemask(srcs, WRITEMASK_Y), src1);
-
- return emit_send(bld, SHADER_OPCODE_TYPED_ATOMIC,
- emit_typed_message_header(bld),
- emit_insert(bld, addr, dims, has_simd4x2),
- has_simd4x2 ? 1 : dims,
- emit_insert(bld, src_reg(srcs), size, has_simd4x2),
- has_simd4x2 ? 1 : size,
- surface, op, rsize, pred);
- }
}
}
const src_reg &src0, const src_reg &src1,
unsigned dims, unsigned rsize, unsigned op,
brw_predicate pred = BRW_PREDICATE_NONE);
-
- src_reg
- emit_typed_read(const vec4_builder &bld, const src_reg &surface,
- const src_reg &addr, unsigned dims, unsigned size);
-
- void
- emit_typed_write(const vec4_builder &bld, const src_reg &surface,
- const src_reg &addr, const src_reg &src,
- unsigned dims, unsigned size);
-
- src_reg
- emit_typed_atomic(const vec4_builder &bld, const src_reg &surface,
- const src_reg &addr,
- const src_reg &src0, const src_reg &src1,
- unsigned dims, unsigned rsize, unsigned op,
- brw_predicate pred = BRW_PREDICATE_NONE);
}
}