#include "brw_eu.h"
#include "brw_fs.h"
#include "brw_cfg.h"
-#include "brw_program.h"
static enum brw_reg_file
brw_file_from_reg(fs_reg *reg)
}
brw_reg = retype(brw_reg, reg->type);
- brw_reg = byte_offset(brw_reg, reg->subreg_offset);
+ brw_reg = byte_offset(brw_reg, reg->offset);
brw_reg.abs = reg->abs;
brw_reg.negate = reg->negate;
break;
case ARF:
case FIXED_GRF:
case IMM:
+ assert(reg->offset == 0);
brw_reg = reg->as_brw_reg();
break;
case BAD_FILE:
{
uint32_t msg_control;
- brw_wm_prog_data *prog_data = (brw_wm_prog_data*) this->prog_data;
+ struct brw_wm_prog_data *prog_data = brw_wm_prog_data(this->prog_data);
if (devinfo->gen < 6) {
brw_push_insn_state(p);
if (inst->opcode == FS_OPCODE_REP_FB_WRITE)
msg_control = BRW_DATAPORT_RENDER_TARGET_WRITE_SIMD16_SINGLE_SOURCE_REPLICATED;
else if (prog_data->dual_src_blend) {
- if (!inst->force_sechalf)
+ if (!inst->group)
msg_control = BRW_DATAPORT_RENDER_TARGET_WRITE_SIMD8_DUAL_SOURCE_SUBSPAN01;
else
msg_control = BRW_DATAPORT_RENDER_TARGET_WRITE_SIMD8_DUAL_SOURCE_SUBSPAN23;
void
fs_generator::generate_fb_write(fs_inst *inst, struct brw_reg payload)
{
- brw_wm_prog_data *prog_data = (brw_wm_prog_data*) this->prog_data;
+ struct brw_wm_prog_data *prog_data = brw_wm_prog_data(this->prog_data);
const brw_wm_prog_key * const key = (brw_wm_prog_key * const) this->key;
struct brw_reg implied_header;
}
}
+void
+fs_generator::generate_fb_read(fs_inst *inst, struct brw_reg dst,
+ struct brw_reg payload)
+{
+ assert(inst->size_written % REG_SIZE == 0);
+ struct brw_wm_prog_data *prog_data = brw_wm_prog_data(this->prog_data);
+ const unsigned surf_index =
+ prog_data->binding_table.render_target_start + inst->target;
+
+ gen9_fb_READ(p, dst, payload, surf_index,
+ inst->header_size, inst->size_written / REG_SIZE,
+ prog_data->persample_dispatch);
+
+ brw_mark_surface_used(&prog_data->base, surf_index);
+}
+
void
fs_generator::generate_mov_indirect(fs_inst *inst,
struct brw_reg dst,
indirect_byte_offset =
retype(spread(indirect_byte_offset, 2), BRW_REGISTER_TYPE_UW);
- struct brw_reg ind_src;
- if (devinfo->gen < 8) {
- /* From the Haswell PRM section "Register Region Restrictions":
- *
- * "The lower bits of the AddressImmediate must not overflow to
- * change the register address. The lower 5 bits of Address
- * Immediate when added to lower 5 bits of address register gives
- * the sub-register offset. The upper bits of Address Immediate
- * when added to upper bits of address register gives the register
- * address. Any overflow from sub-register offset is dropped."
- *
- * This restriction is only listed in the Haswell PRM but emperical
- * testing indicates that it applies on all older generations and is
- * lifted on Broadwell.
- *
- * Since the indirect may cause us to cross a register boundary, this
- * makes the base offset almost useless. We could try and do
- * something clever where we use a actual base offset if
- * base_offset % 32 == 0 but that would mean we were generating
- * different code depending on the base offset. Instead, for the
- * sake of consistency, we'll just do the add ourselves.
- */
- brw_ADD(p, addr, indirect_byte_offset, brw_imm_uw(imm_byte_offset));
- ind_src = brw_VxH_indirect(0, 0);
- } else {
- brw_MOV(p, addr, indirect_byte_offset);
- ind_src = brw_VxH_indirect(0, imm_byte_offset);
- }
+ /* There are a number of reasons why we don't use the base offset here.
+ * One reason is that the field is only 9 bits which means we can only
+ * use it to access the first 16 GRFs. Also, from the Haswell PRM
+ * section "Register Region Restrictions":
+ *
+ * "The lower bits of the AddressImmediate must not overflow to
+ * change the register address. The lower 5 bits of Address
+ * Immediate when added to lower 5 bits of address register gives
+ * the sub-register offset. The upper bits of Address Immediate
+ * when added to upper bits of address register gives the register
+ * address. Any overflow from sub-register offset is dropped."
+ *
+ * Since the indirect may cause us to cross a register boundary, this
+ * makes the base offset almost useless. We could try and do something
+ * clever where we use a actual base offset if base_offset % 32 == 0 but
+ * that would mean we were generating different code depending on the
+ * base offset. Instead, for the sake of consistency, we'll just do the
+ * add ourselves. This restriction is only listed in the Haswell PRM
+ * but empirical testing indicates that it applies on all older
+ * generations and is lifted on Broadwell.
+ *
+ * In the end, while base_offset is nice to look at in the generated
+ * code, using it saves us 0 instructions and would require quite a bit
+ * of case-by-case work. It's just not worth it.
+ */
+ brw_ADD(p, addr, indirect_byte_offset, brw_imm_uw(imm_byte_offset));
+ struct brw_reg ind_src = brw_VxH_indirect(0, 0);
- brw_inst *mov = brw_MOV(p, dst, retype(ind_src, dst.type));
+ brw_inst *mov = brw_MOV(p, dst, retype(ind_src, reg.type));
if (devinfo->gen == 6 && dst.file == BRW_MESSAGE_REGISTER_FILE &&
!inst->get_next()->is_tail_sentinel() &&
struct brw_reg dst,
struct brw_reg header)
{
+ assert(inst->size_written % REG_SIZE == 0);
assert(header.file == BRW_GENERAL_REGISTER_FILE);
assert(header.type == BRW_REGISTER_TYPE_UD);
brw_inst *send = brw_next_insn(p, BRW_OPCODE_SEND);
- brw_set_dest(p, send, dst);
+ brw_set_dest(p, send, retype(dst, BRW_REGISTER_TYPE_UD));
brw_set_src0(p, send, header);
brw_set_src1(p, send, brw_imm_ud(0u));
brw_inst_set_urb_per_slot_offset(p->devinfo, send, true);
brw_inst_set_mlen(p->devinfo, send, inst->mlen);
- brw_inst_set_rlen(p->devinfo, send, inst->regs_written);
+ brw_inst_set_rlen(p->devinfo, send, inst->size_written / REG_SIZE);
brw_inst_set_header_present(p->devinfo, send, true);
brw_inst_set_urb_global_offset(p->devinfo, send, inst->offset);
}
insn = brw_next_insn(p, BRW_OPCODE_SEND);
brw_set_dest(p, insn, retype(brw_null_reg(), BRW_REGISTER_TYPE_UW));
- brw_set_src0(p, insn, payload);
+ brw_set_src0(p, insn, retype(payload, BRW_REGISTER_TYPE_UW));
brw_set_src1(p, insn, brw_imm_d(0));
/* Terminate a compute shader by sending a message to the thread spawner.
struct brw_reg surface_index,
struct brw_reg sampler_index)
{
+ assert(inst->size_written % REG_SIZE == 0);
int msg_type = -1;
uint32_t simd_mode;
uint32_t return_format;
break;
case SHADER_OPCODE_TXD:
if (inst->shadow_compare) {
- /* Gen7.5+. Otherwise, lowered by brw_lower_texture_gradients(). */
+ /* Gen7.5+. Otherwise, lowered in NIR */
assert(devinfo->gen >= 8 || devinfo->is_haswell);
msg_type = HSW_SAMPLER_MESSAGE_SAMPLE_DERIV_COMPARE;
} else {
surface + base_binding_table_index,
sampler % 16,
msg_type,
- inst->regs_written,
+ inst->size_written / REG_SIZE,
inst->mlen,
inst->header_size != 0,
simd_mode,
if (brw_regs_equal(&surface_reg, &sampler_reg)) {
brw_MUL(p, addr, sampler_reg, brw_imm_uw(0x101));
} else {
- brw_SHL(p, addr, sampler_reg, brw_imm_ud(8));
- brw_OR(p, addr, addr, surface_reg);
+ if (sampler_reg.file == BRW_IMMEDIATE_VALUE) {
+ brw_OR(p, addr, surface_reg, brw_imm_ud(sampler_reg.ud << 8));
+ } else {
+ brw_SHL(p, addr, sampler_reg, brw_imm_ud(8));
+ brw_OR(p, addr, addr, surface_reg);
+ }
}
if (base_binding_table_index)
brw_ADD(p, addr, addr, brw_imm_ud(base_binding_table_index));
0 /* surface */,
0 /* sampler */,
msg_type,
- inst->regs_written,
+ inst->size_written / REG_SIZE,
inst->mlen /* mlen */,
inst->header_size != 0 /* header */,
simd_mode,
* current block (or the program).
*/
this->discard_halt_patches.push_tail(new(mem_ctx) ip_record(p->nr_insn));
-
- brw_push_insn_state(p);
- brw_set_default_mask_control(p, BRW_MASK_DISABLE);
gen6_HALT(p);
- brw_pop_insn_state(p);
}
void
fs_generator::generate_scratch_write(fs_inst *inst, struct brw_reg src)
{
- assert(inst->exec_size <= 16 || inst->force_writemask_all);
+ /* The 32-wide messages only respect the first 16-wide half of the channel
+ * enable signals which are replicated identically for the second group of
+ * 16 channels, so we cannot use them unless the write is marked
+ * force_writemask_all.
+ */
+ const unsigned lower_size = inst->force_writemask_all ? inst->exec_size :
+ MIN2(16, inst->exec_size);
+ const unsigned block_size = 4 * lower_size / REG_SIZE;
assert(inst->mlen != 0);
- brw_MOV(p,
- brw_uvec_mrf(inst->exec_size, (inst->base_mrf + 1), 0),
- retype(src, BRW_REGISTER_TYPE_UD));
- brw_oword_block_write_scratch(p, brw_message_reg(inst->base_mrf),
- inst->exec_size / 8, inst->offset);
+ brw_push_insn_state(p);
+ brw_set_default_exec_size(p, cvt(lower_size) - 1);
+ brw_set_default_compression(p, lower_size > 8);
+
+ for (unsigned i = 0; i < inst->exec_size / lower_size; i++) {
+ brw_set_default_group(p, inst->group + lower_size * i);
+
+ brw_MOV(p, brw_uvec_mrf(lower_size, inst->base_mrf + 1, 0),
+ retype(offset(src, block_size * i), BRW_REGISTER_TYPE_UD));
+
+ brw_oword_block_write_scratch(p, brw_message_reg(inst->base_mrf),
+ block_size,
+ inst->offset + block_size * REG_SIZE * i);
+ }
+
+ brw_pop_insn_state(p);
}
void
struct brw_reg index,
struct brw_reg offset)
{
+ assert(type_sz(dst.type) == 4);
assert(inst->mlen != 0);
assert(index.file == BRW_IMMEDIATE_VALUE &&
fs_generator::generate_uniform_pull_constant_load_gen7(fs_inst *inst,
struct brw_reg dst,
struct brw_reg index,
- struct brw_reg offset)
+ struct brw_reg payload)
{
assert(index.type == BRW_REGISTER_TYPE_UD);
-
- assert(offset.file == BRW_GENERAL_REGISTER_FILE);
- /* Reference just the dword we need, to avoid angering validate_reg(). */
- offset = brw_vec1_grf(offset.nr, 0);
-
- /* We use the SIMD4x2 mode because we want to end up with 4 components in
- * the destination loaded consecutively from the same offset (which appears
- * in the first component, and the rest are ignored).
- */
- dst.width = BRW_WIDTH_4;
-
- struct brw_reg src = offset;
- bool header_present = false;
-
- if (devinfo->gen >= 9) {
- /* Skylake requires a message header in order to use SIMD4x2 mode. */
- src = retype(brw_vec4_grf(offset.nr, 0), BRW_REGISTER_TYPE_UD);
- header_present = true;
-
- brw_push_insn_state(p);
- brw_set_default_mask_control(p, BRW_MASK_DISABLE);
- brw_set_default_exec_size(p, BRW_EXECUTE_8);
- brw_MOV(p, vec8(src), retype(brw_vec8_grf(0, 0), BRW_REGISTER_TYPE_UD));
- brw_set_default_access_mode(p, BRW_ALIGN_1);
-
- brw_MOV(p, get_element_ud(src, 2),
- brw_imm_ud(GEN9_SAMPLER_SIMD_MODE_EXTENSION_SIMD4X2));
- brw_pop_insn_state(p);
- }
+ assert(payload.file == BRW_GENERAL_REGISTER_FILE);
+ assert(type_sz(dst.type) == 4);
if (index.file == BRW_IMMEDIATE_VALUE) {
-
- uint32_t surf_index = index.ud;
+ const uint32_t surf_index = index.ud;
brw_push_insn_state(p);
- brw_set_default_compression_control(p, BRW_COMPRESSION_NONE);
brw_set_default_mask_control(p, BRW_MASK_DISABLE);
brw_inst *send = brw_next_insn(p, BRW_OPCODE_SEND);
- brw_inst_set_exec_size(devinfo, send, BRW_EXECUTE_4);
brw_pop_insn_state(p);
- brw_set_dest(p, send, dst);
- brw_set_src0(p, send, src);
- brw_set_sampler_message(p, send,
- surf_index,
- 0, /* LD message ignores sampler unit */
- GEN5_SAMPLER_MESSAGE_SAMPLE_LD,
- 1, /* rlen */
- inst->mlen,
- header_present,
- BRW_SAMPLER_SIMD_MODE_SIMD4X2,
- 0);
- } else {
+ brw_set_dest(p, send, retype(dst, BRW_REGISTER_TYPE_UD));
+ brw_set_src0(p, send, retype(payload, BRW_REGISTER_TYPE_UD));
+ brw_set_dp_read_message(p, send, surf_index,
+ BRW_DATAPORT_OWORD_BLOCK_DWORDS(inst->exec_size),
+ GEN7_DATAPORT_DC_OWORD_BLOCK_READ,
+ GEN6_SFID_DATAPORT_CONSTANT_CACHE,
+ 1, /* mlen */
+ true, /* header */
+ DIV_ROUND_UP(inst->size_written, REG_SIZE));
+ } else {
struct brw_reg addr = vec1(retype(brw_address_reg(0), BRW_REGISTER_TYPE_UD));
brw_push_insn_state(p);
brw_set_default_mask_control(p, BRW_MASK_DISABLE);
- brw_set_default_access_mode(p, BRW_ALIGN_1);
/* a0.0 = surf_index & 0xff */
brw_inst *insn_and = brw_next_insn(p, BRW_OPCODE_AND);
/* dst = send(payload, a0.0 | <descriptor>) */
brw_inst *insn = brw_send_indirect_message(
- p, BRW_SFID_SAMPLER, dst, src, addr);
- brw_set_sampler_message(p, insn,
- 0,
- 0, /* LD message ignores sampler unit */
- GEN5_SAMPLER_MESSAGE_SAMPLE_LD,
- 1, /* rlen */
- inst->mlen,
- header_present,
- BRW_SAMPLER_SIMD_MODE_SIMD4X2,
- 0);
+ p, GEN6_SFID_DATAPORT_CONSTANT_CACHE,
+ retype(dst, BRW_REGISTER_TYPE_UD),
+ retype(payload, BRW_REGISTER_TYPE_UD), addr);
+ brw_set_dp_read_message(p, insn, 0 /* surface */,
+ BRW_DATAPORT_OWORD_BLOCK_DWORDS(inst->exec_size),
+ GEN7_DATAPORT_DC_OWORD_BLOCK_READ,
+ GEN6_SFID_DATAPORT_CONSTANT_CACHE,
+ 1, /* mlen */
+ true, /* header */
+ DIV_ROUND_UP(inst->size_written, REG_SIZE));
brw_pop_insn_state(p);
}
*/
msg_type = BRW_SAMPLER_MESSAGE_SIMD16_LD;
assert(inst->mlen == 3);
- assert(inst->regs_written == 8);
+ assert(inst->size_written == 8 * REG_SIZE);
rlen = 8;
simd_mode = BRW_SAMPLER_SIMD_MODE_SIMD16;
}
brw_push_insn_state(p);
brw_set_default_mask_control(p, BRW_MASK_DISABLE);
- brw_set_default_access_mode(p, BRW_ALIGN_1);
/* a0.0 = surf_index & 0xff */
brw_inst *insn_and = brw_next_insn(p, BRW_OPCODE_AND);
struct brw_reg msg_data,
unsigned msg_type)
{
+ assert(inst->size_written % REG_SIZE == 0);
assert(msg_data.type == BRW_REGISTER_TYPE_UD);
brw_pixel_interpolator_query(p,
msg_type,
msg_data,
inst->mlen,
- inst->regs_written);
-}
-
-
-/**
- * Sets the first word of a vgrf for gen7+ simd4x2 uniform pull constant
- * sampler LD messages.
- *
- * We don't want to bake it into the send message's code generation because
- * that means we don't get a chance to schedule the instructions.
- */
-void
-fs_generator::generate_set_simd4x2_offset(fs_inst *inst,
- struct brw_reg dst,
- struct brw_reg value)
-{
- assert(value.file == BRW_IMMEDIATE_VALUE);
-
- brw_push_insn_state(p);
- brw_set_default_exec_size(p, BRW_EXECUTE_8);
- brw_set_default_compression_control(p, BRW_COMPRESSION_NONE);
- brw_set_default_mask_control(p, BRW_MASK_DISABLE);
- brw_MOV(p, retype(brw_vec1_reg(dst.file, dst.nr, 0), value.type), value);
- brw_pop_insn_state(p);
+ inst->size_written / REG_SIZE);
}
/* Sets vstride=1, width=4, hstride=0 of register src1 during
const bool compressed =
inst->dst.component_size(inst->exec_size) > REG_SIZE;
brw_set_default_compression(p, compressed);
- brw_set_default_group(p, inst->force_sechalf ? 8 : 0);
+ brw_set_default_group(p, inst->group);
for (unsigned int i = 0; i < inst->sources; i++) {
src[i] = brw_reg_from_fs_reg(inst, &inst->src[i], devinfo->gen,
brw_set_default_acc_write_control(p, inst->writes_accumulator);
brw_set_default_exec_size(p, cvt(inst->exec_size) - 1);
- assert(inst->force_writemask_all || inst->exec_size >= 8);
+ assert(inst->force_writemask_all || inst->exec_size >= 4);
+ assert(inst->force_writemask_all || inst->group % inst->exec_size == 0);
assert(inst->base_mrf + inst->mlen <= BRW_MAX_MRF(devinfo->gen));
assert(inst->mlen <= BRW_MAX_MSG_LENGTH);
/* FBL only supports UD type for dst. */
brw_FBL(p, retype(dst, BRW_REGISTER_TYPE_UD), src[0]);
break;
+ case BRW_OPCODE_LZD:
+ brw_LZD(p, dst, src[0]);
+ break;
case BRW_OPCODE_CBIT:
assert(devinfo->gen >= 7);
/* CBIT only supports UD type for dst. */
generate_fb_write(inst, src[0]);
break;
+ case FS_OPCODE_FB_READ:
+ generate_fb_read(inst, dst, src[0]);
+ break;
+
case FS_OPCODE_MOV_DISPATCH_TO_FLAGS:
generate_mov_dispatch_to_flags(inst);
break;
brw_memory_fence(p, dst);
break;
- case FS_OPCODE_SET_SIMD4X2_OFFSET:
- generate_set_simd4x2_offset(inst, dst, src[0]);
- break;
-
- case SHADER_OPCODE_FIND_LIVE_CHANNEL:
- brw_find_live_channel(p, dst);
+ case SHADER_OPCODE_FIND_LIVE_CHANNEL: {
+ const struct brw_reg mask =
+ brw_stage_has_packed_dispatch(devinfo, stage,
+ prog_data) ? brw_imm_ud(~0u) :
+ stage == MESA_SHADER_FRAGMENT ? brw_vmask_reg() :
+ brw_dmask_reg();
+ brw_find_live_channel(p, dst, mask);
break;
+ }
case SHADER_OPCODE_BROADCAST:
+ assert(inst->force_writemask_all);
brw_broadcast(p, dst, src[0], src[1]);
break;
}
break;
- case FS_OPCODE_INTERPOLATE_AT_CENTROID:
- generate_pixel_interpolator_query(inst, dst, src[0], src[1],
- GEN7_PIXEL_INTERPOLATOR_LOC_CENTROID);
- break;
-
case FS_OPCODE_INTERPOLATE_AT_SAMPLE:
generate_pixel_interpolator_query(inst, dst, src[0], src[1],
GEN7_PIXEL_INTERPOLATOR_LOC_SAMPLE);
generate_barrier(inst, src[0]);
break;
+ case BRW_OPCODE_DIM:
+ assert(devinfo->is_haswell);
+ assert(src[0].type == BRW_REGISTER_TYPE_DF);
+ assert(dst.type == BRW_REGISTER_TYPE_DF);
+ brw_DIM(p, dst, retype(src[0], BRW_REGISTER_TYPE_F));
+ break;
+
default:
unreachable("Unsupported opcode");
}
}
- brw_set_uip_jip(p);
+ brw_set_uip_jip(p, start_offset);
annotation_finalize(&annotation, p->next_insn_offset);
#ifndef NDEBUG
projects / mesa.git / blobdiff