#include "brw_program.h"
#include "brw_dead_control_flow.h"
#include "compiler/glsl_types.h"
+#include "program/prog_parameter.h"
using namespace brw;
fs_reg vec4_offset = vgrf(glsl_type::uint_type);
bld.ADD(vec4_offset, varying_offset, brw_imm_ud(const_offset & ~0xf));
- int scale = 1;
- if (devinfo->gen == 4 && bld.dispatch_width() == 8) {
- /* Pre-gen5, we can either use a SIMD8 message that requires (header,
- * u, v, r) as parameters, or we can just use the SIMD16 message
- * consisting of (header, u). We choose the second, at the cost of a
- * longer return length.
- */
- scale = 2;
- }
-
- enum opcode op;
- if (devinfo->gen >= 7)
- op = FS_OPCODE_VARYING_PULL_CONSTANT_LOAD_GEN7;
- else
- op = FS_OPCODE_VARYING_PULL_CONSTANT_LOAD;
-
- int regs_written = 4 * (bld.dispatch_width() / 8) * scale;
- fs_reg vec4_result = fs_reg(VGRF, alloc.allocate(regs_written), dst.type);
- fs_inst *inst = bld.emit(op, vec4_result, surf_index, vec4_offset);
- inst->regs_written = regs_written;
+ /* The pull load message will load a vec4 (16 bytes). If we are loading
+ * a double this means we are only loading 2 elements worth of data.
+ * We also want to use a 32-bit data type for the dst of the load operation
+ * so other parts of the driver don't get confused about the size of the
+ * result.
+ */
+ fs_reg vec4_result = bld.vgrf(BRW_REGISTER_TYPE_F, 4);
+ fs_inst *inst = bld.emit(FS_OPCODE_VARYING_PULL_CONSTANT_LOAD_LOGICAL,
+ vec4_result, surf_index, vec4_offset);
+ inst->regs_written = 4 * bld.dispatch_width() / 8;
- if (devinfo->gen < 7) {
- inst->base_mrf = FIRST_PULL_LOAD_MRF(devinfo->gen);
- inst->header_size = 1;
- if (devinfo->gen == 4)
- inst->mlen = 3;
- else
- inst->mlen = 1 + bld.dispatch_width() / 8;
+ if (type_sz(dst.type) == 8) {
+ shuffle_32bit_load_result_to_64bit_data(
+ bld, retype(vec4_result, dst.type), vec4_result, 2);
}
- bld.MOV(dst, offset(vec4_result, bld, ((const_offset & 0xf) / 4) * scale));
+ vec4_result.type = dst.type;
+ bld.MOV(dst, offset(vec4_result, bld,
+ (const_offset & 0xf) / type_sz(vec4_result.type)));
}
/**
return 4 * type_size_vec4(type);
}
+/* Attribute arrays are loaded as one vec4 per element (or matrix column),
+ * except for double-precision types, which are loaded as one dvec4.
+ */
+extern "C" int
+type_size_vs_input(const struct glsl_type *type)
+{
+ if (type->is_double()) {
+ return type_size_dvec4(type);
+ } else {
+ return type_size_vec4(type);
+ }
+}
+
/**
* Create a MOV to read the timestamp register.
*
{
return ((this->predicate && this->opcode != BRW_OPCODE_SEL) ||
(this->exec_size * type_sz(this->dst.type)) < 32 ||
- !this->dst.is_contiguous());
+ !this->dst.is_contiguous() ||
+ this->dst.subreg_offset > 0);
}
unsigned
case SHADER_OPCODE_LOD_LOGICAL:
case SHADER_OPCODE_TG4_LOGICAL:
case SHADER_OPCODE_TG4_OFFSET_LOGICAL:
+ case SHADER_OPCODE_SAMPLEINFO_LOGICAL:
assert(src[TEX_LOGICAL_SRC_COORD_COMPONENTS].file == IMM &&
src[TEX_LOGICAL_SRC_GRAD_COMPONENTS].file == IMM);
/* Texture coordinates. */
case FS_OPCODE_TXB:
case SHADER_OPCODE_TXD:
case SHADER_OPCODE_TXF:
+ case SHADER_OPCODE_TXF_LZ:
case SHADER_OPCODE_TXF_CMS:
case SHADER_OPCODE_TXF_CMS_W:
case SHADER_OPCODE_TXF_MCS:
case SHADER_OPCODE_TG4:
case SHADER_OPCODE_TG4_OFFSET:
case SHADER_OPCODE_TXL:
+ case SHADER_OPCODE_TXL_LZ:
case SHADER_OPCODE_TXS:
case SHADER_OPCODE_LOD:
case SHADER_OPCODE_SAMPLEINFO:
case FS_OPCODE_UNIFORM_PULL_CONSTANT_LOAD:
case SHADER_OPCODE_GEN4_SCRATCH_READ:
return 1;
- case FS_OPCODE_VARYING_PULL_CONSTANT_LOAD:
+ case FS_OPCODE_VARYING_PULL_CONSTANT_LOAD_GEN4:
return inst->mlen;
case SHADER_OPCODE_GEN4_SCRATCH_WRITE:
return inst->mlen;
}
fs_reg *
-fs_visitor::emit_fragcoord_interpolation(bool pixel_center_integer,
- bool origin_upper_left)
+fs_visitor::emit_fragcoord_interpolation()
{
assert(stage == MESA_SHADER_FRAGMENT);
- brw_wm_prog_key *key = (brw_wm_prog_key*) this->key;
fs_reg *reg = new(this->mem_ctx) fs_reg(vgrf(glsl_type::vec4_type));
fs_reg wpos = *reg;
- bool flip = !origin_upper_left ^ key->render_to_fbo;
/* gl_FragCoord.x */
- if (pixel_center_integer) {
- bld.MOV(wpos, this->pixel_x);
- } else {
- bld.ADD(wpos, this->pixel_x, brw_imm_f(0.5f));
- }
+ bld.MOV(wpos, this->pixel_x);
wpos = offset(wpos, bld, 1);
/* gl_FragCoord.y */
- if (!flip && pixel_center_integer) {
- bld.MOV(wpos, this->pixel_y);
- } else {
- fs_reg pixel_y = this->pixel_y;
- float offset = (pixel_center_integer ? 0.0f : 0.5f);
-
- if (flip) {
- pixel_y.negate = true;
- offset += key->drawable_height - 1.0f;
- }
-
- bld.ADD(wpos, pixel_y, brw_imm_f(offset));
- }
+ bld.MOV(wpos, this->pixel_y);
wpos = offset(wpos, bld, 1);
/* gl_FragCoord.z */
inst->no_dd_clear = true;
inst = emit_linterp(*attr, fs_reg(interp), interpolation_mode,
- mod_centroid && !key->persample_shading,
- mod_sample || key->persample_shading);
+ mod_centroid && !key->persample_interp,
+ mod_sample || key->persample_interp);
inst->predicate = BRW_PREDICATE_NORMAL;
inst->predicate_inverse = false;
if (devinfo->has_pln)
} else {
emit_linterp(*attr, fs_reg(interp), interpolation_mode,
- mod_centroid && !key->persample_shading,
- mod_sample || key->persample_shading);
+ mod_centroid && !key->persample_interp,
+ mod_sample || key->persample_interp);
}
if (devinfo->gen < 6 && interpolation_mode == INTERP_QUALIFIER_SMOOTH) {
bld.MUL(*attr, *attr, this->pixel_w);
fs_visitor::compute_sample_position(fs_reg dst, fs_reg int_sample_pos)
{
assert(stage == MESA_SHADER_FRAGMENT);
- brw_wm_prog_key *key = (brw_wm_prog_key*) this->key;
+ brw_wm_prog_data *wm_prog_data = (brw_wm_prog_data *) this->prog_data;
assert(dst.type == BRW_REGISTER_TYPE_F);
- if (key->compute_pos_offset) {
+ if (wm_prog_data->persample_dispatch) {
/* Convert int_sample_pos to floating point */
bld.MOV(dst, int_sample_pos);
/* Scale to the range [0, 1] */
fs_visitor::emit_samplemaskin_setup()
{
assert(stage == MESA_SHADER_FRAGMENT);
- brw_wm_prog_key *key = (brw_wm_prog_key *) this->key;
+ brw_wm_prog_data *wm_prog_data = (brw_wm_prog_data *) this->prog_data;
assert(devinfo->gen >= 6);
fs_reg *reg = new(this->mem_ctx) fs_reg(vgrf(glsl_type::int_type));
fs_reg coverage_mask(retype(brw_vec8_grf(payload.sample_mask_in_reg, 0),
BRW_REGISTER_TYPE_D));
- if (key->persample_shading) {
+ if (wm_prog_data->persample_dispatch) {
/* gl_SampleMaskIn[] comes from two sources: the input coverage mask,
* and a mask representing which sample is being processed by the
* current shader invocation.
void
fs_visitor::assign_curb_setup()
{
- if (dispatch_width == 8) {
- prog_data->dispatch_grf_start_reg = payload.num_regs;
- } else {
- if (stage == MESA_SHADER_FRAGMENT) {
- brw_wm_prog_data *prog_data = (brw_wm_prog_data*) this->prog_data;
- prog_data->dispatch_grf_start_reg_16 = payload.num_regs;
- } else if (stage == MESA_SHADER_COMPUTE) {
- brw_cs_prog_data *prog_data = (brw_cs_prog_data*) this->prog_data;
- prog_data->dispatch_grf_start_reg_16 = payload.num_regs;
- } else {
- unreachable("Unsupported shader type!");
- }
- }
-
prog_data->curb_read_length = ALIGN(stage_prog_data->nr_params, 8) / 8;
/* Map the offsets in the UNIFORM file to fixed HW regs. */
inst->src[i].nr +
inst->src[i].reg_offset;
- unsigned width = inst->src[i].stride == 0 ? 1 : inst->exec_size;
+ /* As explained at brw_reg_from_fs_reg, From the Haswell PRM:
+ *
+ * VertStride must be used to cross GRF register boundaries. This
+ * rule implies that elements within a 'Width' cannot cross GRF
+ * boundaries.
+ *
+ * So, for registers that are large enough, we have to split the exec
+ * size in two and trust the compression state to sort it out.
+ */
+ unsigned total_size = inst->exec_size *
+ inst->src[i].stride *
+ type_sz(inst->src[i].type);
+
+ assert(total_size <= 2 * REG_SIZE);
+ const unsigned exec_size =
+ (total_size <= REG_SIZE) ? inst->exec_size : inst->exec_size / 2;
+
+ unsigned width = inst->src[i].stride == 0 ? 1 : exec_size;
struct brw_reg reg =
stride(byte_offset(retype(brw_vec8_grf(grf, 0), inst->src[i].type),
inst->src[i].subreg_offset),
- inst->exec_size * inst->src[i].stride,
+ exec_size * inst->src[i].stride,
width, inst->src[i].stride);
reg.abs = inst->src[i].abs;
reg.negate = inst->src[i].negate;
assert(stage == MESA_SHADER_VERTEX);
/* Each attribute is 4 regs. */
- this->first_non_payload_grf += 4 * vs_prog_data->nr_attributes;
+ this->first_non_payload_grf += 4 * vs_prog_data->nr_attribute_slots;
assert(vs_prog_data->base.urb_read_length <= 15);
return progress;
}
+static void
+set_push_pull_constant_loc(unsigned uniform, int *chunk_start, bool contiguous,
+ int *push_constant_loc, int *pull_constant_loc,
+ unsigned *num_push_constants,
+ unsigned *num_pull_constants,
+ const unsigned max_push_components,
+ const unsigned max_chunk_size,
+ struct brw_stage_prog_data *stage_prog_data)
+{
+ /* This is the first live uniform in the chunk */
+ if (*chunk_start < 0)
+ *chunk_start = uniform;
+
+ /* If this element does not need to be contiguous with the next, we
+ * split at this point and everything between chunk_start and u forms a
+ * single chunk.
+ */
+ if (!contiguous) {
+ unsigned chunk_size = uniform - *chunk_start + 1;
+
+ /* Decide whether we should push or pull this parameter. In the
+ * Vulkan driver, push constants are explicitly exposed via the API
+ * so we push everything. In GL, we only push small arrays.
+ */
+ if (stage_prog_data->pull_param == NULL ||
+ (*num_push_constants + chunk_size <= max_push_components &&
+ chunk_size <= max_chunk_size)) {
+ assert(*num_push_constants + chunk_size <= max_push_components);
+ for (unsigned j = *chunk_start; j <= uniform; j++)
+ push_constant_loc[j] = (*num_push_constants)++;
+ } else {
+ for (unsigned j = *chunk_start; j <= uniform; j++)
+ pull_constant_loc[j] = (*num_pull_constants)++;
+ }
+
+ *chunk_start = -1;
+ }
+}
+
/**
* Assign UNIFORM file registers to either push constants or pull constants.
*
bool is_live[uniforms];
memset(is_live, 0, sizeof(is_live));
+ bool is_live_64bit[uniforms];
+ memset(is_live_64bit, 0, sizeof(is_live_64bit));
/* For each uniform slot, a value of true indicates that the given slot and
* the next slot must remain contiguous. This is used to keep us from
for (unsigned j = constant_nr; j < last; j++) {
is_live[j] = true;
contiguous[j] = true;
+ if (type_sz(inst->src[i].type) == 8) {
+ is_live_64bit[j] = true;
+ }
}
is_live[last] = true;
} else {
if (constant_nr >= 0 && constant_nr < (int) uniforms) {
int regs_read = inst->components_read(i) *
type_sz(inst->src[i].type) / 4;
- for (int j = 0; j < regs_read; j++)
+ for (int j = 0; j < regs_read; j++) {
is_live[constant_nr + j] = true;
+ if (type_sz(inst->src[i].type) == 8) {
+ is_live_64bit[constant_nr + j] = true;
+ }
+ }
}
}
}
push_constant_loc = ralloc_array(mem_ctx, int, uniforms);
pull_constant_loc = ralloc_array(mem_ctx, int, uniforms);
+ /* Default to -1 meaning no location */
+ memset(push_constant_loc, -1, uniforms * sizeof(*push_constant_loc));
+ memset(pull_constant_loc, -1, uniforms * sizeof(*pull_constant_loc));
+
int chunk_start = -1;
- for (unsigned u = 0; u < uniforms; u++) {
- push_constant_loc[u] = -1;
- pull_constant_loc[u] = -1;
- if (!is_live[u])
+ /* First push 64-bit uniforms to ensure they are properly aligned */
+ for (unsigned u = 0; u < uniforms; u++) {
+ if (!is_live[u] || !is_live_64bit[u])
continue;
- /* This is the first live uniform in the chunk */
- if (chunk_start < 0)
- chunk_start = u;
+ set_push_pull_constant_loc(u, &chunk_start, contiguous[u],
+ push_constant_loc, pull_constant_loc,
+ &num_push_constants, &num_pull_constants,
+ max_push_components, max_chunk_size,
+ stage_prog_data);
- /* If this element does not need to be contiguous with the next, we
- * split at this point and everthing between chunk_start and u forms a
- * single chunk.
- */
- if (!contiguous[u]) {
- unsigned chunk_size = u - chunk_start + 1;
+ }
- /* Decide whether we should push or pull this parameter. In the
- * Vulkan driver, push constants are explicitly exposed via the API
- * so we push everything. In GL, we only push small arrays.
- */
- if (stage_prog_data->pull_param == NULL ||
- (num_push_constants + chunk_size <= max_push_components &&
- chunk_size <= max_chunk_size)) {
- assert(num_push_constants + chunk_size <= max_push_components);
- for (unsigned j = chunk_start; j <= u; j++)
- push_constant_loc[j] = num_push_constants++;
- } else {
- for (unsigned j = chunk_start; j <= u; j++)
- pull_constant_loc[j] = num_pull_constants++;
- }
+ /* Then push the rest of uniforms */
+ for (unsigned u = 0; u < uniforms; u++) {
+ if (!is_live[u] || is_live_64bit[u])
+ continue;
- chunk_start = -1;
- }
+ set_push_pull_constant_loc(u, &chunk_start, contiguous[u],
+ push_constant_loc, pull_constant_loc,
+ &num_push_constants, &num_pull_constants,
+ max_push_components, max_chunk_size,
+ stage_prog_data);
}
+ /* As the uniforms are going to be reordered, take the data from a temporary
+ * copy of the original param[].
+ */
+ gl_constant_value **param = ralloc_array(NULL, gl_constant_value*,
+ stage_prog_data->nr_params);
+ memcpy(param, stage_prog_data->param,
+ sizeof(gl_constant_value*) * stage_prog_data->nr_params);
stage_prog_data->nr_params = num_push_constants;
stage_prog_data->nr_pull_params = num_pull_constants;
* having to make a copy.
*/
for (unsigned int i = 0; i < uniforms; i++) {
- const gl_constant_value *value = stage_prog_data->param[i];
+ const gl_constant_value *value = param[i];
if (pull_constant_loc[i] != -1) {
stage_prog_data->pull_param[pull_constant_loc[i]] = value;
stage_prog_data->param[push_constant_loc[i]] = value;
}
}
+ ralloc_free(param);
}
/**
if (pull_index == -1)
continue;
+ const unsigned index = stage_prog_data->binding_table.pull_constants_start;
+ fs_reg dst;
+
+ if (type_sz(inst->src[i].type) <= 4)
+ dst = vgrf(glsl_type::float_type);
+ else
+ dst = vgrf(glsl_type::double_type);
+
assert(inst->src[i].stride == 0);
- fs_reg dst = vgrf(glsl_type::float_type);
const fs_builder ubld = ibld.exec_all().group(8, 0);
struct brw_reg offset = brw_imm_ud((unsigned)(pull_index * 4) & ~15);
ubld.emit(FS_OPCODE_UNIFORM_PULL_CONSTANT_LOAD,
inst->src[i].file = VGRF;
inst->src[i].nr = dst.nr;
inst->src[i].reg_offset = 0;
- inst->src[i].set_smear(pull_index & 3);
+ inst->src[i].set_smear((pull_index & 3) * 4 /
+ type_sz(inst->src[i].type));
brw_mark_surface_used(prog_data, index);
}
fs_visitor::emit_repclear_shader()
{
brw_wm_prog_key *key = (brw_wm_prog_key*) this->key;
- int base_mrf = 1;
+ int base_mrf = 0;
int color_mrf = base_mrf + 2;
fs_inst *mov;
if (op == SHADER_OPCODE_TG4 || op == SHADER_OPCODE_TG4_OFFSET ||
offset_value.file != BAD_FILE ||
+ op == SHADER_OPCODE_SAMPLEINFO ||
is_high_sampler(devinfo, sampler)) {
/* For general texture offsets (no txf workaround), we need a header to
* put them in. Note that we're only reserving space for it in the
switch (op) {
case FS_OPCODE_TXB:
case SHADER_OPCODE_TXL:
+ if (devinfo->gen >= 9 && op == SHADER_OPCODE_TXL && lod.is_zero()) {
+ op = SHADER_OPCODE_TXL_LZ;
+ break;
+ }
bld.MOV(sources[length], lod);
length++;
break;
length++;
}
- bld.MOV(retype(sources[length], BRW_REGISTER_TYPE_D), lod);
- length++;
+ if (devinfo->gen >= 9 && lod.is_zero()) {
+ op = SHADER_OPCODE_TXF_LZ;
+ } else {
+ bld.MOV(retype(sources[length], BRW_REGISTER_TYPE_D), lod);
+ length++;
+ }
for (unsigned i = devinfo->gen >= 9 ? 2 : 1; i < coord_components; i++) {
bld.MOV(retype(sources[length], BRW_REGISTER_TYPE_D), coordinate);
coordinate_done = true;
break;
+
case SHADER_OPCODE_TXF_CMS:
case SHADER_OPCODE_TXF_CMS_W:
case SHADER_OPCODE_TXF_UMS:
delete[] components;
}
+static void
+lower_varying_pull_constant_logical_send(const fs_builder &bld, fs_inst *inst)
+{
+ const brw_device_info *devinfo = bld.shader->devinfo;
+
+ if (devinfo->gen >= 7) {
+ inst->opcode = FS_OPCODE_VARYING_PULL_CONSTANT_LOAD_GEN7;
+
+ } else {
+ const fs_reg payload(MRF, FIRST_PULL_LOAD_MRF(devinfo->gen),
+ BRW_REGISTER_TYPE_UD);
+
+ bld.MOV(byte_offset(payload, REG_SIZE), inst->src[1]);
+
+ inst->opcode = FS_OPCODE_VARYING_PULL_CONSTANT_LOAD_GEN4;
+ inst->resize_sources(1);
+ inst->base_mrf = payload.nr;
+ inst->header_size = 1;
+ inst->mlen = 1 + inst->exec_size / 8;
+ }
+}
+
+static void
+lower_math_logical_send(const fs_builder &bld, fs_inst *inst)
+{
+ assert(bld.shader->devinfo->gen < 6);
+
+ inst->base_mrf = 2;
+ inst->mlen = inst->sources * inst->exec_size / 8;
+
+ if (inst->sources > 1) {
+ /* From the Ironlake PRM, Volume 4, Part 1, Section 6.1.13
+ * "Message Payload":
+ *
+ * "Operand0[7]. For the INT DIV functions, this operand is the
+ * denominator."
+ * ...
+ * "Operand1[7]. For the INT DIV functions, this operand is the
+ * numerator."
+ */
+ const bool is_int_div = inst->opcode != SHADER_OPCODE_POW;
+ const fs_reg src0 = is_int_div ? inst->src[1] : inst->src[0];
+ const fs_reg src1 = is_int_div ? inst->src[0] : inst->src[1];
+
+ inst->resize_sources(1);
+ inst->src[0] = src0;
+
+ assert(inst->exec_size == 8);
+ bld.MOV(fs_reg(MRF, inst->base_mrf + 1, src1.type), src1);
+ }
+}
+
bool
fs_visitor::lower_logical_sends()
{
lower_sampler_logical_send(ibld, inst, SHADER_OPCODE_TG4_OFFSET);
break;
+ case SHADER_OPCODE_SAMPLEINFO_LOGICAL:
+ lower_sampler_logical_send(ibld, inst, SHADER_OPCODE_SAMPLEINFO);
+ break;
+
case SHADER_OPCODE_UNTYPED_SURFACE_READ_LOGICAL:
lower_surface_logical_send(ibld, inst,
SHADER_OPCODE_UNTYPED_SURFACE_READ,
ibld.sample_mask_reg());
break;
+ case FS_OPCODE_VARYING_PULL_CONSTANT_LOAD_LOGICAL:
+ lower_varying_pull_constant_logical_send(ibld, inst);
+ break;
+
+ case SHADER_OPCODE_RCP:
+ case SHADER_OPCODE_RSQ:
+ case SHADER_OPCODE_SQRT:
+ case SHADER_OPCODE_EXP2:
+ case SHADER_OPCODE_LOG2:
+ case SHADER_OPCODE_SIN:
+ case SHADER_OPCODE_COS:
+ case SHADER_OPCODE_POW:
+ case SHADER_OPCODE_INT_QUOTIENT:
+ case SHADER_OPCODE_INT_REMAINDER:
+ /* The math opcodes are overloaded for the send-like and
+ * expression-like instructions which seems kind of icky. Gen6+ has
+ * a native (but rather quirky) MATH instruction so we don't need to
+ * do anything here. On Gen4-5 we'll have to lower the Gen6-like
+ * logical instructions (which we can easily recognize because they
+ * have mlen = 0) into send-like virtual instructions.
+ */
+ if (devinfo->gen < 6 && inst->mlen == 0) {
+ lower_math_logical_send(ibld, inst);
+ break;
+
+ } else {
+ continue;
+ }
+
default:
continue;
}
return progress;
}
+/**
+ * Get the closest allowed SIMD width for instruction \p inst accounting for
+ * some common regioning and execution control restrictions that apply to FPU
+ * instructions. These restrictions don't necessarily have any relevance to
+ * instructions not executed by the FPU pipeline like extended math, control
+ * flow or send message instructions.
+ *
+ * For virtual opcodes it's really up to the instruction -- In some cases
+ * (e.g. where a virtual instruction unrolls into a simple sequence of FPU
+ * instructions) it may simplify virtual instruction lowering if we can
+ * enforce FPU-like regioning restrictions already on the virtual instruction,
+ * in other cases (e.g. virtual send-like instructions) this may be
+ * excessively restrictive.
+ */
+static unsigned
+get_fpu_lowered_simd_width(const struct brw_device_info *devinfo,
+ const fs_inst *inst)
+{
+ /* Maximum execution size representable in the instruction controls. */
+ unsigned max_width = MIN2(32, inst->exec_size);
+
+ /* According to the PRMs:
+ * "A. In Direct Addressing mode, a source cannot span more than 2
+ * adjacent GRF registers.
+ * B. A destination cannot span more than 2 adjacent GRF registers."
+ *
+ * Look for the source or destination with the largest register region
+ * which is the one that is going to limit the overall execution size of
+ * the instruction due to this rule.
+ */
+ unsigned reg_count = inst->regs_written;
+
+ for (unsigned i = 0; i < inst->sources; i++)
+ reg_count = MAX2(reg_count, (unsigned)inst->regs_read(i));
+
+ /* Calculate the maximum execution size of the instruction based on the
+ * factor by which it goes over the hardware limit of 2 GRFs.
+ */
+ if (reg_count > 2)
+ max_width = MIN2(max_width, inst->exec_size / DIV_ROUND_UP(reg_count, 2));
+
+ /* According to the IVB PRMs:
+ * "When destination spans two registers, the source MUST span two
+ * registers. The exception to the above rule:
+ *
+ * - When source is scalar, the source registers are not incremented.
+ * - When source is packed integer Word and destination is packed
+ * integer DWord, the source register is not incremented but the
+ * source sub register is incremented."
+ *
+ * The hardware specs from Gen4 to Gen7.5 mention similar regioning
+ * restrictions. The code below intentionally doesn't check whether the
+ * destination type is integer because empirically the hardware doesn't
+ * seem to care what the actual type is as long as it's dword-aligned.
+ */
+ if (devinfo->gen < 8) {
+ for (unsigned i = 0; i < inst->sources; i++) {
+ if (inst->regs_written == 2 &&
+ inst->regs_read(i) != 0 && inst->regs_read(i) != 2 &&
+ !is_uniform(inst->src[i]) &&
+ !(type_sz(inst->dst.type) == 4 && inst->dst.stride == 1 &&
+ type_sz(inst->src[i].type) == 2 && inst->src[i].stride == 1))
+ max_width = MIN2(max_width, inst->exec_size /
+ inst->regs_written);
+ }
+ }
+
+ /* From the IVB PRMs:
+ * "When an instruction is SIMD32, the low 16 bits of the execution mask
+ * are applied for both halves of the SIMD32 instruction. If different
+ * execution mask channels are required, split the instruction into two
+ * SIMD16 instructions."
+ *
+ * There is similar text in the HSW PRMs. Gen4-6 don't even implement
+ * 32-wide control flow support in hardware and will behave similarly.
+ */
+ if (devinfo->gen < 8 && !inst->force_writemask_all)
+ max_width = MIN2(max_width, 16);
+
+ /* From the IVB PRMs (applies to HSW too):
+ * "Instructions with condition modifiers must not use SIMD32."
+ *
+ * From the BDW PRMs (applies to later hardware too):
+ * "Ternary instruction with condition modifiers must not use SIMD32."
+ */
+ if (inst->conditional_mod && (devinfo->gen < 8 || inst->is_3src(devinfo)))
+ max_width = MIN2(max_width, 16);
+
+ /* From the IVB PRMs (applies to other devices that don't have the
+ * brw_device_info::supports_simd16_3src flag set):
+ * "In Align16 access mode, SIMD16 is not allowed for DW operations and
+ * SIMD8 is not allowed for DF operations."
+ */
+ if (inst->is_3src(devinfo) && !devinfo->supports_simd16_3src)
+ max_width = MIN2(max_width, inst->exec_size / reg_count);
+
+ /* Only power-of-two execution sizes are representable in the instruction
+ * control fields.
+ */
+ return 1 << _mesa_logbase2(max_width);
+}
+
/**
* Get the closest native SIMD width supported by the hardware for instruction
* \p inst. The instruction will be left untouched by
case BRW_OPCODE_SHR:
case BRW_OPCODE_SHL:
case BRW_OPCODE_ASR:
- case BRW_OPCODE_CMP:
case BRW_OPCODE_CMPN:
case BRW_OPCODE_CSEL:
case BRW_OPCODE_F32TO16:
case BRW_OPCODE_SAD2:
case BRW_OPCODE_MAD:
case BRW_OPCODE_LRP:
+ case FS_OPCODE_PACK:
+ return get_fpu_lowered_simd_width(devinfo, inst);
+
+ case BRW_OPCODE_CMP: {
+ /* The Ivybridge/BayTrail WaCMPInstFlagDepClearedEarly workaround says that
+ * when the destination is a GRF the dependency-clear bit on the flag
+ * register is cleared early.
+ *
+ * Suggested workarounds are to disable coissuing CMP instructions
+ * or to split CMP(16) instructions into two CMP(8) instructions.
+ *
+ * We choose to split into CMP(8) instructions since disabling
+ * coissuing would affect CMP instructions not otherwise affected by
+ * the errata.
+ */
+ const unsigned max_width = (devinfo->gen == 7 && !devinfo->is_haswell &&
+ !inst->dst.is_null() ? 8 : ~0);
+ return MIN2(max_width, get_fpu_lowered_simd_width(devinfo, inst));
+ }
+
case SHADER_OPCODE_RCP:
case SHADER_OPCODE_RSQ:
case SHADER_OPCODE_SQRT:
case SHADER_OPCODE_EXP2:
case SHADER_OPCODE_LOG2:
- case SHADER_OPCODE_POW:
- case SHADER_OPCODE_INT_QUOTIENT:
- case SHADER_OPCODE_INT_REMAINDER:
case SHADER_OPCODE_SIN:
case SHADER_OPCODE_COS:
- case FS_OPCODE_PACK: {
- /* According to the PRMs:
- * "A. In Direct Addressing mode, a source cannot span more than 2
- * adjacent GRF registers.
- * B. A destination cannot span more than 2 adjacent GRF registers."
- *
- * Look for the source or destination with the largest register region
- * which is the one that is going to limit the overal execution size of
- * the instruction due to this rule.
+ /* Unary extended math instructions are limited to SIMD8 on Gen4 and
+ * Gen6.
*/
- unsigned reg_count = inst->regs_written;
+ return (devinfo->gen >= 7 ? MIN2(16, inst->exec_size) :
+ devinfo->gen == 5 || devinfo->is_g4x ? MIN2(16, inst->exec_size) :
+ MIN2(8, inst->exec_size));
- for (unsigned i = 0; i < inst->sources; i++)
- reg_count = MAX2(reg_count, (unsigned)inst->regs_read(i));
+ case SHADER_OPCODE_POW:
+ /* SIMD16 is only allowed on Gen7+. */
+ return (devinfo->gen >= 7 ? MIN2(16, inst->exec_size) :
+ MIN2(8, inst->exec_size));
- /* Calculate the maximum execution size of the instruction based on the
- * factor by which it goes over the hardware limit of 2 GRFs.
+ case SHADER_OPCODE_INT_QUOTIENT:
+ case SHADER_OPCODE_INT_REMAINDER:
+ /* Integer division is limited to SIMD8 on all generations. */
+ return MIN2(8, inst->exec_size);
+
+ case FS_OPCODE_VARYING_PULL_CONSTANT_LOAD_LOGICAL:
+ /* Pre-ILK hardware doesn't have a SIMD8 variant of the texel fetch
+ * message used to implement varying pull constant loads, so expand it
+ * to SIMD16. An alternative with longer message payload length but
+ * shorter return payload would be to use the SIMD8 sampler message that
+ * takes (header, u, v, r) as parameters instead of (header, u).
*/
- return inst->exec_size / DIV_ROUND_UP(reg_count, 2);
- }
+ return (devinfo->gen == 4 ? 16 : MIN2(16, inst->exec_size));
+
case SHADER_OPCODE_MULH:
/* MULH is lowered to the MUL/MACH sequence using the accumulator, which
* is 8-wide on Gen7+.
return (inst->src[FB_WRITE_LOGICAL_SRC_COLOR1].file != BAD_FILE ?
8 : inst->exec_size);
+ case SHADER_OPCODE_SAMPLEINFO_LOGICAL:
+ return MIN2(16, inst->exec_size);
+
case SHADER_OPCODE_TXD_LOGICAL:
/* TXD is unsupported in SIMD16 mode. */
return 8;
case SHADER_OPCODE_MOV_INDIRECT:
/* Prior to Broadwell, we only have 8 address subregisters */
- return devinfo->gen < 8 ? 8 : MIN2(inst->exec_size, 16);
+ if (devinfo->gen < 8)
+ return 8;
+
+ if (inst->exec_size < 16) {
+ return inst->exec_size;
+ } else {
+ assert(type_sz(inst->dst.type) >= 4);
+ return MIN2(inst->exec_size / (type_sz(inst->dst.type) / 4), 16);
+ }
default:
return inst->exec_size;
}
}
-/**
- * The \p rows array of registers represents a \p num_rows by \p num_columns
- * matrix in row-major order, write it in column-major order into the register
- * passed as destination. \p stride gives the separation between matrix
- * elements in the input in fs_builder::dispatch_width() units.
- */
-static void
-emit_transpose(const fs_builder &bld,
- const fs_reg &dst, const fs_reg *rows,
- unsigned num_rows, unsigned num_columns, unsigned stride)
-{
- fs_reg *const components = new fs_reg[num_rows * num_columns];
-
- for (unsigned i = 0; i < num_columns; ++i) {
- for (unsigned j = 0; j < num_rows; ++j)
- components[num_rows * i + j] = offset(rows[j], bld, stride * i);
- }
-
- bld.LOAD_PAYLOAD(dst, components, num_rows * num_columns, 0);
-
- delete[] components;
-}
-
bool
fs_visitor::lower_simd_width()
{
* instruction.
*/
const fs_builder lbld = ibld.group(lower_width, i);
+ const fs_builder cbld = lbld.group(copy_width, 0);
for (unsigned j = 0; j < inst->sources; j++) {
if (inst->src[j].file != BAD_FILE &&
- !is_uniform(inst->src[j])) {
+ !is_periodic(inst->src[j], lower_width)) {
/* Get the i-th copy_width-wide chunk of the source. */
- const fs_reg src = horiz_offset(inst->src[j], copy_width * i);
+ const fs_reg src = offset(inst->src[j], cbld, i);
const unsigned src_size = inst->components_read(j);
- /* Use a trivial transposition to copy one every n
- * copy_width-wide components of the register into a
- * temporary passed as source to the lowered instruction.
+ /* Copy one every n copy_width-wide components of the
+ * register into a temporary passed as source to the lowered
+ * instruction.
*/
split_inst.src[j] = lbld.vgrf(inst->src[j].type, src_size);
- emit_transpose(lbld.group(copy_width, 0),
- split_inst.src[j], &src, 1, src_size, n);
+
+ for (unsigned k = 0; k < src_size; ++k)
+ cbld.MOV(offset(split_inst.src[j], lbld, k),
+ offset(src, cbld, n * k));
}
}
split_inst.dst = dsts[i] =
lbld.vgrf(inst->dst.type, dst_size);
split_inst.regs_written =
- DIV_ROUND_UP(inst->regs_written * lower_width,
- inst->exec_size);
+ DIV_ROUND_UP(type_sz(inst->dst.type) * dst_size * lower_width,
+ REG_SIZE);
+
+ if (inst->predicate) {
+ /* Handle predication by copying the original contents of
+ * the destination into the temporary before emitting the
+ * lowered instruction.
+ */
+ for (unsigned k = 0; k < dst_size; ++k)
+ cbld.MOV(offset(split_inst.dst, lbld, k),
+ offset(inst->dst, cbld, n * k + i));
+ }
}
lbld.emit(split_inst);
}
if (inst->regs_written) {
- /* Distance between useful channels in the temporaries, skipping
- * garbage if the lowered instruction is wider than the original.
- */
- const unsigned m = lower_width / copy_width;
+ const fs_builder lbld = ibld.group(lower_width, 0);
/* Interleave the components of the result from the lowered
- * instructions. We need to set exec_all() when copying more than
- * one half per component, because LOAD_PAYLOAD (in terms of which
- * emit_transpose is implemented) can only use the same channel
- * enable signals for all of its non-header sources.
+ * instructions.
*/
- emit_transpose(ibld.exec_all(inst->exec_size > copy_width)
- .group(copy_width, 0),
- inst->dst, dsts, n, dst_size, m);
+ for (unsigned i = 0; i < dst_size; ++i) {
+ for (unsigned j = 0; j < n; ++j) {
+ const fs_builder cbld = ibld.group(copy_width, j);
+ cbld.MOV(offset(inst->dst, cbld, n * i + j),
+ offset(dsts[j], lbld, i));
+ }
+ }
}
inst->remove(block);
{
assert(stage == MESA_SHADER_FRAGMENT);
brw_wm_prog_data *prog_data = (brw_wm_prog_data*) this->prog_data;
- brw_wm_prog_key *key = (brw_wm_prog_key*) this->key;
unsigned barycentric_interp_modes =
(stage == MESA_SHADER_FRAGMENT) ?
}
}
- prog_data->uses_pos_offset = key->compute_pos_offset;
/* R31: MSAA position offsets. */
- if (prog_data->uses_pos_offset) {
+ if (prog_data->persample_dispatch &&
+ (nir->info.system_values_read & SYSTEM_BIT_SAMPLE_POS)) {
+ /* From the Ivy Bridge PRM documentation for 3DSTATE_PS:
+ *
+ * "MSDISPMODE_PERSAMPLE is required in order to select
+ * POSOFFSET_SAMPLE"
+ *
+ * So we can only really get sample positions if we are doing real
+ * per-sample dispatch. If we need gl_SamplePosition and we don't have
+ * persample dispatch, we hard-code it to 0.5.
+ */
+ prog_data->uses_pos_offset = true;
payload.sample_pos_reg = payload.num_regs;
payload.num_regs++;
}
payload.num_regs++;
}
- /* Use a maximum of 32 registers for push-model inputs. */
- const unsigned max_push_components = 32;
+ /* Use a maximum of 24 registers for push-model inputs. */
+ const unsigned max_push_components = 24;
/* If pushing our inputs would take too many registers, reduce the URB read
* length (which is in HWords, or 8 registers), and resort to pulling.
OPT(dead_code_eliminate);
}
- if (OPT(lower_d2f)) {
+ if (OPT(lower_d2x)) {
OPT(opt_copy_propagate);
OPT(dead_code_eliminate);
}
}
void
-fs_visitor::allocate_registers()
+fs_visitor::allocate_registers(bool allow_spilling)
{
bool allocated_without_spills;
SCHEDULE_PRE_LIFO,
};
+ bool spill_all = allow_spilling && (INTEL_DEBUG & DEBUG_SPILL_FS);
+
/* Try each scheduling heuristic to see if it can successfully register
* allocate without spilling. They should be ordered by decreasing
* performance but increasing likelihood of allocating.
assign_regs_trivial();
allocated_without_spills = true;
} else {
- allocated_without_spills = assign_regs(false);
+ allocated_without_spills = assign_regs(false, spill_all);
}
if (allocated_without_spills)
break;
/* Since we're out of heuristics, just go spill registers until we
* get an allocation.
*/
- while (!assign_regs(true)) {
+ while (!assign_regs(true, spill_all)) {
if (failed)
break;
}
}
+ assert(last_scratch == 0 || allow_spilling);
+
/* This must come after all optimization and register allocation, since
* it inserts dead code that happens to have side effects, and it does
* so based on the actual physical registers in use.
assign_vs_urb_setup();
fixup_3src_null_dest();
- allocate_registers();
+ allocate_registers(true);
return !failed;
}
assign_tcs_single_patch_urb_setup();
fixup_3src_null_dest();
- allocate_registers();
+ allocate_registers(true);
return !failed;
}
assign_tes_urb_setup();
fixup_3src_null_dest();
- allocate_registers();
+ allocate_registers(true);
return !failed;
}
assign_gs_urb_setup();
fixup_3src_null_dest();
- allocate_registers();
+ allocate_registers(true);
return !failed;
}
bool
-fs_visitor::run_fs(bool do_rep_send)
+fs_visitor::run_fs(bool allow_spilling, bool do_rep_send)
{
brw_wm_prog_data *wm_prog_data = (brw_wm_prog_data *) this->prog_data;
brw_wm_prog_key *wm_key = (brw_wm_prog_key *) this->key;
assign_urb_setup();
fixup_3src_null_dest();
- allocate_registers();
+ allocate_registers(allow_spilling);
if (failed)
return false;
}
- if (dispatch_width == 8)
- wm_prog_data->reg_blocks = brw_register_blocks(grf_used);
- else
- wm_prog_data->reg_blocks_16 = brw_register_blocks(grf_used);
-
return !failed;
}
assign_curb_setup();
fixup_3src_null_dest();
- allocate_registers();
+ allocate_registers(true);
if (failed)
return false;
const nir_shader *src_shader,
struct gl_program *prog,
int shader_time_index8, int shader_time_index16,
+ bool allow_spilling,
bool use_rep_send,
unsigned *final_assembly_size,
char **error_str)
prog_data->computed_stencil =
shader->info.outputs_written & BITFIELD64_BIT(FRAG_RESULT_STENCIL);
+ prog_data->persample_dispatch =
+ key->multisample_fbo &&
+ (key->persample_interp ||
+ (shader->info.system_values_read & (SYSTEM_BIT_SAMPLE_ID |
+ SYSTEM_BIT_SAMPLE_POS)) ||
+ shader->info.fs.uses_sample_qualifier);
+
prog_data->early_fragment_tests = shader->info.fs.early_fragment_tests;
prog_data->barycentric_interp_modes =
brw_compute_barycentric_interp_modes(compiler->devinfo,
key->flat_shade,
- key->persample_shading,
+ key->persample_interp,
shader);
- fs_visitor v(compiler, log_data, mem_ctx, key,
- &prog_data->base, prog, shader, 8,
- shader_time_index8);
- if (!v.run_fs(false /* do_rep_send */)) {
+ cfg_t *simd8_cfg = NULL, *simd16_cfg = NULL;
+ uint8_t simd8_grf_start = 0, simd16_grf_start = 0;
+ unsigned simd8_grf_used = 0, simd16_grf_used = 0;
+
+ fs_visitor v8(compiler, log_data, mem_ctx, key,
+ &prog_data->base, prog, shader, 8,
+ shader_time_index8);
+ if (!v8.run_fs(allow_spilling, false /* do_rep_send */)) {
if (error_str)
- *error_str = ralloc_strdup(mem_ctx, v.fail_msg);
+ *error_str = ralloc_strdup(mem_ctx, v8.fail_msg);
return NULL;
+ } else if (likely(!(INTEL_DEBUG & DEBUG_NO8))) {
+ simd8_cfg = v8.cfg;
+ simd8_grf_start = v8.payload.num_regs;
+ simd8_grf_used = v8.grf_used;
}
- cfg_t *simd16_cfg = NULL;
- fs_visitor v2(compiler, log_data, mem_ctx, key,
- &prog_data->base, prog, shader, 16,
- shader_time_index16);
- if (likely(!(INTEL_DEBUG & DEBUG_NO16) || use_rep_send)) {
- if (!v.simd16_unsupported) {
- /* Try a SIMD16 compile */
- v2.import_uniforms(&v);
- if (!v2.run_fs(use_rep_send)) {
- compiler->shader_perf_log(log_data,
- "SIMD16 shader failed to compile: %s",
- v2.fail_msg);
- } else {
- simd16_cfg = v2.cfg;
- }
+ if (!v8.simd16_unsupported &&
+ likely(!(INTEL_DEBUG & DEBUG_NO16) || use_rep_send)) {
+ /* Try a SIMD16 compile */
+ fs_visitor v16(compiler, log_data, mem_ctx, key,
+ &prog_data->base, prog, shader, 16,
+ shader_time_index16);
+ v16.import_uniforms(&v8);
+ if (!v16.run_fs(allow_spilling, use_rep_send)) {
+ compiler->shader_perf_log(log_data,
+ "SIMD16 shader failed to compile: %s",
+ v16.fail_msg);
+ } else {
+ simd16_cfg = v16.cfg;
+ simd16_grf_start = v16.payload.num_regs;
+ simd16_grf_used = v16.grf_used;
+ }
+ }
+
+ /* When the caller requests a repclear shader, they want SIMD16-only */
+ if (use_rep_send)
+ simd8_cfg = NULL;
+
+ /* Prior to Iron Lake, the PS had a single shader offset with a jump table
+ * at the top to select the shader. We've never implemented that.
+ * Instead, we just give them exactly one shader and we pick the widest one
+ * available.
+ */
+ if (compiler->devinfo->gen < 5 && simd16_cfg)
+ simd8_cfg = NULL;
+
+ if (prog_data->persample_dispatch) {
+ /* Starting with SandyBridge (where we first get MSAA), the different
+ * pixel dispatch combinations are grouped into classifications A
+ * through F (SNB PRM Vol. 2 Part 1 Section 7.7.1). On all hardware
+ * generations, the only configurations supporting persample dispatch
+ * are are this in which only one dispatch width is enabled.
+ *
+ * If computed depth is enabled, SNB only allows SIMD8 while IVB+
+ * allow SIMD8 or SIMD16 so we choose SIMD16 if available.
+ */
+ if (compiler->devinfo->gen == 6 &&
+ prog_data->computed_depth_mode != BRW_PSCDEPTH_OFF) {
+ simd16_cfg = NULL;
+ } else if (simd16_cfg) {
+ simd8_cfg = NULL;
}
}
*/
brw_compute_flat_inputs(prog_data, key->flat_shade, shader);
- cfg_t *simd8_cfg;
- int no_simd8 = (INTEL_DEBUG & DEBUG_NO8) || use_rep_send;
- if ((no_simd8 || compiler->devinfo->gen < 5) && simd16_cfg) {
- simd8_cfg = NULL;
- prog_data->no_8 = true;
- } else {
- simd8_cfg = v.cfg;
- prog_data->no_8 = false;
- }
-
fs_generator g(compiler, log_data, mem_ctx, (void *) key, &prog_data->base,
- v.promoted_constants, v.runtime_check_aads_emit,
+ v8.promoted_constants, v8.runtime_check_aads_emit,
MESA_SHADER_FRAGMENT);
if (unlikely(INTEL_DEBUG & DEBUG_WM)) {
shader->info.name));
}
- if (simd8_cfg)
+ if (simd8_cfg) {
+ prog_data->dispatch_8 = true;
g.generate_code(simd8_cfg, 8);
- if (simd16_cfg)
- prog_data->prog_offset_16 = g.generate_code(simd16_cfg, 16);
+ prog_data->base.dispatch_grf_start_reg = simd8_grf_start;
+ prog_data->reg_blocks_0 = brw_register_blocks(simd8_grf_used);
+
+ if (simd16_cfg) {
+ prog_data->dispatch_16 = true;
+ prog_data->prog_offset_2 = g.generate_code(simd16_cfg, 16);
+ prog_data->dispatch_grf_start_reg_2 = simd16_grf_start;
+ prog_data->reg_blocks_2 = brw_register_blocks(simd16_grf_used);
+ }
+ } else if (simd16_cfg) {
+ prog_data->dispatch_16 = true;
+ g.generate_code(simd16_cfg, 16);
+ prog_data->base.dispatch_grf_start_reg = simd16_grf_start;
+ prog_data->reg_blocks_0 = brw_register_blocks(simd16_grf_used);
+ }
return g.get_assembly(final_assembly_size);
}
} else {
cfg = v8.cfg;
prog_data->simd_size = 8;
+ prog_data->base.dispatch_grf_start_reg = v8.payload.num_regs;
}
}
} else {
cfg = v16.cfg;
prog_data->simd_size = 16;
+ prog_data->dispatch_grf_start_reg_16 = v16.payload.num_regs;
}
}
projects / mesa.git / blobdiff