- emit(MOV(retype(sources[length], BRW_REGISTER_TYPE_UD), lod));
- length++;
- break;
- case ir_query_levels:
- emit(MOV(retype(sources[length], BRW_REGISTER_TYPE_UD), fs_reg(0u)));
- length++;
- break;
- case ir_txf:
- /* Unfortunately, the parameters for LD are intermixed: u, lod, v, r. */
- emit(MOV(retype(sources[length], BRW_REGISTER_TYPE_D), coordinate));
- coordinate = offset(coordinate, 1);
- length++;
-
- emit(MOV(retype(sources[length], BRW_REGISTER_TYPE_D), lod));
- length++;
-
- for (int i = 1; i < ir->coordinate->type->vector_elements; i++) {
- emit(MOV(retype(sources[length], BRW_REGISTER_TYPE_D), coordinate));
- coordinate = offset(coordinate, 1);
- length++;
- }
-
- coordinate_done = true;
- break;
- case ir_txf_ms:
- emit(MOV(retype(sources[length], BRW_REGISTER_TYPE_UD), sample_index));
- length++;
-
- /* data from the multisample control surface */
- emit(MOV(retype(sources[length], BRW_REGISTER_TYPE_UD), mcs));
- length++;
-
- /* there is no offsetting for this message; just copy in the integer
- * texture coordinates
- */
- for (int i = 0; i < ir->coordinate->type->vector_elements; i++) {
- emit(MOV(retype(sources[length], BRW_REGISTER_TYPE_D), coordinate));
- coordinate = offset(coordinate, 1);
- length++;
- }
-
- coordinate_done = true;
- break;
- case ir_tg4:
- if (has_nonconstant_offset) {
- if (shadow_c.file != BAD_FILE)
- no16("Gen7 does not support gather4_po_c in SIMD16 mode.");
-
- /* More crazy intermixing */
- ir->offset->accept(this);
- fs_reg offset_value = this->result;
-
- for (int i = 0; i < 2; i++) { /* u, v */
- emit(MOV(sources[length], coordinate));
- coordinate = offset(coordinate, 1);
- length++;
- }
-
- for (int i = 0; i < 2; i++) { /* offu, offv */
- emit(MOV(retype(sources[length], BRW_REGISTER_TYPE_D), offset_value));
- offset_value = offset(offset_value, 1);
- length++;
- }
-
- if (ir->coordinate->type->vector_elements == 3) { /* r if present */
- emit(MOV(sources[length], coordinate));
- coordinate = offset(coordinate, 1);
- length++;
- }
-
- coordinate_done = true;
- }
- break;
- }
-
- /* Set up the coordinate (except for cases where it was done above) */
- if (ir->coordinate && !coordinate_done) {
- for (int i = 0; i < ir->coordinate->type->vector_elements; i++) {
- emit(MOV(sources[length], coordinate));
- coordinate = offset(coordinate, 1);
- length++;
- }
- }
-
- fs_reg src_payload = fs_reg(GRF, virtual_grf_alloc(length),
- BRW_REGISTER_TYPE_F);
- emit(LOAD_PAYLOAD(src_payload, sources, length));
-
- /* Generate the SEND */
- enum opcode opcode;
- switch (ir->op) {
- case ir_tex: opcode = SHADER_OPCODE_TEX; break;
- case ir_txb: opcode = FS_OPCODE_TXB; break;
- case ir_txl: opcode = SHADER_OPCODE_TXL; break;
- case ir_txd: opcode = SHADER_OPCODE_TXD; break;
- case ir_txf: opcode = SHADER_OPCODE_TXF; break;
- case ir_txf_ms: opcode = SHADER_OPCODE_TXF_CMS; break;
- case ir_txs: opcode = SHADER_OPCODE_TXS; break;
- case ir_query_levels: opcode = SHADER_OPCODE_TXS; break;
- case ir_lod: opcode = SHADER_OPCODE_LOD; break;
- case ir_tg4:
- if (has_nonconstant_offset)
- opcode = SHADER_OPCODE_TG4_OFFSET;
- else
- opcode = SHADER_OPCODE_TG4;
- break;
- default:
- unreachable("not reached");
- }
- fs_inst *inst = emit(opcode, dst, src_payload, sampler);
- inst->base_mrf = -1;
- if (reg_width == 2)
- inst->mlen = length * reg_width - header_present;
- else
- inst->mlen = length * reg_width;
- inst->header_present = header_present;
- inst->regs_written = 4;
-
- if (inst->mlen > MAX_SAMPLER_MESSAGE_SIZE) {
- fail("Message length >" STRINGIFY(MAX_SAMPLER_MESSAGE_SIZE)
- " disallowed by hardware\n");
- }
-
- return inst;
-}
-
-fs_reg
-fs_visitor::rescale_texcoord(ir_texture *ir, fs_reg coordinate,
- bool is_rect, uint32_t sampler, int texunit)
-{
- fs_inst *inst = NULL;
- bool needs_gl_clamp = true;
- fs_reg scale_x, scale_y;
- const struct brw_sampler_prog_key_data *tex =
- (stage == MESA_SHADER_FRAGMENT) ?
- &((brw_wm_prog_key*) this->key)->tex : NULL;
- assert(tex);
-
- /* The 965 requires the EU to do the normalization of GL rectangle
- * texture coordinates. We use the program parameter state
- * tracking to get the scaling factor.
- */
- if (is_rect &&
- (brw->gen < 6 ||
- (brw->gen >= 6 && (tex->gl_clamp_mask[0] & (1 << sampler) ||
- tex->gl_clamp_mask[1] & (1 << sampler))))) {
- struct gl_program_parameter_list *params = prog->Parameters;
- int tokens[STATE_LENGTH] = {
- STATE_INTERNAL,
- STATE_TEXRECT_SCALE,
- texunit,
- 0,
- 0
- };
-
- no16("rectangle scale uniform setup not supported on SIMD16\n");
- if (dispatch_width == 16) {
- return coordinate;
- }
-
- GLuint index = _mesa_add_state_reference(params,
- (gl_state_index *)tokens);
- /* Try to find existing copies of the texrect scale uniforms. */
- for (unsigned i = 0; i < uniforms; i++) {
- if (stage_prog_data->param[i] ==
- &prog->Parameters->ParameterValues[index][0]) {
- scale_x = fs_reg(UNIFORM, i);
- scale_y = fs_reg(UNIFORM, i + 1);
- break;
- }
- }
-
- /* If we didn't already set them up, do so now. */
- if (scale_x.file == BAD_FILE) {
- scale_x = fs_reg(UNIFORM, uniforms);
- scale_y = fs_reg(UNIFORM, uniforms + 1);
-
- stage_prog_data->param[uniforms++] =
- &prog->Parameters->ParameterValues[index][0];
- stage_prog_data->param[uniforms++] =
- &prog->Parameters->ParameterValues[index][1];
- }
- }
-
- /* The 965 requires the EU to do the normalization of GL rectangle
- * texture coordinates. We use the program parameter state
- * tracking to get the scaling factor.
- */
- if (brw->gen < 6 && is_rect) {
- fs_reg dst = fs_reg(this, ir->coordinate->type);
- fs_reg src = coordinate;
- coordinate = dst;
-
- emit(MUL(dst, src, scale_x));
- dst = offset(dst, 1);
- src = offset(src, 1);
- emit(MUL(dst, src, scale_y));
- } else if (is_rect) {
- /* On gen6+, the sampler handles the rectangle coordinates
- * natively, without needing rescaling. But that means we have
- * to do GL_CLAMP clamping at the [0, width], [0, height] scale,
- * not [0, 1] like the default case below.
- */
- needs_gl_clamp = false;
-
- for (int i = 0; i < 2; i++) {
- if (tex->gl_clamp_mask[i] & (1 << sampler)) {
- fs_reg chan = coordinate;
- chan = offset(chan, i);
-
- inst = emit(BRW_OPCODE_SEL, chan, chan, fs_reg(0.0f));
- inst->conditional_mod = BRW_CONDITIONAL_G;
-
- /* Our parameter comes in as 1.0/width or 1.0/height,
- * because that's what people normally want for doing
- * texture rectangle handling. We need width or height
- * for clamping, but we don't care enough to make a new
- * parameter type, so just invert back.
- */
- fs_reg limit = fs_reg(this, glsl_type::float_type);
- emit(MOV(limit, i == 0 ? scale_x : scale_y));
- emit(SHADER_OPCODE_RCP, limit, limit);
-
- inst = emit(BRW_OPCODE_SEL, chan, chan, limit);
- inst->conditional_mod = BRW_CONDITIONAL_L;
- }
- }
- }
-
- if (ir->coordinate && needs_gl_clamp) {
- for (unsigned int i = 0;
- i < MIN2(ir->coordinate->type->vector_elements, 3); i++) {
- if (tex->gl_clamp_mask[i] & (1 << sampler)) {
- fs_reg chan = coordinate;
- chan = offset(chan, i);
-
- fs_inst *inst = emit(MOV(chan, chan));
- inst->saturate = true;
- }
- }
- }
- return coordinate;
-}
-
-/* Sample from the MCS surface attached to this multisample texture. */
-fs_reg
-fs_visitor::emit_mcs_fetch(ir_texture *ir, fs_reg coordinate, fs_reg sampler)
-{
- int reg_width = dispatch_width / 8;
- int length = ir->coordinate->type->vector_elements;
- fs_reg payload = fs_reg(GRF, virtual_grf_alloc(length),
- BRW_REGISTER_TYPE_F);
- fs_reg dest = fs_reg(this, glsl_type::uvec4_type);
- fs_reg *sources = ralloc_array(mem_ctx, fs_reg, length);
-
- /* parameters are: u, v, r; missing parameters are treated as zero */
- for (int i = 0; i < length; i++) {
- sources[i] = fs_reg(this, glsl_type::float_type);
- emit(MOV(retype(sources[i], BRW_REGISTER_TYPE_D), coordinate));
- coordinate = offset(coordinate, 1);
- }
-
- emit(LOAD_PAYLOAD(payload, sources, length));
-
- fs_inst *inst = emit(SHADER_OPCODE_TXF_MCS, dest, payload, sampler);
- inst->base_mrf = -1;
- inst->mlen = length * reg_width;
- inst->header_present = false;
- inst->regs_written = 4; /* we only care about one reg of response,
- * but the sampler always writes 4/8
- */
-
- return dest;
-}
-
-void
-fs_visitor::visit(ir_texture *ir)
-{
- const struct brw_sampler_prog_key_data *tex =
- (stage == MESA_SHADER_FRAGMENT) ?
- &((brw_wm_prog_key*) this->key)->tex : NULL;
- assert(tex);
- fs_inst *inst = NULL;
-
- uint32_t sampler =
- _mesa_get_sampler_uniform_value(ir->sampler, shader_prog, prog);
-
- ir_rvalue *nonconst_sampler_index =
- _mesa_get_sampler_array_nonconst_index(ir->sampler);
-
- /* Handle non-constant sampler array indexing */
- fs_reg sampler_reg;
- if (nonconst_sampler_index) {
- /* The highest sampler which may be used by this operation is
- * the last element of the array. Mark it here, because the generator
- * doesn't have enough information to determine the bound.
- */
- uint32_t array_size = ir->sampler->as_dereference_array()
- ->array->type->array_size();
-
- uint32_t max_used = sampler + array_size - 1;
- if (ir->op == ir_tg4 && brw->gen < 8) {
- max_used += stage_prog_data->binding_table.gather_texture_start;
- } else {
- max_used += stage_prog_data->binding_table.texture_start;
- }
-
- brw_mark_surface_used(prog_data, max_used);
-
- /* Emit code to evaluate the actual indexing expression */
- nonconst_sampler_index->accept(this);
- fs_reg temp(this, glsl_type::uint_type);
- emit(ADD(temp, this->result, fs_reg(sampler)))
- ->force_writemask_all = true;
- sampler_reg = temp;
- } else {
- /* Single sampler, or constant array index; the indexing expression
- * is just an immediate.
- */
- sampler_reg = fs_reg(sampler);
- }
-
- /* FINISHME: We're failing to recompile our programs when the sampler is
- * updated. This only matters for the texture rectangle scale parameters
- * (pre-gen6, or gen6+ with GL_CLAMP).
- */
- int texunit = prog->SamplerUnits[sampler];
-
- if (ir->op == ir_tg4) {
- /* When tg4 is used with the degenerate ZERO/ONE swizzles, don't bother
- * emitting anything other than setting up the constant result.
- */
- ir_constant *chan = ir->lod_info.component->as_constant();
- int swiz = GET_SWZ(tex->swizzles[sampler], chan->value.i[0]);
- if (swiz == SWIZZLE_ZERO || swiz == SWIZZLE_ONE) {
-
- fs_reg res = fs_reg(this, glsl_type::vec4_type);
- this->result = res;
-
- for (int i=0; i<4; i++) {
- emit(MOV(res, fs_reg(swiz == SWIZZLE_ZERO ? 0.0f : 1.0f)));
- res = offset(res, 1);
- }
- return;
- }
- }
-
- /* Should be lowered by do_lower_texture_projection */
- assert(!ir->projector);
-
- /* Should be lowered */
- assert(!ir->offset || !ir->offset->type->is_array());
-
- /* Generate code to compute all the subexpression trees. This has to be
- * done before loading any values into MRFs for the sampler message since
- * generating these values may involve SEND messages that need the MRFs.
- */
- fs_reg coordinate;
- if (ir->coordinate) {
- ir->coordinate->accept(this);
-
- coordinate = rescale_texcoord(ir, this->result,
- ir->sampler->type->sampler_dimensionality ==
- GLSL_SAMPLER_DIM_RECT,
- sampler, texunit);
- }
-
- fs_reg shadow_comparitor;
- if (ir->shadow_comparitor) {
- ir->shadow_comparitor->accept(this);
- shadow_comparitor = this->result;
- }
-
- fs_reg lod, lod2, sample_index, mcs;
- switch (ir->op) {
- case ir_tex:
- case ir_lod:
- case ir_tg4:
- case ir_query_levels:
- break;
- case ir_txb:
- ir->lod_info.bias->accept(this);
- lod = this->result;
- break;
- case ir_txd:
- ir->lod_info.grad.dPdx->accept(this);
- lod = this->result;
-
- ir->lod_info.grad.dPdy->accept(this);
- lod2 = this->result;
- break;
- case ir_txf:
- case ir_txl:
- case ir_txs:
- ir->lod_info.lod->accept(this);
- lod = this->result;
- break;
- case ir_txf_ms:
- ir->lod_info.sample_index->accept(this);
- sample_index = this->result;
-
- if (brw->gen >= 7 && tex->compressed_multisample_layout_mask & (1<<sampler))
- mcs = emit_mcs_fetch(ir, coordinate, sampler_reg);
- else
- mcs = fs_reg(0u);
- break;
- default:
- unreachable("Unrecognized texture opcode");
- };
-
- /* Writemasking doesn't eliminate channels on SIMD8 texture
- * samples, so don't worry about them.
- */
- fs_reg dst = fs_reg(this, glsl_type::get_instance(ir->type->base_type, 4, 1));
-
- if (brw->gen >= 7) {
- inst = emit_texture_gen7(ir, dst, coordinate, shadow_comparitor,
- lod, lod2, sample_index, mcs, sampler_reg);
- } else if (brw->gen >= 5) {
- inst = emit_texture_gen5(ir, dst, coordinate, shadow_comparitor,
- lod, lod2, sample_index, sampler);
- } else {
- inst = emit_texture_gen4(ir, dst, coordinate, shadow_comparitor,
- lod, lod2, sampler);
- }
-
- if (ir->offset != NULL && ir->op != ir_txf)
- inst->texture_offset = brw_texture_offset(ctx, ir->offset->as_constant());
-
- if (ir->op == ir_tg4)
- inst->texture_offset |= gather_channel(ir, sampler) << 16; // M0.2:16-17
-
- if (ir->shadow_comparitor)
- inst->shadow_compare = true;
-
- /* fixup #layers for cube map arrays */
- if (ir->op == ir_txs) {
- glsl_type const *type = ir->sampler->type;
- if (type->sampler_dimensionality == GLSL_SAMPLER_DIM_CUBE &&
- type->sampler_array) {
- fs_reg depth = offset(dst, 2);
- fs_reg fixed_depth = fs_reg(this, glsl_type::int_type);
- emit_math(SHADER_OPCODE_INT_QUOTIENT, fixed_depth, depth, fs_reg(6));
-
- fs_reg *fixed_payload = ralloc_array(mem_ctx, fs_reg, inst->regs_written);
- for (int i = 0; i < inst->regs_written; i++) {
- if (i == 2) {
- fixed_payload[i] = fixed_depth;
- } else {
- fixed_payload[i] = offset(dst, i);
- }
- }
- emit(LOAD_PAYLOAD(dst, fixed_payload, inst->regs_written));
- }
- }
-
- if (brw->gen == 6 && ir->op == ir_tg4) {
- emit_gen6_gather_wa(tex->gen6_gather_wa[sampler], dst);
- }
-
- swizzle_result(ir, dst, sampler);
-}
-
-/**
- * Apply workarounds for Gen6 gather with UINT/SINT
- */
-void
-fs_visitor::emit_gen6_gather_wa(uint8_t wa, fs_reg dst)
-{
- if (!wa)
- return;
-
- int width = (wa & WA_8BIT) ? 8 : 16;
-
- for (int i = 0; i < 4; i++) {
- fs_reg dst_f = retype(dst, BRW_REGISTER_TYPE_F);
- /* Convert from UNORM to UINT */
- emit(MUL(dst_f, dst_f, fs_reg((float)((1 << width) - 1))));
- emit(MOV(dst, dst_f));
-
- if (wa & WA_SIGN) {
- /* Reinterpret the UINT value as a signed INT value by
- * shifting the sign bit into place, then shifting back
- * preserving sign.
- */
- emit(SHL(dst, dst, fs_reg(32 - width)));
- emit(ASR(dst, dst, fs_reg(32 - width)));
- }
-
- dst = offset(dst, 1);
- }
-}
-
-/**
- * Set up the gather channel based on the swizzle, for gather4.
- */
-uint32_t
-fs_visitor::gather_channel(ir_texture *ir, uint32_t sampler)
-{
- const struct brw_sampler_prog_key_data *tex =
- (stage == MESA_SHADER_FRAGMENT) ?
- &((brw_wm_prog_key*) this->key)->tex : NULL;
- assert(tex);
- ir_constant *chan = ir->lod_info.component->as_constant();
- int swiz = GET_SWZ(tex->swizzles[sampler], chan->value.i[0]);
- switch (swiz) {
- case SWIZZLE_X: return 0;
- case SWIZZLE_Y:
- /* gather4 sampler is broken for green channel on RG32F --
- * we must ask for blue instead.
- */
- if (tex->gather_channel_quirk_mask & (1<<sampler))
- return 2;
- return 1;
- case SWIZZLE_Z: return 2;
- case SWIZZLE_W: return 3;
- default:
- unreachable("Not reached"); /* zero, one swizzles handled already */
- }
-}
-
-/**
- * Swizzle the result of a texture result. This is necessary for
- * EXT_texture_swizzle as well as DEPTH_TEXTURE_MODE for shadow comparisons.
- */
-void
-fs_visitor::swizzle_result(ir_texture *ir, fs_reg orig_val, uint32_t sampler)
-{
- if (ir->op == ir_query_levels) {
- /* # levels is in .w */
- this->result = offset(orig_val, 3);
- return;
- }
-
- this->result = orig_val;
-
- /* txs,lod don't actually sample the texture, so swizzling the result
- * makes no sense.
- */
- if (ir->op == ir_txs || ir->op == ir_lod || ir->op == ir_tg4)
- return;
-
- const struct brw_sampler_prog_key_data *tex =
- (stage == MESA_SHADER_FRAGMENT) ?
- &((brw_wm_prog_key*) this->key)->tex : NULL;
- assert(tex);
-
- if (ir->type == glsl_type::float_type) {
- /* Ignore DEPTH_TEXTURE_MODE swizzling. */
- assert(ir->sampler->type->sampler_shadow);
- } else if (tex->swizzles[sampler] != SWIZZLE_NOOP) {
- fs_reg swizzled_result = fs_reg(this, glsl_type::vec4_type);
-
- for (int i = 0; i < 4; i++) {
- int swiz = GET_SWZ(tex->swizzles[sampler], i);
- fs_reg l = swizzled_result;
- l = offset(l, i);
-
- if (swiz == SWIZZLE_ZERO) {
- emit(MOV(l, fs_reg(0.0f)));
- } else if (swiz == SWIZZLE_ONE) {
- emit(MOV(l, fs_reg(1.0f)));
- } else {
- emit(MOV(l, offset(orig_val,
- GET_SWZ(tex->swizzles[sampler], i))));
- }
- }
- this->result = swizzled_result;
- }
-}
-
-void
-fs_visitor::visit(ir_swizzle *ir)
-{
- ir->val->accept(this);
- fs_reg val = this->result;
-
- if (ir->type->vector_elements == 1) {
- this->result = offset(this->result, ir->mask.x);
- return;
- }
-
- fs_reg result = fs_reg(this, ir->type);
- this->result = result;
-
- for (unsigned int i = 0; i < ir->type->vector_elements; i++) {
- fs_reg channel = val;
- int swiz = 0;
-
- switch (i) {
- case 0:
- swiz = ir->mask.x;
- break;
- case 1:
- swiz = ir->mask.y;
- break;
- case 2:
- swiz = ir->mask.z;
- break;
- case 3:
- swiz = ir->mask.w;
- break;
- }
-
- emit(MOV(result, offset(channel, swiz)));
- result = offset(result, 1);
- }
-}
-
-void
-fs_visitor::visit(ir_discard *ir)
-{
- assert(ir->condition == NULL); /* FINISHME */
-
- /* We track our discarded pixels in f0.1. By predicating on it, we can
- * update just the flag bits that aren't yet discarded. By emitting a
- * CMP of g0 != g0, all our currently executing channels will get turned
- * off.
- */
- fs_reg some_reg = fs_reg(retype(brw_vec8_grf(0, 0),
- BRW_REGISTER_TYPE_UW));
- fs_inst *cmp = emit(CMP(reg_null_f, some_reg, some_reg,
- BRW_CONDITIONAL_NZ));
- cmp->predicate = BRW_PREDICATE_NORMAL;
- cmp->flag_subreg = 1;
-
- if (brw->gen >= 6) {
- /* For performance, after a discard, jump to the end of the shader.
- * Only jump if all relevant channels have been discarded.
- */
- fs_inst *discard_jump = emit(FS_OPCODE_DISCARD_JUMP);
- discard_jump->flag_subreg = 1;
-
- discard_jump->predicate = (dispatch_width == 8)
- ? BRW_PREDICATE_ALIGN1_ANY8H
- : BRW_PREDICATE_ALIGN1_ANY16H;
- discard_jump->predicate_inverse = true;
- }
-}
-
-void
-fs_visitor::visit(ir_constant *ir)
-{
- /* Set this->result to reg at the bottom of the function because some code
- * paths will cause this visitor to be applied to other fields. This will
- * cause the value stored in this->result to be modified.
- *
- * Make reg constant so that it doesn't get accidentally modified along the
- * way. Yes, I actually had this problem. :(
- */
- const fs_reg reg(this, ir->type);
- fs_reg dst_reg = reg;
-
- if (ir->type->is_array()) {
- const unsigned size = type_size(ir->type->fields.array);
-
- for (unsigned i = 0; i < ir->type->length; i++) {
- ir->array_elements[i]->accept(this);
- fs_reg src_reg = this->result;
-
- dst_reg.type = src_reg.type;
- for (unsigned j = 0; j < size; j++) {
- emit(MOV(dst_reg, src_reg));
- src_reg = offset(src_reg, 1);
- dst_reg = offset(dst_reg, 1);
- }
- }
- } else if (ir->type->is_record()) {
- foreach_in_list(ir_constant, field, &ir->components) {
- const unsigned size = type_size(field->type);
-
- field->accept(this);
- fs_reg src_reg = this->result;
-
- dst_reg.type = src_reg.type;
- for (unsigned j = 0; j < size; j++) {
- emit(MOV(dst_reg, src_reg));
- src_reg = offset(src_reg, 1);
- dst_reg = offset(dst_reg, 1);
- }
- }
- } else {
- const unsigned size = type_size(ir->type);
-
- for (unsigned i = 0; i < size; i++) {
- switch (ir->type->base_type) {
- case GLSL_TYPE_FLOAT:
- emit(MOV(dst_reg, fs_reg(ir->value.f[i])));
- break;
- case GLSL_TYPE_UINT:
- emit(MOV(dst_reg, fs_reg(ir->value.u[i])));
- break;
- case GLSL_TYPE_INT:
- emit(MOV(dst_reg, fs_reg(ir->value.i[i])));
- break;
- case GLSL_TYPE_BOOL:
- emit(MOV(dst_reg,
- fs_reg(ir->value.b[i] != 0 ? ctx->Const.UniformBooleanTrue
- : 0)));
- break;
- default:
- unreachable("Non-float/uint/int/bool constant");
- }
- dst_reg = offset(dst_reg, 1);
- }
- }
-
- this->result = reg;
-}
-
-void
-fs_visitor::emit_bool_to_cond_code(ir_rvalue *ir)
-{
- ir_expression *expr = ir->as_expression();
-
- if (!expr || expr->operation == ir_binop_ubo_load) {
- ir->accept(this);
-
- fs_inst *inst = emit(AND(reg_null_d, this->result, fs_reg(1)));
- inst->conditional_mod = BRW_CONDITIONAL_NZ;
- return;
- }
-
- fs_reg op[3];
- fs_inst *inst;
-
- assert(expr->get_num_operands() <= 3);
- for (unsigned int i = 0; i < expr->get_num_operands(); i++) {
- assert(expr->operands[i]->type->is_scalar());
-
- expr->operands[i]->accept(this);
- op[i] = this->result;
-
- resolve_ud_negate(&op[i]);
- }
-
- switch (expr->operation) {
- case ir_unop_logic_not:
- inst = emit(AND(reg_null_d, op[0], fs_reg(1)));
- inst->conditional_mod = BRW_CONDITIONAL_Z;
- break;
-
- case ir_binop_logic_xor:
- if (ctx->Const.UniformBooleanTrue == 1) {
- fs_reg dst = fs_reg(this, glsl_type::uint_type);
- emit(XOR(dst, op[0], op[1]));
- inst = emit(AND(reg_null_d, dst, fs_reg(1)));
- inst->conditional_mod = BRW_CONDITIONAL_NZ;
- } else {
- inst = emit(XOR(reg_null_d, op[0], op[1]));
- inst->conditional_mod = BRW_CONDITIONAL_NZ;
- }
- break;
-
- case ir_binop_logic_or:
- if (ctx->Const.UniformBooleanTrue == 1) {
- fs_reg dst = fs_reg(this, glsl_type::uint_type);
- emit(OR(dst, op[0], op[1]));
- inst = emit(AND(reg_null_d, dst, fs_reg(1)));
- inst->conditional_mod = BRW_CONDITIONAL_NZ;
- } else {
- inst = emit(OR(reg_null_d, op[0], op[1]));
- inst->conditional_mod = BRW_CONDITIONAL_NZ;
- }
- break;
-
- case ir_binop_logic_and:
- if (ctx->Const.UniformBooleanTrue == 1) {
- fs_reg dst = fs_reg(this, glsl_type::uint_type);
- emit(AND(dst, op[0], op[1]));
- inst = emit(AND(reg_null_d, dst, fs_reg(1)));
- inst->conditional_mod = BRW_CONDITIONAL_NZ;
- } else {
- inst = emit(AND(reg_null_d, op[0], op[1]));
- inst->conditional_mod = BRW_CONDITIONAL_NZ;
- }
- break;
-
- case ir_unop_f2b:
- if (brw->gen >= 6) {
- emit(CMP(reg_null_d, op[0], fs_reg(0.0f), BRW_CONDITIONAL_NZ));
- } else {
- inst = emit(MOV(reg_null_f, op[0]));
- inst->conditional_mod = BRW_CONDITIONAL_NZ;
- }
- break;
-
- case ir_unop_i2b:
- if (brw->gen >= 6) {
- emit(CMP(reg_null_d, op[0], fs_reg(0), BRW_CONDITIONAL_NZ));
- } else {
- inst = emit(MOV(reg_null_d, op[0]));
- inst->conditional_mod = BRW_CONDITIONAL_NZ;
- }
- break;
-
- case ir_binop_greater:
- case ir_binop_gequal:
- case ir_binop_less:
- case ir_binop_lequal:
- case ir_binop_equal:
- case ir_binop_all_equal:
- case ir_binop_nequal:
- case ir_binop_any_nequal:
- if (ctx->Const.UniformBooleanTrue == 1) {
- resolve_bool_comparison(expr->operands[0], &op[0]);
- resolve_bool_comparison(expr->operands[1], &op[1]);
- }
-
- emit(CMP(reg_null_d, op[0], op[1],
- brw_conditional_for_comparison(expr->operation)));
- break;
-
- case ir_triop_csel: {
- /* Expand the boolean condition into the flag register. */
- inst = emit(MOV(reg_null_d, op[0]));
- inst->conditional_mod = BRW_CONDITIONAL_NZ;
-
- /* Select which boolean to return. */
- fs_reg temp(this, expr->operands[1]->type);
- inst = emit(SEL(temp, op[1], op[2]));
- inst->predicate = BRW_PREDICATE_NORMAL;
-
- /* Expand the result to a condition code. */
- inst = emit(MOV(reg_null_d, temp));
- inst->conditional_mod = BRW_CONDITIONAL_NZ;
- break;
- }
-
- default:
- unreachable("not reached");
- }
-}
-
-/**
- * Emit a gen6 IF statement with the comparison folded into the IF
- * instruction.
- */
-void
-fs_visitor::emit_if_gen6(ir_if *ir)
-{
- ir_expression *expr = ir->condition->as_expression();
-
- if (expr && expr->operation != ir_binop_ubo_load) {
- fs_reg op[3];
- fs_inst *inst;
- fs_reg temp;
-
- assert(expr->get_num_operands() <= 3);
- for (unsigned int i = 0; i < expr->get_num_operands(); i++) {
- assert(expr->operands[i]->type->is_scalar());
-
- expr->operands[i]->accept(this);
- op[i] = this->result;
- }
-
- switch (expr->operation) {
- case ir_unop_logic_not:
- emit(IF(op[0], fs_reg(0), BRW_CONDITIONAL_Z));
- return;
-
- case ir_binop_logic_xor:
- emit(IF(op[0], op[1], BRW_CONDITIONAL_NZ));
- return;
-
- case ir_binop_logic_or:
- temp = fs_reg(this, glsl_type::bool_type);
- emit(OR(temp, op[0], op[1]));
- emit(IF(temp, fs_reg(0), BRW_CONDITIONAL_NZ));
- return;
-
- case ir_binop_logic_and:
- temp = fs_reg(this, glsl_type::bool_type);
- emit(AND(temp, op[0], op[1]));
- emit(IF(temp, fs_reg(0), BRW_CONDITIONAL_NZ));
- return;
-
- case ir_unop_f2b:
- inst = emit(BRW_OPCODE_IF, reg_null_f, op[0], fs_reg(0));
- inst->conditional_mod = BRW_CONDITIONAL_NZ;
- return;
-
- case ir_unop_i2b:
- emit(IF(op[0], fs_reg(0), BRW_CONDITIONAL_NZ));
- return;
-
- case ir_binop_greater:
- case ir_binop_gequal:
- case ir_binop_less:
- case ir_binop_lequal:
- case ir_binop_equal:
- case ir_binop_all_equal:
- case ir_binop_nequal:
- case ir_binop_any_nequal:
- if (ctx->Const.UniformBooleanTrue == 1) {
- resolve_bool_comparison(expr->operands[0], &op[0]);
- resolve_bool_comparison(expr->operands[1], &op[1]);
- }
-
- emit(IF(op[0], op[1],
- brw_conditional_for_comparison(expr->operation)));
- return;
-
- case ir_triop_csel: {
- /* Expand the boolean condition into the flag register. */
- fs_inst *inst = emit(MOV(reg_null_d, op[0]));
- inst->conditional_mod = BRW_CONDITIONAL_NZ;
-
- /* Select which boolean to use as the result. */
- fs_reg temp(this, expr->operands[1]->type);
- inst = emit(SEL(temp, op[1], op[2]));
- inst->predicate = BRW_PREDICATE_NORMAL;
-
- emit(IF(temp, fs_reg(0), BRW_CONDITIONAL_NZ));
- return;
- }
-
- default:
- unreachable("not reached");
- }
- }
-
- ir->condition->accept(this);
- emit(IF(this->result, fs_reg(0), BRW_CONDITIONAL_NZ));
-}
-
-/**
- * Try to replace IF/MOV/ELSE/MOV/ENDIF with SEL.
- *
- * Many GLSL shaders contain the following pattern:
- *
- * x = condition ? foo : bar
- *
- * The compiler emits an ir_if tree for this, since each subexpression might be
- * a complex tree that could have side-effects or short-circuit logic.
- *
- * However, the common case is to simply select one of two constants or
- * variable values---which is exactly what SEL is for. In this case, the
- * assembly looks like:
- *
- * (+f0) IF
- * MOV dst src0
- * ELSE
- * MOV dst src1
- * ENDIF
- *
- * which can be easily translated into:
- *
- * (+f0) SEL dst src0 src1
- *
- * If src0 is an immediate value, we promote it to a temporary GRF.
- */
-void
-fs_visitor::try_replace_with_sel()
-{
- fs_inst *endif_inst = (fs_inst *) instructions.get_tail();
- assert(endif_inst->opcode == BRW_OPCODE_ENDIF);
-
- /* Pattern match in reverse: IF, MOV, ELSE, MOV, ENDIF. */
- int opcodes[] = {
- BRW_OPCODE_IF, BRW_OPCODE_MOV, BRW_OPCODE_ELSE, BRW_OPCODE_MOV,
- };
-
- fs_inst *match = (fs_inst *) endif_inst->prev;
- for (int i = 0; i < 4; i++) {
- if (match->is_head_sentinel() || match->opcode != opcodes[4-i-1])
- return;
- match = (fs_inst *) match->prev;
- }
-
- /* The opcodes match; it looks like the right sequence of instructions. */
- fs_inst *else_mov = (fs_inst *) endif_inst->prev;
- fs_inst *then_mov = (fs_inst *) else_mov->prev->prev;
- fs_inst *if_inst = (fs_inst *) then_mov->prev;
-
- /* Check that the MOVs are the right form. */
- if (then_mov->dst.equals(else_mov->dst) &&
- !then_mov->is_partial_write() &&
- !else_mov->is_partial_write()) {
-
- /* Remove the matched instructions; we'll emit a SEL to replace them. */
- while (!if_inst->next->is_tail_sentinel())
- if_inst->next->exec_node::remove();
- if_inst->exec_node::remove();
-
- /* Only the last source register can be a constant, so if the MOV in
- * the "then" clause uses a constant, we need to put it in a temporary.
- */
- fs_reg src0(then_mov->src[0]);
- if (src0.file == IMM) {
- src0 = fs_reg(this, glsl_type::float_type);
- src0.type = then_mov->src[0].type;
- emit(MOV(src0, then_mov->src[0]));
- }
-
- fs_inst *sel;
- if (if_inst->conditional_mod) {
- /* Sandybridge-specific IF with embedded comparison */
- emit(CMP(reg_null_d, if_inst->src[0], if_inst->src[1],
- if_inst->conditional_mod));
- sel = emit(BRW_OPCODE_SEL, then_mov->dst, src0, else_mov->src[0]);
- sel->predicate = BRW_PREDICATE_NORMAL;
- } else {
- /* Separate CMP and IF instructions */
- sel = emit(BRW_OPCODE_SEL, then_mov->dst, src0, else_mov->src[0]);
- sel->predicate = if_inst->predicate;
- sel->predicate_inverse = if_inst->predicate_inverse;
- }
- }
-}
-
-void
-fs_visitor::visit(ir_if *ir)
-{
- if (brw->gen < 6) {
- no16("Can't support (non-uniform) control flow on SIMD16\n");
- }
-
- /* Don't point the annotation at the if statement, because then it plus
- * the then and else blocks get printed.
- */
- this->base_ir = ir->condition;
-
- if (brw->gen == 6) {
- emit_if_gen6(ir);
- } else {
- emit_bool_to_cond_code(ir->condition);
-
- emit(IF(BRW_PREDICATE_NORMAL));
- }
-
- foreach_in_list(ir_instruction, ir_, &ir->then_instructions) {
- this->base_ir = ir_;
- ir_->accept(this);
- }
-
- if (!ir->else_instructions.is_empty()) {
- emit(BRW_OPCODE_ELSE);
-
- foreach_in_list(ir_instruction, ir_, &ir->else_instructions) {
- this->base_ir = ir_;
- ir_->accept(this);
- }
- }
-
- emit(BRW_OPCODE_ENDIF);
-
- try_replace_with_sel();
-}
-
-void
-fs_visitor::visit(ir_loop *ir)
-{
- if (brw->gen < 6) {
- no16("Can't support (non-uniform) control flow on SIMD16\n");
- }
-
- this->base_ir = NULL;
- emit(BRW_OPCODE_DO);
-
- foreach_in_list(ir_instruction, ir_, &ir->body_instructions) {
- this->base_ir = ir_;
- ir_->accept(this);
- }
-
- this->base_ir = NULL;
- emit(BRW_OPCODE_WHILE);
-}
-
-void
-fs_visitor::visit(ir_loop_jump *ir)
-{
- switch (ir->mode) {
- case ir_loop_jump::jump_break:
- emit(BRW_OPCODE_BREAK);
- break;
- case ir_loop_jump::jump_continue:
- emit(BRW_OPCODE_CONTINUE);
- break;
- }
-}
-
-void
-fs_visitor::visit_atomic_counter_intrinsic(ir_call *ir)
-{
- ir_dereference *deref = static_cast<ir_dereference *>(
- ir->actual_parameters.get_head());
- ir_variable *location = deref->variable_referenced();
- unsigned surf_index = (stage_prog_data->binding_table.abo_start +
- location->data.binding);
-
- /* Calculate the surface offset */
- fs_reg offset(this, glsl_type::uint_type);
- ir_dereference_array *deref_array = deref->as_dereference_array();
-
- if (deref_array) {
- deref_array->array_index->accept(this);
-
- fs_reg tmp(this, glsl_type::uint_type);
- emit(MUL(tmp, this->result, fs_reg(ATOMIC_COUNTER_SIZE)));
- emit(ADD(offset, tmp, fs_reg(location->data.atomic.offset)));
- } else {
- offset = fs_reg(location->data.atomic.offset);
- }
-
- /* Emit the appropriate machine instruction */
- const char *callee = ir->callee->function_name();
- ir->return_deref->accept(this);
- fs_reg dst = this->result;
-
- if (!strcmp("__intrinsic_atomic_read", callee)) {
- emit_untyped_surface_read(surf_index, dst, offset);
-
- } else if (!strcmp("__intrinsic_atomic_increment", callee)) {
- emit_untyped_atomic(BRW_AOP_INC, surf_index, dst, offset,
- fs_reg(), fs_reg());
-
- } else if (!strcmp("__intrinsic_atomic_predecrement", callee)) {
- emit_untyped_atomic(BRW_AOP_PREDEC, surf_index, dst, offset,
- fs_reg(), fs_reg());
- }
-}
-
-void
-fs_visitor::visit(ir_call *ir)
-{
- const char *callee = ir->callee->function_name();
-
- if (!strcmp("__intrinsic_atomic_read", callee) ||
- !strcmp("__intrinsic_atomic_increment", callee) ||
- !strcmp("__intrinsic_atomic_predecrement", callee)) {
- visit_atomic_counter_intrinsic(ir);
- } else {
- unreachable("Unsupported intrinsic.");