+ vec4_instruction *inst;
+
+ dst_reg dst = get_nir_dest(instr->dest.dest,
+ nir_op_infos[instr->op].output_type);
+ dst.writemask = instr->dest.write_mask;
+
+ src_reg op[4];
+ for (unsigned i = 0; i < nir_op_infos[instr->op].num_inputs; i++) {
+ op[i] = get_nir_src(instr->src[i].src,
+ nir_op_infos[instr->op].input_types[i], 4);
+ op[i].swizzle = brw_swizzle_for_nir_swizzle(instr->src[i].swizzle);
+ op[i].abs = instr->src[i].abs;
+ op[i].negate = instr->src[i].negate;
+ }
+
+ switch (instr->op) {
+ case nir_op_imov:
+ case nir_op_fmov:
+ inst = emit(MOV(dst, op[0]));
+ inst->saturate = instr->dest.saturate;
+ break;
+
+ case nir_op_vec2:
+ case nir_op_vec3:
+ case nir_op_vec4:
+ unreachable("not reached: should be handled by lower_vec_to_movs()");
+
+ case nir_op_i2f:
+ case nir_op_u2f:
+ inst = emit(MOV(dst, op[0]));
+ inst->saturate = instr->dest.saturate;
+ break;
+
+ case nir_op_f2i:
+ case nir_op_f2u:
+ inst = emit(MOV(dst, op[0]));
+ break;
+
+ case nir_op_fadd:
+ /* fall through */
+ case nir_op_iadd:
+ inst = emit(ADD(dst, op[0], op[1]));
+ inst->saturate = instr->dest.saturate;
+ break;
+
+ case nir_op_fmul:
+ inst = emit(MUL(dst, op[0], op[1]));
+ inst->saturate = instr->dest.saturate;
+ break;
+
+ case nir_op_imul: {
+ if (devinfo->gen < 8) {
+ nir_const_value *value0 = nir_src_as_const_value(instr->src[0].src);
+ nir_const_value *value1 = nir_src_as_const_value(instr->src[1].src);
+
+ /* For integer multiplication, the MUL uses the low 16 bits of one of
+ * the operands (src0 through SNB, src1 on IVB and later). The MACH
+ * accumulates in the contribution of the upper 16 bits of that
+ * operand. If we can determine that one of the args is in the low
+ * 16 bits, though, we can just emit a single MUL.
+ */
+ if (value0 && value0->u[0] < (1 << 16)) {
+ if (devinfo->gen < 7)
+ emit(MUL(dst, op[0], op[1]));
+ else
+ emit(MUL(dst, op[1], op[0]));
+ } else if (value1 && value1->u[0] < (1 << 16)) {
+ if (devinfo->gen < 7)
+ emit(MUL(dst, op[1], op[0]));
+ else
+ emit(MUL(dst, op[0], op[1]));
+ } else {
+ struct brw_reg acc = retype(brw_acc_reg(8), dst.type);
+
+ emit(MUL(acc, op[0], op[1]));
+ emit(MACH(dst_null_d(), op[0], op[1]));
+ emit(MOV(dst, src_reg(acc)));
+ }
+ } else {
+ emit(MUL(dst, op[0], op[1]));
+ }
+ break;
+ }
+
+ case nir_op_imul_high:
+ case nir_op_umul_high: {
+ struct brw_reg acc = retype(brw_acc_reg(8), dst.type);
+
+ emit(MUL(acc, op[0], op[1]));
+ emit(MACH(dst, op[0], op[1]));
+ break;
+ }
+
+ case nir_op_frcp:
+ inst = emit_math(SHADER_OPCODE_RCP, dst, op[0]);
+ inst->saturate = instr->dest.saturate;
+ break;
+
+ case nir_op_fexp2:
+ inst = emit_math(SHADER_OPCODE_EXP2, dst, op[0]);
+ inst->saturate = instr->dest.saturate;
+ break;
+
+ case nir_op_flog2:
+ inst = emit_math(SHADER_OPCODE_LOG2, dst, op[0]);
+ inst->saturate = instr->dest.saturate;
+ break;
+
+ case nir_op_fsin:
+ inst = emit_math(SHADER_OPCODE_SIN, dst, op[0]);
+ inst->saturate = instr->dest.saturate;
+ break;
+
+ case nir_op_fcos:
+ inst = emit_math(SHADER_OPCODE_COS, dst, op[0]);
+ inst->saturate = instr->dest.saturate;
+ break;
+
+ case nir_op_idiv:
+ case nir_op_udiv:
+ emit_math(SHADER_OPCODE_INT_QUOTIENT, dst, op[0], op[1]);
+ break;
+
+ case nir_op_umod:
+ emit_math(SHADER_OPCODE_INT_REMAINDER, dst, op[0], op[1]);
+ break;
+
+ case nir_op_ldexp:
+ unreachable("not reached: should be handled by ldexp_to_arith()");
+
+ case nir_op_fsqrt:
+ inst = emit_math(SHADER_OPCODE_SQRT, dst, op[0]);
+ inst->saturate = instr->dest.saturate;
+ break;
+
+ case nir_op_frsq:
+ inst = emit_math(SHADER_OPCODE_RSQ, dst, op[0]);
+ inst->saturate = instr->dest.saturate;
+ break;
+
+ case nir_op_fpow:
+ inst = emit_math(SHADER_OPCODE_POW, dst, op[0], op[1]);
+ inst->saturate = instr->dest.saturate;
+ break;
+
+ case nir_op_uadd_carry: {
+ struct brw_reg acc = retype(brw_acc_reg(8), BRW_REGISTER_TYPE_UD);
+
+ emit(ADDC(dst_null_ud(), op[0], op[1]));
+ emit(MOV(dst, src_reg(acc)));
+ break;
+ }
+
+ case nir_op_usub_borrow: {
+ struct brw_reg acc = retype(brw_acc_reg(8), BRW_REGISTER_TYPE_UD);
+
+ emit(SUBB(dst_null_ud(), op[0], op[1]));
+ emit(MOV(dst, src_reg(acc)));
+ break;
+ }
+
+ case nir_op_ftrunc:
+ inst = emit(RNDZ(dst, op[0]));
+ inst->saturate = instr->dest.saturate;
+ break;
+
+ case nir_op_fceil: {
+ src_reg tmp = src_reg(this, glsl_type::float_type);
+ tmp.swizzle =
+ brw_swizzle_for_size(instr->src[0].src.is_ssa ?
+ instr->src[0].src.ssa->num_components :
+ instr->src[0].src.reg.reg->num_components);
+
+ op[0].negate = !op[0].negate;
+ emit(RNDD(dst_reg(tmp), op[0]));
+ tmp.negate = true;
+ inst = emit(MOV(dst, tmp));
+ inst->saturate = instr->dest.saturate;
+ break;
+ }
+
+ case nir_op_ffloor:
+ inst = emit(RNDD(dst, op[0]));
+ inst->saturate = instr->dest.saturate;
+ break;
+
+ case nir_op_ffract:
+ inst = emit(FRC(dst, op[0]));
+ inst->saturate = instr->dest.saturate;
+ break;
+
+ case nir_op_fround_even:
+ inst = emit(RNDE(dst, op[0]));
+ inst->saturate = instr->dest.saturate;
+ break;
+
+ case nir_op_fmin:
+ case nir_op_imin:
+ case nir_op_umin:
+ inst = emit_minmax(BRW_CONDITIONAL_L, dst, op[0], op[1]);
+ inst->saturate = instr->dest.saturate;
+ break;
+
+ case nir_op_fmax:
+ case nir_op_imax:
+ case nir_op_umax:
+ inst = emit_minmax(BRW_CONDITIONAL_GE, dst, op[0], op[1]);
+ inst->saturate = instr->dest.saturate;
+ break;
+
+ case nir_op_fddx:
+ case nir_op_fddx_coarse:
+ case nir_op_fddx_fine:
+ case nir_op_fddy:
+ case nir_op_fddy_coarse:
+ case nir_op_fddy_fine:
+ unreachable("derivatives are not valid in vertex shaders");
+
+ case nir_op_flt:
+ case nir_op_ilt:
+ case nir_op_ult:
+ case nir_op_fge:
+ case nir_op_ige:
+ case nir_op_uge:
+ case nir_op_feq:
+ case nir_op_ieq:
+ case nir_op_fne:
+ case nir_op_ine:
+ emit(CMP(dst, op[0], op[1],
+ brw_conditional_for_nir_comparison(instr->op)));
+ break;
+
+ case nir_op_ball_fequal2:
+ case nir_op_ball_iequal2:
+ case nir_op_ball_fequal3:
+ case nir_op_ball_iequal3:
+ case nir_op_ball_fequal4:
+ case nir_op_ball_iequal4: {
+ dst_reg tmp = dst_reg(this, glsl_type::bool_type);
+
+ switch (instr->op) {
+ case nir_op_ball_fequal2:
+ case nir_op_ball_iequal2:
+ tmp.writemask = WRITEMASK_XY;
+ break;
+ case nir_op_ball_fequal3:
+ case nir_op_ball_iequal3:
+ tmp.writemask = WRITEMASK_XYZ;
+ break;
+ case nir_op_ball_fequal4:
+ case nir_op_ball_iequal4:
+ tmp.writemask = WRITEMASK_XYZW;
+ break;
+ default:
+ unreachable("not reached");
+ }
+
+ emit(CMP(tmp, op[0], op[1],
+ brw_conditional_for_nir_comparison(instr->op)));
+ emit(MOV(dst, src_reg(0)));
+ inst = emit(MOV(dst, src_reg(~0)));
+ inst->predicate = BRW_PREDICATE_ALIGN16_ALL4H;
+ break;
+ }
+
+ case nir_op_bany_fnequal2:
+ case nir_op_bany_inequal2:
+ case nir_op_bany_fnequal3:
+ case nir_op_bany_inequal3:
+ case nir_op_bany_fnequal4:
+ case nir_op_bany_inequal4: {
+ dst_reg tmp = dst_reg(this, glsl_type::bool_type);
+
+ switch (instr->op) {
+ case nir_op_bany_fnequal2:
+ case nir_op_bany_inequal2:
+ tmp.writemask = WRITEMASK_XY;
+ break;
+ case nir_op_bany_fnequal3:
+ case nir_op_bany_inequal3:
+ tmp.writemask = WRITEMASK_XYZ;
+ break;
+ case nir_op_bany_fnequal4:
+ case nir_op_bany_inequal4:
+ tmp.writemask = WRITEMASK_XYZW;
+ break;
+ default:
+ unreachable("not reached");
+ }
+
+ emit(CMP(tmp, op[0], op[1],
+ brw_conditional_for_nir_comparison(instr->op)));
+
+ emit(MOV(dst, src_reg(0)));
+ inst = emit(MOV(dst, src_reg(~0)));
+ inst->predicate = BRW_PREDICATE_ALIGN16_ANY4H;
+ break;
+ }
+
+ case nir_op_inot:
+ if (devinfo->gen >= 8) {
+ op[0] = resolve_source_modifiers(op[0]);
+ }
+ emit(NOT(dst, op[0]));
+ break;
+
+ case nir_op_ixor:
+ if (devinfo->gen >= 8) {
+ op[0] = resolve_source_modifiers(op[0]);
+ op[1] = resolve_source_modifiers(op[1]);
+ }
+ emit(XOR(dst, op[0], op[1]));
+ break;
+
+ case nir_op_ior:
+ if (devinfo->gen >= 8) {
+ op[0] = resolve_source_modifiers(op[0]);
+ op[1] = resolve_source_modifiers(op[1]);
+ }
+ emit(OR(dst, op[0], op[1]));
+ break;
+
+ case nir_op_iand:
+ if (devinfo->gen >= 8) {
+ op[0] = resolve_source_modifiers(op[0]);
+ op[1] = resolve_source_modifiers(op[1]);
+ }
+ emit(AND(dst, op[0], op[1]));
+ break;
+
+ case nir_op_b2i:
+ emit(AND(dst, op[0], src_reg(1)));
+ break;
+
+ case nir_op_b2f:
+ op[0].type = BRW_REGISTER_TYPE_D;
+ dst.type = BRW_REGISTER_TYPE_D;
+ emit(AND(dst, op[0], src_reg(0x3f800000u)));
+ dst.type = BRW_REGISTER_TYPE_F;
+ break;
+
+ case nir_op_f2b:
+ emit(CMP(dst, op[0], src_reg(0.0f), BRW_CONDITIONAL_NZ));
+ break;
+
+ case nir_op_i2b:
+ emit(CMP(dst, op[0], src_reg(0), BRW_CONDITIONAL_NZ));
+ break;
+
+ case nir_op_fnoise1_1:
+ case nir_op_fnoise1_2:
+ case nir_op_fnoise1_3:
+ case nir_op_fnoise1_4:
+ case nir_op_fnoise2_1:
+ case nir_op_fnoise2_2:
+ case nir_op_fnoise2_3:
+ case nir_op_fnoise2_4:
+ case nir_op_fnoise3_1:
+ case nir_op_fnoise3_2:
+ case nir_op_fnoise3_3:
+ case nir_op_fnoise3_4:
+ case nir_op_fnoise4_1:
+ case nir_op_fnoise4_2:
+ case nir_op_fnoise4_3:
+ case nir_op_fnoise4_4:
+ unreachable("not reached: should be handled by lower_noise");
+
+ case nir_op_unpack_half_2x16_split_x:
+ case nir_op_unpack_half_2x16_split_y:
+ case nir_op_pack_half_2x16_split:
+ unreachable("not reached: should not occur in vertex shader");
+
+ case nir_op_unpack_snorm_2x16:
+ case nir_op_unpack_unorm_2x16:
+ case nir_op_pack_snorm_2x16:
+ case nir_op_pack_unorm_2x16:
+ unreachable("not reached: should be handled by lower_packing_builtins");
+
+ case nir_op_unpack_half_2x16:
+ /* As NIR does not guarantee that we have a correct swizzle outside the
+ * boundaries of a vector, and the implementation of emit_unpack_half_2x16
+ * uses the source operand in an operation with WRITEMASK_Y while our
+ * source operand has only size 1, it accessed incorrect data producing
+ * regressions in Piglit. We repeat the swizzle of the first component on the
+ * rest of components to avoid regressions. In the vec4_visitor IR code path
+ * this is not needed because the operand has already the correct swizzle.
+ */
+ op[0].swizzle = brw_compose_swizzle(BRW_SWIZZLE_XXXX, op[0].swizzle);
+ emit_unpack_half_2x16(dst, op[0]);
+ break;
+
+ case nir_op_pack_half_2x16:
+ emit_pack_half_2x16(dst, op[0]);
+ break;
+
+ case nir_op_unpack_unorm_4x8:
+ emit_unpack_unorm_4x8(dst, op[0]);
+ break;
+
+ case nir_op_pack_unorm_4x8:
+ emit_pack_unorm_4x8(dst, op[0]);
+ break;
+
+ case nir_op_unpack_snorm_4x8:
+ emit_unpack_snorm_4x8(dst, op[0]);
+ break;
+
+ case nir_op_pack_snorm_4x8:
+ emit_pack_snorm_4x8(dst, op[0]);
+ break;
+
+ case nir_op_bitfield_reverse:
+ emit(BFREV(dst, op[0]));
+ break;
+
+ case nir_op_bit_count:
+ emit(CBIT(dst, op[0]));
+ break;
+
+ case nir_op_ufind_msb:
+ case nir_op_ifind_msb: {
+ src_reg temp = src_reg(this, glsl_type::uint_type);
+
+ inst = emit(FBH(dst_reg(temp), op[0]));
+ inst->dst.writemask = WRITEMASK_XYZW;
+
+ /* FBH counts from the MSB side, while GLSL's findMSB() wants the count
+ * from the LSB side. If FBH didn't return an error (0xFFFFFFFF), then
+ * subtract the result from 31 to convert the MSB count into an LSB count.
+ */
+
+ /* FBH only supports UD type for dst, so use a MOV to convert UD to D. */
+ temp.swizzle = BRW_SWIZZLE_NOOP;
+ emit(MOV(dst, temp));
+
+ src_reg src_tmp = src_reg(dst);
+ emit(CMP(dst_null_d(), src_tmp, src_reg(-1), BRW_CONDITIONAL_NZ));
+
+ src_tmp.negate = true;
+ inst = emit(ADD(dst, src_tmp, src_reg(31)));
+ inst->predicate = BRW_PREDICATE_NORMAL;
+ break;
+ }
+
+ case nir_op_find_lsb:
+ emit(FBL(dst, op[0]));
+ break;
+
+ case nir_op_ubitfield_extract:
+ case nir_op_ibitfield_extract:
+ op[0] = fix_3src_operand(op[0]);
+ op[1] = fix_3src_operand(op[1]);
+ op[2] = fix_3src_operand(op[2]);
+
+ emit(BFE(dst, op[2], op[1], op[0]));
+ break;
+
+ case nir_op_bfm:
+ emit(BFI1(dst, op[0], op[1]));
+ break;
+
+ case nir_op_bfi:
+ op[0] = fix_3src_operand(op[0]);
+ op[1] = fix_3src_operand(op[1]);
+ op[2] = fix_3src_operand(op[2]);
+
+ emit(BFI2(dst, op[0], op[1], op[2]));
+ break;
+
+ case nir_op_bitfield_insert:
+ unreachable("not reached: should be handled by "
+ "lower_instructions::bitfield_insert_to_bfm_bfi");
+
+ case nir_op_fsign:
+ /* AND(val, 0x80000000) gives the sign bit.
+ *
+ * Predicated OR ORs 1.0 (0x3f800000) with the sign bit if val is not
+ * zero.
+ */
+ emit(CMP(dst_null_f(), op[0], src_reg(0.0f), BRW_CONDITIONAL_NZ));
+
+ op[0].type = BRW_REGISTER_TYPE_UD;
+ dst.type = BRW_REGISTER_TYPE_UD;
+ emit(AND(dst, op[0], src_reg(0x80000000u)));
+
+ inst = emit(OR(dst, src_reg(dst), src_reg(0x3f800000u)));
+ inst->predicate = BRW_PREDICATE_NORMAL;
+ dst.type = BRW_REGISTER_TYPE_F;
+
+ if (instr->dest.saturate) {
+ inst = emit(MOV(dst, src_reg(dst)));
+ inst->saturate = true;
+ }
+ break;
+
+ case nir_op_isign:
+ /* ASR(val, 31) -> negative val generates 0xffffffff (signed -1).
+ * -> non-negative val generates 0x00000000.
+ * Predicated OR sets 1 if val is positive.
+ */
+ emit(CMP(dst_null_d(), op[0], src_reg(0), BRW_CONDITIONAL_G));
+ emit(ASR(dst, op[0], src_reg(31)));
+ inst = emit(OR(dst, src_reg(dst), src_reg(1)));
+ inst->predicate = BRW_PREDICATE_NORMAL;
+ break;
+
+ case nir_op_ishl:
+ emit(SHL(dst, op[0], op[1]));
+ break;
+
+ case nir_op_ishr:
+ emit(ASR(dst, op[0], op[1]));
+ break;
+
+ case nir_op_ushr:
+ emit(SHR(dst, op[0], op[1]));
+ break;
+
+ case nir_op_ffma:
+ op[0] = fix_3src_operand(op[0]);
+ op[1] = fix_3src_operand(op[1]);
+ op[2] = fix_3src_operand(op[2]);
+
+ inst = emit(MAD(dst, op[2], op[1], op[0]));
+ inst->saturate = instr->dest.saturate;
+ break;
+
+ case nir_op_flrp:
+ inst = emit_lrp(dst, op[0], op[1], op[2]);
+ inst->saturate = instr->dest.saturate;
+ break;
+
+ case nir_op_bcsel:
+ emit(CMP(dst_null_d(), op[0], src_reg(0), BRW_CONDITIONAL_NZ));
+ inst = emit(BRW_OPCODE_SEL, dst, op[1], op[2]);
+ inst->predicate = BRW_PREDICATE_NORMAL;
+ break;
+
+ case nir_op_fdot_replicated2:
+ inst = emit(BRW_OPCODE_DP2, dst, op[0], op[1]);
+ inst->saturate = instr->dest.saturate;
+ break;
+
+ case nir_op_fdot_replicated3:
+ inst = emit(BRW_OPCODE_DP3, dst, op[0], op[1]);
+ inst->saturate = instr->dest.saturate;
+ break;
+
+ case nir_op_fdot_replicated4:
+ inst = emit(BRW_OPCODE_DP4, dst, op[0], op[1]);
+ inst->saturate = instr->dest.saturate;
+ break;
+
+ case nir_op_bany2:
+ case nir_op_bany3:
+ case nir_op_bany4: {
+ dst_reg tmp = dst_reg(this, glsl_type::bool_type);
+ tmp.writemask = brw_writemask_for_size(nir_op_infos[instr->op].input_sizes[0]);
+
+ emit(CMP(tmp, op[0], src_reg(0), BRW_CONDITIONAL_NZ));
+
+ emit(MOV(dst, src_reg(0)));
+ inst = emit(MOV(dst, src_reg(~0)));
+ inst->predicate = BRW_PREDICATE_ALIGN16_ANY4H;
+ break;
+ }
+
+ case nir_op_fabs:
+ case nir_op_iabs:
+ case nir_op_fneg:
+ case nir_op_ineg:
+ case nir_op_fsat:
+ unreachable("not reached: should be lowered by lower_source mods");
+
+ case nir_op_fdiv:
+ unreachable("not reached: should be lowered by DIV_TO_MUL_RCP in the compiler");
+
+ case nir_op_fmod:
+ unreachable("not reached: should be lowered by MOD_TO_FLOOR in the compiler");
+
+ case nir_op_fsub:
+ case nir_op_isub:
+ unreachable("not reached: should be handled by ir_sub_to_add_neg");
+
+ default:
+ unreachable("Unimplemented ALU operation");
+ }
+
+ /* If we need to do a boolean resolve, replace the result with -(x & 1)
+ * to sign extend the low bit to 0/~0
+ */
+ if (devinfo->gen <= 5 &&
+ (instr->instr.pass_flags & BRW_NIR_BOOLEAN_MASK) ==
+ BRW_NIR_BOOLEAN_NEEDS_RESOLVE) {
+ dst_reg masked = dst_reg(this, glsl_type::int_type);
+ masked.writemask = dst.writemask;
+ emit(AND(masked, src_reg(dst), src_reg(1)));
+ src_reg masked_neg = src_reg(masked);
+ masked_neg.negate = true;
+ emit(MOV(retype(dst, BRW_REGISTER_TYPE_D), masked_neg));
+ }