}
)
-;; Predicated integer comparison.
-(define_insn "*vec_cmp<cmp_op>_<mode>"
+;; Integer comparisons predicated with a PTRUE.
+(define_insn "*cmp<cmp_op><mode>"
[(set (match_operand:<VPRED> 0 "register_operand" "=Upa, Upa")
(unspec:<VPRED>
[(match_operand:<VPRED> 1 "register_operand" "Upl, Upl")
- (match_operand:SVE_I 2 "register_operand" "w, w")
- (match_operand:SVE_I 3 "aarch64_sve_cmp_<imm_con>_operand" "<imm_con>, w")]
- SVE_COND_INT_CMP))
+ (SVE_INT_CMP:<VPRED>
+ (match_operand:SVE_I 2 "register_operand" "w, w")
+ (match_operand:SVE_I 3 "aarch64_sve_cmp_<sve_imm_con>_operand" "<sve_imm_con>, w"))]
+ UNSPEC_MERGE_PTRUE))
(clobber (reg:CC CC_REGNUM))]
"TARGET_SVE"
"@
cmp<cmp_op>\t%0.<Vetype>, %1/z, %2.<Vetype>, %3.<Vetype>"
)
-;; Predicated integer comparison in which only the flags result is interesting.
-(define_insn "*vec_cmp<cmp_op>_<mode>_ptest"
+;; Integer comparisons predicated with a PTRUE in which only the flags result
+;; is interesting.
+(define_insn "*cmp<cmp_op><mode>_ptest"
[(set (reg:CC CC_REGNUM)
(compare:CC
(unspec:SI
[(match_operand:<VPRED> 1 "register_operand" "Upl, Upl")
(unspec:<VPRED>
[(match_dup 1)
- (match_operand:SVE_I 2 "register_operand" "w, w")
- (match_operand:SVE_I 3 "aarch64_sve_cmp_<imm_con>_operand" "<imm_con>, w")]
- SVE_COND_INT_CMP)]
+ (SVE_INT_CMP:<VPRED>
+ (match_operand:SVE_I 2 "register_operand" "w, w")
+ (match_operand:SVE_I 3 "aarch64_sve_cmp_<sve_imm_con>_operand" "<sve_imm_con>, w"))]
+ UNSPEC_MERGE_PTRUE)]
UNSPEC_PTEST_PTRUE)
(const_int 0)))
(clobber (match_scratch:<VPRED> 0 "=Upa, Upa"))]
cmp<cmp_op>\t%0.<Vetype>, %1/z, %2.<Vetype>, %3.<Vetype>"
)
-;; Predicated comparison in which both the flag and predicate results
-;; are interesting.
-(define_insn "*vec_cmp<cmp_op>_<mode>_cc"
+;; Integer comparisons predicated with a PTRUE in which both the flag and
+;; predicate results are interesting.
+(define_insn "*cmp<cmp_op><mode>_cc"
[(set (reg:CC CC_REGNUM)
(compare:CC
(unspec:SI
[(match_operand:<VPRED> 1 "register_operand" "Upl, Upl")
(unspec:<VPRED>
[(match_dup 1)
- (match_operand:SVE_I 2 "register_operand" "w, w")
- (match_operand:SVE_I 3 "aarch64_sve_cmp_<imm_con>_operand" "<imm_con>, w")]
- SVE_COND_INT_CMP)]
+ (SVE_INT_CMP:<VPRED>
+ (match_operand:SVE_I 2 "register_operand" "w, w")
+ (match_operand:SVE_I 3 "aarch64_sve_cmp_<sve_imm_con>_operand" "<sve_imm_con>, w"))]
+ UNSPEC_MERGE_PTRUE)]
UNSPEC_PTEST_PTRUE)
(const_int 0)))
(set (match_operand:<VPRED> 0 "register_operand" "=Upa, Upa")
(unspec:<VPRED>
[(match_dup 1)
- (match_dup 2)
- (match_dup 3)]
- SVE_COND_INT_CMP))]
+ (SVE_INT_CMP:<VPRED>
+ (match_dup 2)
+ (match_dup 3))]
+ UNSPEC_MERGE_PTRUE))]
"TARGET_SVE"
"@
cmp<cmp_op>\t%0.<Vetype>, %1/z, %2.<Vetype>, #%3
cmp<cmp_op>\t%0.<Vetype>, %1/z, %2.<Vetype>, %3.<Vetype>"
)
-;; Predicated floating-point comparison (excluding FCMUO, which doesn't
-;; allow #0.0 as an operand).
-(define_insn "*vec_fcm<cmp_op><mode>"
+;; Floating-point comparisons predicated with a PTRUE.
+(define_insn "*fcm<cmp_op><mode>"
[(set (match_operand:<VPRED> 0 "register_operand" "=Upa, Upa")
(unspec:<VPRED>
[(match_operand:<VPRED> 1 "register_operand" "Upl, Upl")
- (match_operand:SVE_F 2 "register_operand" "w, w")
- (match_operand:SVE_F 3 "aarch64_simd_reg_or_zero" "Dz, w")]
- SVE_COND_FP_CMP))]
+ (SVE_FP_CMP:<VPRED>
+ (match_operand:SVE_F 2 "register_operand" "w, w")
+ (match_operand:SVE_F 3 "aarch64_simd_reg_or_zero" "Dz, w"))]
+ UNSPEC_MERGE_PTRUE))]
"TARGET_SVE"
"@
fcm<cmp_op>\t%0.<Vetype>, %1/z, %2.<Vetype>, #0.0
fcm<cmp_op>\t%0.<Vetype>, %1/z, %2.<Vetype>, %3.<Vetype>"
)
-;; Predicated FCMUO.
-(define_insn "*vec_fcmuo<mode>"
+(define_insn "*fcmuo<mode>"
[(set (match_operand:<VPRED> 0 "register_operand" "=Upa")
(unspec:<VPRED>
[(match_operand:<VPRED> 1 "register_operand" "Upl")
- (match_operand:SVE_F 2 "register_operand" "w")
- (match_operand:SVE_F 3 "register_operand" "w")]
- UNSPEC_COND_UO))]
+ (unordered:<VPRED>
+ (match_operand:SVE_F 2 "register_operand" "w")
+ (match_operand:SVE_F 3 "register_operand" "w"))]
+ UNSPEC_MERGE_PTRUE))]
"TARGET_SVE"
"fcmuo\t%0.<Vetype>, %1/z, %2.<Vetype>, %3.<Vetype>"
)
+;; Predicated floating-point comparisons. We don't need a version
+;; of this for unordered comparisons.
+(define_insn "*pred_fcm<cmp_op><mode>"
+ [(set (match_operand:<VPRED> 0 "register_operand" "=Upa, Upa")
+ (unspec:<VPRED>
+ [(match_operand:<VPRED> 1 "register_operand" "Upl, Upl")
+ (match_operand:SVE_F 2 "register_operand" "w, w")
+ (match_operand:SVE_F 3 "aarch64_simd_reg_or_zero" "Dz, w")]
+ SVE_COND_FP_CMP))]
+ "TARGET_SVE"
+ "@
+ fcm<cmp_op>\t%0.<Vetype>, %1/z, %2.<Vetype>, #0.0
+ fcm<cmp_op>\t%0.<Vetype>, %1/z, %2.<Vetype>, %3.<Vetype>"
+)
+
;; vcond_mask operand order: true, false, mask
;; UNSPEC_SEL operand order: mask, true, false (as for VEC_COND_EXPR)
;; SEL operand order: mask, true, false
: emit_move_insn_1 (dest, src));
}
+/* Apply UNOPTAB to OP and store the result in DEST. */
+
+static void
+aarch64_emit_unop (rtx dest, optab unoptab, rtx op)
+{
+ rtx tmp = expand_unop (GET_MODE (dest), unoptab, op, dest, 0);
+ if (dest != tmp)
+ emit_move_insn (dest, tmp);
+}
+
+/* Apply BINOPTAB to OP0 and OP1 and store the result in DEST. */
+
+static void
+aarch64_emit_binop (rtx dest, optab binoptab, rtx op0, rtx op1)
+{
+ rtx tmp = expand_binop (GET_MODE (dest), binoptab, op0, op1, dest, 0,
+ OPTAB_DIRECT);
+ if (dest != tmp)
+ emit_move_insn (dest, tmp);
+}
+
/* Split a 128-bit move operation into two 64-bit move operations,
taking care to handle partial overlap of register to register
copies. Special cases are needed when moving between GP regs and
}
}
+/* Use predicated SVE instructions to implement the equivalent of:
+
+ (set TARGET OP)
+
+ given that PTRUE is an all-true predicate of the appropriate mode. */
+
+static void
+aarch64_emit_sve_ptrue_op (rtx target, rtx ptrue, rtx op)
+{
+ rtx unspec = gen_rtx_UNSPEC (GET_MODE (target),
+ gen_rtvec (2, ptrue, op),
+ UNSPEC_MERGE_PTRUE);
+ rtx_insn *insn = emit_set_insn (target, unspec);
+ set_unique_reg_note (insn, REG_EQUAL, copy_rtx (op));
+}
+
+/* Likewise, but also clobber the condition codes. */
+
+static void
+aarch64_emit_sve_ptrue_op_cc (rtx target, rtx ptrue, rtx op)
+{
+ rtx unspec = gen_rtx_UNSPEC (GET_MODE (target),
+ gen_rtvec (2, ptrue, op),
+ UNSPEC_MERGE_PTRUE);
+ rtx_insn *insn = emit_insn (gen_set_clobber_cc (target, unspec));
+ set_unique_reg_note (insn, REG_EQUAL, copy_rtx (op));
+}
+
/* Return the UNSPEC_COND_* code for comparison CODE. */
static unsigned int
return UNSPEC_COND_LE;
case GE:
return UNSPEC_COND_GE;
- case LTU:
- return UNSPEC_COND_LO;
- case GTU:
- return UNSPEC_COND_HI;
- case LEU:
- return UNSPEC_COND_LS;
- case GEU:
- return UNSPEC_COND_HS;
- case UNORDERED:
- return UNSPEC_COND_UO;
default:
gcc_unreachable ();
}
}
-/* Return an (unspec:PRED_MODE [PRED OP0 OP1] UNSPEC_COND_<X>) expression,
- where <X> is the operation associated with comparison CODE. */
+/* Emit:
-static rtx
-aarch64_gen_unspec_cond (rtx_code code, machine_mode pred_mode,
- rtx pred, rtx op0, rtx op1)
+ (set TARGET (unspec [PRED OP0 OP1] UNSPEC_COND_<X>))
+
+ where <X> is the operation associated with comparison CODE. This form
+ of instruction is used when (and (CODE OP0 OP1) PRED) would have different
+ semantics, such as when PRED might not be all-true and when comparing
+ inactive lanes could have side effects. */
+
+static void
+aarch64_emit_sve_predicated_cond (rtx target, rtx_code code,
+ rtx pred, rtx op0, rtx op1)
{
- rtvec vec = gen_rtvec (3, pred, op0, op1);
- return gen_rtx_UNSPEC (pred_mode, vec, aarch64_unspec_cond_code (code));
+ rtx unspec = gen_rtx_UNSPEC (GET_MODE (pred),
+ gen_rtvec (3, pred, op0, op1),
+ aarch64_unspec_cond_code (code));
+ emit_set_insn (target, unspec);
}
-/* Expand an SVE integer comparison:
+/* Expand an SVE integer comparison using the SVE equivalent of:
- TARGET = CODE (OP0, OP1). */
+ (set TARGET (CODE OP0 OP1)). */
void
aarch64_expand_sve_vec_cmp_int (rtx target, rtx_code code, rtx op0, rtx op1)
op1 = force_reg (data_mode, op1);
rtx ptrue = force_reg (pred_mode, CONSTM1_RTX (pred_mode));
- rtx unspec = aarch64_gen_unspec_cond (code, pred_mode, ptrue, op0, op1);
- emit_insn (gen_set_clobber_cc (target, unspec));
-}
-
-/* Emit an instruction:
-
- (set TARGET (unspec:PRED_MODE [PRED OP0 OP1] UNSPEC_COND_<X>))
-
- where <X> is the operation associated with comparison CODE. */
-
-static void
-aarch64_emit_unspec_cond (rtx target, rtx_code code, machine_mode pred_mode,
- rtx pred, rtx op0, rtx op1)
-{
- rtx unspec = aarch64_gen_unspec_cond (code, pred_mode, pred, op0, op1);
- emit_set_insn (target, unspec);
+ rtx cond = gen_rtx_fmt_ee (code, pred_mode, op0, op1);
+ aarch64_emit_sve_ptrue_op_cc (target, ptrue, cond);
}
-/* Emit:
+/* Emit the SVE equivalent of:
- (set TMP1 (unspec:PRED_MODE [PTRUE OP0 OP1] UNSPEC_COND_<X1>))
- (set TMP2 (unspec:PRED_MODE [PTRUE OP0 OP1] UNSPEC_COND_<X2>))
- (set TARGET (and:PRED_MODE (ior:PRED_MODE TMP1 TMP2) PTRUE))
+ (set TMP1 (CODE1 OP0 OP1))
+ (set TMP2 (CODE2 OP0 OP1))
+ (set TARGET (ior:PRED_MODE TMP1 TMP2))
- where <Xi> is the operation associated with comparison CODEi. */
+ PTRUE is an all-true predicate with the same mode as TARGET. */
static void
-aarch64_emit_unspec_cond_or (rtx target, rtx_code code1, rtx_code code2,
- machine_mode pred_mode, rtx ptrue,
- rtx op0, rtx op1)
+aarch64_emit_sve_or_conds (rtx target, rtx_code code1, rtx_code code2,
+ rtx ptrue, rtx op0, rtx op1)
{
+ machine_mode pred_mode = GET_MODE (ptrue);
rtx tmp1 = gen_reg_rtx (pred_mode);
- aarch64_emit_unspec_cond (tmp1, code1, pred_mode, ptrue, op0, op1);
+ aarch64_emit_sve_ptrue_op (tmp1, ptrue,
+ gen_rtx_fmt_ee (code1, pred_mode, op0, op1));
rtx tmp2 = gen_reg_rtx (pred_mode);
- aarch64_emit_unspec_cond (tmp2, code2, pred_mode, ptrue, op0, op1);
- emit_set_insn (target, gen_rtx_AND (pred_mode,
- gen_rtx_IOR (pred_mode, tmp1, tmp2),
- ptrue));
+ aarch64_emit_sve_ptrue_op (tmp2, ptrue,
+ gen_rtx_fmt_ee (code2, pred_mode, op0, op1));
+ aarch64_emit_binop (target, ior_optab, tmp1, tmp2);
}
-/* If CAN_INVERT_P, emit an instruction:
+/* Emit the SVE equivalent of:
- (set TARGET (unspec:PRED_MODE [PRED OP0 OP1] UNSPEC_COND_<X>))
+ (set TMP (CODE OP0 OP1))
+ (set TARGET (not TMP))
- where <X> is the operation associated with comparison CODE. Otherwise
- emit:
-
- (set TMP (unspec:PRED_MODE [PRED OP0 OP1] UNSPEC_COND_<X>))
- (set TARGET (and:PRED_MODE (not:PRED_MODE TMP) PTRUE))
-
- where the second instructions sets TARGET to the inverse of TMP. */
+ PTRUE is an all-true predicate with the same mode as TARGET. */
static void
-aarch64_emit_inverted_unspec_cond (rtx target, rtx_code code,
- machine_mode pred_mode, rtx ptrue, rtx pred,
- rtx op0, rtx op1, bool can_invert_p)
+aarch64_emit_sve_inverted_cond (rtx target, rtx ptrue, rtx_code code,
+ rtx op0, rtx op1)
{
- if (can_invert_p)
- aarch64_emit_unspec_cond (target, code, pred_mode, pred, op0, op1);
- else
- {
- rtx tmp = gen_reg_rtx (pred_mode);
- aarch64_emit_unspec_cond (tmp, code, pred_mode, pred, op0, op1);
- emit_set_insn (target, gen_rtx_AND (pred_mode,
- gen_rtx_NOT (pred_mode, tmp),
- ptrue));
- }
+ machine_mode pred_mode = GET_MODE (ptrue);
+ rtx tmp = gen_reg_rtx (pred_mode);
+ aarch64_emit_sve_ptrue_op (tmp, ptrue,
+ gen_rtx_fmt_ee (code, pred_mode, op0, op1));
+ aarch64_emit_unop (target, one_cmpl_optab, tmp);
}
-/* Expand an SVE floating-point comparison:
+/* Expand an SVE floating-point comparison using the SVE equivalent of:
- TARGET = CODE (OP0, OP1)
+ (set TARGET (CODE OP0 OP1))
If CAN_INVERT_P is true, the caller can also handle inverted results;
return true if the result is in fact inverted. */
case UNORDERED:
/* UNORDERED has no immediate form. */
op1 = force_reg (data_mode, op1);
- aarch64_emit_unspec_cond (target, code, pred_mode, ptrue, op0, op1);
- return false;
-
+ /* fall through */
case LT:
case LE:
case GT:
case GE:
case EQ:
case NE:
- /* There is native support for the comparison. */
- aarch64_emit_unspec_cond (target, code, pred_mode, ptrue, op0, op1);
- return false;
-
- case ORDERED:
- /* There is native support for the inverse comparison. */
- op1 = force_reg (data_mode, op1);
- aarch64_emit_inverted_unspec_cond (target, UNORDERED,
- pred_mode, ptrue, ptrue, op0, op1,
- can_invert_p);
- return can_invert_p;
+ {
+ /* There is native support for the comparison. */
+ rtx cond = gen_rtx_fmt_ee (code, pred_mode, op0, op1);
+ aarch64_emit_sve_ptrue_op (target, ptrue, cond);
+ return false;
+ }
case LTGT:
/* This is a trapping operation (LT or GT). */
- aarch64_emit_unspec_cond_or (target, LT, GT, pred_mode, ptrue, op0, op1);
+ aarch64_emit_sve_or_conds (target, LT, GT, ptrue, op0, op1);
return false;
case UNEQ:
{
/* This would trap for signaling NaNs. */
op1 = force_reg (data_mode, op1);
- aarch64_emit_unspec_cond_or (target, UNORDERED, EQ,
- pred_mode, ptrue, op0, op1);
+ aarch64_emit_sve_or_conds (target, UNORDERED, EQ, ptrue, op0, op1);
return false;
}
/* fall through */
-
case UNLT:
case UNLE:
case UNGT:
case UNGE:
- {
- rtx ordered = ptrue;
- if (flag_trapping_math)
- {
- /* Only compare the elements that are known to be ordered. */
- ordered = gen_reg_rtx (pred_mode);
- op1 = force_reg (data_mode, op1);
- aarch64_emit_inverted_unspec_cond (ordered, UNORDERED, pred_mode,
- ptrue, ptrue, op0, op1, false);
- }
- if (code == UNEQ)
- code = NE;
- else
- code = reverse_condition_maybe_unordered (code);
- aarch64_emit_inverted_unspec_cond (target, code, pred_mode, ptrue,
- ordered, op0, op1, can_invert_p);
- return can_invert_p;
- }
+ if (flag_trapping_math)
+ {
+ /* Work out which elements are ordered. */
+ rtx ordered = gen_reg_rtx (pred_mode);
+ op1 = force_reg (data_mode, op1);
+ aarch64_emit_sve_inverted_cond (ordered, ptrue, UNORDERED, op0, op1);
+
+ /* Test the opposite condition for the ordered elements,
+ then invert the result. */
+ if (code == UNEQ)
+ code = NE;
+ else
+ code = reverse_condition_maybe_unordered (code);
+ if (can_invert_p)
+ {
+ aarch64_emit_sve_predicated_cond (target, code,
+ ordered, op0, op1);
+ return true;
+ }
+ rtx tmp = gen_reg_rtx (pred_mode);
+ aarch64_emit_sve_predicated_cond (tmp, code, ordered, op0, op1);
+ aarch64_emit_unop (target, one_cmpl_optab, tmp);
+ return false;
+ }
+ break;
+
+ case ORDERED:
+ /* ORDERED has no immediate form. */
+ op1 = force_reg (data_mode, op1);
+ break;
default:
gcc_unreachable ();
}
+
+ /* There is native support for the inverse comparison. */
+ code = reverse_condition_maybe_unordered (code);
+ if (can_invert_p)
+ {
+ rtx cond = gen_rtx_fmt_ee (code, pred_mode, op0, op1);
+ aarch64_emit_sve_ptrue_op (target, ptrue, cond);
+ return true;
+ }
+ aarch64_emit_sve_inverted_cond (target, ptrue, code, op0, op1);
+ return false;
}
/* Expand an SVE vcond pattern with operands OPS. DATA_MODE is the mode
projects / gcc.git / commitdiff