/* Expand the basic unary and binary arithmetic operations, for GNU compiler.
- Copyright (C) 1987-2016 Free Software Foundation, Inc.
+ Copyright (C) 1987-2019 Free Software Foundation, Inc.
This file is part of GCC.
#include "emit-rtl.h"
#include "recog.h"
#include "diagnostic-core.h"
+#include "rtx-vector-builder.h"
/* Include insn-config.h before expr.h so that HAVE_conditional_move
is properly defined. */
if (GET_MODE (op0) != VOIDmode && GET_MODE (target) != GET_MODE (op0))
{
note = gen_rtx_fmt_e (code, GET_MODE (op0), copy_rtx (op0));
- if (GET_MODE_SIZE (GET_MODE (op0))
- > GET_MODE_SIZE (GET_MODE (target)))
+ if (GET_MODE_UNIT_SIZE (GET_MODE (op0))
+ > GET_MODE_UNIT_SIZE (GET_MODE (target)))
note = simplify_gen_unary (TRUNCATE, GET_MODE (target),
note, GET_MODE (op0));
else
if (m0 == VOIDmode && m1 == VOIDmode)
return to_mode;
- else if (m0 == VOIDmode || GET_MODE_SIZE (m0) < GET_MODE_SIZE (m1))
+ else if (m0 == VOIDmode || GET_MODE_UNIT_SIZE (m0) < GET_MODE_UNIT_SIZE (m1))
result = m1;
else
result = m0;
- if (GET_MODE_SIZE (result) > GET_MODE_SIZE (to_mode))
+ if (GET_MODE_UNIT_SIZE (result) > GET_MODE_UNIT_SIZE (to_mode))
return to_mode;
return result;
int unsignedp, int no_extend)
{
rtx result;
+ scalar_int_mode int_mode;
/* If we don't have to extend and this is a constant, return it. */
if (no_extend && GET_MODE (op) == VOIDmode)
extend since it will be more efficient to do so unless the signedness of
a promoted object differs from our extension. */
if (! no_extend
+ || !is_a <scalar_int_mode> (mode, &int_mode)
|| (GET_CODE (op) == SUBREG && SUBREG_PROMOTED_VAR_P (op)
&& SUBREG_CHECK_PROMOTED_SIGN (op, unsignedp)))
return convert_modes (mode, oldmode, op, unsignedp);
/* If MODE is no wider than a single word, we return a lowpart or paradoxical
SUBREG. */
- if (GET_MODE_SIZE (mode) <= UNITS_PER_WORD)
- return gen_lowpart (mode, force_reg (GET_MODE (op), op));
+ if (GET_MODE_SIZE (int_mode) <= UNITS_PER_WORD)
+ return gen_lowpart (int_mode, force_reg (GET_MODE (op), op));
/* Otherwise, get an object of MODE, clobber it, and set the low-order
part to OP. */
- result = gen_reg_rtx (mode);
+ result = gen_reg_rtx (int_mode);
emit_clobber (result);
emit_move_insn (gen_lowpart (GET_MODE (op), result), op);
return result;
enum insn_code icode;
int nops = TREE_CODE_LENGTH (ops->code);
int op;
+ bool sbool = false;
oprnd0 = ops->op0;
tmode0 = TYPE_MODE (TREE_TYPE (oprnd0));
- widen_pattern_optab =
- optab_for_tree_code (ops->code, TREE_TYPE (oprnd0), optab_default);
+ if (ops->code == VEC_UNPACK_FIX_TRUNC_HI_EXPR
+ || ops->code == VEC_UNPACK_FIX_TRUNC_LO_EXPR)
+ /* The sign is from the result type rather than operand's type
+ for these ops. */
+ widen_pattern_optab
+ = optab_for_tree_code (ops->code, ops->type, optab_default);
+ else if ((ops->code == VEC_UNPACK_HI_EXPR
+ || ops->code == VEC_UNPACK_LO_EXPR)
+ && VECTOR_BOOLEAN_TYPE_P (ops->type)
+ && VECTOR_BOOLEAN_TYPE_P (TREE_TYPE (oprnd0))
+ && TYPE_MODE (ops->type) == TYPE_MODE (TREE_TYPE (oprnd0))
+ && SCALAR_INT_MODE_P (TYPE_MODE (ops->type)))
+ {
+ /* For VEC_UNPACK_{LO,HI}_EXPR if the mode of op0 and result is
+ the same scalar mode for VECTOR_BOOLEAN_TYPE_P vectors, use
+ vec_unpacks_sbool_{lo,hi}_optab, so that we can pass in
+ the pattern number of elements in the wider vector. */
+ widen_pattern_optab
+ = (ops->code == VEC_UNPACK_HI_EXPR
+ ? vec_unpacks_sbool_hi_optab : vec_unpacks_sbool_lo_optab);
+ sbool = true;
+ }
+ else
+ widen_pattern_optab
+ = optab_for_tree_code (ops->code, TREE_TYPE (oprnd0), optab_default);
if (ops->code == WIDEN_MULT_PLUS_EXPR
|| ops->code == WIDEN_MULT_MINUS_EXPR)
icode = find_widening_optab_handler (widen_pattern_optab,
TYPE_MODE (TREE_TYPE (ops->op2)),
- tmode0, 0);
+ tmode0);
else
icode = optab_handler (widen_pattern_optab, tmode0);
gcc_assert (icode != CODE_FOR_nothing);
oprnd1 = ops->op1;
tmode1 = TYPE_MODE (TREE_TYPE (oprnd1));
}
+ else if (sbool)
+ {
+ nops = 2;
+ op1 = GEN_INT (TYPE_VECTOR_SUBPARTS (TREE_TYPE (oprnd0)).to_constant ());
+ tmode1 = tmode0;
+ }
/* The last operand is of a wider mode than the rest of the operands. */
if (nops == 2)
mode of OP must be the element mode of VMODE. If OP is a constant,
then the return value will be a constant. */
-static rtx
+rtx
expand_vector_broadcast (machine_mode vmode, rtx op)
{
- enum insn_code icode;
+ int n;
rtvec vec;
- rtx ret;
- int i, n;
gcc_checking_assert (VECTOR_MODE_P (vmode));
- n = GET_MODE_NUNITS (vmode);
- vec = rtvec_alloc (n);
- for (i = 0; i < n; ++i)
- RTVEC_ELT (vec, i) = op;
+ if (valid_for_const_vector_p (vmode, op))
+ return gen_const_vec_duplicate (vmode, op);
- if (CONSTANT_P (op))
- return gen_rtx_CONST_VECTOR (vmode, vec);
+ insn_code icode = optab_handler (vec_duplicate_optab, vmode);
+ if (icode != CODE_FOR_nothing)
+ {
+ struct expand_operand ops[2];
+ create_output_operand (&ops[0], NULL_RTX, vmode);
+ create_input_operand (&ops[1], op, GET_MODE (op));
+ expand_insn (icode, 2, ops);
+ return ops[0].value;
+ }
+
+ if (!GET_MODE_NUNITS (vmode).is_constant (&n))
+ return NULL;
/* ??? If the target doesn't have a vec_init, then we have no easy way
of performing this operation. Most of this sort of generic support
is hidden away in the vector lowering support in gimple. */
- icode = optab_handler (vec_init_optab, vmode);
+ icode = convert_optab_handler (vec_init_optab, vmode,
+ GET_MODE_INNER (vmode));
if (icode == CODE_FOR_nothing)
return NULL;
- ret = gen_reg_rtx (vmode);
+ vec = rtvec_alloc (n);
+ for (int i = 0; i < n; ++i)
+ RTVEC_ELT (vec, i) = op;
+ rtx ret = gen_reg_rtx (vmode);
emit_insn (GEN_FCN (icode) (ret, gen_rtx_PARALLEL (vmode, vec)));
return ret;
if (binoptab != ashr_optab)
emit_move_insn (outof_target, CONST0_RTX (word_mode));
else
- if (!force_expand_binop (word_mode, binoptab,
- outof_input, GEN_INT (BITS_PER_WORD - 1),
+ if (!force_expand_binop (word_mode, binoptab, outof_input,
+ gen_int_shift_amount (word_mode,
+ BITS_PER_WORD - 1),
outof_target, unsignedp, methods))
return false;
}
value are the same as for the parent routine. */
static bool
-expand_subword_shift (machine_mode op1_mode, optab binoptab,
+expand_subword_shift (scalar_int_mode op1_mode, optab binoptab,
rtx outof_input, rtx into_input, rtx op1,
rtx outof_target, rtx into_target,
int unsignedp, enum optab_methods methods,
arguments are the same as the parent routine. */
static bool
-expand_doubleword_shift_condmove (machine_mode op1_mode, optab binoptab,
+expand_doubleword_shift_condmove (scalar_int_mode op1_mode, optab binoptab,
enum rtx_code cmp_code, rtx cmp1, rtx cmp2,
rtx outof_input, rtx into_input,
rtx subword_op1, rtx superword_op1,
Return true if the shift could be successfully synthesized. */
static bool
-expand_doubleword_shift (machine_mode op1_mode, optab binoptab,
+expand_doubleword_shift (scalar_int_mode op1_mode, optab binoptab,
rtx outof_input, rtx into_input, rtx op1,
rtx outof_target, rtx into_target,
int unsignedp, enum optab_methods methods,
NO_DEFER_POP;
do_compare_rtx_and_jump (cmp1, cmp2, cmp_code, false, op1_mode,
- 0, 0, subword_label, -1);
+ 0, 0, subword_label,
+ profile_probability::uninitialized ());
OK_DEFER_POP;
if (!expand_superword_shift (binoptab, outof_input, superword_op1,
{
int low = (WORDS_BIG_ENDIAN ? 1 : 0);
int high = (WORDS_BIG_ENDIAN ? 0 : 1);
- rtx wordm1 = umulp ? NULL_RTX : GEN_INT (BITS_PER_WORD - 1);
+ rtx wordm1 = (umulp ? NULL_RTX
+ : gen_int_shift_amount (word_mode, BITS_PER_WORD - 1));
rtx product, adjust, product_high, temp;
rtx op0_high = operand_subword_force (op0, high, mode);
if (target && !REG_P (target))
target = NULL_RTX;
+ /* *_widen_optab needs to determine operand mode, make sure at least
+ one operand has non-VOID mode. */
+ if (GET_MODE (op0_low) == VOIDmode && GET_MODE (op1_low) == VOIDmode)
+ op0_low = force_reg (word_mode, op0_low);
+
if (umulp)
product = expand_binop (mode, umul_widen_optab, op0_low, op1_low,
target, 1, OPTAB_DIRECT);
}
/* Helper function for expand_binop: handle the case where there
- is an insn that directly implements the indicated operation.
+ is an insn ICODE that directly implements the indicated operation.
Returns null if this is not possible. */
static rtx
-expand_binop_directly (machine_mode mode, optab binoptab,
+expand_binop_directly (enum insn_code icode, machine_mode mode, optab binoptab,
rtx op0, rtx op1,
rtx target, int unsignedp, enum optab_methods methods,
rtx_insn *last)
{
- machine_mode from_mode = widened_mode (mode, op0, op1);
- enum insn_code icode = find_widening_optab_handler (binoptab, mode,
- from_mode, 1);
machine_mode xmode0 = insn_data[(int) icode].operand[1].mode;
machine_mode xmode1 = insn_data[(int) icode].operand[2].mode;
machine_mode mode0, mode1, tmp_mode;
|| binoptab == vec_pack_usat_optab
|| binoptab == vec_pack_ssat_optab
|| binoptab == vec_pack_ufix_trunc_optab
- || binoptab == vec_pack_sfix_trunc_optab)
+ || binoptab == vec_pack_sfix_trunc_optab
+ || binoptab == vec_packu_float_optab
+ || binoptab == vec_packs_float_optab)
{
/* The mode of the result is different then the mode of the
arguments. */
tmp_mode = insn_data[(int) icode].operand[0].mode;
if (VECTOR_MODE_P (mode)
- && GET_MODE_NUNITS (tmp_mode) != 2 * GET_MODE_NUNITS (mode))
+ && maybe_ne (GET_MODE_NUNITS (tmp_mode), 2 * GET_MODE_NUNITS (mode)))
{
delete_insns_since (last);
return NULL_RTX;
= (methods == OPTAB_LIB || methods == OPTAB_LIB_WIDEN
? OPTAB_WIDEN : methods);
enum mode_class mclass;
+ enum insn_code icode;
machine_mode wider_mode;
+ scalar_int_mode int_mode;
rtx libfunc;
rtx temp;
rtx_insn *entry_last = get_last_insn ();
/* If we can do it with a three-operand insn, do so. */
- if (methods != OPTAB_MUST_WIDEN
- && find_widening_optab_handler (binoptab, mode,
- widened_mode (mode, op0, op1), 1)
- != CODE_FOR_nothing)
+ if (methods != OPTAB_MUST_WIDEN)
{
- temp = expand_binop_directly (mode, binoptab, op0, op1, target,
- unsignedp, methods, last);
- if (temp)
- return temp;
+ if (convert_optab_p (binoptab))
+ {
+ machine_mode from_mode = widened_mode (mode, op0, op1);
+ icode = find_widening_optab_handler (binoptab, mode, from_mode);
+ }
+ else
+ icode = optab_handler (binoptab, mode);
+ if (icode != CODE_FOR_nothing)
+ {
+ temp = expand_binop_directly (icode, mode, binoptab, op0, op1,
+ target, unsignedp, methods, last);
+ if (temp)
+ return temp;
+ }
}
/* If we were trying to rotate, and that didn't work, try rotating
the other direction before falling back to shifts and bitwise-or. */
if (((binoptab == rotl_optab
- && optab_handler (rotr_optab, mode) != CODE_FOR_nothing)
+ && (icode = optab_handler (rotr_optab, mode)) != CODE_FOR_nothing)
|| (binoptab == rotr_optab
- && optab_handler (rotl_optab, mode) != CODE_FOR_nothing))
- && mclass == MODE_INT)
+ && (icode = optab_handler (rotl_optab, mode)) != CODE_FOR_nothing))
+ && is_int_mode (mode, &int_mode))
{
optab otheroptab = (binoptab == rotl_optab ? rotr_optab : rotl_optab);
rtx newop1;
- unsigned int bits = GET_MODE_PRECISION (mode);
+ unsigned int bits = GET_MODE_PRECISION (int_mode);
if (CONST_INT_P (op1))
- newop1 = GEN_INT (bits - INTVAL (op1));
- else if (targetm.shift_truncation_mask (mode) == bits - 1)
+ newop1 = gen_int_shift_amount (int_mode, bits - INTVAL (op1));
+ else if (targetm.shift_truncation_mask (int_mode) == bits - 1)
newop1 = negate_rtx (GET_MODE (op1), op1);
else
newop1 = expand_binop (GET_MODE (op1), sub_optab,
gen_int_mode (bits, GET_MODE (op1)), op1,
NULL_RTX, unsignedp, OPTAB_DIRECT);
- temp = expand_binop_directly (mode, otheroptab, op0, newop1,
+ temp = expand_binop_directly (icode, int_mode, otheroptab, op0, newop1,
target, unsignedp, methods, last);
if (temp)
return temp;
takes operands of this mode and makes a wider mode. */
if (binoptab == smul_optab
- && GET_MODE_2XWIDER_MODE (mode) != VOIDmode
- && (widening_optab_handler ((unsignedp ? umul_widen_optab
- : smul_widen_optab),
- GET_MODE_2XWIDER_MODE (mode), mode)
- != CODE_FOR_nothing))
- {
- temp = expand_binop (GET_MODE_2XWIDER_MODE (mode),
+ && GET_MODE_2XWIDER_MODE (mode).exists (&wider_mode)
+ && (convert_optab_handler ((unsignedp
+ ? umul_widen_optab
+ : smul_widen_optab),
+ wider_mode, mode) != CODE_FOR_nothing))
+ {
+ /* *_widen_optab needs to determine operand mode, make sure at least
+ one operand has non-VOID mode. */
+ if (GET_MODE (op0) == VOIDmode && GET_MODE (op1) == VOIDmode)
+ op0 = force_reg (mode, op0);
+ temp = expand_binop (wider_mode,
unsignedp ? umul_widen_optab : smul_widen_optab,
op0, op1, NULL_RTX, unsignedp, OPTAB_DIRECT);
else if (binoptab == rotr_optab)
otheroptab = vrotr_optab;
- if (otheroptab && optab_handler (otheroptab, mode) != CODE_FOR_nothing)
+ if (otheroptab
+ && (icode = optab_handler (otheroptab, mode)) != CODE_FOR_nothing)
{
/* The scalar may have been extended to be too wide. Truncate
it back to the proper size to fit in the broadcast vector. */
- machine_mode inner_mode = GET_MODE_INNER (mode);
+ scalar_mode inner_mode = GET_MODE_INNER (mode);
if (!CONST_INT_P (op1)
- && (GET_MODE_BITSIZE (inner_mode)
- < GET_MODE_BITSIZE (GET_MODE (op1))))
+ && (GET_MODE_BITSIZE (as_a <scalar_int_mode> (GET_MODE (op1)))
+ > GET_MODE_BITSIZE (inner_mode)))
op1 = force_reg (inner_mode,
simplify_gen_unary (TRUNCATE, inner_mode, op1,
GET_MODE (op1)));
rtx vop1 = expand_vector_broadcast (mode, op1);
if (vop1)
{
- temp = expand_binop_directly (mode, otheroptab, op0, vop1,
+ temp = expand_binop_directly (icode, mode, otheroptab, op0, vop1,
target, unsignedp, methods, last);
if (temp)
return temp;
if (CLASS_HAS_WIDER_MODES_P (mclass)
&& methods != OPTAB_DIRECT && methods != OPTAB_LIB)
- for (wider_mode = GET_MODE_WIDER_MODE (mode);
- wider_mode != VOIDmode;
- wider_mode = GET_MODE_WIDER_MODE (wider_mode))
+ FOR_EACH_WIDER_MODE (wider_mode, mode)
{
+ machine_mode next_mode;
if (optab_handler (binoptab, wider_mode) != CODE_FOR_nothing
|| (binoptab == smul_optab
- && GET_MODE_WIDER_MODE (wider_mode) != VOIDmode
+ && GET_MODE_WIDER_MODE (wider_mode).exists (&next_mode)
&& (find_widening_optab_handler ((unsignedp
? umul_widen_optab
: smul_widen_optab),
- GET_MODE_WIDER_MODE (wider_mode),
- mode, 0)
+ next_mode, mode)
!= CODE_FOR_nothing)))
{
rtx xop0 = op0, xop1 = op1;
/* These can be done a word at a time. */
if ((binoptab == and_optab || binoptab == ior_optab || binoptab == xor_optab)
- && mclass == MODE_INT
- && GET_MODE_SIZE (mode) > UNITS_PER_WORD
+ && is_int_mode (mode, &int_mode)
+ && GET_MODE_SIZE (int_mode) > UNITS_PER_WORD
&& optab_handler (binoptab, word_mode) != CODE_FOR_nothing)
{
int i;
|| target == op0
|| target == op1
|| !valid_multiword_target_p (target))
- target = gen_reg_rtx (mode);
+ target = gen_reg_rtx (int_mode);
start_sequence ();
/* Do the actual arithmetic. */
- for (i = 0; i < GET_MODE_BITSIZE (mode) / BITS_PER_WORD; i++)
+ machine_mode op0_mode = GET_MODE (op0);
+ machine_mode op1_mode = GET_MODE (op1);
+ if (op0_mode == VOIDmode)
+ op0_mode = int_mode;
+ if (op1_mode == VOIDmode)
+ op1_mode = int_mode;
+ for (i = 0; i < GET_MODE_BITSIZE (int_mode) / BITS_PER_WORD; i++)
{
- rtx target_piece = operand_subword (target, i, 1, mode);
+ rtx target_piece = operand_subword (target, i, 1, int_mode);
rtx x = expand_binop (word_mode, binoptab,
- operand_subword_force (op0, i, mode),
- operand_subword_force (op1, i, mode),
+ operand_subword_force (op0, i, op0_mode),
+ operand_subword_force (op1, i, op1_mode),
target_piece, unsignedp, next_methods);
if (x == 0)
insns = get_insns ();
end_sequence ();
- if (i == GET_MODE_BITSIZE (mode) / BITS_PER_WORD)
+ if (i == GET_MODE_BITSIZE (int_mode) / BITS_PER_WORD)
{
emit_insn (insns);
return target;
/* Synthesize double word shifts from single word shifts. */
if ((binoptab == lshr_optab || binoptab == ashl_optab
|| binoptab == ashr_optab)
- && mclass == MODE_INT
+ && is_int_mode (mode, &int_mode)
&& (CONST_INT_P (op1) || optimize_insn_for_speed_p ())
- && GET_MODE_SIZE (mode) == 2 * UNITS_PER_WORD
- && GET_MODE_PRECISION (mode) == GET_MODE_BITSIZE (mode)
+ && GET_MODE_SIZE (int_mode) == 2 * UNITS_PER_WORD
+ && GET_MODE_PRECISION (int_mode) == GET_MODE_BITSIZE (int_mode)
&& optab_handler (binoptab, word_mode) != CODE_FOR_nothing
&& optab_handler (ashl_optab, word_mode) != CODE_FOR_nothing
&& optab_handler (lshr_optab, word_mode) != CODE_FOR_nothing)
{
unsigned HOST_WIDE_INT shift_mask, double_shift_mask;
- machine_mode op1_mode;
+ scalar_int_mode op1_mode;
- double_shift_mask = targetm.shift_truncation_mask (mode);
+ double_shift_mask = targetm.shift_truncation_mask (int_mode);
shift_mask = targetm.shift_truncation_mask (word_mode);
- op1_mode = GET_MODE (op1) != VOIDmode ? GET_MODE (op1) : word_mode;
+ op1_mode = (GET_MODE (op1) != VOIDmode
+ ? as_a <scalar_int_mode> (GET_MODE (op1))
+ : word_mode);
/* Apply the truncation to constant shifts. */
if (double_shift_mask > 0 && CONST_INT_P (op1))
- op1 = GEN_INT (INTVAL (op1) & double_shift_mask);
+ op1 = gen_int_mode (INTVAL (op1) & double_shift_mask, op1_mode);
if (op1 == CONST0_RTX (op1_mode))
return op0;
|| target == op0
|| target == op1
|| !valid_multiword_target_p (target))
- target = gen_reg_rtx (mode);
+ target = gen_reg_rtx (int_mode);
start_sequence ();
left_shift = binoptab == ashl_optab;
outof_word = left_shift ^ ! WORDS_BIG_ENDIAN;
- outof_target = operand_subword (target, outof_word, 1, mode);
- into_target = operand_subword (target, 1 - outof_word, 1, mode);
+ outof_target = operand_subword (target, outof_word, 1, int_mode);
+ into_target = operand_subword (target, 1 - outof_word, 1, int_mode);
- outof_input = operand_subword_force (op0, outof_word, mode);
- into_input = operand_subword_force (op0, 1 - outof_word, mode);
+ outof_input = operand_subword_force (op0, outof_word, int_mode);
+ into_input = operand_subword_force (op0, 1 - outof_word, int_mode);
if (expand_doubleword_shift (op1_mode, binoptab,
outof_input, into_input, op1,
/* Synthesize double word rotates from single word shifts. */
if ((binoptab == rotl_optab || binoptab == rotr_optab)
- && mclass == MODE_INT
+ && is_int_mode (mode, &int_mode)
&& CONST_INT_P (op1)
- && GET_MODE_PRECISION (mode) == 2 * BITS_PER_WORD
+ && GET_MODE_PRECISION (int_mode) == 2 * BITS_PER_WORD
&& optab_handler (ashl_optab, word_mode) != CODE_FOR_nothing
&& optab_handler (lshr_optab, word_mode) != CODE_FOR_nothing)
{
|| target == op1
|| !REG_P (target)
|| !valid_multiword_target_p (target))
- target = gen_reg_rtx (mode);
+ target = gen_reg_rtx (int_mode);
start_sequence ();
left_shift = (binoptab == rotl_optab);
outof_word = left_shift ^ ! WORDS_BIG_ENDIAN;
- outof_target = operand_subword (target, outof_word, 1, mode);
- into_target = operand_subword (target, 1 - outof_word, 1, mode);
+ outof_target = operand_subword (target, outof_word, 1, int_mode);
+ into_target = operand_subword (target, 1 - outof_word, 1, int_mode);
- outof_input = operand_subword_force (op0, outof_word, mode);
- into_input = operand_subword_force (op0, 1 - outof_word, mode);
+ outof_input = operand_subword_force (op0, outof_word, int_mode);
+ into_input = operand_subword_force (op0, 1 - outof_word, int_mode);
if (shift_count == BITS_PER_WORD)
{
else
{
rtx into_temp1, into_temp2, outof_temp1, outof_temp2;
- rtx first_shift_count, second_shift_count;
+ HOST_WIDE_INT first_shift_count, second_shift_count;
optab reverse_unsigned_shift, unsigned_shift;
reverse_unsigned_shift = (left_shift ^ (shift_count < BITS_PER_WORD)
if (shift_count > BITS_PER_WORD)
{
- first_shift_count = GEN_INT (shift_count - BITS_PER_WORD);
- second_shift_count = GEN_INT (2 * BITS_PER_WORD - shift_count);
+ first_shift_count = shift_count - BITS_PER_WORD;
+ second_shift_count = 2 * BITS_PER_WORD - shift_count;
}
else
{
- first_shift_count = GEN_INT (BITS_PER_WORD - shift_count);
- second_shift_count = GEN_INT (shift_count);
+ first_shift_count = BITS_PER_WORD - shift_count;
+ second_shift_count = shift_count;
}
+ rtx first_shift_count_rtx
+ = gen_int_shift_amount (word_mode, first_shift_count);
+ rtx second_shift_count_rtx
+ = gen_int_shift_amount (word_mode, second_shift_count);
into_temp1 = expand_binop (word_mode, unsigned_shift,
- outof_input, first_shift_count,
+ outof_input, first_shift_count_rtx,
NULL_RTX, unsignedp, next_methods);
into_temp2 = expand_binop (word_mode, reverse_unsigned_shift,
- into_input, second_shift_count,
+ into_input, second_shift_count_rtx,
NULL_RTX, unsignedp, next_methods);
if (into_temp1 != 0 && into_temp2 != 0)
emit_move_insn (into_target, inter);
outof_temp1 = expand_binop (word_mode, unsigned_shift,
- into_input, first_shift_count,
+ into_input, first_shift_count_rtx,
NULL_RTX, unsignedp, next_methods);
outof_temp2 = expand_binop (word_mode, reverse_unsigned_shift,
- outof_input, second_shift_count,
+ outof_input, second_shift_count_rtx,
NULL_RTX, unsignedp, next_methods);
if (inter != 0 && outof_temp1 != 0 && outof_temp2 != 0)
/* These can be done a word at a time by propagating carries. */
if ((binoptab == add_optab || binoptab == sub_optab)
- && mclass == MODE_INT
- && GET_MODE_SIZE (mode) >= 2 * UNITS_PER_WORD
+ && is_int_mode (mode, &int_mode)
+ && GET_MODE_SIZE (int_mode) >= 2 * UNITS_PER_WORD
&& optab_handler (binoptab, word_mode) != CODE_FOR_nothing)
{
unsigned int i;
optab otheroptab = binoptab == add_optab ? sub_optab : add_optab;
- const unsigned int nwords = GET_MODE_BITSIZE (mode) / BITS_PER_WORD;
+ const unsigned int nwords = GET_MODE_BITSIZE (int_mode) / BITS_PER_WORD;
rtx carry_in = NULL_RTX, carry_out = NULL_RTX;
rtx xop0, xop1, xtarget;
#endif
/* Prepare the operands. */
- xop0 = force_reg (mode, op0);
- xop1 = force_reg (mode, op1);
+ xop0 = force_reg (int_mode, op0);
+ xop1 = force_reg (int_mode, op1);
- xtarget = gen_reg_rtx (mode);
+ xtarget = gen_reg_rtx (int_mode);
if (target == 0 || !REG_P (target) || !valid_multiword_target_p (target))
target = xtarget;
for (i = 0; i < nwords; i++)
{
int index = (WORDS_BIG_ENDIAN ? nwords - i - 1 : i);
- rtx target_piece = operand_subword (xtarget, index, 1, mode);
- rtx op0_piece = operand_subword_force (xop0, index, mode);
- rtx op1_piece = operand_subword_force (xop1, index, mode);
+ rtx target_piece = operand_subword (xtarget, index, 1, int_mode);
+ rtx op0_piece = operand_subword_force (xop0, index, int_mode);
+ rtx op1_piece = operand_subword_force (xop1, index, int_mode);
rtx x;
/* Main add/subtract of the input operands. */
carry_in = carry_out;
}
- if (i == GET_MODE_BITSIZE (mode) / (unsigned) BITS_PER_WORD)
+ if (i == GET_MODE_BITSIZE (int_mode) / (unsigned) BITS_PER_WORD)
{
- if (optab_handler (mov_optab, mode) != CODE_FOR_nothing
+ if (optab_handler (mov_optab, int_mode) != CODE_FOR_nothing
|| ! rtx_equal_p (target, xtarget))
{
rtx_insn *temp = emit_move_insn (target, xtarget);
set_dst_reg_note (temp, REG_EQUAL,
gen_rtx_fmt_ee (optab_to_code (binoptab),
- mode, copy_rtx (xop0),
+ int_mode, copy_rtx (xop0),
copy_rtx (xop1)),
target);
}
try using a signed widening multiply. */
if (binoptab == smul_optab
- && mclass == MODE_INT
- && GET_MODE_SIZE (mode) == 2 * UNITS_PER_WORD
+ && is_int_mode (mode, &int_mode)
+ && GET_MODE_SIZE (int_mode) == 2 * UNITS_PER_WORD
&& optab_handler (smul_optab, word_mode) != CODE_FOR_nothing
&& optab_handler (add_optab, word_mode) != CODE_FOR_nothing)
{
rtx product = NULL_RTX;
- if (widening_optab_handler (umul_widen_optab, mode, word_mode)
- != CODE_FOR_nothing)
+ if (convert_optab_handler (umul_widen_optab, int_mode, word_mode)
+ != CODE_FOR_nothing)
{
- product = expand_doubleword_mult (mode, op0, op1, target,
+ product = expand_doubleword_mult (int_mode, op0, op1, target,
true, methods);
if (!product)
delete_insns_since (last);
}
if (product == NULL_RTX
- && widening_optab_handler (smul_widen_optab, mode, word_mode)
- != CODE_FOR_nothing)
+ && (convert_optab_handler (smul_widen_optab, int_mode, word_mode)
+ != CODE_FOR_nothing))
{
- product = expand_doubleword_mult (mode, op0, op1, target,
+ product = expand_doubleword_mult (int_mode, op0, op1, target,
false, methods);
if (!product)
delete_insns_since (last);
if (product != NULL_RTX)
{
- if (optab_handler (mov_optab, mode) != CODE_FOR_nothing)
+ if (optab_handler (mov_optab, int_mode) != CODE_FOR_nothing)
{
- temp = emit_move_insn (target ? target : product, product);
- set_dst_reg_note (temp,
+ rtx_insn *move = emit_move_insn (target ? target : product,
+ product);
+ set_dst_reg_note (move,
REG_EQUAL,
- gen_rtx_fmt_ee (MULT, mode,
+ gen_rtx_fmt_ee (MULT, int_mode,
copy_rtx (op0),
copy_rtx (op1)),
target ? target : product);
/* Pass 1 for NO_QUEUE so we don't lose any increments
if the libcall is cse'd or moved. */
value = emit_library_call_value (libfunc,
- NULL_RTX, LCT_CONST, mode, 2,
+ NULL_RTX, LCT_CONST, mode,
op0, mode, op1x, op1_mode);
insns = get_insns ();
if (CLASS_HAS_WIDER_MODES_P (mclass))
{
- for (wider_mode = GET_MODE_WIDER_MODE (mode);
- wider_mode != VOIDmode;
- wider_mode = GET_MODE_WIDER_MODE (wider_mode))
+ /* This code doesn't make sense for conversion optabs, since we
+ wouldn't then want to extend the operands to be the same size
+ as the result. */
+ gcc_assert (!convert_optab_p (binoptab));
+ FOR_EACH_WIDER_MODE (wider_mode, mode)
{
- if (find_widening_optab_handler (binoptab, wider_mode, mode, 1)
- != CODE_FOR_nothing
+ if (optab_handler (binoptab, wider_mode)
|| (methods == OPTAB_LIB
&& optab_libfunc (binoptab, wider_mode)))
{
if (CLASS_HAS_WIDER_MODES_P (mclass))
{
- for (wider_mode = GET_MODE_WIDER_MODE (mode);
- wider_mode != VOIDmode;
- wider_mode = GET_MODE_WIDER_MODE (wider_mode))
+ FOR_EACH_WIDER_MODE (wider_mode, mode)
{
if (optab_handler (unoptab, wider_mode) != CODE_FOR_nothing)
{
if (CLASS_HAS_WIDER_MODES_P (mclass))
{
- for (wider_mode = GET_MODE_WIDER_MODE (mode);
- wider_mode != VOIDmode;
- wider_mode = GET_MODE_WIDER_MODE (wider_mode))
+ FOR_EACH_WIDER_MODE (wider_mode, mode)
{
if (optab_handler (binoptab, wider_mode) != CODE_FOR_nothing)
{
/* The value returned by the library function will have twice as
many bits as the nominal MODE. */
- libval_mode = smallest_mode_for_size (2 * GET_MODE_BITSIZE (mode),
- MODE_INT);
+ libval_mode = smallest_int_mode_for_size (2 * GET_MODE_BITSIZE (mode));
start_sequence ();
libval = emit_library_call_value (libfunc, NULL_RTX, LCT_CONST,
- libval_mode, 2,
+ libval_mode,
op0, mode,
op1, mode);
/* Get the part of VAL containing the value that we want. */
A similar operation can be used for clrsb. UNOPTAB says which operation
we are trying to expand. */
static rtx
-widen_leading (machine_mode mode, rtx op0, rtx target, optab unoptab)
+widen_leading (scalar_int_mode mode, rtx op0, rtx target, optab unoptab)
{
- enum mode_class mclass = GET_MODE_CLASS (mode);
- if (CLASS_HAS_WIDER_MODES_P (mclass))
+ opt_scalar_int_mode wider_mode_iter;
+ FOR_EACH_WIDER_MODE (wider_mode_iter, mode)
{
- machine_mode wider_mode;
- for (wider_mode = GET_MODE_WIDER_MODE (mode);
- wider_mode != VOIDmode;
- wider_mode = GET_MODE_WIDER_MODE (wider_mode))
+ scalar_int_mode wider_mode = wider_mode_iter.require ();
+ if (optab_handler (unoptab, wider_mode) != CODE_FOR_nothing)
{
- if (optab_handler (unoptab, wider_mode) != CODE_FOR_nothing)
- {
- rtx xop0, temp;
- rtx_insn *last;
+ rtx xop0, temp;
+ rtx_insn *last;
- last = get_last_insn ();
+ last = get_last_insn ();
- if (target == 0)
- target = gen_reg_rtx (mode);
- xop0 = widen_operand (op0, wider_mode, mode,
- unoptab != clrsb_optab, false);
- temp = expand_unop (wider_mode, unoptab, xop0, NULL_RTX,
- unoptab != clrsb_optab);
- if (temp != 0)
- temp = expand_binop
- (wider_mode, sub_optab, temp,
- gen_int_mode (GET_MODE_PRECISION (wider_mode)
- - GET_MODE_PRECISION (mode),
- wider_mode),
- target, true, OPTAB_DIRECT);
- if (temp == 0)
- delete_insns_since (last);
+ if (target == 0)
+ target = gen_reg_rtx (mode);
+ xop0 = widen_operand (op0, wider_mode, mode,
+ unoptab != clrsb_optab, false);
+ temp = expand_unop (wider_mode, unoptab, xop0, NULL_RTX,
+ unoptab != clrsb_optab);
+ if (temp != 0)
+ temp = expand_binop
+ (wider_mode, sub_optab, temp,
+ gen_int_mode (GET_MODE_PRECISION (wider_mode)
+ - GET_MODE_PRECISION (mode),
+ wider_mode),
+ target, true, OPTAB_DIRECT);
+ if (temp == 0)
+ delete_insns_since (last);
- return temp;
- }
+ return temp;
}
}
return 0;
/* Try calculating clz of a double-word quantity as two clz's of word-sized
quantities, choosing which based on whether the high word is nonzero. */
static rtx
-expand_doubleword_clz (machine_mode mode, rtx op0, rtx target)
+expand_doubleword_clz (scalar_int_mode mode, rtx op0, rtx target)
{
rtx xop0 = force_reg (mode, op0);
rtx subhi = gen_highpart (word_mode, xop0);
/* Try calculating popcount of a double-word quantity as two popcount's of
word-sized quantities and summing up the results. */
static rtx
-expand_doubleword_popcount (machine_mode mode, rtx op0, rtx target)
+expand_doubleword_popcount (scalar_int_mode mode, rtx op0, rtx target)
{
rtx t0, t1, t;
rtx_insn *seq;
as
(parity:narrow (low (x) ^ high (x))) */
static rtx
-expand_doubleword_parity (machine_mode mode, rtx op0, rtx target)
+expand_doubleword_parity (scalar_int_mode mode, rtx op0, rtx target)
{
rtx t = expand_binop (word_mode, xor_optab,
operand_subword_force (op0, 0, mode),
as
(lshiftrt:wide (bswap:wide x) ((width wide) - (width narrow))). */
static rtx
-widen_bswap (machine_mode mode, rtx op0, rtx target)
+widen_bswap (scalar_int_mode mode, rtx op0, rtx target)
{
- enum mode_class mclass = GET_MODE_CLASS (mode);
- machine_mode wider_mode;
rtx x;
rtx_insn *last;
+ opt_scalar_int_mode wider_mode_iter;
- if (!CLASS_HAS_WIDER_MODES_P (mclass))
- return NULL_RTX;
+ FOR_EACH_WIDER_MODE (wider_mode_iter, mode)
+ if (optab_handler (bswap_optab, wider_mode_iter.require ())
+ != CODE_FOR_nothing)
+ break;
- for (wider_mode = GET_MODE_WIDER_MODE (mode);
- wider_mode != VOIDmode;
- wider_mode = GET_MODE_WIDER_MODE (wider_mode))
- if (optab_handler (bswap_optab, wider_mode) != CODE_FOR_nothing)
- goto found;
- return NULL_RTX;
+ if (!wider_mode_iter.exists ())
+ return NULL_RTX;
- found:
+ scalar_int_mode wider_mode = wider_mode_iter.require ();
last = get_last_insn ();
x = widen_operand (op0, wider_mode, mode, true, true);
/* Try calculating (parity x) as (and (popcount x) 1), where
popcount can also be done in a wider mode. */
static rtx
-expand_parity (machine_mode mode, rtx op0, rtx target)
+expand_parity (scalar_int_mode mode, rtx op0, rtx target)
{
enum mode_class mclass = GET_MODE_CLASS (mode);
- if (CLASS_HAS_WIDER_MODES_P (mclass))
+ opt_scalar_int_mode wider_mode_iter;
+ FOR_EACH_MODE_FROM (wider_mode_iter, mode)
{
- machine_mode wider_mode;
- for (wider_mode = mode; wider_mode != VOIDmode;
- wider_mode = GET_MODE_WIDER_MODE (wider_mode))
+ scalar_int_mode wider_mode = wider_mode_iter.require ();
+ if (optab_handler (popcount_optab, wider_mode) != CODE_FOR_nothing)
{
- if (optab_handler (popcount_optab, wider_mode) != CODE_FOR_nothing)
- {
- rtx xop0, temp;
- rtx_insn *last;
-
- last = get_last_insn ();
-
- if (target == 0)
- target = gen_reg_rtx (mode);
- xop0 = widen_operand (op0, wider_mode, mode, true, false);
- temp = expand_unop (wider_mode, popcount_optab, xop0, NULL_RTX,
- true);
- if (temp != 0)
- temp = expand_binop (wider_mode, and_optab, temp, const1_rtx,
- target, true, OPTAB_DIRECT);
- if (temp == 0)
- delete_insns_since (last);
+ rtx xop0, temp;
+ rtx_insn *last;
+
+ last = get_last_insn ();
+
+ if (target == 0 || GET_MODE (target) != wider_mode)
+ target = gen_reg_rtx (wider_mode);
+
+ xop0 = widen_operand (op0, wider_mode, mode, true, false);
+ temp = expand_unop (wider_mode, popcount_optab, xop0, NULL_RTX,
+ true);
+ if (temp != 0)
+ temp = expand_binop (wider_mode, and_optab, temp, const1_rtx,
+ target, true, OPTAB_DIRECT);
- return temp;
+ if (temp)
+ {
+ if (mclass != MODE_INT
+ || !TRULY_NOOP_TRUNCATION_MODES_P (mode, wider_mode))
+ return convert_to_mode (mode, temp, 0);
+ else
+ return gen_lowpart (mode, temp);
}
+ else
+ delete_insns_since (last);
}
}
return 0;
less convenient for expand_ffs anyway. */
static rtx
-expand_ctz (machine_mode mode, rtx op0, rtx target)
+expand_ctz (scalar_int_mode mode, rtx op0, rtx target)
{
rtx_insn *seq;
rtx temp;
may have an undefined value in that case. If they do not give us a
convenient value, we have to generate a test and branch. */
static rtx
-expand_ffs (machine_mode mode, rtx op0, rtx target)
+expand_ffs (scalar_int_mode mode, rtx op0, rtx target)
{
HOST_WIDE_INT val = 0;
bool defined_at_zero = false;
logical operation on the sign bit. */
static rtx
-expand_absneg_bit (enum rtx_code code, machine_mode mode,
+expand_absneg_bit (enum rtx_code code, scalar_float_mode mode,
rtx op0, rtx target)
{
const struct real_format *fmt;
int bitpos, word, nwords, i;
- machine_mode imode;
+ scalar_int_mode imode;
rtx temp;
rtx_insn *insns;
if (GET_MODE_SIZE (mode) <= UNITS_PER_WORD)
{
- imode = int_mode_for_mode (mode);
- if (imode == BLKmode)
+ if (!int_mode_for_mode (mode).exists (&imode))
return NULL_RTX;
word = 0;
nwords = 1;
{
enum mode_class mclass = GET_MODE_CLASS (mode);
machine_mode wider_mode;
+ scalar_int_mode int_mode;
+ scalar_float_mode float_mode;
rtx temp;
rtx libfunc;
/* Widening (or narrowing) clz needs special treatment. */
if (unoptab == clz_optab)
{
- temp = widen_leading (mode, op0, target, unoptab);
- if (temp)
- return temp;
-
- if (GET_MODE_SIZE (mode) == 2 * UNITS_PER_WORD
- && optab_handler (unoptab, word_mode) != CODE_FOR_nothing)
+ if (is_a <scalar_int_mode> (mode, &int_mode))
{
- temp = expand_doubleword_clz (mode, op0, target);
+ temp = widen_leading (int_mode, op0, target, unoptab);
if (temp)
return temp;
+
+ if (GET_MODE_SIZE (int_mode) == 2 * UNITS_PER_WORD
+ && optab_handler (unoptab, word_mode) != CODE_FOR_nothing)
+ {
+ temp = expand_doubleword_clz (int_mode, op0, target);
+ if (temp)
+ return temp;
+ }
}
goto try_libcall;
if (unoptab == clrsb_optab)
{
- temp = widen_leading (mode, op0, target, unoptab);
- if (temp)
- return temp;
+ if (is_a <scalar_int_mode> (mode, &int_mode))
+ {
+ temp = widen_leading (int_mode, op0, target, unoptab);
+ if (temp)
+ return temp;
+ }
goto try_libcall;
}
if (unoptab == popcount_optab
- && GET_MODE_SIZE (mode) == 2 * UNITS_PER_WORD
+ && is_a <scalar_int_mode> (mode, &int_mode)
+ && GET_MODE_SIZE (int_mode) == 2 * UNITS_PER_WORD
&& optab_handler (unoptab, word_mode) != CODE_FOR_nothing
&& optimize_insn_for_speed_p ())
{
- temp = expand_doubleword_popcount (mode, op0, target);
+ temp = expand_doubleword_popcount (int_mode, op0, target);
if (temp)
return temp;
}
if (unoptab == parity_optab
- && GET_MODE_SIZE (mode) == 2 * UNITS_PER_WORD
+ && is_a <scalar_int_mode> (mode, &int_mode)
+ && GET_MODE_SIZE (int_mode) == 2 * UNITS_PER_WORD
&& (optab_handler (unoptab, word_mode) != CODE_FOR_nothing
|| optab_handler (popcount_optab, word_mode) != CODE_FOR_nothing)
&& optimize_insn_for_speed_p ())
{
- temp = expand_doubleword_parity (mode, op0, target);
+ temp = expand_doubleword_parity (int_mode, op0, target);
if (temp)
return temp;
}
if (optab_handler (rotl_optab, mode) != CODE_FOR_nothing)
{
- temp = expand_binop (mode, rotl_optab, op0, GEN_INT (8), target,
- unsignedp, OPTAB_DIRECT);
+ temp = expand_binop (mode, rotl_optab, op0,
+ gen_int_shift_amount (mode, 8),
+ target, unsignedp, OPTAB_DIRECT);
if (temp)
return temp;
}
if (optab_handler (rotr_optab, mode) != CODE_FOR_nothing)
{
- temp = expand_binop (mode, rotr_optab, op0, GEN_INT (8), target,
- unsignedp, OPTAB_DIRECT);
+ temp = expand_binop (mode, rotr_optab, op0,
+ gen_int_shift_amount (mode, 8),
+ target, unsignedp, OPTAB_DIRECT);
if (temp)
return temp;
}
last = get_last_insn ();
- temp1 = expand_binop (mode, ashl_optab, op0, GEN_INT (8), NULL_RTX,
+ temp1 = expand_binop (mode, ashl_optab, op0,
+ gen_int_shift_amount (mode, 8), NULL_RTX,
unsignedp, OPTAB_WIDEN);
- temp2 = expand_binop (mode, lshr_optab, op0, GEN_INT (8), NULL_RTX,
+ temp2 = expand_binop (mode, lshr_optab, op0,
+ gen_int_shift_amount (mode, 8), NULL_RTX,
unsignedp, OPTAB_WIDEN);
if (temp1 && temp2)
{
delete_insns_since (last);
}
- temp = widen_bswap (mode, op0, target);
- if (temp)
- return temp;
-
- if (GET_MODE_SIZE (mode) == 2 * UNITS_PER_WORD
- && optab_handler (unoptab, word_mode) != CODE_FOR_nothing)
+ if (is_a <scalar_int_mode> (mode, &int_mode))
{
- temp = expand_doubleword_bswap (mode, op0, target);
+ temp = widen_bswap (int_mode, op0, target);
if (temp)
return temp;
+
+ if (GET_MODE_SIZE (int_mode) == 2 * UNITS_PER_WORD
+ && optab_handler (unoptab, word_mode) != CODE_FOR_nothing)
+ {
+ temp = expand_doubleword_bswap (mode, op0, target);
+ if (temp)
+ return temp;
+ }
}
goto try_libcall;
}
if (CLASS_HAS_WIDER_MODES_P (mclass))
- for (wider_mode = GET_MODE_WIDER_MODE (mode);
- wider_mode != VOIDmode;
- wider_mode = GET_MODE_WIDER_MODE (wider_mode))
+ FOR_EACH_WIDER_MODE (wider_mode, mode)
{
if (optab_handler (unoptab, wider_mode) != CODE_FOR_nothing)
{
/* These can be done a word at a time. */
if (unoptab == one_cmpl_optab
- && mclass == MODE_INT
- && GET_MODE_SIZE (mode) > UNITS_PER_WORD
+ && is_int_mode (mode, &int_mode)
+ && GET_MODE_SIZE (int_mode) > UNITS_PER_WORD
&& optab_handler (unoptab, word_mode) != CODE_FOR_nothing)
{
int i;
rtx_insn *insns;
if (target == 0 || target == op0 || !valid_multiword_target_p (target))
- target = gen_reg_rtx (mode);
+ target = gen_reg_rtx (int_mode);
start_sequence ();
/* Do the actual arithmetic. */
- for (i = 0; i < GET_MODE_BITSIZE (mode) / BITS_PER_WORD; i++)
+ for (i = 0; i < GET_MODE_BITSIZE (int_mode) / BITS_PER_WORD; i++)
{
- rtx target_piece = operand_subword (target, i, 1, mode);
+ rtx target_piece = operand_subword (target, i, 1, int_mode);
rtx x = expand_unop (word_mode, unoptab,
- operand_subword_force (op0, i, mode),
+ operand_subword_force (op0, i, int_mode),
target_piece, unsignedp);
if (target_piece != x)
if (optab_to_code (unoptab) == NEG)
{
/* Try negating floating point values by flipping the sign bit. */
- if (SCALAR_FLOAT_MODE_P (mode))
+ if (is_a <scalar_float_mode> (mode, &float_mode))
{
- temp = expand_absneg_bit (NEG, mode, op0, target);
+ temp = expand_absneg_bit (NEG, float_mode, op0, target);
if (temp)
return temp;
}
}
/* Try calculating parity (x) as popcount (x) % 2. */
- if (unoptab == parity_optab)
+ if (unoptab == parity_optab && is_a <scalar_int_mode> (mode, &int_mode))
{
- temp = expand_parity (mode, op0, target);
+ temp = expand_parity (int_mode, op0, target);
if (temp)
return temp;
}
/* Try implementing ffs (x) in terms of clz (x). */
- if (unoptab == ffs_optab)
+ if (unoptab == ffs_optab && is_a <scalar_int_mode> (mode, &int_mode))
{
- temp = expand_ffs (mode, op0, target);
+ temp = expand_ffs (int_mode, op0, target);
if (temp)
return temp;
}
/* Try implementing ctz (x) in terms of clz (x). */
- if (unoptab == ctz_optab)
+ if (unoptab == ctz_optab && is_a <scalar_int_mode> (mode, &int_mode))
{
- temp = expand_ctz (mode, op0, target);
+ temp = expand_ctz (int_mode, op0, target);
if (temp)
return temp;
}
/* Pass 1 for NO_QUEUE so we don't lose any increments
if the libcall is cse'd or moved. */
value = emit_library_call_value (libfunc, NULL_RTX, LCT_CONST, outmode,
- 1, op0, mode);
+ op0, mode);
insns = get_insns ();
end_sequence ();
else
{
eq_value = gen_rtx_fmt_e (optab_to_code (unoptab), mode, op0);
- if (GET_MODE_SIZE (outmode) < GET_MODE_SIZE (mode))
+ if (GET_MODE_UNIT_SIZE (outmode) < GET_MODE_UNIT_SIZE (mode))
eq_value = simplify_gen_unary (TRUNCATE, outmode, eq_value, mode);
- else if (GET_MODE_SIZE (outmode) > GET_MODE_SIZE (mode))
+ else if (GET_MODE_UNIT_SIZE (outmode) > GET_MODE_UNIT_SIZE (mode))
eq_value = simplify_gen_unary (ZERO_EXTEND,
outmode, eq_value, mode);
}
if (CLASS_HAS_WIDER_MODES_P (mclass))
{
- for (wider_mode = GET_MODE_WIDER_MODE (mode);
- wider_mode != VOIDmode;
- wider_mode = GET_MODE_WIDER_MODE (wider_mode))
+ FOR_EACH_WIDER_MODE (wider_mode, mode)
{
if (optab_handler (unoptab, wider_mode) != CODE_FOR_nothing
|| optab_libfunc (unoptab, wider_mode))
result. Similarly for clrsb. */
if ((unoptab == clz_optab || unoptab == clrsb_optab)
&& temp != 0)
- temp = expand_binop
- (wider_mode, sub_optab, temp,
- gen_int_mode (GET_MODE_PRECISION (wider_mode)
- - GET_MODE_PRECISION (mode),
- wider_mode),
- target, true, OPTAB_DIRECT);
+ {
+ scalar_int_mode wider_int_mode
+ = as_a <scalar_int_mode> (wider_mode);
+ int_mode = as_a <scalar_int_mode> (mode);
+ temp = expand_binop
+ (wider_mode, sub_optab, temp,
+ gen_int_mode (GET_MODE_PRECISION (wider_int_mode)
+ - GET_MODE_PRECISION (int_mode),
+ wider_int_mode),
+ target, true, OPTAB_DIRECT);
+ }
/* Likewise for bswap. */
if (unoptab == bswap_optab && temp != 0)
{
- gcc_assert (GET_MODE_PRECISION (wider_mode)
- == GET_MODE_BITSIZE (wider_mode)
- && GET_MODE_PRECISION (mode)
- == GET_MODE_BITSIZE (mode));
-
- temp = expand_shift (RSHIFT_EXPR, wider_mode, temp,
- GET_MODE_BITSIZE (wider_mode)
- - GET_MODE_BITSIZE (mode),
+ scalar_int_mode wider_int_mode
+ = as_a <scalar_int_mode> (wider_mode);
+ int_mode = as_a <scalar_int_mode> (mode);
+ gcc_assert (GET_MODE_PRECISION (wider_int_mode)
+ == GET_MODE_BITSIZE (wider_int_mode)
+ && GET_MODE_PRECISION (int_mode)
+ == GET_MODE_BITSIZE (int_mode));
+
+ temp = expand_shift (RSHIFT_EXPR, wider_int_mode, temp,
+ GET_MODE_BITSIZE (wider_int_mode)
+ - GET_MODE_BITSIZE (int_mode),
NULL_RTX, true);
}
return temp;
/* For floating point modes, try clearing the sign bit. */
- if (SCALAR_FLOAT_MODE_P (mode))
+ scalar_float_mode float_mode;
+ if (is_a <scalar_float_mode> (mode, &float_mode))
{
- temp = expand_absneg_bit (ABS, mode, op0, target);
+ temp = expand_absneg_bit (ABS, float_mode, op0, target);
if (temp)
return temp;
}
value of X as (((signed) x >> (W-1)) ^ x) - ((signed) x >> (W-1)),
where W is the width of MODE. */
- if (GET_MODE_CLASS (mode) == MODE_INT
+ scalar_int_mode int_mode;
+ if (is_int_mode (mode, &int_mode)
&& BRANCH_COST (optimize_insn_for_speed_p (),
false) >= 2)
{
- rtx extended = expand_shift (RSHIFT_EXPR, mode, op0,
- GET_MODE_PRECISION (mode) - 1,
+ rtx extended = expand_shift (RSHIFT_EXPR, int_mode, op0,
+ GET_MODE_PRECISION (int_mode) - 1,
NULL_RTX, 0);
- temp = expand_binop (mode, xor_optab, extended, op0, target, 0,
+ temp = expand_binop (int_mode, xor_optab, extended, op0, target, 0,
OPTAB_LIB_WIDEN);
if (temp != 0)
- temp = expand_binop (mode, result_unsignedp ? sub_optab : subv_optab,
+ temp = expand_binop (int_mode,
+ result_unsignedp ? sub_optab : subv_optab,
temp, extended, target, 0, OPTAB_LIB_WIDEN);
if (temp != 0)
NO_DEFER_POP;
do_compare_rtx_and_jump (target, CONST0_RTX (mode), GE, 0, mode,
- NULL_RTX, NULL, op1, -1);
+ NULL_RTX, NULL, op1,
+ profile_probability::uninitialized ());
op0 = expand_unop (mode, result_unsignedp ? neg_optab : negv_optab,
target, target, 0);
/* If this machine has expensive jumps, we can do one's complement
absolute value of X as (((signed) x >> (W-1)) ^ x). */
- if (GET_MODE_CLASS (mode) == MODE_INT
+ scalar_int_mode int_mode;
+ if (is_int_mode (mode, &int_mode)
&& BRANCH_COST (optimize_insn_for_speed_p (),
false) >= 2)
{
- rtx extended = expand_shift (RSHIFT_EXPR, mode, op0,
- GET_MODE_PRECISION (mode) - 1,
+ rtx extended = expand_shift (RSHIFT_EXPR, int_mode, op0,
+ GET_MODE_PRECISION (int_mode) - 1,
NULL_RTX, 0);
- temp = expand_binop (mode, xor_optab, extended, op0, target, 0,
+ temp = expand_binop (int_mode, xor_optab, extended, op0, target, 0,
OPTAB_LIB_WIDEN);
if (temp != 0)
and not playing with subregs so much, will help the register allocator. */
static rtx
-expand_copysign_absneg (machine_mode mode, rtx op0, rtx op1, rtx target,
+expand_copysign_absneg (scalar_float_mode mode, rtx op0, rtx op1, rtx target,
int bitpos, bool op0_is_abs)
{
- machine_mode imode;
+ scalar_int_mode imode;
enum insn_code icode;
rtx sign;
rtx_code_label *label;
icode = optab_handler (signbit_optab, mode);
if (icode != CODE_FOR_nothing)
{
- imode = insn_data[(int) icode].operand[0].mode;
+ imode = as_a <scalar_int_mode> (insn_data[(int) icode].operand[0].mode);
sign = gen_reg_rtx (imode);
emit_unop_insn (icode, sign, op1, UNKNOWN);
}
{
if (GET_MODE_SIZE (mode) <= UNITS_PER_WORD)
{
- imode = int_mode_for_mode (mode);
- if (imode == BLKmode)
+ if (!int_mode_for_mode (mode).exists (&imode))
return NULL_RTX;
op1 = gen_lowpart (imode, op1);
}
is true if op0 is known to have its sign bit clear. */
static rtx
-expand_copysign_bit (machine_mode mode, rtx op0, rtx op1, rtx target,
+expand_copysign_bit (scalar_float_mode mode, rtx op0, rtx op1, rtx target,
int bitpos, bool op0_is_abs)
{
- machine_mode imode;
+ scalar_int_mode imode;
int word, nwords, i;
rtx temp;
rtx_insn *insns;
if (GET_MODE_SIZE (mode) <= UNITS_PER_WORD)
{
- imode = int_mode_for_mode (mode);
- if (imode == BLKmode)
+ if (!int_mode_for_mode (mode).exists (&imode))
return NULL_RTX;
word = 0;
nwords = 1;
rtx
expand_copysign (rtx op0, rtx op1, rtx target)
{
- machine_mode mode = GET_MODE (op0);
+ scalar_float_mode mode;
const struct real_format *fmt;
bool op0_is_abs;
rtx temp;
- gcc_assert (SCALAR_FLOAT_MODE_P (mode));
+ mode = as_a <scalar_float_mode> (GET_MODE (op0));
gcc_assert (GET_MODE (op1) == mode);
/* First try to do it with a special instruction. */
}
void
-emit_libcall_block (rtx insns, rtx target, rtx result, rtx equiv)
+emit_libcall_block (rtx_insn *insns, rtx target, rtx result, rtx equiv)
{
- emit_libcall_block_1 (safe_as_a <rtx_insn *> (insns),
- target, result, equiv, false);
+ emit_libcall_block_1 (insns, target, result, equiv, false);
}
\f
/* Nonzero if we can perform a comparison of mode MODE straightforwardly.
&& optab_handler (cmov_optab, mode) != CODE_FOR_nothing)
return 1;
- mode = GET_MODE_WIDER_MODE (mode);
+ mode = GET_MODE_WIDER_MODE (mode).else_void ();
PUT_MODE (test, mode);
}
while (mode != VOIDmode);
gcc_assert (methods == OPTAB_DIRECT || methods == OPTAB_WIDEN
|| methods == OPTAB_LIB_WIDEN);
+ if (CONST_SCALAR_INT_P (y))
+ canonicalize_comparison (mode, &comparison, &y);
+
/* If we are optimizing, force expensive constants into a register. */
if (CONSTANT_P (x) && optimize
&& (rtx_cost (x, mode, COMPARE, 0, optimize_insn_for_speed_p ())
/* Try to use a memory block compare insn - either cmpstr
or cmpmem will do. */
- for (cmp_mode = GET_CLASS_NARROWEST_MODE (MODE_INT);
- cmp_mode != VOIDmode;
- cmp_mode = GET_MODE_WIDER_MODE (cmp_mode))
+ opt_scalar_int_mode cmp_mode_iter;
+ FOR_EACH_MODE_IN_CLASS (cmp_mode_iter, MODE_INT)
{
+ scalar_int_mode cmp_mode = cmp_mode_iter.require ();
cmp_code = direct_optab_handler (cmpmem_optab, cmp_mode);
if (cmp_code == CODE_FOR_nothing)
cmp_code = direct_optab_handler (cmpstr_optab, cmp_mode);
continue;
/* Must make sure the size fits the insn's mode. */
- if ((CONST_INT_P (size)
- && INTVAL (size) >= (1 << GET_MODE_BITSIZE (cmp_mode)))
- || (GET_MODE_BITSIZE (GET_MODE (size))
- > GET_MODE_BITSIZE (cmp_mode)))
+ if (CONST_INT_P (size)
+ ? INTVAL (size) >= (1 << GET_MODE_BITSIZE (cmp_mode))
+ : (GET_MODE_BITSIZE (as_a <scalar_int_mode> (GET_MODE (size)))
+ > GET_MODE_BITSIZE (cmp_mode)))
continue;
result_mode = insn_data[cmp_code].operand[0].mode;
if (cfun->can_throw_non_call_exceptions)
{
if (may_trap_p (x))
- x = force_reg (mode, x);
+ x = copy_to_reg (x);
if (may_trap_p (y))
- y = force_reg (mode, y);
+ y = copy_to_reg (y);
}
if (GET_MODE_CLASS (mode) == MODE_CC)
mclass = GET_MODE_CLASS (mode);
test = gen_rtx_fmt_ee (comparison, VOIDmode, x, y);
- cmp_mode = mode;
- do
- {
+ FOR_EACH_MODE_FROM (cmp_mode, mode)
+ {
enum insn_code icode;
icode = optab_handler (cbranch_optab, cmp_mode);
if (icode != CODE_FOR_nothing
if (methods == OPTAB_DIRECT || !CLASS_HAS_WIDER_MODES_P (mclass))
break;
- cmp_mode = GET_MODE_WIDER_MODE (cmp_mode);
}
- while (cmp_mode != VOIDmode);
if (methods != OPTAB_LIB_WIDEN)
goto fail;
- if (!SCALAR_FLOAT_MODE_P (mode))
+ if (SCALAR_FLOAT_MODE_P (mode))
+ {
+ /* Small trick if UNORDERED isn't implemented by the hardware. */
+ if (comparison == UNORDERED && rtx_equal_p (x, y))
+ {
+ prepare_cmp_insn (x, y, UNLT, NULL_RTX, unsignedp, OPTAB_WIDEN,
+ ptest, pmode);
+ if (*ptest)
+ return;
+ }
+
+ prepare_float_lib_cmp (x, y, comparison, ptest, pmode);
+ }
+ else
{
rtx result;
machine_mode ret_mode;
ret_mode = targetm.libgcc_cmp_return_mode ();
result = emit_library_call_value (libfunc, NULL_RTX, LCT_CONST,
- ret_mode, 2, x, mode, y, mode);
+ ret_mode, x, mode, y, mode);
/* There are two kinds of comparison routines. Biased routines
return 0/1/2, and unbiased routines return -1/0/1. Other parts
prepare_cmp_insn (x, y, comparison, NULL_RTX, unsignedp, methods,
ptest, pmode);
}
- else
- prepare_float_lib_cmp (x, y, comparison, ptest, pmode);
return;
we can do the branch. */
static void
-emit_cmp_and_jump_insn_1 (rtx test, machine_mode mode, rtx label, int prob)
+emit_cmp_and_jump_insn_1 (rtx test, machine_mode mode, rtx label,
+ profile_probability prob)
{
machine_mode optab_mode;
enum mode_class mclass;
gcc_assert (insn_operand_matches (icode, 0, test));
insn = emit_jump_insn (GEN_FCN (icode) (test, XEXP (test, 0),
XEXP (test, 1), label));
- if (prob != -1
+ if (prob.initialized_p ()
&& profile_status_for_fn (cfun) != PROFILE_ABSENT
&& insn
&& JUMP_P (insn)
&& any_condjump_p (insn)
&& !find_reg_note (insn, REG_BR_PROB, 0))
- add_int_reg_note (insn, REG_BR_PROB, prob);
+ add_reg_br_prob_note (insn, prob);
}
/* Generate code to compare X with Y so that the condition codes are
void
emit_cmp_and_jump_insns (rtx x, rtx y, enum rtx_code comparison, rtx size,
machine_mode mode, int unsignedp, rtx label,
- int prob)
+ profile_probability prob)
{
rtx op0 = x, op1 = y;
rtx test;
enum rtx_code swapped = swap_condition (comparison);
enum rtx_code reversed = reverse_condition_maybe_unordered (comparison);
machine_mode orig_mode = GET_MODE (x);
- machine_mode mode, cmp_mode;
+ machine_mode mode;
rtx true_rtx, false_rtx;
rtx value, target, equiv;
rtx_insn *insns;
rtx libfunc = 0;
bool reversed_p = false;
- cmp_mode = targetm.libgcc_cmp_return_mode ();
+ scalar_int_mode cmp_mode = targetm.libgcc_cmp_return_mode ();
- for (mode = orig_mode;
- mode != VOIDmode;
- mode = GET_MODE_WIDER_MODE (mode))
+ FOR_EACH_MODE_FROM (mode, orig_mode)
{
if (code_to_optab (comparison)
&& (libfunc = optab_libfunc (code_to_optab (comparison), mode)))
start_sequence ();
value = emit_library_call_value (libfunc, NULL_RTX, LCT_CONST,
- cmp_mode, 2, x, mode, y, mode);
+ cmp_mode, x, mode, y, mode);
insns = get_insns ();
end_sequence ();
if (cmode == VOIDmode)
cmode = GET_MODE (op0);
+ enum rtx_code orig_code = code;
+ bool swapped = false;
if (swap_commutative_operands_p (op2, op3)
&& ((reversed = reversed_comparison_code_parts (code, op0, op1, NULL))
!= UNKNOWN))
{
std::swap (op2, op3);
code = reversed;
+ swapped = true;
}
if (mode == VOIDmode)
icode = direct_optab_handler (movcc_optab, mode);
if (icode == CODE_FOR_nothing)
- return 0;
+ return NULL_RTX;
if (!target)
target = gen_reg_rtx (mode);
- code = unsignedp ? unsigned_condition (code) : code;
- comparison = simplify_gen_relational (code, VOIDmode, cmode, op0, op1);
-
- /* We can get const0_rtx or const_true_rtx in some circumstances. Just
- return NULL and let the caller figure out how best to deal with this
- situation. */
- if (!COMPARISON_P (comparison))
- return NULL_RTX;
-
- saved_pending_stack_adjust save;
- save_pending_stack_adjust (&save);
- last = get_last_insn ();
- do_pending_stack_adjust ();
- prepare_cmp_insn (XEXP (comparison, 0), XEXP (comparison, 1),
- GET_CODE (comparison), NULL_RTX, unsignedp, OPTAB_WIDEN,
- &comparison, &cmode);
- if (comparison)
+ for (int pass = 0; ; pass++)
{
- struct expand_operand ops[4];
+ code = unsignedp ? unsigned_condition (code) : code;
+ comparison = simplify_gen_relational (code, VOIDmode, cmode, op0, op1);
- create_output_operand (&ops[0], target, mode);
- create_fixed_operand (&ops[1], comparison);
- create_input_operand (&ops[2], op2, mode);
- create_input_operand (&ops[3], op3, mode);
- if (maybe_expand_insn (icode, 4, ops))
+ /* We can get const0_rtx or const_true_rtx in some circumstances. Just
+ punt and let the caller figure out how best to deal with this
+ situation. */
+ if (COMPARISON_P (comparison))
{
- if (ops[0].value != target)
- convert_move (target, ops[0].value, false);
- return target;
+ saved_pending_stack_adjust save;
+ save_pending_stack_adjust (&save);
+ last = get_last_insn ();
+ do_pending_stack_adjust ();
+ machine_mode cmpmode = cmode;
+ prepare_cmp_insn (XEXP (comparison, 0), XEXP (comparison, 1),
+ GET_CODE (comparison), NULL_RTX, unsignedp,
+ OPTAB_WIDEN, &comparison, &cmpmode);
+ if (comparison)
+ {
+ struct expand_operand ops[4];
+
+ create_output_operand (&ops[0], target, mode);
+ create_fixed_operand (&ops[1], comparison);
+ create_input_operand (&ops[2], op2, mode);
+ create_input_operand (&ops[3], op3, mode);
+ if (maybe_expand_insn (icode, 4, ops))
+ {
+ if (ops[0].value != target)
+ convert_move (target, ops[0].value, false);
+ return target;
+ }
+ }
+ delete_insns_since (last);
+ restore_pending_stack_adjust (&save);
}
+
+ if (pass == 1)
+ return NULL_RTX;
+
+ /* If the preferred op2/op3 order is not usable, retry with other
+ operand order, perhaps it will expand successfully. */
+ if (swapped)
+ code = orig_code;
+ else if ((reversed = reversed_comparison_code_parts (orig_code, op0, op1,
+ NULL))
+ != UNKNOWN)
+ code = reversed;
+ else
+ return NULL_RTX;
+ std::swap (op2, op3);
}
- delete_insns_since (last);
- restore_pending_stack_adjust (&save);
- return NULL_RTX;
}
{
enum insn_code icode;
rtx target = to;
+ scalar_mode from_mode, to_mode;
machine_mode fmode, imode;
bool can_do_signed = false;
wider mode. If the integer mode is wider than the mode of FROM,
we can do the conversion signed even if the input is unsigned. */
- for (fmode = GET_MODE (to); fmode != VOIDmode;
- fmode = GET_MODE_WIDER_MODE (fmode))
- for (imode = GET_MODE (from); imode != VOIDmode;
- imode = GET_MODE_WIDER_MODE (imode))
+ FOR_EACH_MODE_FROM (fmode, GET_MODE (to))
+ FOR_EACH_MODE_FROM (imode, GET_MODE (from))
{
int doing_unsigned = unsignedp;
if (fmode != GET_MODE (to)
- && significand_size (fmode) < GET_MODE_PRECISION (GET_MODE (from)))
+ && (significand_size (fmode)
+ < GET_MODE_UNIT_PRECISION (GET_MODE (from))))
continue;
icode = can_float_p (fmode, imode, unsignedp);
/* Unsigned integer, and no way to convert directly. Convert as signed,
then unconditionally adjust the result. */
- if (unsignedp && can_do_signed)
+ if (unsignedp
+ && can_do_signed
+ && is_a <scalar_mode> (GET_MODE (to), &to_mode)
+ && is_a <scalar_mode> (GET_MODE (from), &from_mode))
{
+ opt_scalar_mode fmode_iter;
rtx_code_label *label = gen_label_rtx ();
rtx temp;
REAL_VALUE_TYPE offset;
least as wide as the target. Using FMODE will avoid rounding woes
with unsigned values greater than the signed maximum value. */
- for (fmode = GET_MODE (to); fmode != VOIDmode;
- fmode = GET_MODE_WIDER_MODE (fmode))
- if (GET_MODE_PRECISION (GET_MODE (from)) < GET_MODE_BITSIZE (fmode)
- && can_float_p (fmode, GET_MODE (from), 0) != CODE_FOR_nothing)
- break;
+ FOR_EACH_MODE_FROM (fmode_iter, to_mode)
+ {
+ scalar_mode fmode = fmode_iter.require ();
+ if (GET_MODE_PRECISION (from_mode) < GET_MODE_BITSIZE (fmode)
+ && can_float_p (fmode, from_mode, 0) != CODE_FOR_nothing)
+ break;
+ }
- if (fmode == VOIDmode)
+ if (!fmode_iter.exists (&fmode))
{
/* There is no such mode. Pretend the target is wide enough. */
- fmode = GET_MODE (to);
+ fmode = to_mode;
/* Avoid double-rounding when TO is narrower than FROM. */
if ((significand_size (fmode) + 1)
- < GET_MODE_PRECISION (GET_MODE (from)))
+ < GET_MODE_PRECISION (from_mode))
{
rtx temp1;
rtx_code_label *neglabel = gen_label_rtx ();
|| GET_MODE (target) != fmode)
target = gen_reg_rtx (fmode);
- imode = GET_MODE (from);
+ imode = from_mode;
do_pending_stack_adjust ();
/* Test whether the sign bit is set. */
/* If we are about to do some arithmetic to correct for an
unsigned operand, do it in a pseudo-register. */
- if (GET_MODE (to) != fmode
+ if (to_mode != fmode
|| !REG_P (to) || REGNO (to) < FIRST_PSEUDO_REGISTER)
target = gen_reg_rtx (fmode);
correct its value by 2**bitwidth. */
do_pending_stack_adjust ();
- emit_cmp_and_jump_insns (from, const0_rtx, GE, NULL_RTX, GET_MODE (from),
+ emit_cmp_and_jump_insns (from, const0_rtx, GE, NULL_RTX, from_mode,
0, label);
- real_2expN (&offset, GET_MODE_PRECISION (GET_MODE (from)), fmode);
+ real_2expN (&offset, GET_MODE_PRECISION (from_mode), fmode);
temp = expand_binop (fmode, add_optab, target,
const_double_from_real_value (offset, fmode),
target, 0, OPTAB_LIB_WIDEN);
rtx value;
convert_optab tab = unsignedp ? ufloat_optab : sfloat_optab;
- if (GET_MODE_PRECISION (GET_MODE (from)) < GET_MODE_PRECISION (SImode))
+ if (is_narrower_int_mode (GET_MODE (from), SImode))
from = convert_to_mode (SImode, from, unsignedp);
libfunc = convert_optab_libfunc (tab, GET_MODE (to), GET_MODE (from));
start_sequence ();
value = emit_library_call_value (libfunc, NULL_RTX, LCT_CONST,
- GET_MODE (to), 1, from,
- GET_MODE (from));
+ GET_MODE (to), from, GET_MODE (from));
insns = get_insns ();
end_sequence ();
enum insn_code icode;
rtx target = to;
machine_mode fmode, imode;
+ opt_scalar_mode fmode_iter;
bool must_trunc = false;
/* We first try to find a pair of modes, one real and one integer, at
this conversion. If the integer mode is wider than the mode of TO,
we can do the conversion either signed or unsigned. */
- for (fmode = GET_MODE (from); fmode != VOIDmode;
- fmode = GET_MODE_WIDER_MODE (fmode))
- for (imode = GET_MODE (to); imode != VOIDmode;
- imode = GET_MODE_WIDER_MODE (imode))
+ FOR_EACH_MODE_FROM (fmode, GET_MODE (from))
+ FOR_EACH_MODE_FROM (imode, GET_MODE (to))
{
int doing_unsigned = unsignedp;
2^63. The subtraction of 2^63 should not generate any rounding as it
simply clears out that bit. The rest is trivial. */
- if (unsignedp && GET_MODE_PRECISION (GET_MODE (to)) <= HOST_BITS_PER_WIDE_INT)
- for (fmode = GET_MODE (from); fmode != VOIDmode;
- fmode = GET_MODE_WIDER_MODE (fmode))
- if (CODE_FOR_nothing != can_fix_p (GET_MODE (to), fmode, 0, &must_trunc)
- && (!DECIMAL_FLOAT_MODE_P (fmode)
- || GET_MODE_BITSIZE (fmode) > GET_MODE_PRECISION (GET_MODE (to))))
- {
- int bitsize;
- REAL_VALUE_TYPE offset;
- rtx limit;
- rtx_code_label *lab1, *lab2;
- rtx_insn *insn;
-
- bitsize = GET_MODE_PRECISION (GET_MODE (to));
- real_2expN (&offset, bitsize - 1, fmode);
- limit = const_double_from_real_value (offset, fmode);
- lab1 = gen_label_rtx ();
- lab2 = gen_label_rtx ();
-
- if (fmode != GET_MODE (from))
- from = convert_to_mode (fmode, from, 0);
-
- /* See if we need to do the subtraction. */
- do_pending_stack_adjust ();
- emit_cmp_and_jump_insns (from, limit, GE, NULL_RTX, GET_MODE (from),
- 0, lab1);
-
- /* If not, do the signed "fix" and branch around fixup code. */
- expand_fix (to, from, 0);
- emit_jump_insn (targetm.gen_jump (lab2));
- emit_barrier ();
-
- /* Otherwise, subtract 2**(N-1), convert to signed number,
- then add 2**(N-1). Do the addition using XOR since this
- will often generate better code. */
- emit_label (lab1);
- target = expand_binop (GET_MODE (from), sub_optab, from, limit,
- NULL_RTX, 0, OPTAB_LIB_WIDEN);
- expand_fix (to, target, 0);
- target = expand_binop (GET_MODE (to), xor_optab, to,
- gen_int_mode
- (HOST_WIDE_INT_1 << (bitsize - 1),
- GET_MODE (to)),
- to, 1, OPTAB_LIB_WIDEN);
-
- if (target != to)
- emit_move_insn (to, target);
-
- emit_label (lab2);
-
- if (optab_handler (mov_optab, GET_MODE (to)) != CODE_FOR_nothing)
- {
- /* Make a place for a REG_NOTE and add it. */
- insn = emit_move_insn (to, to);
- set_dst_reg_note (insn, REG_EQUAL,
- gen_rtx_fmt_e (UNSIGNED_FIX, GET_MODE (to),
- copy_rtx (from)),
- to);
- }
+ scalar_int_mode to_mode;
+ if (unsignedp
+ && is_a <scalar_int_mode> (GET_MODE (to), &to_mode)
+ && HWI_COMPUTABLE_MODE_P (to_mode))
+ FOR_EACH_MODE_FROM (fmode_iter, as_a <scalar_mode> (GET_MODE (from)))
+ {
+ scalar_mode fmode = fmode_iter.require ();
+ if (CODE_FOR_nothing != can_fix_p (to_mode, fmode,
+ 0, &must_trunc)
+ && (!DECIMAL_FLOAT_MODE_P (fmode)
+ || (GET_MODE_BITSIZE (fmode) > GET_MODE_PRECISION (to_mode))))
+ {
+ int bitsize;
+ REAL_VALUE_TYPE offset;
+ rtx limit;
+ rtx_code_label *lab1, *lab2;
+ rtx_insn *insn;
+
+ bitsize = GET_MODE_PRECISION (to_mode);
+ real_2expN (&offset, bitsize - 1, fmode);
+ limit = const_double_from_real_value (offset, fmode);
+ lab1 = gen_label_rtx ();
+ lab2 = gen_label_rtx ();
- return;
- }
+ if (fmode != GET_MODE (from))
+ from = convert_to_mode (fmode, from, 0);
+
+ /* See if we need to do the subtraction. */
+ do_pending_stack_adjust ();
+ emit_cmp_and_jump_insns (from, limit, GE, NULL_RTX,
+ GET_MODE (from), 0, lab1);
+
+ /* If not, do the signed "fix" and branch around fixup code. */
+ expand_fix (to, from, 0);
+ emit_jump_insn (targetm.gen_jump (lab2));
+ emit_barrier ();
+
+ /* Otherwise, subtract 2**(N-1), convert to signed number,
+ then add 2**(N-1). Do the addition using XOR since this
+ will often generate better code. */
+ emit_label (lab1);
+ target = expand_binop (GET_MODE (from), sub_optab, from, limit,
+ NULL_RTX, 0, OPTAB_LIB_WIDEN);
+ expand_fix (to, target, 0);
+ target = expand_binop (to_mode, xor_optab, to,
+ gen_int_mode
+ (HOST_WIDE_INT_1 << (bitsize - 1),
+ to_mode),
+ to, 1, OPTAB_LIB_WIDEN);
+
+ if (target != to)
+ emit_move_insn (to, target);
+
+ emit_label (lab2);
+
+ if (optab_handler (mov_optab, to_mode) != CODE_FOR_nothing)
+ {
+ /* Make a place for a REG_NOTE and add it. */
+ insn = emit_move_insn (to, to);
+ set_dst_reg_note (insn, REG_EQUAL,
+ gen_rtx_fmt_e (UNSIGNED_FIX, to_mode,
+ copy_rtx (from)),
+ to);
+ }
+
+ return;
+ }
+ }
/* We can't do it with an insn, so use a library call. But first ensure
that the mode of TO is at least as wide as SImode, since those are the
only library calls we know about. */
- if (GET_MODE_PRECISION (GET_MODE (to)) < GET_MODE_PRECISION (SImode))
+ if (is_narrower_int_mode (GET_MODE (to), SImode))
{
target = gen_reg_rtx (SImode);
start_sequence ();
value = emit_library_call_value (libfunc, NULL_RTX, LCT_CONST,
- GET_MODE (to), 1, from,
- GET_MODE (from));
+ GET_MODE (to), from, GET_MODE (from));
insns = get_insns ();
end_sequence ();
static rtx
prepare_libcall_arg (rtx arg, int uintp)
{
- machine_mode mode = GET_MODE (arg);
+ scalar_int_mode mode;
machine_mode arg_mode;
- if (SCALAR_INT_MODE_P (mode))
+ if (is_a <scalar_int_mode> (GET_MODE (arg), &mode))
{
/* If we need to promote the integer function argument we need to do
it here instead of inside emit_library_call_value because in
start_sequence ();
value = emit_library_call_value (libfunc, NULL_RTX, LCT_CONST, to_mode,
- 1, from, from_mode);
+ from, from_mode);
insns = get_insns ();
end_sequence ();
this conversion. If the integer mode is wider than the mode of TO,
we can do the conversion either signed or unsigned. */
- for (fmode = GET_MODE (from); fmode != VOIDmode;
- fmode = GET_MODE_WIDER_MODE (fmode))
- for (imode = GET_MODE (to); imode != VOIDmode;
- imode = GET_MODE_WIDER_MODE (imode))
+ FOR_EACH_MODE_FROM (fmode, GET_MODE (from))
+ FOR_EACH_MODE_FROM (imode, GET_MODE (to))
{
icode = convert_optab_handler (tab, imode, fmode);
if (icode != CODE_FOR_nothing)
return code;
}
-/* Return comparison rtx for COND. Use UNSIGNEDP to select signed or
- unsigned operators. OPNO holds an index of the first comparison
- operand in insn with code ICODE. Do not generate compare instruction. */
+/* Return a comparison rtx of mode CMP_MODE for COND. Use UNSIGNEDP to
+ select signed or unsigned operators. OPNO holds the index of the
+ first comparison operand for insn ICODE. Do not generate the
+ compare instruction itself. */
static rtx
-vector_compare_rtx (enum tree_code tcode, tree t_op0, tree t_op1,
- bool unsignedp, enum insn_code icode,
- unsigned int opno)
+vector_compare_rtx (machine_mode cmp_mode, enum tree_code tcode,
+ tree t_op0, tree t_op1, bool unsignedp,
+ enum insn_code icode, unsigned int opno)
{
struct expand_operand ops[2];
rtx rtx_op0, rtx_op1;
create_input_operand (&ops[1], rtx_op1, m1);
if (!maybe_legitimize_operands (icode, opno, 2, ops))
gcc_unreachable ();
- return gen_rtx_fmt_ee (rcode, VOIDmode, ops[0].value, ops[1].value);
+ return gen_rtx_fmt_ee (rcode, cmp_mode, ops[0].value, ops[1].value);
}
-/* Checks if vec_perm mask SEL is a constant equivalent to a shift of the first
- vec_perm operand, assuming the second operand is a constant vector of zeroes.
- Return the shift distance in bits if so, or NULL_RTX if the vec_perm is not a
- shift. */
+/* Check if vec_perm mask SEL is a constant equivalent to a shift of
+ the first vec_perm operand, assuming the second operand is a constant
+ vector of zeros. Return the shift distance in bits if so, or NULL_RTX
+ if the vec_perm is not a shift. MODE is the mode of the value being
+ shifted. */
static rtx
-shift_amt_for_vec_perm_mask (rtx sel)
+shift_amt_for_vec_perm_mask (machine_mode mode, const vec_perm_indices &sel)
{
- unsigned int i, first, nelt = GET_MODE_NUNITS (GET_MODE (sel));
- unsigned int bitsize = GET_MODE_UNIT_BITSIZE (GET_MODE (sel));
-
- if (GET_CODE (sel) != CONST_VECTOR)
+ unsigned int bitsize = GET_MODE_UNIT_BITSIZE (mode);
+ poly_int64 first = sel[0];
+ if (maybe_ge (sel[0], GET_MODE_NUNITS (mode)))
return NULL_RTX;
- first = INTVAL (CONST_VECTOR_ELT (sel, 0));
- if (first >= nelt)
- return NULL_RTX;
- for (i = 1; i < nelt; i++)
+ if (!sel.series_p (0, 1, first, 1))
{
- int idx = INTVAL (CONST_VECTOR_ELT (sel, i));
- unsigned int expected = i + first;
- /* Indices into the second vector are all equivalent. */
- if (idx < 0 || (MIN (nelt, (unsigned) idx) != MIN (nelt, expected)))
+ unsigned int nelt;
+ if (!GET_MODE_NUNITS (mode).is_constant (&nelt))
return NULL_RTX;
+ for (unsigned int i = 1; i < nelt; i++)
+ {
+ poly_int64 expected = i + first;
+ /* Indices into the second vector are all equivalent. */
+ if (maybe_lt (sel[i], nelt)
+ ? maybe_ne (sel[i], expected)
+ : maybe_lt (expected, nelt))
+ return NULL_RTX;
+ }
}
- return GEN_INT (first * bitsize);
+ return gen_int_shift_amount (mode, first * bitsize);
}
-/* A subroutine of expand_vec_perm for expanding one vec_perm insn. */
+/* A subroutine of expand_vec_perm_var for expanding one vec_perm insn. */
static rtx
expand_vec_perm_1 (enum insn_code icode, rtx target,
machine_mode smode = GET_MODE (sel);
struct expand_operand ops[4];
+ gcc_assert (GET_MODE_CLASS (smode) == MODE_VECTOR_INT
+ || mode_for_int_vector (tmode).require () == smode);
create_output_operand (&ops[0], target, tmode);
create_input_operand (&ops[3], sel, smode);
return NULL_RTX;
}
-/* Generate instructions for vec_perm optab given its mode
- and three operands. */
+/* Implement a permutation of vectors v0 and v1 using the permutation
+ vector in SEL and return the result. Use TARGET to hold the result
+ if nonnull and convenient.
+
+ MODE is the mode of the vectors being permuted (V0 and V1). SEL_MODE
+ is the TYPE_MODE associated with SEL, or BLKmode if SEL isn't known
+ to have a particular mode. */
rtx
-expand_vec_perm (machine_mode mode, rtx v0, rtx v1, rtx sel, rtx target)
+expand_vec_perm_const (machine_mode mode, rtx v0, rtx v1,
+ const vec_perm_builder &sel, machine_mode sel_mode,
+ rtx target)
{
- enum insn_code icode;
- machine_mode qimode;
- unsigned int i, w, e, u;
- rtx tmp, sel_qi = NULL;
- rtvec vec;
-
- if (!target || GET_MODE (target) != mode)
+ if (!target || !register_operand (target, mode))
target = gen_reg_rtx (mode);
- w = GET_MODE_SIZE (mode);
- e = GET_MODE_NUNITS (mode);
- u = GET_MODE_UNIT_SIZE (mode);
-
/* Set QIMODE to a different vector mode with byte elements.
If no such mode, or if MODE already has byte elements, use VOIDmode. */
- qimode = VOIDmode;
- if (GET_MODE_INNER (mode) != QImode)
- {
- qimode = mode_for_vector (QImode, w);
- if (!VECTOR_MODE_P (qimode))
- qimode = VOIDmode;
- }
-
- /* If the input is a constant, expand it specially. */
- gcc_assert (GET_MODE_CLASS (GET_MODE (sel)) == MODE_VECTOR_INT);
- if (GET_CODE (sel) == CONST_VECTOR)
- {
- /* See if this can be handled with a vec_shr. We only do this if the
- second vector is all zeroes. */
- enum insn_code shift_code = optab_handler (vec_shr_optab, mode);
- enum insn_code shift_code_qi = ((qimode != VOIDmode && qimode != mode)
- ? optab_handler (vec_shr_optab, qimode)
- : CODE_FOR_nothing);
- rtx shift_amt = NULL_RTX;
- if (v1 == CONST0_RTX (GET_MODE (v1))
- && (shift_code != CODE_FOR_nothing
- || shift_code_qi != CODE_FOR_nothing))
+ machine_mode qimode;
+ if (!qimode_for_vec_perm (mode).exists (&qimode))
+ qimode = VOIDmode;
+
+ rtx_insn *last = get_last_insn ();
+
+ bool single_arg_p = rtx_equal_p (v0, v1);
+ /* Always specify two input vectors here and leave the target to handle
+ cases in which the inputs are equal. Not all backends can cope with
+ the single-input representation when testing for a double-input
+ target instruction. */
+ vec_perm_indices indices (sel, 2, GET_MODE_NUNITS (mode));
+
+ /* See if this can be handled with a vec_shr. We only do this if the
+ second vector is all zeroes. */
+ insn_code shift_code = optab_handler (vec_shr_optab, mode);
+ insn_code shift_code_qi = ((qimode != VOIDmode && qimode != mode)
+ ? optab_handler (vec_shr_optab, qimode)
+ : CODE_FOR_nothing);
+
+ if (v1 == CONST0_RTX (GET_MODE (v1))
+ && (shift_code != CODE_FOR_nothing
+ || shift_code_qi != CODE_FOR_nothing))
+ {
+ rtx shift_amt = shift_amt_for_vec_perm_mask (mode, indices);
+ if (shift_amt)
{
- shift_amt = shift_amt_for_vec_perm_mask (sel);
- if (shift_amt)
+ struct expand_operand ops[3];
+ if (shift_code != CODE_FOR_nothing)
{
- struct expand_operand ops[3];
- if (shift_code != CODE_FOR_nothing)
- {
- create_output_operand (&ops[0], target, mode);
- create_input_operand (&ops[1], v0, mode);
- create_convert_operand_from_type (&ops[2], shift_amt,
- sizetype);
- if (maybe_expand_insn (shift_code, 3, ops))
- return ops[0].value;
- }
- if (shift_code_qi != CODE_FOR_nothing)
- {
- tmp = gen_reg_rtx (qimode);
- create_output_operand (&ops[0], tmp, qimode);
- create_input_operand (&ops[1], gen_lowpart (qimode, v0),
- qimode);
- create_convert_operand_from_type (&ops[2], shift_amt,
- sizetype);
- if (maybe_expand_insn (shift_code_qi, 3, ops))
- return gen_lowpart (mode, ops[0].value);
- }
+ create_output_operand (&ops[0], target, mode);
+ create_input_operand (&ops[1], v0, mode);
+ create_convert_operand_from_type (&ops[2], shift_amt, sizetype);
+ if (maybe_expand_insn (shift_code, 3, ops))
+ return ops[0].value;
+ }
+ if (shift_code_qi != CODE_FOR_nothing)
+ {
+ rtx tmp = gen_reg_rtx (qimode);
+ create_output_operand (&ops[0], tmp, qimode);
+ create_input_operand (&ops[1], gen_lowpart (qimode, v0), qimode);
+ create_convert_operand_from_type (&ops[2], shift_amt, sizetype);
+ if (maybe_expand_insn (shift_code_qi, 3, ops))
+ return gen_lowpart (mode, ops[0].value);
}
}
+ }
+
+ if (targetm.vectorize.vec_perm_const != NULL)
+ {
+ v0 = force_reg (mode, v0);
+ if (single_arg_p)
+ v1 = v0;
+ else
+ v1 = force_reg (mode, v1);
+
+ if (targetm.vectorize.vec_perm_const (mode, target, v0, v1, indices))
+ return target;
+ }
+
+ /* Fall back to a constant byte-based permutation. */
+ vec_perm_indices qimode_indices;
+ rtx target_qi = NULL_RTX, v0_qi = NULL_RTX, v1_qi = NULL_RTX;
+ if (qimode != VOIDmode)
+ {
+ qimode_indices.new_expanded_vector (indices, GET_MODE_UNIT_SIZE (mode));
+ target_qi = gen_reg_rtx (qimode);
+ v0_qi = gen_lowpart (qimode, v0);
+ v1_qi = gen_lowpart (qimode, v1);
+ if (targetm.vectorize.vec_perm_const != NULL
+ && targetm.vectorize.vec_perm_const (qimode, target_qi, v0_qi,
+ v1_qi, qimode_indices))
+ return gen_lowpart (mode, target_qi);
+ }
+
+ /* Otherwise expand as a fully variable permuation. */
+
+ /* The optabs are only defined for selectors with the same width
+ as the values being permuted. */
+ machine_mode required_sel_mode;
+ if (!mode_for_int_vector (mode).exists (&required_sel_mode)
+ || !VECTOR_MODE_P (required_sel_mode))
+ {
+ delete_insns_since (last);
+ return NULL_RTX;
+ }
+
+ /* We know that it is semantically valid to treat SEL as having SEL_MODE.
+ If that isn't the mode we want then we need to prove that using
+ REQUIRED_SEL_MODE is OK. */
+ if (sel_mode != required_sel_mode)
+ {
+ if (!selector_fits_mode_p (required_sel_mode, indices))
+ {
+ delete_insns_since (last);
+ return NULL_RTX;
+ }
+ sel_mode = required_sel_mode;
+ }
+
+ insn_code icode = direct_optab_handler (vec_perm_optab, mode);
+ if (icode != CODE_FOR_nothing)
+ {
+ rtx sel_rtx = vec_perm_indices_to_rtx (sel_mode, indices);
+ rtx tmp = expand_vec_perm_1 (icode, target, v0, v1, sel_rtx);
+ if (tmp)
+ return tmp;
+ }
- icode = direct_optab_handler (vec_perm_const_optab, mode);
+ if (qimode != VOIDmode
+ && selector_fits_mode_p (qimode, qimode_indices))
+ {
+ icode = direct_optab_handler (vec_perm_optab, qimode);
if (icode != CODE_FOR_nothing)
{
- tmp = expand_vec_perm_1 (icode, target, v0, v1, sel);
+ rtx sel_qi = vec_perm_indices_to_rtx (qimode, qimode_indices);
+ rtx tmp = expand_vec_perm_1 (icode, target_qi, v0_qi, v1_qi, sel_qi);
if (tmp)
- return tmp;
+ return gen_lowpart (mode, tmp);
}
+ }
- /* Fall back to a constant byte-based permutation. */
- if (qimode != VOIDmode)
- {
- vec = rtvec_alloc (w);
- for (i = 0; i < e; ++i)
- {
- unsigned int j, this_e;
+ delete_insns_since (last);
+ return NULL_RTX;
+}
- this_e = INTVAL (CONST_VECTOR_ELT (sel, i));
- this_e &= 2 * e - 1;
- this_e *= u;
+/* Implement a permutation of vectors v0 and v1 using the permutation
+ vector in SEL and return the result. Use TARGET to hold the result
+ if nonnull and convenient.
- for (j = 0; j < u; ++j)
- RTVEC_ELT (vec, i * u + j) = GEN_INT (this_e + j);
- }
- sel_qi = gen_rtx_CONST_VECTOR (qimode, vec);
+ MODE is the mode of the vectors being permuted (V0 and V1).
+ SEL must have the integer equivalent of MODE and is known to be
+ unsuitable for permutes with a constant permutation vector. */
- icode = direct_optab_handler (vec_perm_const_optab, qimode);
- if (icode != CODE_FOR_nothing)
- {
- tmp = mode != qimode ? gen_reg_rtx (qimode) : target;
- tmp = expand_vec_perm_1 (icode, tmp, gen_lowpart (qimode, v0),
- gen_lowpart (qimode, v1), sel_qi);
- if (tmp)
- return gen_lowpart (mode, tmp);
- }
- }
- }
+rtx
+expand_vec_perm_var (machine_mode mode, rtx v0, rtx v1, rtx sel, rtx target)
+{
+ enum insn_code icode;
+ unsigned int i, u;
+ rtx tmp, sel_qi;
+
+ u = GET_MODE_UNIT_SIZE (mode);
+
+ if (!target || GET_MODE (target) != mode)
+ target = gen_reg_rtx (mode);
- /* Otherwise expand as a fully variable permuation. */
icode = direct_optab_handler (vec_perm_optab, mode);
if (icode != CODE_FOR_nothing)
{
/* As a special case to aid several targets, lower the element-based
permutation to a byte-based permutation and try again. */
- if (qimode == VOIDmode)
+ machine_mode qimode;
+ if (!qimode_for_vec_perm (mode).exists (&qimode)
+ || maybe_gt (GET_MODE_NUNITS (qimode), GET_MODE_MASK (QImode) + 1))
return NULL_RTX;
icode = direct_optab_handler (vec_perm_optab, qimode);
if (icode == CODE_FOR_nothing)
return NULL_RTX;
- if (sel_qi == NULL)
- {
- /* Multiply each element by its byte size. */
- machine_mode selmode = GET_MODE (sel);
- if (u == 2)
- sel = expand_simple_binop (selmode, PLUS, sel, sel,
- NULL, 0, OPTAB_DIRECT);
- else
- sel = expand_simple_binop (selmode, ASHIFT, sel,
- GEN_INT (exact_log2 (u)),
- NULL, 0, OPTAB_DIRECT);
- gcc_assert (sel != NULL);
-
- /* Broadcast the low byte each element into each of its bytes. */
- vec = rtvec_alloc (w);
- for (i = 0; i < w; ++i)
- {
- int this_e = i / u * u;
- if (BYTES_BIG_ENDIAN)
- this_e += u - 1;
- RTVEC_ELT (vec, i) = GEN_INT (this_e);
- }
- tmp = gen_rtx_CONST_VECTOR (qimode, vec);
- sel = gen_lowpart (qimode, sel);
- sel = expand_vec_perm (qimode, sel, sel, tmp, NULL);
- gcc_assert (sel != NULL);
-
- /* Add the byte offset to each byte element. */
- /* Note that the definition of the indicies here is memory ordering,
- so there should be no difference between big and little endian. */
- vec = rtvec_alloc (w);
- for (i = 0; i < w; ++i)
- RTVEC_ELT (vec, i) = GEN_INT (i % u);
- tmp = gen_rtx_CONST_VECTOR (qimode, vec);
- sel_qi = expand_simple_binop (qimode, PLUS, sel, tmp,
- sel, 0, OPTAB_DIRECT);
- gcc_assert (sel_qi != NULL);
- }
+ /* Multiply each element by its byte size. */
+ machine_mode selmode = GET_MODE (sel);
+ if (u == 2)
+ sel = expand_simple_binop (selmode, PLUS, sel, sel,
+ NULL, 0, OPTAB_DIRECT);
+ else
+ sel = expand_simple_binop (selmode, ASHIFT, sel,
+ gen_int_shift_amount (selmode, exact_log2 (u)),
+ NULL, 0, OPTAB_DIRECT);
+ gcc_assert (sel != NULL);
+
+ /* Broadcast the low byte each element into each of its bytes.
+ The encoding has U interleaved stepped patterns, one for each
+ byte of an element. */
+ vec_perm_builder const_sel (GET_MODE_SIZE (mode), u, 3);
+ unsigned int low_byte_in_u = BYTES_BIG_ENDIAN ? u - 1 : 0;
+ for (i = 0; i < 3; ++i)
+ for (unsigned int j = 0; j < u; ++j)
+ const_sel.quick_push (i * u + low_byte_in_u);
+ sel = gen_lowpart (qimode, sel);
+ sel = expand_vec_perm_const (qimode, sel, sel, const_sel, qimode, NULL);
+ gcc_assert (sel != NULL);
+
+ /* Add the byte offset to each byte element. */
+ /* Note that the definition of the indicies here is memory ordering,
+ so there should be no difference between big and little endian. */
+ rtx_vector_builder byte_indices (qimode, u, 1);
+ for (i = 0; i < u; ++i)
+ byte_indices.quick_push (GEN_INT (i));
+ tmp = byte_indices.build ();
+ sel_qi = expand_simple_binop (qimode, PLUS, sel, tmp,
+ sel, 0, OPTAB_DIRECT);
+ gcc_assert (sel_qi != NULL);
tmp = mode != qimode ? gen_reg_rtx (qimode) : target;
tmp = expand_vec_perm_1 (icode, tmp, gen_lowpart (qimode, v0),
unsignedp = TYPE_UNSIGNED (TREE_TYPE (op0a));
- gcc_assert (GET_MODE_SIZE (mode) == GET_MODE_SIZE (cmp_op_mode)
- && GET_MODE_NUNITS (mode) == GET_MODE_NUNITS (cmp_op_mode));
+ gcc_assert (known_eq (GET_MODE_SIZE (mode), GET_MODE_SIZE (cmp_op_mode))
+ && known_eq (GET_MODE_NUNITS (mode),
+ GET_MODE_NUNITS (cmp_op_mode)));
icode = get_vcond_icode (mode, cmp_op_mode, unsignedp);
if (icode == CODE_FOR_nothing)
- return 0;
+ {
+ if (tcode == EQ_EXPR || tcode == NE_EXPR)
+ icode = get_vcond_eq_icode (mode, cmp_op_mode);
+ if (icode == CODE_FOR_nothing)
+ return 0;
+ }
- comparison = vector_compare_rtx (tcode, op0a, op0b, unsignedp, icode, 4);
+ comparison = vector_compare_rtx (VOIDmode, tcode, op0a, op0b, unsignedp,
+ icode, 4);
rtx_op1 = expand_normal (op1);
rtx_op2 = expand_normal (op2);
return ops[0].value;
}
+/* Generate VEC_SERIES_EXPR <OP0, OP1>, returning a value of mode VMODE.
+ Use TARGET for the result if nonnull and convenient. */
+
+rtx
+expand_vec_series_expr (machine_mode vmode, rtx op0, rtx op1, rtx target)
+{
+ struct expand_operand ops[3];
+ enum insn_code icode;
+ machine_mode emode = GET_MODE_INNER (vmode);
+
+ icode = direct_optab_handler (vec_series_optab, vmode);
+ gcc_assert (icode != CODE_FOR_nothing);
+
+ create_output_operand (&ops[0], target, vmode);
+ create_input_operand (&ops[1], op0, emode);
+ create_input_operand (&ops[2], op1, emode);
+
+ expand_insn (icode, 3, ops);
+ return ops[0].value;
+}
+
/* Generate insns for a vector comparison into a mask. */
rtx
icode = get_vec_cmp_icode (vmode, mask_mode, unsignedp);
if (icode == CODE_FOR_nothing)
- return 0;
+ {
+ if (tcode == EQ_EXPR || tcode == NE_EXPR)
+ icode = get_vec_cmp_eq_icode (vmode, mask_mode);
+ if (icode == CODE_FOR_nothing)
+ return 0;
+ }
- comparison = vector_compare_rtx (tcode, op0a, op0b, unsignedp, icode, 2);
+ comparison = vector_compare_rtx (mask_mode, tcode, op0a, op0b,
+ unsignedp, icode, 2);
create_output_operand (&ops[0], target, mask_mode);
create_fixed_operand (&ops[1], comparison);
create_fixed_operand (&ops[2], XEXP (comparison, 0));
{
struct expand_operand eops[3];
enum insn_code icode;
- int method, i, nunits;
+ int method, i;
machine_mode wmode;
- rtx m1, m2, perm;
+ rtx m1, m2;
optab tab1, tab2;
- rtvec v;
method = can_mult_highpart_p (mode, uns_p);
switch (method)
}
icode = optab_handler (tab1, mode);
- nunits = GET_MODE_NUNITS (mode);
wmode = insn_data[icode].operand[0].mode;
- gcc_checking_assert (2 * GET_MODE_NUNITS (wmode) == nunits);
- gcc_checking_assert (GET_MODE_SIZE (wmode) == GET_MODE_SIZE (mode));
+ gcc_checking_assert (known_eq (2 * GET_MODE_NUNITS (wmode),
+ GET_MODE_NUNITS (mode)));
+ gcc_checking_assert (known_eq (GET_MODE_SIZE (wmode), GET_MODE_SIZE (mode)));
create_output_operand (&eops[0], gen_reg_rtx (wmode), wmode);
create_input_operand (&eops[1], op0, mode);
expand_insn (optab_handler (tab2, mode), 3, eops);
m2 = gen_lowpart (mode, eops[0].value);
- v = rtvec_alloc (nunits);
+ vec_perm_builder sel;
if (method == 2)
{
- for (i = 0; i < nunits; ++i)
- RTVEC_ELT (v, i) = GEN_INT (!BYTES_BIG_ENDIAN + (i & ~1)
- + ((i & 1) ? nunits : 0));
+ /* The encoding has 2 interleaved stepped patterns. */
+ sel.new_vector (GET_MODE_NUNITS (mode), 2, 3);
+ for (i = 0; i < 6; ++i)
+ sel.quick_push (!BYTES_BIG_ENDIAN + (i & ~1)
+ + ((i & 1) ? GET_MODE_NUNITS (mode) : 0));
}
else
{
- for (i = 0; i < nunits; ++i)
- RTVEC_ELT (v, i) = GEN_INT (2 * i + (BYTES_BIG_ENDIAN ? 0 : 1));
+ /* The encoding has a single interleaved stepped pattern. */
+ sel.new_vector (GET_MODE_NUNITS (mode), 1, 3);
+ for (i = 0; i < 3; ++i)
+ sel.quick_push (2 * i + (BYTES_BIG_ENDIAN ? 0 : 1));
}
- perm = gen_rtx_CONST_VECTOR (mode, v);
- return expand_vec_perm (mode, m1, m2, perm, target);
+ return expand_vec_perm_const (mode, m1, m2, sel, BLKmode, target);
}
\f
/* Helper function to find the MODE_CC set in a sync_compare_and_swap
/* Mark this jump predicted not taken. */
emit_cmp_and_jump_insns (success, const0_rtx, EQ, const0_rtx,
- GET_MODE (success), 1, label, 0);
+ GET_MODE (success), 1, label,
+ profile_probability::guessed_never ());
return true;
}
addr = convert_memory_address (ptr_mode, XEXP (mem, 0));
return emit_library_call_value (libfunc, NULL_RTX, LCT_NORMAL,
- mode, 2, addr, ptr_mode,
+ mode, addr, ptr_mode,
val, mode);
}
}
rtx
expand_atomic_exchange (rtx target, rtx mem, rtx val, enum memmodel model)
{
+ machine_mode mode = GET_MODE (mem);
rtx ret;
+ /* If loads are not atomic for the required size and we are not called to
+ provide a __sync builtin, do not do anything so that we stay consistent
+ with atomic loads of the same size. */
+ if (!can_atomic_load_p (mode) && !is_mm_sync (model))
+ return NULL_RTX;
+
ret = maybe_emit_atomic_exchange (target, mem, val, model);
/* Next try a compare-and-swap loop for the exchange. */
rtx target_oval, target_bool = NULL_RTX;
rtx libfunc;
+ /* If loads are not atomic for the required size and we are not called to
+ provide a __sync builtin, do not do anything so that we stay consistent
+ with atomic loads of the same size. */
+ if (!can_atomic_load_p (mode) && !is_mm_sync (succ_model))
+ return false;
+
/* Load expected into a register for the compare and swap. */
if (MEM_P (expected))
expected = copy_to_reg (expected);
{
rtx addr = convert_memory_address (ptr_mode, XEXP (mem, 0));
rtx target = emit_library_call_value (libfunc, NULL_RTX, LCT_NORMAL,
- mode, 3, addr, ptr_mode,
+ mode, addr, ptr_mode,
expected, mode, desired, mode);
emit_move_insn (target_oval, target);
return true;
}
-/* Generate asm volatile("" : : : "memory") as the memory barrier. */
+/* Generate asm volatile("" : : : "memory") as the memory blockage. */
static void
-expand_asm_memory_barrier (void)
+expand_asm_memory_blockage (void)
{
rtx asm_op, clob;
- asm_op = gen_rtx_ASM_OPERANDS (VOIDmode, empty_string, empty_string, 0,
+ asm_op = gen_rtx_ASM_OPERANDS (VOIDmode, "", "", 0,
rtvec_alloc (0), rtvec_alloc (0),
rtvec_alloc (0), UNKNOWN_LOCATION);
MEM_VOLATILE_P (asm_op) = 1;
emit_insn (gen_rtx_PARALLEL (VOIDmode, gen_rtvec (2, asm_op, clob)));
}
+/* Do not propagate memory accesses across this point. */
+
+static void
+expand_memory_blockage (void)
+{
+ if (targetm.have_memory_blockage ())
+ emit_insn (targetm.gen_memory_blockage ());
+ else
+ expand_asm_memory_blockage ();
+}
+
/* This routine will either emit the mem_thread_fence pattern or issue a
sync_synchronize to generate a fence for memory model MEMMODEL. */
void
expand_mem_thread_fence (enum memmodel model)
{
+ if (is_mm_relaxed (model))
+ return;
if (targetm.have_mem_thread_fence ())
- emit_insn (targetm.gen_mem_thread_fence (GEN_INT (model)));
- else if (!is_mm_relaxed (model))
{
- if (targetm.have_memory_barrier ())
- emit_insn (targetm.gen_memory_barrier ());
- else if (synchronize_libfunc != NULL_RTX)
- emit_library_call (synchronize_libfunc, LCT_NORMAL, VOIDmode, 0);
- else
- expand_asm_memory_barrier ();
+ emit_insn (targetm.gen_mem_thread_fence (GEN_INT (model)));
+ expand_memory_blockage ();
}
+ else if (targetm.have_memory_barrier ())
+ emit_insn (targetm.gen_memory_barrier ());
+ else if (synchronize_libfunc != NULL_RTX)
+ emit_library_call (synchronize_libfunc, LCT_NORMAL, VOIDmode);
+ else
+ expand_memory_blockage ();
}
-/* This routine will either emit the mem_signal_fence pattern or issue a
- sync_synchronize to generate a fence for memory model MEMMODEL. */
+/* Emit a signal fence with given memory model. */
void
expand_mem_signal_fence (enum memmodel model)
{
- if (targetm.have_mem_signal_fence ())
- emit_insn (targetm.gen_mem_signal_fence (GEN_INT (model)));
- else if (!is_mm_relaxed (model))
- {
- /* By default targets are coherent between a thread and the signal
- handler running on the same thread. Thus this really becomes a
- compiler barrier, in that stores must not be sunk past
- (or raised above) a given point. */
- expand_asm_memory_barrier ();
- }
+ /* No machine barrier is required to implement a signal fence, but
+ a compiler memory barrier must be issued, except for relaxed MM. */
+ if (!is_mm_relaxed (model))
+ expand_memory_blockage ();
}
/* This function expands the atomic load operation:
if (icode != CODE_FOR_nothing)
{
struct expand_operand ops[3];
+ rtx_insn *last = get_last_insn ();
+ if (is_mm_seq_cst (model))
+ expand_memory_blockage ();
create_output_operand (&ops[0], target, mode);
create_fixed_operand (&ops[1], mem);
create_integer_operand (&ops[2], model);
if (maybe_expand_insn (icode, 3, ops))
- return ops[0].value;
+ {
+ if (!is_mm_relaxed (model))
+ expand_memory_blockage ();
+ return ops[0].value;
+ }
+ delete_insns_since (last);
}
/* If the size of the object is greater than word size on this target,
- then we assume that a load will not be atomic. */
- if (GET_MODE_PRECISION (mode) > BITS_PER_WORD)
- {
- /* Issue val = compare_and_swap (mem, 0, 0).
- This may cause the occasional harmless store of 0 when the value is
- already 0, but it seems to be OK according to the standards guys. */
- if (expand_atomic_compare_and_swap (NULL, &target, mem, const0_rtx,
- const0_rtx, false, model, model))
- return target;
- else
- /* Otherwise there is no atomic load, leave the library call. */
- return NULL_RTX;
- }
+ then we assume that a load will not be atomic. We could try to
+ emulate a load with a compare-and-swap operation, but the store that
+ doing this could result in would be incorrect if this is a volatile
+ atomic load or targetting read-only-mapped memory. */
+ if (maybe_gt (GET_MODE_PRECISION (mode), BITS_PER_WORD))
+ /* If there is no atomic load, leave the library call. */
+ return NULL_RTX;
/* Otherwise assume loads are atomic, and emit the proper barriers. */
if (!target || target == const0_rtx)
icode = direct_optab_handler (atomic_store_optab, mode);
if (icode != CODE_FOR_nothing)
{
+ rtx_insn *last = get_last_insn ();
+ if (!is_mm_relaxed (model))
+ expand_memory_blockage ();
create_fixed_operand (&ops[0], mem);
create_input_operand (&ops[1], val, mode);
create_integer_operand (&ops[2], model);
if (maybe_expand_insn (icode, 3, ops))
- return const0_rtx;
+ {
+ if (is_mm_seq_cst (model))
+ expand_memory_blockage ();
+ return const0_rtx;
+ }
+ delete_insns_since (last);
}
- /* If using __sync_lock_release is a viable alternative, try it. */
+ /* If using __sync_lock_release is a viable alternative, try it.
+ Note that this will not be set to true if we are expanding a generic
+ __atomic_store_n. */
if (use_release)
{
icode = direct_optab_handler (sync_lock_release_optab, mode);
}
/* If the size of the object is greater than word size on this target,
- a default store will not be atomic, Try a mem_exchange and throw away
- the result. If that doesn't work, don't do anything. */
- if (GET_MODE_PRECISION (mode) > BITS_PER_WORD)
- {
- rtx target = maybe_emit_atomic_exchange (NULL_RTX, mem, val, model);
- if (!target)
- target = maybe_emit_compare_and_swap_exchange_loop (NULL_RTX, mem, val);
- if (target)
- return const0_rtx;
- else
+ a default store will not be atomic. */
+ if (maybe_gt (GET_MODE_PRECISION (mode), BITS_PER_WORD))
+ {
+ /* If loads are atomic or we are called to provide a __sync builtin,
+ we can try a atomic_exchange and throw away the result. Otherwise,
+ don't do anything so that we do not create an inconsistency between
+ loads and stores. */
+ if (can_atomic_load_p (mode) || is_mm_sync (model))
+ {
+ rtx target = maybe_emit_atomic_exchange (NULL_RTX, mem, val, model);
+ if (!target)
+ target = maybe_emit_compare_and_swap_exchange_loop (NULL_RTX, mem,
+ val);
+ if (target)
+ return const0_rtx;
+ }
return NULL_RTX;
}
rtx result;
bool unused_result = (target == const0_rtx);
+ /* If loads are not atomic for the required size and we are not called to
+ provide a __sync builtin, do not do anything so that we stay consistent
+ with atomic loads of the same size. */
+ if (!can_atomic_load_p (mode) && !is_mm_sync (model))
+ return NULL_RTX;
+
result = expand_atomic_fetch_op_no_fallback (target, mem, val, code, model,
after);
{
rtx addr = convert_memory_address (ptr_mode, XEXP (mem, 0));
result = emit_library_call_value (libfunc, NULL, LCT_NORMAL, mode,
- 2, addr, ptr_mode, val, mode);
+ addr, ptr_mode, val, mode);
if (!unused_result && fixup)
result = expand_simple_binop (mode, code, result, val, target,
valid_multiword_target_p (rtx target)
{
machine_mode mode;
- int i;
+ int i, size;
mode = GET_MODE (target);
- for (i = 0; i < GET_MODE_SIZE (mode); i += UNITS_PER_WORD)
+ if (!GET_MODE_SIZE (mode).is_constant (&size))
+ return false;
+ for (i = 0; i < size; i += UNITS_PER_WORD)
if (!validate_subreg (word_mode, mode, target, i))
return false;
return true;
}
+/* Make OP describe an input operand that has value INTVAL and that has
+ no inherent mode. This function should only be used for operands that
+ are always expand-time constants. The backend may request that INTVAL
+ be copied into a different kind of rtx, but it must specify the mode
+ of that rtx if so. */
+
+void
+create_integer_operand (struct expand_operand *op, poly_int64 intval)
+{
+ create_expand_operand (op, EXPAND_INTEGER,
+ gen_int_mode (intval, MAX_MODE_INT),
+ VOIDmode, false, intval);
+}
+
/* Like maybe_legitimize_operand, but do not change the code of the
current rtx value. */
return true;
op->value = gen_reg_rtx (mode);
+ op->target = 0;
break;
case EXPAND_INPUT:
goto input;
case EXPAND_ADDRESS:
- gcc_assert (mode != VOIDmode);
- op->value = convert_memory_address (mode, op->value);
+ op->value = convert_memory_address (as_a <scalar_int_mode> (mode),
+ op->value);
goto input;
case EXPAND_INTEGER:
mode = insn_data[(int) icode].operand[opno].mode;
- if (mode != VOIDmode && const_int_operand (op->value, mode))
- goto input;
+ if (mode != VOIDmode
+ && known_eq (trunc_int_for_mode (op->int_value, mode),
+ op->int_value))
+ {
+ op->value = gen_int_mode (op->int_value, mode);
+ goto input;
+ }
break;
}
return insn_operand_matches (icode, opno, op->value);
TYPE_UNSIGNED (type));
}
+/* Return true if the requirements on operands OP1 and OP2 of instruction
+ ICODE are similar enough for the result of legitimizing OP1 to be
+ reusable for OP2. OPNO1 and OPNO2 are the operand numbers associated
+ with OP1 and OP2 respectively. */
+
+static inline bool
+can_reuse_operands_p (enum insn_code icode,
+ unsigned int opno1, unsigned int opno2,
+ const struct expand_operand *op1,
+ const struct expand_operand *op2)
+{
+ /* Check requirements that are common to all types. */
+ if (op1->type != op2->type
+ || op1->mode != op2->mode
+ || (insn_data[(int) icode].operand[opno1].mode
+ != insn_data[(int) icode].operand[opno2].mode))
+ return false;
+
+ /* Check the requirements for specific types. */
+ switch (op1->type)
+ {
+ case EXPAND_OUTPUT:
+ /* Outputs must remain distinct. */
+ return false;
+
+ case EXPAND_FIXED:
+ case EXPAND_INPUT:
+ case EXPAND_ADDRESS:
+ case EXPAND_INTEGER:
+ return true;
+
+ case EXPAND_CONVERT_TO:
+ case EXPAND_CONVERT_FROM:
+ return op1->unsigned_p == op2->unsigned_p;
+ }
+ gcc_unreachable ();
+}
+
/* Try to make operands [OPS, OPS + NOPS) match operands [OPNO, OPNO + NOPS)
of instruction ICODE. Return true on success, leaving the new operand
values in the OPS themselves. Emit no code on failure. */
maybe_legitimize_operands (enum insn_code icode, unsigned int opno,
unsigned int nops, struct expand_operand *ops)
{
- rtx_insn *last;
- unsigned int i;
+ rtx_insn *last = get_last_insn ();
+ rtx *orig_values = XALLOCAVEC (rtx, nops);
+ for (unsigned int i = 0; i < nops; i++)
+ {
+ orig_values[i] = ops[i].value;
+
+ /* First try reusing the result of an earlier legitimization.
+ This avoids duplicate rtl and ensures that tied operands
+ remain tied.
+
+ This search is linear, but NOPS is bounded at compile time
+ to a small number (current a single digit). */
+ unsigned int j = 0;
+ for (; j < i; ++j)
+ if (can_reuse_operands_p (icode, opno + j, opno + i, &ops[j], &ops[i])
+ && rtx_equal_p (orig_values[j], orig_values[i])
+ && ops[j].value
+ && insn_operand_matches (icode, opno + i, ops[j].value))
+ {
+ ops[i].value = copy_rtx (ops[j].value);
+ break;
+ }
- last = get_last_insn ();
- for (i = 0; i < nops; i++)
- if (!maybe_legitimize_operand (icode, opno + i, &ops[i]))
- {
- delete_insns_since (last);
- return false;
- }
+ /* Otherwise try legitimizing the operand on its own. */
+ if (j == i && !maybe_legitimize_operand (icode, opno + i, &ops[i]))
+ {
+ delete_insns_since (last);
+ return false;
+ }
+ }
return true;
}
projects / gcc.git / blobdiff