#include "target-def.h"
#include "common/common-target.h"
#include "langhooks.h"
+#include "reload.h"
#include "cgraph.h"
#include "gimple.h"
#include "dwarf2.h"
#include "fibheap.h"
#include "opts.h"
#include "diagnostic.h"
+#include "dumpfile.h"
enum upper_128bits_state
{
1, /* cond_not_taken_branch_cost. */
};
+/* Set by -mtune. */
+const struct processor_costs *ix86_tune_cost = &pentium_cost;
+
+/* Set by -mtune or -Os. */
const struct processor_costs *ix86_cost = &pentium_cost;
/* Processor feature/optimization bitmasks. */
/* X86_TUNE_PARTIAL_FLAG_REG_STALL */
m_CORE2I7 | m_GENERIC,
+ /* X86_TUNE_LCP_STALL: Avoid an expensive length-changing prefix stall
+ * on 16-bit immediate moves into memory on Core2 and Corei7. */
+ m_CORE2I7 | m_GENERIC,
+
/* X86_TUNE_USE_HIMODE_FIOP */
m_386 | m_486 | m_K6_GEODE,
/* Feature tests against the various architecture variations, used to create
ix86_arch_features based on the processor mask. */
static unsigned int initial_ix86_arch_features[X86_ARCH_LAST] = {
- /* X86_ARCH_CMOVE: Conditional move was added for pentiumpro. */
+ /* X86_ARCH_CMOV: Conditional move was added for pentiumpro. */
~(m_386 | m_486 | m_PENT | m_K6),
/* X86_ARCH_CMPXCHG: Compare and exchange was added for 80486. */
int va_arg_size;
int red_zone_size;
int outgoing_arguments_size;
- HOST_WIDE_INT frame;
/* The offsets relative to ARG_POINTER. */
HOST_WIDE_INT frame_pointer_offset;
preceding options while match those first. */
static struct ix86_target_opts isa_opts[] =
{
- { "-m64", OPTION_MASK_ISA_64BIT },
{ "-mfma4", OPTION_MASK_ISA_FMA4 },
{ "-mfma", OPTION_MASK_ISA_FMA },
{ "-mxop", OPTION_MASK_ISA_XOP },
{ "-mbmi", OPTION_MASK_ISA_BMI },
{ "-mbmi2", OPTION_MASK_ISA_BMI2 },
{ "-mlzcnt", OPTION_MASK_ISA_LZCNT },
+ { "-mhle", OPTION_MASK_ISA_HLE },
{ "-mtbm", OPTION_MASK_ISA_TBM },
{ "-mpopcnt", OPTION_MASK_ISA_POPCNT },
{ "-mmovbe", OPTION_MASK_ISA_MOVBE },
size_t len;
size_t line_len;
size_t sep_len;
+ const char *abi;
memset (opts, '\0', sizeof (opts));
opts[num++][1] = tune;
}
+ /* Add -m32/-m64/-mx32. */
+ if ((isa & OPTION_MASK_ISA_64BIT) != 0)
+ {
+ if ((isa & OPTION_MASK_ABI_64) != 0)
+ abi = "-m64";
+ else
+ abi = "-mx32";
+ isa &= ~ (OPTION_MASK_ISA_64BIT
+ | OPTION_MASK_ABI_64
+ | OPTION_MASK_ABI_X32);
+ }
+ else
+ abi = "-m32";
+ opts[num++][0] = abi;
+
/* Pick out the options in isa options. */
for (i = 0; i < ARRAY_SIZE (isa_opts); i++)
{
#define PTA_AVX2 (HOST_WIDE_INT_1 << 30)
#define PTA_BMI2 (HOST_WIDE_INT_1 << 31)
#define PTA_RTM (HOST_WIDE_INT_1 << 32)
+#define PTA_HLE (HOST_WIDE_INT_1 << 33)
/* if this reaches 64, need to widen struct pta flags below */
static struct pta
| PTA_SSSE3 | PTA_SSE4_1 | PTA_SSE4_2 | PTA_AVX | PTA_AVX2
| PTA_CX16 | PTA_POPCNT | PTA_AES | PTA_PCLMUL | PTA_FSGSBASE
| PTA_RDRND | PTA_F16C | PTA_BMI | PTA_BMI2 | PTA_LZCNT
- | PTA_FMA | PTA_MOVBE | PTA_RTM},
+ | PTA_FMA | PTA_MOVBE | PTA_RTM | PTA_HLE},
{"atom", PROCESSOR_ATOM, CPU_ATOM,
PTA_64BIT | PTA_MMX | PTA_SSE | PTA_SSE2 | PTA_SSE3
| PTA_SSSE3 | PTA_CX16 | PTA_MOVBE},
PTA_64BIT | PTA_MMX | PTA_SSE | PTA_SSE2 | PTA_SSE3
| PTA_SSSE3 | PTA_SSE4A |PTA_ABM | PTA_CX16},
{"generic32", PROCESSOR_GENERIC32, CPU_PENTIUMPRO,
- 0 /* flags are only used for -march switch. */ },
+ PTA_HLE /* flags are only used for -march switch. */ },
{"generic64", PROCESSOR_GENERIC64, CPU_GENERIC64,
- PTA_64BIT /* flags are only used for -march switch. */ },
+ PTA_64BIT
+ | PTA_HLE /* flags are only used for -march switch. */ },
};
/* -mrecip options. */
sw = "attribute";
}
+ /* Turn off both OPTION_MASK_ABI_64 and OPTION_MASK_ABI_X32 if
+ TARGET_64BIT_DEFAULT is true and TARGET_64BIT is false. */
+ if (TARGET_64BIT_DEFAULT && !TARGET_64BIT)
+ ix86_isa_flags &= ~(OPTION_MASK_ABI_64 | OPTION_MASK_ABI_X32);
+#ifdef TARGET_BI_ARCH
+ else
+ {
+#if TARGET_BI_ARCH == 1
+ /* When TARGET_BI_ARCH == 1, by default, OPTION_MASK_ABI_64
+ is on and OPTION_MASK_ABI_X32 is off. We turn off
+ OPTION_MASK_ABI_64 if OPTION_MASK_ABI_X32 is turned on by
+ -mx32. */
+ if (TARGET_X32)
+ ix86_isa_flags &= ~OPTION_MASK_ABI_64;
+#else
+ /* When TARGET_BI_ARCH == 2, by default, OPTION_MASK_ABI_X32 is
+ on and OPTION_MASK_ABI_64 is off. We turn off
+ OPTION_MASK_ABI_X32 if OPTION_MASK_ABI_64 is turned on by
+ -m64. */
+ if (TARGET_LP64)
+ ix86_isa_flags &= ~OPTION_MASK_ABI_X32;
+#endif
+ }
+#endif
+
+ if (TARGET_X32)
+ {
+ /* Always turn on OPTION_MASK_ISA_64BIT and turn off
+ OPTION_MASK_ABI_64 for TARGET_X32. */
+ ix86_isa_flags |= OPTION_MASK_ISA_64BIT;
+ ix86_isa_flags &= ~OPTION_MASK_ABI_64;
+ }
+ else if (TARGET_LP64)
+ {
+ /* Always turn on OPTION_MASK_ISA_64BIT and turn off
+ OPTION_MASK_ABI_X32 for TARGET_LP64. */
+ ix86_isa_flags |= OPTION_MASK_ISA_64BIT;
+ ix86_isa_flags &= ~OPTION_MASK_ABI_X32;
+ }
+
#ifdef SUBTARGET_OVERRIDE_OPTIONS
SUBTARGET_OVERRIDE_OPTIONS;
#endif
SUBSUBTARGET_OVERRIDE_OPTIONS;
#endif
- if (TARGET_X32)
- ix86_isa_flags |= OPTION_MASK_ISA_64BIT;
-
/* -fPIC is the default for x86_64. */
if (TARGET_MACHO && TARGET_64BIT)
flag_pic = 2;
"large", "32");
else if (TARGET_X32)
error ("code model %qs not supported in x32 mode",
- "medium");
+ "large");
break;
case CM_32:
if (processor_alias_table[i].flags & PTA_RTM
&& !(ix86_isa_flags_explicit & OPTION_MASK_ISA_RTM))
ix86_isa_flags |= OPTION_MASK_ISA_RTM;
+ if (processor_alias_table[i].flags & PTA_HLE
+ && !(ix86_isa_flags_explicit & OPTION_MASK_ISA_HLE))
+ ix86_isa_flags |= OPTION_MASK_ISA_HLE;
if (processor_alias_table[i].flags & (PTA_PREFETCH_SSE | PTA_SSE))
x86_prefetch_sse = true;
-mtune (rather than -march) points us to a processor that has them.
However, the VIA C3 gives a SIGILL, so we only do that for i686 and
higher processors. */
- if (TARGET_CMOVE
+ if (TARGET_CMOV
&& (processor_alias_table[i].flags & (PTA_PREFETCH_SSE | PTA_SSE)))
x86_prefetch_sse = true;
break;
flag_pcc_struct_return = DEFAULT_PCC_STRUCT_RETURN;
}
+ ix86_tune_cost = processor_target_table[ix86_tune].cost;
if (optimize_size)
ix86_cost = &ix86_size_cost;
else
- ix86_cost = processor_target_table[ix86_tune].cost;
+ ix86_cost = ix86_tune_cost;
/* Arrange to set up i386_stack_locals for all functions. */
init_machine_status = ix86_init_machine_status;
ix86_preferred_stack_boundary = PREFERRED_STACK_BOUNDARY_DEFAULT;
if (global_options_set.x_ix86_preferred_stack_boundary_arg)
{
- int min = (TARGET_64BIT ? 4 : 2);
+ int min = (TARGET_64BIT ? (TARGET_SSE ? 4 : 3) : 2);
int max = (TARGET_SEH ? 4 : 12);
if (ix86_preferred_stack_boundary_arg < min
target_flags |= MASK_ACCUMULATE_OUTGOING_ARGS;
}
- /* For sane SSE instruction set generation we need fcomi instruction.
- It is safe to enable all CMOVE instructions. Also, RDRAND intrinsic
- expands to a sequence that includes conditional move. */
- if (TARGET_SSE || TARGET_RDRND)
- TARGET_CMOVE = 1;
-
/* Figure out what ASM_GENERATE_INTERNAL_LABEL builds as a prefix. */
{
char *p;
flag_schedule_insns_after_reload = flag_schedule_insns = 0;
maybe_set_param_value (PARAM_SIMULTANEOUS_PREFETCHES,
- ix86_cost->simultaneous_prefetches,
+ ix86_tune_cost->simultaneous_prefetches,
global_options.x_param_values,
global_options_set.x_param_values);
- maybe_set_param_value (PARAM_L1_CACHE_LINE_SIZE, ix86_cost->prefetch_block,
+ maybe_set_param_value (PARAM_L1_CACHE_LINE_SIZE,
+ ix86_tune_cost->prefetch_block,
global_options.x_param_values,
global_options_set.x_param_values);
- maybe_set_param_value (PARAM_L1_CACHE_SIZE, ix86_cost->l1_cache_size,
+ maybe_set_param_value (PARAM_L1_CACHE_SIZE,
+ ix86_tune_cost->l1_cache_size,
global_options.x_param_values,
global_options_set.x_param_values);
- maybe_set_param_value (PARAM_L2_CACHE_SIZE, ix86_cost->l2_cache_size,
+ maybe_set_param_value (PARAM_L2_CACHE_SIZE,
+ ix86_tune_cost->l2_cache_size,
global_options.x_param_values,
global_options_set.x_param_values);
IX86_ATTR_ISA ("rdrnd", OPT_mrdrnd),
IX86_ATTR_ISA ("f16c", OPT_mf16c),
IX86_ATTR_ISA ("rtm", OPT_mrtm),
+ IX86_ATTR_ISA ("hle", OPT_mhle),
/* enum options */
IX86_ATTR_ENUM ("fpmath=", OPT_mfpmath_),
}
return TYPE_MODE (type);
}
+ else if ((size == 8 || size == 16) && !TARGET_SSE)
+ {
+ static bool warnedsse;
+
+ if (cum
+ && !warnedsse
+ && cum->warn_sse)
+ {
+ warnedsse = true;
+ warning (0, "SSE vector argument without SSE "
+ "enabled changes the ABI");
+ }
+ return mode;
+ }
else
return mode;
}
for (i = cum->regno; i < max; i++)
{
mem = gen_rtx_MEM (word_mode,
- plus_constant (save_area, i * UNITS_PER_WORD));
+ plus_constant (Pmode, save_area, i * UNITS_PER_WORD));
MEM_NOTRAP_P (mem) = 1;
set_mem_alias_set (mem, set);
emit_move_insn (mem,
for (i = cum->sse_regno; i < max; ++i)
{
- mem = plus_constant (save_area, i * 16 + ix86_varargs_gpr_size);
+ mem = plus_constant (Pmode, save_area,
+ i * 16 + ix86_varargs_gpr_size);
mem = gen_rtx_MEM (smode, mem);
MEM_NOTRAP_P (mem) = 1;
set_mem_alias_set (mem, set);
rtx reg, mem;
mem = gen_rtx_MEM (Pmode,
- plus_constant (virtual_incoming_args_rtx,
+ plus_constant (Pmode, virtual_incoming_args_rtx,
i * UNITS_PER_WORD));
MEM_NOTRAP_P (mem) = 1;
set_mem_alias_set (mem, set);
switch (get_attr_mode (insn))
{
case MODE_TI:
- if (!TARGET_SSE_PACKED_SINGLE_INSN_OPTIMAL)
- return "%vpxor\t%0, %d0";
+ return "%vpxor\t%0, %d0";
case MODE_V2DF:
- if (!TARGET_SSE_PACKED_SINGLE_INSN_OPTIMAL)
- return "%vxorpd\t%0, %d0";
+ return "%vxorpd\t%0, %d0";
case MODE_V4SF:
return "%vxorps\t%0, %d0";
case MODE_OI:
- if (!TARGET_SSE_PACKED_SINGLE_INSN_OPTIMAL)
- return "vpxor\t%x0, %x0, %x0";
+ return "vpxor\t%x0, %x0, %x0";
case MODE_V4DF:
- if (!TARGET_SSE_PACKED_SINGLE_INSN_OPTIMAL)
- return "vxorpd\t%x0, %x0, %x0";
+ return "vxorpd\t%x0, %x0, %x0";
case MODE_V8SF:
return "vxorps\t%x0, %x0, %x0";
if (TARGET_32BIT_MS_ABI && cfun->calls_setjmp)
return true;
+ /* Win64 SEH, very large frames need a frame-pointer as maximum stack
+ allocation is 4GB. */
+ if (TARGET_64BIT_MS_ABI && get_frame_size () > SEH_MAX_FRAME_SIZE)
+ return true;
+
/* In ix86_option_override_internal, TARGET_OMIT_LEAF_FRAME_POINTER
turns off the frame pointer by default. Turn it back on now if
we've not got a leaf function. */
if (TARGET_OMIT_LEAF_FRAME_POINTER
- && (!current_function_is_leaf
+ && (!crtl->is_leaf
|| ix86_current_function_calls_tls_descriptor))
return true;
NULL_TREE, void_type_node);
TREE_PUBLIC (decl) = 1;
TREE_STATIC (decl) = 1;
+ DECL_IGNORED_P (decl) = 1;
#if TARGET_MACHO
if (TARGET_MACHO)
static unsigned int
ix86_select_alt_pic_regnum (void)
{
- if (current_function_is_leaf
+ if (crtl->is_leaf
&& !crtl->profile
&& !ix86_current_function_calls_tls_descriptor)
{
static void
ix86_compute_frame_layout (struct ix86_frame *frame)
{
- unsigned int stack_alignment_needed;
+ unsigned HOST_WIDE_INT stack_alignment_needed;
HOST_WIDE_INT offset;
- unsigned int preferred_alignment;
+ unsigned HOST_WIDE_INT preferred_alignment;
HOST_WIDE_INT size = get_frame_size ();
HOST_WIDE_INT to_allocate;
/* 64-bit MS ABI seem to require stack alignment to be always 16 except for
function prologues and leaf. */
if ((TARGET_64BIT_MS_ABI && preferred_alignment < 16)
- && (!current_function_is_leaf || cfun->calls_alloca != 0
+ && (!crtl->is_leaf || cfun->calls_alloca != 0
|| ix86_current_function_calls_tls_descriptor))
{
preferred_alignment = 16;
offset += frame->nregs * UNITS_PER_WORD;
frame->reg_save_offset = offset;
+ /* On SEH target, registers are pushed just before the frame pointer
+ location. */
+ if (TARGET_SEH)
+ frame->hard_frame_pointer_offset = offset;
+
/* Align and set SSE register save area. */
if (frame->nsseregs)
{
if (stack_realign_fp
|| offset != frame->sse_reg_save_offset
|| size != 0
- || !current_function_is_leaf
+ || !crtl->is_leaf
|| cfun->calls_alloca
|| ix86_current_function_calls_tls_descriptor)
offset = (offset + stack_alignment_needed - 1) & -stack_alignment_needed;
expander assumes that last crtl->outgoing_args_size
of stack frame are unused. */
if (ACCUMULATE_OUTGOING_ARGS
- && (!current_function_is_leaf || cfun->calls_alloca
+ && (!crtl->is_leaf || cfun->calls_alloca
|| ix86_current_function_calls_tls_descriptor))
{
offset += crtl->outgoing_args_size;
/* Align stack boundary. Only needed if we're calling another function
or using alloca. */
- if (!current_function_is_leaf || cfun->calls_alloca
+ if (!crtl->is_leaf || cfun->calls_alloca
|| ix86_current_function_calls_tls_descriptor)
offset = (offset + preferred_alignment - 1) & -preferred_alignment;
frame->save_regs_using_mov = false;
if (ix86_using_red_zone ()
- && current_function_sp_is_unchanging
- && current_function_is_leaf
+ && crtl->sp_is_unchanging
+ && crtl->is_leaf
&& !ix86_current_function_calls_tls_descriptor)
{
frame->red_zone_size = to_allocate;
{
HOST_WIDE_INT diff;
- /* If we can leave the frame pointer where it is, do so. */
+ /* If we can leave the frame pointer where it is, do so. Also, returns
+ the establisher frame for __builtin_frame_address (0). */
diff = frame->stack_pointer_offset - frame->hard_frame_pointer_offset;
- if (diff > 240 || (diff & 15) != 0)
+ if (diff <= SEH_MAX_FRAME_SIZE
+ && (diff > 240 || (diff & 15) != 0)
+ && !crtl->accesses_prior_frames)
{
/* Ideally we'd determine what portion of the local stack frame
(within the constraint of the lowest 240) is most heavily used.
if (m->use_fast_prologue_epilogue)
{
/* Choose the base register most likely to allow the most scheduling
- opportunities. Generally FP is valid througout the function,
+ opportunities. Generally FP is valid throughout the function,
while DRAP must be reloaded within the epilogue. But choose either
over the SP due to increased encoding size. */
}
gcc_assert (base_reg != NULL);
- return plus_constant (base_reg, base_offset);
+ return plus_constant (Pmode, base_reg, base_offset);
}
/* Emit code to save registers in the prologue. */
the re-aligned stack frame, which provides us with a copy
of the CFA that will last past the prologue. Install it. */
gcc_checking_assert (cfun->machine->fs.fp_valid);
- addr = plus_constant (hard_frame_pointer_rtx,
+ addr = plus_constant (Pmode, hard_frame_pointer_rtx,
cfun->machine->fs.fp_offset - cfa_offset);
mem = gen_rtx_MEM (mode, addr);
add_reg_note (insn, REG_CFA_DEF_CFA, mem);
/* The frame pointer is a stable reference within the
aligned frame. Use it. */
gcc_checking_assert (cfun->machine->fs.fp_valid);
- addr = plus_constant (hard_frame_pointer_rtx,
+ addr = plus_constant (Pmode, hard_frame_pointer_rtx,
cfun->machine->fs.fp_offset - cfa_offset);
mem = gen_rtx_MEM (mode, addr);
add_reg_note (insn, REG_CFA_EXPRESSION,
use by the unwind info. */
else if (base != m->fs.cfa_reg)
{
- addr = plus_constant (m->fs.cfa_reg, m->fs.cfa_offset - cfa_offset);
+ addr = plus_constant (Pmode, m->fs.cfa_reg,
+ m->fs.cfa_offset - cfa_offset);
mem = gen_rtx_MEM (mode, addr);
add_reg_note (insn, REG_CFA_OFFSET, gen_rtx_SET (VOIDmode, mem, reg));
}
adjust = PROBE_INTERVAL;
emit_insn (gen_rtx_SET (VOIDmode, stack_pointer_rtx,
- plus_constant (stack_pointer_rtx, -adjust)));
+ plus_constant (Pmode, stack_pointer_rtx,
+ -adjust)));
emit_stack_probe (stack_pointer_rtx);
}
adjust = size + PROBE_INTERVAL - i;
emit_insn (gen_rtx_SET (VOIDmode, stack_pointer_rtx,
- plus_constant (stack_pointer_rtx, -adjust)));
+ plus_constant (Pmode, stack_pointer_rtx,
+ -adjust)));
emit_stack_probe (stack_pointer_rtx);
/* Adjust back to account for the additional first interval. */
last = emit_insn (gen_rtx_SET (VOIDmode, stack_pointer_rtx,
- plus_constant (stack_pointer_rtx,
+ plus_constant (Pmode, stack_pointer_rtx,
PROBE_INTERVAL + dope)));
}
/* SP = SP_0 + PROBE_INTERVAL. */
emit_insn (gen_rtx_SET (VOIDmode, stack_pointer_rtx,
- plus_constant (stack_pointer_rtx,
+ plus_constant (Pmode, stack_pointer_rtx,
- (PROBE_INTERVAL + dope))));
/* LAST_ADDR = SP_0 + PROBE_INTERVAL + ROUNDED_SIZE. */
if (size != rounded_size)
{
emit_insn (gen_rtx_SET (VOIDmode, stack_pointer_rtx,
- plus_constant (stack_pointer_rtx,
+ plus_constant (Pmode, stack_pointer_rtx,
rounded_size - size)));
emit_stack_probe (stack_pointer_rtx);
}
/* Adjust back to account for the additional first interval. */
last = emit_insn (gen_rtx_SET (VOIDmode, stack_pointer_rtx,
- plus_constant (stack_pointer_rtx,
+ plus_constant (Pmode, stack_pointer_rtx,
PROBE_INTERVAL + dope)));
release_scratch_register_on_entry (&sr);
rtx expr = gen_rtx_SEQUENCE (VOIDmode, rtvec_alloc (2));
XVECEXP (expr, 0, 0)
= gen_rtx_SET (VOIDmode, stack_pointer_rtx,
- plus_constant (stack_pointer_rtx, -size));
+ plus_constant (Pmode, stack_pointer_rtx, -size));
XVECEXP (expr, 0, 1)
= gen_rtx_SET (VOIDmode, stack_pointer_rtx,
- plus_constant (stack_pointer_rtx,
+ plus_constant (Pmode, stack_pointer_rtx,
PROBE_INTERVAL + dope + size));
add_reg_note (last, REG_FRAME_RELATED_EXPR, expr);
RTX_FRAME_RELATED_P (last) = 1;
it exceeds SIZE. If only one probe is needed, this will not
generate any code. Then probe at FIRST + SIZE. */
for (i = PROBE_INTERVAL; i < size; i += PROBE_INTERVAL)
- emit_stack_probe (plus_constant (stack_pointer_rtx, -(first + i)));
+ emit_stack_probe (plus_constant (Pmode, stack_pointer_rtx,
+ -(first + i)));
- emit_stack_probe (plus_constant (stack_pointer_rtx, -(first + size)));
+ emit_stack_probe (plus_constant (Pmode, stack_pointer_rtx,
+ -(first + size)));
}
/* Otherwise, do the same as above, but in a loop. Note that we must be
that SIZE is equal to ROUNDED_SIZE. */
if (size != rounded_size)
- emit_stack_probe (plus_constant (gen_rtx_PLUS (Pmode,
+ emit_stack_probe (plus_constant (Pmode,
+ gen_rtx_PLUS (Pmode,
stack_pointer_rtx,
sr.reg),
rounded_size - size));
= (crtl->parm_stack_boundary > ix86_incoming_stack_boundary
? crtl->parm_stack_boundary : ix86_incoming_stack_boundary);
unsigned int stack_realign = (incoming_stack_boundary
- < (current_function_is_leaf
+ < (crtl->is_leaf
? crtl->max_used_stack_slot_alignment
: crtl->stack_alignment_needed));
if (stack_realign
&& !crtl->need_drap
&& frame_pointer_needed
- && current_function_is_leaf
+ && crtl->is_leaf
&& flag_omit_frame_pointer
- && current_function_sp_is_unchanging
+ && crtl->sp_is_unchanging
&& !ix86_current_function_calls_tls_descriptor
&& !crtl->accesses_prior_frames
&& !cfun->calls_alloca
struct ix86_frame frame;
HOST_WIDE_INT allocate;
bool int_registers_saved;
+ bool sse_registers_saved;
ix86_finalize_stack_realign_flags ();
/* We don't want to interpret this push insn as a register save,
only as a stack adjustment. The real copy of the register as
a save will be done later, if needed. */
- t = plus_constant (stack_pointer_rtx, -UNITS_PER_WORD);
+ t = plus_constant (Pmode, stack_pointer_rtx, -UNITS_PER_WORD);
t = gen_rtx_SET (VOIDmode, stack_pointer_rtx, t);
add_reg_note (insn, REG_CFA_ADJUST_CFA, t);
RTX_FRAME_RELATED_P (insn) = 1;
}
/* Grab the argument pointer. */
- t = plus_constant (stack_pointer_rtx, m->fs.sp_offset);
+ t = plus_constant (Pmode, stack_pointer_rtx, m->fs.sp_offset);
insn = emit_insn (gen_rtx_SET (VOIDmode, crtl->drap_reg, t));
RTX_FRAME_RELATED_P (insn) = 1;
m->fs.cfa_reg = crtl->drap_reg;
address can be reached via (argp - 1) slot. This is needed
to implement macro RETURN_ADDR_RTX and intrinsic function
expand_builtin_return_addr etc. */
- t = plus_constant (crtl->drap_reg, -UNITS_PER_WORD);
+ t = plus_constant (Pmode, crtl->drap_reg, -UNITS_PER_WORD);
t = gen_frame_mem (word_mode, t);
insn = emit_insn (gen_push (t));
RTX_FRAME_RELATED_P (insn) = 1;
m->fs.realigned = true;
}
+ int_registers_saved = (frame.nregs == 0);
+ sse_registers_saved = (frame.nsseregs == 0);
+
if (frame_pointer_needed && !m->fs.fp_valid)
{
/* Note: AT&T enter does NOT have reversed args. Enter is probably
insn = emit_insn (gen_push (hard_frame_pointer_rtx));
RTX_FRAME_RELATED_P (insn) = 1;
+ /* Push registers now, before setting the frame pointer
+ on SEH target. */
+ if (!int_registers_saved
+ && TARGET_SEH
+ && !frame.save_regs_using_mov)
+ {
+ ix86_emit_save_regs ();
+ int_registers_saved = true;
+ gcc_assert (m->fs.sp_offset == frame.reg_save_offset);
+ }
+
if (m->fs.sp_offset == frame.hard_frame_pointer_offset)
{
insn = emit_move_insn (hard_frame_pointer_rtx, stack_pointer_rtx);
}
}
- int_registers_saved = (frame.nregs == 0);
-
if (!int_registers_saved)
{
/* If saving registers via PUSH, do so now. */
current_function_static_stack_size = stack_size;
}
+ /* On SEH target with very large frame size, allocate an area to save
+ SSE registers (as the very large allocation won't be described). */
+ if (TARGET_SEH
+ && frame.stack_pointer_offset > SEH_MAX_FRAME_SIZE
+ && !sse_registers_saved)
+ {
+ HOST_WIDE_INT sse_size =
+ frame.sse_reg_save_offset - frame.reg_save_offset;
+
+ gcc_assert (int_registers_saved);
+
+ /* No need to do stack checking as the area will be immediately
+ written. */
+ pro_epilogue_adjust_stack (stack_pointer_rtx, stack_pointer_rtx,
+ GEN_INT (-sse_size), -1,
+ m->fs.cfa_reg == stack_pointer_rtx);
+ allocate -= sse_size;
+ ix86_emit_save_sse_regs_using_mov (frame.sse_reg_save_offset);
+ sse_registers_saved = true;
+ }
+
/* The stack has already been decremented by the instruction calling us
so probe if the size is non-negative to preserve the protection area. */
if (allocate >= 0 && flag_stack_check == STATIC_BUILTIN_STACK_CHECK)
RTX_FRAME_RELATED_P (insn) = 1;
add_reg_note (insn, REG_FRAME_RELATED_EXPR,
gen_rtx_SET (VOIDmode, stack_pointer_rtx,
- plus_constant (stack_pointer_rtx,
+ plus_constant (Pmode, stack_pointer_rtx,
-allocate)));
}
m->fs.sp_offset += allocate;
if (!int_registers_saved)
ix86_emit_save_regs_using_mov (frame.reg_save_offset);
- if (frame.nsseregs)
+ if (!sse_registers_saved)
ix86_emit_save_sse_regs_using_mov (frame.sse_reg_save_offset);
pic_reg_used = false;
if (m->fs.cfa_reg == stack_pointer_rtx)
{
- rtx x = plus_constant (stack_pointer_rtx, UNITS_PER_WORD);
+ rtx x = plus_constant (Pmode, stack_pointer_rtx, UNITS_PER_WORD);
x = gen_rtx_SET (VOIDmode, stack_pointer_rtx, x);
add_reg_note (insn, REG_CFA_ADJUST_CFA, x);
RTX_FRAME_RELATED_P (insn) = 1;
m->fs.cfa_offset = m->fs.sp_offset;
add_reg_note (insn, REG_CFA_DEF_CFA,
- plus_constant (stack_pointer_rtx, m->fs.sp_offset));
+ plus_constant (Pmode, stack_pointer_rtx,
+ m->fs.sp_offset));
RTX_FRAME_RELATED_P (insn) = 1;
}
ix86_add_cfa_restore_note (insn, hard_frame_pointer_rtx,
ix86_compute_frame_layout (&frame);
m->fs.sp_valid = (!frame_pointer_needed
- || (current_function_sp_is_unchanging
+ || (crtl->sp_is_unchanging
&& !stack_realign_fp));
gcc_assert (!m->fs.sp_valid
|| m->fs.sp_offset == frame.stack_pointer_offset);
if (frame_pointer_needed)
{
t = gen_rtx_PLUS (Pmode, hard_frame_pointer_rtx, sa);
- t = plus_constant (t, m->fs.fp_offset - UNITS_PER_WORD);
+ t = plus_constant (Pmode, t, m->fs.fp_offset - UNITS_PER_WORD);
emit_insn (gen_rtx_SET (VOIDmode, sa, t));
t = gen_frame_mem (Pmode, hard_frame_pointer_rtx);
bother resetting the CFA to the SP for the duration of
the return insn. */
add_reg_note (insn, REG_CFA_DEF_CFA,
- plus_constant (sa, UNITS_PER_WORD));
+ plus_constant (Pmode, sa, UNITS_PER_WORD));
ix86_add_queued_cfa_restore_notes (insn);
add_reg_note (insn, REG_CFA_RESTORE, hard_frame_pointer_rtx);
RTX_FRAME_RELATED_P (insn) = 1;
else
{
t = gen_rtx_PLUS (Pmode, stack_pointer_rtx, sa);
- t = plus_constant (t, m->fs.sp_offset - UNITS_PER_WORD);
+ t = plus_constant (Pmode, t, m->fs.sp_offset - UNITS_PER_WORD);
insn = emit_insn (gen_rtx_SET (VOIDmode, stack_pointer_rtx, t));
ix86_add_queued_cfa_restore_notes (insn);
{
m->fs.cfa_offset = UNITS_PER_WORD;
add_reg_note (insn, REG_CFA_DEF_CFA,
- plus_constant (stack_pointer_rtx,
+ plus_constant (Pmode, stack_pointer_rtx,
UNITS_PER_WORD));
RTX_FRAME_RELATED_P (insn) = 1;
}
}
/* First step is to deallocate the stack frame so that we can
- pop the registers. */
- if (!m->fs.sp_valid)
+ pop the registers. Also do it on SEH target for very large
+ frame as the emitted instructions aren't allowed by the ABI in
+ epilogues. */
+ if (!m->fs.sp_valid
+ || (TARGET_SEH
+ && (m->fs.sp_offset - frame.reg_save_offset
+ >= SEH_MAX_FRAME_SIZE)))
{
pro_epilogue_adjust_stack (stack_pointer_rtx, hard_frame_pointer_rtx,
GEN_INT (m->fs.fp_offset
int retval = 1;
enum ix86_address_seg seg = SEG_DEFAULT;
+ /* Allow SImode subregs of DImode addresses,
+ they will be emitted with addr32 prefix. */
+ if (TARGET_64BIT && GET_MODE (addr) == SImode)
+ {
+ if (GET_CODE (addr) == SUBREG
+ && GET_MODE (XEXP (addr, 0)) == DImode)
+ addr = SUBREG_REG (addr);
+ }
+
/* Allow zero-extended SImode addresses,
they will be emitted with addr32 prefix. */
- if (TARGET_64BIT && GET_MODE (addr) == DImode)
+ else if (TARGET_64BIT && GET_MODE (addr) == DImode)
{
if (GET_CODE (addr) == ZERO_EXTEND
&& GET_MODE (XEXP (addr, 0)) == SImode)
addr = SUBREG_REG (addr);
else if (GET_MODE (addr) == DImode)
addr = gen_rtx_SUBREG (SImode, addr, 0);
- else
+ else if (GET_MODE (addr) != VOIDmode)
return 0;
}
}
return false;
}
+/* Our implementation of LEGITIMIZE_RELOAD_ADDRESS. Returns a value to
+ replace the input X, or the original X if no replacement is called for.
+ The output parameter *WIN is 1 if the calling macro should goto WIN,
+ 0 if it should not. */
+
+bool
+ix86_legitimize_reload_address (rtx x,
+ enum machine_mode mode ATTRIBUTE_UNUSED,
+ int opnum, int type,
+ int ind_levels ATTRIBUTE_UNUSED)
+{
+ /* Reload can generate:
+
+ (plus:DI (plus:DI (unspec:DI [(const_int 0 [0])] UNSPEC_TP)
+ (reg:DI 97))
+ (reg:DI 2 cx))
+
+ This RTX is rejected from ix86_legitimate_address_p due to
+ non-strictness of base register 97. Following this rejection,
+ reload pushes all three components into separate registers,
+ creating invalid memory address RTX.
+
+ Following code reloads only the invalid part of the
+ memory address RTX. */
+
+ if (GET_CODE (x) == PLUS
+ && REG_P (XEXP (x, 1))
+ && GET_CODE (XEXP (x, 0)) == PLUS
+ && REG_P (XEXP (XEXP (x, 0), 1)))
+ {
+ rtx base, index;
+ bool something_reloaded = false;
+
+ base = XEXP (XEXP (x, 0), 1);
+ if (!REG_OK_FOR_BASE_STRICT_P (base))
+ {
+ push_reload (base, NULL_RTX, &XEXP (XEXP (x, 0), 1), NULL,
+ BASE_REG_CLASS, GET_MODE (x), VOIDmode, 0, 0,
+ opnum, (enum reload_type) type);
+ something_reloaded = true;
+ }
+
+ index = XEXP (x, 1);
+ if (!REG_OK_FOR_INDEX_STRICT_P (index))
+ {
+ push_reload (index, NULL_RTX, &XEXP (x, 1), NULL,
+ INDEX_REG_CLASS, GET_MODE (x), VOIDmode, 0, 0,
+ opnum, (enum reload_type) type);
+ something_reloaded = true;
+ }
+
+ gcc_assert (something_reloaded);
+ return true;
+ }
+
+ return false;
+}
+
/* Recognizes RTL expressions that are valid memory addresses for an
instruction. The MODE argument is the machine mode for the MEM
expression that wants to use this address.
base == reg ? NULL_RTX : reg);
if (CONST_INT_P (new_rtx))
- new_rtx = plus_constant (base, INTVAL (new_rtx));
+ new_rtx = plus_constant (Pmode, base, INTVAL (new_rtx));
else
{
if (GET_CODE (new_rtx) == PLUS && CONSTANT_P (XEXP (new_rtx, 1)))
case TLS_MODEL_INITIAL_EXEC:
if (TARGET_64BIT)
{
- if (TARGET_SUN_TLS)
+ if (TARGET_SUN_TLS && !TARGET_X32)
{
/* The Sun linker took the AMD64 TLS spec literally
and can only handle %rax as destination of the
initial executable code sequence. */
- dest = gen_reg_rtx (Pmode);
+ dest = gen_reg_rtx (DImode);
emit_insn (gen_tls_initial_exec_64_sun (dest, x));
return dest;
}
x = gen_rtx_PLUS (Pmode,
gen_rtx_PLUS (Pmode, XEXP (XEXP (x, 0), 0),
XEXP (XEXP (XEXP (x, 0), 1), 0)),
- plus_constant (other, INTVAL (constant)));
+ plus_constant (Pmode, other,
+ INTVAL (constant)));
}
}
}
\f
static void
-put_condition_code (enum rtx_code code, enum machine_mode mode, int reverse,
- int fp, FILE *file)
+put_condition_code (enum rtx_code code, enum machine_mode mode, bool reverse,
+ bool fp, FILE *file)
{
const char *suffix;
c -- like C, but print reversed condition
F,f -- likewise, but for floating-point.
O -- if HAVE_AS_IX86_CMOV_SUN_SYNTAX, expand to "w.", "l." or "q.",
- otherwise nothing
+ otherwise nothing
R -- print the prefix for register names.
z -- print the opcode suffix for the size of the current operand.
Z -- likewise, with special suffixes for x87 instructions.
{
switch (code)
{
- case '*':
- if (ASSEMBLER_DIALECT == ASM_ATT)
- putc ('*', file);
- return;
-
- case '&':
- {
- const char *name = get_some_local_dynamic_name ();
- if (name == NULL)
- output_operand_lossage ("'%%&' used without any "
- "local dynamic TLS references");
- else
- assemble_name (file, name);
- return;
- }
-
case 'A':
switch (ASSEMBLER_DIALECT)
{
putc ('t', file);
return;
+ case 'O':
+#ifdef HAVE_AS_IX86_CMOV_SUN_SYNTAX
+ if (ASSEMBLER_DIALECT != ASM_ATT)
+ return;
+
+ switch (GET_MODE_SIZE (GET_MODE (x)))
+ {
+ case 2:
+ putc ('w', file);
+ break;
+
+ case 4:
+ putc ('l', file);
+ break;
+
+ case 8:
+ putc ('q', file);
+ break;
+
+ default:
+ output_operand_lossage
+ ("invalid operand size for operand code 'O'");
+ return;
+ }
+
+ putc ('.', file);
+#endif
+ return;
+
case 'z':
if (GET_MODE_CLASS (GET_MODE (x)) == MODE_INT)
{
default:
output_operand_lossage
- ("invalid operand size for operand code '%c'", code);
+ ("invalid operand size for operand code 'z'");
return;
}
}
if (GET_MODE_CLASS (GET_MODE (x)) == MODE_FLOAT)
warning
- (0, "non-integer operand used with operand code '%c'", code);
+ (0, "non-integer operand used with operand code 'z'");
/* FALLTHRU */
case 'Z':
else
{
output_operand_lossage
- ("invalid operand type used with operand code '%c'", code);
+ ("invalid operand type used with operand code 'Z'");
return;
}
output_operand_lossage
- ("invalid operand size for operand code '%c'", code);
+ ("invalid operand size for operand code 'Z'");
return;
case 'd':
}
return;
- case 'D':
- /* Little bit of braindamage here. The SSE compare instructions
- does use completely different names for the comparisons that the
- fp conditional moves. */
- if (TARGET_AVX)
- {
- switch (GET_CODE (x))
- {
- case EQ:
- fputs ("eq", file);
- break;
- case UNEQ:
- fputs ("eq_us", file);
- break;
- case LT:
- fputs ("lt", file);
- break;
- case UNLT:
- fputs ("nge", file);
- break;
- case LE:
- fputs ("le", file);
- break;
- case UNLE:
- fputs ("ngt", file);
- break;
- case UNORDERED:
- fputs ("unord", file);
- break;
- case NE:
- fputs ("neq", file);
- break;
- case LTGT:
- fputs ("neq_oq", file);
- break;
- case GE:
- fputs ("ge", file);
- break;
- case UNGE:
- fputs ("nlt", file);
- break;
- case GT:
- fputs ("gt", file);
- break;
- case UNGT:
- fputs ("nle", file);
- break;
- case ORDERED:
- fputs ("ord", file);
- break;
- default:
- output_operand_lossage ("operand is not a condition code, "
- "invalid operand code 'D'");
- return;
- }
- }
- else
- {
- switch (GET_CODE (x))
- {
- case EQ:
- case UNEQ:
- fputs ("eq", file);
- break;
- case LT:
- case UNLT:
- fputs ("lt", file);
- break;
- case LE:
- case UNLE:
- fputs ("le", file);
- break;
- case UNORDERED:
- fputs ("unord", file);
- break;
- case NE:
- case LTGT:
- fputs ("neq", file);
- break;
- case UNGE:
- case GE:
- fputs ("nlt", file);
- break;
- case UNGT:
- case GT:
- fputs ("nle", file);
- break;
- case ORDERED:
- fputs ("ord", file);
- break;
- default:
- output_operand_lossage ("operand is not a condition code, "
- "invalid operand code 'D'");
- return;
- }
- }
- return;
- case 'O':
-#ifdef HAVE_AS_IX86_CMOV_SUN_SYNTAX
- if (ASSEMBLER_DIALECT == ASM_ATT)
- {
- switch (GET_MODE (x))
- {
- case HImode: putc ('w', file); break;
- case SImode:
- case SFmode: putc ('l', file); break;
- case DImode:
- case DFmode: putc ('q', file); break;
- default: gcc_unreachable ();
- }
- putc ('.', file);
- }
-#endif
- return;
- case 'C':
- if (!COMPARISON_P (x))
- {
- output_operand_lossage ("operand is neither a constant nor a "
- "condition code, invalid operand code "
- "'C'");
- return;
- }
- put_condition_code (GET_CODE (x), GET_MODE (XEXP (x, 0)), 0, 0, file);
- return;
- case 'F':
- if (!COMPARISON_P (x))
- {
- output_operand_lossage ("operand is neither a constant nor a "
- "condition code, invalid operand code "
- "'F'");
- return;
- }
-#ifdef HAVE_AS_IX86_CMOV_SUN_SYNTAX
- if (ASSEMBLER_DIALECT == ASM_ATT)
- putc ('.', file);
-#endif
- put_condition_code (GET_CODE (x), GET_MODE (XEXP (x, 0)), 0, 1, file);
- return;
-
- /* Like above, but reverse condition */
- case 'c':
- /* Check to see if argument to %c is really a constant
- and not a condition code which needs to be reversed. */
- if (!COMPARISON_P (x))
- {
- output_operand_lossage ("operand is neither a constant nor a "
- "condition code, invalid operand "
- "code 'c'");
- return;
- }
- put_condition_code (GET_CODE (x), GET_MODE (XEXP (x, 0)), 1, 0, file);
- return;
- case 'f':
- if (!COMPARISON_P (x))
- {
- output_operand_lossage ("operand is neither a constant nor a "
- "condition code, invalid operand "
- "code 'f'");
- return;
- }
-#ifdef HAVE_AS_IX86_CMOV_SUN_SYNTAX
- if (ASSEMBLER_DIALECT == ASM_ATT)
- putc ('.', file);
-#endif
- put_condition_code (GET_CODE (x), GET_MODE (XEXP (x, 0)), 1, 1, file);
- return;
-
- case 'H':
- if (!offsettable_memref_p (x))
- {
- output_operand_lossage ("operand is not an offsettable memory "
- "reference, invalid operand "
- "code 'H'");
- return;
- }
- /* It doesn't actually matter what mode we use here, as we're
- only going to use this for printing. */
- x = adjust_address_nv (x, DImode, 8);
- break;
-
- case '+':
- {
- rtx x;
-
- if (!optimize
- || optimize_function_for_size_p (cfun)
- || !TARGET_BRANCH_PREDICTION_HINTS)
- return;
-
- x = find_reg_note (current_output_insn, REG_BR_PROB, 0);
- if (x)
- {
- int pred_val = INTVAL (XEXP (x, 0));
-
- if (pred_val < REG_BR_PROB_BASE * 45 / 100
- || pred_val > REG_BR_PROB_BASE * 55 / 100)
- {
- bool taken = pred_val > REG_BR_PROB_BASE / 2;
- bool cputaken
- = final_forward_branch_p (current_output_insn) == 0;
-
- /* Emit hints only in the case default branch prediction
- heuristics would fail. */
- if (taken != cputaken)
- {
- /* We use 3e (DS) prefix for taken branches and
- 2e (CS) prefix for not taken branches. */
- if (taken)
- fputs ("ds ; ", file);
- else
- fputs ("cs ; ", file);
- }
- }
- }
- return;
- }
-
case 'Y':
switch (GET_CODE (x))
{
}
return;
+ case 'D':
+ /* Little bit of braindamage here. The SSE compare instructions
+ does use completely different names for the comparisons that the
+ fp conditional moves. */
+ switch (GET_CODE (x))
+ {
+ case UNEQ:
+ if (TARGET_AVX)
+ {
+ fputs ("eq_us", file);
+ break;
+ }
+ case EQ:
+ fputs ("eq", file);
+ break;
+ case UNLT:
+ if (TARGET_AVX)
+ {
+ fputs ("nge", file);
+ break;
+ }
+ case LT:
+ fputs ("lt", file);
+ break;
+ case UNLE:
+ if (TARGET_AVX)
+ {
+ fputs ("ngt", file);
+ break;
+ }
+ case LE:
+ fputs ("le", file);
+ break;
+ case UNORDERED:
+ fputs ("unord", file);
+ break;
+ case LTGT:
+ if (TARGET_AVX)
+ {
+ fputs ("neq_oq", file);
+ break;
+ }
+ case NE:
+ fputs ("neq", file);
+ break;
+ case GE:
+ if (TARGET_AVX)
+ {
+ fputs ("ge", file);
+ break;
+ }
+ case UNGE:
+ fputs ("nlt", file);
+ break;
+ case GT:
+ if (TARGET_AVX)
+ {
+ fputs ("gt", file);
+ break;
+ }
+ case UNGT:
+ fputs ("nle", file);
+ break;
+ case ORDERED:
+ fputs ("ord", file);
+ break;
+ default:
+ output_operand_lossage ("operand is not a condition code, "
+ "invalid operand code 'D'");
+ return;
+ }
+ return;
+
+ case 'F':
+ case 'f':
+#ifdef HAVE_AS_IX86_CMOV_SUN_SYNTAX
+ if (ASSEMBLER_DIALECT == ASM_ATT)
+ putc ('.', file);
+#endif
+
+ case 'C':
+ case 'c':
+ if (!COMPARISON_P (x))
+ {
+ output_operand_lossage ("operand is not a condition code, "
+ "invalid operand code '%c'", code);
+ return;
+ }
+ put_condition_code (GET_CODE (x), GET_MODE (XEXP (x, 0)),
+ code == 'c' || code == 'f',
+ code == 'F' || code == 'f',
+ file);
+ return;
+
+ case 'H':
+ if (!offsettable_memref_p (x))
+ {
+ output_operand_lossage ("operand is not an offsettable memory "
+ "reference, invalid operand code 'H'");
+ return;
+ }
+ /* It doesn't actually matter what mode we use here, as we're
+ only going to use this for printing. */
+ x = adjust_address_nv (x, DImode, 8);
+ break;
+
+ case 'K':
+ gcc_assert (CONST_INT_P (x));
+
+ if (INTVAL (x) & IX86_HLE_ACQUIRE)
+#ifdef HAVE_AS_IX86_HLE
+ fputs ("xacquire ", file);
+#else
+ fputs ("\n" ASM_BYTE "0xf2\n\t", file);
+#endif
+ else if (INTVAL (x) & IX86_HLE_RELEASE)
+#ifdef HAVE_AS_IX86_HLE
+ fputs ("xrelease ", file);
+#else
+ fputs ("\n" ASM_BYTE "0xf3\n\t", file);
+#endif
+ /* We do not want to print value of the operand. */
+ return;
+
+ case '*':
+ if (ASSEMBLER_DIALECT == ASM_ATT)
+ putc ('*', file);
+ return;
+
+ case '&':
+ {
+ const char *name = get_some_local_dynamic_name ();
+ if (name == NULL)
+ output_operand_lossage ("'%%&' used without any "
+ "local dynamic TLS references");
+ else
+ assemble_name (file, name);
+ return;
+ }
+
+ case '+':
+ {
+ rtx x;
+
+ if (!optimize
+ || optimize_function_for_size_p (cfun)
+ || !TARGET_BRANCH_PREDICTION_HINTS)
+ return;
+
+ x = find_reg_note (current_output_insn, REG_BR_PROB, 0);
+ if (x)
+ {
+ int pred_val = INTVAL (XEXP (x, 0));
+
+ if (pred_val < REG_BR_PROB_BASE * 45 / 100
+ || pred_val > REG_BR_PROB_BASE * 55 / 100)
+ {
+ bool taken = pred_val > REG_BR_PROB_BASE / 2;
+ bool cputaken
+ = final_forward_branch_p (current_output_insn) == 0;
+
+ /* Emit hints only in the case default branch prediction
+ heuristics would fail. */
+ if (taken != cputaken)
+ {
+ /* We use 3e (DS) prefix for taken branches and
+ 2e (CS) prefix for not taken branches. */
+ if (taken)
+ fputs ("ds ; ", file);
+ else
+ fputs ("cs ; ", file);
+ }
+ }
+ }
+ return;
+ }
+
case ';':
#ifndef HAVE_AS_IX86_REP_LOCK_PREFIX
putc (';', file);
}
else
{
- /* Print SImode register names for zero-extended
- addresses to force addr32 prefix. */
+ /* Print SImode register names to force addr32 prefix. */
if (TARGET_64BIT
- && (GET_CODE (addr) == ZERO_EXTEND
+ && (GET_CODE (addr) == SUBREG
+ || GET_CODE (addr) == ZERO_EXTEND
|| GET_CODE (addr) == AND))
{
gcc_assert (!code);
op0 = operands[0];
op1 = operands[1];
- if (TARGET_AVX)
+ if (TARGET_AVX
+ && GET_MODE_SIZE (mode) == 32)
{
switch (GET_MODE_CLASS (mode))
{
case MODE_VECTOR_INT:
case MODE_INT:
- switch (GET_MODE_SIZE (mode))
- {
- case 16:
- /* If we're optimizing for size, movups is the smallest. */
- if (TARGET_SSE_PACKED_SINGLE_INSN_OPTIMAL)
- {
- op0 = gen_lowpart (V4SFmode, op0);
- op1 = gen_lowpart (V4SFmode, op1);
- emit_insn (gen_sse_movups (op0, op1));
- return;
- }
- op0 = gen_lowpart (V16QImode, op0);
- op1 = gen_lowpart (V16QImode, op1);
- emit_insn (gen_sse2_movdqu (op0, op1));
- break;
- case 32:
- op0 = gen_lowpart (V32QImode, op0);
- op1 = gen_lowpart (V32QImode, op1);
- ix86_avx256_split_vector_move_misalign (op0, op1);
- break;
- default:
- gcc_unreachable ();
- }
- break;
- case MODE_VECTOR_FLOAT:
- op0 = gen_lowpart (mode, op0);
- op1 = gen_lowpart (mode, op1);
+ op0 = gen_lowpart (V32QImode, op0);
+ op1 = gen_lowpart (V32QImode, op1);
+ /* FALLTHRU */
- switch (mode)
- {
- case V4SFmode:
- emit_insn (gen_sse_movups (op0, op1));
- break;
- case V8SFmode:
- ix86_avx256_split_vector_move_misalign (op0, op1);
- break;
- case V2DFmode:
- if (TARGET_SSE_PACKED_SINGLE_INSN_OPTIMAL)
- {
- op0 = gen_lowpart (V4SFmode, op0);
- op1 = gen_lowpart (V4SFmode, op1);
- emit_insn (gen_sse_movups (op0, op1));
- return;
- }
- emit_insn (gen_sse2_movupd (op0, op1));
- break;
- case V4DFmode:
- ix86_avx256_split_vector_move_misalign (op0, op1);
- break;
- default:
- gcc_unreachable ();
- }
+ case MODE_VECTOR_FLOAT:
+ ix86_avx256_split_vector_move_misalign (op0, op1);
break;
default:
if (MEM_P (op1))
{
- /* If we're optimizing for size, movups is the smallest. */
- if (optimize_insn_for_size_p ()
- || TARGET_SSE_PACKED_SINGLE_INSN_OPTIMAL)
- {
- op0 = gen_lowpart (V4SFmode, op0);
- op1 = gen_lowpart (V4SFmode, op1);
- emit_insn (gen_sse_movups (op0, op1));
- return;
- }
-
/* ??? If we have typed data, then it would appear that using
movdqu is the only way to get unaligned data loaded with
integer type. */
{
op0 = gen_lowpart (V16QImode, op0);
op1 = gen_lowpart (V16QImode, op1);
+ /* We will eventually emit movups based on insn attributes. */
emit_insn (gen_sse2_movdqu (op0, op1));
- return;
}
-
- if (TARGET_SSE2 && mode == V2DFmode)
+ else if (TARGET_SSE2 && mode == V2DFmode)
{
rtx zero;
- if (TARGET_SSE_UNALIGNED_LOAD_OPTIMAL)
+ if (TARGET_AVX
+ || TARGET_SSE_UNALIGNED_LOAD_OPTIMAL
+ || TARGET_SSE_PACKED_SINGLE_INSN_OPTIMAL
+ || optimize_function_for_size_p (cfun))
{
- op0 = gen_lowpart (V2DFmode, op0);
- op1 = gen_lowpart (V2DFmode, op1);
+ /* We will eventually emit movups based on insn attributes. */
emit_insn (gen_sse2_movupd (op0, op1));
return;
}
}
else
{
- if (TARGET_SSE_UNALIGNED_LOAD_OPTIMAL)
+ if (TARGET_AVX
+ || TARGET_SSE_UNALIGNED_LOAD_OPTIMAL
+ || TARGET_SSE_PACKED_SINGLE_INSN_OPTIMAL
+ || optimize_function_for_size_p (cfun))
{
op0 = gen_lowpart (V4SFmode, op0);
op1 = gen_lowpart (V4SFmode, op1);
if (mode != V4SFmode)
op0 = gen_lowpart (V4SFmode, op0);
+
m = adjust_address (op1, V2SFmode, 0);
emit_insn (gen_sse_loadlps (op0, op0, m));
m = adjust_address (op1, V2SFmode, 8);
}
else if (MEM_P (op0))
{
- /* If we're optimizing for size, movups is the smallest. */
- if (optimize_insn_for_size_p ()
- || TARGET_SSE_PACKED_SINGLE_INSN_OPTIMAL)
- {
- op0 = gen_lowpart (V4SFmode, op0);
- op1 = gen_lowpart (V4SFmode, op1);
- emit_insn (gen_sse_movups (op0, op1));
- return;
- }
-
- /* ??? Similar to above, only less clear because of quote
- typeless stores unquote. */
- if (TARGET_SSE2 && !TARGET_SSE_TYPELESS_STORES
- && GET_MODE_CLASS (mode) == MODE_VECTOR_INT)
+ if (TARGET_SSE2 && GET_MODE_CLASS (mode) == MODE_VECTOR_INT)
{
op0 = gen_lowpart (V16QImode, op0);
op1 = gen_lowpart (V16QImode, op1);
+ /* We will eventually emit movups based on insn attributes. */
emit_insn (gen_sse2_movdqu (op0, op1));
- return;
}
-
- if (TARGET_SSE2 && mode == V2DFmode)
+ else if (TARGET_SSE2 && mode == V2DFmode)
{
- if (TARGET_SSE_UNALIGNED_STORE_OPTIMAL)
- {
- op0 = gen_lowpart (V2DFmode, op0);
- op1 = gen_lowpart (V2DFmode, op1);
- emit_insn (gen_sse2_movupd (op0, op1));
- }
+ if (TARGET_AVX
+ || TARGET_SSE_UNALIGNED_STORE_OPTIMAL
+ || TARGET_SSE_PACKED_SINGLE_INSN_OPTIMAL
+ || optimize_function_for_size_p (cfun))
+ /* We will eventually emit movups based on insn attributes. */
+ emit_insn (gen_sse2_movupd (op0, op1));
else
{
m = adjust_address (op0, DFmode, 0);
if (mode != V4SFmode)
op1 = gen_lowpart (V4SFmode, op1);
- if (TARGET_SSE_UNALIGNED_STORE_OPTIMAL)
+ if (TARGET_AVX
+ || TARGET_SSE_UNALIGNED_STORE_OPTIMAL
+ || TARGET_SSE_PACKED_SINGLE_INSN_OPTIMAL
+ || optimize_function_for_size_p (cfun))
{
op0 = gen_lowpart (V4SFmode, op0);
emit_insn (gen_sse_movups (op0, op1));
unsigned int regno1 = true_regnum (operands[1]);
unsigned int regno2 = true_regnum (operands[2]);
+ /* FIXME: Handle zero-extended addresses. */
+ if (GET_CODE (operands[1]) == ZERO_EXTEND
+ || GET_CODE (operands[1]) == AND)
+ return false;
+
/* Check if we need to optimize. */
if (!TARGET_OPT_AGU || optimize_function_for_size_p (cfun))
return false;
rtx op0 = XEXP (operands[1], 0);
rtx op1 = XEXP (operands[1], 1);
+ if (GET_MODE (op0) == TImode
+ || (GET_MODE (op0) == DImode
+ && !TARGET_64BIT))
+ return false;
+
start_sequence ();
compare_op = ix86_expand_compare (code, op0, op1);
compare_seq = get_insns ();
vt = force_reg (maskmode, vt);
mask = gen_lowpart (maskmode, mask);
if (maskmode == V8SImode)
- emit_insn (gen_avx2_permvarv8si (t1, vt, mask));
+ emit_insn (gen_avx2_permvarv8si (t1, mask, vt));
else
emit_insn (gen_avx2_pshufbv32qi3 (t1, mask, vt));
the high bits of the shuffle elements. No need for us to
perform an AND ourselves. */
if (one_operand_shuffle)
- emit_insn (gen_avx2_permvarv8si (target, mask, op0));
+ emit_insn (gen_avx2_permvarv8si (target, op0, mask));
else
{
t1 = gen_reg_rtx (V8SImode);
t2 = gen_reg_rtx (V8SImode);
- emit_insn (gen_avx2_permvarv8si (t1, mask, op0));
- emit_insn (gen_avx2_permvarv8si (t2, mask, op1));
+ emit_insn (gen_avx2_permvarv8si (t1, op0, mask));
+ emit_insn (gen_avx2_permvarv8si (t2, op1, mask));
goto merge_two;
}
return;
case V8SFmode:
mask = gen_lowpart (V8SFmode, mask);
if (one_operand_shuffle)
- emit_insn (gen_avx2_permvarv8sf (target, mask, op0));
+ emit_insn (gen_avx2_permvarv8sf (target, op0, mask));
else
{
t1 = gen_reg_rtx (V8SFmode);
t2 = gen_reg_rtx (V8SFmode);
- emit_insn (gen_avx2_permvarv8sf (t1, mask, op0));
- emit_insn (gen_avx2_permvarv8sf (t2, mask, op1));
+ emit_insn (gen_avx2_permvarv8sf (t1, op0, mask));
+ emit_insn (gen_avx2_permvarv8sf (t2, op1, mask));
goto merge_two;
}
return;
t2 = gen_reg_rtx (V8SImode);
emit_insn (gen_avx_vec_concatv8si (t1, op0, op1));
emit_insn (gen_avx_vec_concatv8si (t2, mask, mask));
- emit_insn (gen_avx2_permvarv8si (t1, t2, t1));
+ emit_insn (gen_avx2_permvarv8si (t1, t1, t2));
emit_insn (gen_avx_vextractf128v8si (target, t1, const0_rtx));
return;
case V4SFmode:
t1 = gen_reg_rtx (V8SFmode);
- t2 = gen_reg_rtx (V8SFmode);
- mask = gen_lowpart (V4SFmode, mask);
+ t2 = gen_reg_rtx (V8SImode);
+ mask = gen_lowpart (V4SImode, mask);
emit_insn (gen_avx_vec_concatv8sf (t1, op0, op1));
- emit_insn (gen_avx_vec_concatv8sf (t2, mask, mask));
- emit_insn (gen_avx2_permvarv8sf (t1, t2, t1));
+ emit_insn (gen_avx_vec_concatv8si (t2, mask, mask));
+ emit_insn (gen_avx2_permvarv8sf (t1, t1, t2));
emit_insn (gen_avx_vextractf128v8sf (target, t1, const0_rtx));
return;
true if we want the N/2 high elements, else the low elements. */
void
-ix86_expand_sse_unpack (rtx operands[2], bool unsigned_p, bool high_p)
+ix86_expand_sse_unpack (rtx dest, rtx src, bool unsigned_p, bool high_p)
{
- enum machine_mode imode = GET_MODE (operands[1]);
- rtx tmp, dest;
+ enum machine_mode imode = GET_MODE (src);
+ rtx tmp;
if (TARGET_SSE4_1)
{
if (GET_MODE_SIZE (imode) == 32)
{
tmp = gen_reg_rtx (halfmode);
- emit_insn (extract (tmp, operands[1]));
+ emit_insn (extract (tmp, src));
}
else if (high_p)
{
/* Shift higher 8 bytes to lower 8 bytes. */
tmp = gen_reg_rtx (imode);
emit_insn (gen_sse2_lshrv1ti3 (gen_lowpart (V1TImode, tmp),
- gen_lowpart (V1TImode, operands[1]),
+ gen_lowpart (V1TImode, src),
GEN_INT (64)));
}
else
- tmp = operands[1];
+ tmp = src;
- emit_insn (unpack (operands[0], tmp));
+ emit_insn (unpack (dest, tmp));
}
else
{
gcc_unreachable ();
}
- dest = gen_lowpart (imode, operands[0]);
-
if (unsigned_p)
tmp = force_reg (imode, CONST0_RTX (imode));
else
tmp = ix86_expand_sse_cmp (gen_reg_rtx (imode), GT, CONST0_RTX (imode),
- operands[1], pc_rtx, pc_rtx);
+ src, pc_rtx, pc_rtx);
- emit_insn (unpack (dest, operands[1], tmp));
+ emit_insn (unpack (gen_lowpart (imode, dest), src, tmp));
}
}
/* Compensate for the stack decrement by 4. */
if (!TARGET_64BIT && nparts == 3
&& mode == XFmode && TARGET_128BIT_LONG_DOUBLE)
- src_base = plus_constant (src_base, 4);
+ src_base = plus_constant (Pmode, src_base, 4);
/* src_base refers to the stack pointer and is
automatically decreased by emitted push. */
part[1][0] = replace_equiv_address (part[1][0], base);
for (i = 1; i < nparts; i++)
{
- tmp = plus_constant (base, UNITS_PER_WORD * i);
+ tmp = plus_constant (Pmode, base, UNITS_PER_WORD * i);
part[1][i] = replace_equiv_address (part[1][i], tmp);
}
}
sufficiently aligned, maintain aliasing info accurately. */
dst = expand_constant_movmem_prologue (dst, &src, destreg, srcreg,
desired_align, align_bytes);
- count_exp = plus_constant (count_exp, -align_bytes);
+ count_exp = plus_constant (counter_mode (count_exp),
+ count_exp, -align_bytes);
count -= align_bytes;
}
if (need_zero_guard
sufficiently aligned, maintain aliasing info accurately. */
dst = expand_constant_setmem_prologue (dst, destreg, promoted_val,
desired_align, align_bytes);
- count_exp = plus_constant (count_exp, -align_bytes);
+ count_exp = plus_constant (counter_mode (count_exp),
+ count_exp, -align_bytes);
count -= align_bytes;
}
if (need_zero_guard
/* For given symbol (function) construct code to compute address of it's PLT
entry in large x86-64 PIC model. */
-rtx
+static rtx
construct_plt_address (rtx symbol)
{
rtx tmp, unspec;
}
}
+/* Try to reorder ready list to take advantage of Atom pipelined IMUL
+ execution. It is applied if
+ (1) IMUL instruction is on the top of list;
+ (2) There exists the only producer of independent IMUL instruction in
+ ready list;
+ (3) Put found producer on the top of ready list.
+ Returns issue rate. */
+
+static int
+ix86_sched_reorder(FILE *dump, int sched_verbose, rtx *ready, int *pn_ready,
+ int clock_var ATTRIBUTE_UNUSED)
+{
+ static int issue_rate = -1;
+ int n_ready = *pn_ready;
+ rtx insn, insn1, insn2;
+ int i;
+ sd_iterator_def sd_it;
+ dep_t dep;
+ int index = -1;
+
+ /* Set up issue rate. */
+ issue_rate = ix86_issue_rate();
+
+ /* Do reodering for Atom only. */
+ if (ix86_tune != PROCESSOR_ATOM)
+ return issue_rate;
+ /* Nothing to do if ready list contains only 1 instruction. */
+ if (n_ready <= 1)
+ return issue_rate;
+
+ /* Check that IMUL instruction is on the top of ready list. */
+ insn = ready[n_ready - 1];
+ if (!NONDEBUG_INSN_P (insn))
+ return issue_rate;
+ insn = PATTERN (insn);
+ if (GET_CODE (insn) == PARALLEL)
+ insn = XVECEXP (insn, 0, 0);
+ if (GET_CODE (insn) != SET)
+ return issue_rate;
+ if (!(GET_CODE (SET_SRC (insn)) == MULT
+ && GET_MODE (SET_SRC (insn)) == SImode))
+ return issue_rate;
+
+ /* Search for producer of independent IMUL instruction. */
+ for (i = n_ready - 2; i>= 0; i--)
+ {
+ insn = ready[i];
+ if (!NONDEBUG_INSN_P (insn))
+ continue;
+ /* Skip IMUL instruction. */
+ insn2 = PATTERN (insn);
+ if (GET_CODE (insn2) == PARALLEL)
+ insn2 = XVECEXP (insn2, 0, 0);
+ if (GET_CODE (insn2) == SET
+ && GET_CODE (SET_SRC (insn2)) == MULT
+ && GET_MODE (SET_SRC (insn2)) == SImode)
+ continue;
+
+ FOR_EACH_DEP (insn, SD_LIST_FORW, sd_it, dep)
+ {
+ rtx con;
+ con = DEP_CON (dep);
+ if (!NONDEBUG_INSN_P (con))
+ continue;
+ insn1 = PATTERN (con);
+ if (GET_CODE (insn1) == PARALLEL)
+ insn1 = XVECEXP (insn1, 0, 0);
+
+ if (GET_CODE (insn1) == SET
+ && GET_CODE (SET_SRC (insn1)) == MULT
+ && GET_MODE (SET_SRC (insn1)) == SImode)
+ {
+ sd_iterator_def sd_it1;
+ dep_t dep1;
+ /* Check if there is no other dependee for IMUL. */
+ index = i;
+ FOR_EACH_DEP (con, SD_LIST_BACK, sd_it1, dep1)
+ {
+ rtx pro;
+ pro = DEP_PRO (dep1);
+ if (!NONDEBUG_INSN_P (pro))
+ continue;
+ if (pro != insn)
+ index = -1;
+ }
+ if (index >= 0)
+ break;
+ }
+ }
+ if (index >= 0)
+ break;
+ }
+ if (index < 0)
+ return issue_rate; /* Didn't find IMUL producer. */
+
+ if (sched_verbose > 1)
+ fprintf(dump, ";;\tatom sched_reorder: swap %d and %d insns\n",
+ INSN_UID (ready[index]), INSN_UID (ready[n_ready - 1]));
+
+ /* Put IMUL producer (ready[index]) at the top of ready list. */
+ insn1= ready[index];
+ for (i = index; i < n_ready - 1; i++)
+ ready[i] = ready[i + 1];
+ ready[n_ready - 1] = insn1;
+
+ return issue_rate;
+}
+
\f
/* Model decoder of Core 2/i7.
if (fndecl == current_function_decl)
ix86_static_chain_on_stack = true;
return gen_frame_mem (SImode,
- plus_constant (arg_pointer_rtx, -8));
+ plus_constant (Pmode,
+ arg_pointer_rtx, -8));
}
regno = SI_REG;
}
(call-saved) register static chain; this push is 1 byte. */
offset += 5;
disp = expand_binop (SImode, sub_optab, fnaddr,
- plus_constant (XEXP (m_tramp, 0),
+ plus_constant (Pmode, XEXP (m_tramp, 0),
offset - (MEM_P (chain) ? 1 : 0)),
NULL_RTX, 1, OPTAB_DIRECT);
emit_move_insn (mem, disp);
/* CFString built-in for darwin */
IX86_BUILTIN_CFSTRING,
+ /* Builtins to get CPU type and supported features. */
+ IX86_BUILTIN_CPU_INIT,
+ IX86_BUILTIN_CPU_IS,
+ IX86_BUILTIN_CPU_SUPPORTS,
+
IX86_BUILTIN_MAX
};
{ OPTION_MASK_ISA_SSE, CODE_FOR_sse_vmrsqrtv4sf2, "__builtin_ia32_rsqrtss", IX86_BUILTIN_RSQRTSS, UNKNOWN, (int) V4SF_FTYPE_V4SF_VEC_MERGE },
{ OPTION_MASK_ISA_SSE, CODE_FOR_sse_vmrcpv4sf2, "__builtin_ia32_rcpss", IX86_BUILTIN_RCPSS, UNKNOWN, (int) V4SF_FTYPE_V4SF_VEC_MERGE },
+ { OPTION_MASK_ISA_SSE, CODE_FOR_abstf2, 0, IX86_BUILTIN_FABSQ, UNKNOWN, (int) FLOAT128_FTYPE_FLOAT128 },
+ { OPTION_MASK_ISA_SSE, CODE_FOR_copysigntf3, 0, IX86_BUILTIN_COPYSIGNQ, UNKNOWN, (int) FLOAT128_FTYPE_FLOAT128_FLOAT128 },
+
/* SSE MMX or 3Dnow!A */
{ OPTION_MASK_ISA_SSE | OPTION_MASK_ISA_3DNOW_A, CODE_FOR_mmx_uavgv8qi3, "__builtin_ia32_pavgb", IX86_BUILTIN_PAVGB, UNKNOWN, (int) V8QI_FTYPE_V8QI_V8QI },
{ OPTION_MASK_ISA_SSE | OPTION_MASK_ISA_3DNOW_A, CODE_FOR_mmx_uavgv4hi3, "__builtin_ia32_pavgw", IX86_BUILTIN_PAVGW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
{ OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_psadbw, "__builtin_ia32_psadbw128", IX86_BUILTIN_PSADBW128, UNKNOWN, (int) V2DI_FTYPE_V16QI_V16QI },
{ OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_umulv1siv1di3, "__builtin_ia32_pmuludq", IX86_BUILTIN_PMULUDQ, UNKNOWN, (int) V1DI_FTYPE_V2SI_V2SI },
- { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_umulv2siv2di3, "__builtin_ia32_pmuludq128", IX86_BUILTIN_PMULUDQ128, UNKNOWN, (int) V2DI_FTYPE_V4SI_V4SI },
+ { OPTION_MASK_ISA_SSE2, CODE_FOR_vec_widen_umult_even_v4si, "__builtin_ia32_pmuludq128", IX86_BUILTIN_PMULUDQ128, UNKNOWN, (int) V2DI_FTYPE_V4SI_V4SI },
{ OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_pmaddwd, "__builtin_ia32_pmaddwd128", IX86_BUILTIN_PMADDWD128, UNKNOWN, (int) V4SI_FTYPE_V8HI_V8HI },
{ OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_vmsqrtv2df2, "__builtin_ia32_sqrtsd", IX86_BUILTIN_SQRTSD, UNKNOWN, (int) V2DF_FTYPE_V2DF_VEC_MERGE },
- { OPTION_MASK_ISA_SSE2, CODE_FOR_abstf2, 0, IX86_BUILTIN_FABSQ, UNKNOWN, (int) FLOAT128_FTYPE_FLOAT128 },
- { OPTION_MASK_ISA_SSE2, CODE_FOR_copysigntf3, 0, IX86_BUILTIN_COPYSIGNQ, UNKNOWN, (int) FLOAT128_FTYPE_FLOAT128_FLOAT128 },
-
{ OPTION_MASK_ISA_SSE, CODE_FOR_sse2_movq128, "__builtin_ia32_movq128", IX86_BUILTIN_MOVQ128, UNKNOWN, (int) V2DI_FTYPE_V2DI },
/* SSE2 MMX */
{ OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_zero_extendv8hiv8si2 , "__builtin_ia32_pmovzxwd256", IX86_BUILTIN_PMOVZXWD256, UNKNOWN, (int) V8SI_FTYPE_V8HI },
{ OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_zero_extendv4hiv4di2 , "__builtin_ia32_pmovzxwq256", IX86_BUILTIN_PMOVZXWQ256, UNKNOWN, (int) V4DI_FTYPE_V8HI },
{ OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_zero_extendv4siv4di2 , "__builtin_ia32_pmovzxdq256", IX86_BUILTIN_PMOVZXDQ256, UNKNOWN, (int) V4DI_FTYPE_V4SI },
- { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_mulv4siv4di3 , "__builtin_ia32_pmuldq256" , IX86_BUILTIN_PMULDQ256 , UNKNOWN, (int) V4DI_FTYPE_V8SI_V8SI },
+ { OPTION_MASK_ISA_AVX2, CODE_FOR_vec_widen_smult_even_v8si, "__builtin_ia32_pmuldq256", IX86_BUILTIN_PMULDQ256, UNKNOWN, (int) V4DI_FTYPE_V8SI_V8SI },
{ OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_umulhrswv16hi3 , "__builtin_ia32_pmulhrsw256", IX86_BUILTIN_PMULHRSW256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI },
{ OPTION_MASK_ISA_AVX2, CODE_FOR_umulv16hi3_highpart, "__builtin_ia32_pmulhuw256" , IX86_BUILTIN_PMULHUW256 , UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI },
{ OPTION_MASK_ISA_AVX2, CODE_FOR_smulv16hi3_highpart, "__builtin_ia32_pmulhw256" , IX86_BUILTIN_PMULHW256 , UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI },
{ OPTION_MASK_ISA_AVX2, CODE_FOR_mulv16hi3, "__builtin_ia32_pmullw256" , IX86_BUILTIN_PMULLW256 , UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI },
{ OPTION_MASK_ISA_AVX2, CODE_FOR_mulv8si3, "__builtin_ia32_pmulld256" , IX86_BUILTIN_PMULLD256 , UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI },
- { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_umulv4siv4di3 , "__builtin_ia32_pmuludq256" , IX86_BUILTIN_PMULUDQ256 , UNKNOWN, (int) V4DI_FTYPE_V8SI_V8SI },
+ { OPTION_MASK_ISA_AVX2, CODE_FOR_vec_widen_umult_even_v8si, "__builtin_ia32_pmuludq256", IX86_BUILTIN_PMULUDQ256, UNKNOWN, (int) V4DI_FTYPE_V8SI_V8SI },
{ OPTION_MASK_ISA_AVX2, CODE_FOR_iorv4di3, "__builtin_ia32_por256", IX86_BUILTIN_POR256, UNKNOWN, (int) V4DI_FTYPE_V4DI_V4DI },
{ OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_psadbw, "__builtin_ia32_psadbw256", IX86_BUILTIN_PSADBW256, UNKNOWN, (int) V16HI_FTYPE_V32QI_V32QI },
{ OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_pshufbv32qi3, "__builtin_ia32_pshufb256", IX86_BUILTIN_PSHUFB256, UNKNOWN, (int) V32QI_FTYPE_V32QI_V32QI },
{ OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_pbroadcastv4si, "__builtin_ia32_pbroadcastd128", IX86_BUILTIN_PBROADCASTD128, UNKNOWN, (int) V4SI_FTYPE_V4SI },
{ OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_pbroadcastv2di, "__builtin_ia32_pbroadcastq128", IX86_BUILTIN_PBROADCASTQ128, UNKNOWN, (int) V2DI_FTYPE_V2DI },
{ OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_permvarv8si, "__builtin_ia32_permvarsi256", IX86_BUILTIN_VPERMVARSI256, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI },
+ { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_permvarv8sf, "__builtin_ia32_permvarsf256", IX86_BUILTIN_VPERMVARSF256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SI },
{ OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_permv4df, "__builtin_ia32_permdf256", IX86_BUILTIN_VPERMDF256, UNKNOWN, (int) V4DF_FTYPE_V4DF_INT },
- { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_permvarv8sf, "__builtin_ia32_permvarsf256", IX86_BUILTIN_VPERMVARSF256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF },
{ OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_permv4di, "__builtin_ia32_permdi256", IX86_BUILTIN_VPERMDI256, UNKNOWN, (int) V4DI_FTYPE_V4DI_INT },
{ OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_permv2ti, "__builtin_ia32_permti256", IX86_BUILTIN_VPERMTI256, UNKNOWN, (int) V4DI_FTYPE_V4DI_V4DI_INT },
{ OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_extracti128, "__builtin_ia32_extract128i256", IX86_BUILTIN_VEXTRACT128I256, UNKNOWN, (int) V2DI_FTYPE_V4DI_INT },
}
}
+/* This builds the processor_model struct type defined in
+ libgcc/config/i386/cpuinfo.c */
+
+static tree
+build_processor_model_struct (void)
+{
+ const char *field_name[] = {"__cpu_vendor", "__cpu_type", "__cpu_subtype",
+ "__cpu_features"};
+ tree field = NULL_TREE, field_chain = NULL_TREE;
+ int i;
+ tree type = make_node (RECORD_TYPE);
+
+ /* The first 3 fields are unsigned int. */
+ for (i = 0; i < 3; ++i)
+ {
+ field = build_decl (UNKNOWN_LOCATION, FIELD_DECL,
+ get_identifier (field_name[i]), unsigned_type_node);
+ if (field_chain != NULL_TREE)
+ DECL_CHAIN (field) = field_chain;
+ field_chain = field;
+ }
+
+ /* The last field is an array of unsigned integers of size one. */
+ field = build_decl (UNKNOWN_LOCATION, FIELD_DECL,
+ get_identifier (field_name[3]),
+ build_array_type (unsigned_type_node,
+ build_index_type (size_one_node)));
+ if (field_chain != NULL_TREE)
+ DECL_CHAIN (field) = field_chain;
+ field_chain = field;
+
+ finish_builtin_struct (type, "__processor_model", field_chain, NULL_TREE);
+ return type;
+}
+
+/* Returns a extern, comdat VAR_DECL of type TYPE and name NAME. */
+
+static tree
+make_var_decl (tree type, const char *name)
+{
+ tree new_decl;
+
+ new_decl = build_decl (UNKNOWN_LOCATION,
+ VAR_DECL,
+ get_identifier(name),
+ type);
+
+ DECL_EXTERNAL (new_decl) = 1;
+ TREE_STATIC (new_decl) = 1;
+ TREE_PUBLIC (new_decl) = 1;
+ DECL_INITIAL (new_decl) = 0;
+ DECL_ARTIFICIAL (new_decl) = 0;
+ DECL_PRESERVE_P (new_decl) = 1;
+
+ make_decl_one_only (new_decl, DECL_ASSEMBLER_NAME (new_decl));
+ assemble_variable (new_decl, 0, 0, 0);
+
+ return new_decl;
+}
+
+/* FNDECL is a __builtin_cpu_is or a __builtin_cpu_supports call that is folded
+ into an integer defined in libgcc/config/i386/cpuinfo.c */
+
+static tree
+fold_builtin_cpu (tree fndecl, tree *args)
+{
+ unsigned int i;
+ enum ix86_builtins fn_code = (enum ix86_builtins)
+ DECL_FUNCTION_CODE (fndecl);
+ tree param_string_cst = NULL;
+
+ /* This is the order of bit-fields in __processor_features in cpuinfo.c */
+ enum processor_features
+ {
+ F_CMOV = 0,
+ F_MMX,
+ F_POPCNT,
+ F_SSE,
+ F_SSE2,
+ F_SSE3,
+ F_SSSE3,
+ F_SSE4_1,
+ F_SSE4_2,
+ F_AVX,
+ F_AVX2,
+ F_MAX
+ };
+
+ /* These are the values for vendor types and cpu types and subtypes
+ in cpuinfo.c. Cpu types and subtypes should be subtracted by
+ the corresponding start value. */
+ enum processor_model
+ {
+ M_INTEL = 1,
+ M_AMD,
+ M_CPU_TYPE_START,
+ M_INTEL_ATOM,
+ M_INTEL_CORE2,
+ M_INTEL_COREI7,
+ M_AMDFAM10H,
+ M_AMDFAM15H,
+ M_CPU_SUBTYPE_START,
+ M_INTEL_COREI7_NEHALEM,
+ M_INTEL_COREI7_WESTMERE,
+ M_INTEL_COREI7_SANDYBRIDGE,
+ M_AMDFAM10H_BARCELONA,
+ M_AMDFAM10H_SHANGHAI,
+ M_AMDFAM10H_ISTANBUL,
+ M_AMDFAM15H_BDVER1,
+ M_AMDFAM15H_BDVER2
+ };
+
+ static struct _arch_names_table
+ {
+ const char *const name;
+ const enum processor_model model;
+ }
+ const arch_names_table[] =
+ {
+ {"amd", M_AMD},
+ {"intel", M_INTEL},
+ {"atom", M_INTEL_ATOM},
+ {"core2", M_INTEL_CORE2},
+ {"corei7", M_INTEL_COREI7},
+ {"nehalem", M_INTEL_COREI7_NEHALEM},
+ {"westmere", M_INTEL_COREI7_WESTMERE},
+ {"sandybridge", M_INTEL_COREI7_SANDYBRIDGE},
+ {"amdfam10h", M_AMDFAM10H},
+ {"barcelona", M_AMDFAM10H_BARCELONA},
+ {"shanghai", M_AMDFAM10H_SHANGHAI},
+ {"istanbul", M_AMDFAM10H_ISTANBUL},
+ {"amdfam15h", M_AMDFAM15H},
+ {"bdver1", M_AMDFAM15H_BDVER1},
+ {"bdver2", M_AMDFAM15H_BDVER2},
+ };
+
+ static struct _isa_names_table
+ {
+ const char *const name;
+ const enum processor_features feature;
+ }
+ const isa_names_table[] =
+ {
+ {"cmov", F_CMOV},
+ {"mmx", F_MMX},
+ {"popcnt", F_POPCNT},
+ {"sse", F_SSE},
+ {"sse2", F_SSE2},
+ {"sse3", F_SSE3},
+ {"ssse3", F_SSSE3},
+ {"sse4.1", F_SSE4_1},
+ {"sse4.2", F_SSE4_2},
+ {"avx", F_AVX},
+ {"avx2", F_AVX2}
+ };
+
+ static tree __processor_model_type = NULL_TREE;
+ static tree __cpu_model_var = NULL_TREE;
+
+ if (__processor_model_type == NULL_TREE)
+ __processor_model_type = build_processor_model_struct ();
+
+ if (__cpu_model_var == NULL_TREE)
+ __cpu_model_var = make_var_decl (__processor_model_type,
+ "__cpu_model");
+
+ gcc_assert ((args != NULL) && (*args != NULL));
+
+ param_string_cst = *args;
+ while (param_string_cst
+ && TREE_CODE (param_string_cst) != STRING_CST)
+ {
+ /* *args must be a expr that can contain other EXPRS leading to a
+ STRING_CST. */
+ if (!EXPR_P (param_string_cst))
+ {
+ error ("Parameter to builtin must be a string constant or literal");
+ return integer_zero_node;
+ }
+ param_string_cst = TREE_OPERAND (EXPR_CHECK (param_string_cst), 0);
+ }
+
+ gcc_assert (param_string_cst);
+
+ if (fn_code == IX86_BUILTIN_CPU_IS)
+ {
+ tree ref;
+ tree field;
+ unsigned int field_val = 0;
+ unsigned int NUM_ARCH_NAMES
+ = sizeof (arch_names_table) / sizeof (struct _arch_names_table);
+
+ for (i = 0; i < NUM_ARCH_NAMES; i++)
+ if (strcmp (arch_names_table[i].name,
+ TREE_STRING_POINTER (param_string_cst)) == 0)
+ break;
+
+ if (i == NUM_ARCH_NAMES)
+ {
+ error ("Parameter to builtin not valid: %s",
+ TREE_STRING_POINTER (param_string_cst));
+ return integer_zero_node;
+ }
+
+ field = TYPE_FIELDS (__processor_model_type);
+ field_val = arch_names_table[i].model;
+
+ /* CPU types are stored in the next field. */
+ if (field_val > M_CPU_TYPE_START
+ && field_val < M_CPU_SUBTYPE_START)
+ {
+ field = DECL_CHAIN (field);
+ field_val -= M_CPU_TYPE_START;
+ }
+
+ /* CPU subtypes are stored in the next field. */
+ if (field_val > M_CPU_SUBTYPE_START)
+ {
+ field = DECL_CHAIN ( DECL_CHAIN (field));
+ field_val -= M_CPU_SUBTYPE_START;
+ }
+
+ /* Get the appropriate field in __cpu_model. */
+ ref = build3 (COMPONENT_REF, TREE_TYPE (field), __cpu_model_var,
+ field, NULL_TREE);
+
+ /* Check the value. */
+ return build2 (EQ_EXPR, unsigned_type_node, ref,
+ build_int_cstu (unsigned_type_node, field_val));
+ }
+ else if (fn_code == IX86_BUILTIN_CPU_SUPPORTS)
+ {
+ tree ref;
+ tree array_elt;
+ tree field;
+ unsigned int field_val = 0;
+ unsigned int NUM_ISA_NAMES
+ = sizeof (isa_names_table) / sizeof (struct _isa_names_table);
+
+ for (i = 0; i < NUM_ISA_NAMES; i++)
+ if (strcmp (isa_names_table[i].name,
+ TREE_STRING_POINTER (param_string_cst)) == 0)
+ break;
+
+ if (i == NUM_ISA_NAMES)
+ {
+ error ("Parameter to builtin not valid: %s",
+ TREE_STRING_POINTER (param_string_cst));
+ return integer_zero_node;
+ }
+
+ field = TYPE_FIELDS (__processor_model_type);
+ /* Get the last field, which is __cpu_features. */
+ while (DECL_CHAIN (field))
+ field = DECL_CHAIN (field);
+
+ /* Get the appropriate field: __cpu_model.__cpu_features */
+ ref = build3 (COMPONENT_REF, TREE_TYPE (field), __cpu_model_var,
+ field, NULL_TREE);
+
+ /* Access the 0th element of __cpu_features array. */
+ array_elt = build4 (ARRAY_REF, unsigned_type_node, ref,
+ integer_zero_node, NULL_TREE, NULL_TREE);
+
+ field_val = (1 << isa_names_table[i].feature);
+ /* Return __cpu_model.__cpu_features[0] & field_val */
+ return build2 (BIT_AND_EXPR, unsigned_type_node, array_elt,
+ build_int_cstu (unsigned_type_node, field_val));
+ }
+ gcc_unreachable ();
+}
+
+static tree
+ix86_fold_builtin (tree fndecl, int n_args,
+ tree *args, bool ignore ATTRIBUTE_UNUSED)
+{
+ if (DECL_BUILT_IN_CLASS (fndecl) == BUILT_IN_MD)
+ {
+ enum ix86_builtins fn_code = (enum ix86_builtins)
+ DECL_FUNCTION_CODE (fndecl);
+ if (fn_code == IX86_BUILTIN_CPU_IS
+ || fn_code == IX86_BUILTIN_CPU_SUPPORTS)
+ {
+ gcc_assert (n_args == 1);
+ return fold_builtin_cpu (fndecl, args);
+ }
+ }
+
+#ifdef SUBTARGET_FOLD_BUILTIN
+ return SUBTARGET_FOLD_BUILTIN (fndecl, n_args, args, ignore);
+#endif
+
+ return NULL_TREE;
+}
+
+/* Make builtins to detect cpu type and features supported. NAME is
+ the builtin name, CODE is the builtin code, and FTYPE is the function
+ type of the builtin. */
+
+static void
+make_cpu_type_builtin (const char* name, int code,
+ enum ix86_builtin_func_type ftype, bool is_const)
+{
+ tree decl;
+ tree type;
+
+ type = ix86_get_builtin_func_type (ftype);
+ decl = add_builtin_function (name, type, code, BUILT_IN_MD,
+ NULL, NULL_TREE);
+ gcc_assert (decl != NULL_TREE);
+ ix86_builtins[(int) code] = decl;
+ TREE_READONLY (decl) = is_const;
+}
+
+/* Make builtins to get CPU type and features supported. The created
+ builtins are :
+
+ __builtin_cpu_init (), to detect cpu type and features,
+ __builtin_cpu_is ("<CPUNAME>"), to check if cpu is of type <CPUNAME>,
+ __builtin_cpu_supports ("<FEATURE>"), to check if cpu supports <FEATURE>
+ */
+
+static void
+ix86_init_platform_type_builtins (void)
+{
+ make_cpu_type_builtin ("__builtin_cpu_init", IX86_BUILTIN_CPU_INIT,
+ INT_FTYPE_VOID, false);
+ make_cpu_type_builtin ("__builtin_cpu_is", IX86_BUILTIN_CPU_IS,
+ INT_FTYPE_PCCHAR, true);
+ make_cpu_type_builtin ("__builtin_cpu_supports", IX86_BUILTIN_CPU_SUPPORTS,
+ INT_FTYPE_PCCHAR, true);
+}
+
/* Internal method for ix86_init_builtins. */
static void
ix86_init_builtin_types ();
+ /* Builtins to get CPU type and features. */
+ ix86_init_platform_type_builtins ();
+
/* TFmode support builtins. */
def_builtin_const (0, "__builtin_infq",
FLOAT128_FTYPE_VOID, IX86_BUILTIN_INFQ);
def_builtin_const (0, "__builtin_huge_valq",
FLOAT128_FTYPE_VOID, IX86_BUILTIN_HUGE_VALQ);
- /* We will expand them to normal call if SSE2 isn't available since
+ /* We will expand them to normal call if SSE isn't available since
they are used by libgcc. */
t = ix86_get_builtin_func_type (FLOAT128_FTYPE_FLOAT128);
t = add_builtin_function ("__builtin_fabsq", t, IX86_BUILTIN_FABSQ,
case V2DI_FTYPE_V2DI_V2DI:
case V2DI_FTYPE_V16QI_V16QI:
case V2DI_FTYPE_V4SI_V4SI:
+ case V2UDI_FTYPE_V4USI_V4USI:
case V2DI_FTYPE_V2DI_V16QI:
case V2DI_FTYPE_V2DF_V2DF:
case V2SI_FTYPE_V2SI_V2SI:
case V8SI_FTYPE_V16HI_V16HI:
case V4DI_FTYPE_V4DI_V4DI:
case V4DI_FTYPE_V8SI_V8SI:
+ case V4UDI_FTYPE_V8USI_V8USI:
if (comparison == UNKNOWN)
return ix86_expand_binop_builtin (icode, exp, target);
nargs = 2;
arg_adjust = 0;
if (optimize
|| target == 0
- || GET_MODE (target) != tmode
- || !insn_p->operand[0].predicate (target, tmode))
+ || !register_operand (target, tmode)
+ || GET_MODE (target) != tmode)
target = gen_reg_rtx (tmode);
}
enum machine_mode mode0, mode1, mode2, mode3, mode4;
unsigned int fcode = DECL_FUNCTION_CODE (fndecl);
+ /* For CPU builtins that can be folded, fold first and expand the fold. */
+ switch (fcode)
+ {
+ case IX86_BUILTIN_CPU_INIT:
+ {
+ /* Make it call __cpu_indicator_init in libgcc. */
+ tree call_expr, fndecl, type;
+ type = build_function_type_list (integer_type_node, NULL_TREE);
+ fndecl = build_fn_decl ("__cpu_indicator_init", type);
+ call_expr = build_call_expr (fndecl, 0);
+ return expand_expr (call_expr, target, mode, EXPAND_NORMAL);
+ }
+ case IX86_BUILTIN_CPU_IS:
+ case IX86_BUILTIN_CPU_SUPPORTS:
+ {
+ tree arg0 = CALL_EXPR_ARG (exp, 0);
+ tree fold_expr = fold_builtin_cpu (fndecl, &arg0);
+ gcc_assert (fold_expr != NULL_TREE);
+ return expand_expr (fold_expr, target, mode, EXPAND_NORMAL);
+ }
+ }
+
/* Determine whether the builtin function is available under the current ISA.
Originally the builtin was not created if it wasn't applicable to the
current ISA based on the command line switches. With function specific
{
case IX86_BUILTIN_FABSQ:
case IX86_BUILTIN_COPYSIGNQ:
- if (!TARGET_SSE2)
- /* Emit a normal call if SSE2 isn't available. */
+ if (!TARGET_SSE)
+ /* Emit a normal call if SSE isn't available. */
return expand_call (exp, target, ignore);
default:
return ix86_expand_args_builtin (d, exp, target);
/* If MODE2 is only appropriate for an SSE register, then tie with
any other mode acceptable to SSE registers. */
+ if (GET_MODE_SIZE (mode2) == 32
+ && ix86_hard_regno_mode_ok (FIRST_SSE_REG, mode2))
+ return (GET_MODE_SIZE (mode1) == 32
+ && ix86_hard_regno_mode_ok (FIRST_SSE_REG, mode1));
if (GET_MODE_SIZE (mode2) == 16
&& ix86_hard_regno_mode_ok (FIRST_SSE_REG, mode2))
return (GET_MODE_SIZE (mode1) == 16
return false;
}
+/* Return the cost of moving between two registers of mode MODE. */
+
+static int
+ix86_set_reg_reg_cost (enum machine_mode mode)
+{
+ unsigned int units = UNITS_PER_WORD;
+
+ switch (GET_MODE_CLASS (mode))
+ {
+ default:
+ break;
+
+ case MODE_CC:
+ units = GET_MODE_SIZE (CCmode);
+ break;
+
+ case MODE_FLOAT:
+ if ((TARGET_SSE && mode == TFmode)
+ || (TARGET_80387 && mode == XFmode)
+ || ((TARGET_80387 || TARGET_SSE2) && mode == DFmode)
+ || ((TARGET_80387 || TARGET_SSE) && mode == SFmode))
+ units = GET_MODE_SIZE (mode);
+ break;
+
+ case MODE_COMPLEX_FLOAT:
+ if ((TARGET_SSE && mode == TCmode)
+ || (TARGET_80387 && mode == XCmode)
+ || ((TARGET_80387 || TARGET_SSE2) && mode == DCmode)
+ || ((TARGET_80387 || TARGET_SSE) && mode == SCmode))
+ units = GET_MODE_SIZE (mode);
+ break;
+
+ case MODE_VECTOR_INT:
+ case MODE_VECTOR_FLOAT:
+ if ((TARGET_AVX && VALID_AVX256_REG_MODE (mode))
+ || (TARGET_SSE2 && VALID_SSE2_REG_MODE (mode))
+ || (TARGET_SSE && VALID_SSE_REG_MODE (mode))
+ || (TARGET_MMX && VALID_MMX_REG_MODE (mode)))
+ units = GET_MODE_SIZE (mode);
+ }
+
+ /* Return the cost of moving between two registers of mode MODE,
+ assuming that the move will be in pieces of at most UNITS bytes. */
+ return COSTS_N_INSNS ((GET_MODE_SIZE (mode) + units - 1) / units);
+}
+
/* Compute a (partial) cost for rtx X. Return true if the complete
cost has been computed, and false if subexpressions should be
scanned. In either case, *TOTAL contains the cost result. */
static bool
-ix86_rtx_costs (rtx x, int code, int outer_code_i, int opno, int *total,
+ix86_rtx_costs (rtx x, int code_i, int outer_code_i, int opno, int *total,
bool speed)
{
+ enum rtx_code code = (enum rtx_code) code_i;
enum rtx_code outer_code = (enum rtx_code) outer_code_i;
enum machine_mode mode = GET_MODE (x);
const struct processor_costs *cost = speed ? ix86_cost : &ix86_size_cost;
switch (code)
{
+ case SET:
+ if (register_operand (SET_DEST (x), VOIDmode)
+ && reg_or_0_operand (SET_SRC (x), VOIDmode))
+ {
+ *total = ix86_set_reg_reg_cost (GET_MODE (SET_DEST (x)));
+ return true;
+ }
+ return false;
+
case CONST_INT:
case CONST:
case LABEL_REF:
case CONST_DOUBLE:
if (mode == VOIDmode)
- *total = 0;
- else
- switch (standard_80387_constant_p (x))
- {
- case 1: /* 0.0 */
- *total = 1;
- break;
- default: /* Other constants */
- *total = 2;
- break;
- case 0:
- case -1:
- /* Start with (MEM (SYMBOL_REF)), since that's where
- it'll probably end up. Add a penalty for size. */
- *total = (COSTS_N_INSNS (1)
- + (flag_pic != 0 && !TARGET_64BIT)
- + (mode == SFmode ? 0 : mode == DFmode ? 1 : 2));
- break;
- }
+ {
+ *total = 0;
+ return true;
+ }
+ switch (standard_80387_constant_p (x))
+ {
+ case 1: /* 0.0 */
+ *total = 1;
+ return true;
+ default: /* Other constants */
+ *total = 2;
+ return true;
+ case 0:
+ case -1:
+ break;
+ }
+ if (SSE_FLOAT_MODE_P (mode))
+ {
+ case CONST_VECTOR:
+ switch (standard_sse_constant_p (x))
+ {
+ case 0:
+ break;
+ case 1: /* 0: xor eliminates false dependency */
+ *total = 0;
+ return true;
+ default: /* -1: cmp contains false dependency */
+ *total = 1;
+ return true;
+ }
+ }
+ /* Fall back to (MEM (SYMBOL_REF)), since that's where
+ it'll probably end up. Add a penalty for size. */
+ *total = (COSTS_N_INSNS (1)
+ + (flag_pic != 0 && !TARGET_64BIT)
+ + (mode == SFmode ? 0 : mode == DFmode ? 1 : 2));
return true;
case ZERO_EXTEND:
return false;
case ASHIFT:
- if (CONST_INT_P (XEXP (x, 1))
- && (GET_MODE (XEXP (x, 0)) != DImode || TARGET_64BIT))
+ if (SCALAR_INT_MODE_P (mode)
+ && GET_MODE_SIZE (mode) < UNITS_PER_WORD
+ && CONST_INT_P (XEXP (x, 1)))
{
HOST_WIDE_INT value = INTVAL (XEXP (x, 1));
if (value == 1)
case ASHIFTRT:
case LSHIFTRT:
case ROTATERT:
- if (!TARGET_64BIT && GET_MODE (XEXP (x, 0)) == DImode)
+ if (GET_MODE_CLASS (mode) == MODE_VECTOR_INT)
+ {
+ /* ??? Should be SSE vector operation cost. */
+ /* At least for published AMD latencies, this really is the same
+ as the latency for a simple fpu operation like fabs. */
+ /* V*QImode is emulated with 1-11 insns. */
+ if (mode == V16QImode || mode == V32QImode)
+ {
+ int count = 11;
+ if (TARGET_XOP && mode == V16QImode)
+ {
+ /* For XOP we use vpshab, which requires a broadcast of the
+ value to the variable shift insn. For constants this
+ means a V16Q const in mem; even when we can perform the
+ shift with one insn set the cost to prefer paddb. */
+ if (CONSTANT_P (XEXP (x, 1)))
+ {
+ *total = (cost->fabs
+ + rtx_cost (XEXP (x, 0), code, 0, speed)
+ + (speed ? 2 : COSTS_N_BYTES (16)));
+ return true;
+ }
+ count = 3;
+ }
+ else if (TARGET_SSSE3)
+ count = 7;
+ *total = cost->fabs * count;
+ }
+ else
+ *total = cost->fabs;
+ }
+ else if (GET_MODE_SIZE (mode) > UNITS_PER_WORD)
{
if (CONST_INT_P (XEXP (x, 1)))
{
*total = cost->fmul;
return false;
}
+ else if (GET_MODE_CLASS (mode) == MODE_VECTOR_INT)
+ {
+ /* V*QImode is emulated with 7-13 insns. */
+ if (mode == V16QImode || mode == V32QImode)
+ {
+ int extra = 11;
+ if (TARGET_XOP && mode == V16QImode)
+ extra = 5;
+ else if (TARGET_SSSE3)
+ extra = 6;
+ *total = cost->fmul * 2 + cost->fabs * extra;
+ }
+ /* V*DImode is emulated with 5-8 insns. */
+ else if (mode == V2DImode || mode == V4DImode)
+ {
+ if (TARGET_XOP && mode == V2DImode)
+ *total = cost->fmul * 2 + cost->fabs * 3;
+ else
+ *total = cost->fmul * 3 + cost->fabs * 5;
+ }
+ /* Without sse4.1, we don't have PMULLD; it's emulated with 7
+ insns, including two PMULUDQ. */
+ else if (mode == V4SImode && !(TARGET_SSE4_1 || TARGET_AVX))
+ *total = cost->fmul * 2 + cost->fabs * 5;
+ else
+ *total = cost->fmul;
+ return false;
+ }
else
{
rtx op0 = XEXP (x, 0);
case PLUS:
if (GET_MODE_CLASS (mode) == MODE_INT
- && GET_MODE_BITSIZE (mode) <= GET_MODE_BITSIZE (Pmode))
+ && GET_MODE_SIZE (mode) <= UNITS_PER_WORD)
{
if (GET_CODE (XEXP (x, 0)) == PLUS
&& GET_CODE (XEXP (XEXP (x, 0), 0)) == MULT
case AND:
case IOR:
case XOR:
- if (!TARGET_64BIT && mode == DImode)
+ if (GET_MODE_SIZE (mode) > UNITS_PER_WORD)
{
*total = (cost->add * 2
+ (rtx_cost (XEXP (x, 0), outer_code, opno, speed)
/* FALLTHRU */
case NOT:
- if (!TARGET_64BIT && mode == DImode)
+ if (GET_MODE_CLASS (mode) == MODE_VECTOR_INT)
+ {
+ /* ??? Should be SSE vector operation cost. */
+ /* At least for published AMD latencies, this really is the same
+ as the latency for a simple fpu operation like fabs. */
+ *total = cost->fabs;
+ }
+ else if (GET_MODE_SIZE (mode) > UNITS_PER_WORD)
*total = cost->add * 2;
else
*total = cost->add;
/* ??? Assume all of these vector manipulation patterns are
recognizable. In which case they all pretty much have the
same cost. */
- *total = COSTS_N_INSNS (1);
+ *total = cost->fabs;
return true;
default:
else
type = node;
- if (!(type && (TREE_CODE (*type) == RECORD_TYPE
- || TREE_CODE (*type) == UNION_TYPE)))
+ if (!(type && RECORD_OR_UNION_TYPE_P (*type)))
{
warning (OPT_Wattributes, "%qE attribute ignored",
name);
regno = CX_REG;
if (aggr)
return gen_rtx_MEM (SImode,
- plus_constant (stack_pointer_rtx, 4));
+ plus_constant (Pmode, stack_pointer_rtx, 4));
}
else
{
regno = DX_REG;
if (nregs == 1)
return gen_rtx_MEM (SImode,
- plus_constant (stack_pointer_rtx, 4));
+ plus_constant (Pmode,
+ stack_pointer_rtx, 4));
}
}
return gen_rtx_REG (SImode, regno);
}
- return gen_rtx_MEM (SImode, plus_constant (stack_pointer_rtx, aggr ? 8 : 4));
+ return gen_rtx_MEM (SImode, plus_constant (Pmode, stack_pointer_rtx,
+ aggr ? 8 : 4));
}
/* Determine whether x86_output_mi_thunk can succeed. */
{
rtx this_param = x86_this_parameter (function);
rtx this_reg, tmp, fnaddr;
+ unsigned int tmp_regno;
+
+ if (TARGET_64BIT)
+ tmp_regno = R10_REG;
+ else
+ {
+ unsigned int ccvt = ix86_get_callcvt (TREE_TYPE (function));
+ if ((ccvt & (IX86_CALLCVT_FASTCALL | IX86_CALLCVT_THISCALL)) != 0)
+ tmp_regno = AX_REG;
+ else
+ tmp_regno = CX_REG;
+ }
emit_note (NOTE_INSN_PROLOGUE_END);
{
if (!x86_64_general_operand (delta_rtx, Pmode))
{
- tmp = gen_rtx_REG (Pmode, R10_REG);
+ tmp = gen_rtx_REG (Pmode, tmp_regno);
emit_move_insn (tmp, delta_rtx);
delta_rtx = tmp;
}
if (vcall_offset)
{
rtx vcall_addr, vcall_mem, this_mem;
- unsigned int tmp_regno;
- if (TARGET_64BIT)
- tmp_regno = R10_REG;
- else
- {
- unsigned int ccvt = ix86_get_callcvt (TREE_TYPE (function));
- if ((ccvt & (IX86_CALLCVT_FASTCALL | IX86_CALLCVT_THISCALL)) != 0)
- tmp_regno = AX_REG;
- else
- tmp_regno = CX_REG;
- }
tmp = gen_rtx_REG (Pmode, tmp_regno);
this_mem = gen_rtx_MEM (ptr_mode, this_reg);
emit_move_insn (tmp, this_mem);
/* Adjust the this parameter. */
- vcall_addr = plus_constant (tmp, vcall_offset);
+ vcall_addr = plus_constant (Pmode, tmp, vcall_offset);
if (TARGET_64BIT
&& !ix86_legitimate_address_p (ptr_mode, vcall_addr, true))
{
emit_jump_insn (gen_indirect_jump (fnaddr));
else
{
+ if (ix86_cmodel == CM_LARGE_PIC && SYMBOLIC_CONST (fnaddr))
+ fnaddr = legitimize_pic_address (fnaddr,
+ gen_rtx_REG (Pmode, tmp_regno));
+
+ if (!sibcall_insn_operand (fnaddr, word_mode))
+ {
+ tmp = gen_rtx_REG (word_mode, tmp_regno);
+ if (GET_MODE (fnaddr) != word_mode)
+ fnaddr = gen_rtx_ZERO_EXTEND (word_mode, fnaddr);
+ emit_move_insn (tmp, fnaddr);
+ fnaddr = tmp;
+ }
+
tmp = gen_rtx_MEM (QImode, fnaddr);
tmp = gen_rtx_CALL (VOIDmode, tmp, const0_rtx);
tmp = emit_call_insn (tmp);
return min_prev_count;
}
-/* Pad short funtion to 4 instructions. */
+/* Pad short function to 4 instructions. */
static void
ix86_pad_short_function (void)
unsigned char perm[MAX_VECT_LEN];
enum machine_mode vmode;
unsigned char nelt;
+ bool one_operand_p;
bool testing_p;
};
+static bool canonicalize_perm (struct expand_vec_perm_d *d);
static bool expand_vec_perm_1 (struct expand_vec_perm_d *d);
static bool expand_vec_perm_broadcast_1 (struct expand_vec_perm_d *d);
dperm.vmode = mode;
dperm.nelt = GET_MODE_NUNITS (mode);
dperm.op0 = dperm.op1 = gen_reg_rtx (mode);
+ dperm.one_operand_p = true;
/* Extend to SImode using a paradoxical SUBREG. */
tmp1 = gen_reg_rtx (SImode);
}
else
{
- rtx mem = assign_stack_temp (mode, GET_MODE_SIZE (mode), false);
+ rtx mem = assign_stack_temp (mode, GET_MODE_SIZE (mode));
emit_move_insn (mem, target);
}
else
{
- rtx mem = assign_stack_temp (mode, GET_MODE_SIZE (mode), false);
+ rtx mem = assign_stack_temp (mode, GET_MODE_SIZE (mode));
emit_move_insn (mem, vec);
/* Implement targetm.vectorize.builtin_vectorization_cost. */
static int
ix86_builtin_vectorization_cost (enum vect_cost_for_stmt type_of_cost,
- tree vectype ATTRIBUTE_UNUSED,
+ tree vectype,
int misalign ATTRIBUTE_UNUSED)
{
+ unsigned elements;
+
switch (type_of_cost)
{
case scalar_stmt:
case vec_promote_demote:
return ix86_cost->vec_stmt_cost;
+ case vec_construct:
+ elements = TYPE_VECTOR_SUBPARTS (vectype);
+ return elements / 2 + 1;
+
default:
gcc_unreachable ();
}
}
+/* A cached (set (nil) (vselect (vconcat (nil) (nil)) (parallel [])))
+ insn, so that expand_vselect{,_vconcat} doesn't have to create a fresh
+ insn every time. */
+
+static GTY(()) rtx vselect_insn;
+
+/* Initialize vselect_insn. */
+
+static void
+init_vselect_insn (void)
+{
+ unsigned i;
+ rtx x;
+
+ x = gen_rtx_PARALLEL (VOIDmode, rtvec_alloc (MAX_VECT_LEN));
+ for (i = 0; i < MAX_VECT_LEN; ++i)
+ XVECEXP (x, 0, i) = const0_rtx;
+ x = gen_rtx_VEC_SELECT (V2DFmode, gen_rtx_VEC_CONCAT (V4DFmode, const0_rtx,
+ const0_rtx), x);
+ x = gen_rtx_SET (VOIDmode, const0_rtx, x);
+ start_sequence ();
+ vselect_insn = emit_insn (x);
+ end_sequence ();
+}
+
/* Construct (set target (vec_select op0 (parallel perm))) and
return true if that's a valid instruction in the active ISA. */
static bool
-expand_vselect (rtx target, rtx op0, const unsigned char *perm, unsigned nelt)
+expand_vselect (rtx target, rtx op0, const unsigned char *perm,
+ unsigned nelt, bool testing_p)
{
- rtx rperm[MAX_VECT_LEN], x;
- unsigned i;
+ unsigned int i;
+ rtx x, save_vconcat;
+ int icode;
+
+ if (vselect_insn == NULL_RTX)
+ init_vselect_insn ();
+ x = XEXP (SET_SRC (PATTERN (vselect_insn)), 1);
+ PUT_NUM_ELEM (XVEC (x, 0), nelt);
for (i = 0; i < nelt; ++i)
- rperm[i] = GEN_INT (perm[i]);
+ XVECEXP (x, 0, i) = GEN_INT (perm[i]);
+ save_vconcat = XEXP (SET_SRC (PATTERN (vselect_insn)), 0);
+ XEXP (SET_SRC (PATTERN (vselect_insn)), 0) = op0;
+ PUT_MODE (SET_SRC (PATTERN (vselect_insn)), GET_MODE (target));
+ SET_DEST (PATTERN (vselect_insn)) = target;
+ icode = recog_memoized (vselect_insn);
- x = gen_rtx_PARALLEL (VOIDmode, gen_rtvec_v (nelt, rperm));
- x = gen_rtx_VEC_SELECT (GET_MODE (target), op0, x);
- x = gen_rtx_SET (VOIDmode, target, x);
+ if (icode >= 0 && !testing_p)
+ emit_insn (copy_rtx (PATTERN (vselect_insn)));
- x = emit_insn (x);
- if (recog_memoized (x) < 0)
- {
- remove_insn (x);
- return false;
- }
- return true;
+ SET_DEST (PATTERN (vselect_insn)) = const0_rtx;
+ XEXP (SET_SRC (PATTERN (vselect_insn)), 0) = save_vconcat;
+ INSN_CODE (vselect_insn) = -1;
+
+ return icode >= 0;
}
/* Similar, but generate a vec_concat from op0 and op1 as well. */
static bool
expand_vselect_vconcat (rtx target, rtx op0, rtx op1,
- const unsigned char *perm, unsigned nelt)
+ const unsigned char *perm, unsigned nelt,
+ bool testing_p)
{
enum machine_mode v2mode;
rtx x;
+ bool ok;
+
+ if (vselect_insn == NULL_RTX)
+ init_vselect_insn ();
v2mode = GET_MODE_2XWIDER_MODE (GET_MODE (op0));
- x = gen_rtx_VEC_CONCAT (v2mode, op0, op1);
- return expand_vselect (target, x, perm, nelt);
+ x = XEXP (SET_SRC (PATTERN (vselect_insn)), 0);
+ PUT_MODE (x, v2mode);
+ XEXP (x, 0) = op0;
+ XEXP (x, 1) = op1;
+ ok = expand_vselect (target, x, perm, nelt, testing_p);
+ XEXP (x, 0) = const0_rtx;
+ XEXP (x, 1) = const0_rtx;
+ return ok;
}
/* A subroutine of ix86_expand_vec_perm_builtin_1. Try to implement D
rtx target, op0, op1, x;
rtx rperm[32], vperm;
- if (d->op0 == d->op1)
+ if (d->one_operand_p)
return false;
if (TARGET_AVX2 && GET_MODE_SIZE (vmode) == 32)
;
rtx rperm[8], vperm;
unsigned i;
- if (!TARGET_AVX || d->vmode != V8SFmode || d->op0 != d->op1)
+ if (!TARGET_AVX || d->vmode != V8SFmode || !d->one_operand_p)
return false;
/* We can only permute within the 128-bit lane. */
}
/* A subroutine of ix86_expand_vec_perm_builtin_1. Try to implement D
- in terms of pshufb, vpperm, vpermq, vpermd or vperm2i128. */
+ in terms of pshufb, vpperm, vpermq, vpermd, vpermps or vperm2i128. */
static bool
expand_vec_perm_pshufb (struct expand_vec_perm_d *d)
nelt = d->nelt;
- if (d->op0 != d->op1)
+ if (!d->one_operand_p)
{
if (!TARGET_XOP || GET_MODE_SIZE (d->vmode) != 16)
{
return true;
return expand_vselect (gen_lowpart (V4DImode, d->target),
gen_lowpart (V4DImode, d->op0),
- perm, 4);
+ perm, 4, false);
}
/* Next see if vpermd can be used. */
if (valid_perm_using_mode_p (V8SImode, d))
vmode = V8SImode;
}
+ /* Or if vpermps can be used. */
+ else if (d->vmode == V8SFmode)
+ vmode = V8SImode;
if (vmode == V32QImode)
{
else
{
eltsz = GET_MODE_SIZE (GET_MODE_INNER (d->vmode));
- if (d->op0 != d->op1)
+ if (!d->one_operand_p)
mask = 2 * nelt - 1;
else if (vmode == V16QImode)
mask = nelt - 1;
target = gen_lowpart (vmode, d->target);
op0 = gen_lowpart (vmode, d->op0);
- if (d->op0 == d->op1)
+ if (d->one_operand_p)
{
if (vmode == V16QImode)
emit_insn (gen_ssse3_pshufbv16qi3 (target, op0, vperm));
else if (vmode == V32QImode)
emit_insn (gen_avx2_pshufbv32qi3 (target, op0, vperm));
+ else if (vmode == V8SFmode)
+ emit_insn (gen_avx2_permvarv8sf (target, op0, vperm));
else
- emit_insn (gen_avx2_permvarv8si (target, vperm, op0));
+ emit_insn (gen_avx2_permvarv8si (target, op0, vperm));
}
else
{
/* Check plain VEC_SELECT first, because AVX has instructions that could
match both SEL and SEL+CONCAT, but the plain SEL will allow a memory
input where SEL+CONCAT may not. */
- if (d->op0 == d->op1)
+ if (d->one_operand_p)
{
int mask = nelt - 1;
bool identity_perm = true;
else if (broadcast_perm && TARGET_AVX2)
{
/* Use vpbroadcast{b,w,d}. */
- rtx op = d->op0, (*gen) (rtx, rtx) = NULL;
+ rtx (*gen) (rtx, rtx) = NULL;
switch (d->vmode)
{
case V32QImode:
- op = gen_lowpart (V16QImode, op);
- gen = gen_avx2_pbroadcastv32qi;
+ gen = gen_avx2_pbroadcastv32qi_1;
break;
case V16HImode:
- op = gen_lowpart (V8HImode, op);
- gen = gen_avx2_pbroadcastv16hi;
+ gen = gen_avx2_pbroadcastv16hi_1;
break;
case V8SImode:
- op = gen_lowpart (V4SImode, op);
- gen = gen_avx2_pbroadcastv8si;
+ gen = gen_avx2_pbroadcastv8si_1;
break;
case V16QImode:
gen = gen_avx2_pbroadcastv16qi;
case V8HImode:
gen = gen_avx2_pbroadcastv8hi;
break;
+ case V8SFmode:
+ gen = gen_avx2_vec_dupv8sf_1;
+ break;
/* For other modes prefer other shuffles this function creates. */
default: break;
}
if (gen != NULL)
{
if (!d->testing_p)
- emit_insn (gen (d->target, op));
+ emit_insn (gen (d->target, d->op0));
return true;
}
}
- if (expand_vselect (d->target, d->op0, perm2, nelt))
+ if (expand_vselect (d->target, d->op0, perm2, nelt, d->testing_p))
return true;
/* There are plenty of patterns in sse.md that are written for
perm2[i] = d->perm[i] & mask;
perm2[i + 1] = (d->perm[i + 1] & mask) + nelt;
}
- if (expand_vselect_vconcat (d->target, d->op0, d->op0, perm2, nelt))
+ if (expand_vselect_vconcat (d->target, d->op0, d->op0, perm2, nelt,
+ d->testing_p))
return true;
/* Recognize shufps, which means adding {0, 0, nelt, nelt}. */
perm2[i + 3] = (d->perm[i + 3] & mask) + nelt;
}
- if (expand_vselect_vconcat (d->target, d->op0, d->op0, perm2, nelt))
+ if (expand_vselect_vconcat (d->target, d->op0, d->op0, perm2, nelt,
+ d->testing_p))
return true;
}
}
/* Finally, try the fully general two operand permute. */
- if (expand_vselect_vconcat (d->target, d->op0, d->op1, d->perm, nelt))
+ if (expand_vselect_vconcat (d->target, d->op0, d->op1, d->perm, nelt,
+ d->testing_p))
return true;
/* Recognize interleave style patterns with reversed operands. */
- if (d->op0 != d->op1)
+ if (!d->one_operand_p)
{
for (i = 0; i < nelt; ++i)
{
perm2[i] = e;
}
- if (expand_vselect_vconcat (d->target, d->op1, d->op0, perm2, nelt))
+ if (expand_vselect_vconcat (d->target, d->op1, d->op0, perm2, nelt,
+ d->testing_p))
return true;
}
return true;
/* Try the SSSE3 pshufb or XOP vpperm or AVX2 vperm2i128,
- vpshufb, vpermd or vpermq variable permutation. */
+ vpshufb, vpermd, vpermps or vpermq variable permutation. */
if (expand_vec_perm_pshufb (d))
return true;
unsigned i;
bool ok;
- if (d->vmode != V8HImode || d->op0 != d->op1)
+ if (d->vmode != V8HImode || !d->one_operand_p)
return false;
/* The two permutations only operate in 64-bit lanes. */
memcpy (perm2, d->perm, 4);
for (i = 4; i < 8; ++i)
perm2[i] = i;
- ok = expand_vselect (d->target, d->op0, perm2, 8);
+ ok = expand_vselect (d->target, d->op0, perm2, 8, d->testing_p);
gcc_assert (ok);
/* Emit the pshufhw. */
memcpy (perm2 + 4, d->perm + 4, 4);
for (i = 0; i < 4; ++i)
perm2[i] = i;
- ok = expand_vselect (d->target, d->target, perm2, 8);
+ ok = expand_vselect (d->target, d->target, perm2, 8, d->testing_p);
gcc_assert (ok);
return true;
gen_lowpart (TImode, d->op0), shift));
d->op0 = d->op1 = d->target;
+ d->one_operand_p = true;
in_order = true;
for (i = 0; i < nelt; ++i)
if (GET_MODE_SIZE (d->vmode) == 16)
{
- if (d->op0 == d->op1)
+ if (d->one_operand_p)
return false;
}
else if (GET_MODE_SIZE (d->vmode) == 32)
{
if (!TARGET_AVX)
return false;
- /* For 32-byte modes allow even d->op0 == d->op1.
+ /* For 32-byte modes allow even d->one_operand_p.
The lack of cross-lane shuffling in some instructions
might prevent a single insn shuffle. */
dfinal = *d;
if (nzcnt == 1)
{
- gcc_assert (d->op0 == d->op1);
+ gcc_assert (d->one_operand_p);
nonzero_halves[1] = nonzero_halves[0];
same_halves = true;
}
- else if (d->op0 == d->op1)
+ else if (d->one_operand_p)
{
gcc_assert (nonzero_halves[0] == 0);
gcc_assert (nonzero_halves[1] == 1);
{
if (d->perm[0] / nelt2 == nonzero_halves[1])
{
- /* Attempt to increase the likelyhood that dfinal
+ /* Attempt to increase the likelihood that dfinal
shuffle will be intra-lane. */
char tmph = nonzero_halves[0];
nonzero_halves[0] = nonzero_halves[1];
}
}
}
- else if (d->op0 == d->op1)
+ else if (d->one_operand_p)
return false;
else if (TARGET_AVX2
&& (contents & (q[0] | q[2] | q[4] | q[6])) == contents)
}
dfinal.op0 = gen_reg_rtx (dfinal.vmode);
dfinal.op1 = dfinal.op0;
+ dfinal.one_operand_p = true;
dremap.target = dfinal.op0;
/* Test if the final remap can be done with a single insn. For V4SFmode or
if (!(TARGET_AVX2
&& (d->vmode == V32QImode || d->vmode == V16HImode)
- && d->op0 == d->op1))
+ && d->one_operand_p))
return false;
contents[0] = 0;
dremap.target = gen_reg_rtx (V4DImode);
dremap.op0 = gen_lowpart (V4DImode, d->op0);
dremap.op1 = dremap.op0;
+ dremap.one_operand_p = true;
for (i = 0; i < 2; ++i)
{
unsigned int cnt = 0;
dfinal = *d;
dfinal.op0 = gen_lowpart (dfinal.vmode, dremap.target);
dfinal.op1 = dfinal.op0;
+ dfinal.one_operand_p = true;
for (i = 0, j = 0; i < nelt; ++i)
{
if (i == nelt2)
return true;
}
+/* A subroutine of ix86_expand_vec_perm_builtin_1. Try to expand
+ a vector permutation using two instructions, vperm2f128 resp.
+ vperm2i128 followed by any single in-lane permutation. */
+
+static bool
+expand_vec_perm_vperm2f128 (struct expand_vec_perm_d *d)
+{
+ struct expand_vec_perm_d dfirst, dsecond;
+ unsigned i, j, nelt = d->nelt, nelt2 = nelt / 2, perm;
+ bool ok;
+
+ if (!TARGET_AVX
+ || GET_MODE_SIZE (d->vmode) != 32
+ || (d->vmode != V8SFmode && d->vmode != V4DFmode && !TARGET_AVX2))
+ return false;
+
+ dsecond = *d;
+ dsecond.one_operand_p = false;
+ dsecond.testing_p = true;
+
+ /* ((perm << 2)|perm) & 0x33 is the vperm2[fi]128
+ immediate. For perm < 16 the second permutation uses
+ d->op0 as first operand, for perm >= 16 it uses d->op1
+ as first operand. The second operand is the result of
+ vperm2[fi]128. */
+ for (perm = 0; perm < 32; perm++)
+ {
+ /* Ignore permutations which do not move anything cross-lane. */
+ if (perm < 16)
+ {
+ /* The second shuffle for e.g. V4DFmode has
+ 0123 and ABCD operands.
+ Ignore AB23, as 23 is already in the second lane
+ of the first operand. */
+ if ((perm & 0xc) == (1 << 2)) continue;
+ /* And 01CD, as 01 is in the first lane of the first
+ operand. */
+ if ((perm & 3) == 0) continue;
+ /* And 4567, as then the vperm2[fi]128 doesn't change
+ anything on the original 4567 second operand. */
+ if ((perm & 0xf) == ((3 << 2) | 2)) continue;
+ }
+ else
+ {
+ /* The second shuffle for e.g. V4DFmode has
+ 4567 and ABCD operands.
+ Ignore AB67, as 67 is already in the second lane
+ of the first operand. */
+ if ((perm & 0xc) == (3 << 2)) continue;
+ /* And 45CD, as 45 is in the first lane of the first
+ operand. */
+ if ((perm & 3) == 2) continue;
+ /* And 0123, as then the vperm2[fi]128 doesn't change
+ anything on the original 0123 first operand. */
+ if ((perm & 0xf) == (1 << 2)) continue;
+ }
+
+ for (i = 0; i < nelt; i++)
+ {
+ j = d->perm[i] / nelt2;
+ if (j == ((perm >> (2 * (i >= nelt2))) & 3))
+ dsecond.perm[i] = nelt + (i & nelt2) + (d->perm[i] & (nelt2 - 1));
+ else if (j == (unsigned) (i >= nelt2) + 2 * (perm >= 16))
+ dsecond.perm[i] = d->perm[i] & (nelt - 1);
+ else
+ break;
+ }
+
+ if (i == nelt)
+ {
+ start_sequence ();
+ ok = expand_vec_perm_1 (&dsecond);
+ end_sequence ();
+ }
+ else
+ ok = false;
+
+ if (ok)
+ {
+ if (d->testing_p)
+ return true;
+
+ /* Found a usable second shuffle. dfirst will be
+ vperm2f128 on d->op0 and d->op1. */
+ dsecond.testing_p = false;
+ dfirst = *d;
+ dfirst.target = gen_reg_rtx (d->vmode);
+ for (i = 0; i < nelt; i++)
+ dfirst.perm[i] = (i & (nelt2 - 1))
+ + ((perm >> (2 * (i >= nelt2))) & 3) * nelt2;
+
+ ok = expand_vec_perm_1 (&dfirst);
+ gcc_assert (ok);
+
+ /* And dsecond is some single insn shuffle, taking
+ d->op0 and result of vperm2f128 (if perm < 16) or
+ d->op1 and result of vperm2f128 (otherwise). */
+ dsecond.op1 = dfirst.target;
+ if (perm >= 16)
+ dsecond.op0 = dfirst.op1;
+
+ ok = expand_vec_perm_1 (&dsecond);
+ gcc_assert (ok);
+
+ return true;
+ }
+
+ /* For one operand, the only useful vperm2f128 permutation is 0x10. */
+ if (d->one_operand_p)
+ return false;
+ }
+
+ return false;
+}
+
/* A subroutine of ix86_expand_vec_perm_builtin_1. Try to simplify
a two vector permutation using 2 intra-lane interleave insns
and cross-lane shuffle for 32-byte vectors. */
unsigned i, nelt;
rtx (*gen) (rtx, rtx, rtx);
- if (d->op0 == d->op1)
+ if (d->one_operand_p)
return false;
if (TARGET_AVX2 && GET_MODE_SIZE (d->vmode) == 32)
;
if (!TARGET_AVX
|| TARGET_AVX2
|| (d->vmode != V8SFmode && d->vmode != V4DFmode)
- || d->op0 != d->op1)
+ || !d->one_operand_p)
return false;
dfirst = *d;
dsecond = *d;
dsecond.op0 = dfirst.target;
dsecond.op1 = dfirst.target;
+ dsecond.one_operand_p = true;
dsecond.target = gen_reg_rtx (dsecond.vmode);
for (i = 0; i < nelt; i++)
dsecond.perm[i] = i ^ nelt2;
return true;
}
+/* A subroutine of ix86_expand_vec_perm_builtin_1. Implement a V4DF
+ permutation using two vperm2f128, followed by a vshufpd insn blending
+ the two vectors together. */
+
+static bool
+expand_vec_perm_2vperm2f128_vshuf (struct expand_vec_perm_d *d)
+{
+ struct expand_vec_perm_d dfirst, dsecond, dthird;
+ bool ok;
+
+ if (!TARGET_AVX || (d->vmode != V4DFmode))
+ return false;
+
+ if (d->testing_p)
+ return true;
+
+ dfirst = *d;
+ dsecond = *d;
+ dthird = *d;
+
+ dfirst.perm[0] = (d->perm[0] & ~1);
+ dfirst.perm[1] = (d->perm[0] & ~1) + 1;
+ dfirst.perm[2] = (d->perm[2] & ~1);
+ dfirst.perm[3] = (d->perm[2] & ~1) + 1;
+ dsecond.perm[0] = (d->perm[1] & ~1);
+ dsecond.perm[1] = (d->perm[1] & ~1) + 1;
+ dsecond.perm[2] = (d->perm[3] & ~1);
+ dsecond.perm[3] = (d->perm[3] & ~1) + 1;
+ dthird.perm[0] = (d->perm[0] % 2);
+ dthird.perm[1] = (d->perm[1] % 2) + 4;
+ dthird.perm[2] = (d->perm[2] % 2) + 2;
+ dthird.perm[3] = (d->perm[3] % 2) + 6;
+
+ dfirst.target = gen_reg_rtx (dfirst.vmode);
+ dsecond.target = gen_reg_rtx (dsecond.vmode);
+ dthird.op0 = dfirst.target;
+ dthird.op1 = dsecond.target;
+ dthird.one_operand_p = false;
+
+ canonicalize_perm (&dfirst);
+ canonicalize_perm (&dsecond);
+
+ ok = expand_vec_perm_1 (&dfirst)
+ && expand_vec_perm_1 (&dsecond)
+ && expand_vec_perm_1 (&dthird);
+
+ gcc_assert (ok);
+
+ return true;
+}
+
/* A subroutine of expand_vec_perm_even_odd_1. Implement the double-word
permutation with two pshufb insns and an ior. We should have already
failed all two instruction sequences. */
if (!TARGET_SSSE3 || GET_MODE_SIZE (d->vmode) != 16)
return false;
- gcc_assert (d->op0 != d->op1);
+ gcc_assert (!d->one_operand_p);
nelt = d->nelt;
eltsz = GET_MODE_SIZE (GET_MODE_INNER (d->vmode));
unsigned int i, nelt, eltsz;
if (!TARGET_AVX2
- || d->op0 != d->op1
+ || !d->one_operand_p
|| (d->vmode != V32QImode && d->vmode != V16HImode))
return false;
unsigned int i, nelt, eltsz;
if (!TARGET_AVX2
- || d->op0 == d->op1
+ || d->one_operand_p
|| (d->vmode != V32QImode && d->vmode != V16HImode))
return false;
while (vmode != V4SImode);
memset (perm2, elt, 4);
- ok = expand_vselect (gen_lowpart (V4SImode, d->target), op0, perm2, 4);
+ ok = expand_vselect (gen_lowpart (V4SImode, d->target), op0, perm2, 4,
+ d->testing_p);
gcc_assert (ok);
return true;
{
unsigned i, elt, nelt = d->nelt;
- if (d->op0 != d->op1)
+ if (!d->one_operand_p)
return false;
elt = d->perm[0];
bool used[4];
if (!TARGET_AVX2
- || d->op0 == d->op1
+ || d->one_operand_p
|| (d->vmode != V32QImode && d->vmode != V16HImode))
return false;
if (expand_vec_perm_vpermq_perm_1 (d))
return true;
+ if (expand_vec_perm_vperm2f128 (d))
+ return true;
+
/* Try sequences of three instructions. */
+ if (expand_vec_perm_2vperm2f128_vshuf (d))
+ return true;
+
if (expand_vec_perm_pshufb2 (d))
return true;
return false;
}
+/* If a permutation only uses one operand, make it clear. Returns true
+ if the permutation references both operands. */
+
+static bool
+canonicalize_perm (struct expand_vec_perm_d *d)
+{
+ int i, which, nelt = d->nelt;
+
+ for (i = which = 0; i < nelt; ++i)
+ which |= (d->perm[i] < nelt ? 1 : 2);
+
+ d->one_operand_p = true;
+ switch (which)
+ {
+ default:
+ gcc_unreachable();
+
+ case 3:
+ if (!rtx_equal_p (d->op0, d->op1))
+ {
+ d->one_operand_p = false;
+ break;
+ }
+ /* The elements of PERM do not suggest that only the first operand
+ is used, but both operands are identical. Allow easier matching
+ of the permutation by folding the permutation into the single
+ input vector. */
+ /* FALLTHRU */
+
+ case 2:
+ for (i = 0; i < nelt; ++i)
+ d->perm[i] &= nelt - 1;
+ d->op0 = d->op1;
+ break;
+
+ case 1:
+ d->op1 = d->op0;
+ break;
+ }
+
+ return (which == 3);
+}
+
bool
ix86_expand_vec_perm_const (rtx operands[4])
{
struct expand_vec_perm_d d;
unsigned char perm[MAX_VECT_LEN];
- int i, nelt, which;
+ int i, nelt;
+ bool two_args;
rtx sel;
d.target = operands[0];
gcc_assert (XVECLEN (sel, 0) == nelt);
gcc_checking_assert (sizeof (d.perm) == sizeof (perm));
- for (i = which = 0; i < nelt; ++i)
+ for (i = 0; i < nelt; ++i)
{
rtx e = XVECEXP (sel, 0, i);
int ei = INTVAL (e) & (2 * nelt - 1);
-
- which |= (ei < nelt ? 1 : 2);
d.perm[i] = ei;
perm[i] = ei;
}
- switch (which)
- {
- default:
- gcc_unreachable();
-
- case 3:
- if (!rtx_equal_p (d.op0, d.op1))
- break;
-
- /* The elements of PERM do not suggest that only the first operand
- is used, but both operands are identical. Allow easier matching
- of the permutation by folding the permutation into the single
- input vector. */
- for (i = 0; i < nelt; ++i)
- if (d.perm[i] >= nelt)
- d.perm[i] -= nelt;
- /* FALLTHRU */
-
- case 1:
- d.op1 = d.op0;
- break;
-
- case 2:
- for (i = 0; i < nelt; ++i)
- d.perm[i] -= nelt;
- d.op0 = d.op1;
- break;
- }
+ two_args = canonicalize_perm (&d);
if (ix86_expand_vec_perm_const_1 (&d))
return true;
- /* If the mask says both arguments are needed, but they are the same,
- the above tried to expand with d.op0 == d.op1. If that didn't work,
- retry with d.op0 != d.op1 as that is what testing has been done with. */
- if (which == 3 && d.op0 == d.op1)
+ /* If the selector says both arguments are needed, but the operands are the
+ same, the above tried to expand with one_operand_p and flattened selector.
+ If that didn't work, retry without one_operand_p; we succeeded with that
+ during testing. */
+ if (two_args && d.one_operand_p)
{
- rtx seq;
- bool ok;
-
+ d.one_operand_p = false;
memcpy (d.perm, perm, sizeof (perm));
- d.op1 = gen_reg_rtx (d.vmode);
- start_sequence ();
- ok = ix86_expand_vec_perm_const_1 (&d);
- seq = get_insns ();
- end_sequence ();
- if (ok)
- {
- emit_move_insn (d.op1, d.op0);
- emit_insn (seq);
- return true;
- }
+ return ix86_expand_vec_perm_const_1 (&d);
}
return false;
{
struct expand_vec_perm_d d;
unsigned int i, nelt, which;
- bool ret, one_vec;
+ bool ret;
d.vmode = vmode;
d.nelt = nelt = GET_MODE_NUNITS (d.vmode);
d.perm[i] -= nelt;
/* Check whether the mask can be applied to the vector type. */
- one_vec = (which != 3);
+ d.one_operand_p = (which != 3);
/* Implementable with shufps or pshufd. */
- if (one_vec && (d.vmode == V4SFmode || d.vmode == V4SImode))
+ if (d.one_operand_p && (d.vmode == V4SFmode || d.vmode == V4SImode))
return true;
/* Otherwise we have to go through the motions and see if we can
figure out how to generate the requested permutation. */
d.target = gen_raw_REG (d.vmode, LAST_VIRTUAL_REGISTER + 1);
d.op1 = d.op0 = gen_raw_REG (d.vmode, LAST_VIRTUAL_REGISTER + 2);
- if (!one_vec)
+ if (!d.one_operand_p)
d.op1 = gen_raw_REG (d.vmode, LAST_VIRTUAL_REGISTER + 3);
start_sequence ();
d.op1 = op1;
d.vmode = GET_MODE (targ);
d.nelt = nelt = GET_MODE_NUNITS (d.vmode);
+ d.one_operand_p = false;
d.testing_p = false;
for (i = 0; i < nelt; ++i)
expand_vec_perm_even_odd_1 (&d, odd);
}
+static void
+ix86_expand_vec_interleave (rtx targ, rtx op0, rtx op1, bool high_p)
+{
+ struct expand_vec_perm_d d;
+ unsigned i, nelt, base;
+ bool ok;
+
+ d.target = targ;
+ d.op0 = op0;
+ d.op1 = op1;
+ d.vmode = GET_MODE (targ);
+ d.nelt = nelt = GET_MODE_NUNITS (d.vmode);
+ d.one_operand_p = false;
+ d.testing_p = false;
+
+ base = high_p ? nelt / 2 : 0;
+ for (i = 0; i < nelt / 2; ++i)
+ {
+ d.perm[i * 2] = i + base;
+ d.perm[i * 2 + 1] = i + base + nelt;
+ }
+
+ /* Note that for AVX this isn't one instruction. */
+ ok = ix86_expand_vec_perm_const_1 (&d);
+ gcc_assert (ok);
+}
+
+
+/* Expand a vector operation CODE for a V*QImode in terms of the
+ same operation on V*HImode. */
+
+void
+ix86_expand_vecop_qihi (enum rtx_code code, rtx dest, rtx op1, rtx op2)
+{
+ enum machine_mode qimode = GET_MODE (dest);
+ enum machine_mode himode;
+ rtx (*gen_il) (rtx, rtx, rtx);
+ rtx (*gen_ih) (rtx, rtx, rtx);
+ rtx op1_l, op1_h, op2_l, op2_h, res_l, res_h;
+ struct expand_vec_perm_d d;
+ bool ok, full_interleave;
+ bool uns_p = false;
+ int i;
+
+ switch (qimode)
+ {
+ case V16QImode:
+ himode = V8HImode;
+ gen_il = gen_vec_interleave_lowv16qi;
+ gen_ih = gen_vec_interleave_highv16qi;
+ break;
+ case V32QImode:
+ himode = V16HImode;
+ gen_il = gen_avx2_interleave_lowv32qi;
+ gen_ih = gen_avx2_interleave_highv32qi;
+ break;
+ default:
+ gcc_unreachable ();
+ }
+
+ op2_l = op2_h = op2;
+ switch (code)
+ {
+ case MULT:
+ /* Unpack data such that we've got a source byte in each low byte of
+ each word. We don't care what goes into the high byte of each word.
+ Rather than trying to get zero in there, most convenient is to let
+ it be a copy of the low byte. */
+ op2_l = gen_reg_rtx (qimode);
+ op2_h = gen_reg_rtx (qimode);
+ emit_insn (gen_il (op2_l, op2, op2));
+ emit_insn (gen_ih (op2_h, op2, op2));
+ /* FALLTHRU */
+
+ op1_l = gen_reg_rtx (qimode);
+ op1_h = gen_reg_rtx (qimode);
+ emit_insn (gen_il (op1_l, op1, op1));
+ emit_insn (gen_ih (op1_h, op1, op1));
+ full_interleave = qimode == V16QImode;
+ break;
+
+ case ASHIFT:
+ case LSHIFTRT:
+ uns_p = true;
+ /* FALLTHRU */
+ case ASHIFTRT:
+ op1_l = gen_reg_rtx (himode);
+ op1_h = gen_reg_rtx (himode);
+ ix86_expand_sse_unpack (op1_l, op1, uns_p, false);
+ ix86_expand_sse_unpack (op1_h, op1, uns_p, true);
+ full_interleave = true;
+ break;
+ default:
+ gcc_unreachable ();
+ }
+
+ /* Perform the operation. */
+ res_l = expand_simple_binop (himode, code, op1_l, op2_l, NULL_RTX,
+ 1, OPTAB_DIRECT);
+ res_h = expand_simple_binop (himode, code, op1_h, op2_h, NULL_RTX,
+ 1, OPTAB_DIRECT);
+ gcc_assert (res_l && res_h);
+
+ /* Merge the data back into the right place. */
+ d.target = dest;
+ d.op0 = gen_lowpart (qimode, res_l);
+ d.op1 = gen_lowpart (qimode, res_h);
+ d.vmode = qimode;
+ d.nelt = GET_MODE_NUNITS (qimode);
+ d.one_operand_p = false;
+ d.testing_p = false;
+
+ if (full_interleave)
+ {
+ /* For SSE2, we used an full interleave, so the desired
+ results are in the even elements. */
+ for (i = 0; i < 32; ++i)
+ d.perm[i] = i * 2;
+ }
+ else
+ {
+ /* For AVX, the interleave used above was not cross-lane. So the
+ extraction is evens but with the second and third quarter swapped.
+ Happily, that is even one insn shorter than even extraction. */
+ for (i = 0; i < 32; ++i)
+ d.perm[i] = i * 2 + ((i & 24) == 8 ? 16 : (i & 24) == 16 ? -16 : 0);
+ }
+
+ ok = ix86_expand_vec_perm_const_1 (&d);
+ gcc_assert (ok);
+
+ set_unique_reg_note (get_last_insn (), REG_EQUAL,
+ gen_rtx_fmt_ee (code, qimode, op1, op2));
+}
+
+void
+ix86_expand_mul_widen_evenodd (rtx dest, rtx op1, rtx op2,
+ bool uns_p, bool odd_p)
+{
+ enum machine_mode mode = GET_MODE (op1);
+ enum machine_mode wmode = GET_MODE (dest);
+ rtx x;
+
+ /* We only play even/odd games with vectors of SImode. */
+ gcc_assert (mode == V4SImode || mode == V8SImode);
+
+ /* If we're looking for the odd results, shift those members down to
+ the even slots. For some cpus this is faster than a PSHUFD. */
+ if (odd_p)
+ {
+ if (TARGET_XOP && mode == V4SImode)
+ {
+ x = force_reg (wmode, CONST0_RTX (wmode));
+ emit_insn (gen_xop_pmacsdqh (dest, op1, op2, x));
+ return;
+ }
+
+ x = GEN_INT (GET_MODE_UNIT_BITSIZE (mode));
+ op1 = expand_binop (wmode, lshr_optab, gen_lowpart (wmode, op1),
+ x, NULL, 1, OPTAB_DIRECT);
+ op2 = expand_binop (wmode, lshr_optab, gen_lowpart (wmode, op2),
+ x, NULL, 1, OPTAB_DIRECT);
+ op1 = gen_lowpart (mode, op1);
+ op2 = gen_lowpart (mode, op2);
+ }
+
+ if (mode == V8SImode)
+ {
+ if (uns_p)
+ x = gen_vec_widen_umult_even_v8si (dest, op1, op2);
+ else
+ x = gen_vec_widen_smult_even_v8si (dest, op1, op2);
+ }
+ else if (uns_p)
+ x = gen_vec_widen_umult_even_v4si (dest, op1, op2);
+ else if (TARGET_SSE4_1)
+ x = gen_sse4_1_mulv2siv2di3 (dest, op1, op2);
+ else
+ {
+ rtx s1, s2, t0, t1, t2;
+
+ /* The easiest way to implement this without PMULDQ is to go through
+ the motions as if we are performing a full 64-bit multiply. With
+ the exception that we need to do less shuffling of the elements. */
+
+ /* Compute the sign-extension, aka highparts, of the two operands. */
+ s1 = ix86_expand_sse_cmp (gen_reg_rtx (mode), GT, CONST0_RTX (mode),
+ op1, pc_rtx, pc_rtx);
+ s2 = ix86_expand_sse_cmp (gen_reg_rtx (mode), GT, CONST0_RTX (mode),
+ op2, pc_rtx, pc_rtx);
+
+ /* Multiply LO(A) * HI(B), and vice-versa. */
+ t1 = gen_reg_rtx (wmode);
+ t2 = gen_reg_rtx (wmode);
+ emit_insn (gen_vec_widen_umult_even_v4si (t1, s1, op2));
+ emit_insn (gen_vec_widen_umult_even_v4si (t2, s2, op1));
+
+ /* Multiply LO(A) * LO(B). */
+ t0 = gen_reg_rtx (wmode);
+ emit_insn (gen_vec_widen_umult_even_v4si (t0, op1, op2));
+
+ /* Combine and shift the highparts into place. */
+ t1 = expand_binop (wmode, add_optab, t1, t2, t1, 1, OPTAB_DIRECT);
+ t1 = expand_binop (wmode, ashl_optab, t1, GEN_INT (32), t1,
+ 1, OPTAB_DIRECT);
+
+ /* Combine high and low parts. */
+ force_expand_binop (wmode, add_optab, t0, t1, dest, 1, OPTAB_DIRECT);
+ return;
+ }
+ emit_insn (x);
+}
+
+void
+ix86_expand_mul_widen_hilo (rtx dest, rtx op1, rtx op2,
+ bool uns_p, bool high_p)
+{
+ enum machine_mode wmode = GET_MODE (dest);
+ enum machine_mode mode = GET_MODE (op1);
+ rtx t1, t2, t3, t4, mask;
+
+ switch (mode)
+ {
+ case V4SImode:
+ t1 = gen_reg_rtx (mode);
+ t2 = gen_reg_rtx (mode);
+ if (TARGET_XOP && !uns_p)
+ {
+ /* With XOP, we have pmacsdqh, aka mul_widen_odd. In this case,
+ shuffle the elements once so that all elements are in the right
+ place for immediate use: { A C B D }. */
+ emit_insn (gen_sse2_pshufd_1 (t1, op1, const0_rtx, const2_rtx,
+ const1_rtx, GEN_INT (3)));
+ emit_insn (gen_sse2_pshufd_1 (t2, op2, const0_rtx, const2_rtx,
+ const1_rtx, GEN_INT (3)));
+ }
+ else
+ {
+ /* Put the elements into place for the multiply. */
+ ix86_expand_vec_interleave (t1, op1, op1, high_p);
+ ix86_expand_vec_interleave (t2, op2, op2, high_p);
+ high_p = false;
+ }
+ ix86_expand_mul_widen_evenodd (dest, t1, t2, uns_p, high_p);
+ break;
+
+ case V8SImode:
+ /* Shuffle the elements between the lanes. After this we
+ have { A B E F | C D G H } for each operand. */
+ t1 = gen_reg_rtx (V4DImode);
+ t2 = gen_reg_rtx (V4DImode);
+ emit_insn (gen_avx2_permv4di_1 (t1, gen_lowpart (V4DImode, op1),
+ const0_rtx, const2_rtx,
+ const1_rtx, GEN_INT (3)));
+ emit_insn (gen_avx2_permv4di_1 (t2, gen_lowpart (V4DImode, op2),
+ const0_rtx, const2_rtx,
+ const1_rtx, GEN_INT (3)));
+
+ /* Shuffle the elements within the lanes. After this we
+ have { A A B B | C C D D } or { E E F F | G G H H }. */
+ t3 = gen_reg_rtx (V8SImode);
+ t4 = gen_reg_rtx (V8SImode);
+ mask = GEN_INT (high_p
+ ? 2 + (2 << 2) + (3 << 4) + (3 << 6)
+ : 0 + (0 << 2) + (1 << 4) + (1 << 6));
+ emit_insn (gen_avx2_pshufdv3 (t3, gen_lowpart (V8SImode, t1), mask));
+ emit_insn (gen_avx2_pshufdv3 (t4, gen_lowpart (V8SImode, t2), mask));
+
+ ix86_expand_mul_widen_evenodd (dest, t3, t4, uns_p, false);
+ break;
+
+ case V8HImode:
+ case V16HImode:
+ t1 = expand_binop (mode, smul_optab, op1, op2, NULL_RTX,
+ uns_p, OPTAB_DIRECT);
+ t2 = expand_binop (mode,
+ uns_p ? umul_highpart_optab : smul_highpart_optab,
+ op1, op2, NULL_RTX, uns_p, OPTAB_DIRECT);
+ gcc_assert (t1 && t2);
+
+ ix86_expand_vec_interleave (gen_lowpart (mode, dest), t1, t2, high_p);
+ break;
+
+ case V16QImode:
+ case V32QImode:
+ t1 = gen_reg_rtx (wmode);
+ t2 = gen_reg_rtx (wmode);
+ ix86_expand_sse_unpack (t1, op1, uns_p, high_p);
+ ix86_expand_sse_unpack (t2, op2, uns_p, high_p);
+
+ emit_insn (gen_rtx_SET (VOIDmode, dest, gen_rtx_MULT (wmode, t1, t2)));
+ break;
+
+ default:
+ gcc_unreachable ();
+ }
+}
+
+void
+ix86_expand_sse2_mulv4si3 (rtx op0, rtx op1, rtx op2)
+{
+ rtx res_1, res_2;
+
+ res_1 = gen_reg_rtx (V4SImode);
+ res_2 = gen_reg_rtx (V4SImode);
+ ix86_expand_mul_widen_evenodd (gen_lowpart (V2DImode, res_1),
+ op1, op2, true, false);
+ ix86_expand_mul_widen_evenodd (gen_lowpart (V2DImode, res_2),
+ op1, op2, true, true);
+
+ /* Move the results in element 2 down to element 1; we don't care
+ what goes in elements 2 and 3. Then we can merge the parts
+ back together with an interleave.
+
+ Note that two other sequences were tried:
+ (1) Use interleaves at the start instead of psrldq, which allows
+ us to use a single shufps to merge things back at the end.
+ (2) Use shufps here to combine the two vectors, then pshufd to
+ put the elements in the correct order.
+ In both cases the cost of the reformatting stall was too high
+ and the overall sequence slower. */
+
+ emit_insn (gen_sse2_pshufd_1 (res_1, res_1, const0_rtx, const2_rtx,
+ const0_rtx, const0_rtx));
+ emit_insn (gen_sse2_pshufd_1 (res_2, res_2, const0_rtx, const2_rtx,
+ const0_rtx, const0_rtx));
+ res_1 = emit_insn (gen_vec_interleave_lowv4si (op0, res_1, res_2));
+
+ set_unique_reg_note (res_1, REG_EQUAL, gen_rtx_MULT (V4SImode, op1, op2));
+}
+
+void
+ix86_expand_sse2_mulvxdi3 (rtx op0, rtx op1, rtx op2)
+{
+ enum machine_mode mode = GET_MODE (op0);
+ rtx t1, t2, t3, t4, t5, t6;
+
+ if (TARGET_XOP && mode == V2DImode)
+ {
+ /* op1: A,B,C,D, op2: E,F,G,H */
+ op1 = gen_lowpart (V4SImode, op1);
+ op2 = gen_lowpart (V4SImode, op2);
+
+ t1 = gen_reg_rtx (V4SImode);
+ t2 = gen_reg_rtx (V4SImode);
+ t3 = gen_reg_rtx (V2DImode);
+ t4 = gen_reg_rtx (V2DImode);
+
+ /* t1: B,A,D,C */
+ emit_insn (gen_sse2_pshufd_1 (t1, op1,
+ GEN_INT (1),
+ GEN_INT (0),
+ GEN_INT (3),
+ GEN_INT (2)));
+
+ /* t2: (B*E),(A*F),(D*G),(C*H) */
+ emit_insn (gen_mulv4si3 (t2, t1, op2));
+
+ /* t3: (B*E)+(A*F), (D*G)+(C*H) */
+ emit_insn (gen_xop_phadddq (t3, t2));
+
+ /* t4: ((B*E)+(A*F))<<32, ((D*G)+(C*H))<<32 */
+ emit_insn (gen_ashlv2di3 (t4, t3, GEN_INT (32)));
+
+ /* op0: (((B*E)+(A*F))<<32)+(B*F), (((D*G)+(C*H))<<32)+(D*H) */
+ emit_insn (gen_xop_pmacsdql (op0, op1, op2, t4));
+ }
+ else
+ {
+ enum machine_mode nmode;
+ rtx (*umul) (rtx, rtx, rtx);
+
+ if (mode == V2DImode)
+ {
+ umul = gen_vec_widen_umult_even_v4si;
+ nmode = V4SImode;
+ }
+ else if (mode == V4DImode)
+ {
+ umul = gen_vec_widen_umult_even_v8si;
+ nmode = V8SImode;
+ }
+ else
+ gcc_unreachable ();
+
+
+ /* Multiply low parts. */
+ t1 = gen_reg_rtx (mode);
+ emit_insn (umul (t1, gen_lowpart (nmode, op1), gen_lowpart (nmode, op2)));
+
+ /* Shift input vectors right 32 bits so we can multiply high parts. */
+ t6 = GEN_INT (32);
+ t2 = expand_binop (mode, lshr_optab, op1, t6, NULL, 1, OPTAB_DIRECT);
+ t3 = expand_binop (mode, lshr_optab, op2, t6, NULL, 1, OPTAB_DIRECT);
+
+ /* Multiply high parts by low parts. */
+ t4 = gen_reg_rtx (mode);
+ t5 = gen_reg_rtx (mode);
+ emit_insn (umul (t4, gen_lowpart (nmode, t2), gen_lowpart (nmode, op2)));
+ emit_insn (umul (t5, gen_lowpart (nmode, t3), gen_lowpart (nmode, op1)));
+
+ /* Combine and shift the highparts back. */
+ t4 = expand_binop (mode, add_optab, t4, t5, t4, 1, OPTAB_DIRECT);
+ t4 = expand_binop (mode, ashl_optab, t4, t6, t4, 1, OPTAB_DIRECT);
+
+ /* Combine high and low parts. */
+ force_expand_binop (mode, add_optab, t1, t4, op0, 1, OPTAB_DIRECT);
+ }
+
+ set_unique_reg_note (get_last_insn (), REG_EQUAL,
+ gen_rtx_MULT (mode, op1, op2));
+}
+
/* Expand an insert into a vector register through pinsr insn.
Return true if successful. */
#define TARGET_SCHED_DISPATCH_DO do_dispatch
#undef TARGET_SCHED_REASSOCIATION_WIDTH
#define TARGET_SCHED_REASSOCIATION_WIDTH ix86_reassociation_width
+#undef TARGET_SCHED_REORDER
+#define TARGET_SCHED_REORDER ix86_sched_reorder
/* The size of the dispatch window is the total number of bytes of
object code allowed in a window. */
/* Make disp_cmp and disp_jcc get scheduled at the latest. These
instructions should be given the lowest priority in the
scheduling process in Haifa scheduler to make sure they will be
- scheduled in the same dispatch window as the refrence to them. */
+ scheduled in the same dispatch window as the reference to them. */
if (group == disp_jcc || group == disp_cmp)
return false;
static bool
has_dispatch (rtx insn, int action)
{
- if ((ix86_tune == PROCESSOR_BDVER1 || ix86_tune == PROCESSOR_BDVER2)
+ if ((TARGET_BDVER1 || TARGET_BDVER2)
&& flag_dispatch_scheduler)
switch (action)
{
return (TARGET_AVX && !TARGET_PREFER_AVX128) ? 32 | 16 : 0;
}
+/* Implement targetm.vectorize.init_cost. */
+
+static void *
+ix86_init_cost (struct loop *loop_info ATTRIBUTE_UNUSED)
+{
+ unsigned *cost = XNEW (unsigned);
+ *cost = 0;
+ return cost;
+}
+
+/* Implement targetm.vectorize.add_stmt_cost. */
+
+static unsigned
+ix86_add_stmt_cost (void *data, int count, enum vect_cost_for_stmt kind,
+ struct _stmt_vec_info *stmt_info, int misalign)
+{
+ unsigned *cost = (unsigned *) data;
+ unsigned retval = 0;
+
+ if (flag_vect_cost_model)
+ {
+ tree vectype = stmt_vectype (stmt_info);
+ int stmt_cost = ix86_builtin_vectorization_cost (kind, vectype, misalign);
+
+ /* Statements in an inner loop relative to the loop being
+ vectorized are weighted more heavily. The value here is
+ arbitrary and could potentially be improved with analysis. */
+ if (stmt_in_inner_loop_p (stmt_info))
+ count *= 50; /* FIXME. */
+
+ retval = (unsigned) (count * stmt_cost);
+ *cost += retval;
+ }
+
+ return retval;
+}
+
+/* Implement targetm.vectorize.finish_cost. */
+
+static unsigned
+ix86_finish_cost (void *data)
+{
+ return *((unsigned *) data);
+}
+
+/* Implement targetm.vectorize.destroy_cost_data. */
+
+static void
+ix86_destroy_cost_data (void *data)
+{
+ free (data);
+}
+
+/* Validate target specific memory model bits in VAL. */
+
+static unsigned HOST_WIDE_INT
+ix86_memmodel_check (unsigned HOST_WIDE_INT val)
+{
+ unsigned HOST_WIDE_INT model = val & MEMMODEL_MASK;
+ unsigned HOST_WIDE_INT strong;
+
+ if (val & ~(unsigned HOST_WIDE_INT)(IX86_HLE_ACQUIRE|IX86_HLE_RELEASE
+ |MEMMODEL_MASK)
+ || ((val & IX86_HLE_ACQUIRE) && (val & IX86_HLE_RELEASE)))
+ {
+ warning (OPT_Winvalid_memory_model,
+ "Unknown architecture specific memory model");
+ return MEMMODEL_SEQ_CST;
+ }
+ strong = (model == MEMMODEL_ACQ_REL || model == MEMMODEL_SEQ_CST);
+ if (val & IX86_HLE_ACQUIRE && !(model == MEMMODEL_ACQUIRE || strong))
+ {
+ warning (OPT_Winvalid_memory_model,
+ "HLE_ACQUIRE not used with ACQUIRE or stronger memory model");
+ return MEMMODEL_SEQ_CST | IX86_HLE_ACQUIRE;
+ }
+ if (val & IX86_HLE_RELEASE && !(model == MEMMODEL_RELEASE || strong))
+ {
+ warning (OPT_Winvalid_memory_model,
+ "HLE_RELEASE not used with RELEASE or stronger memory model");
+ return MEMMODEL_SEQ_CST | IX86_HLE_RELEASE;
+ }
+ return val;
+}
+
/* Initialize the GCC target structure. */
#undef TARGET_RETURN_IN_MEMORY
#define TARGET_RETURN_IN_MEMORY ix86_return_in_memory
#undef TARGET_FUNCTION_OK_FOR_SIBCALL
#define TARGET_FUNCTION_OK_FOR_SIBCALL ix86_function_ok_for_sibcall
+#undef TARGET_MEMMODEL_CHECK
+#define TARGET_MEMMODEL_CHECK ix86_memmodel_check
+
#ifdef HAVE_AS_TLS
#undef TARGET_HAVE_TLS
#define TARGET_HAVE_TLS true
#undef TARGET_BUILD_BUILTIN_VA_LIST
#define TARGET_BUILD_BUILTIN_VA_LIST ix86_build_builtin_va_list
+#undef TARGET_FOLD_BUILTIN
+#define TARGET_FOLD_BUILTIN ix86_fold_builtin
+
#undef TARGET_ENUM_VA_LIST_P
#define TARGET_ENUM_VA_LIST_P ix86_enum_va_list
#undef TARGET_VECTORIZE_AUTOVECTORIZE_VECTOR_SIZES
#define TARGET_VECTORIZE_AUTOVECTORIZE_VECTOR_SIZES \
ix86_autovectorize_vector_sizes
+#undef TARGET_VECTORIZE_INIT_COST
+#define TARGET_VECTORIZE_INIT_COST ix86_init_cost
+#undef TARGET_VECTORIZE_ADD_STMT_COST
+#define TARGET_VECTORIZE_ADD_STMT_COST ix86_add_stmt_cost
+#undef TARGET_VECTORIZE_FINISH_COST
+#define TARGET_VECTORIZE_FINISH_COST ix86_finish_cost
+#undef TARGET_VECTORIZE_DESTROY_COST_DATA
+#define TARGET_VECTORIZE_DESTROY_COST_DATA ix86_destroy_cost_data
#undef TARGET_SET_CURRENT_FUNCTION
#define TARGET_SET_CURRENT_FUNCTION ix86_set_current_function
projects / gcc.git / blobdiff