ivopts previously treated pointer arguments to internal functions
like IFN_MASK_LOAD and IFN_MASK_STORE as normal gimple values.
This patch makes it treat them as addresses instead. This makes
a significant difference to the code quality for SVE loops,
since we can then use loads and stores with scaled indices.
2018-01-13 Richard Sandiford <richard.sandiford@linaro.org>
Alan Hayward <alan.hayward@arm.com>
David Sherwood <david.sherwood@arm.com>
gcc/
* tree-ssa-loop-ivopts.c (USE_ADDRESS): Split into...
(USE_REF_ADDRESS, USE_PTR_ADDRESS): ...these new use types.
(dump_groups): Update accordingly.
(iv_use::mem_type): New member variable.
(address_p): New function.
(record_use): Add a mem_type argument and initialize the new
mem_type field.
(record_group_use): Add a mem_type argument. Use address_p.
Remove obsolete null checks of base_object. Update call to record_use.
(find_interesting_uses_op): Update call to record_group_use.
(find_interesting_uses_cond): Likewise.
(find_interesting_uses_address): Likewise.
(get_mem_type_for_internal_fn): New function.
(find_address_like_use): Likewise.
(find_interesting_uses_stmt): Try find_address_like_use before
calling find_interesting_uses_op.
(addr_offset_valid_p): Use the iv mem_type field as the type
of the addressed memory.
(add_autoinc_candidates): Likewise.
(get_address_cost): Likewise.
(split_small_address_groups_p): Use address_p.
(split_address_groups): Likewise.
(add_iv_candidate_for_use): Likewise.
(autoinc_possible_for_pair): Likewise.
(rewrite_groups): Likewise.
(get_use_type): Check for USE_REF_ADDRESS instead of USE_ADDRESS.
(determine_group_iv_cost): Update after split of USE_ADDRESS.
(get_alias_ptr_type_for_ptr_address): New function.
(rewrite_use_address): Rewrite address uses in calls that were
identified by find_address_like_use.
gcc/testsuite/
* gcc.dg/tree-ssa/scev-9.c: Expected REFERENCE ADDRESS
instead of just ADDRESS.
* gcc.dg/tree-ssa/scev-10.c: Likewise.
* gcc.dg/tree-ssa/scev-11.c: Likewise.
* gcc.dg/tree-ssa/scev-12.c: Likewise.
* gcc.target/aarch64/sve/index_offset_1.c: New test.
* gcc.target/aarch64/sve/index_offset_1_run.c: Likewise.
* gcc.target/aarch64/sve/loop_add_2.c: Likewise.
* gcc.target/aarch64/sve/loop_add_3.c: Likewise.
* gcc.target/aarch64/sve/while_1.c: Check for indexed addressing modes.
* gcc.target/aarch64/sve/while_2.c: Likewise.
* gcc.target/aarch64/sve/while_3.c: Likewise.
* gcc.target/aarch64/sve/while_4.c: Likewise.
Co-Authored-By: Alan Hayward <alan.hayward@arm.com>
Co-Authored-By: David Sherwood <david.sherwood@arm.com>
From-SVN: r256628
+2018-01-13 Richard Sandiford <richard.sandiford@linaro.org>
+ Alan Hayward <alan.hayward@arm.com>
+ David Sherwood <david.sherwood@arm.com>
+
+ * tree-ssa-loop-ivopts.c (USE_ADDRESS): Split into...
+ (USE_REF_ADDRESS, USE_PTR_ADDRESS): ...these new use types.
+ (dump_groups): Update accordingly.
+ (iv_use::mem_type): New member variable.
+ (address_p): New function.
+ (record_use): Add a mem_type argument and initialize the new
+ mem_type field.
+ (record_group_use): Add a mem_type argument. Use address_p.
+ Remove obsolete null checks of base_object. Update call to record_use.
+ (find_interesting_uses_op): Update call to record_group_use.
+ (find_interesting_uses_cond): Likewise.
+ (find_interesting_uses_address): Likewise.
+ (get_mem_type_for_internal_fn): New function.
+ (find_address_like_use): Likewise.
+ (find_interesting_uses_stmt): Try find_address_like_use before
+ calling find_interesting_uses_op.
+ (addr_offset_valid_p): Use the iv mem_type field as the type
+ of the addressed memory.
+ (add_autoinc_candidates): Likewise.
+ (get_address_cost): Likewise.
+ (split_small_address_groups_p): Use address_p.
+ (split_address_groups): Likewise.
+ (add_iv_candidate_for_use): Likewise.
+ (autoinc_possible_for_pair): Likewise.
+ (rewrite_groups): Likewise.
+ (get_use_type): Check for USE_REF_ADDRESS instead of USE_ADDRESS.
+ (determine_group_iv_cost): Update after split of USE_ADDRESS.
+ (get_alias_ptr_type_for_ptr_address): New function.
+ (rewrite_use_address): Rewrite address uses in calls that were
+ identified by find_address_like_use.
+
2018-01-13 Richard Sandiford <richard.sandiford@linaro.org>
Alan Hayward <alan.hayward@arm.com>
David Sherwood <david.sherwood@arm.com>
+2018-01-13 Richard Sandiford <richard.sandiford@linaro.org>
+ Alan Hayward <alan.hayward@arm.com>
+ David Sherwood <david.sherwood@arm.com>
+
+ * gcc.dg/tree-ssa/scev-9.c: Expected REFERENCE ADDRESS
+ instead of just ADDRESS.
+ * gcc.dg/tree-ssa/scev-10.c: Likewise.
+ * gcc.dg/tree-ssa/scev-11.c: Likewise.
+ * gcc.dg/tree-ssa/scev-12.c: Likewise.
+ * gcc.target/aarch64/sve/index_offset_1.c: New test.
+ * gcc.target/aarch64/sve/index_offset_1_run.c: Likewise.
+ * gcc.target/aarch64/sve/loop_add_2.c: Likewise.
+ * gcc.target/aarch64/sve/loop_add_3.c: Likewise.
+ * gcc.target/aarch64/sve/while_1.c: Check for indexed addressing modes.
+ * gcc.target/aarch64/sve/while_2.c: Likewise.
+ * gcc.target/aarch64/sve/while_3.c: Likewise.
+ * gcc.target/aarch64/sve/while_4.c: Likewise.
+
2018-01-13 Richard Sandiford <richard.sandiford@linaro.org>
Alan Hayward <alan.hayward@arm.com>
David Sherwood <david.sherwood@arm.com>
}
/* Address of array reference is scev. */
-/* { dg-final { scan-tree-dump-times " Type:\\tADDRESS\n Use \[0-9\].\[0-9\]:" 1 "ivopts" } } */
+/* { dg-final { scan-tree-dump-times " Type:\\tREFERENCE ADDRESS\n Use \[0-9\].\[0-9\]:" 1 "ivopts" } } */
}
/* Address of array reference to b is scev. */
-/* { dg-final { scan-tree-dump-times " Type:\\tADDRESS\n Use \[0-9\].\[0-9\]:" 2 "ivopts" } } */
+/* { dg-final { scan-tree-dump-times " Type:\\tREFERENCE ADDRESS\n Use \[0-9\].\[0-9\]:" 2 "ivopts" } } */
}
/* Address of array reference to b is not scev. */
-/* { dg-final { scan-tree-dump-times " Type:\\tADDRESS\n Use \[0-9\].\[0-9\]:" 1 "ivopts" } } */
+/* { dg-final { scan-tree-dump-times " Type:\\tREFERENCE ADDRESS\n Use \[0-9\].\[0-9\]:" 1 "ivopts" } } */
}
/* Address of array reference is scev. */
-/* { dg-final { scan-tree-dump-times " Type:\\tADDRESS\n Use \[0-9\].\[0-9\]:" 1 "ivopts" } } */
+/* { dg-final { scan-tree-dump-times " Type:\\tREFERENCE ADDRESS\n Use \[0-9\].\[0-9\]:" 1 "ivopts" } } */
--- /dev/null
+/* { dg-do compile } */
+/* { dg-options "-O2 -ftree-vectorize -msve-vector-bits=256" } */
+
+#define SIZE (15 * 8 + 3)
+
+#define DEF_INDEX_OFFSET(SIGNED, TYPE, ITERTYPE) \
+void __attribute__ ((noinline, noclone)) \
+set_##SIGNED##_##TYPE##_##ITERTYPE (SIGNED TYPE *restrict out, \
+ SIGNED TYPE *restrict in) \
+{ \
+ SIGNED ITERTYPE i; \
+ for (i = 0; i < SIZE; i++) \
+ { \
+ out[i] = in[i]; \
+ } \
+} \
+void __attribute__ ((noinline, noclone)) \
+set_##SIGNED##_##TYPE##_##ITERTYPE##_var (SIGNED TYPE *restrict out, \
+ SIGNED TYPE *restrict in, \
+ SIGNED ITERTYPE n) \
+{ \
+ SIGNED ITERTYPE i; \
+ for (i = 0; i < n; i++) \
+ { \
+ out[i] = in[i]; \
+ } \
+}
+
+#define TEST_TYPE(T, SIGNED, TYPE) \
+ T (SIGNED, TYPE, char) \
+ T (SIGNED, TYPE, short) \
+ T (SIGNED, TYPE, int) \
+ T (SIGNED, TYPE, long)
+
+#define TEST_ALL(T) \
+ TEST_TYPE (T, signed, long) \
+ TEST_TYPE (T, unsigned, long) \
+ TEST_TYPE (T, signed, int) \
+ TEST_TYPE (T, unsigned, int) \
+ TEST_TYPE (T, signed, short) \
+ TEST_TYPE (T, unsigned, short) \
+ TEST_TYPE (T, signed, char) \
+ TEST_TYPE (T, unsigned, char)
+
+TEST_ALL (DEF_INDEX_OFFSET)
+
+/* { dg-final { scan-assembler-times "ld1d\\tz\[0-9\]+.d, p\[0-9\]+/z, \\\[x\[0-9\]+, x\[0-9\]+, lsl 3\\\]" 16 } } */
+/* { dg-final { scan-assembler-times "st1d\\tz\[0-9\]+.d, p\[0-9\]+, \\\[x\[0-9\]+, x\[0-9\]+, lsl 3\\\]" 16 } } */
+/* { dg-final { scan-assembler-times "ld1w\\tz\[0-9\]+.s, p\[0-9\]+/z, \\\[x\[0-9\]+, x\[0-9\]+, lsl 2\\\]" 16 } } */
+/* { dg-final { scan-assembler-times "st1w\\tz\[0-9\]+.s, p\[0-9\]+, \\\[x\[0-9\]+, x\[0-9\]+, lsl 2\\\]" 16 } } */
+/* { dg-final { scan-assembler-times "ld1h\\tz\[0-9\]+.h, p\[0-9\]+/z, \\\[x\[0-9\]+, x\[0-9\]+, lsl 1\\\]" 16 } } */
+/* { dg-final { scan-assembler-times "st1h\\tz\[0-9\]+.h, p\[0-9\]+, \\\[x\[0-9\]+, x\[0-9\]+, lsl 1\\\]" 16 } } */
+/* { dg-final { scan-assembler-times "ld1b\\tz\[0-9\]+.b, p\[0-9\]+/z, \\\[x\[0-9\]+, x\[0-9\]+\\\]" 16 } } */
+/* { dg-final { scan-assembler-times "st1b\\tz\[0-9\]+.b, p\[0-9\]+, \\\[x\[0-9\]+, x\[0-9\]+\\\]" 16 } } */
--- /dev/null
+/* { dg-do run { target aarch64_sve_hw } } */
+/* { dg-options "-O2 -ftree-vectorize" } */
+/* { dg-options "-O2 -ftree-vectorize -msve-vector-bits=256" { target aarch64_sve256_hw } } */
+
+#include "index_offset_1.c"
+
+#define TEST_INDEX_OFFSET(SIGNED, TYPE, ITERTYPE) \
+{ \
+ SIGNED TYPE out[SIZE + 1]; \
+ SIGNED TYPE in1[SIZE + 1]; \
+ SIGNED TYPE in2[SIZE + 1]; \
+ for (int i = 0; i < SIZE + 1; ++i) \
+ { \
+ in1[i] = (i * 4) ^ i; \
+ in2[i] = (i * 2) ^ i; \
+ asm volatile ("" ::: "memory"); \
+ } \
+ out[SIZE] = 42; \
+ set_##SIGNED##_##TYPE##_##ITERTYPE (out, in1); \
+ if (0 != __builtin_memcmp (out, in1, SIZE * sizeof (TYPE))) \
+ __builtin_abort (); \
+ set_##SIGNED##_##TYPE##_##ITERTYPE##_var (out, in2, SIZE); \
+ if (0 != __builtin_memcmp (out, in2, SIZE * sizeof (TYPE))) \
+ __builtin_abort (); \
+ if (out[SIZE] != 42) \
+ __builtin_abort (); \
+}
+
+int __attribute__ ((optimize (1)))
+main (void)
+{
+ TEST_ALL (TEST_INDEX_OFFSET);
+ return 0;
+}
--- /dev/null
+/* { dg-do compile } */
+/* { dg-options "-std=c99 -O3" } */
+
+void
+foo (int *__restrict a, int *__restrict b)
+{
+ for (int i = 0; i < 512; ++i)
+ a[i] += b[i];
+}
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+.s, p[0-7]+/z, \[x[0-9]+, x[0-9]+, lsl 2\]\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tst1w\tz[0-9]+.s, p[0-7]+, \[x[0-9]+, x[0-9]+, lsl 2\]\n} 1 } } */
--- /dev/null
+/* { dg-do compile } */
+/* { dg-options "-std=c99 -O3" } */
+
+void
+f (int *__restrict a,
+ int *__restrict b,
+ int *__restrict c,
+ int *__restrict d,
+ int *__restrict e,
+ int *__restrict f,
+ int *__restrict g,
+ int *__restrict h,
+ int count)
+{
+ for (int i = 0; i < count; ++i)
+ a[i] = b[i] + c[i] + d[i] + e[i] + f[i] + g[i] + h[i];
+}
+
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+.s, p[0-7]+/z, \[x[0-9]+, x[0-9]+, lsl 2\]\n} 7 } } */
+/* { dg-final { scan-assembler-times {\tst1w\tz[0-9]+.s, p[0-7]+, \[x[0-9]+, x[0-9]+, lsl 2\]\n} 1 } } */
/* { dg-final { scan-assembler-times {\twhilelo\tp[0-7]\.s, x[0-9]+,} 3 } } */
/* { dg-final { scan-assembler-times {\twhilelo\tp[0-7]\.d, xzr,} 3 } } */
/* { dg-final { scan-assembler-times {\twhilelo\tp[0-7]\.d, x[0-9]+,} 3 } } */
+/* { dg-final { scan-assembler-times {\tld1b\tz[0-9]+\.b, p[0-7]/z, \[x0, x[0-9]+\]\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tst1b\tz[0-9]+\.b, p[0-7], \[x0, x[0-9]+\]\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.h, p[0-7]/z, \[x0, x[0-9]+, lsl 1\]\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tst1h\tz[0-9]+\.h, p[0-7], \[x0, x[0-9]+, lsl 1\]\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.s, p[0-7]/z, \[x0, x[0-9]+, lsl 2\]\n} 3 } } */
+/* { dg-final { scan-assembler-times {\tst1w\tz[0-9]+\.s, p[0-7], \[x0, x[0-9]+, lsl 2\]\n} 3 } } */
+/* { dg-final { scan-assembler-times {\tld1d\tz[0-9]+\.d, p[0-7]/z, \[x0, x[0-9]+, lsl 3\]\n} 3 } } */
+/* { dg-final { scan-assembler-times {\tst1d\tz[0-9]+\.d, p[0-7], \[x0, x[0-9]+, lsl 3\]\n} 3 } } */
/* { dg-final { scan-assembler-times {\twhilelo\tp[0-7]\.s, x[0-9]+,} 3 } } */
/* { dg-final { scan-assembler-times {\twhilelo\tp[0-7]\.d, xzr,} 3 } } */
/* { dg-final { scan-assembler-times {\twhilelo\tp[0-7]\.d, x[0-9]+,} 3 } } */
+/* { dg-final { scan-assembler-times {\tld1b\tz[0-9]+\.b, p[0-7]/z, \[x0, x[0-9]+\]\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tst1b\tz[0-9]+\.b, p[0-7], \[x0, x[0-9]+\]\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.h, p[0-7]/z, \[x0, x[0-9]+, lsl 1\]\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tst1h\tz[0-9]+\.h, p[0-7], \[x0, x[0-9]+, lsl 1\]\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.s, p[0-7]/z, \[x0, x[0-9]+, lsl 2\]\n} 3 } } */
+/* { dg-final { scan-assembler-times {\tst1w\tz[0-9]+\.s, p[0-7], \[x0, x[0-9]+, lsl 2\]\n} 3 } } */
+/* { dg-final { scan-assembler-times {\tld1d\tz[0-9]+\.d, p[0-7]/z, \[x0, x[0-9]+, lsl 3\]\n} 3 } } */
+/* { dg-final { scan-assembler-times {\tst1d\tz[0-9]+\.d, p[0-7], \[x0, x[0-9]+, lsl 3\]\n} 3 } } */
/* { dg-final { scan-assembler-times {\twhilelo\tp[0-7]\.s, x[0-9]+,} 3 } } */
/* { dg-final { scan-assembler-times {\twhilelo\tp[0-7]\.d, xzr,} 3 } } */
/* { dg-final { scan-assembler-times {\twhilelo\tp[0-7]\.d, x[0-9]+,} 3 } } */
+/* { dg-final { scan-assembler-times {\tld1b\tz[0-9]+\.b, p[0-7]/z, \[x0, x[0-9]+\]\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tst1b\tz[0-9]+\.b, p[0-7], \[x0, x[0-9]+\]\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.h, p[0-7]/z, \[x0, x[0-9]+, lsl 1\]\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tst1h\tz[0-9]+\.h, p[0-7], \[x0, x[0-9]+, lsl 1\]\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.s, p[0-7]/z, \[x0, x[0-9]+, lsl 2\]\n} 3 } } */
+/* { dg-final { scan-assembler-times {\tst1w\tz[0-9]+\.s, p[0-7], \[x0, x[0-9]+, lsl 2\]\n} 3 } } */
+/* { dg-final { scan-assembler-times {\tld1d\tz[0-9]+\.d, p[0-7]/z, \[x0, x[0-9]+, lsl 3\]\n} 3 } } */
+/* { dg-final { scan-assembler-times {\tst1d\tz[0-9]+\.d, p[0-7], \[x0, x[0-9]+, lsl 3\]\n} 3 } } */
/* { dg-final { scan-assembler-times {\twhilelo\tp[0-7]\.s, x[0-9]+,} 3 } } */
/* { dg-final { scan-assembler-times {\twhilelo\tp[0-7]\.d, xzr,} 3 } } */
/* { dg-final { scan-assembler-times {\twhilelo\tp[0-7]\.d, x[0-9]+,} 3 } } */
+/* { dg-final { scan-assembler-times {\tld1b\tz[0-9]+\.b, p[0-7]/z, \[x0, x[0-9]+\]\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tst1b\tz[0-9]+\.b, p[0-7], \[x0, x[0-9]+\]\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.h, p[0-7]/z, \[x0, x[0-9]+, lsl 1\]\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tst1h\tz[0-9]+\.h, p[0-7], \[x0, x[0-9]+, lsl 1\]\n} 2 } } */
+/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.s, p[0-7]/z, \[x0, x[0-9]+, lsl 2\]\n} 3 } } */
+/* { dg-final { scan-assembler-times {\tst1w\tz[0-9]+\.s, p[0-7], \[x0, x[0-9]+, lsl 2\]\n} 3 } } */
+/* { dg-final { scan-assembler-times {\tld1d\tz[0-9]+\.d, p[0-7]/z, \[x0, x[0-9]+, lsl 3\]\n} 3 } } */
+/* { dg-final { scan-assembler-times {\tst1d\tz[0-9]+\.d, p[0-7], \[x0, x[0-9]+, lsl 3\]\n} 3 } } */
enum use_type
{
USE_NONLINEAR_EXPR, /* Use in a nonlinear expression. */
- USE_ADDRESS, /* Use in an address. */
+ USE_REF_ADDRESS, /* Use is an address for an explicit memory
+ reference. */
+ USE_PTR_ADDRESS, /* Use is a pointer argument to a function in
+ cases where the expansion of the function
+ will turn the argument into a normal address. */
USE_COMPARE /* Use is a compare. */
};
unsigned id; /* The id of the use. */
unsigned group_id; /* The group id the use belongs to. */
enum use_type type; /* Type of the use. */
+ tree mem_type; /* The memory type to use when testing whether an
+ address is legitimate, and what the address's
+ cost is. */
struct iv *iv; /* The induction variable it is based on. */
gimple *stmt; /* Statement in that it occurs. */
tree *op_p; /* The place where it occurs. */
static inline bool equal (const iv_inv_expr_ent *, const iv_inv_expr_ent *);
};
+/* Return true if uses of type TYPE represent some form of address. */
+
+inline bool
+address_p (use_type type)
+{
+ return type == USE_REF_ADDRESS || type == USE_PTR_ADDRESS;
+}
+
/* Hash function for loop invariant expressions. */
inline hashval_t
fprintf (file, "Group %d:\n", group->id);
if (group->type == USE_NONLINEAR_EXPR)
fprintf (file, " Type:\tGENERIC\n");
- else if (group->type == USE_ADDRESS)
- fprintf (file, " Type:\tADDRESS\n");
+ else if (group->type == USE_REF_ADDRESS)
+ fprintf (file, " Type:\tREFERENCE ADDRESS\n");
+ else if (group->type == USE_PTR_ADDRESS)
+ fprintf (file, " Type:\tPOINTER ARGUMENT ADDRESS\n");
else
{
gcc_assert (group->type == USE_COMPARE);
/* Records a use of TYPE at *USE_P in STMT whose value is IV in GROUP.
For address type use, ADDR_BASE is the stripped IV base, ADDR_OFFSET
- is the const offset stripped from IV base; for other types use, both
- are zero by default. */
+ is the const offset stripped from IV base and MEM_TYPE is the type
+ of the memory being addressed. For uses of other types, ADDR_BASE
+ and ADDR_OFFSET are zero by default and MEM_TYPE is NULL_TREE. */
static struct iv_use *
record_use (struct iv_group *group, tree *use_p, struct iv *iv,
- gimple *stmt, enum use_type type, tree addr_base,
- poly_uint64 addr_offset)
+ gimple *stmt, enum use_type type, tree mem_type,
+ tree addr_base, poly_uint64 addr_offset)
{
struct iv_use *use = XCNEW (struct iv_use);
use->id = group->vuses.length ();
use->group_id = group->id;
use->type = type;
+ use->mem_type = mem_type;
use->iv = iv;
use->stmt = stmt;
use->op_p = use_p;
}
/* Record a use of TYPE at *USE_P in STMT whose value is IV in a group.
- New group will be created if there is no existing group for the use. */
+ New group will be created if there is no existing group for the use.
+ MEM_TYPE is the type of memory being addressed, or NULL if this
+ isn't an address reference. */
static struct iv_use *
record_group_use (struct ivopts_data *data, tree *use_p,
- struct iv *iv, gimple *stmt, enum use_type type)
+ struct iv *iv, gimple *stmt, enum use_type type,
+ tree mem_type)
{
tree addr_base = NULL;
struct iv_group *group = NULL;
poly_uint64 addr_offset = 0;
/* Record non address type use in a new group. */
- if (type == USE_ADDRESS && iv->base_object)
+ if (address_p (type))
{
unsigned int i;
group = data->vgroups[i];
use = group->vuses[0];
- if (use->type != USE_ADDRESS || !use->iv->base_object)
+ if (!address_p (use->type))
continue;
/* Check if it has the same stripped base and step. */
if (!group)
group = record_group (data, type);
- return record_use (group, use_p, iv, stmt, type, addr_base, addr_offset);
+ return record_use (group, use_p, iv, stmt, type, mem_type,
+ addr_base, addr_offset);
}
/* Checks whether the use OP is interesting and if so, records it. */
stmt = SSA_NAME_DEF_STMT (op);
gcc_assert (gimple_code (stmt) == GIMPLE_PHI || is_gimple_assign (stmt));
- use = record_group_use (data, NULL, iv, stmt, USE_NONLINEAR_EXPR);
+ use = record_group_use (data, NULL, iv, stmt, USE_NONLINEAR_EXPR, NULL_TREE);
iv->nonlin_use = use;
return use;
}
return;
}
- record_group_use (data, var_p, var_iv, stmt, USE_COMPARE);
+ record_group_use (data, var_p, var_iv, stmt, USE_COMPARE, NULL_TREE);
/* Record compare type iv_use for iv on the other side of comparison. */
if (ret == COMP_IV_EXPR_2)
- record_group_use (data, bound_p, bound_iv, stmt, USE_COMPARE);
+ record_group_use (data, bound_p, bound_iv, stmt, USE_COMPARE, NULL_TREE);
}
/* Returns the outermost loop EXPR is obviously invariant in
if (civ->base_object == NULL_TREE)
goto fail;
- record_group_use (data, op_p, civ, stmt, USE_ADDRESS);
+ record_group_use (data, op_p, civ, stmt, USE_REF_ADDRESS, TREE_TYPE (*op_p));
return;
fail:
}
}
+/* CALL calls an internal function. If operand *OP_P will become an
+ address when the call is expanded, return the type of the memory
+ being addressed, otherwise return null. */
+
+static tree
+get_mem_type_for_internal_fn (gcall *call, tree *op_p)
+{
+ switch (gimple_call_internal_fn (call))
+ {
+ case IFN_MASK_LOAD:
+ if (op_p == gimple_call_arg_ptr (call, 0))
+ return TREE_TYPE (gimple_call_lhs (call));
+ return NULL_TREE;
+
+ case IFN_MASK_STORE:
+ if (op_p == gimple_call_arg_ptr (call, 0))
+ return TREE_TYPE (gimple_call_arg (call, 3));
+ return NULL_TREE;
+
+ default:
+ return NULL_TREE;
+ }
+}
+
+/* IV is a (non-address) iv that describes operand *OP_P of STMT.
+ Return true if the operand will become an address when STMT
+ is expanded and record the associated address use if so. */
+
+static bool
+find_address_like_use (struct ivopts_data *data, gimple *stmt, tree *op_p,
+ struct iv *iv)
+{
+ /* Fail if base object of this memory reference is unknown. */
+ if (iv->base_object == NULL_TREE)
+ return false;
+
+ tree mem_type = NULL_TREE;
+ if (gcall *call = dyn_cast <gcall *> (stmt))
+ if (gimple_call_internal_p (call))
+ mem_type = get_mem_type_for_internal_fn (call, op_p);
+ if (mem_type)
+ {
+ iv = alloc_iv (data, iv->base, iv->step);
+ record_group_use (data, op_p, iv, stmt, USE_PTR_ADDRESS, mem_type);
+ return true;
+ }
+ return false;
+}
+
/* Finds interesting uses of induction variables in the statement STMT. */
static void
if (!iv)
continue;
- find_interesting_uses_op (data, op);
+ if (!find_address_like_use (data, stmt, use_p->use, iv))
+ find_interesting_uses_op (data, op);
}
}
rtx reg, addr;
unsigned list_index;
addr_space_t as = TYPE_ADDR_SPACE (TREE_TYPE (use->iv->base));
- machine_mode addr_mode, mem_mode = TYPE_MODE (TREE_TYPE (*use->op_p));
+ machine_mode addr_mode, mem_mode = TYPE_MODE (use->mem_type);
list_index = (unsigned) as * MAX_MACHINE_MODE + (unsigned) mem_mode;
if (list_index >= vec_safe_length (addr_list))
if (group->vuses.length () == 1)
continue;
- gcc_assert (group->type == USE_ADDRESS);
+ gcc_assert (address_p (group->type));
if (group->vuses.length () == 2)
{
if (compare_sizes_for_sort (group->vuses[0]->addr_offset,
if (group->vuses.length () == 1)
continue;
- gcc_assert (group->type == USE_ADDRESS);
+ gcc_assert (address_p (use->type));
for (j = 1; j < group->vuses.length ();)
{
cstepi = int_cst_value (step);
- mem_mode = TYPE_MODE (TREE_TYPE (*use->op_p));
+ mem_mode = TYPE_MODE (use->mem_type);
if (((USE_LOAD_PRE_INCREMENT (mem_mode)
|| USE_STORE_PRE_INCREMENT (mem_mode))
&& known_eq (GET_MODE_SIZE (mem_mode), cstepi))
/* At last, add auto-incremental candidates. Make such variables
important since other iv uses with same base object may be based
on it. */
- if (use != NULL && use->type == USE_ADDRESS)
+ if (use != NULL && address_p (use->type))
add_autoinc_candidates (data, iv->base, iv->step, true, use);
}
tree base_type = TREE_TYPE (use->iv->base);
tree type;
- if (use->type == USE_ADDRESS)
+ if (use->type == USE_REF_ADDRESS)
{
/* The base_type may be a void pointer. Create a pointer type based on
the mem_ref instead. */
struct mem_address parts = {NULL_TREE, integer_one_node,
NULL_TREE, NULL_TREE, NULL_TREE};
machine_mode addr_mode = TYPE_MODE (type);
- machine_mode mem_mode = TYPE_MODE (TREE_TYPE (*use->op_p));
+ machine_mode mem_mode = TYPE_MODE (use->mem_type);
addr_space_t as = TYPE_ADDR_SPACE (TREE_TYPE (use->iv->base));
/* Only true if ratio != 1. */
bool ok_with_ratio_p = false;
case USE_NONLINEAR_EXPR:
return determine_group_iv_cost_generic (data, group, cand);
- case USE_ADDRESS:
+ case USE_REF_ADDRESS:
+ case USE_PTR_ADDRESS:
return determine_group_iv_cost_address (data, group, cand);
case USE_COMPARE:
autoinc_possible_for_pair (struct ivopts_data *data, struct iv_use *use,
struct iv_cand *cand)
{
- if (use->type != USE_ADDRESS)
+ if (!address_p (use->type))
return false;
bool can_autoinc = false;
cand->incremented_at = use->stmt;
}
+/* Return the alias pointer type that should be used for a MEM_REF
+ associated with USE, which has type USE_PTR_ADDRESS. */
+
+static tree
+get_alias_ptr_type_for_ptr_address (iv_use *use)
+{
+ gcall *call = as_a <gcall *> (use->stmt);
+ switch (gimple_call_internal_fn (call))
+ {
+ case IFN_MASK_LOAD:
+ case IFN_MASK_STORE:
+ /* The second argument contains the correct alias type. */
+ gcc_assert (use->op_p = gimple_call_arg_ptr (call, 0));
+ return TREE_TYPE (gimple_call_arg (call, 1));
+
+ default:
+ gcc_unreachable ();
+ }
+}
+
+
/* Rewrites USE (address that is an iv) using candidate CAND. */
static void
tree iv = var_at_stmt (data->current_loop, cand, use->stmt);
tree base_hint = (cand->iv->base_object) ? iv : NULL_TREE;
gimple_stmt_iterator bsi = gsi_for_stmt (use->stmt);
- tree type = TREE_TYPE (*use->op_p);
- unsigned int align = get_object_alignment (*use->op_p);
- if (align != TYPE_ALIGN (type))
- type = build_aligned_type (type, align);
-
- tree ref = create_mem_ref (&bsi, type, &aff,
- reference_alias_ptr_type (*use->op_p),
+ tree type = use->mem_type;
+ tree alias_ptr_type;
+ if (use->type == USE_PTR_ADDRESS)
+ alias_ptr_type = get_alias_ptr_type_for_ptr_address (use);
+ else
+ {
+ gcc_assert (type == TREE_TYPE (*use->op_p));
+ unsigned int align = get_object_alignment (*use->op_p);
+ if (align != TYPE_ALIGN (type))
+ type = build_aligned_type (type, align);
+ alias_ptr_type = reference_alias_ptr_type (*use->op_p);
+ }
+ tree ref = create_mem_ref (&bsi, type, &aff, alias_ptr_type,
iv, base_hint, data->speed);
- copy_ref_info (ref, *use->op_p);
+ if (use->type == USE_PTR_ADDRESS)
+ {
+ ref = fold_build1 (ADDR_EXPR, build_pointer_type (use->mem_type), ref);
+ ref = fold_convert (get_use_type (use), ref);
+ ref = force_gimple_operand_gsi (&bsi, ref, true, NULL_TREE,
+ true, GSI_SAME_STMT);
+ }
+ else
+ copy_ref_info (ref, *use->op_p);
+
*use->op_p = ref;
}
update_stmt (group->vuses[j]->stmt);
}
}
- else if (group->type == USE_ADDRESS)
+ else if (address_p (group->type))
{
for (j = 0; j < group->vuses.length (); j++)
{
projects / gcc.git / commitdiff