From: Richard Sandiford Date: Sat, 13 Jan 2018 17:59:15 +0000 (+0000) Subject: Make ivopts handle calls to internal functions X-Git-Url: https://git.libre-soc.org/?a=commitdiff_plain;h=8277ddf9eeae431d432855e41537df1c3a4fa323;p=gcc.git Make ivopts handle calls to internal functions ivopts previously treated pointer arguments to internal functions like IFN_MASK_LOAD and IFN_MASK_STORE as normal gimple values. This patch makes it treat them as addresses instead. This makes a significant difference to the code quality for SVE loops, since we can then use loads and stores with scaled indices. 2018-01-13 Richard Sandiford Alan Hayward David Sherwood gcc/ * tree-ssa-loop-ivopts.c (USE_ADDRESS): Split into... (USE_REF_ADDRESS, USE_PTR_ADDRESS): ...these new use types. (dump_groups): Update accordingly. (iv_use::mem_type): New member variable. (address_p): New function. (record_use): Add a mem_type argument and initialize the new mem_type field. (record_group_use): Add a mem_type argument. Use address_p. Remove obsolete null checks of base_object. Update call to record_use. (find_interesting_uses_op): Update call to record_group_use. (find_interesting_uses_cond): Likewise. (find_interesting_uses_address): Likewise. (get_mem_type_for_internal_fn): New function. (find_address_like_use): Likewise. (find_interesting_uses_stmt): Try find_address_like_use before calling find_interesting_uses_op. (addr_offset_valid_p): Use the iv mem_type field as the type of the addressed memory. (add_autoinc_candidates): Likewise. (get_address_cost): Likewise. (split_small_address_groups_p): Use address_p. (split_address_groups): Likewise. (add_iv_candidate_for_use): Likewise. (autoinc_possible_for_pair): Likewise. (rewrite_groups): Likewise. (get_use_type): Check for USE_REF_ADDRESS instead of USE_ADDRESS. (determine_group_iv_cost): Update after split of USE_ADDRESS. (get_alias_ptr_type_for_ptr_address): New function. (rewrite_use_address): Rewrite address uses in calls that were identified by find_address_like_use. gcc/testsuite/ * gcc.dg/tree-ssa/scev-9.c: Expected REFERENCE ADDRESS instead of just ADDRESS. * gcc.dg/tree-ssa/scev-10.c: Likewise. * gcc.dg/tree-ssa/scev-11.c: Likewise. * gcc.dg/tree-ssa/scev-12.c: Likewise. * gcc.target/aarch64/sve/index_offset_1.c: New test. * gcc.target/aarch64/sve/index_offset_1_run.c: Likewise. * gcc.target/aarch64/sve/loop_add_2.c: Likewise. * gcc.target/aarch64/sve/loop_add_3.c: Likewise. * gcc.target/aarch64/sve/while_1.c: Check for indexed addressing modes. * gcc.target/aarch64/sve/while_2.c: Likewise. * gcc.target/aarch64/sve/while_3.c: Likewise. * gcc.target/aarch64/sve/while_4.c: Likewise. Co-Authored-By: Alan Hayward Co-Authored-By: David Sherwood From-SVN: r256628 --- diff --git a/gcc/ChangeLog b/gcc/ChangeLog index 79ddaf32d89..6d198be273f 100644 --- a/gcc/ChangeLog +++ b/gcc/ChangeLog @@ -1,3 +1,38 @@ +2018-01-13 Richard Sandiford + Alan Hayward + David Sherwood + + * tree-ssa-loop-ivopts.c (USE_ADDRESS): Split into... + (USE_REF_ADDRESS, USE_PTR_ADDRESS): ...these new use types. + (dump_groups): Update accordingly. + (iv_use::mem_type): New member variable. + (address_p): New function. + (record_use): Add a mem_type argument and initialize the new + mem_type field. + (record_group_use): Add a mem_type argument. Use address_p. + Remove obsolete null checks of base_object. Update call to record_use. + (find_interesting_uses_op): Update call to record_group_use. + (find_interesting_uses_cond): Likewise. + (find_interesting_uses_address): Likewise. + (get_mem_type_for_internal_fn): New function. + (find_address_like_use): Likewise. + (find_interesting_uses_stmt): Try find_address_like_use before + calling find_interesting_uses_op. + (addr_offset_valid_p): Use the iv mem_type field as the type + of the addressed memory. + (add_autoinc_candidates): Likewise. + (get_address_cost): Likewise. + (split_small_address_groups_p): Use address_p. + (split_address_groups): Likewise. + (add_iv_candidate_for_use): Likewise. + (autoinc_possible_for_pair): Likewise. + (rewrite_groups): Likewise. + (get_use_type): Check for USE_REF_ADDRESS instead of USE_ADDRESS. + (determine_group_iv_cost): Update after split of USE_ADDRESS. + (get_alias_ptr_type_for_ptr_address): New function. + (rewrite_use_address): Rewrite address uses in calls that were + identified by find_address_like_use. + 2018-01-13 Richard Sandiford Alan Hayward David Sherwood diff --git a/gcc/testsuite/ChangeLog b/gcc/testsuite/ChangeLog index 7363c0d04a4..c59919ab3c4 100644 --- a/gcc/testsuite/ChangeLog +++ b/gcc/testsuite/ChangeLog @@ -1,3 +1,21 @@ +2018-01-13 Richard Sandiford + Alan Hayward + David Sherwood + + * gcc.dg/tree-ssa/scev-9.c: Expected REFERENCE ADDRESS + instead of just ADDRESS. + * gcc.dg/tree-ssa/scev-10.c: Likewise. + * gcc.dg/tree-ssa/scev-11.c: Likewise. + * gcc.dg/tree-ssa/scev-12.c: Likewise. + * gcc.target/aarch64/sve/index_offset_1.c: New test. + * gcc.target/aarch64/sve/index_offset_1_run.c: Likewise. + * gcc.target/aarch64/sve/loop_add_2.c: Likewise. + * gcc.target/aarch64/sve/loop_add_3.c: Likewise. + * gcc.target/aarch64/sve/while_1.c: Check for indexed addressing modes. + * gcc.target/aarch64/sve/while_2.c: Likewise. + * gcc.target/aarch64/sve/while_3.c: Likewise. + * gcc.target/aarch64/sve/while_4.c: Likewise. + 2018-01-13 Richard Sandiford Alan Hayward David Sherwood diff --git a/gcc/testsuite/gcc.dg/tree-ssa/scev-10.c b/gcc/testsuite/gcc.dg/tree-ssa/scev-10.c index e402bf92b75..6c4fca79fa4 100644 --- a/gcc/testsuite/gcc.dg/tree-ssa/scev-10.c +++ b/gcc/testsuite/gcc.dg/tree-ssa/scev-10.c @@ -18,5 +18,5 @@ foo (signed char s, signed char l) } /* Address of array reference is scev. */ -/* { dg-final { scan-tree-dump-times " Type:\\tADDRESS\n Use \[0-9\].\[0-9\]:" 1 "ivopts" } } */ +/* { dg-final { scan-tree-dump-times " Type:\\tREFERENCE ADDRESS\n Use \[0-9\].\[0-9\]:" 1 "ivopts" } } */ diff --git a/gcc/testsuite/gcc.dg/tree-ssa/scev-11.c b/gcc/testsuite/gcc.dg/tree-ssa/scev-11.c index 088771ebe40..c665fd7f20d 100644 --- a/gcc/testsuite/gcc.dg/tree-ssa/scev-11.c +++ b/gcc/testsuite/gcc.dg/tree-ssa/scev-11.c @@ -23,4 +23,4 @@ foo (int n) } /* Address of array reference to b is scev. */ -/* { dg-final { scan-tree-dump-times " Type:\\tADDRESS\n Use \[0-9\].\[0-9\]:" 2 "ivopts" } } */ +/* { dg-final { scan-tree-dump-times " Type:\\tREFERENCE ADDRESS\n Use \[0-9\].\[0-9\]:" 2 "ivopts" } } */ diff --git a/gcc/testsuite/gcc.dg/tree-ssa/scev-12.c b/gcc/testsuite/gcc.dg/tree-ssa/scev-12.c index c112639d4cb..f598c7b2955 100644 --- a/gcc/testsuite/gcc.dg/tree-ssa/scev-12.c +++ b/gcc/testsuite/gcc.dg/tree-ssa/scev-12.c @@ -24,4 +24,4 @@ foo (int x, int n) } /* Address of array reference to b is not scev. */ -/* { dg-final { scan-tree-dump-times " Type:\\tADDRESS\n Use \[0-9\].\[0-9\]:" 1 "ivopts" } } */ +/* { dg-final { scan-tree-dump-times " Type:\\tREFERENCE ADDRESS\n Use \[0-9\].\[0-9\]:" 1 "ivopts" } } */ diff --git a/gcc/testsuite/gcc.dg/tree-ssa/scev-9.c b/gcc/testsuite/gcc.dg/tree-ssa/scev-9.c index b11b2f1b482..2863918a687 100644 --- a/gcc/testsuite/gcc.dg/tree-ssa/scev-9.c +++ b/gcc/testsuite/gcc.dg/tree-ssa/scev-9.c @@ -18,5 +18,5 @@ foo (unsigned char s, unsigned char l) } /* Address of array reference is scev. */ -/* { dg-final { scan-tree-dump-times " Type:\\tADDRESS\n Use \[0-9\].\[0-9\]:" 1 "ivopts" } } */ +/* { dg-final { scan-tree-dump-times " Type:\\tREFERENCE ADDRESS\n Use \[0-9\].\[0-9\]:" 1 "ivopts" } } */ diff --git a/gcc/testsuite/gcc.target/aarch64/sve/index_offset_1.c b/gcc/testsuite/gcc.target/aarch64/sve/index_offset_1.c new file mode 100644 index 00000000000..31d46aab960 --- /dev/null +++ b/gcc/testsuite/gcc.target/aarch64/sve/index_offset_1.c @@ -0,0 +1,54 @@ +/* { dg-do compile } */ +/* { dg-options "-O2 -ftree-vectorize -msve-vector-bits=256" } */ + +#define SIZE (15 * 8 + 3) + +#define DEF_INDEX_OFFSET(SIGNED, TYPE, ITERTYPE) \ +void __attribute__ ((noinline, noclone)) \ +set_##SIGNED##_##TYPE##_##ITERTYPE (SIGNED TYPE *restrict out, \ + SIGNED TYPE *restrict in) \ +{ \ + SIGNED ITERTYPE i; \ + for (i = 0; i < SIZE; i++) \ + { \ + out[i] = in[i]; \ + } \ +} \ +void __attribute__ ((noinline, noclone)) \ +set_##SIGNED##_##TYPE##_##ITERTYPE##_var (SIGNED TYPE *restrict out, \ + SIGNED TYPE *restrict in, \ + SIGNED ITERTYPE n) \ +{ \ + SIGNED ITERTYPE i; \ + for (i = 0; i < n; i++) \ + { \ + out[i] = in[i]; \ + } \ +} + +#define TEST_TYPE(T, SIGNED, TYPE) \ + T (SIGNED, TYPE, char) \ + T (SIGNED, TYPE, short) \ + T (SIGNED, TYPE, int) \ + T (SIGNED, TYPE, long) + +#define TEST_ALL(T) \ + TEST_TYPE (T, signed, long) \ + TEST_TYPE (T, unsigned, long) \ + TEST_TYPE (T, signed, int) \ + TEST_TYPE (T, unsigned, int) \ + TEST_TYPE (T, signed, short) \ + TEST_TYPE (T, unsigned, short) \ + TEST_TYPE (T, signed, char) \ + TEST_TYPE (T, unsigned, char) + +TEST_ALL (DEF_INDEX_OFFSET) + +/* { dg-final { scan-assembler-times "ld1d\\tz\[0-9\]+.d, p\[0-9\]+/z, \\\[x\[0-9\]+, x\[0-9\]+, lsl 3\\\]" 16 } } */ +/* { dg-final { scan-assembler-times "st1d\\tz\[0-9\]+.d, p\[0-9\]+, \\\[x\[0-9\]+, x\[0-9\]+, lsl 3\\\]" 16 } } */ +/* { dg-final { scan-assembler-times "ld1w\\tz\[0-9\]+.s, p\[0-9\]+/z, \\\[x\[0-9\]+, x\[0-9\]+, lsl 2\\\]" 16 } } */ +/* { dg-final { scan-assembler-times "st1w\\tz\[0-9\]+.s, p\[0-9\]+, \\\[x\[0-9\]+, x\[0-9\]+, lsl 2\\\]" 16 } } */ +/* { dg-final { scan-assembler-times "ld1h\\tz\[0-9\]+.h, p\[0-9\]+/z, \\\[x\[0-9\]+, x\[0-9\]+, lsl 1\\\]" 16 } } */ +/* { dg-final { scan-assembler-times "st1h\\tz\[0-9\]+.h, p\[0-9\]+, \\\[x\[0-9\]+, x\[0-9\]+, lsl 1\\\]" 16 } } */ +/* { dg-final { scan-assembler-times "ld1b\\tz\[0-9\]+.b, p\[0-9\]+/z, \\\[x\[0-9\]+, x\[0-9\]+\\\]" 16 } } */ +/* { dg-final { scan-assembler-times "st1b\\tz\[0-9\]+.b, p\[0-9\]+, \\\[x\[0-9\]+, x\[0-9\]+\\\]" 16 } } */ diff --git a/gcc/testsuite/gcc.target/aarch64/sve/index_offset_1_run.c b/gcc/testsuite/gcc.target/aarch64/sve/index_offset_1_run.c new file mode 100644 index 00000000000..00114acfe49 --- /dev/null +++ b/gcc/testsuite/gcc.target/aarch64/sve/index_offset_1_run.c @@ -0,0 +1,34 @@ +/* { dg-do run { target aarch64_sve_hw } } */ +/* { dg-options "-O2 -ftree-vectorize" } */ +/* { dg-options "-O2 -ftree-vectorize -msve-vector-bits=256" { target aarch64_sve256_hw } } */ + +#include "index_offset_1.c" + +#define TEST_INDEX_OFFSET(SIGNED, TYPE, ITERTYPE) \ +{ \ + SIGNED TYPE out[SIZE + 1]; \ + SIGNED TYPE in1[SIZE + 1]; \ + SIGNED TYPE in2[SIZE + 1]; \ + for (int i = 0; i < SIZE + 1; ++i) \ + { \ + in1[i] = (i * 4) ^ i; \ + in2[i] = (i * 2) ^ i; \ + asm volatile ("" ::: "memory"); \ + } \ + out[SIZE] = 42; \ + set_##SIGNED##_##TYPE##_##ITERTYPE (out, in1); \ + if (0 != __builtin_memcmp (out, in1, SIZE * sizeof (TYPE))) \ + __builtin_abort (); \ + set_##SIGNED##_##TYPE##_##ITERTYPE##_var (out, in2, SIZE); \ + if (0 != __builtin_memcmp (out, in2, SIZE * sizeof (TYPE))) \ + __builtin_abort (); \ + if (out[SIZE] != 42) \ + __builtin_abort (); \ +} + +int __attribute__ ((optimize (1))) +main (void) +{ + TEST_ALL (TEST_INDEX_OFFSET); + return 0; +} diff --git a/gcc/testsuite/gcc.target/aarch64/sve/loop_add_2.c b/gcc/testsuite/gcc.target/aarch64/sve/loop_add_2.c new file mode 100644 index 00000000000..a58ba0b5d5d --- /dev/null +++ b/gcc/testsuite/gcc.target/aarch64/sve/loop_add_2.c @@ -0,0 +1,12 @@ +/* { dg-do compile } */ +/* { dg-options "-std=c99 -O3" } */ + +void +foo (int *__restrict a, int *__restrict b) +{ + for (int i = 0; i < 512; ++i) + a[i] += b[i]; +} + +/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+.s, p[0-7]+/z, \[x[0-9]+, x[0-9]+, lsl 2\]\n} 2 } } */ +/* { dg-final { scan-assembler-times {\tst1w\tz[0-9]+.s, p[0-7]+, \[x[0-9]+, x[0-9]+, lsl 2\]\n} 1 } } */ diff --git a/gcc/testsuite/gcc.target/aarch64/sve/loop_add_3.c b/gcc/testsuite/gcc.target/aarch64/sve/loop_add_3.c new file mode 100644 index 00000000000..9d2c5981cd7 --- /dev/null +++ b/gcc/testsuite/gcc.target/aarch64/sve/loop_add_3.c @@ -0,0 +1,20 @@ +/* { dg-do compile } */ +/* { dg-options "-std=c99 -O3" } */ + +void +f (int *__restrict a, + int *__restrict b, + int *__restrict c, + int *__restrict d, + int *__restrict e, + int *__restrict f, + int *__restrict g, + int *__restrict h, + int count) +{ + for (int i = 0; i < count; ++i) + a[i] = b[i] + c[i] + d[i] + e[i] + f[i] + g[i] + h[i]; +} + +/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+.s, p[0-7]+/z, \[x[0-9]+, x[0-9]+, lsl 2\]\n} 7 } } */ +/* { dg-final { scan-assembler-times {\tst1w\tz[0-9]+.s, p[0-7]+, \[x[0-9]+, x[0-9]+, lsl 2\]\n} 1 } } */ diff --git a/gcc/testsuite/gcc.target/aarch64/sve/while_1.c b/gcc/testsuite/gcc.target/aarch64/sve/while_1.c index 70ceec86975..a93a04baa3b 100644 --- a/gcc/testsuite/gcc.target/aarch64/sve/while_1.c +++ b/gcc/testsuite/gcc.target/aarch64/sve/while_1.c @@ -34,3 +34,11 @@ TEST_ALL (ADD_LOOP) /* { dg-final { scan-assembler-times {\twhilelo\tp[0-7]\.s, x[0-9]+,} 3 } } */ /* { dg-final { scan-assembler-times {\twhilelo\tp[0-7]\.d, xzr,} 3 } } */ /* { dg-final { scan-assembler-times {\twhilelo\tp[0-7]\.d, x[0-9]+,} 3 } } */ +/* { dg-final { scan-assembler-times {\tld1b\tz[0-9]+\.b, p[0-7]/z, \[x0, x[0-9]+\]\n} 2 } } */ +/* { dg-final { scan-assembler-times {\tst1b\tz[0-9]+\.b, p[0-7], \[x0, x[0-9]+\]\n} 2 } } */ +/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.h, p[0-7]/z, \[x0, x[0-9]+, lsl 1\]\n} 2 } } */ +/* { dg-final { scan-assembler-times {\tst1h\tz[0-9]+\.h, p[0-7], \[x0, x[0-9]+, lsl 1\]\n} 2 } } */ +/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.s, p[0-7]/z, \[x0, x[0-9]+, lsl 2\]\n} 3 } } */ +/* { dg-final { scan-assembler-times {\tst1w\tz[0-9]+\.s, p[0-7], \[x0, x[0-9]+, lsl 2\]\n} 3 } } */ +/* { dg-final { scan-assembler-times {\tld1d\tz[0-9]+\.d, p[0-7]/z, \[x0, x[0-9]+, lsl 3\]\n} 3 } } */ +/* { dg-final { scan-assembler-times {\tst1d\tz[0-9]+\.d, p[0-7], \[x0, x[0-9]+, lsl 3\]\n} 3 } } */ diff --git a/gcc/testsuite/gcc.target/aarch64/sve/while_2.c b/gcc/testsuite/gcc.target/aarch64/sve/while_2.c index 651bb574042..69bb6cb0e1a 100644 --- a/gcc/testsuite/gcc.target/aarch64/sve/while_2.c +++ b/gcc/testsuite/gcc.target/aarch64/sve/while_2.c @@ -34,3 +34,11 @@ TEST_ALL (ADD_LOOP) /* { dg-final { scan-assembler-times {\twhilelo\tp[0-7]\.s, x[0-9]+,} 3 } } */ /* { dg-final { scan-assembler-times {\twhilelo\tp[0-7]\.d, xzr,} 3 } } */ /* { dg-final { scan-assembler-times {\twhilelo\tp[0-7]\.d, x[0-9]+,} 3 } } */ +/* { dg-final { scan-assembler-times {\tld1b\tz[0-9]+\.b, p[0-7]/z, \[x0, x[0-9]+\]\n} 2 } } */ +/* { dg-final { scan-assembler-times {\tst1b\tz[0-9]+\.b, p[0-7], \[x0, x[0-9]+\]\n} 2 } } */ +/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.h, p[0-7]/z, \[x0, x[0-9]+, lsl 1\]\n} 2 } } */ +/* { dg-final { scan-assembler-times {\tst1h\tz[0-9]+\.h, p[0-7], \[x0, x[0-9]+, lsl 1\]\n} 2 } } */ +/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.s, p[0-7]/z, \[x0, x[0-9]+, lsl 2\]\n} 3 } } */ +/* { dg-final { scan-assembler-times {\tst1w\tz[0-9]+\.s, p[0-7], \[x0, x[0-9]+, lsl 2\]\n} 3 } } */ +/* { dg-final { scan-assembler-times {\tld1d\tz[0-9]+\.d, p[0-7]/z, \[x0, x[0-9]+, lsl 3\]\n} 3 } } */ +/* { dg-final { scan-assembler-times {\tst1d\tz[0-9]+\.d, p[0-7], \[x0, x[0-9]+, lsl 3\]\n} 3 } } */ diff --git a/gcc/testsuite/gcc.target/aarch64/sve/while_3.c b/gcc/testsuite/gcc.target/aarch64/sve/while_3.c index 68d524e04dd..f805c2fc57d 100644 --- a/gcc/testsuite/gcc.target/aarch64/sve/while_3.c +++ b/gcc/testsuite/gcc.target/aarch64/sve/while_3.c @@ -34,3 +34,11 @@ TEST_ALL (ADD_LOOP) /* { dg-final { scan-assembler-times {\twhilelo\tp[0-7]\.s, x[0-9]+,} 3 } } */ /* { dg-final { scan-assembler-times {\twhilelo\tp[0-7]\.d, xzr,} 3 } } */ /* { dg-final { scan-assembler-times {\twhilelo\tp[0-7]\.d, x[0-9]+,} 3 } } */ +/* { dg-final { scan-assembler-times {\tld1b\tz[0-9]+\.b, p[0-7]/z, \[x0, x[0-9]+\]\n} 2 } } */ +/* { dg-final { scan-assembler-times {\tst1b\tz[0-9]+\.b, p[0-7], \[x0, x[0-9]+\]\n} 2 } } */ +/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.h, p[0-7]/z, \[x0, x[0-9]+, lsl 1\]\n} 2 } } */ +/* { dg-final { scan-assembler-times {\tst1h\tz[0-9]+\.h, p[0-7], \[x0, x[0-9]+, lsl 1\]\n} 2 } } */ +/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.s, p[0-7]/z, \[x0, x[0-9]+, lsl 2\]\n} 3 } } */ +/* { dg-final { scan-assembler-times {\tst1w\tz[0-9]+\.s, p[0-7], \[x0, x[0-9]+, lsl 2\]\n} 3 } } */ +/* { dg-final { scan-assembler-times {\tld1d\tz[0-9]+\.d, p[0-7]/z, \[x0, x[0-9]+, lsl 3\]\n} 3 } } */ +/* { dg-final { scan-assembler-times {\tst1d\tz[0-9]+\.d, p[0-7], \[x0, x[0-9]+, lsl 3\]\n} 3 } } */ diff --git a/gcc/testsuite/gcc.target/aarch64/sve/while_4.c b/gcc/testsuite/gcc.target/aarch64/sve/while_4.c index d8ec5364ab1..e7aa99f3b87 100644 --- a/gcc/testsuite/gcc.target/aarch64/sve/while_4.c +++ b/gcc/testsuite/gcc.target/aarch64/sve/while_4.c @@ -35,3 +35,11 @@ TEST_ALL (ADD_LOOP) /* { dg-final { scan-assembler-times {\twhilelo\tp[0-7]\.s, x[0-9]+,} 3 } } */ /* { dg-final { scan-assembler-times {\twhilelo\tp[0-7]\.d, xzr,} 3 } } */ /* { dg-final { scan-assembler-times {\twhilelo\tp[0-7]\.d, x[0-9]+,} 3 } } */ +/* { dg-final { scan-assembler-times {\tld1b\tz[0-9]+\.b, p[0-7]/z, \[x0, x[0-9]+\]\n} 2 } } */ +/* { dg-final { scan-assembler-times {\tst1b\tz[0-9]+\.b, p[0-7], \[x0, x[0-9]+\]\n} 2 } } */ +/* { dg-final { scan-assembler-times {\tld1h\tz[0-9]+\.h, p[0-7]/z, \[x0, x[0-9]+, lsl 1\]\n} 2 } } */ +/* { dg-final { scan-assembler-times {\tst1h\tz[0-9]+\.h, p[0-7], \[x0, x[0-9]+, lsl 1\]\n} 2 } } */ +/* { dg-final { scan-assembler-times {\tld1w\tz[0-9]+\.s, p[0-7]/z, \[x0, x[0-9]+, lsl 2\]\n} 3 } } */ +/* { dg-final { scan-assembler-times {\tst1w\tz[0-9]+\.s, p[0-7], \[x0, x[0-9]+, lsl 2\]\n} 3 } } */ +/* { dg-final { scan-assembler-times {\tld1d\tz[0-9]+\.d, p[0-7]/z, \[x0, x[0-9]+, lsl 3\]\n} 3 } } */ +/* { dg-final { scan-assembler-times {\tst1d\tz[0-9]+\.d, p[0-7], \[x0, x[0-9]+, lsl 3\]\n} 3 } } */ diff --git a/gcc/tree-ssa-loop-ivopts.c b/gcc/tree-ssa-loop-ivopts.c index 63f01d22344..b3135717f22 100644 --- a/gcc/tree-ssa-loop-ivopts.c +++ b/gcc/tree-ssa-loop-ivopts.c @@ -166,7 +166,11 @@ struct version_info enum use_type { USE_NONLINEAR_EXPR, /* Use in a nonlinear expression. */ - USE_ADDRESS, /* Use in an address. */ + USE_REF_ADDRESS, /* Use is an address for an explicit memory + reference. */ + USE_PTR_ADDRESS, /* Use is a pointer argument to a function in + cases where the expansion of the function + will turn the argument into a normal address. */ USE_COMPARE /* Use is a compare. */ }; @@ -362,6 +366,9 @@ struct iv_use unsigned id; /* The id of the use. */ unsigned group_id; /* The group id the use belongs to. */ enum use_type type; /* Type of the use. */ + tree mem_type; /* The memory type to use when testing whether an + address is legitimate, and what the address's + cost is. */ struct iv *iv; /* The induction variable it is based on. */ gimple *stmt; /* Statement in that it occurs. */ tree *op_p; /* The place where it occurs. */ @@ -506,6 +513,14 @@ struct iv_inv_expr_hasher : free_ptr_hash static inline bool equal (const iv_inv_expr_ent *, const iv_inv_expr_ent *); }; +/* Return true if uses of type TYPE represent some form of address. */ + +inline bool +address_p (use_type type) +{ + return type == USE_REF_ADDRESS || type == USE_PTR_ADDRESS; +} + /* Hash function for loop invariant expressions. */ inline hashval_t @@ -768,8 +783,10 @@ dump_groups (FILE *file, struct ivopts_data *data) fprintf (file, "Group %d:\n", group->id); if (group->type == USE_NONLINEAR_EXPR) fprintf (file, " Type:\tGENERIC\n"); - else if (group->type == USE_ADDRESS) - fprintf (file, " Type:\tADDRESS\n"); + else if (group->type == USE_REF_ADDRESS) + fprintf (file, " Type:\tREFERENCE ADDRESS\n"); + else if (group->type == USE_PTR_ADDRESS) + fprintf (file, " Type:\tPOINTER ARGUMENT ADDRESS\n"); else { gcc_assert (group->type == USE_COMPARE); @@ -1502,19 +1519,21 @@ find_induction_variables (struct ivopts_data *data) /* Records a use of TYPE at *USE_P in STMT whose value is IV in GROUP. For address type use, ADDR_BASE is the stripped IV base, ADDR_OFFSET - is the const offset stripped from IV base; for other types use, both - are zero by default. */ + is the const offset stripped from IV base and MEM_TYPE is the type + of the memory being addressed. For uses of other types, ADDR_BASE + and ADDR_OFFSET are zero by default and MEM_TYPE is NULL_TREE. */ static struct iv_use * record_use (struct iv_group *group, tree *use_p, struct iv *iv, - gimple *stmt, enum use_type type, tree addr_base, - poly_uint64 addr_offset) + gimple *stmt, enum use_type type, tree mem_type, + tree addr_base, poly_uint64 addr_offset) { struct iv_use *use = XCNEW (struct iv_use); use->id = group->vuses.length (); use->group_id = group->id; use->type = type; + use->mem_type = mem_type; use->iv = iv; use->stmt = stmt; use->op_p = use_p; @@ -1569,18 +1588,21 @@ record_group (struct ivopts_data *data, enum use_type type) } /* Record a use of TYPE at *USE_P in STMT whose value is IV in a group. - New group will be created if there is no existing group for the use. */ + New group will be created if there is no existing group for the use. + MEM_TYPE is the type of memory being addressed, or NULL if this + isn't an address reference. */ static struct iv_use * record_group_use (struct ivopts_data *data, tree *use_p, - struct iv *iv, gimple *stmt, enum use_type type) + struct iv *iv, gimple *stmt, enum use_type type, + tree mem_type) { tree addr_base = NULL; struct iv_group *group = NULL; poly_uint64 addr_offset = 0; /* Record non address type use in a new group. */ - if (type == USE_ADDRESS && iv->base_object) + if (address_p (type)) { unsigned int i; @@ -1591,7 +1613,7 @@ record_group_use (struct ivopts_data *data, tree *use_p, group = data->vgroups[i]; use = group->vuses[0]; - if (use->type != USE_ADDRESS || !use->iv->base_object) + if (!address_p (use->type)) continue; /* Check if it has the same stripped base and step. */ @@ -1607,7 +1629,8 @@ record_group_use (struct ivopts_data *data, tree *use_p, if (!group) group = record_group (data, type); - return record_use (group, use_p, iv, stmt, type, addr_base, addr_offset); + return record_use (group, use_p, iv, stmt, type, mem_type, + addr_base, addr_offset); } /* Checks whether the use OP is interesting and if so, records it. */ @@ -1641,7 +1664,7 @@ find_interesting_uses_op (struct ivopts_data *data, tree op) stmt = SSA_NAME_DEF_STMT (op); gcc_assert (gimple_code (stmt) == GIMPLE_PHI || is_gimple_assign (stmt)); - use = record_group_use (data, NULL, iv, stmt, USE_NONLINEAR_EXPR); + use = record_group_use (data, NULL, iv, stmt, USE_NONLINEAR_EXPR, NULL_TREE); iv->nonlin_use = use; return use; } @@ -1757,10 +1780,10 @@ find_interesting_uses_cond (struct ivopts_data *data, gimple *stmt) return; } - record_group_use (data, var_p, var_iv, stmt, USE_COMPARE); + record_group_use (data, var_p, var_iv, stmt, USE_COMPARE, NULL_TREE); /* Record compare type iv_use for iv on the other side of comparison. */ if (ret == COMP_IV_EXPR_2) - record_group_use (data, bound_p, bound_iv, stmt, USE_COMPARE); + record_group_use (data, bound_p, bound_iv, stmt, USE_COMPARE, NULL_TREE); } /* Returns the outermost loop EXPR is obviously invariant in @@ -2375,7 +2398,7 @@ find_interesting_uses_address (struct ivopts_data *data, gimple *stmt, if (civ->base_object == NULL_TREE) goto fail; - record_group_use (data, op_p, civ, stmt, USE_ADDRESS); + record_group_use (data, op_p, civ, stmt, USE_REF_ADDRESS, TREE_TYPE (*op_p)); return; fail: @@ -2398,6 +2421,55 @@ find_invariants_stmt (struct ivopts_data *data, gimple *stmt) } } +/* CALL calls an internal function. If operand *OP_P will become an + address when the call is expanded, return the type of the memory + being addressed, otherwise return null. */ + +static tree +get_mem_type_for_internal_fn (gcall *call, tree *op_p) +{ + switch (gimple_call_internal_fn (call)) + { + case IFN_MASK_LOAD: + if (op_p == gimple_call_arg_ptr (call, 0)) + return TREE_TYPE (gimple_call_lhs (call)); + return NULL_TREE; + + case IFN_MASK_STORE: + if (op_p == gimple_call_arg_ptr (call, 0)) + return TREE_TYPE (gimple_call_arg (call, 3)); + return NULL_TREE; + + default: + return NULL_TREE; + } +} + +/* IV is a (non-address) iv that describes operand *OP_P of STMT. + Return true if the operand will become an address when STMT + is expanded and record the associated address use if so. */ + +static bool +find_address_like_use (struct ivopts_data *data, gimple *stmt, tree *op_p, + struct iv *iv) +{ + /* Fail if base object of this memory reference is unknown. */ + if (iv->base_object == NULL_TREE) + return false; + + tree mem_type = NULL_TREE; + if (gcall *call = dyn_cast (stmt)) + if (gimple_call_internal_p (call)) + mem_type = get_mem_type_for_internal_fn (call, op_p); + if (mem_type) + { + iv = alloc_iv (data, iv->base, iv->step); + record_group_use (data, op_p, iv, stmt, USE_PTR_ADDRESS, mem_type); + return true; + } + return false; +} + /* Finds interesting uses of induction variables in the statement STMT. */ static void @@ -2482,7 +2554,8 @@ find_interesting_uses_stmt (struct ivopts_data *data, gimple *stmt) if (!iv) continue; - find_interesting_uses_op (data, op); + if (!find_address_like_use (data, stmt, use_p->use, iv)) + find_interesting_uses_op (data, op); } } @@ -2516,7 +2589,7 @@ addr_offset_valid_p (struct iv_use *use, poly_int64 offset) rtx reg, addr; unsigned list_index; addr_space_t as = TYPE_ADDR_SPACE (TREE_TYPE (use->iv->base)); - machine_mode addr_mode, mem_mode = TYPE_MODE (TREE_TYPE (*use->op_p)); + machine_mode addr_mode, mem_mode = TYPE_MODE (use->mem_type); list_index = (unsigned) as * MAX_MACHINE_MODE + (unsigned) mem_mode; if (list_index >= vec_safe_length (addr_list)) @@ -2573,7 +2646,7 @@ split_small_address_groups_p (struct ivopts_data *data) if (group->vuses.length () == 1) continue; - gcc_assert (group->type == USE_ADDRESS); + gcc_assert (address_p (group->type)); if (group->vuses.length () == 2) { if (compare_sizes_for_sort (group->vuses[0]->addr_offset, @@ -2625,7 +2698,7 @@ split_address_groups (struct ivopts_data *data) if (group->vuses.length () == 1) continue; - gcc_assert (group->type == USE_ADDRESS); + gcc_assert (address_p (use->type)); for (j = 1; j < group->vuses.length ();) { @@ -3145,7 +3218,7 @@ add_autoinc_candidates (struct ivopts_data *data, tree base, tree step, cstepi = int_cst_value (step); - mem_mode = TYPE_MODE (TREE_TYPE (*use->op_p)); + mem_mode = TYPE_MODE (use->mem_type); if (((USE_LOAD_PRE_INCREMENT (mem_mode) || USE_STORE_PRE_INCREMENT (mem_mode)) && known_eq (GET_MODE_SIZE (mem_mode), cstepi)) @@ -3436,7 +3509,7 @@ add_iv_candidate_for_use (struct ivopts_data *data, struct iv_use *use) /* At last, add auto-incremental candidates. Make such variables important since other iv uses with same base object may be based on it. */ - if (use != NULL && use->type == USE_ADDRESS) + if (use != NULL && address_p (use->type)) add_autoinc_candidates (data, iv->base, iv->step, true, use); } @@ -3903,7 +3976,7 @@ get_use_type (struct iv_use *use) tree base_type = TREE_TYPE (use->iv->base); tree type; - if (use->type == USE_ADDRESS) + if (use->type == USE_REF_ADDRESS) { /* The base_type may be a void pointer. Create a pointer type based on the mem_ref instead. */ @@ -4331,7 +4404,7 @@ get_address_cost (struct ivopts_data *data, struct iv_use *use, struct mem_address parts = {NULL_TREE, integer_one_node, NULL_TREE, NULL_TREE, NULL_TREE}; machine_mode addr_mode = TYPE_MODE (type); - machine_mode mem_mode = TYPE_MODE (TREE_TYPE (*use->op_p)); + machine_mode mem_mode = TYPE_MODE (use->mem_type); addr_space_t as = TYPE_ADDR_SPACE (TREE_TYPE (use->iv->base)); /* Only true if ratio != 1. */ bool ok_with_ratio_p = false; @@ -5220,7 +5293,8 @@ determine_group_iv_cost (struct ivopts_data *data, case USE_NONLINEAR_EXPR: return determine_group_iv_cost_generic (data, group, cand); - case USE_ADDRESS: + case USE_REF_ADDRESS: + case USE_PTR_ADDRESS: return determine_group_iv_cost_address (data, group, cand); case USE_COMPARE: @@ -5238,7 +5312,7 @@ static bool autoinc_possible_for_pair (struct ivopts_data *data, struct iv_use *use, struct iv_cand *cand) { - if (use->type != USE_ADDRESS) + if (!address_p (use->type)) return false; bool can_autoinc = false; @@ -6997,6 +7071,27 @@ adjust_iv_update_pos (struct iv_cand *cand, struct iv_use *use) cand->incremented_at = use->stmt; } +/* Return the alias pointer type that should be used for a MEM_REF + associated with USE, which has type USE_PTR_ADDRESS. */ + +static tree +get_alias_ptr_type_for_ptr_address (iv_use *use) +{ + gcall *call = as_a (use->stmt); + switch (gimple_call_internal_fn (call)) + { + case IFN_MASK_LOAD: + case IFN_MASK_STORE: + /* The second argument contains the correct alias type. */ + gcc_assert (use->op_p = gimple_call_arg_ptr (call, 0)); + return TREE_TYPE (gimple_call_arg (call, 1)); + + default: + gcc_unreachable (); + } +} + + /* Rewrites USE (address that is an iv) using candidate CAND. */ static void @@ -7025,16 +7120,31 @@ rewrite_use_address (struct ivopts_data *data, tree iv = var_at_stmt (data->current_loop, cand, use->stmt); tree base_hint = (cand->iv->base_object) ? iv : NULL_TREE; gimple_stmt_iterator bsi = gsi_for_stmt (use->stmt); - tree type = TREE_TYPE (*use->op_p); - unsigned int align = get_object_alignment (*use->op_p); - if (align != TYPE_ALIGN (type)) - type = build_aligned_type (type, align); - - tree ref = create_mem_ref (&bsi, type, &aff, - reference_alias_ptr_type (*use->op_p), + tree type = use->mem_type; + tree alias_ptr_type; + if (use->type == USE_PTR_ADDRESS) + alias_ptr_type = get_alias_ptr_type_for_ptr_address (use); + else + { + gcc_assert (type == TREE_TYPE (*use->op_p)); + unsigned int align = get_object_alignment (*use->op_p); + if (align != TYPE_ALIGN (type)) + type = build_aligned_type (type, align); + alias_ptr_type = reference_alias_ptr_type (*use->op_p); + } + tree ref = create_mem_ref (&bsi, type, &aff, alias_ptr_type, iv, base_hint, data->speed); - copy_ref_info (ref, *use->op_p); + if (use->type == USE_PTR_ADDRESS) + { + ref = fold_build1 (ADDR_EXPR, build_pointer_type (use->mem_type), ref); + ref = fold_convert (get_use_type (use), ref); + ref = force_gimple_operand_gsi (&bsi, ref, true, NULL_TREE, + true, GSI_SAME_STMT); + } + else + copy_ref_info (ref, *use->op_p); + *use->op_p = ref; } @@ -7110,7 +7220,7 @@ rewrite_groups (struct ivopts_data *data) update_stmt (group->vuses[j]->stmt); } } - else if (group->type == USE_ADDRESS) + else if (address_p (group->type)) { for (j = 0; j < group->vuses.length (); j++) {