1 /* Builtins' description for AArch64 SIMD architecture.
2 Copyright (C) 2011-2019 Free Software Foundation, Inc.
3 Contributed by ARM Ltd.
5 This file is part of GCC.
7 GCC is free software; you can redistribute it and/or modify it
8 under the terms of the GNU General Public License as published by
9 the Free Software Foundation; either version 3, or (at your option)
12 GCC is distributed in the hope that it will be useful, but
13 WITHOUT ANY WARRANTY; without even the implied warranty of
14 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15 General Public License for more details.
17 You should have received a copy of the GNU General Public License
18 along with GCC; see the file COPYING3. If not see
19 <http://www.gnu.org/licenses/>. */
21 #define IN_TARGET_CODE 1
25 #include "coretypes.h"
28 #include "basic-block.h"
37 #include "diagnostic-core.h"
38 #include "fold-const.h"
39 #include "stor-layout.h"
42 #include "langhooks.h"
43 #include "gimple-iterator.h"
44 #include "case-cfn-macros.h"
47 #define v8qi_UP E_V8QImode
48 #define v4hi_UP E_V4HImode
49 #define v4hf_UP E_V4HFmode
50 #define v2si_UP E_V2SImode
51 #define v2sf_UP E_V2SFmode
52 #define v1df_UP E_V1DFmode
53 #define di_UP E_DImode
54 #define df_UP E_DFmode
55 #define v16qi_UP E_V16QImode
56 #define v8hi_UP E_V8HImode
57 #define v8hf_UP E_V8HFmode
58 #define v4si_UP E_V4SImode
59 #define v4sf_UP E_V4SFmode
60 #define v2di_UP E_V2DImode
61 #define v2df_UP E_V2DFmode
62 #define ti_UP E_TImode
63 #define oi_UP E_OImode
64 #define ci_UP E_CImode
65 #define xi_UP E_XImode
66 #define si_UP E_SImode
67 #define sf_UP E_SFmode
68 #define hi_UP E_HImode
69 #define hf_UP E_HFmode
70 #define qi_UP E_QImode
73 #define SIMD_MAX_BUILTIN_ARGS 5
75 enum aarch64_type_qualifiers
80 qualifier_unsigned
= 0x1, /* 1 << 0 */
82 qualifier_const
= 0x2, /* 1 << 1 */
84 qualifier_pointer
= 0x4, /* 1 << 2 */
85 /* Used when expanding arguments if an operand could
87 qualifier_immediate
= 0x8, /* 1 << 3 */
88 qualifier_maybe_immediate
= 0x10, /* 1 << 4 */
90 qualifier_void
= 0x20, /* 1 << 5 */
91 /* Some patterns may have internal operands, this qualifier is an
92 instruction to the initialisation code to skip this operand. */
93 qualifier_internal
= 0x40, /* 1 << 6 */
94 /* Some builtins should use the T_*mode* encoded in a simd_builtin_datum
95 rather than using the type of the operand. */
96 qualifier_map_mode
= 0x80, /* 1 << 7 */
97 /* qualifier_pointer | qualifier_map_mode */
98 qualifier_pointer_map_mode
= 0x84,
99 /* qualifier_const | qualifier_pointer | qualifier_map_mode */
100 qualifier_const_pointer_map_mode
= 0x86,
101 /* Polynomial types. */
102 qualifier_poly
= 0x100,
103 /* Lane indices - must be in range, and flipped for bigendian. */
104 qualifier_lane_index
= 0x200,
105 /* Lane indices for single lane structure loads and stores. */
106 qualifier_struct_load_store_lane_index
= 0x400,
107 /* Lane indices selected in pairs. - must be in range, and flipped for
109 qualifier_lane_pair_index
= 0x800,
116 const enum insn_code code
;
118 enum aarch64_type_qualifiers
*qualifiers
;
119 } aarch64_simd_builtin_datum
;
121 static enum aarch64_type_qualifiers
122 aarch64_types_unop_qualifiers
[SIMD_MAX_BUILTIN_ARGS
]
123 = { qualifier_none
, qualifier_none
};
124 #define TYPES_UNOP (aarch64_types_unop_qualifiers)
125 static enum aarch64_type_qualifiers
126 aarch64_types_unopu_qualifiers
[SIMD_MAX_BUILTIN_ARGS
]
127 = { qualifier_unsigned
, qualifier_unsigned
};
128 #define TYPES_UNOPU (aarch64_types_unopu_qualifiers)
129 static enum aarch64_type_qualifiers
130 aarch64_types_unopus_qualifiers
[SIMD_MAX_BUILTIN_ARGS
]
131 = { qualifier_unsigned
, qualifier_none
};
132 #define TYPES_UNOPUS (aarch64_types_unopus_qualifiers)
133 static enum aarch64_type_qualifiers
134 aarch64_types_binop_qualifiers
[SIMD_MAX_BUILTIN_ARGS
]
135 = { qualifier_none
, qualifier_none
, qualifier_maybe_immediate
};
136 #define TYPES_BINOP (aarch64_types_binop_qualifiers)
137 static enum aarch64_type_qualifiers
138 aarch64_types_binopu_qualifiers
[SIMD_MAX_BUILTIN_ARGS
]
139 = { qualifier_unsigned
, qualifier_unsigned
, qualifier_unsigned
};
140 #define TYPES_BINOPU (aarch64_types_binopu_qualifiers)
141 static enum aarch64_type_qualifiers
142 aarch64_types_binop_uus_qualifiers
[SIMD_MAX_BUILTIN_ARGS
]
143 = { qualifier_unsigned
, qualifier_unsigned
, qualifier_none
};
144 #define TYPES_BINOP_UUS (aarch64_types_binop_uus_qualifiers)
145 static enum aarch64_type_qualifiers
146 aarch64_types_binop_ssu_qualifiers
[SIMD_MAX_BUILTIN_ARGS
]
147 = { qualifier_none
, qualifier_none
, qualifier_unsigned
};
148 #define TYPES_BINOP_SSU (aarch64_types_binop_ssu_qualifiers)
149 static enum aarch64_type_qualifiers
150 aarch64_types_binop_uss_qualifiers
[SIMD_MAX_BUILTIN_ARGS
]
151 = { qualifier_unsigned
, qualifier_none
, qualifier_none
};
152 #define TYPES_BINOP_USS (aarch64_types_binop_uss_qualifiers)
153 static enum aarch64_type_qualifiers
154 aarch64_types_binopp_qualifiers
[SIMD_MAX_BUILTIN_ARGS
]
155 = { qualifier_poly
, qualifier_poly
, qualifier_poly
};
156 #define TYPES_BINOPP (aarch64_types_binopp_qualifiers)
158 static enum aarch64_type_qualifiers
159 aarch64_types_ternop_qualifiers
[SIMD_MAX_BUILTIN_ARGS
]
160 = { qualifier_none
, qualifier_none
, qualifier_none
, qualifier_none
};
161 #define TYPES_TERNOP (aarch64_types_ternop_qualifiers)
162 static enum aarch64_type_qualifiers
163 aarch64_types_ternop_lane_qualifiers
[SIMD_MAX_BUILTIN_ARGS
]
164 = { qualifier_none
, qualifier_none
, qualifier_none
, qualifier_lane_index
};
165 #define TYPES_TERNOP_LANE (aarch64_types_ternop_lane_qualifiers)
166 static enum aarch64_type_qualifiers
167 aarch64_types_ternopu_qualifiers
[SIMD_MAX_BUILTIN_ARGS
]
168 = { qualifier_unsigned
, qualifier_unsigned
,
169 qualifier_unsigned
, qualifier_unsigned
};
170 #define TYPES_TERNOPU (aarch64_types_ternopu_qualifiers)
171 static enum aarch64_type_qualifiers
172 aarch64_types_ternopu_imm_qualifiers
[SIMD_MAX_BUILTIN_ARGS
]
173 = { qualifier_unsigned
, qualifier_unsigned
,
174 qualifier_unsigned
, qualifier_immediate
};
175 #define TYPES_TERNOPUI (aarch64_types_ternopu_imm_qualifiers)
178 static enum aarch64_type_qualifiers
179 aarch64_types_quadop_lane_pair_qualifiers
[SIMD_MAX_BUILTIN_ARGS
]
180 = { qualifier_none
, qualifier_none
, qualifier_none
,
181 qualifier_none
, qualifier_lane_pair_index
};
182 #define TYPES_QUADOP_LANE_PAIR (aarch64_types_quadop_lane_pair_qualifiers)
183 static enum aarch64_type_qualifiers
184 aarch64_types_quadop_lane_qualifiers
[SIMD_MAX_BUILTIN_ARGS
]
185 = { qualifier_none
, qualifier_none
, qualifier_none
,
186 qualifier_none
, qualifier_lane_index
};
187 #define TYPES_QUADOP_LANE (aarch64_types_quadop_lane_qualifiers)
188 static enum aarch64_type_qualifiers
189 aarch64_types_quadopu_lane_qualifiers
[SIMD_MAX_BUILTIN_ARGS
]
190 = { qualifier_unsigned
, qualifier_unsigned
, qualifier_unsigned
,
191 qualifier_unsigned
, qualifier_lane_index
};
192 #define TYPES_QUADOPU_LANE (aarch64_types_quadopu_lane_qualifiers)
194 static enum aarch64_type_qualifiers
195 aarch64_types_quadopu_imm_qualifiers
[SIMD_MAX_BUILTIN_ARGS
]
196 = { qualifier_unsigned
, qualifier_unsigned
, qualifier_unsigned
,
197 qualifier_unsigned
, qualifier_immediate
};
198 #define TYPES_QUADOPUI (aarch64_types_quadopu_imm_qualifiers)
200 static enum aarch64_type_qualifiers
201 aarch64_types_binop_imm_p_qualifiers
[SIMD_MAX_BUILTIN_ARGS
]
202 = { qualifier_poly
, qualifier_none
, qualifier_immediate
};
203 #define TYPES_GETREGP (aarch64_types_binop_imm_p_qualifiers)
204 static enum aarch64_type_qualifiers
205 aarch64_types_binop_imm_qualifiers
[SIMD_MAX_BUILTIN_ARGS
]
206 = { qualifier_none
, qualifier_none
, qualifier_immediate
};
207 #define TYPES_GETREG (aarch64_types_binop_imm_qualifiers)
208 #define TYPES_SHIFTIMM (aarch64_types_binop_imm_qualifiers)
209 static enum aarch64_type_qualifiers
210 aarch64_types_shift_to_unsigned_qualifiers
[SIMD_MAX_BUILTIN_ARGS
]
211 = { qualifier_unsigned
, qualifier_none
, qualifier_immediate
};
212 #define TYPES_SHIFTIMM_USS (aarch64_types_shift_to_unsigned_qualifiers)
213 static enum aarch64_type_qualifiers
214 aarch64_types_fcvt_from_unsigned_qualifiers
[SIMD_MAX_BUILTIN_ARGS
]
215 = { qualifier_none
, qualifier_unsigned
, qualifier_immediate
};
216 #define TYPES_FCVTIMM_SUS (aarch64_types_fcvt_from_unsigned_qualifiers)
217 static enum aarch64_type_qualifiers
218 aarch64_types_unsigned_shift_qualifiers
[SIMD_MAX_BUILTIN_ARGS
]
219 = { qualifier_unsigned
, qualifier_unsigned
, qualifier_immediate
};
220 #define TYPES_USHIFTIMM (aarch64_types_unsigned_shift_qualifiers)
222 static enum aarch64_type_qualifiers
223 aarch64_types_ternop_s_imm_p_qualifiers
[SIMD_MAX_BUILTIN_ARGS
]
224 = { qualifier_none
, qualifier_none
, qualifier_poly
, qualifier_immediate
};
225 #define TYPES_SETREGP (aarch64_types_ternop_s_imm_p_qualifiers)
226 static enum aarch64_type_qualifiers
227 aarch64_types_ternop_s_imm_qualifiers
[SIMD_MAX_BUILTIN_ARGS
]
228 = { qualifier_none
, qualifier_none
, qualifier_none
, qualifier_immediate
};
229 #define TYPES_SETREG (aarch64_types_ternop_s_imm_qualifiers)
230 #define TYPES_SHIFTINSERT (aarch64_types_ternop_s_imm_qualifiers)
231 #define TYPES_SHIFTACC (aarch64_types_ternop_s_imm_qualifiers)
233 static enum aarch64_type_qualifiers
234 aarch64_types_ternop_p_imm_qualifiers
[SIMD_MAX_BUILTIN_ARGS
]
235 = { qualifier_poly
, qualifier_poly
, qualifier_poly
, qualifier_immediate
};
236 #define TYPES_SHIFTINSERTP (aarch64_types_ternop_p_imm_qualifiers)
238 static enum aarch64_type_qualifiers
239 aarch64_types_unsigned_shiftacc_qualifiers
[SIMD_MAX_BUILTIN_ARGS
]
240 = { qualifier_unsigned
, qualifier_unsigned
, qualifier_unsigned
,
241 qualifier_immediate
};
242 #define TYPES_USHIFTACC (aarch64_types_unsigned_shiftacc_qualifiers)
245 static enum aarch64_type_qualifiers
246 aarch64_types_combine_qualifiers
[SIMD_MAX_BUILTIN_ARGS
]
247 = { qualifier_none
, qualifier_none
, qualifier_none
};
248 #define TYPES_COMBINE (aarch64_types_combine_qualifiers)
250 static enum aarch64_type_qualifiers
251 aarch64_types_combine_p_qualifiers
[SIMD_MAX_BUILTIN_ARGS
]
252 = { qualifier_poly
, qualifier_poly
, qualifier_poly
};
253 #define TYPES_COMBINEP (aarch64_types_combine_p_qualifiers)
255 static enum aarch64_type_qualifiers
256 aarch64_types_load1_qualifiers
[SIMD_MAX_BUILTIN_ARGS
]
257 = { qualifier_none
, qualifier_const_pointer_map_mode
};
258 #define TYPES_LOAD1 (aarch64_types_load1_qualifiers)
259 #define TYPES_LOADSTRUCT (aarch64_types_load1_qualifiers)
260 static enum aarch64_type_qualifiers
261 aarch64_types_loadstruct_lane_qualifiers
[SIMD_MAX_BUILTIN_ARGS
]
262 = { qualifier_none
, qualifier_const_pointer_map_mode
,
263 qualifier_none
, qualifier_struct_load_store_lane_index
};
264 #define TYPES_LOADSTRUCT_LANE (aarch64_types_loadstruct_lane_qualifiers)
266 static enum aarch64_type_qualifiers
267 aarch64_types_bsl_p_qualifiers
[SIMD_MAX_BUILTIN_ARGS
]
268 = { qualifier_poly
, qualifier_unsigned
,
269 qualifier_poly
, qualifier_poly
};
270 #define TYPES_BSL_P (aarch64_types_bsl_p_qualifiers)
271 static enum aarch64_type_qualifiers
272 aarch64_types_bsl_s_qualifiers
[SIMD_MAX_BUILTIN_ARGS
]
273 = { qualifier_none
, qualifier_unsigned
,
274 qualifier_none
, qualifier_none
};
275 #define TYPES_BSL_S (aarch64_types_bsl_s_qualifiers)
276 static enum aarch64_type_qualifiers
277 aarch64_types_bsl_u_qualifiers
[SIMD_MAX_BUILTIN_ARGS
]
278 = { qualifier_unsigned
, qualifier_unsigned
,
279 qualifier_unsigned
, qualifier_unsigned
};
280 #define TYPES_BSL_U (aarch64_types_bsl_u_qualifiers)
282 /* The first argument (return type) of a store should be void type,
283 which we represent with qualifier_void. Their first operand will be
284 a DImode pointer to the location to store to, so we must use
285 qualifier_map_mode | qualifier_pointer to build a pointer to the
286 element type of the vector. */
287 static enum aarch64_type_qualifiers
288 aarch64_types_store1_p_qualifiers
[SIMD_MAX_BUILTIN_ARGS
]
289 = { qualifier_void
, qualifier_pointer_map_mode
, qualifier_poly
};
290 #define TYPES_STORE1P (aarch64_types_store1_p_qualifiers)
291 static enum aarch64_type_qualifiers
292 aarch64_types_store1_qualifiers
[SIMD_MAX_BUILTIN_ARGS
]
293 = { qualifier_void
, qualifier_pointer_map_mode
, qualifier_none
};
294 #define TYPES_STORE1 (aarch64_types_store1_qualifiers)
295 #define TYPES_STORESTRUCT (aarch64_types_store1_qualifiers)
296 static enum aarch64_type_qualifiers
297 aarch64_types_storestruct_lane_qualifiers
[SIMD_MAX_BUILTIN_ARGS
]
298 = { qualifier_void
, qualifier_pointer_map_mode
,
299 qualifier_none
, qualifier_struct_load_store_lane_index
};
300 #define TYPES_STORESTRUCT_LANE (aarch64_types_storestruct_lane_qualifiers)
302 #define CF0(N, X) CODE_FOR_aarch64_##N##X
303 #define CF1(N, X) CODE_FOR_##N##X##1
304 #define CF2(N, X) CODE_FOR_##N##X##2
305 #define CF3(N, X) CODE_FOR_##N##X##3
306 #define CF4(N, X) CODE_FOR_##N##X##4
307 #define CF10(N, X) CODE_FOR_##N##X
309 #define VAR1(T, N, MAP, A) \
310 {#N #A, UP (A), CF##MAP (N, A), 0, TYPES_##T},
311 #define VAR2(T, N, MAP, A, B) \
312 VAR1 (T, N, MAP, A) \
314 #define VAR3(T, N, MAP, A, B, C) \
315 VAR2 (T, N, MAP, A, B) \
317 #define VAR4(T, N, MAP, A, B, C, D) \
318 VAR3 (T, N, MAP, A, B, C) \
320 #define VAR5(T, N, MAP, A, B, C, D, E) \
321 VAR4 (T, N, MAP, A, B, C, D) \
323 #define VAR6(T, N, MAP, A, B, C, D, E, F) \
324 VAR5 (T, N, MAP, A, B, C, D, E) \
326 #define VAR7(T, N, MAP, A, B, C, D, E, F, G) \
327 VAR6 (T, N, MAP, A, B, C, D, E, F) \
329 #define VAR8(T, N, MAP, A, B, C, D, E, F, G, H) \
330 VAR7 (T, N, MAP, A, B, C, D, E, F, G) \
332 #define VAR9(T, N, MAP, A, B, C, D, E, F, G, H, I) \
333 VAR8 (T, N, MAP, A, B, C, D, E, F, G, H) \
335 #define VAR10(T, N, MAP, A, B, C, D, E, F, G, H, I, J) \
336 VAR9 (T, N, MAP, A, B, C, D, E, F, G, H, I) \
338 #define VAR11(T, N, MAP, A, B, C, D, E, F, G, H, I, J, K) \
339 VAR10 (T, N, MAP, A, B, C, D, E, F, G, H, I, J) \
341 #define VAR12(T, N, MAP, A, B, C, D, E, F, G, H, I, J, K, L) \
342 VAR11 (T, N, MAP, A, B, C, D, E, F, G, H, I, J, K) \
344 #define VAR13(T, N, MAP, A, B, C, D, E, F, G, H, I, J, K, L, M) \
345 VAR12 (T, N, MAP, A, B, C, D, E, F, G, H, I, J, K, L) \
347 #define VAR14(T, X, MAP, A, B, C, D, E, F, G, H, I, J, K, L, M, N) \
348 VAR13 (T, X, MAP, A, B, C, D, E, F, G, H, I, J, K, L, M) \
351 #include "aarch64-builtin-iterators.h"
353 static aarch64_simd_builtin_datum aarch64_simd_builtin_data
[] = {
354 #include "aarch64-simd-builtins.def"
357 /* There's only 8 CRC32 builtins. Probably not worth their own .def file. */
358 #define AARCH64_CRC32_BUILTINS \
359 CRC32_BUILTIN (crc32b, QI) \
360 CRC32_BUILTIN (crc32h, HI) \
361 CRC32_BUILTIN (crc32w, SI) \
362 CRC32_BUILTIN (crc32x, DI) \
363 CRC32_BUILTIN (crc32cb, QI) \
364 CRC32_BUILTIN (crc32ch, HI) \
365 CRC32_BUILTIN (crc32cw, SI) \
366 CRC32_BUILTIN (crc32cx, DI)
368 /* The next 8 FCMLA instrinsics require some special handling compared the
369 normal simd intrinsics. */
370 #define AARCH64_SIMD_FCMLA_LANEQ_BUILTINS \
371 FCMLA_LANEQ_BUILTIN (0, v2sf, fcmla, V2SF, false) \
372 FCMLA_LANEQ_BUILTIN (90, v2sf, fcmla, V2SF, false) \
373 FCMLA_LANEQ_BUILTIN (180, v2sf, fcmla, V2SF, false) \
374 FCMLA_LANEQ_BUILTIN (270, v2sf, fcmla, V2SF, false) \
375 FCMLA_LANEQ_BUILTIN (0, v4hf, fcmla_laneq, V4HF, true) \
376 FCMLA_LANEQ_BUILTIN (90, v4hf, fcmla_laneq, V4HF, true) \
377 FCMLA_LANEQ_BUILTIN (180, v4hf, fcmla_laneq, V4HF, true) \
378 FCMLA_LANEQ_BUILTIN (270, v4hf, fcmla_laneq, V4HF, true) \
384 const enum insn_code icode
;
386 } aarch64_crc_builtin_datum
;
388 /* Hold information about how to expand the FCMLA_LANEQ builtins. */
393 const enum insn_code icode
;
396 } aarch64_fcmla_laneq_builtin_datum
;
398 #define CRC32_BUILTIN(N, M) \
401 #define FCMLA_LANEQ_BUILTIN(I, N, X, M, T) \
402 AARCH64_SIMD_BUILTIN_FCMLA_LANEQ##I##_##M,
405 #define VAR1(T, N, MAP, A) \
406 AARCH64_SIMD_BUILTIN_##T##_##N##A,
408 enum aarch64_builtins
412 AARCH64_BUILTIN_GET_FPCR
,
413 AARCH64_BUILTIN_SET_FPCR
,
414 AARCH64_BUILTIN_GET_FPSR
,
415 AARCH64_BUILTIN_SET_FPSR
,
417 AARCH64_BUILTIN_RSQRT_DF
,
418 AARCH64_BUILTIN_RSQRT_SF
,
419 AARCH64_BUILTIN_RSQRT_V2DF
,
420 AARCH64_BUILTIN_RSQRT_V2SF
,
421 AARCH64_BUILTIN_RSQRT_V4SF
,
422 AARCH64_SIMD_BUILTIN_BASE
,
423 AARCH64_SIMD_BUILTIN_LANE_CHECK
,
424 #include "aarch64-simd-builtins.def"
425 /* The first enum element which is based on an insn_data pattern. */
426 AARCH64_SIMD_PATTERN_START
= AARCH64_SIMD_BUILTIN_LANE_CHECK
+ 1,
427 AARCH64_SIMD_BUILTIN_MAX
= AARCH64_SIMD_PATTERN_START
428 + ARRAY_SIZE (aarch64_simd_builtin_data
) - 1,
429 AARCH64_CRC32_BUILTIN_BASE
,
430 AARCH64_CRC32_BUILTINS
431 AARCH64_CRC32_BUILTIN_MAX
,
432 /* ARMv8.3-A Pointer Authentication Builtins. */
433 AARCH64_PAUTH_BUILTIN_AUTIA1716
,
434 AARCH64_PAUTH_BUILTIN_PACIA1716
,
435 AARCH64_PAUTH_BUILTIN_AUTIB1716
,
436 AARCH64_PAUTH_BUILTIN_PACIB1716
,
437 AARCH64_PAUTH_BUILTIN_XPACLRI
,
438 /* Special cased Armv8.3-A Complex FMA by Lane quad Builtins. */
439 AARCH64_SIMD_FCMLA_LANEQ_BUILTIN_BASE
,
440 AARCH64_SIMD_FCMLA_LANEQ_BUILTINS
441 /* Builtin for Arm8.3-a Javascript conversion instruction. */
444 AARCH64_TME_BUILTIN_TSTART
,
445 AARCH64_TME_BUILTIN_TCOMMIT
,
446 AARCH64_TME_BUILTIN_TTEST
,
447 AARCH64_TME_BUILTIN_TCANCEL
,
452 #define CRC32_BUILTIN(N, M) \
453 {"__builtin_aarch64_"#N, E_##M##mode, CODE_FOR_aarch64_##N, AARCH64_BUILTIN_##N},
455 static aarch64_crc_builtin_datum aarch64_crc_builtin_data
[] = {
456 AARCH64_CRC32_BUILTINS
460 #undef FCMLA_LANEQ_BUILTIN
461 #define FCMLA_LANEQ_BUILTIN(I, N, X, M, T) \
462 {"__builtin_aarch64_fcmla_laneq"#I#N, E_##M##mode, CODE_FOR_aarch64_##X##I##N, \
463 AARCH64_SIMD_BUILTIN_FCMLA_LANEQ##I##_##M, T},
465 /* This structure contains how to manage the mapping form the builtin to the
466 instruction to generate in the backend and how to invoke the instruction. */
467 static aarch64_fcmla_laneq_builtin_datum aarch64_fcmla_lane_builtin_data
[] = {
468 AARCH64_SIMD_FCMLA_LANEQ_BUILTINS
473 static GTY(()) tree aarch64_builtin_decls
[AARCH64_BUILTIN_MAX
];
475 #define NUM_DREG_TYPES 6
476 #define NUM_QREG_TYPES 6
478 /* Internal scalar builtin types. These types are used to support
479 neon intrinsic builtins. They are _not_ user-visible types. Therefore
480 the mangling for these types are implementation defined. */
481 const char *aarch64_scalar_builtin_types
[] = {
482 "__builtin_aarch64_simd_qi",
483 "__builtin_aarch64_simd_hi",
484 "__builtin_aarch64_simd_si",
485 "__builtin_aarch64_simd_hf",
486 "__builtin_aarch64_simd_sf",
487 "__builtin_aarch64_simd_di",
488 "__builtin_aarch64_simd_df",
489 "__builtin_aarch64_simd_poly8",
490 "__builtin_aarch64_simd_poly16",
491 "__builtin_aarch64_simd_poly64",
492 "__builtin_aarch64_simd_poly128",
493 "__builtin_aarch64_simd_ti",
494 "__builtin_aarch64_simd_uqi",
495 "__builtin_aarch64_simd_uhi",
496 "__builtin_aarch64_simd_usi",
497 "__builtin_aarch64_simd_udi",
498 "__builtin_aarch64_simd_ei",
499 "__builtin_aarch64_simd_oi",
500 "__builtin_aarch64_simd_ci",
501 "__builtin_aarch64_simd_xi",
505 #define ENTRY(E, M, Q, G) E,
506 enum aarch64_simd_type
508 #include "aarch64-simd-builtin-types.def"
509 ARM_NEON_H_TYPES_LAST
513 struct aarch64_simd_type_info
515 enum aarch64_simd_type type
;
517 /* Internal type name. */
520 /* Internal type name(mangled). The mangled names conform to the
521 AAPCS64 (see "Procedure Call Standard for the ARM 64-bit Architecture",
522 Appendix A). To qualify for emission with the mangled names defined in
523 that document, a vector type must not only be of the correct mode but also
524 be of the correct internal AdvSIMD vector type (e.g. __Int8x8_t); these
525 types are registered by aarch64_init_simd_builtin_types (). In other
526 words, vector types defined in other ways e.g. via vector_size attribute
527 will get default mangled names. */
536 /* Machine mode the internal type maps to. */
537 enum machine_mode mode
;
540 enum aarch64_type_qualifiers q
;
543 #define ENTRY(E, M, Q, G) \
544 {E, "__" #E, #G "__" #E, NULL_TREE, NULL_TREE, E_##M##mode, qualifier_##Q},
545 static struct aarch64_simd_type_info aarch64_simd_types
[] = {
546 #include "aarch64-simd-builtin-types.def"
550 static tree aarch64_simd_intOI_type_node
= NULL_TREE
;
551 static tree aarch64_simd_intCI_type_node
= NULL_TREE
;
552 static tree aarch64_simd_intXI_type_node
= NULL_TREE
;
554 /* The user-visible __fp16 type, and a pointer to that type. Used
555 across the back-end. */
556 tree aarch64_fp16_type_node
= NULL_TREE
;
557 tree aarch64_fp16_ptr_type_node
= NULL_TREE
;
560 aarch64_mangle_builtin_scalar_type (const_tree type
)
564 while (aarch64_scalar_builtin_types
[i
] != NULL
)
566 const char *name
= aarch64_scalar_builtin_types
[i
];
568 if (TREE_CODE (TYPE_NAME (type
)) == TYPE_DECL
569 && DECL_NAME (TYPE_NAME (type
))
570 && !strcmp (IDENTIFIER_POINTER (DECL_NAME (TYPE_NAME (type
))), name
))
571 return aarch64_scalar_builtin_types
[i
];
578 aarch64_mangle_builtin_vector_type (const_tree type
)
581 int nelts
= sizeof (aarch64_simd_types
) / sizeof (aarch64_simd_types
[0]);
583 for (i
= 0; i
< nelts
; i
++)
584 if (aarch64_simd_types
[i
].mode
== TYPE_MODE (type
)
586 && TREE_CODE (TYPE_NAME (type
)) == TYPE_DECL
587 && DECL_NAME (TYPE_NAME (type
))
589 (IDENTIFIER_POINTER (DECL_NAME (TYPE_NAME (type
))),
590 aarch64_simd_types
[i
].name
))
591 return aarch64_simd_types
[i
].mangle
;
597 aarch64_mangle_builtin_type (const_tree type
)
600 /* Walk through all the AArch64 builtins types tables to filter out the
602 if ((mangle
= aarch64_mangle_builtin_vector_type (type
))
603 || (mangle
= aarch64_mangle_builtin_scalar_type (type
)))
610 aarch64_simd_builtin_std_type (machine_mode mode
,
611 enum aarch64_type_qualifiers q
)
613 #define QUAL_TYPE(M) \
614 ((q == qualifier_none) ? int##M##_type_node : unsigned_int##M##_type_node);
618 return QUAL_TYPE (QI
);
620 return QUAL_TYPE (HI
);
622 return QUAL_TYPE (SI
);
624 return QUAL_TYPE (DI
);
626 return QUAL_TYPE (TI
);
628 return aarch64_simd_intOI_type_node
;
630 return aarch64_simd_intCI_type_node
;
632 return aarch64_simd_intXI_type_node
;
634 return aarch64_fp16_type_node
;
636 return float_type_node
;
638 return double_type_node
;
646 aarch64_lookup_simd_builtin_type (machine_mode mode
,
647 enum aarch64_type_qualifiers q
)
650 int nelts
= sizeof (aarch64_simd_types
) / sizeof (aarch64_simd_types
[0]);
652 /* Non-poly scalar modes map to standard types not in the table. */
653 if (q
!= qualifier_poly
&& !VECTOR_MODE_P (mode
))
654 return aarch64_simd_builtin_std_type (mode
, q
);
656 for (i
= 0; i
< nelts
; i
++)
657 if (aarch64_simd_types
[i
].mode
== mode
658 && aarch64_simd_types
[i
].q
== q
)
659 return aarch64_simd_types
[i
].itype
;
665 aarch64_simd_builtin_type (machine_mode mode
,
666 bool unsigned_p
, bool poly_p
)
669 return aarch64_lookup_simd_builtin_type (mode
, qualifier_poly
);
671 return aarch64_lookup_simd_builtin_type (mode
, qualifier_unsigned
);
673 return aarch64_lookup_simd_builtin_type (mode
, qualifier_none
);
677 aarch64_init_simd_builtin_types (void)
680 int nelts
= sizeof (aarch64_simd_types
) / sizeof (aarch64_simd_types
[0]);
683 /* Init all the element types built by the front-end. */
684 aarch64_simd_types
[Int8x8_t
].eltype
= intQI_type_node
;
685 aarch64_simd_types
[Int8x16_t
].eltype
= intQI_type_node
;
686 aarch64_simd_types
[Int16x4_t
].eltype
= intHI_type_node
;
687 aarch64_simd_types
[Int16x8_t
].eltype
= intHI_type_node
;
688 aarch64_simd_types
[Int32x2_t
].eltype
= intSI_type_node
;
689 aarch64_simd_types
[Int32x4_t
].eltype
= intSI_type_node
;
690 aarch64_simd_types
[Int64x1_t
].eltype
= intDI_type_node
;
691 aarch64_simd_types
[Int64x2_t
].eltype
= intDI_type_node
;
692 aarch64_simd_types
[Uint8x8_t
].eltype
= unsigned_intQI_type_node
;
693 aarch64_simd_types
[Uint8x16_t
].eltype
= unsigned_intQI_type_node
;
694 aarch64_simd_types
[Uint16x4_t
].eltype
= unsigned_intHI_type_node
;
695 aarch64_simd_types
[Uint16x8_t
].eltype
= unsigned_intHI_type_node
;
696 aarch64_simd_types
[Uint32x2_t
].eltype
= unsigned_intSI_type_node
;
697 aarch64_simd_types
[Uint32x4_t
].eltype
= unsigned_intSI_type_node
;
698 aarch64_simd_types
[Uint64x1_t
].eltype
= unsigned_intDI_type_node
;
699 aarch64_simd_types
[Uint64x2_t
].eltype
= unsigned_intDI_type_node
;
701 /* Poly types are a world of their own. */
702 aarch64_simd_types
[Poly8_t
].eltype
= aarch64_simd_types
[Poly8_t
].itype
=
703 build_distinct_type_copy (unsigned_intQI_type_node
);
704 /* Prevent front-ends from transforming Poly8_t arrays into string
706 TYPE_STRING_FLAG (aarch64_simd_types
[Poly8_t
].eltype
) = false;
708 aarch64_simd_types
[Poly16_t
].eltype
= aarch64_simd_types
[Poly16_t
].itype
=
709 build_distinct_type_copy (unsigned_intHI_type_node
);
710 aarch64_simd_types
[Poly64_t
].eltype
= aarch64_simd_types
[Poly64_t
].itype
=
711 build_distinct_type_copy (unsigned_intDI_type_node
);
712 aarch64_simd_types
[Poly128_t
].eltype
= aarch64_simd_types
[Poly128_t
].itype
=
713 build_distinct_type_copy (unsigned_intTI_type_node
);
714 /* Init poly vector element types with scalar poly types. */
715 aarch64_simd_types
[Poly8x8_t
].eltype
= aarch64_simd_types
[Poly8_t
].itype
;
716 aarch64_simd_types
[Poly8x16_t
].eltype
= aarch64_simd_types
[Poly8_t
].itype
;
717 aarch64_simd_types
[Poly16x4_t
].eltype
= aarch64_simd_types
[Poly16_t
].itype
;
718 aarch64_simd_types
[Poly16x8_t
].eltype
= aarch64_simd_types
[Poly16_t
].itype
;
719 aarch64_simd_types
[Poly64x1_t
].eltype
= aarch64_simd_types
[Poly64_t
].itype
;
720 aarch64_simd_types
[Poly64x2_t
].eltype
= aarch64_simd_types
[Poly64_t
].itype
;
722 /* Continue with standard types. */
723 aarch64_simd_types
[Float16x4_t
].eltype
= aarch64_fp16_type_node
;
724 aarch64_simd_types
[Float16x8_t
].eltype
= aarch64_fp16_type_node
;
725 aarch64_simd_types
[Float32x2_t
].eltype
= float_type_node
;
726 aarch64_simd_types
[Float32x4_t
].eltype
= float_type_node
;
727 aarch64_simd_types
[Float64x1_t
].eltype
= double_type_node
;
728 aarch64_simd_types
[Float64x2_t
].eltype
= double_type_node
;
730 for (i
= 0; i
< nelts
; i
++)
732 tree eltype
= aarch64_simd_types
[i
].eltype
;
733 machine_mode mode
= aarch64_simd_types
[i
].mode
;
735 if (aarch64_simd_types
[i
].itype
== NULL
)
737 aarch64_simd_types
[i
].itype
738 = build_distinct_type_copy
739 (build_vector_type (eltype
, GET_MODE_NUNITS (mode
)));
740 SET_TYPE_STRUCTURAL_EQUALITY (aarch64_simd_types
[i
].itype
);
743 tdecl
= add_builtin_type (aarch64_simd_types
[i
].name
,
744 aarch64_simd_types
[i
].itype
);
745 TYPE_NAME (aarch64_simd_types
[i
].itype
) = tdecl
;
748 #define AARCH64_BUILD_SIGNED_TYPE(mode) \
749 make_signed_type (GET_MODE_PRECISION (mode));
750 aarch64_simd_intOI_type_node
= AARCH64_BUILD_SIGNED_TYPE (OImode
);
751 aarch64_simd_intCI_type_node
= AARCH64_BUILD_SIGNED_TYPE (CImode
);
752 aarch64_simd_intXI_type_node
= AARCH64_BUILD_SIGNED_TYPE (XImode
);
753 #undef AARCH64_BUILD_SIGNED_TYPE
755 tdecl
= add_builtin_type
756 ("__builtin_aarch64_simd_oi" , aarch64_simd_intOI_type_node
);
757 TYPE_NAME (aarch64_simd_intOI_type_node
) = tdecl
;
758 tdecl
= add_builtin_type
759 ("__builtin_aarch64_simd_ci" , aarch64_simd_intCI_type_node
);
760 TYPE_NAME (aarch64_simd_intCI_type_node
) = tdecl
;
761 tdecl
= add_builtin_type
762 ("__builtin_aarch64_simd_xi" , aarch64_simd_intXI_type_node
);
763 TYPE_NAME (aarch64_simd_intXI_type_node
) = tdecl
;
767 aarch64_init_simd_builtin_scalar_types (void)
769 /* Define typedefs for all the standard scalar types. */
770 (*lang_hooks
.types
.register_builtin_type
) (intQI_type_node
,
771 "__builtin_aarch64_simd_qi");
772 (*lang_hooks
.types
.register_builtin_type
) (intHI_type_node
,
773 "__builtin_aarch64_simd_hi");
774 (*lang_hooks
.types
.register_builtin_type
) (aarch64_fp16_type_node
,
775 "__builtin_aarch64_simd_hf");
776 (*lang_hooks
.types
.register_builtin_type
) (intSI_type_node
,
777 "__builtin_aarch64_simd_si");
778 (*lang_hooks
.types
.register_builtin_type
) (float_type_node
,
779 "__builtin_aarch64_simd_sf");
780 (*lang_hooks
.types
.register_builtin_type
) (intDI_type_node
,
781 "__builtin_aarch64_simd_di");
782 (*lang_hooks
.types
.register_builtin_type
) (double_type_node
,
783 "__builtin_aarch64_simd_df");
784 (*lang_hooks
.types
.register_builtin_type
) (unsigned_intQI_type_node
,
785 "__builtin_aarch64_simd_poly8");
786 (*lang_hooks
.types
.register_builtin_type
) (unsigned_intHI_type_node
,
787 "__builtin_aarch64_simd_poly16");
788 (*lang_hooks
.types
.register_builtin_type
) (unsigned_intDI_type_node
,
789 "__builtin_aarch64_simd_poly64");
790 (*lang_hooks
.types
.register_builtin_type
) (unsigned_intTI_type_node
,
791 "__builtin_aarch64_simd_poly128");
792 (*lang_hooks
.types
.register_builtin_type
) (intTI_type_node
,
793 "__builtin_aarch64_simd_ti");
794 /* Unsigned integer types for various mode sizes. */
795 (*lang_hooks
.types
.register_builtin_type
) (unsigned_intQI_type_node
,
796 "__builtin_aarch64_simd_uqi");
797 (*lang_hooks
.types
.register_builtin_type
) (unsigned_intHI_type_node
,
798 "__builtin_aarch64_simd_uhi");
799 (*lang_hooks
.types
.register_builtin_type
) (unsigned_intSI_type_node
,
800 "__builtin_aarch64_simd_usi");
801 (*lang_hooks
.types
.register_builtin_type
) (unsigned_intDI_type_node
,
802 "__builtin_aarch64_simd_udi");
805 static bool aarch64_simd_builtins_initialized_p
= false;
807 /* Due to the architecture not providing lane variant of the lane instructions
808 for fcmla we can't use the standard simd builtin expansion code, but we
809 still want the majority of the validation that would normally be done. */
812 aarch64_init_fcmla_laneq_builtins (void)
816 for (i
= 0; i
< ARRAY_SIZE (aarch64_fcmla_lane_builtin_data
); ++i
)
818 aarch64_fcmla_laneq_builtin_datum
* d
819 = &aarch64_fcmla_lane_builtin_data
[i
];
820 tree argtype
= aarch64_lookup_simd_builtin_type (d
->mode
, qualifier_none
);
821 machine_mode quadmode
= GET_MODE_2XWIDER_MODE (d
->mode
).require ();
823 = aarch64_lookup_simd_builtin_type (quadmode
, qualifier_none
);
825 = aarch64_simd_builtin_std_type (SImode
, qualifier_lane_pair_index
);
826 tree ftype
= build_function_type_list (argtype
, argtype
, argtype
,
827 quadtype
, lanetype
, NULL_TREE
);
828 tree fndecl
= add_builtin_function (d
->name
, ftype
, d
->fcode
,
829 BUILT_IN_MD
, NULL
, NULL_TREE
);
831 aarch64_builtin_decls
[d
->fcode
] = fndecl
;
836 aarch64_init_simd_builtins (void)
838 unsigned int i
, fcode
= AARCH64_SIMD_PATTERN_START
;
840 if (aarch64_simd_builtins_initialized_p
)
843 aarch64_simd_builtins_initialized_p
= true;
845 aarch64_init_simd_builtin_types ();
847 /* Strong-typing hasn't been implemented for all AdvSIMD builtin intrinsics.
848 Therefore we need to preserve the old __builtin scalar types. It can be
849 removed once all the intrinsics become strongly typed using the qualifier
851 aarch64_init_simd_builtin_scalar_types ();
853 tree lane_check_fpr
= build_function_type_list (void_type_node
,
858 aarch64_builtin_decls
[AARCH64_SIMD_BUILTIN_LANE_CHECK
] =
859 add_builtin_function ("__builtin_aarch64_im_lane_boundsi", lane_check_fpr
,
860 AARCH64_SIMD_BUILTIN_LANE_CHECK
, BUILT_IN_MD
,
863 for (i
= 0; i
< ARRAY_SIZE (aarch64_simd_builtin_data
); i
++, fcode
++)
865 bool print_type_signature_p
= false;
866 char type_signature
[SIMD_MAX_BUILTIN_ARGS
+ 1] = { 0 };
867 aarch64_simd_builtin_datum
*d
= &aarch64_simd_builtin_data
[i
];
874 /* We must track two variables here. op_num is
875 the operand number as in the RTL pattern. This is
876 required to access the mode (e.g. V4SF mode) of the
877 argument, from which the base type can be derived.
878 arg_num is an index in to the qualifiers data, which
879 gives qualifiers to the type (e.g. const unsigned).
880 The reason these two variables may differ by one is the
881 void return type. While all return types take the 0th entry
882 in the qualifiers array, there is no operand for them in the
884 int op_num
= insn_data
[d
->code
].n_operands
- 1;
885 int arg_num
= d
->qualifiers
[0] & qualifier_void
888 tree return_type
= void_type_node
, args
= void_list_node
;
891 /* Build a function type directly from the insn_data for this
892 builtin. The build_function_type () function takes care of
893 removing duplicates for us. */
894 for (; op_num
>= 0; arg_num
--, op_num
--)
896 machine_mode op_mode
= insn_data
[d
->code
].operand
[op_num
].mode
;
897 enum aarch64_type_qualifiers qualifiers
= d
->qualifiers
[arg_num
];
899 if (qualifiers
& qualifier_unsigned
)
901 type_signature
[op_num
] = 'u';
902 print_type_signature_p
= true;
904 else if (qualifiers
& qualifier_poly
)
906 type_signature
[op_num
] = 'p';
907 print_type_signature_p
= true;
910 type_signature
[op_num
] = 's';
912 /* Skip an internal operand for vget_{low, high}. */
913 if (qualifiers
& qualifier_internal
)
916 /* Some builtins have different user-facing types
917 for certain arguments, encoded in d->mode. */
918 if (qualifiers
& qualifier_map_mode
)
921 /* For pointers, we want a pointer to the basic type
923 if (qualifiers
& qualifier_pointer
&& VECTOR_MODE_P (op_mode
))
924 op_mode
= GET_MODE_INNER (op_mode
);
926 eltype
= aarch64_simd_builtin_type
928 (qualifiers
& qualifier_unsigned
) != 0,
929 (qualifiers
& qualifier_poly
) != 0);
930 gcc_assert (eltype
!= NULL
);
932 /* Add qualifiers. */
933 if (qualifiers
& qualifier_const
)
934 eltype
= build_qualified_type (eltype
, TYPE_QUAL_CONST
);
936 if (qualifiers
& qualifier_pointer
)
937 eltype
= build_pointer_type (eltype
);
939 /* If we have reached arg_num == 0, we are at a non-void
940 return type. Otherwise, we are still processing
943 return_type
= eltype
;
945 args
= tree_cons (NULL_TREE
, eltype
, args
);
948 ftype
= build_function_type (return_type
, args
);
950 gcc_assert (ftype
!= NULL
);
952 if (print_type_signature_p
)
953 snprintf (namebuf
, sizeof (namebuf
), "__builtin_aarch64_%s_%s",
954 d
->name
, type_signature
);
956 snprintf (namebuf
, sizeof (namebuf
), "__builtin_aarch64_%s",
959 fndecl
= add_builtin_function (namebuf
, ftype
, fcode
, BUILT_IN_MD
,
961 aarch64_builtin_decls
[fcode
] = fndecl
;
964 /* Initialize the remaining fcmla_laneq intrinsics. */
965 aarch64_init_fcmla_laneq_builtins ();
969 aarch64_init_crc32_builtins ()
971 tree usi_type
= aarch64_simd_builtin_std_type (SImode
, qualifier_unsigned
);
974 for (i
= 0; i
< ARRAY_SIZE (aarch64_crc_builtin_data
); ++i
)
976 aarch64_crc_builtin_datum
* d
= &aarch64_crc_builtin_data
[i
];
977 tree argtype
= aarch64_simd_builtin_std_type (d
->mode
,
979 tree ftype
= build_function_type_list (usi_type
, usi_type
, argtype
, NULL_TREE
);
980 tree fndecl
= add_builtin_function (d
->name
, ftype
, d
->fcode
,
981 BUILT_IN_MD
, NULL
, NULL_TREE
);
983 aarch64_builtin_decls
[d
->fcode
] = fndecl
;
987 /* Add builtins for reciprocal square root. */
990 aarch64_init_builtin_rsqrt (void)
995 tree V2SF_type_node
= build_vector_type (float_type_node
, 2);
996 tree V2DF_type_node
= build_vector_type (double_type_node
, 2);
997 tree V4SF_type_node
= build_vector_type (float_type_node
, 4);
999 struct builtin_decls_data
1002 const char *builtin_name
;
1006 builtin_decls_data bdda
[] =
1008 { double_type_node
, "__builtin_aarch64_rsqrt_df", AARCH64_BUILTIN_RSQRT_DF
},
1009 { float_type_node
, "__builtin_aarch64_rsqrt_sf", AARCH64_BUILTIN_RSQRT_SF
},
1010 { V2DF_type_node
, "__builtin_aarch64_rsqrt_v2df", AARCH64_BUILTIN_RSQRT_V2DF
},
1011 { V2SF_type_node
, "__builtin_aarch64_rsqrt_v2sf", AARCH64_BUILTIN_RSQRT_V2SF
},
1012 { V4SF_type_node
, "__builtin_aarch64_rsqrt_v4sf", AARCH64_BUILTIN_RSQRT_V4SF
}
1015 builtin_decls_data
*bdd
= bdda
;
1016 builtin_decls_data
*bdd_end
= bdd
+ (sizeof (bdda
) / sizeof (builtin_decls_data
));
1018 for (; bdd
< bdd_end
; bdd
++)
1020 ftype
= build_function_type_list (bdd
->type_node
, bdd
->type_node
, NULL_TREE
);
1021 fndecl
= add_builtin_function (bdd
->builtin_name
,
1022 ftype
, bdd
->function_code
, BUILT_IN_MD
, NULL
, NULL_TREE
);
1023 aarch64_builtin_decls
[bdd
->function_code
] = fndecl
;
1027 /* Initialize the backend types that support the user-visible __fp16
1028 type, also initialize a pointer to that type, to be used when
1032 aarch64_init_fp16_types (void)
1034 aarch64_fp16_type_node
= make_node (REAL_TYPE
);
1035 TYPE_PRECISION (aarch64_fp16_type_node
) = 16;
1036 layout_type (aarch64_fp16_type_node
);
1038 (*lang_hooks
.types
.register_builtin_type
) (aarch64_fp16_type_node
, "__fp16");
1039 aarch64_fp16_ptr_type_node
= build_pointer_type (aarch64_fp16_type_node
);
1042 /* Pointer authentication builtins that will become NOP on legacy platform.
1043 Currently, these builtins are for internal use only (libgcc EH unwinder). */
1046 aarch64_init_pauth_hint_builtins (void)
1048 /* Pointer Authentication builtins. */
1049 tree ftype_pointer_auth
1050 = build_function_type_list (ptr_type_node
, ptr_type_node
,
1051 unsigned_intDI_type_node
, NULL_TREE
);
1052 tree ftype_pointer_strip
1053 = build_function_type_list (ptr_type_node
, ptr_type_node
, NULL_TREE
);
1055 aarch64_builtin_decls
[AARCH64_PAUTH_BUILTIN_AUTIA1716
]
1056 = add_builtin_function ("__builtin_aarch64_autia1716", ftype_pointer_auth
,
1057 AARCH64_PAUTH_BUILTIN_AUTIA1716
, BUILT_IN_MD
, NULL
,
1059 aarch64_builtin_decls
[AARCH64_PAUTH_BUILTIN_PACIA1716
]
1060 = add_builtin_function ("__builtin_aarch64_pacia1716", ftype_pointer_auth
,
1061 AARCH64_PAUTH_BUILTIN_PACIA1716
, BUILT_IN_MD
, NULL
,
1063 aarch64_builtin_decls
[AARCH64_PAUTH_BUILTIN_AUTIB1716
]
1064 = add_builtin_function ("__builtin_aarch64_autib1716", ftype_pointer_auth
,
1065 AARCH64_PAUTH_BUILTIN_AUTIB1716
, BUILT_IN_MD
, NULL
,
1067 aarch64_builtin_decls
[AARCH64_PAUTH_BUILTIN_PACIB1716
]
1068 = add_builtin_function ("__builtin_aarch64_pacib1716", ftype_pointer_auth
,
1069 AARCH64_PAUTH_BUILTIN_PACIB1716
, BUILT_IN_MD
, NULL
,
1071 aarch64_builtin_decls
[AARCH64_PAUTH_BUILTIN_XPACLRI
]
1072 = add_builtin_function ("__builtin_aarch64_xpaclri", ftype_pointer_strip
,
1073 AARCH64_PAUTH_BUILTIN_XPACLRI
, BUILT_IN_MD
, NULL
,
1077 /* Initialize the transactional memory extension (TME) builtins. */
1079 aarch64_init_tme_builtins (void)
1081 tree ftype_uint64_void
1082 = build_function_type_list (uint64_type_node
, NULL
);
1083 tree ftype_void_void
1084 = build_function_type_list (void_type_node
, NULL
);
1085 tree ftype_void_uint64
1086 = build_function_type_list (void_type_node
, uint64_type_node
, NULL
);
1088 aarch64_builtin_decls
[AARCH64_TME_BUILTIN_TSTART
]
1089 = add_builtin_function ("__builtin_aarch64_tstart", ftype_uint64_void
,
1090 AARCH64_TME_BUILTIN_TSTART
, BUILT_IN_MD
,
1092 aarch64_builtin_decls
[AARCH64_TME_BUILTIN_TTEST
]
1093 = add_builtin_function ("__builtin_aarch64_ttest", ftype_uint64_void
,
1094 AARCH64_TME_BUILTIN_TTEST
, BUILT_IN_MD
,
1096 aarch64_builtin_decls
[AARCH64_TME_BUILTIN_TCOMMIT
]
1097 = add_builtin_function ("__builtin_aarch64_tcommit", ftype_void_void
,
1098 AARCH64_TME_BUILTIN_TCOMMIT
, BUILT_IN_MD
,
1100 aarch64_builtin_decls
[AARCH64_TME_BUILTIN_TCANCEL
]
1101 = add_builtin_function ("__builtin_aarch64_tcancel", ftype_void_uint64
,
1102 AARCH64_TME_BUILTIN_TCANCEL
, BUILT_IN_MD
,
1107 aarch64_init_builtins (void)
1110 = build_function_type_list (void_type_node
, unsigned_type_node
, NULL
);
1112 = build_function_type_list (unsigned_type_node
, NULL
);
1114 aarch64_builtin_decls
[AARCH64_BUILTIN_GET_FPCR
]
1115 = add_builtin_function ("__builtin_aarch64_get_fpcr", ftype_get_fpr
,
1116 AARCH64_BUILTIN_GET_FPCR
, BUILT_IN_MD
, NULL
, NULL_TREE
);
1117 aarch64_builtin_decls
[AARCH64_BUILTIN_SET_FPCR
]
1118 = add_builtin_function ("__builtin_aarch64_set_fpcr", ftype_set_fpr
,
1119 AARCH64_BUILTIN_SET_FPCR
, BUILT_IN_MD
, NULL
, NULL_TREE
);
1120 aarch64_builtin_decls
[AARCH64_BUILTIN_GET_FPSR
]
1121 = add_builtin_function ("__builtin_aarch64_get_fpsr", ftype_get_fpr
,
1122 AARCH64_BUILTIN_GET_FPSR
, BUILT_IN_MD
, NULL
, NULL_TREE
);
1123 aarch64_builtin_decls
[AARCH64_BUILTIN_SET_FPSR
]
1124 = add_builtin_function ("__builtin_aarch64_set_fpsr", ftype_set_fpr
,
1125 AARCH64_BUILTIN_SET_FPSR
, BUILT_IN_MD
, NULL
, NULL_TREE
);
1127 aarch64_init_fp16_types ();
1130 aarch64_init_simd_builtins ();
1132 aarch64_init_crc32_builtins ();
1133 aarch64_init_builtin_rsqrt ();
1136 = build_function_type_list (intSI_type_node
, double_type_node
, NULL
);
1137 aarch64_builtin_decls
[AARCH64_JSCVT
]
1138 = add_builtin_function ("__builtin_aarch64_jcvtzs", ftype_jcvt
,
1139 AARCH64_JSCVT
, BUILT_IN_MD
, NULL
, NULL_TREE
);
1141 /* Initialize pointer authentication builtins which are backed by instructions
1142 in NOP encoding space.
1144 NOTE: these builtins are supposed to be used by libgcc unwinder only, as
1145 there is no support on return address signing under ILP32, we don't
1148 aarch64_init_pauth_hint_builtins ();
1151 aarch64_init_tme_builtins ();
1155 aarch64_builtin_decl (unsigned code
, bool initialize_p ATTRIBUTE_UNUSED
)
1157 if (code
>= AARCH64_BUILTIN_MAX
)
1158 return error_mark_node
;
1160 return aarch64_builtin_decls
[code
];
1165 SIMD_ARG_COPY_TO_REG
,
1167 SIMD_ARG_LANE_INDEX
,
1168 SIMD_ARG_STRUCT_LOAD_STORE_LANE_INDEX
,
1169 SIMD_ARG_LANE_PAIR_INDEX
,
1175 aarch64_simd_expand_args (rtx target
, int icode
, int have_retval
,
1176 tree exp
, builtin_simd_arg
*args
,
1177 machine_mode builtin_mode
)
1180 rtx op
[SIMD_MAX_BUILTIN_ARGS
+ 1]; /* First element for result operand. */
1185 machine_mode tmode
= insn_data
[icode
].operand
[0].mode
;
1187 || GET_MODE (target
) != tmode
1188 || !(*insn_data
[icode
].operand
[0].predicate
) (target
, tmode
))
1189 target
= gen_reg_rtx (tmode
);
1195 builtin_simd_arg thisarg
= args
[opc
- have_retval
];
1197 if (thisarg
== SIMD_ARG_STOP
)
1201 tree arg
= CALL_EXPR_ARG (exp
, opc
- have_retval
);
1202 machine_mode mode
= insn_data
[icode
].operand
[opc
].mode
;
1203 op
[opc
] = expand_normal (arg
);
1207 case SIMD_ARG_COPY_TO_REG
:
1208 if (POINTER_TYPE_P (TREE_TYPE (arg
)))
1209 op
[opc
] = convert_memory_address (Pmode
, op
[opc
]);
1210 /*gcc_assert (GET_MODE (op[opc]) == mode); */
1211 if (!(*insn_data
[icode
].operand
[opc
].predicate
)
1213 op
[opc
] = copy_to_mode_reg (mode
, op
[opc
]);
1216 case SIMD_ARG_STRUCT_LOAD_STORE_LANE_INDEX
:
1217 gcc_assert (opc
> 1);
1218 if (CONST_INT_P (op
[opc
]))
1221 = GET_MODE_NUNITS (builtin_mode
).to_constant ();
1222 aarch64_simd_lane_bounds (op
[opc
], 0, nunits
, exp
);
1223 /* Keep to GCC-vector-extension lane indices in the RTL. */
1224 op
[opc
] = aarch64_endian_lane_rtx (builtin_mode
,
1229 case SIMD_ARG_LANE_INDEX
:
1230 /* Must be a previous operand into which this is an index. */
1231 gcc_assert (opc
> 0);
1232 if (CONST_INT_P (op
[opc
]))
1234 machine_mode vmode
= insn_data
[icode
].operand
[opc
- 1].mode
;
1236 = GET_MODE_NUNITS (vmode
).to_constant ();
1237 aarch64_simd_lane_bounds (op
[opc
], 0, nunits
, exp
);
1238 /* Keep to GCC-vector-extension lane indices in the RTL. */
1239 op
[opc
] = aarch64_endian_lane_rtx (vmode
, INTVAL (op
[opc
]));
1241 /* If the lane index isn't a constant then error out. */
1244 case SIMD_ARG_LANE_PAIR_INDEX
:
1245 /* Must be a previous operand into which this is an index and
1246 index is restricted to nunits / 2. */
1247 gcc_assert (opc
> 0);
1248 if (CONST_INT_P (op
[opc
]))
1250 machine_mode vmode
= insn_data
[icode
].operand
[opc
- 1].mode
;
1252 = GET_MODE_NUNITS (vmode
).to_constant ();
1253 aarch64_simd_lane_bounds (op
[opc
], 0, nunits
/ 2, exp
);
1254 /* Keep to GCC-vector-extension lane indices in the RTL. */
1255 int lane
= INTVAL (op
[opc
]);
1256 op
[opc
] = gen_int_mode (ENDIAN_LANE_N (nunits
/ 2, lane
),
1259 /* Fall through - if the lane index isn't a constant then
1260 the next case will error. */
1262 case SIMD_ARG_CONSTANT
:
1264 if (!(*insn_data
[icode
].operand
[opc
].predicate
)
1267 error ("%Kargument %d must be a constant immediate",
1268 exp
, opc
+ 1 - have_retval
);
1284 pat
= GEN_FCN (icode
) (op
[0]);
1288 pat
= GEN_FCN (icode
) (op
[0], op
[1]);
1292 pat
= GEN_FCN (icode
) (op
[0], op
[1], op
[2]);
1296 pat
= GEN_FCN (icode
) (op
[0], op
[1], op
[2], op
[3]);
1300 pat
= GEN_FCN (icode
) (op
[0], op
[1], op
[2], op
[3], op
[4]);
1304 pat
= GEN_FCN (icode
) (op
[0], op
[1], op
[2], op
[3], op
[4], op
[5]);
1319 /* Expand an AArch64 AdvSIMD builtin(intrinsic). */
1321 aarch64_simd_expand_builtin (int fcode
, tree exp
, rtx target
)
1323 if (fcode
== AARCH64_SIMD_BUILTIN_LANE_CHECK
)
1325 rtx totalsize
= expand_normal (CALL_EXPR_ARG (exp
, 0));
1326 rtx elementsize
= expand_normal (CALL_EXPR_ARG (exp
, 1));
1327 if (CONST_INT_P (totalsize
) && CONST_INT_P (elementsize
)
1328 && UINTVAL (elementsize
) != 0
1329 && UINTVAL (totalsize
) != 0)
1331 rtx lane_idx
= expand_normal (CALL_EXPR_ARG (exp
, 2));
1332 if (CONST_INT_P (lane_idx
))
1333 aarch64_simd_lane_bounds (lane_idx
, 0,
1335 / UINTVAL (elementsize
),
1338 error ("%Klane index must be a constant immediate", exp
);
1341 error ("%Ktotal size and element size must be a non-zero constant immediate", exp
);
1342 /* Don't generate any RTL. */
1345 aarch64_simd_builtin_datum
*d
=
1346 &aarch64_simd_builtin_data
[fcode
- AARCH64_SIMD_PATTERN_START
];
1347 enum insn_code icode
= d
->code
;
1348 builtin_simd_arg args
[SIMD_MAX_BUILTIN_ARGS
+ 1];
1349 int num_args
= insn_data
[d
->code
].n_operands
;
1353 is_void
= !!(d
->qualifiers
[0] & qualifier_void
);
1355 num_args
+= is_void
;
1357 for (k
= 1; k
< num_args
; k
++)
1359 /* We have four arrays of data, each indexed in a different fashion.
1360 qualifiers - element 0 always describes the function return type.
1361 operands - element 0 is either the operand for return value (if
1362 the function has a non-void return type) or the operand for the
1364 expr_args - element 0 always holds the first argument.
1365 args - element 0 is always used for the return type. */
1366 int qualifiers_k
= k
;
1367 int operands_k
= k
- is_void
;
1368 int expr_args_k
= k
- 1;
1370 if (d
->qualifiers
[qualifiers_k
] & qualifier_lane_index
)
1371 args
[k
] = SIMD_ARG_LANE_INDEX
;
1372 else if (d
->qualifiers
[qualifiers_k
] & qualifier_lane_pair_index
)
1373 args
[k
] = SIMD_ARG_LANE_PAIR_INDEX
;
1374 else if (d
->qualifiers
[qualifiers_k
] & qualifier_struct_load_store_lane_index
)
1375 args
[k
] = SIMD_ARG_STRUCT_LOAD_STORE_LANE_INDEX
;
1376 else if (d
->qualifiers
[qualifiers_k
] & qualifier_immediate
)
1377 args
[k
] = SIMD_ARG_CONSTANT
;
1378 else if (d
->qualifiers
[qualifiers_k
] & qualifier_maybe_immediate
)
1381 = expand_normal (CALL_EXPR_ARG (exp
,
1383 /* Handle constants only if the predicate allows it. */
1384 bool op_const_int_p
=
1386 && (*insn_data
[icode
].operand
[operands_k
].predicate
)
1387 (arg
, insn_data
[icode
].operand
[operands_k
].mode
));
1388 args
[k
] = op_const_int_p
? SIMD_ARG_CONSTANT
: SIMD_ARG_COPY_TO_REG
;
1391 args
[k
] = SIMD_ARG_COPY_TO_REG
;
1394 args
[k
] = SIMD_ARG_STOP
;
1396 /* The interface to aarch64_simd_expand_args expects a 0 if
1397 the function is void, and a 1 if it is not. */
1398 return aarch64_simd_expand_args
1399 (target
, icode
, !is_void
, exp
, &args
[1], d
->mode
);
1403 aarch64_crc32_expand_builtin (int fcode
, tree exp
, rtx target
)
1406 aarch64_crc_builtin_datum
*d
1407 = &aarch64_crc_builtin_data
[fcode
- (AARCH64_CRC32_BUILTIN_BASE
+ 1)];
1408 enum insn_code icode
= d
->icode
;
1409 tree arg0
= CALL_EXPR_ARG (exp
, 0);
1410 tree arg1
= CALL_EXPR_ARG (exp
, 1);
1411 rtx op0
= expand_normal (arg0
);
1412 rtx op1
= expand_normal (arg1
);
1413 machine_mode tmode
= insn_data
[icode
].operand
[0].mode
;
1414 machine_mode mode0
= insn_data
[icode
].operand
[1].mode
;
1415 machine_mode mode1
= insn_data
[icode
].operand
[2].mode
;
1418 || GET_MODE (target
) != tmode
1419 || ! (*insn_data
[icode
].operand
[0].predicate
) (target
, tmode
))
1420 target
= gen_reg_rtx (tmode
);
1422 gcc_assert ((GET_MODE (op0
) == mode0
|| GET_MODE (op0
) == VOIDmode
)
1423 && (GET_MODE (op1
) == mode1
|| GET_MODE (op1
) == VOIDmode
));
1425 if (! (*insn_data
[icode
].operand
[1].predicate
) (op0
, mode0
))
1426 op0
= copy_to_mode_reg (mode0
, op0
);
1427 if (! (*insn_data
[icode
].operand
[2].predicate
) (op1
, mode1
))
1428 op1
= copy_to_mode_reg (mode1
, op1
);
1430 pat
= GEN_FCN (icode
) (target
, op0
, op1
);
1438 /* Function to expand reciprocal square root builtins. */
1441 aarch64_expand_builtin_rsqrt (int fcode
, tree exp
, rtx target
)
1443 tree arg0
= CALL_EXPR_ARG (exp
, 0);
1444 rtx op0
= expand_normal (arg0
);
1446 rtx (*gen
) (rtx
, rtx
);
1450 case AARCH64_BUILTIN_RSQRT_DF
:
1453 case AARCH64_BUILTIN_RSQRT_SF
:
1456 case AARCH64_BUILTIN_RSQRT_V2DF
:
1457 gen
= gen_rsqrtv2df2
;
1459 case AARCH64_BUILTIN_RSQRT_V2SF
:
1460 gen
= gen_rsqrtv2sf2
;
1462 case AARCH64_BUILTIN_RSQRT_V4SF
:
1463 gen
= gen_rsqrtv4sf2
;
1465 default: gcc_unreachable ();
1469 target
= gen_reg_rtx (GET_MODE (op0
));
1471 emit_insn (gen (target
, op0
));
1476 /* Expand a FCMLA lane expression EXP with code FCODE and
1477 result going to TARGET if that is convenient. */
1480 aarch64_expand_fcmla_builtin (tree exp
, rtx target
, int fcode
)
1482 int bcode
= fcode
- AARCH64_SIMD_FCMLA_LANEQ_BUILTIN_BASE
- 1;
1483 aarch64_fcmla_laneq_builtin_datum
* d
1484 = &aarch64_fcmla_lane_builtin_data
[bcode
];
1485 machine_mode quadmode
= GET_MODE_2XWIDER_MODE (d
->mode
).require ();
1486 rtx op0
= force_reg (d
->mode
, expand_normal (CALL_EXPR_ARG (exp
, 0)));
1487 rtx op1
= force_reg (d
->mode
, expand_normal (CALL_EXPR_ARG (exp
, 1)));
1488 rtx op2
= force_reg (quadmode
, expand_normal (CALL_EXPR_ARG (exp
, 2)));
1489 tree tmp
= CALL_EXPR_ARG (exp
, 3);
1490 rtx lane_idx
= expand_expr (tmp
, NULL_RTX
, VOIDmode
, EXPAND_INITIALIZER
);
1492 /* Validate that the lane index is a constant. */
1493 if (!CONST_INT_P (lane_idx
))
1495 error ("%Kargument %d must be a constant immediate", exp
, 4);
1499 /* Validate that the index is within the expected range. */
1500 int nunits
= GET_MODE_NUNITS (quadmode
).to_constant ();
1501 aarch64_simd_lane_bounds (lane_idx
, 0, nunits
/ 2, exp
);
1503 /* Generate the correct register and mode. */
1504 int lane
= INTVAL (lane_idx
);
1506 if (lane
< nunits
/ 4)
1507 op2
= simplify_gen_subreg (d
->mode
, op2
, quadmode
,
1508 subreg_lowpart_offset (d
->mode
, quadmode
));
1511 /* Select the upper 64 bits, either a V2SF or V4HF, this however
1512 is quite messy, as the operation required even though simple
1513 doesn't have a simple RTL pattern, and seems it's quite hard to
1514 define using a single RTL pattern. The target generic version
1515 gen_highpart_mode generates code that isn't optimal. */
1516 rtx temp1
= gen_reg_rtx (d
->mode
);
1517 rtx temp2
= gen_reg_rtx (DImode
);
1518 temp1
= simplify_gen_subreg (d
->mode
, op2
, quadmode
,
1519 subreg_lowpart_offset (d
->mode
, quadmode
));
1520 temp1
= simplify_gen_subreg (V2DImode
, temp1
, d
->mode
, 0);
1521 if (BYTES_BIG_ENDIAN
)
1522 emit_insn (gen_aarch64_get_lanev2di (temp2
, temp1
, const0_rtx
));
1524 emit_insn (gen_aarch64_get_lanev2di (temp2
, temp1
, const1_rtx
));
1525 op2
= simplify_gen_subreg (d
->mode
, temp2
, GET_MODE (temp2
), 0);
1527 /* And recalculate the index. */
1531 /* Keep to GCC-vector-extension lane indices in the RTL, only nunits / 4
1532 (max nunits in range check) are valid. Which means only 0-1, so we
1533 only need to know the order in a V2mode. */
1534 lane_idx
= aarch64_endian_lane_rtx (V2DImode
, lane
);
1537 target
= gen_reg_rtx (d
->mode
);
1539 target
= force_reg (d
->mode
, target
);
1544 pat
= GEN_FCN (d
->icode
) (target
, op0
, op1
, op2
, lane_idx
);
1546 pat
= GEN_FCN (d
->icode
) (target
, op0
, op1
, op2
);
1555 /* Function to expand an expression EXP which calls one of the Transactional
1556 Memory Extension (TME) builtins FCODE with the result going to TARGET. */
1558 aarch64_expand_builtin_tme (int fcode
, tree exp
, rtx target
)
1562 case AARCH64_TME_BUILTIN_TSTART
:
1563 target
= gen_reg_rtx (DImode
);
1564 emit_insn (GEN_FCN (CODE_FOR_tstart
) (target
));
1567 case AARCH64_TME_BUILTIN_TTEST
:
1568 target
= gen_reg_rtx (DImode
);
1569 emit_insn (GEN_FCN (CODE_FOR_ttest
) (target
));
1572 case AARCH64_TME_BUILTIN_TCOMMIT
:
1573 emit_insn (GEN_FCN (CODE_FOR_tcommit
) ());
1576 case AARCH64_TME_BUILTIN_TCANCEL
:
1578 tree arg0
= CALL_EXPR_ARG (exp
, 0);
1579 rtx op0
= expand_normal (arg0
);
1580 if (CONST_INT_P (op0
) && UINTVAL (op0
) <= 65536)
1581 emit_insn (GEN_FCN (CODE_FOR_tcancel
) (op0
));
1584 error ("%Kargument must be a 16-bit constant immediate", exp
);
1596 /* Expand an expression EXP that calls a built-in function,
1597 with result going to TARGET if that's convenient. */
1599 aarch64_expand_builtin (tree exp
,
1601 rtx subtarget ATTRIBUTE_UNUSED
,
1602 machine_mode mode ATTRIBUTE_UNUSED
,
1603 int ignore ATTRIBUTE_UNUSED
)
1605 tree fndecl
= TREE_OPERAND (CALL_EXPR_FN (exp
), 0);
1606 int fcode
= DECL_MD_FUNCTION_CODE (fndecl
);
1613 case AARCH64_BUILTIN_GET_FPCR
:
1614 case AARCH64_BUILTIN_SET_FPCR
:
1615 case AARCH64_BUILTIN_GET_FPSR
:
1616 case AARCH64_BUILTIN_SET_FPSR
:
1617 if ((fcode
== AARCH64_BUILTIN_GET_FPCR
)
1618 || (fcode
== AARCH64_BUILTIN_GET_FPSR
))
1620 icode
= (fcode
== AARCH64_BUILTIN_GET_FPSR
) ?
1621 CODE_FOR_get_fpsr
: CODE_FOR_get_fpcr
;
1622 target
= gen_reg_rtx (SImode
);
1623 pat
= GEN_FCN (icode
) (target
);
1628 icode
= (fcode
== AARCH64_BUILTIN_SET_FPSR
) ?
1629 CODE_FOR_set_fpsr
: CODE_FOR_set_fpcr
;
1630 arg0
= CALL_EXPR_ARG (exp
, 0);
1631 op0
= force_reg (SImode
, expand_normal (arg0
));
1632 pat
= GEN_FCN (icode
) (op0
);
1637 case AARCH64_PAUTH_BUILTIN_AUTIA1716
:
1638 case AARCH64_PAUTH_BUILTIN_PACIA1716
:
1639 case AARCH64_PAUTH_BUILTIN_AUTIB1716
:
1640 case AARCH64_PAUTH_BUILTIN_PACIB1716
:
1641 case AARCH64_PAUTH_BUILTIN_XPACLRI
:
1642 arg0
= CALL_EXPR_ARG (exp
, 0);
1643 op0
= force_reg (Pmode
, expand_normal (arg0
));
1646 target
= gen_reg_rtx (Pmode
);
1648 target
= force_reg (Pmode
, target
);
1650 emit_move_insn (target
, op0
);
1652 if (fcode
== AARCH64_PAUTH_BUILTIN_XPACLRI
)
1654 rtx lr
= gen_rtx_REG (Pmode
, R30_REGNUM
);
1655 icode
= CODE_FOR_xpaclri
;
1656 emit_move_insn (lr
, op0
);
1657 emit_insn (GEN_FCN (icode
) ());
1658 emit_move_insn (target
, lr
);
1662 tree arg1
= CALL_EXPR_ARG (exp
, 1);
1663 rtx op1
= force_reg (Pmode
, expand_normal (arg1
));
1666 case AARCH64_PAUTH_BUILTIN_AUTIA1716
:
1667 icode
= CODE_FOR_autia1716
;
1669 case AARCH64_PAUTH_BUILTIN_AUTIB1716
:
1670 icode
= CODE_FOR_autib1716
;
1672 case AARCH64_PAUTH_BUILTIN_PACIA1716
:
1673 icode
= CODE_FOR_pacia1716
;
1675 case AARCH64_PAUTH_BUILTIN_PACIB1716
:
1676 icode
= CODE_FOR_pacib1716
;
1683 rtx x16_reg
= gen_rtx_REG (Pmode
, R16_REGNUM
);
1684 rtx x17_reg
= gen_rtx_REG (Pmode
, R17_REGNUM
);
1685 emit_move_insn (x17_reg
, op0
);
1686 emit_move_insn (x16_reg
, op1
);
1687 emit_insn (GEN_FCN (icode
) ());
1688 emit_move_insn (target
, x17_reg
);
1694 arg0
= CALL_EXPR_ARG (exp
, 0);
1695 op0
= force_reg (DFmode
, expand_normal (arg0
));
1697 target
= gen_reg_rtx (SImode
);
1699 target
= force_reg (SImode
, target
);
1700 emit_insn (GEN_FCN (CODE_FOR_aarch64_fjcvtzs
) (target
, op0
));
1703 case AARCH64_SIMD_BUILTIN_FCMLA_LANEQ0_V2SF
:
1704 case AARCH64_SIMD_BUILTIN_FCMLA_LANEQ90_V2SF
:
1705 case AARCH64_SIMD_BUILTIN_FCMLA_LANEQ180_V2SF
:
1706 case AARCH64_SIMD_BUILTIN_FCMLA_LANEQ270_V2SF
:
1707 case AARCH64_SIMD_BUILTIN_FCMLA_LANEQ0_V4HF
:
1708 case AARCH64_SIMD_BUILTIN_FCMLA_LANEQ90_V4HF
:
1709 case AARCH64_SIMD_BUILTIN_FCMLA_LANEQ180_V4HF
:
1710 case AARCH64_SIMD_BUILTIN_FCMLA_LANEQ270_V4HF
:
1711 return aarch64_expand_fcmla_builtin (exp
, target
, fcode
);
1714 if (fcode
>= AARCH64_SIMD_BUILTIN_BASE
&& fcode
<= AARCH64_SIMD_BUILTIN_MAX
)
1715 return aarch64_simd_expand_builtin (fcode
, exp
, target
);
1716 else if (fcode
>= AARCH64_CRC32_BUILTIN_BASE
&& fcode
<= AARCH64_CRC32_BUILTIN_MAX
)
1717 return aarch64_crc32_expand_builtin (fcode
, exp
, target
);
1719 if (fcode
== AARCH64_BUILTIN_RSQRT_DF
1720 || fcode
== AARCH64_BUILTIN_RSQRT_SF
1721 || fcode
== AARCH64_BUILTIN_RSQRT_V2DF
1722 || fcode
== AARCH64_BUILTIN_RSQRT_V2SF
1723 || fcode
== AARCH64_BUILTIN_RSQRT_V4SF
)
1724 return aarch64_expand_builtin_rsqrt (fcode
, exp
, target
);
1726 if (fcode
== AARCH64_TME_BUILTIN_TSTART
1727 || fcode
== AARCH64_TME_BUILTIN_TCOMMIT
1728 || fcode
== AARCH64_TME_BUILTIN_TTEST
1729 || fcode
== AARCH64_TME_BUILTIN_TCANCEL
)
1730 return aarch64_expand_builtin_tme (fcode
, exp
, target
);
1736 aarch64_builtin_vectorized_function (unsigned int fn
, tree type_out
,
1739 machine_mode in_mode
, out_mode
;
1740 unsigned HOST_WIDE_INT in_n
, out_n
;
1742 if (TREE_CODE (type_out
) != VECTOR_TYPE
1743 || TREE_CODE (type_in
) != VECTOR_TYPE
)
1746 out_mode
= TYPE_MODE (TREE_TYPE (type_out
));
1747 in_mode
= TYPE_MODE (TREE_TYPE (type_in
));
1748 if (!TYPE_VECTOR_SUBPARTS (type_out
).is_constant (&out_n
)
1749 || !TYPE_VECTOR_SUBPARTS (type_in
).is_constant (&in_n
))
1752 #undef AARCH64_CHECK_BUILTIN_MODE
1753 #define AARCH64_CHECK_BUILTIN_MODE(C, N) 1
1754 #define AARCH64_FIND_FRINT_VARIANT(N) \
1755 (AARCH64_CHECK_BUILTIN_MODE (2, D) \
1756 ? aarch64_builtin_decls[AARCH64_SIMD_BUILTIN_UNOP_##N##v2df] \
1757 : (AARCH64_CHECK_BUILTIN_MODE (4, S) \
1758 ? aarch64_builtin_decls[AARCH64_SIMD_BUILTIN_UNOP_##N##v4sf] \
1759 : (AARCH64_CHECK_BUILTIN_MODE (2, S) \
1760 ? aarch64_builtin_decls[AARCH64_SIMD_BUILTIN_UNOP_##N##v2sf] \
1764 #undef AARCH64_CHECK_BUILTIN_MODE
1765 #define AARCH64_CHECK_BUILTIN_MODE(C, N) \
1766 (out_mode == N##Fmode && out_n == C \
1767 && in_mode == N##Fmode && in_n == C)
1769 return AARCH64_FIND_FRINT_VARIANT (floor
);
1771 return AARCH64_FIND_FRINT_VARIANT (ceil
);
1773 return AARCH64_FIND_FRINT_VARIANT (btrunc
);
1775 return AARCH64_FIND_FRINT_VARIANT (round
);
1777 return AARCH64_FIND_FRINT_VARIANT (nearbyint
);
1779 return AARCH64_FIND_FRINT_VARIANT (sqrt
);
1780 #undef AARCH64_CHECK_BUILTIN_MODE
1781 #define AARCH64_CHECK_BUILTIN_MODE(C, N) \
1782 (out_mode == SImode && out_n == C \
1783 && in_mode == N##Imode && in_n == C)
1786 if (AARCH64_CHECK_BUILTIN_MODE (4, S
))
1787 return aarch64_builtin_decls
[AARCH64_SIMD_BUILTIN_UNOP_clzv4si
];
1792 if (AARCH64_CHECK_BUILTIN_MODE (2, S
))
1793 return aarch64_builtin_decls
[AARCH64_SIMD_BUILTIN_UNOP_ctzv2si
];
1794 else if (AARCH64_CHECK_BUILTIN_MODE (4, S
))
1795 return aarch64_builtin_decls
[AARCH64_SIMD_BUILTIN_UNOP_ctzv4si
];
1798 #undef AARCH64_CHECK_BUILTIN_MODE
1799 #define AARCH64_CHECK_BUILTIN_MODE(C, N) \
1800 (out_mode == N##Imode && out_n == C \
1801 && in_mode == N##Fmode && in_n == C)
1806 enum aarch64_builtins builtin
;
1807 if (AARCH64_CHECK_BUILTIN_MODE (2, D
))
1808 builtin
= AARCH64_SIMD_BUILTIN_UNOP_lfloorv2dfv2di
;
1809 else if (AARCH64_CHECK_BUILTIN_MODE (4, S
))
1810 builtin
= AARCH64_SIMD_BUILTIN_UNOP_lfloorv4sfv4si
;
1811 else if (AARCH64_CHECK_BUILTIN_MODE (2, S
))
1812 builtin
= AARCH64_SIMD_BUILTIN_UNOP_lfloorv2sfv2si
;
1816 return aarch64_builtin_decls
[builtin
];
1822 enum aarch64_builtins builtin
;
1823 if (AARCH64_CHECK_BUILTIN_MODE (2, D
))
1824 builtin
= AARCH64_SIMD_BUILTIN_UNOP_lceilv2dfv2di
;
1825 else if (AARCH64_CHECK_BUILTIN_MODE (4, S
))
1826 builtin
= AARCH64_SIMD_BUILTIN_UNOP_lceilv4sfv4si
;
1827 else if (AARCH64_CHECK_BUILTIN_MODE (2, S
))
1828 builtin
= AARCH64_SIMD_BUILTIN_UNOP_lceilv2sfv2si
;
1832 return aarch64_builtin_decls
[builtin
];
1838 enum aarch64_builtins builtin
;
1839 if (AARCH64_CHECK_BUILTIN_MODE (2, D
))
1840 builtin
= AARCH64_SIMD_BUILTIN_UNOP_lroundv2dfv2di
;
1841 else if (AARCH64_CHECK_BUILTIN_MODE (4, S
))
1842 builtin
= AARCH64_SIMD_BUILTIN_UNOP_lroundv4sfv4si
;
1843 else if (AARCH64_CHECK_BUILTIN_MODE (2, S
))
1844 builtin
= AARCH64_SIMD_BUILTIN_UNOP_lroundv2sfv2si
;
1848 return aarch64_builtin_decls
[builtin
];
1850 case CFN_BUILT_IN_BSWAP16
:
1851 #undef AARCH64_CHECK_BUILTIN_MODE
1852 #define AARCH64_CHECK_BUILTIN_MODE(C, N) \
1853 (out_mode == N##Imode && out_n == C \
1854 && in_mode == N##Imode && in_n == C)
1855 if (AARCH64_CHECK_BUILTIN_MODE (4, H
))
1856 return aarch64_builtin_decls
[AARCH64_SIMD_BUILTIN_UNOPU_bswapv4hi
];
1857 else if (AARCH64_CHECK_BUILTIN_MODE (8, H
))
1858 return aarch64_builtin_decls
[AARCH64_SIMD_BUILTIN_UNOPU_bswapv8hi
];
1861 case CFN_BUILT_IN_BSWAP32
:
1862 if (AARCH64_CHECK_BUILTIN_MODE (2, S
))
1863 return aarch64_builtin_decls
[AARCH64_SIMD_BUILTIN_UNOPU_bswapv2si
];
1864 else if (AARCH64_CHECK_BUILTIN_MODE (4, S
))
1865 return aarch64_builtin_decls
[AARCH64_SIMD_BUILTIN_UNOPU_bswapv4si
];
1868 case CFN_BUILT_IN_BSWAP64
:
1869 if (AARCH64_CHECK_BUILTIN_MODE (2, D
))
1870 return aarch64_builtin_decls
[AARCH64_SIMD_BUILTIN_UNOPU_bswapv2di
];
1880 /* Return builtin for reciprocal square root. */
1883 aarch64_builtin_rsqrt (unsigned int fn
)
1885 if (fn
== AARCH64_SIMD_BUILTIN_UNOP_sqrtv2df
)
1886 return aarch64_builtin_decls
[AARCH64_BUILTIN_RSQRT_V2DF
];
1887 if (fn
== AARCH64_SIMD_BUILTIN_UNOP_sqrtv2sf
)
1888 return aarch64_builtin_decls
[AARCH64_BUILTIN_RSQRT_V2SF
];
1889 if (fn
== AARCH64_SIMD_BUILTIN_UNOP_sqrtv4sf
)
1890 return aarch64_builtin_decls
[AARCH64_BUILTIN_RSQRT_V4SF
];
1895 #define VAR1(T, N, MAP, A) \
1896 case AARCH64_SIMD_BUILTIN_##T##_##N##A:
1899 aarch64_fold_builtin (tree fndecl
, int n_args ATTRIBUTE_UNUSED
, tree
*args
,
1900 bool ignore ATTRIBUTE_UNUSED
)
1902 int fcode
= DECL_MD_FUNCTION_CODE (fndecl
);
1903 tree type
= TREE_TYPE (TREE_TYPE (fndecl
));
1907 BUILTIN_VDQF (UNOP
, abs
, 2)
1908 return fold_build1 (ABS_EXPR
, type
, args
[0]);
1909 VAR1 (UNOP
, floatv2si
, 2, v2sf
)
1910 VAR1 (UNOP
, floatv4si
, 2, v4sf
)
1911 VAR1 (UNOP
, floatv2di
, 2, v2df
)
1912 return fold_build1 (FLOAT_EXPR
, type
, args
[0]);
1921 aarch64_gimple_fold_builtin (gimple_stmt_iterator
*gsi
)
1923 bool changed
= false;
1924 gimple
*stmt
= gsi_stmt (*gsi
);
1925 tree call
= gimple_call_fn (stmt
);
1927 gimple
*new_stmt
= NULL
;
1931 fndecl
= gimple_call_fndecl (stmt
);
1934 int fcode
= DECL_MD_FUNCTION_CODE (fndecl
);
1935 unsigned nargs
= gimple_call_num_args (stmt
);
1936 tree
*args
= (nargs
> 0
1937 ? gimple_call_arg_ptr (stmt
, 0)
1938 : &error_mark_node
);
1940 /* We use gimple's IFN_REDUC_(PLUS|MIN|MAX)s for float, signed int
1941 and unsigned int; it will distinguish according to the types of
1942 the arguments to the __builtin. */
1945 BUILTIN_VALL (UNOP
, reduc_plus_scal_
, 10)
1946 new_stmt
= gimple_build_call_internal (IFN_REDUC_PLUS
,
1948 gimple_call_set_lhs (new_stmt
, gimple_call_lhs (stmt
));
1950 BUILTIN_VDQIF (UNOP
, reduc_smax_scal_
, 10)
1951 BUILTIN_VDQ_BHSI (UNOPU
, reduc_umax_scal_
, 10)
1952 new_stmt
= gimple_build_call_internal (IFN_REDUC_MAX
,
1954 gimple_call_set_lhs (new_stmt
, gimple_call_lhs (stmt
));
1956 BUILTIN_VDQIF (UNOP
, reduc_smin_scal_
, 10)
1957 BUILTIN_VDQ_BHSI (UNOPU
, reduc_umin_scal_
, 10)
1958 new_stmt
= gimple_build_call_internal (IFN_REDUC_MIN
,
1960 gimple_call_set_lhs (new_stmt
, gimple_call_lhs (stmt
));
1962 BUILTIN_GPF (BINOP
, fmulx
, 0)
1964 gcc_assert (nargs
== 2);
1965 bool a0_cst_p
= TREE_CODE (args
[0]) == REAL_CST
;
1966 bool a1_cst_p
= TREE_CODE (args
[1]) == REAL_CST
;
1967 if (a0_cst_p
|| a1_cst_p
)
1969 if (a0_cst_p
&& a1_cst_p
)
1971 tree t0
= TREE_TYPE (args
[0]);
1972 real_value a0
= (TREE_REAL_CST (args
[0]));
1973 real_value a1
= (TREE_REAL_CST (args
[1]));
1974 if (real_equal (&a1
, &dconst0
))
1976 /* According to real_equal (), +0 equals -0. */
1977 if (real_equal (&a0
, &dconst0
) && real_isinf (&a1
))
1979 real_value res
= dconst2
;
1980 res
.sign
= a0
.sign
^ a1
.sign
;
1982 gimple_build_assign (gimple_call_lhs (stmt
),
1984 build_real (t0
, res
));
1988 gimple_build_assign (gimple_call_lhs (stmt
),
1992 else /* a0_cst_p ^ a1_cst_p. */
1994 real_value const_part
= a0_cst_p
1995 ? TREE_REAL_CST (args
[0]) : TREE_REAL_CST (args
[1]);
1996 if (!real_equal (&const_part
, &dconst0
)
1997 && !real_isinf (&const_part
))
1999 gimple_build_assign (gimple_call_lhs (stmt
),
2000 MULT_EXPR
, args
[0], args
[1]);
2005 gimple_set_vuse (new_stmt
, gimple_vuse (stmt
));
2006 gimple_set_vdef (new_stmt
, gimple_vdef (stmt
));
2018 gsi_replace (gsi
, new_stmt
, true);
2026 aarch64_atomic_assign_expand_fenv (tree
*hold
, tree
*clear
, tree
*update
)
2028 const unsigned AARCH64_FE_INVALID
= 1;
2029 const unsigned AARCH64_FE_DIVBYZERO
= 2;
2030 const unsigned AARCH64_FE_OVERFLOW
= 4;
2031 const unsigned AARCH64_FE_UNDERFLOW
= 8;
2032 const unsigned AARCH64_FE_INEXACT
= 16;
2033 const unsigned HOST_WIDE_INT AARCH64_FE_ALL_EXCEPT
= (AARCH64_FE_INVALID
2034 | AARCH64_FE_DIVBYZERO
2035 | AARCH64_FE_OVERFLOW
2036 | AARCH64_FE_UNDERFLOW
2037 | AARCH64_FE_INEXACT
);
2038 const unsigned HOST_WIDE_INT AARCH64_FE_EXCEPT_SHIFT
= 8;
2039 tree fenv_cr
, fenv_sr
, get_fpcr
, set_fpcr
, mask_cr
, mask_sr
;
2040 tree ld_fenv_cr
, ld_fenv_sr
, masked_fenv_cr
, masked_fenv_sr
, hold_fnclex_cr
;
2041 tree hold_fnclex_sr
, new_fenv_var
, reload_fenv
, restore_fnenv
, get_fpsr
, set_fpsr
;
2042 tree update_call
, atomic_feraiseexcept
, hold_fnclex
, masked_fenv
, ld_fenv
;
2044 /* Generate the equivalence of :
2045 unsigned int fenv_cr;
2046 fenv_cr = __builtin_aarch64_get_fpcr ();
2048 unsigned int fenv_sr;
2049 fenv_sr = __builtin_aarch64_get_fpsr ();
2051 Now set all exceptions to non-stop
2052 unsigned int mask_cr
2053 = ~(AARCH64_FE_ALL_EXCEPT << AARCH64_FE_EXCEPT_SHIFT);
2054 unsigned int masked_cr;
2055 masked_cr = fenv_cr & mask_cr;
2057 And clear all exception flags
2058 unsigned int maske_sr = ~AARCH64_FE_ALL_EXCEPT;
2059 unsigned int masked_cr;
2060 masked_sr = fenv_sr & mask_sr;
2062 __builtin_aarch64_set_cr (masked_cr);
2063 __builtin_aarch64_set_sr (masked_sr); */
2065 fenv_cr
= create_tmp_var_raw (unsigned_type_node
);
2066 fenv_sr
= create_tmp_var_raw (unsigned_type_node
);
2068 get_fpcr
= aarch64_builtin_decls
[AARCH64_BUILTIN_GET_FPCR
];
2069 set_fpcr
= aarch64_builtin_decls
[AARCH64_BUILTIN_SET_FPCR
];
2070 get_fpsr
= aarch64_builtin_decls
[AARCH64_BUILTIN_GET_FPSR
];
2071 set_fpsr
= aarch64_builtin_decls
[AARCH64_BUILTIN_SET_FPSR
];
2073 mask_cr
= build_int_cst (unsigned_type_node
,
2074 ~(AARCH64_FE_ALL_EXCEPT
<< AARCH64_FE_EXCEPT_SHIFT
));
2075 mask_sr
= build_int_cst (unsigned_type_node
,
2076 ~(AARCH64_FE_ALL_EXCEPT
));
2078 ld_fenv_cr
= build2 (MODIFY_EXPR
, unsigned_type_node
,
2079 fenv_cr
, build_call_expr (get_fpcr
, 0));
2080 ld_fenv_sr
= build2 (MODIFY_EXPR
, unsigned_type_node
,
2081 fenv_sr
, build_call_expr (get_fpsr
, 0));
2083 masked_fenv_cr
= build2 (BIT_AND_EXPR
, unsigned_type_node
, fenv_cr
, mask_cr
);
2084 masked_fenv_sr
= build2 (BIT_AND_EXPR
, unsigned_type_node
, fenv_sr
, mask_sr
);
2086 hold_fnclex_cr
= build_call_expr (set_fpcr
, 1, masked_fenv_cr
);
2087 hold_fnclex_sr
= build_call_expr (set_fpsr
, 1, masked_fenv_sr
);
2089 hold_fnclex
= build2 (COMPOUND_EXPR
, void_type_node
, hold_fnclex_cr
,
2091 masked_fenv
= build2 (COMPOUND_EXPR
, void_type_node
, masked_fenv_cr
,
2093 ld_fenv
= build2 (COMPOUND_EXPR
, void_type_node
, ld_fenv_cr
, ld_fenv_sr
);
2095 *hold
= build2 (COMPOUND_EXPR
, void_type_node
,
2096 build2 (COMPOUND_EXPR
, void_type_node
, masked_fenv
, ld_fenv
),
2099 /* Store the value of masked_fenv to clear the exceptions:
2100 __builtin_aarch64_set_fpsr (masked_fenv_sr); */
2102 *clear
= build_call_expr (set_fpsr
, 1, masked_fenv_sr
);
2104 /* Generate the equivalent of :
2105 unsigned int new_fenv_var;
2106 new_fenv_var = __builtin_aarch64_get_fpsr ();
2108 __builtin_aarch64_set_fpsr (fenv_sr);
2110 __atomic_feraiseexcept (new_fenv_var); */
2112 new_fenv_var
= create_tmp_var_raw (unsigned_type_node
);
2113 reload_fenv
= build2 (MODIFY_EXPR
, unsigned_type_node
,
2114 new_fenv_var
, build_call_expr (get_fpsr
, 0));
2115 restore_fnenv
= build_call_expr (set_fpsr
, 1, fenv_sr
);
2116 atomic_feraiseexcept
= builtin_decl_implicit (BUILT_IN_ATOMIC_FERAISEEXCEPT
);
2117 update_call
= build_call_expr (atomic_feraiseexcept
, 1,
2118 fold_convert (integer_type_node
, new_fenv_var
));
2119 *update
= build2 (COMPOUND_EXPR
, void_type_node
,
2120 build2 (COMPOUND_EXPR
, void_type_node
,
2121 reload_fenv
, restore_fnenv
), update_call
);
2125 #undef AARCH64_CHECK_BUILTIN_MODE
2126 #undef AARCH64_FIND_FRINT_VARIANT
2145 #include "gt-aarch64-builtins.h"