1 /* SLP - Pattern matcher on SLP trees
2 Copyright (C) 2020-2021 Free Software Foundation, Inc.
4 This file is part of GCC.
6 GCC is free software; you can redistribute it and/or modify it under
7 the terms of the GNU General Public License as published by the Free
8 Software Foundation; either version 3, or (at your option) any later
11 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
12 WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
16 You should have received a copy of the GNU General Public License
17 along with GCC; see the file COPYING3. If not see
18 <http://www.gnu.org/licenses/>. */
22 #include "coretypes.h"
28 #include "tree-pass.h"
30 #include "optabs-tree.h"
31 #include "insn-config.h"
32 #include "recog.h" /* FIXME: for insn_data */
33 #include "fold-const.h"
34 #include "stor-layout.h"
35 #include "gimple-iterator.h"
37 #include "tree-vectorizer.h"
38 #include "langhooks.h"
39 #include "gimple-walk.h"
41 #include "tree-vector-builder.h"
42 #include "vec-perm-indices.h"
43 #include "gimple-fold.h"
44 #include "internal-fn.h"
46 /* SLP Pattern matching mechanism.
48 This extension to the SLP vectorizer allows one to transform the generated SLP
49 tree based on any pattern. The difference between this and the normal vect
50 pattern matcher is that unlike the former, this matcher allows you to match
51 with instructions that do not belong to the same SSA dominator graph.
53 The only requirement that this pattern matcher has is that you are only
54 only allowed to either match an entire group or none.
56 The pattern matcher currently only allows you to perform replacements to
59 Once the patterns are matched it is one way, these cannot be undone. It is
60 currently not supported to match patterns recursively.
62 To add a new pattern, implement the vect_pattern class and add the type to
67 /*******************************************************************************
69 ******************************************************************************/
71 /* Default implementation of recognize that performs matching, validation and
72 replacement of nodes but that can be overriden if required. */
75 vect_pattern_validate_optab (internal_fn ifn
, slp_tree node
)
77 tree vectype
= SLP_TREE_VECTYPE (node
);
78 if (ifn
== IFN_LAST
|| !vectype
)
81 if (dump_enabled_p ())
82 dump_printf_loc (MSG_NOTE
, vect_location
,
83 "Found %s pattern in SLP tree\n",
84 internal_fn_name (ifn
));
86 if (direct_internal_fn_supported_p (ifn
, vectype
, OPTIMIZE_FOR_SPEED
))
88 if (dump_enabled_p ())
89 dump_printf_loc (MSG_NOTE
, vect_location
,
90 "Target supports %s vectorization with mode %T\n",
91 internal_fn_name (ifn
), vectype
);
95 if (dump_enabled_p ())
98 dump_printf_loc (MSG_NOTE
, vect_location
,
99 "Target does not support vector type for %T\n",
100 SLP_TREE_DEF_TYPE (node
));
102 dump_printf_loc (MSG_NOTE
, vect_location
,
103 "Target does not support %s for vector type "
104 "%T\n", internal_fn_name (ifn
), vectype
);
111 /*******************************************************************************
112 * General helper types
113 ******************************************************************************/
115 /* The COMPLEX_OPERATION enum denotes the possible pair of operations that can
116 be matched when looking for expressions that we are interested matching for
117 complex numbers addition and mla. */
119 typedef enum _complex_operation
: unsigned {
125 } complex_operation_t
;
127 /*******************************************************************************
128 * General helper functions
129 ******************************************************************************/
131 /* Helper function of linear_loads_p that checks to see if the load permutation
132 is sequential and in monotonically increasing order of loads with no gaps.
135 static inline complex_perm_kinds_t
136 is_linear_load_p (load_permutation_t loads
)
138 if (loads
.length() == 0)
142 complex_perm_kinds_t candidates
[4]
149 int valid_patterns
= 4;
150 FOR_EACH_VEC_ELT (loads
, i
, load
)
152 if (candidates
[0] != PERM_UNKNOWN
&& load
!= 1)
154 candidates
[0] = PERM_UNKNOWN
;
157 if (candidates
[1] != PERM_UNKNOWN
&& load
!= 0)
159 candidates
[1] = PERM_UNKNOWN
;
162 if (candidates
[2] != PERM_UNKNOWN
&& load
!= i
)
164 candidates
[2] = PERM_UNKNOWN
;
167 if (candidates
[3] != PERM_UNKNOWN
168 && load
!= (i
% 2 == 0 ? i
+ 1 : i
- 1))
170 candidates
[3] = PERM_UNKNOWN
;
174 if (valid_patterns
== 0)
178 for (i
= 0; i
< sizeof(candidates
); i
++)
179 if (candidates
[i
] != PERM_UNKNOWN
)
180 return candidates
[i
];
185 /* Combine complex_perm_kinds A and B into a new permute kind that describes the
186 resulting operation. */
188 static inline complex_perm_kinds_t
189 vect_merge_perms (complex_perm_kinds_t a
, complex_perm_kinds_t b
)
203 /* Check to see if all loads rooted in ROOT are linear. Linearity is
204 defined as having no gaps between values loaded. */
206 static complex_load_perm_t
207 linear_loads_p (slp_tree_to_load_perm_map_t
*perm_cache
, slp_tree root
)
210 return std::make_pair (PERM_UNKNOWN
, vNULL
);
213 complex_load_perm_t
*tmp
;
215 if ((tmp
= perm_cache
->get (root
)) != NULL
)
218 complex_load_perm_t retval
= std::make_pair (PERM_UNKNOWN
, vNULL
);
219 perm_cache
->put (root
, retval
);
221 /* If it's a load node, then just read the load permute. */
222 if (SLP_TREE_LOAD_PERMUTATION (root
).exists ())
224 retval
.first
= is_linear_load_p (SLP_TREE_LOAD_PERMUTATION (root
));
225 retval
.second
= SLP_TREE_LOAD_PERMUTATION (root
);
226 perm_cache
->put (root
, retval
);
229 else if (SLP_TREE_DEF_TYPE (root
) != vect_internal_def
)
231 retval
.first
= PERM_TOP
;
232 perm_cache
->put (root
, retval
);
236 auto_vec
<load_permutation_t
> all_loads
;
237 complex_perm_kinds_t kind
= PERM_TOP
;
240 FOR_EACH_VEC_ELT (SLP_TREE_CHILDREN (root
), i
, child
)
242 complex_load_perm_t res
= linear_loads_p (perm_cache
, child
);
243 kind
= vect_merge_perms (kind
, res
.first
);
244 /* Unknown and Top are not valid on blends as they produce no permute. */
246 if (kind
== PERM_UNKNOWN
|| kind
== PERM_TOP
)
248 all_loads
.safe_push (res
.second
);
251 if (SLP_TREE_LANE_PERMUTATION (root
).exists ())
253 lane_permutation_t perm
= SLP_TREE_LANE_PERMUTATION (root
);
254 load_permutation_t nloads
;
255 nloads
.create (SLP_TREE_LANES (root
));
256 nloads
.quick_grow (SLP_TREE_LANES (root
));
257 for (i
= 0; i
< SLP_TREE_LANES (root
); i
++)
258 nloads
[i
] = all_loads
[perm
[i
].first
][perm
[i
].second
];
261 retval
.second
= nloads
;
266 retval
.second
= all_loads
[0];
269 perm_cache
->put (root
, retval
);
274 /* This function attempts to make a node rooted in NODE is linear. If the node
275 if already linear than the node itself is returned in RESULT.
277 If the node is not linear then a new VEC_PERM_EXPR node is created with a
278 lane permute that when applied will make the node linear. If such a
279 permute cannot be created then FALSE is returned from the function.
281 Here linearity is defined as having a sequential, monotically increasing
282 load position inside the load permute generated by the loads reachable from
286 vect_build_swap_evenodd_node (slp_tree node
)
288 /* Attempt to linearise the permute. */
289 vec
<std::pair
<unsigned, unsigned> > zipped
;
290 zipped
.create (SLP_TREE_LANES (node
));
292 for (unsigned x
= 0; x
< SLP_TREE_LANES (node
); x
+=2)
294 zipped
.quick_push (std::make_pair (0, x
+1));
295 zipped
.quick_push (std::make_pair (0, x
));
298 /* Create the new permute node and store it instead. */
299 slp_tree vnode
= vect_create_new_slp_node (1, VEC_PERM_EXPR
);
300 SLP_TREE_LANE_PERMUTATION (vnode
) = zipped
;
301 SLP_TREE_VECTYPE (vnode
) = SLP_TREE_VECTYPE (node
);
302 SLP_TREE_CHILDREN (vnode
).quick_push (node
);
303 SLP_TREE_REF_COUNT (vnode
) = 1;
304 SLP_TREE_LANES (vnode
) = SLP_TREE_LANES (node
);
305 SLP_TREE_REPRESENTATIVE (vnode
) = SLP_TREE_REPRESENTATIVE (node
);
306 SLP_TREE_REF_COUNT (node
)++;
310 /* Checks to see of the expression represented by NODE is a gimple assign with
314 vect_match_expression_p (slp_tree node
, tree_code code
)
317 || !SLP_TREE_REPRESENTATIVE (node
))
320 gimple
* expr
= STMT_VINFO_STMT (SLP_TREE_REPRESENTATIVE (node
));
321 if (!is_gimple_assign (expr
)
322 || gimple_assign_rhs_code (expr
) != code
)
328 /* Checks to see if the expression represented by NODE is a call to the internal
332 vect_match_call_p (slp_tree node
, internal_fn fn
)
335 || !SLP_TREE_REPRESENTATIVE (node
))
338 gimple
* expr
= STMT_VINFO_STMT (SLP_TREE_REPRESENTATIVE (node
));
340 || !gimple_call_internal_p (expr
, fn
))
346 /* Check if the given lane permute in PERMUTES matches an alternating sequence
347 of {even odd even odd ...}. This to account for unrolled loops. Further
348 mode there resulting permute must be linear. */
351 vect_check_evenodd_blend (lane_permutation_t
&permutes
,
352 unsigned even
, unsigned odd
)
354 if (permutes
.length () == 0)
357 unsigned val
[2] = {even
, odd
};
359 for (unsigned i
= 0; i
< permutes
.length (); i
++)
360 if (permutes
[i
].first
!= val
[i
% 2]
361 || permutes
[i
].second
!= seed
++)
367 /* This function will match the two gimple expressions representing NODE1 and
368 NODE2 in parallel and returns the pair operation that represents the two
369 expressions in the two statements.
371 If match is successful then the corresponding complex_operation is
372 returned and the arguments to the two matched operations are returned in OPS.
374 If TWO_OPERANDS it is expected that the LANES of the parent VEC_PERM select
375 from the two nodes alternatingly.
377 If unsuccessful then CMPLX_NONE is returned and OPS is untouched.
379 e.g. the following gimple statements
381 stmt 0 _39 = _37 + _12;
382 stmt 1 _6 = _38 - _36;
384 will return PLUS_MINUS along with OPS containing {_37, _12, _38, _36}.
387 static complex_operation_t
388 vect_detect_pair_op (slp_tree node1
, slp_tree node2
, lane_permutation_t
&lanes
,
389 bool two_operands
= true, vec
<slp_tree
> *ops
= NULL
)
391 complex_operation_t result
= CMPLX_NONE
;
393 if (vect_match_expression_p (node1
, MINUS_EXPR
)
394 && vect_match_expression_p (node2
, PLUS_EXPR
)
395 && (!two_operands
|| vect_check_evenodd_blend (lanes
, 0, 1)))
397 else if (vect_match_expression_p (node1
, PLUS_EXPR
)
398 && vect_match_expression_p (node2
, MINUS_EXPR
)
399 && (!two_operands
|| vect_check_evenodd_blend (lanes
, 0, 1)))
401 else if (vect_match_expression_p (node1
, PLUS_EXPR
)
402 && vect_match_expression_p (node2
, PLUS_EXPR
))
404 else if (vect_match_expression_p (node1
, MULT_EXPR
)
405 && vect_match_expression_p (node2
, MULT_EXPR
))
408 if (result
!= CMPLX_NONE
&& ops
!= NULL
)
411 ops
->quick_push (node1
);
412 ops
->quick_push (node2
);
417 /* Overload of vect_detect_pair_op that matches against the representative
418 statements in the children of NODE. It is expected that NODE has exactly
419 two children and when TWO_OPERANDS then NODE must be a VEC_PERM. */
421 static complex_operation_t
422 vect_detect_pair_op (slp_tree node
, bool two_operands
= true,
423 vec
<slp_tree
> *ops
= NULL
)
425 if (!two_operands
&& SLP_TREE_CODE (node
) == VEC_PERM_EXPR
)
428 if (SLP_TREE_CHILDREN (node
).length () != 2)
431 vec
<slp_tree
> children
= SLP_TREE_CHILDREN (node
);
432 lane_permutation_t
&lanes
= SLP_TREE_LANE_PERMUTATION (node
);
434 return vect_detect_pair_op (children
[0], children
[1], lanes
, two_operands
,
438 /*******************************************************************************
439 * complex_pattern class
440 ******************************************************************************/
442 /* SLP Complex Numbers pattern matching.
444 As an example, the following simple loop:
446 double a[restrict N]; double b[restrict N]; double c[restrict N];
448 for (int i=0; i < N; i+=2)
450 c[i] = a[i] - b[i+1];
451 c[i+1] = a[i+1] + b[i];
454 which represents a complex addition on with a rotation of 90* around the
455 argand plane. i.e. if `a` and `b` were complex numbers then this would be the
456 same as `a + (b * I)`.
458 Here the expressions for `c[i]` and `c[i+1]` are independent but have to be
459 both recognized in order for the pattern to work. As an SLP tree this is
462 +--------------------------------+
463 | stmt 0 *_9 = _10; |
464 | stmt 1 *_15 = _16; |
465 +--------------------------------+
469 +--------------------------------+
470 | stmt 0 _10 = _4 - _8; |
471 | stmt 1 _16 = _12 + _14; |
472 | lane permutation { 0[0] 1[1] } |
473 +--------------------------------+
479 +-----| { } |<-----+ +----->| { } --------+
480 | | | +------------------| | |
481 | +-----+ | +-----+ |
484 | +------|------------------+ |
487 +--------------------------+ +--------------------------------+
488 | stmt 0 _8 = *_7; | | stmt 0 _4 = *_3; |
489 | stmt 1 _14 = *_13; | | stmt 1 _12 = *_11; |
490 | load permutation { 1 0 } | | load permutation { 0 1 } |
491 +--------------------------+ +--------------------------------+
493 The pattern matcher allows you to replace both statements 0 and 1 or none at
494 all. Because this operation is a two operands operation the actual nodes
495 being replaced are those in the { } nodes. The actual scalar statements
496 themselves are not replaced or used during the matching but instead the
497 SLP_TREE_REPRESENTATIVE statements are inspected. You are also allowed to
498 replace and match on any number of nodes.
500 Because the pattern matcher matches on the representative statement for the
501 SLP node the case of two_operators it allows you to match the children of the
502 node. This is done using the method `recognize ()`.
506 /* The complex_pattern class contains common code for pattern matchers that work
507 on complex numbers. These provide functionality to allow de-construction and
508 validation of sequences depicting/transforming REAL and IMAG pairs. */
510 class complex_pattern
: public vect_pattern
513 auto_vec
<slp_tree
> m_workset
;
514 complex_pattern (slp_tree
*node
, vec
<slp_tree
> *m_ops
, internal_fn ifn
)
515 : vect_pattern (node
, m_ops
, ifn
)
517 this->m_workset
.safe_push (*node
);
521 void build (vec_info
*);
524 matches (complex_operation_t op
, slp_tree_to_load_perm_map_t
*, slp_tree
*,
528 /* Create a replacement pattern statement for each node in m_node and inserts
529 the new statement into m_node as the new representative statement. The old
530 statement is marked as being in a pattern defined by the new statement. The
531 statement is created as call to internal function IFN with m_num_args
534 Futhermore the new pattern is also added to the vectorization information
535 structure VINFO and the old statement STMT_INFO is marked as unused while
536 the new statement is marked as used and the number of SLP uses of the new
537 statement is incremented.
539 The newly created SLP nodes are marked as SLP only and will be dissolved
542 The newly created gimple call is returned and the BB remains unchanged.
544 This default method is designed to only match against simple operands where
545 all the input and output types are the same.
549 complex_pattern::build (vec_info
*vinfo
)
551 stmt_vec_info stmt_info
;
554 args
.create (this->m_num_args
);
555 args
.quick_grow_cleared (this->m_num_args
);
558 stmt_vec_info call_stmt_info
;
559 gcall
*call_stmt
= NULL
;
561 /* Now modify the nodes themselves. */
562 FOR_EACH_VEC_ELT (this->m_workset
, ix
, node
)
564 /* Calculate the location of the statement in NODE to replace. */
565 stmt_info
= SLP_TREE_REPRESENTATIVE (node
);
566 gimple
* old_stmt
= STMT_VINFO_STMT (stmt_info
);
567 tree lhs_old_stmt
= gimple_get_lhs (old_stmt
);
568 tree type
= TREE_TYPE (lhs_old_stmt
);
570 /* Create the argument set for use by gimple_build_call_internal_vec. */
571 for (unsigned i
= 0; i
< this->m_num_args
; i
++)
572 args
[i
] = lhs_old_stmt
;
574 /* Create the new pattern statements. */
575 call_stmt
= gimple_build_call_internal_vec (this->m_ifn
, args
);
576 tree var
= make_temp_ssa_name (type
, call_stmt
, "slp_patt");
577 gimple_call_set_lhs (call_stmt
, var
);
578 gimple_set_location (call_stmt
, gimple_location (old_stmt
));
579 gimple_call_set_nothrow (call_stmt
, true);
581 /* Adjust the book-keeping for the new and old statements for use during
582 SLP. This is required to get the right VF and statement during SLP
583 analysis. These changes are created after relevancy has been set for
584 the nodes as such we need to manually update them. Any changes will be
585 undone if SLP is cancelled. */
587 = vinfo
->add_pattern_stmt (call_stmt
, stmt_info
);
589 /* Make sure to mark the representative statement pure_slp and
591 STMT_VINFO_RELEVANT (call_stmt_info
) = vect_used_in_scope
;
592 STMT_SLP_TYPE (call_stmt_info
) = pure_slp
;
594 /* add_pattern_stmt can't be done in vect_mark_pattern_stmts because
595 the non-SLP pattern matchers already have added the statement to VINFO
596 by the time it is called. Some of them need to modify the returned
597 stmt_info. vect_mark_pattern_stmts is called by recog_pattern and it
598 would increase the size of each pattern with boilerplate code to make
600 vect_mark_pattern_stmts (vinfo
, stmt_info
, call_stmt
,
601 SLP_TREE_VECTYPE (node
));
602 STMT_VINFO_SLP_VECT_ONLY (call_stmt_info
) = true;
604 /* Since we are replacing all the statements in the group with the same
605 thing it doesn't really matter. So just set it every time a new stmt
607 SLP_TREE_REPRESENTATIVE (node
) = call_stmt_info
;
608 SLP_TREE_LANE_PERMUTATION (node
).release ();
609 SLP_TREE_CODE (node
) = CALL_EXPR
;
613 /*******************************************************************************
614 * complex_add_pattern class
615 ******************************************************************************/
617 class complex_add_pattern
: public complex_pattern
620 complex_add_pattern (slp_tree
*node
, vec
<slp_tree
> *m_ops
, internal_fn ifn
)
621 : complex_pattern (node
, m_ops
, ifn
)
623 this->m_num_args
= 2;
627 void build (vec_info
*);
629 matches (complex_operation_t op
, slp_tree_to_load_perm_map_t
*, slp_tree
*,
633 recognize (slp_tree_to_load_perm_map_t
*, slp_tree
*);
636 mkInstance (slp_tree
*node
, vec
<slp_tree
> *m_ops
, internal_fn ifn
)
638 return new complex_add_pattern (node
, m_ops
, ifn
);
642 /* Perform a replacement of the detected complex add pattern with the new
643 instruction sequences. */
646 complex_add_pattern::build (vec_info
*vinfo
)
648 SLP_TREE_CHILDREN (*this->m_node
).reserve_exact (2);
650 slp_tree node
= this->m_ops
[0];
651 vec
<slp_tree
> children
= SLP_TREE_CHILDREN (node
);
653 /* First re-arrange the children. */
654 SLP_TREE_CHILDREN (*this->m_node
)[0] = children
[0];
655 SLP_TREE_CHILDREN (*this->m_node
)[1] =
656 vect_build_swap_evenodd_node (children
[1]);
658 SLP_TREE_REF_COUNT (SLP_TREE_CHILDREN (*this->m_node
)[0])++;
659 SLP_TREE_REF_COUNT (SLP_TREE_CHILDREN (*this->m_node
)[1])++;
660 vect_free_slp_tree (this->m_ops
[0]);
661 vect_free_slp_tree (this->m_ops
[1]);
663 complex_pattern::build (vinfo
);
666 /* Pattern matcher for trying to match complex addition pattern in SLP tree.
668 If no match is found then IFN is set to IFN_LAST.
669 This function matches the patterns shaped as:
671 c[i] = a[i] - b[i+1];
672 c[i+1] = a[i+1] + b[i];
674 If a match occurred then TRUE is returned, else FALSE. The initial match is
675 expected to be in OP1 and the initial match operands in args0. */
678 complex_add_pattern::matches (complex_operation_t op
,
679 slp_tree_to_load_perm_map_t
*perm_cache
,
680 slp_tree
*node
, vec
<slp_tree
> *ops
)
682 internal_fn ifn
= IFN_LAST
;
684 /* Find the two components. Rotation in the complex plane will modify
692 Rotation 0 and 180 can be handled by normal SIMD code, so we don't need
693 to care about them here. */
694 if (op
== MINUS_PLUS
)
695 ifn
= IFN_COMPLEX_ADD_ROT90
;
696 else if (op
== PLUS_MINUS
)
697 ifn
= IFN_COMPLEX_ADD_ROT270
;
701 /* verify that there is a permute, otherwise this isn't a pattern we
703 gcc_assert (ops
->length () == 2);
705 vec
<slp_tree
> children
= SLP_TREE_CHILDREN ((*ops
)[0]);
707 /* First node must be unpermuted. */
708 if (linear_loads_p (perm_cache
, children
[0]).first
!= PERM_EVENODD
)
711 /* Second node must be permuted. */
712 if (linear_loads_p (perm_cache
, children
[1]).first
!= PERM_ODDEVEN
)
715 if (!vect_pattern_validate_optab (ifn
, *node
))
721 /* Attempt to recognize a complex add pattern. */
724 complex_add_pattern::recognize (slp_tree_to_load_perm_map_t
*perm_cache
,
727 auto_vec
<slp_tree
> ops
;
728 complex_operation_t op
729 = vect_detect_pair_op (*node
, true, &ops
);
731 = complex_add_pattern::matches (op
, perm_cache
, node
, &ops
);
735 return new complex_add_pattern (node
, &ops
, ifn
);
738 /*******************************************************************************
739 * complex_mul_pattern
740 ******************************************************************************/
742 /* Helper function of that looks for a match in the CHILDth child of NODE. The
743 child used is stored in RES.
745 If the match is successful then ARGS will contain the operands matched
746 and the complex_operation_t type is returned. If match is not successful
747 then CMPLX_NONE is returned and ARGS is left unmodified. */
749 static inline complex_operation_t
750 vect_match_call_complex_mla (slp_tree node
, unsigned child
,
751 vec
<slp_tree
> *args
= NULL
, slp_tree
*res
= NULL
)
753 gcc_assert (child
< SLP_TREE_CHILDREN (node
).length ());
755 slp_tree data
= SLP_TREE_CHILDREN (node
)[child
];
760 return vect_detect_pair_op (data
, false, args
);
763 /* Check to see if either of the trees in ARGS are a NEGATE_EXPR. If the first
764 child (args[0]) is a NEGATE_EXPR then NEG_FIRST_P is set to TRUE.
766 If a negate is found then the values in ARGS are reordered such that the
767 negate node is always the second one and the entry is replaced by the child
768 of the negate node. */
771 vect_normalize_conj_loc (vec
<slp_tree
> args
, bool *neg_first_p
= NULL
)
773 gcc_assert (args
.length () == 2);
774 bool neg_found
= false;
776 if (vect_match_expression_p (args
[0], NEGATE_EXPR
))
778 std::swap (args
[0], args
[1]);
783 else if (vect_match_expression_p (args
[1], NEGATE_EXPR
))
787 *neg_first_p
= false;
791 args
[1] = SLP_TREE_CHILDREN (args
[1])[0];
796 /* Helper function to check if PERM is KIND or PERM_TOP. */
799 is_eq_or_top (complex_load_perm_t perm
, complex_perm_kinds_t kind
)
801 return perm
.first
== kind
|| perm
.first
== PERM_TOP
;
804 /* Helper function that checks to see if LEFT_OP and RIGHT_OP are both MULT_EXPR
805 nodes but also that they represent an operation that is either a complex
806 multiplication or a complex multiplication by conjugated value.
808 Of the negation is expected to be in the first half of the tree (As required
809 by an FMS pattern) then NEG_FIRST is true. If the operation is a conjugate
810 operation then CONJ_FIRST_OPERAND is set to indicate whether the first or
811 second operand contains the conjugate operation. */
814 vect_validate_multiplication (slp_tree_to_load_perm_map_t
*perm_cache
,
815 vec
<slp_tree
> left_op
, vec
<slp_tree
> right_op
,
816 bool neg_first
, bool *conj_first_operand
,
819 /* The presence of a negation indicates that we have either a conjugate or a
820 rotation. We need to distinguish which one. */
821 *conj_first_operand
= false;
822 complex_perm_kinds_t kind
;
824 /* Complex conjugates have the negation on the imaginary part of the
825 number where rotations affect the real component. So check if the
826 negation is on a dup of lane 1. */
829 /* Canonicalization for fms is not consistent. So have to test both
830 variants to be sure. This needs to be fixed in the mid-end so
831 this part can be simpler. */
832 kind
= linear_loads_p (perm_cache
, right_op
[0]).first
;
833 if (!((is_eq_or_top (linear_loads_p (perm_cache
, right_op
[0]), PERM_ODDODD
)
834 && is_eq_or_top (linear_loads_p (perm_cache
, right_op
[1]),
836 || (kind
== PERM_ODDEVEN
837 && is_eq_or_top (linear_loads_p (perm_cache
, right_op
[1]),
843 if (linear_loads_p (perm_cache
, right_op
[1]).first
!= PERM_ODDODD
844 && !is_eq_or_top (linear_loads_p (perm_cache
, right_op
[0]),
849 /* Deal with differences in indexes. */
850 int index1
= fms
? 1 : 0;
851 int index2
= fms
? 0 : 1;
853 /* Check if the conjugate is on the second first or second operand. The
854 order of the node with the conjugate value determines this, and the dup
855 node must be one of lane 0 of the same DR as the neg node. */
856 kind
= linear_loads_p (perm_cache
, left_op
[index1
]).first
;
857 if (kind
== PERM_TOP
)
859 if (linear_loads_p (perm_cache
, left_op
[index2
]).first
== PERM_EVENODD
)
862 else if (kind
== PERM_EVENODD
)
864 if ((kind
= linear_loads_p (perm_cache
, left_op
[index2
]).first
) == PERM_EVENODD
)
869 *conj_first_operand
= true;
873 if (kind
!= PERM_EVENEVEN
)
879 /* Helper function to help distinguish between a conjugate and a rotation in a
880 complex multiplication. The operations have similar shapes but the order of
881 the load permutes are different. This function returns TRUE when the order
882 is consistent with a multiplication or multiplication by conjugated
883 operand but returns FALSE if it's a multiplication by rotated operand. */
886 vect_validate_multiplication (slp_tree_to_load_perm_map_t
*perm_cache
,
887 vec
<slp_tree
> op
, complex_perm_kinds_t permKind
)
889 /* The left node is the more common case, test it first. */
890 if (!is_eq_or_top (linear_loads_p (perm_cache
, op
[0]), permKind
))
892 if (!is_eq_or_top (linear_loads_p (perm_cache
, op
[1]), permKind
))
898 /* This function combines two nodes containing only even and only odd lanes
899 together into a single node which contains the nodes in even/odd order
900 by using a lane permute.
902 The lanes in EVEN and ODD are duplicated 2 times inside the vectors.
903 So for a lanes = 4 EVEN contains {EVEN1, EVEN1, EVEN2, EVEN2}.
905 The tree REPRESENTATION is taken from the supplied REP along with the
906 vectype which must be the same between all three nodes.
910 vect_build_combine_node (slp_tree even
, slp_tree odd
, slp_tree rep
)
912 vec
<std::pair
<unsigned, unsigned> > perm
;
913 perm
.create (SLP_TREE_LANES (rep
));
915 for (unsigned x
= 0; x
< SLP_TREE_LANES (rep
); x
+=2)
917 perm
.quick_push (std::make_pair (0, x
));
918 perm
.quick_push (std::make_pair (1, x
+1));
921 slp_tree vnode
= vect_create_new_slp_node (2, SLP_TREE_CODE (even
));
922 SLP_TREE_CODE (vnode
) = VEC_PERM_EXPR
;
923 SLP_TREE_LANE_PERMUTATION (vnode
) = perm
;
925 SLP_TREE_CHILDREN (vnode
).create (2);
926 SLP_TREE_CHILDREN (vnode
).quick_push (even
);
927 SLP_TREE_CHILDREN (vnode
).quick_push (odd
);
928 SLP_TREE_REF_COUNT (even
)++;
929 SLP_TREE_REF_COUNT (odd
)++;
930 SLP_TREE_REF_COUNT (vnode
) = 1;
932 SLP_TREE_LANES (vnode
) = SLP_TREE_LANES (rep
);
933 gcc_assert (perm
.length () == SLP_TREE_LANES (vnode
));
934 /* Representation is set to that of the current node as the vectorizer
935 can't deal with VEC_PERMs with no representation, as would be the
936 case with invariants. */
937 SLP_TREE_REPRESENTATIVE (vnode
) = SLP_TREE_REPRESENTATIVE (rep
);
938 SLP_TREE_VECTYPE (vnode
) = SLP_TREE_VECTYPE (rep
);
942 class complex_mul_pattern
: public complex_pattern
945 complex_mul_pattern (slp_tree
*node
, vec
<slp_tree
> *m_ops
, internal_fn ifn
)
946 : complex_pattern (node
, m_ops
, ifn
)
948 this->m_num_args
= 2;
952 void build (vec_info
*);
954 matches (complex_operation_t op
, slp_tree_to_load_perm_map_t
*, slp_tree
*,
958 recognize (slp_tree_to_load_perm_map_t
*, slp_tree
*);
961 mkInstance (slp_tree
*node
, vec
<slp_tree
> *m_ops
, internal_fn ifn
)
963 return new complex_mul_pattern (node
, m_ops
, ifn
);
968 /* Pattern matcher for trying to match complex multiply pattern in SLP tree
969 If the operation matches then IFN is set to the operation it matched
970 and the arguments to the two replacement statements are put in m_ops.
972 If no match is found then IFN is set to IFN_LAST and m_ops is unchanged.
974 This function matches the patterns shaped as:
976 double ax = (b[i+1] * a[i]);
977 double bx = (a[i+1] * b[i]);
980 c[i+1] = c[i+1] + bx;
982 If a match occurred then TRUE is returned, else FALSE. The initial match is
983 expected to be in OP1 and the initial match operands in args0. */
986 complex_mul_pattern::matches (complex_operation_t op
,
987 slp_tree_to_load_perm_map_t
*perm_cache
,
988 slp_tree
*node
, vec
<slp_tree
> *ops
)
990 internal_fn ifn
= IFN_LAST
;
992 if (op
!= MINUS_PLUS
)
995 slp_tree root
= *node
;
996 /* First two nodes must be a multiply. */
997 auto_vec
<slp_tree
> muls
;
998 if (vect_match_call_complex_mla (root
, 0) != MULT_MULT
999 || vect_match_call_complex_mla (root
, 1, &muls
) != MULT_MULT
)
1002 /* Now operand2+4 may lead to another expression. */
1003 auto_vec
<slp_tree
> left_op
, right_op
;
1004 left_op
.safe_splice (SLP_TREE_CHILDREN (muls
[0]));
1005 right_op
.safe_splice (SLP_TREE_CHILDREN (muls
[1]));
1007 if (linear_loads_p (perm_cache
, left_op
[1]).first
== PERM_ODDEVEN
)
1010 bool neg_first
= false;
1011 bool conj_first_operand
= false;
1012 bool is_neg
= vect_normalize_conj_loc (right_op
, &neg_first
);
1016 /* A multiplication needs to multiply agains the real pair, otherwise
1017 the pattern matches that of FMS. */
1018 if (!vect_validate_multiplication (perm_cache
, left_op
, PERM_EVENEVEN
)
1019 || vect_normalize_conj_loc (left_op
))
1021 ifn
= IFN_COMPLEX_MUL
;
1025 if (!vect_validate_multiplication (perm_cache
, left_op
, right_op
,
1026 neg_first
, &conj_first_operand
,
1030 ifn
= IFN_COMPLEX_MUL_CONJ
;
1033 if (!vect_pattern_validate_optab (ifn
, *node
))
1039 complex_perm_kinds_t kind
= linear_loads_p (perm_cache
, left_op
[0]).first
;
1040 if (kind
== PERM_EVENODD
)
1042 ops
->quick_push (left_op
[1]);
1043 ops
->quick_push (right_op
[1]);
1044 ops
->quick_push (left_op
[0]);
1046 else if (kind
== PERM_TOP
)
1048 ops
->quick_push (left_op
[1]);
1049 ops
->quick_push (right_op
[1]);
1050 ops
->quick_push (left_op
[0]);
1052 else if (kind
== PERM_EVENEVEN
&& !conj_first_operand
)
1054 ops
->quick_push (left_op
[0]);
1055 ops
->quick_push (right_op
[0]);
1056 ops
->quick_push (left_op
[1]);
1060 ops
->quick_push (left_op
[0]);
1061 ops
->quick_push (right_op
[1]);
1062 ops
->quick_push (left_op
[1]);
1068 /* Attempt to recognize a complex mul pattern. */
1071 complex_mul_pattern::recognize (slp_tree_to_load_perm_map_t
*perm_cache
,
1074 auto_vec
<slp_tree
> ops
;
1075 complex_operation_t op
1076 = vect_detect_pair_op (*node
, true, &ops
);
1078 = complex_mul_pattern::matches (op
, perm_cache
, node
, &ops
);
1079 if (ifn
== IFN_LAST
)
1082 return new complex_mul_pattern (node
, &ops
, ifn
);
1085 /* Perform a replacement of the detected complex mul pattern with the new
1086 instruction sequences. */
1089 complex_mul_pattern::build (vec_info
*vinfo
)
1093 FOR_EACH_VEC_ELT (SLP_TREE_CHILDREN (*this->m_node
), i
, node
)
1094 vect_free_slp_tree (node
);
1096 /* First re-arrange the children. */
1097 SLP_TREE_CHILDREN (*this->m_node
).reserve_exact (2);
1098 SLP_TREE_CHILDREN (*this->m_node
)[0] = this->m_ops
[2];
1099 SLP_TREE_CHILDREN (*this->m_node
)[1] =
1100 vect_build_combine_node (this->m_ops
[0], this->m_ops
[1], *this->m_node
);
1101 SLP_TREE_REF_COUNT (this->m_ops
[2])++;
1103 /* And then rewrite the node itself. */
1104 complex_pattern::build (vinfo
);
1107 /*******************************************************************************
1108 * complex_fma_pattern class
1109 ******************************************************************************/
1111 class complex_fma_pattern
: public complex_pattern
1114 complex_fma_pattern (slp_tree
*node
, vec
<slp_tree
> *m_ops
, internal_fn ifn
)
1115 : complex_pattern (node
, m_ops
, ifn
)
1117 this->m_num_args
= 3;
1121 void build (vec_info
*);
1123 matches (complex_operation_t op
, slp_tree_to_load_perm_map_t
*, slp_tree
*,
1126 static vect_pattern
*
1127 recognize (slp_tree_to_load_perm_map_t
*, slp_tree
*);
1129 static vect_pattern
*
1130 mkInstance (slp_tree
*node
, vec
<slp_tree
> *m_ops
, internal_fn ifn
)
1132 return new complex_fma_pattern (node
, m_ops
, ifn
);
1136 /* Helper function to "reset" a previously matched node and undo the changes
1137 made enough so that the node is treated as an irrelevant node. */
1140 vect_slp_reset_pattern (slp_tree node
)
1142 stmt_vec_info stmt_info
= vect_orig_stmt (SLP_TREE_REPRESENTATIVE (node
));
1143 STMT_VINFO_IN_PATTERN_P (stmt_info
) = false;
1144 STMT_SLP_TYPE (stmt_info
) = pure_slp
;
1145 SLP_TREE_REPRESENTATIVE (node
) = stmt_info
;
1148 /* Pattern matcher for trying to match complex multiply and accumulate
1149 and multiply and subtract patterns in SLP tree.
1150 If the operation matches then IFN is set to the operation it matched and
1151 the arguments to the two replacement statements are put in m_ops.
1153 If no match is found then IFN is set to IFN_LAST and m_ops is unchanged.
1155 This function matches the patterns shaped as:
1157 double ax = (b[i+1] * a[i]) + (b[i] * a[i]);
1158 double bx = (a[i+1] * b[i]) - (a[i+1] * b[i+1]);
1161 c[i+1] = c[i+1] + bx;
1163 If a match occurred then TRUE is returned, else FALSE. The match is
1164 performed after COMPLEX_MUL which would have done the majority of the work.
1165 This function merely matches an ADD with a COMPLEX_MUL IFN. The initial
1166 match is expected to be in OP1 and the initial match operands in args0. */
1169 complex_fma_pattern::matches (complex_operation_t op
,
1170 slp_tree_to_load_perm_map_t
* /* perm_cache */,
1171 slp_tree
*ref_node
, vec
<slp_tree
> *ops
)
1173 internal_fn ifn
= IFN_LAST
;
1175 /* Find the two components. We match Complex MUL first which reduces the
1176 amount of work this pattern has to do. After that we just match the
1177 head node and we're done.:
1181 We need to ignore the two_operands nodes that may also match.
1182 For that we can check if they have any scalar statements and also
1183 check that it's not a permute node as we're looking for a normal
1184 PLUS_EXPR operation. */
1185 if (op
!= CMPLX_NONE
)
1188 /* Find the two components. We match Complex MUL first which reduces the
1189 amount of work this pattern has to do. After that we just match the
1190 head node and we're done.:
1192 * FMA: + + on a non-two_operands node. */
1193 slp_tree vnode
= *ref_node
;
1194 if (SLP_TREE_LANE_PERMUTATION (vnode
).exists ()
1195 || !SLP_TREE_CHILDREN (vnode
).exists ()
1196 || !vect_match_expression_p (vnode
, PLUS_EXPR
))
1199 slp_tree node
= SLP_TREE_CHILDREN (vnode
)[1];
1201 if (vect_match_call_p (node
, IFN_COMPLEX_MUL
))
1202 ifn
= IFN_COMPLEX_FMA
;
1203 else if (vect_match_call_p (node
, IFN_COMPLEX_MUL_CONJ
))
1204 ifn
= IFN_COMPLEX_FMA_CONJ
;
1208 if (!vect_pattern_validate_optab (ifn
, vnode
))
1211 /* FMA matched ADD + CMUL. During the matching of CMUL the
1212 stmt that starts the pattern is marked as being in a pattern,
1213 namely the CMUL. When replacing this with a CFMA we have to
1214 unmark this statement as being in a pattern. This is because
1215 vect_mark_pattern_stmts will only mark the current stmt as being
1216 in a pattern. Later on when the scalar stmts are examined the
1217 old statement which is supposed to be irrelevant will point to
1218 CMUL unless we undo the pattern relationship here. */
1219 vect_slp_reset_pattern (node
);
1223 if (ifn
== IFN_COMPLEX_FMA
)
1225 ops
->quick_push (SLP_TREE_CHILDREN (vnode
)[0]);
1226 ops
->quick_push (SLP_TREE_CHILDREN (node
)[1]);
1227 ops
->quick_push (SLP_TREE_CHILDREN (node
)[0]);
1231 ops
->quick_push (SLP_TREE_CHILDREN (vnode
)[0]);
1232 ops
->quick_push (SLP_TREE_CHILDREN (node
)[0]);
1233 ops
->quick_push (SLP_TREE_CHILDREN (node
)[1]);
1239 /* Attempt to recognize a complex mul pattern. */
1242 complex_fma_pattern::recognize (slp_tree_to_load_perm_map_t
*perm_cache
,
1245 auto_vec
<slp_tree
> ops
;
1246 complex_operation_t op
1247 = vect_detect_pair_op (*node
, true, &ops
);
1249 = complex_fma_pattern::matches (op
, perm_cache
, node
, &ops
);
1250 if (ifn
== IFN_LAST
)
1253 return new complex_fma_pattern (node
, &ops
, ifn
);
1256 /* Perform a replacement of the detected complex mul pattern with the new
1257 instruction sequences. */
1260 complex_fma_pattern::build (vec_info
*vinfo
)
1262 SLP_TREE_CHILDREN (*this->m_node
).release ();
1263 SLP_TREE_CHILDREN (*this->m_node
).create (3);
1264 SLP_TREE_CHILDREN (*this->m_node
).safe_splice (this->m_ops
);
1266 complex_pattern::build (vinfo
);
1269 /*******************************************************************************
1270 * complex_fms_pattern class
1271 ******************************************************************************/
1273 class complex_fms_pattern
: public complex_pattern
1276 complex_fms_pattern (slp_tree
*node
, vec
<slp_tree
> *m_ops
, internal_fn ifn
)
1277 : complex_pattern (node
, m_ops
, ifn
)
1279 this->m_num_args
= 3;
1283 void build (vec_info
*);
1285 matches (complex_operation_t op
, slp_tree_to_load_perm_map_t
*, slp_tree
*,
1288 static vect_pattern
*
1289 recognize (slp_tree_to_load_perm_map_t
*, slp_tree
*);
1291 static vect_pattern
*
1292 mkInstance (slp_tree
*node
, vec
<slp_tree
> *m_ops
, internal_fn ifn
)
1294 return new complex_fms_pattern (node
, m_ops
, ifn
);
1299 /* Pattern matcher for trying to match complex multiply and accumulate
1300 and multiply and subtract patterns in SLP tree.
1301 If the operation matches then IFN is set to the operation it matched and
1302 the arguments to the two replacement statements are put in m_ops.
1304 If no match is found then IFN is set to IFN_LAST and m_ops is unchanged.
1306 This function matches the patterns shaped as:
1308 double ax = (b[i+1] * a[i]) + (b[i] * a[i]);
1309 double bx = (a[i+1] * b[i]) - (a[i+1] * b[i+1]);
1312 c[i+1] = c[i+1] + bx;
1314 If a match occurred then TRUE is returned, else FALSE. The initial match is
1315 expected to be in OP1 and the initial match operands in args0. */
1318 complex_fms_pattern::matches (complex_operation_t op
,
1319 slp_tree_to_load_perm_map_t
*perm_cache
,
1320 slp_tree
* ref_node
, vec
<slp_tree
> *ops
)
1322 internal_fn ifn
= IFN_LAST
;
1324 /* Find the two components. We match Complex MUL first which reduces the
1325 amount of work this pattern has to do. After that we just match the
1326 head node and we're done.:
1329 slp_tree child
= NULL
;
1331 /* We need to ignore the two_operands nodes that may also match,
1332 for that we can check if they have any scalar statements and also
1333 check that it's not a permute node as we're looking for a normal
1334 PLUS_EXPR operation. */
1335 if (op
!= PLUS_MINUS
)
1338 child
= SLP_TREE_CHILDREN ((*ops
)[1])[1];
1339 if (vect_detect_pair_op (child
) != MINUS_PLUS
)
1342 /* First two nodes must be a multiply. */
1343 auto_vec
<slp_tree
> muls
;
1344 if (vect_match_call_complex_mla (child
, 0) != MULT_MULT
1345 || vect_match_call_complex_mla (child
, 1, &muls
) != MULT_MULT
)
1348 /* Now operand2+4 may lead to another expression. */
1349 auto_vec
<slp_tree
> left_op
, right_op
;
1350 left_op
.safe_splice (SLP_TREE_CHILDREN (muls
[0]));
1351 right_op
.safe_splice (SLP_TREE_CHILDREN (muls
[1]));
1353 bool is_neg
= vect_normalize_conj_loc (left_op
);
1355 child
= SLP_TREE_CHILDREN ((*ops
)[1])[0];
1356 bool conj_first_operand
= false;
1357 if (!vect_validate_multiplication (perm_cache
, right_op
, left_op
, false,
1358 &conj_first_operand
, true))
1362 ifn
= IFN_COMPLEX_FMS
;
1364 ifn
= IFN_COMPLEX_FMS_CONJ
;
1366 if (!vect_pattern_validate_optab (ifn
, *ref_node
))
1372 complex_perm_kinds_t kind
= linear_loads_p (perm_cache
, right_op
[0]).first
;
1373 if (kind
== PERM_EVENODD
)
1375 ops
->quick_push (child
);
1376 ops
->quick_push (right_op
[0]);
1377 ops
->quick_push (right_op
[1]);
1378 ops
->quick_push (left_op
[1]);
1380 else if (kind
== PERM_TOP
)
1382 ops
->quick_push (child
);
1383 ops
->quick_push (right_op
[1]);
1384 ops
->quick_push (right_op
[0]);
1385 ops
->quick_push (left_op
[0]);
1387 else if (kind
== PERM_EVENEVEN
&& !is_neg
)
1389 ops
->quick_push (child
);
1390 ops
->quick_push (right_op
[1]);
1391 ops
->quick_push (right_op
[0]);
1392 ops
->quick_push (left_op
[0]);
1396 ops
->quick_push (child
);
1397 ops
->quick_push (right_op
[1]);
1398 ops
->quick_push (right_op
[0]);
1399 ops
->quick_push (left_op
[1]);
1405 /* Attempt to recognize a complex mul pattern. */
1408 complex_fms_pattern::recognize (slp_tree_to_load_perm_map_t
*perm_cache
,
1411 auto_vec
<slp_tree
> ops
;
1412 complex_operation_t op
1413 = vect_detect_pair_op (*node
, true, &ops
);
1415 = complex_fms_pattern::matches (op
, perm_cache
, node
, &ops
);
1416 if (ifn
== IFN_LAST
)
1419 return new complex_fms_pattern (node
, &ops
, ifn
);
1422 /* Perform a replacement of the detected complex mul pattern with the new
1423 instruction sequences. */
1426 complex_fms_pattern::build (vec_info
*vinfo
)
1430 FOR_EACH_VEC_ELT (SLP_TREE_CHILDREN (*this->m_node
), i
, node
)
1431 vect_free_slp_tree (node
);
1433 SLP_TREE_CHILDREN (*this->m_node
).release ();
1434 SLP_TREE_CHILDREN (*this->m_node
).create (3);
1436 /* First re-arrange the children. */
1437 SLP_TREE_CHILDREN (*this->m_node
).quick_push (this->m_ops
[0]);
1438 SLP_TREE_CHILDREN (*this->m_node
).quick_push (this->m_ops
[1]);
1439 SLP_TREE_CHILDREN (*this->m_node
).quick_push (
1440 vect_build_combine_node (this->m_ops
[2], this->m_ops
[3], *this->m_node
));
1441 SLP_TREE_REF_COUNT (this->m_ops
[0])++;
1442 SLP_TREE_REF_COUNT (this->m_ops
[1])++;
1444 /* And then rewrite the node itself. */
1445 complex_pattern::build (vinfo
);
1448 /*******************************************************************************
1449 * complex_operations_pattern class
1450 ******************************************************************************/
1452 /* This function combines all the existing pattern matchers above into one class
1453 that shares the functionality between them. The initial match is shared
1454 between all complex operations. */
1456 class complex_operations_pattern
: public complex_pattern
1459 complex_operations_pattern (slp_tree
*node
, vec
<slp_tree
> *m_ops
,
1461 : complex_pattern (node
, m_ops
, ifn
)
1463 this->m_num_args
= 0;
1467 void build (vec_info
*);
1469 matches (complex_operation_t op
, slp_tree_to_load_perm_map_t
*, slp_tree
*,
1472 static vect_pattern
*
1473 recognize (slp_tree_to_load_perm_map_t
*, slp_tree
*);
1476 /* Dummy matches implementation for proxy object. */
1479 complex_operations_pattern::
1480 matches (complex_operation_t
/* op */,
1481 slp_tree_to_load_perm_map_t
* /* perm_cache */,
1482 slp_tree
* /* ref_node */, vec
<slp_tree
> * /* ops */)
1487 /* Attempt to recognize a complex mul pattern. */
1490 complex_operations_pattern::recognize (slp_tree_to_load_perm_map_t
*perm_cache
,
1493 auto_vec
<slp_tree
> ops
;
1494 complex_operation_t op
1495 = vect_detect_pair_op (*node
, true, &ops
);
1496 internal_fn ifn
= IFN_LAST
;
1498 ifn
= complex_fms_pattern::matches (op
, perm_cache
, node
, &ops
);
1499 if (ifn
!= IFN_LAST
)
1500 return complex_fms_pattern::mkInstance (node
, &ops
, ifn
);
1502 ifn
= complex_mul_pattern::matches (op
, perm_cache
, node
, &ops
);
1503 if (ifn
!= IFN_LAST
)
1504 return complex_mul_pattern::mkInstance (node
, &ops
, ifn
);
1506 ifn
= complex_fma_pattern::matches (op
, perm_cache
, node
, &ops
);
1507 if (ifn
!= IFN_LAST
)
1508 return complex_fma_pattern::mkInstance (node
, &ops
, ifn
);
1510 ifn
= complex_add_pattern::matches (op
, perm_cache
, node
, &ops
);
1511 if (ifn
!= IFN_LAST
)
1512 return complex_add_pattern::mkInstance (node
, &ops
, ifn
);
1517 /* Dummy implementation of build. */
1520 complex_operations_pattern::build (vec_info
* /* vinfo */)
1525 /*******************************************************************************
1526 * Pattern matching definitions
1527 ******************************************************************************/
1529 #define SLP_PATTERN(x) &x::recognize
1530 vect_pattern_decl_t slp_patterns
[]
1532 /* For least amount of back-tracking and more efficient matching
1533 order patterns from the largest to the smallest. Especially if they
1534 overlap in what they can detect. */
1536 SLP_PATTERN (complex_operations_pattern
),
1540 /* Set the number of SLP pattern matchers available. */
1541 size_t num__slp_patterns
= sizeof(slp_patterns
)/sizeof(vect_pattern_decl_t
);