gcc/tree-vectorizer.h

   1 /* Loop Vectorization
   2    Copyright (C) 2003, 2004, 2005, 2006, 2007 Free Software Foundation, Inc.
   3    Contributed by Dorit Naishlos <dorit@il.ibm.com>
   4
   5 This file is part of GCC.
   6
   7 GCC is free software; you can redistribute it and/or modify it under
   8 the terms of the GNU General Public License as published by the Free
   9 Software Foundation; either version 3, or (at your option) any later
  10 version.
  11
  12 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
  13 WARRANTY; without even the implied warranty of MERCHANTABILITY or
  14 FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  15 for more details.
  16
  17 You should have received a copy of the GNU General Public License
  18 along with GCC; see the file COPYING3.  If not see
  19 <http://www.gnu.org/licenses/>.  */
  20
  21 #ifndef GCC_TREE_VECTORIZER_H
  22 #define GCC_TREE_VECTORIZER_H
  23
  24 #ifdef USE_MAPPED_LOCATION
  25   typedef source_location LOC;
  26   #define UNKNOWN_LOC UNKNOWN_LOCATION
  27   #define EXPR_LOC(e) EXPR_LOCATION(e)
  28   #define LOC_FILE(l) LOCATION_FILE (l)
  29   #define LOC_LINE(l) LOCATION_LINE (l)
  30 #else
  31   typedef source_locus LOC;
  32   #define UNKNOWN_LOC NULL
  33   #define EXPR_LOC(e) EXPR_LOCUS(e)
  34   #define LOC_FILE(l) (l)->file
  35   #define LOC_LINE(l) (l)->line
  36 #endif
  37
  38 /* Used for naming of new temporaries.  */
  39 enum vect_var_kind {
  40   vect_simple_var,
  41   vect_pointer_var,
  42   vect_scalar_var
  43 };
  44
  45 /* Defines type of operation.  */
  46 enum operation_type {
  47   unary_op = 1,
  48   binary_op,
  49   ternary_op
  50 };
  51
  52 /* Define type of available alignment support.  */
  53 enum dr_alignment_support {
  54   dr_unaligned_unsupported,
  55   dr_unaligned_supported,
  56   dr_explicit_realign,
  57   dr_explicit_realign_optimized,
  58   dr_aligned
  59 };
  60
  61 /* Define type of def-use cross-iteration cycle.  */
  62 enum vect_def_type {
  63   vect_constant_def = 1,
  64   vect_invariant_def,
  65   vect_loop_def,
  66   vect_induction_def,
  67   vect_reduction_def,
  68   vect_unknown_def_type
  69 };
  70
  71 /* Define verbosity levels.  */
  72 enum verbosity_levels {
  73   REPORT_NONE,
  74   REPORT_VECTORIZED_LOOPS,
  75   REPORT_UNVECTORIZED_LOOPS,
  76   REPORT_COST,
  77   REPORT_ALIGNMENT,
  78   REPORT_DR_DETAILS,
  79   REPORT_BAD_FORM_LOOPS,
  80   REPORT_OUTER_LOOPS,
  81   REPORT_SLP,
  82   REPORT_DETAILS,
  83   /* New verbosity levels should be added before this one.  */
  84   MAX_VERBOSITY_LEVEL
  85 };
  86
  87 /************************************************************************
  88   SLP
  89  ************************************************************************/
  90
  91 /* A computation tree of an SLP instance. Each node corresponds to a group of
  92    stmts to be packed in a SIMD stmt.  */
  93 typedef struct _slp_tree {
  94   /* Only binary and unary operations are supported. LEFT child corresponds to
  95      the first operand and RIGHT child to the second if the operation is
  96      binary.  */
  97   struct _slp_tree *left;
  98   struct _slp_tree *right;
  99   /* A group of scalar stmts to be vectorized together.  */
 100   VEC (tree, heap) *stmts;
 101   /* Vectorized stmt/s.  */
 102   VEC (tree, heap) *vec_stmts;
 103   /* Number of vector stmts that are created to replace the group of scalar
 104      stmts. It is calculated during the transformation phase as the number of
 105      scalar elements in one scalar iteration (GROUP_SIZE) multiplied by VF
 106      divided by vector size.  */
 107   unsigned int vec_stmts_size;
 108   /* Vectorization costs associated with SLP node.  */
 109   struct
 110   {
 111     int outside_of_loop;     /* Statements generated outside loop.  */
 112     int inside_of_loop;      /* Statements generated inside loop.  */
 113   } cost;
 114 } *slp_tree;
 115
 116
 117 /* SLP instance is a sequence of stmts in a loop that can be packed into
 118    SIMD stmts.  */
 119 typedef struct _slp_instance {
 120   /* The root of SLP tree.  */
 121   slp_tree root;
 122
 123   /* Size of groups of scalar stmts that will be replaced by SIMD stmt/s.  */
 124   unsigned int group_size;
 125
 126   /* The unrolling factor required to vectorized this SLP instance.  */
 127   unsigned int unrolling_factor;
 128
 129   /* Vectorization costs associated with SLP instance.  */
 130   struct
 131   {
 132     int outside_of_loop;     /* Statements generated outside loop.  */
 133     int inside_of_loop;      /* Statements generated inside loop.  */
 134   } cost;
 135 } *slp_instance;
 136
 137 DEF_VEC_P(slp_instance);
 138 DEF_VEC_ALLOC_P(slp_instance, heap);
 139
 140 /* Access Functions.  */
 141 #define SLP_INSTANCE_TREE(S)                     (S)->root
 142 #define SLP_INSTANCE_GROUP_SIZE(S)               (S)->group_size
 143 #define SLP_INSTANCE_UNROLLING_FACTOR(S)         (S)->unrolling_factor
 144 #define SLP_INSTANCE_OUTSIDE_OF_LOOP_COST(S)     (S)->cost.outside_of_loop
 145 #define SLP_INSTANCE_INSIDE_OF_LOOP_COST(S)      (S)->cost.inside_of_loop
 146
 147 #define SLP_TREE_LEFT(S)                         (S)->left
 148 #define SLP_TREE_RIGHT(S)                        (S)->right
 149 #define SLP_TREE_SCALAR_STMTS(S)                 (S)->stmts
 150 #define SLP_TREE_VEC_STMTS(S)                    (S)->vec_stmts
 151 #define SLP_TREE_NUMBER_OF_VEC_STMTS(S)          (S)->vec_stmts_size
 152 #define SLP_TREE_OUTSIDE_OF_LOOP_COST(S)         (S)->cost.outside_of_loop
 153 #define SLP_TREE_INSIDE_OF_LOOP_COST(S)          (S)->cost.inside_of_loop
 154
 155 /*-----------------------------------------------------------------*/
 156 /* Info on vectorized loops.                                       */
 157 /*-----------------------------------------------------------------*/
 158 typedef struct _loop_vec_info {
 159
 160   /* The loop to which this info struct refers to.  */
 161   struct loop *loop;
 162
 163   /* The loop basic blocks.  */
 164   basic_block *bbs;
 165
 166   /* Number of iterations.  */
 167   tree num_iters;
 168   tree num_iters_unchanged;
 169
 170   /* Minimum number of iterations below which vectorization is expected to
 171      not be profitable (as estimated by the cost model).
 172      -1 indicates that vectorization will not be profitable.
 173      FORNOW: This field is an int. Will be a tree in the future, to represent
 174              values unknown at compile time.  */
 175   int min_profitable_iters;
 176
 177   /* Is the loop vectorizable? */
 178   bool vectorizable;
 179
 180   /* Unrolling factor  */
 181   int vectorization_factor;
 182
 183   /* Unknown DRs according to which loop was peeled.  */
 184   struct data_reference *unaligned_dr;
 185
 186   /* peeling_for_alignment indicates whether peeling for alignment will take
 187      place, and what the peeling factor should be:
 188      peeling_for_alignment = X means:
 189         If X=0: Peeling for alignment will not be applied.
 190         If X>0: Peel first X iterations.
 191         If X=-1: Generate a runtime test to calculate the number of iterations
 192                  to be peeled, using the dataref recorded in the field
 193                  unaligned_dr.  */
 194   int peeling_for_alignment;
 195
 196   /* The mask used to check the alignment of pointers or arrays.  */
 197   int ptr_mask;
 198
 199   /* All data references in the loop.  */
 200   VEC (data_reference_p, heap) *datarefs;
 201
 202   /* All data dependences in the loop.  */
 203   VEC (ddr_p, heap) *ddrs;
 204
 205   /* Data Dependence Relations defining address ranges that are candidates
 206      for a run-time aliasing check.  */
 207   VEC (ddr_p, heap) *may_alias_ddrs;
 208
 209   /* Statements in the loop that have data references that are candidates for a
 210      runtime (loop versioning) misalignment check.  */
 211   VEC(tree,heap) *may_misalign_stmts;
 212
 213   /* The loop location in the source.  */
 214   LOC loop_line_number;
 215
 216   /* All interleaving chains of stores in the loop, represented by the first
 217      stmt in the chain.  */
 218   VEC(tree, heap) *strided_stores;
 219
 220   /* All SLP instances in the loop. This is a subset of the set of STRIDED_STORES
 221      of the loop.  */
 222   VEC(slp_instance, heap) *slp_instances;
 223
 224   /* The unrolling factor needed to SLP the loop. In case of that pure SLP is
 225      applied to the loop, i.e., no unrolling is needed, this is 1.  */
 226   unsigned slp_unrolling_factor;
 227 } *loop_vec_info;
 228
 229 /* Access Functions.  */
 230 #define LOOP_VINFO_LOOP(L)            (L)->loop
 231 #define LOOP_VINFO_BBS(L)             (L)->bbs
 232 #define LOOP_VINFO_NITERS(L)          (L)->num_iters
 233 /* Since LOOP_VINFO_NITERS can change after prologue peeling
 234    retain total unchanged scalar loop iterations for cost model.  */
 235 #define LOOP_VINFO_NITERS_UNCHANGED(L)          (L)->num_iters_unchanged
 236 #define LOOP_VINFO_COST_MODEL_MIN_ITERS(L)      (L)->min_profitable_iters
 237 #define LOOP_VINFO_VECTORIZABLE_P(L)  (L)->vectorizable
 238 #define LOOP_VINFO_VECT_FACTOR(L)     (L)->vectorization_factor
 239 #define LOOP_VINFO_PTR_MASK(L)        (L)->ptr_mask
 240 #define LOOP_VINFO_DATAREFS(L)        (L)->datarefs
 241 #define LOOP_VINFO_DDRS(L)            (L)->ddrs
 242 #define LOOP_VINFO_INT_NITERS(L)      (TREE_INT_CST_LOW ((L)->num_iters))
 243 #define LOOP_PEELING_FOR_ALIGNMENT(L) (L)->peeling_for_alignment
 244 #define LOOP_VINFO_UNALIGNED_DR(L)    (L)->unaligned_dr
 245 #define LOOP_VINFO_MAY_MISALIGN_STMTS(L) (L)->may_misalign_stmts
 246 #define LOOP_VINFO_LOC(L)             (L)->loop_line_number
 247 #define LOOP_VINFO_MAY_ALIAS_DDRS(L)  (L)->may_alias_ddrs
 248 #define LOOP_VINFO_STRIDED_STORES(L)  (L)->strided_stores
 249 #define LOOP_VINFO_SLP_INSTANCES(L)   (L)->slp_instances
 250 #define LOOP_VINFO_SLP_UNROLLING_FACTOR(L) (L)->slp_unrolling_factor
 251
 252 #define NITERS_KNOWN_P(n)                     \
 253 (host_integerp ((n),0)                        \
 254 && TREE_INT_CST_LOW ((n)) > 0)
 255
 256 #define LOOP_VINFO_NITERS_KNOWN_P(L)                     \
 257 NITERS_KNOWN_P((L)->num_iters)
 258
 259 static inline loop_vec_info
 260 loop_vec_info_for_loop (struct loop *loop)
 261 {
 262   return (loop_vec_info) loop->aux;
 263 }
 264
 265 static inline bool
 266 nested_in_vect_loop_p (struct loop *loop, tree stmt)
 267 {
 268   return (loop->inner
 269           && (loop->inner == (bb_for_stmt (stmt))->loop_father));
 270 }
 271
 272 /*-----------------------------------------------------------------*/
 273 /* Info on vectorized defs.                                        */
 274 /*-----------------------------------------------------------------*/
 275 enum stmt_vec_info_type {
 276   undef_vec_info_type = 0,
 277   load_vec_info_type,
 278   store_vec_info_type,
 279   op_vec_info_type,
 280   call_vec_info_type,
 281   assignment_vec_info_type,
 282   condition_vec_info_type,
 283   reduc_vec_info_type,
 284   induc_vec_info_type,
 285   type_promotion_vec_info_type,
 286   type_demotion_vec_info_type,
 287   type_conversion_vec_info_type,
 288   loop_exit_ctrl_vec_info_type
 289 };
 290
 291 /* Indicates whether/how a variable is used in the loop.  */
 292 enum vect_relevant {
 293   vect_unused_in_loop = 0,
 294   vect_used_in_outer_by_reduction,
 295   vect_used_in_outer,
 296
 297   /* defs that feed computations that end up (only) in a reduction. These
 298      defs may be used by non-reduction stmts, but eventually, any
 299      computations/values that are affected by these defs are used to compute
 300      a reduction (i.e. don't get stored to memory, for example). We use this
 301      to identify computations that we can change the order in which they are
 302      computed.  */
 303   vect_used_by_reduction,
 304
 305   vect_used_in_loop
 306 };
 307
 308 /* The type of vectorization that can be applied to the stmt: regular loop-based
 309    vectorization; pure SLP - the stmt is a part of SLP instances and does not
 310    have uses outside SLP instances; or hybrid SLP and loop-based - the stmt is
 311    a part of SLP instance and also must be loop-based vectorized, since it has
 312    uses outside SLP sequences.
 313
 314    In the loop context the meanings of pure and hybrid SLP are slightly
 315    different. By saying that pure SLP is applied to the loop, we mean that we
 316    exploit only intra-iteration parallelism in the loop; i.e., the loop can be
 317    vectorized without doing any conceptual unrolling, cause we don't pack
 318    together stmts from different iterations, only within a single iteration.
 319    Loop hybrid SLP means that we exploit both intra-iteration and
 320    inter-iteration parallelism (e.g., number of elements in the vector is 4
 321    and the slp-group-size is 2, in which case we don't have enough parallelism
 322    within an iteration, so we obtain the rest of the parallelism from subsequent
 323    iterations by unrolling the loop by 2).  */
 324 enum slp_vect_type {
 325   loop_vect = 0,
 326   pure_slp,
 327   hybrid
 328 };
 329
 330
 331 typedef struct data_reference *dr_p;
 332 DEF_VEC_P(dr_p);
 333 DEF_VEC_ALLOC_P(dr_p,heap);
 334
 335 typedef struct _stmt_vec_info {
 336
 337   enum stmt_vec_info_type type;
 338
 339   /* The stmt to which this info struct refers to.  */
 340   tree stmt;
 341
 342   /* The loop_vec_info with respect to which STMT is vectorized.  */
 343   loop_vec_info loop_vinfo;
 344
 345   /* Not all stmts in the loop need to be vectorized. e.g, the increment
 346      of the loop induction variable and computation of array indexes. relevant
 347      indicates whether the stmt needs to be vectorized.  */
 348   enum vect_relevant relevant;
 349
 350   /* Indicates whether this stmts is part of a computation whose result is
 351      used outside the loop.  */
 352   bool live;
 353
 354   /* The vector type to be used.  */
 355   tree vectype;
 356
 357   /* The vectorized version of the stmt.  */
 358   tree vectorized_stmt;
 359
 360
 361   /** The following is relevant only for stmts that contain a non-scalar
 362      data-ref (array/pointer/struct access). A GIMPLE stmt is expected to have
 363      at most one such data-ref.  **/
 364
 365   /* Information about the data-ref (access function, etc),
 366      relative to the inner-most containing loop.  */
 367   struct data_reference *data_ref_info;
 368
 369   /* Information about the data-ref relative to this loop
 370      nest (the loop that is being considered for vectorization).  */
 371   tree dr_base_address;
 372   tree dr_init;
 373   tree dr_offset;
 374   tree dr_step;
 375   tree dr_aligned_to;
 376
 377   /* Stmt is part of some pattern (computation idiom)  */
 378   bool in_pattern_p;
 379
 380   /* Used for various bookkeeping purposes, generally holding a pointer to
 381      some other stmt S that is in some way "related" to this stmt.
 382      Current use of this field is:
 383         If this stmt is part of a pattern (i.e. the field 'in_pattern_p' is
 384         true): S is the "pattern stmt" that represents (and replaces) the
 385         sequence of stmts that constitutes the pattern.  Similarly, the
 386         related_stmt of the "pattern stmt" points back to this stmt (which is
 387         the last stmt in the original sequence of stmts that constitutes the
 388         pattern).  */
 389   tree related_stmt;
 390
 391   /* List of datarefs that are known to have the same alignment as the dataref
 392      of this stmt.  */
 393   VEC(dr_p,heap) *same_align_refs;
 394
 395   /* Classify the def of this stmt.  */
 396   enum vect_def_type def_type;
 397
 398   /* Interleaving info.  */
 399   /* First data-ref in the interleaving group.  */
 400   tree first_dr;
 401   /* Pointer to the next data-ref in the group.  */
 402   tree next_dr;
 403   /* The size of the interleaving group.  */
 404   unsigned int size;
 405   /* For stores, number of stores from this group seen. We vectorize the last
 406      one.  */
 407   unsigned int store_count;
 408   /* For loads only, the gap from the previous load. For consecutive loads, GAP
 409      is 1.  */
 410   unsigned int gap;
 411   /* In case that two or more stmts share data-ref, this is the pointer to the
 412      previously detected stmt with the same dr.  */
 413   tree same_dr_stmt;
 414   /* For loads only, if there is a store with the same location, this field is
 415      TRUE.  */
 416   bool read_write_dep;
 417
 418   /* Vectorization costs associated with statement.  */
 419   struct
 420   {
 421     int outside_of_loop;     /* Statements generated outside loop.  */
 422     int inside_of_loop;      /* Statements generated inside loop.  */
 423   } cost;
 424
 425   /*  Whether the stmt is SLPed, loop-based vectorized, or both.  */
 426   enum slp_vect_type slp_type;
 427 } *stmt_vec_info;
 428
 429 /* Access Functions.  */
 430 #define STMT_VINFO_TYPE(S)                 (S)->type
 431 #define STMT_VINFO_STMT(S)                 (S)->stmt
 432 #define STMT_VINFO_LOOP_VINFO(S)           (S)->loop_vinfo
 433 #define STMT_VINFO_RELEVANT(S)             (S)->relevant
 434 #define STMT_VINFO_LIVE_P(S)               (S)->live
 435 #define STMT_VINFO_VECTYPE(S)              (S)->vectype
 436 #define STMT_VINFO_VEC_STMT(S)             (S)->vectorized_stmt
 437 #define STMT_VINFO_DATA_REF(S)             (S)->data_ref_info
 438
 439 #define STMT_VINFO_DR_BASE_ADDRESS(S)      (S)->dr_base_address
 440 #define STMT_VINFO_DR_INIT(S)              (S)->dr_init
 441 #define STMT_VINFO_DR_OFFSET(S)            (S)->dr_offset
 442 #define STMT_VINFO_DR_STEP(S)              (S)->dr_step
 443 #define STMT_VINFO_DR_ALIGNED_TO(S)        (S)->dr_aligned_to
 444
 445 #define STMT_VINFO_IN_PATTERN_P(S)         (S)->in_pattern_p
 446 #define STMT_VINFO_RELATED_STMT(S)         (S)->related_stmt
 447 #define STMT_VINFO_SAME_ALIGN_REFS(S)      (S)->same_align_refs
 448 #define STMT_VINFO_DEF_TYPE(S)             (S)->def_type
 449 #define STMT_VINFO_DR_GROUP_FIRST_DR(S)    (S)->first_dr
 450 #define STMT_VINFO_DR_GROUP_NEXT_DR(S)     (S)->next_dr
 451 #define STMT_VINFO_DR_GROUP_SIZE(S)        (S)->size
 452 #define STMT_VINFO_DR_GROUP_STORE_COUNT(S) (S)->store_count
 453 #define STMT_VINFO_DR_GROUP_GAP(S)         (S)->gap
 454 #define STMT_VINFO_DR_GROUP_SAME_DR_STMT(S)(S)->same_dr_stmt
 455 #define STMT_VINFO_DR_GROUP_READ_WRITE_DEPENDENCE(S)  (S)->read_write_dep
 456 #define STMT_VINFO_STRIDED_ACCESS(S)      ((S)->first_dr != NULL)
 457
 458 #define DR_GROUP_FIRST_DR(S)               (S)->first_dr
 459 #define DR_GROUP_NEXT_DR(S)                (S)->next_dr
 460 #define DR_GROUP_SIZE(S)                   (S)->size
 461 #define DR_GROUP_STORE_COUNT(S)            (S)->store_count
 462 #define DR_GROUP_GAP(S)                    (S)->gap
 463 #define DR_GROUP_SAME_DR_STMT(S)           (S)->same_dr_stmt
 464 #define DR_GROUP_READ_WRITE_DEPENDENCE(S)  (S)->read_write_dep
 465
 466 #define STMT_VINFO_RELEVANT_P(S)          ((S)->relevant != vect_unused_in_loop)
 467 #define STMT_VINFO_OUTSIDE_OF_LOOP_COST(S) (S)->cost.outside_of_loop
 468 #define STMT_VINFO_INSIDE_OF_LOOP_COST(S)  (S)->cost.inside_of_loop
 469
 470 #define HYBRID_SLP_STMT(S)                ((S)->slp_type == hybrid)
 471 #define PURE_SLP_STMT(S)                  ((S)->slp_type == pure_slp)
 472 #define STMT_SLP_TYPE(S)                   (S)->slp_type
 473
 474 /* These are some defines for the initial implementation of the vectorizer's
 475    cost model.  These will later be target specific hooks.  */
 476
 477 /* Cost of conditional taken branch.  */
 478 #ifndef TARG_COND_TAKEN_BRANCH_COST
 479 #define TARG_COND_TAKEN_BRANCH_COST        3
 480 #endif
 481
 482 /* Cost of conditional not taken branch.  */
 483 #ifndef TARG_COND_NOT_TAKEN_BRANCH_COST
 484 #define TARG_COND_NOT_TAKEN_BRANCH_COST        1
 485 #endif
 486
 487 /* Cost of any scalar operation, excluding load and store.  */
 488 #ifndef TARG_SCALAR_STMT_COST
 489 #define TARG_SCALAR_STMT_COST           1
 490 #endif
 491
 492 /* Cost of scalar load.  */
 493 #ifndef TARG_SCALAR_LOAD_COST
 494 #define TARG_SCALAR_LOAD_COST           1
 495 #endif
 496
 497 /* Cost of scalar store.  */
 498 #ifndef TARG_SCALAR_STORE_COST
 499 #define TARG_SCALAR_STORE_COST           1
 500 #endif
 501
 502 /* Cost of any vector operation, excluding load, store or vector to scalar
 503    operation.  */
 504 #ifndef TARG_VEC_STMT_COST
 505 #define TARG_VEC_STMT_COST           1
 506 #endif
 507
 508 /* Cost of vector to scalar operation.  */
 509 #ifndef TARG_VEC_TO_SCALAR_COST
 510 #define TARG_VEC_TO_SCALAR_COST      1
 511 #endif
 512
 513 /* Cost of scalar to vector operation.  */
 514 #ifndef TARG_SCALAR_TO_VEC_COST
 515 #define TARG_SCALAR_TO_VEC_COST      1
 516 #endif
 517
 518 /* Cost of aligned vector load.  */
 519 #ifndef TARG_VEC_LOAD_COST
 520 #define TARG_VEC_LOAD_COST           1
 521 #endif
 522
 523 /* Cost of misaligned vector load.  */
 524 #ifndef TARG_VEC_UNALIGNED_LOAD_COST
 525 #define TARG_VEC_UNALIGNED_LOAD_COST 2
 526 #endif
 527
 528 /* Cost of vector store.  */
 529 #ifndef TARG_VEC_STORE_COST
 530 #define TARG_VEC_STORE_COST          1
 531 #endif
 532
 533 static inline void set_stmt_info (stmt_ann_t ann, stmt_vec_info stmt_info);
 534 static inline stmt_vec_info vinfo_for_stmt (tree stmt);
 535
 536 static inline void
 537 set_stmt_info (stmt_ann_t ann, stmt_vec_info stmt_info)
 538 {
 539   if (ann)
 540     ann->common.aux = (char *) stmt_info;
 541 }
 542
 543 static inline stmt_vec_info
 544 vinfo_for_stmt (tree stmt)
 545 {
 546   stmt_ann_t ann = stmt_ann (stmt);
 547   return ann ? (stmt_vec_info) ann->common.aux : NULL;
 548 }
 549
 550 static inline bool
 551 is_pattern_stmt_p (stmt_vec_info stmt_info)
 552 {
 553   tree related_stmt;
 554   stmt_vec_info related_stmt_info;
 555
 556   related_stmt = STMT_VINFO_RELATED_STMT (stmt_info);
 557   if (related_stmt
 558       && (related_stmt_info = vinfo_for_stmt (related_stmt))
 559       && STMT_VINFO_IN_PATTERN_P (related_stmt_info))
 560     return true;
 561
 562   return false;
 563 }
 564
 565 static inline bool
 566 is_loop_header_bb_p (basic_block bb)
 567 {
 568   if (bb == (bb->loop_father)->header)
 569     return true;
 570   gcc_assert (EDGE_COUNT (bb->preds) == 1);
 571   return false;
 572 }
 573
 574 static inline void
 575 stmt_vinfo_set_inside_of_loop_cost (stmt_vec_info stmt_info, slp_tree slp_node,
 576                                     int cost)
 577 {
 578   if (slp_node)
 579     SLP_TREE_INSIDE_OF_LOOP_COST (slp_node) = cost;
 580   else
 581     STMT_VINFO_INSIDE_OF_LOOP_COST (stmt_info) = cost;
 582 }
 583
 584 static inline void
 585 stmt_vinfo_set_outside_of_loop_cost (stmt_vec_info stmt_info, slp_tree slp_node,
 586                                      int cost)
 587 {
 588   if (slp_node)
 589     SLP_TREE_OUTSIDE_OF_LOOP_COST (slp_node) = cost;
 590   else
 591     STMT_VINFO_OUTSIDE_OF_LOOP_COST (stmt_info) = cost;
 592 }
 593
 594
 595 /*-----------------------------------------------------------------*/
 596 /* Info on data references alignment.                              */
 597 /*-----------------------------------------------------------------*/
 598
 599 /* Reflects actual alignment of first access in the vectorized loop,
 600    taking into account peeling/versioning if applied.  */
 601 #define DR_MISALIGNMENT(DR)   ((int) (size_t) (DR)->aux)
 602 #define SET_DR_MISALIGNMENT(DR, VAL)   ((DR)->aux = (void *) (size_t) (VAL))
 603
 604 static inline bool
 605 aligned_access_p (struct data_reference *data_ref_info)
 606 {
 607   return (DR_MISALIGNMENT (data_ref_info) == 0);
 608 }
 609
 610 static inline bool
 611 known_alignment_for_access_p (struct data_reference *data_ref_info)
 612 {
 613   return (DR_MISALIGNMENT (data_ref_info) != -1);
 614 }
 615
 616 /* vect_dump will be set to stderr or dump_file if exist.  */
 617 extern FILE *vect_dump;
 618 extern enum verbosity_levels vect_verbosity_level;
 619
 620 /* Bitmap of virtual variables to be renamed.  */
 621 extern bitmap vect_memsyms_to_rename;
 622
 623 /*-----------------------------------------------------------------*/
 624 /* Function prototypes.                                            */
 625 /*-----------------------------------------------------------------*/
 626
 627 /*************************************************************************
 628   Simple Loop Peeling Utilities - in tree-vectorizer.c
 629  *************************************************************************/
 630 /* Entry point for peeling of simple loops.
 631    Peel the first/last iterations of a loop.
 632    It can be used outside of the vectorizer for loops that are simple enough
 633    (see function documentation).  In the vectorizer it is used to peel the
 634    last few iterations when the loop bound is unknown or does not evenly
 635    divide by the vectorization factor, and to peel the first few iterations
 636    to force the alignment of data references in the loop.  */
 637 extern struct loop *slpeel_tree_peel_loop_to_edge
 638   (struct loop *, edge, tree, tree, bool, unsigned int, bool);
 639 extern void set_prologue_iterations (basic_block, tree,
 640                                      struct loop *, unsigned int);
 641 extern void slpeel_make_loop_iterate_ntimes (struct loop *, tree);
 642 extern bool slpeel_can_duplicate_loop_p (const struct loop *, const_edge);
 643 #ifdef ENABLE_CHECKING
 644 extern void slpeel_verify_cfg_after_peeling (struct loop *, struct loop *);
 645 #endif
 646
 647
 648 /*************************************************************************
 649   General Vectorization Utilities
 650  *************************************************************************/
 651 /** In tree-vectorizer.c **/
 652 extern tree get_vectype_for_scalar_type (tree);
 653 extern bool vect_is_simple_use (tree, loop_vec_info, tree *, tree *,
 654                                 enum vect_def_type *);
 655 extern bool vect_is_simple_iv_evolution (unsigned, tree, tree *, tree *);
 656 extern tree vect_is_simple_reduction (loop_vec_info, tree);
 657 extern bool vect_can_force_dr_alignment_p (const_tree, unsigned int);
 658 extern enum dr_alignment_support vect_supportable_dr_alignment
 659   (struct data_reference *);
 660 extern bool reduction_code_for_scalar_code (enum tree_code, enum tree_code *);
 661 extern bool supportable_widening_operation (enum tree_code, tree, tree,
 662   tree *, tree *, enum tree_code *, enum tree_code *);
 663 extern bool supportable_narrowing_operation (enum tree_code, const_tree,
 664                                              const_tree, enum tree_code *);
 665
 666 /* Creation and deletion of loop and stmt info structs.  */
 667 extern loop_vec_info new_loop_vec_info (struct loop *loop);
 668 extern void destroy_loop_vec_info (loop_vec_info, bool);
 669 extern stmt_vec_info new_stmt_vec_info (tree stmt, loop_vec_info);
 670
 671
 672 /** In tree-vect-analyze.c  **/
 673 /* Driver for analysis stage.  */
 674 extern loop_vec_info vect_analyze_loop (struct loop *);
 675 extern void vect_free_slp_tree (slp_tree);
 676 extern loop_vec_info vect_analyze_loop_form (struct loop *);
 677
 678 /** In tree-vect-patterns.c  **/
 679 /* Pattern recognition functions.
 680    Additional pattern recognition functions can (and will) be added
 681    in the future.  */
 682 typedef tree (* vect_recog_func_ptr) (tree, tree *, tree *);
 683 #define NUM_PATTERNS 4
 684 void vect_pattern_recog (loop_vec_info);
 685
 686
 687 /** In tree-vect-transform.c  **/
 688 extern bool vectorizable_load (tree, block_stmt_iterator *, tree *, slp_tree);
 689 extern bool vectorizable_store (tree, block_stmt_iterator *, tree *, slp_tree);
 690 extern bool vectorizable_operation (tree, block_stmt_iterator *, tree *,
 691                                     slp_tree);
 692 extern bool vectorizable_type_promotion (tree, block_stmt_iterator *, tree *);
 693 extern bool vectorizable_type_demotion (tree, block_stmt_iterator *, tree *);
 694 extern bool vectorizable_conversion (tree, block_stmt_iterator *,
 695                                      tree *, slp_tree);
 696 extern bool vectorizable_assignment (tree, block_stmt_iterator *, tree *,
 697                                      slp_tree);
 698 extern tree vectorizable_function (tree, tree, tree);
 699 extern bool vectorizable_call (tree, block_stmt_iterator *, tree *);
 700 extern bool vectorizable_condition (tree, block_stmt_iterator *, tree *);
 701 extern bool vectorizable_live_operation (tree, block_stmt_iterator *, tree *);
 702 extern bool vectorizable_reduction (tree, block_stmt_iterator *, tree *);
 703 extern bool vectorizable_induction (tree, block_stmt_iterator *, tree *);
 704 extern int  vect_estimate_min_profitable_iters (loop_vec_info);
 705 extern void vect_model_simple_cost (stmt_vec_info, int, enum vect_def_type *,
 706                                     slp_tree);
 707 extern void vect_model_store_cost (stmt_vec_info, int, enum vect_def_type,
 708                                    slp_tree);
 709 extern void vect_model_load_cost (stmt_vec_info, int, slp_tree);
 710 /* Driver for transformation stage.  */
 711 extern void vect_transform_loop (loop_vec_info);
 712
 713 /*************************************************************************
 714   Vectorization Debug Information - in tree-vectorizer.c
 715  *************************************************************************/
 716 extern bool vect_print_dump_info (enum verbosity_levels);
 717 extern void vect_set_verbosity_level (const char *);
 718 extern LOC find_loop_location (struct loop *);
 719
 720 #endif  /* GCC_TREE_VECTORIZER_H  */