GCC Middle and Back End API Reference
tree-vectorizer.h
Go to the documentation of this file.
1 /* Vectorizer
2  Copyright (C) 2003-2013 Free Software Foundation, Inc.
3  Contributed by Dorit Naishlos <dorit@il.ibm.com>
4 
5 This file is part of GCC.
6 
7 GCC is free software; you can redistribute it and/or modify it under
8 the terms of the GNU General Public License as published by the Free
9 Software Foundation; either version 3, or (at your option) any later
10 version.
11 
12 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
13 WARRANTY; without even the implied warranty of MERCHANTABILITY or
14 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
15 for more details.
16 
17 You should have received a copy of the GNU General Public License
18 along with GCC; see the file COPYING3. If not see
19 <http://www.gnu.org/licenses/>. */
20 
21 #ifndef GCC_TREE_VECTORIZER_H
22 #define GCC_TREE_VECTORIZER_H
23 
24 #include "tree-data-ref.h"
25 #include "target.h"
26 #include "hash-table.h"
27 
28 typedef source_location LOC;
29 #define UNKNOWN_LOC UNKNOWN_LOCATION
30 #define EXPR_LOC(e) EXPR_LOCATION (e)
31 #define LOC_FILE(l) LOCATION_FILE (l)
32 #define LOC_LINE(l) LOCATION_LINE (l)
33 
34 /* Used for naming of new temporaries. */
35 enum vect_var_kind {
39 };
40 
41 /* Defines type of operation. */
42 enum operation_type {
43  unary_op = 1,
44  binary_op,
46 };
47 
48 /* Define type of available alignment support. */
55 };
56 
57 /* Define type of def-use cross-iteration cycle. */
68 };
69 
70 #define VECTORIZABLE_CYCLE_DEF(D) (((D) == vect_reduction_def) \
71  || ((D) == vect_double_reduction_def) \
72  || ((D) == vect_nested_cycle))
73 
74 /* Structure to encapsulate information about a group of like
75  instructions to be presented to the target cost model. */
76 typedef struct _stmt_info_for_cost {
77  int count;
79  gimple stmt;
80  int misalign;
82 
83 
85 
86 static inline void
88  enum vect_cost_for_stmt kind, gimple stmt, int misalign)
89 {
91  si.count = count;
92  si.kind = kind;
93  si.stmt = stmt;
94  si.misalign = misalign;
95  stmt_cost_vec->safe_push (si);
96 }
97 
98 /************************************************************************
99  SLP
100  ************************************************************************/
101 typedef struct _slp_tree *slp_tree;
102 
103 /* A computation tree of an SLP instance. Each node corresponds to a group of
104  stmts to be packed in a SIMD stmt. */
105 struct _slp_tree {
106  /* Nodes that contain def-stmts of this node statements operands. */
108  /* A group of scalar stmts to be vectorized together. */
110  /* Load permutation relative to the stores, NULL if there is no
111  permutation. */
113  /* Vectorized stmt/s. */
115  /* Number of vector stmts that are created to replace the group of scalar
116  stmts. It is calculated during the transformation phase as the number of
117  scalar elements in one scalar iteration (GROUP_SIZE) multiplied by VF
118  divided by vector size. */
119  unsigned int vec_stmts_size;
120 };
121 
123 /* SLP instance is a sequence of stmts in a loop that can be packed into
124  SIMD stmts. */
125 typedef struct _slp_instance {
126  /* The root of SLP tree. */
127  slp_tree root;
128 
129  /* Size of groups of scalar stmts that will be replaced by SIMD stmt/s. */
130  unsigned int group_size;
132  /* The unrolling factor required to vectorized this SLP instance. */
133  unsigned int unrolling_factor;
134 
135  /* Vectorization costs associated with SLP instance. */
137 
138  /* The group of nodes that contain loads of this SLP instance. */
140 
141  /* The first scalar load of the instance. The created vector loads will be
142  inserted before this statement. */
144 } *slp_instance;
146 
147 /* Access Functions. */
148 #define SLP_INSTANCE_TREE(S) (S)->root
149 #define SLP_INSTANCE_GROUP_SIZE(S) (S)->group_size
150 #define SLP_INSTANCE_UNROLLING_FACTOR(S) (S)->unrolling_factor
151 #define SLP_INSTANCE_BODY_COST_VEC(S) (S)->body_cost_vec
152 #define SLP_INSTANCE_LOADS(S) (S)->loads
153 #define SLP_INSTANCE_FIRST_LOAD_STMT(S) (S)->first_load
154 
155 #define SLP_TREE_CHILDREN(S) (S)->children
156 #define SLP_TREE_SCALAR_STMTS(S) (S)->stmts
157 #define SLP_TREE_VEC_STMTS(S) (S)->vec_stmts
158 #define SLP_TREE_NUMBER_OF_VEC_STMTS(S) (S)->vec_stmts_size
159 #define SLP_TREE_LOAD_PERMUTATION(S) (S)->load_permutation
160 
161 /* This structure is used in creation of an SLP tree. Each instance
162  corresponds to the same operand in a group of scalar stmts in an SLP
163  node. */
164 typedef struct _slp_oprnd_info
165 {
166  /* Def-stmts for the operands. */
168  /* Information about the first statement, its vector def-type, type, the
169  operand itself in case it's constant, and an indication if it's a pattern
170  stmt. */
174 } *slp_oprnd_info;
178 typedef struct _vect_peel_info
179 {
180  int npeel;
181  struct data_reference *dr;
182  unsigned int count;
183 } *vect_peel_info;
184 
186 {
187  struct _vect_peel_info peel_info;
188  unsigned int inside_cost;
189  unsigned int outside_cost;
192 
193 
194 /* Peeling hashtable helpers. */
196 struct peel_info_hasher : typed_free_remove <_vect_peel_info>
197 {
198  typedef _vect_peel_info value_type;
200  static inline hashval_t hash (const value_type *);
201  static inline bool equal (const value_type *, const compare_type *);
202 };
204 inline hashval_t
205 peel_info_hasher::hash (const value_type *peel_info)
206 {
207  return (hashval_t) peel_info->npeel;
208 }
209 
210 inline bool
211 peel_info_hasher::equal (const value_type *a, const compare_type *b)
212 {
213  return (a->npeel == b->npeel);
214 }
215 
216 
217 /*-----------------------------------------------------------------*/
218 /* Info on vectorized loops. */
219 /*-----------------------------------------------------------------*/
220 typedef struct _loop_vec_info {
221 
222  /* The loop to which this info struct refers to. */
223  struct loop *loop;
224 
225  /* The loop basic blocks. */
226  basic_block *bbs;
227 
228  /* Number of iterations. */
231 
232  /* Minimum number of iterations below which vectorization is expected to
233  not be profitable (as estimated by the cost model).
234  -1 indicates that vectorization will not be profitable.
235  FORNOW: This field is an int. Will be a tree in the future, to represent
236  values unknown at compile time. */
238 
239  /* Is the loop vectorizable? */
240  bool vectorizable;
241 
242  /* Unrolling factor */
244 
245  /* The loop location in the source. */
247 
248  /* Unknown DRs according to which loop was peeled. */
250 
251  /* peeling_for_alignment indicates whether peeling for alignment will take
252  place, and what the peeling factor should be:
253  peeling_for_alignment = X means:
254  If X=0: Peeling for alignment will not be applied.
255  If X>0: Peel first X iterations.
256  If X=-1: Generate a runtime test to calculate the number of iterations
257  to be peeled, using the dataref recorded in the field
258  unaligned_dr. */
260 
261  /* The mask used to check the alignment of pointers or arrays. */
262  int ptr_mask;
263 
264  /* The loop nest in which the data dependences are computed. */
267  /* All data references in the loop. */
269 
270  /* All data dependences in the loop. */
272 
273  /* Data Dependence Relations defining address ranges that are candidates
274  for a run-time aliasing check. */
276 
277  /* Statements in the loop that have data references that are candidates for a
278  runtime (loop versioning) misalignment check. */
280 
281  /* All interleaving chains of stores in the loop, represented by the first
282  stmt in the chain. */
284 
285  /* All SLP instances in the loop. This is a subset of the set of GROUP_STORES
286  of the loop. */
288 
289  /* The unrolling factor needed to SLP the loop. In case of that pure SLP is
290  applied to the loop, i.e., no unrolling is needed, this is 1. */
291  unsigned slp_unrolling_factor;
292 
293  /* Reduction cycles detected in the loop. Used in loop-aware SLP. */
295 
296  /* All reduction chains in the loop, represented by the first
297  stmt in the chain. */
299 
300  /* Hash table used to choose the best peeling option. */
302 
303  /* Cost data used by the target cost model. */
304  void *target_cost_data;
306  /* When we have grouped data accesses with gaps, we may introduce invalid
307  memory accesses. We peel the last iteration of the loop to prevent
308  this. */
310 
311  /* Reductions are canonicalized so that the last operand is the reduction
312  operand. If this places a constant into RHS1, this decanonicalizes
313  GIMPLE for other phases, so we must track when this has occurred and
314  fix it up. */
315  bool operands_swapped;
316 
317 } *loop_vec_info;
318 
319 /* Access Functions. */
320 #define LOOP_VINFO_LOOP(L) (L)->loop
321 #define LOOP_VINFO_BBS(L) (L)->bbs
322 #define LOOP_VINFO_NITERS(L) (L)->num_iters
323 /* Since LOOP_VINFO_NITERS can change after prologue peeling
324  retain total unchanged scalar loop iterations for cost model. */
325 #define LOOP_VINFO_NITERS_UNCHANGED(L) (L)->num_iters_unchanged
326 #define LOOP_VINFO_COST_MODEL_MIN_ITERS(L) (L)->min_profitable_iters
327 #define LOOP_VINFO_VECTORIZABLE_P(L) (L)->vectorizable
328 #define LOOP_VINFO_VECT_FACTOR(L) (L)->vectorization_factor
329 #define LOOP_VINFO_PTR_MASK(L) (L)->ptr_mask
330 #define LOOP_VINFO_LOOP_NEST(L) (L)->loop_nest
331 #define LOOP_VINFO_DATAREFS(L) (L)->datarefs
332 #define LOOP_VINFO_DDRS(L) (L)->ddrs
333 #define LOOP_VINFO_INT_NITERS(L) (TREE_INT_CST_LOW ((L)->num_iters))
334 #define LOOP_PEELING_FOR_ALIGNMENT(L) (L)->peeling_for_alignment
335 #define LOOP_VINFO_UNALIGNED_DR(L) (L)->unaligned_dr
336 #define LOOP_VINFO_MAY_MISALIGN_STMTS(L) (L)->may_misalign_stmts
337 #define LOOP_VINFO_LOC(L) (L)->loop_line_number
338 #define LOOP_VINFO_MAY_ALIAS_DDRS(L) (L)->may_alias_ddrs
339 #define LOOP_VINFO_GROUPED_STORES(L) (L)->grouped_stores
340 #define LOOP_VINFO_SLP_INSTANCES(L) (L)->slp_instances
341 #define LOOP_VINFO_SLP_UNROLLING_FACTOR(L) (L)->slp_unrolling_factor
342 #define LOOP_VINFO_REDUCTIONS(L) (L)->reductions
343 #define LOOP_VINFO_REDUCTION_CHAINS(L) (L)->reduction_chains
344 #define LOOP_VINFO_PEELING_HTAB(L) (L)->peeling_htab
345 #define LOOP_VINFO_TARGET_COST_DATA(L) (L)->target_cost_data
346 #define LOOP_VINFO_PEELING_FOR_GAPS(L) (L)->peeling_for_gaps
347 #define LOOP_VINFO_OPERANDS_SWAPPED(L) (L)->operands_swapped
348 
349 #define LOOP_REQUIRES_VERSIONING_FOR_ALIGNMENT(L) \
350 (L)->may_misalign_stmts.length () > 0
351 #define LOOP_REQUIRES_VERSIONING_FOR_ALIAS(L) \
352 (L)->may_alias_ddrs.length () > 0
353 
354 #define NITERS_KNOWN_P(n) \
355 (host_integerp ((n),0) \
356 && TREE_INT_CST_LOW ((n)) > 0)
358 #define LOOP_VINFO_NITERS_KNOWN_P(L) \
359 NITERS_KNOWN_P ((L)->num_iters)
360 
361 static inline loop_vec_info
363 {
364  return (loop_vec_info) loop->aux;
365 }
366 
367 static inline bool
368 nested_in_vect_loop_p (struct loop *loop, gimple stmt)
369 {
370  return (loop->inner
371  && (loop->inner == (gimple_bb (stmt))->loop_father));
372 }
373 
374 typedef struct _bb_vec_info {
375 
377  /* All interleaving chains of stores in the basic block, represented by the
378  first stmt in the chain. */
381  /* All SLP instances in the basic block. This is a subset of the set of
382  GROUP_STORES of the basic block. */
385  /* All data references in the basic block. */
388  /* All data dependences in the basic block. */
391  /* Cost data used by the target cost model. */
396 #define BB_VINFO_BB(B) (B)->bb
397 #define BB_VINFO_GROUPED_STORES(B) (B)->grouped_stores
398 #define BB_VINFO_SLP_INSTANCES(B) (B)->slp_instances
399 #define BB_VINFO_DATAREFS(B) (B)->datarefs
400 #define BB_VINFO_DDRS(B) (B)->ddrs
401 #define BB_VINFO_TARGET_COST_DATA(B) (B)->target_cost_data
403 static inline bb_vec_info
405 {
406  return (bb_vec_info) bb->aux;
407 }
408 
409 /*-----------------------------------------------------------------*/
410 /* Info on vectorized defs. */
411 /*-----------------------------------------------------------------*/
412 enum stmt_vec_info_type {
427 };
428 
429 /* Indicates whether/how a variable is used in the scope of loop/basic
430  block. */
433  /* The def is in the inner loop, and the use is in the outer loop, and the
434  use is a reduction stmt. */
436  /* The def is in the inner loop, and the use is in the outer loop (and is
437  not part of reduction). */
439 
440  /* defs that feed computations that end up (only) in a reduction. These
441  defs may be used by non-reduction stmts, but eventually, any
442  computations/values that are affected by these defs are used to compute
443  a reduction (i.e. don't get stored to memory, for example). We use this
444  to identify computations that we can change the order in which they are
445  computed. */
447 
449 };
450 
451 /* The type of vectorization that can be applied to the stmt: regular loop-based
452  vectorization; pure SLP - the stmt is a part of SLP instances and does not
453  have uses outside SLP instances; or hybrid SLP and loop-based - the stmt is
454  a part of SLP instance and also must be loop-based vectorized, since it has
455  uses outside SLP sequences.
457  In the loop context the meanings of pure and hybrid SLP are slightly
458  different. By saying that pure SLP is applied to the loop, we mean that we
459  exploit only intra-iteration parallelism in the loop; i.e., the loop can be
460  vectorized without doing any conceptual unrolling, cause we don't pack
461  together stmts from different iterations, only within a single iteration.
462  Loop hybrid SLP means that we exploit both intra-iteration and
463  inter-iteration parallelism (e.g., number of elements in the vector is 4
464  and the slp-group-size is 2, in which case we don't have enough parallelism
465  within an iteration, so we obtain the rest of the parallelism from subsequent
466  iterations by unrolling the loop by 2). */
467 enum slp_vect_type {
468  loop_vect = 0,
471 };
474 typedef struct data_reference *dr_p;
476 typedef struct _stmt_vec_info {
480  /* Indicates whether this stmts is part of a computation whose result is
481  used outside the loop. */
482  bool live;
484  /* Stmt is part of some pattern (computation idiom) */
485  bool in_pattern_p;
486 
487  /* The stmt to which this info struct refers to. */
488  gimple stmt;
490  /* The loop_vec_info with respect to which STMT is vectorized. */
492 
493  /* The vector type to be used for the LHS of this statement. */
495 
496  /* The vectorized version of the stmt. */
499 
504  /* Information about the data-ref (access function, etc),
505  relative to the inner-most containing loop. */
508  /* Information about the data-ref relative to this loop
509  nest (the loop that is being considered for vectorization). */
511  tree dr_init;
512  tree dr_offset;
513  tree dr_step;
515 
516  /* For loop PHI nodes, the evolution part of it. This makes sure
517  this information is still available in vect_update_ivs_after_vectorizer
518  where we may not be able to re-analyze the PHI nodes evolution as
519  peeling for the prologue loop can make it unanalyzable. The evolution
520  part is still correct though. */
522 
523  /* Used for various bookkeeping purposes, generally holding a pointer to
524  some other stmt S that is in some way "related" to this stmt.
525  Current use of this field is:
526  If this stmt is part of a pattern (i.e. the field 'in_pattern_p' is
527  true): S is the "pattern stmt" that represents (and replaces) the
528  sequence of stmts that constitutes the pattern. Similarly, the
529  related_stmt of the "pattern stmt" points back to this stmt (which is
530  the last stmt in the original sequence of stmts that constitutes the
531  pattern). */
533 
534  /* Used to keep a sequence of def stmts of a pattern stmt if such exists. */
537  /* List of datarefs that are known to have the same alignment as the dataref
538  of this stmt. */
541  /* Classify the def of this stmt. */
542  enum vect_def_type def_type;
543 
544  /* Whether the stmt is SLPed, loop-based vectorized, or both. */
546 
547  /* Interleaving and reduction chains info. */
548  /* First element in the group. */
550  /* Pointer to the next element in the group. */
552  /* For data-refs, in case that two or more stmts share data-ref, this is the
553  pointer to the previously detected stmt with the same dr. */
555  /* The size of the group. */
556  unsigned int size;
557  /* For stores, number of stores from this group seen. We vectorize the last
558  one. */
559  unsigned int store_count;
560  /* For loads only, the gap from the previous load. For consecutive loads, GAP
561  is 1. */
562  unsigned int gap;
563 
564  /* Not all stmts in the loop need to be vectorized. e.g, the increment
565  of the loop induction variable and computation of array indexes. relevant
566  indicates whether the stmt needs to be vectorized. */
567  enum vect_relevant relevant;
568 
569  /* The bb_vec_info with respect to which STMT is vectorized. */
571 
572  /* Is this statement vectorizable or should it be skipped in (partial)
573  vectorization. */
574  bool vectorizable;
575 
576  /* For loads only, true if this is a gather load. */
577  bool gather_p;
578  bool stride_load_p;
579 
580  /* For both loads and stores. */
584 /* Access Functions. */
585 #define STMT_VINFO_TYPE(S) (S)->type
586 #define STMT_VINFO_STMT(S) (S)->stmt
587 #define STMT_VINFO_LOOP_VINFO(S) (S)->loop_vinfo
588 #define STMT_VINFO_BB_VINFO(S) (S)->bb_vinfo
589 #define STMT_VINFO_RELEVANT(S) (S)->relevant
590 #define STMT_VINFO_LIVE_P(S) (S)->live
591 #define STMT_VINFO_VECTYPE(S) (S)->vectype
592 #define STMT_VINFO_VEC_STMT(S) (S)->vectorized_stmt
593 #define STMT_VINFO_VECTORIZABLE(S) (S)->vectorizable
594 #define STMT_VINFO_DATA_REF(S) (S)->data_ref_info
595 #define STMT_VINFO_GATHER_P(S) (S)->gather_p
596 #define STMT_VINFO_STRIDE_LOAD_P(S) (S)->stride_load_p
597 #define STMT_VINFO_SIMD_LANE_ACCESS_P(S) (S)->simd_lane_access_p
598 
599 #define STMT_VINFO_DR_BASE_ADDRESS(S) (S)->dr_base_address
600 #define STMT_VINFO_DR_INIT(S) (S)->dr_init
601 #define STMT_VINFO_DR_OFFSET(S) (S)->dr_offset
602 #define STMT_VINFO_DR_STEP(S) (S)->dr_step
603 #define STMT_VINFO_DR_ALIGNED_TO(S) (S)->dr_aligned_to
604 
605 #define STMT_VINFO_IN_PATTERN_P(S) (S)->in_pattern_p
606 #define STMT_VINFO_RELATED_STMT(S) (S)->related_stmt
607 #define STMT_VINFO_PATTERN_DEF_SEQ(S) (S)->pattern_def_seq
608 #define STMT_VINFO_SAME_ALIGN_REFS(S) (S)->same_align_refs
609 #define STMT_VINFO_DEF_TYPE(S) (S)->def_type
610 #define STMT_VINFO_GROUP_FIRST_ELEMENT(S) (S)->first_element
611 #define STMT_VINFO_GROUP_NEXT_ELEMENT(S) (S)->next_element
612 #define STMT_VINFO_GROUP_SIZE(S) (S)->size
613 #define STMT_VINFO_GROUP_STORE_COUNT(S) (S)->store_count
614 #define STMT_VINFO_GROUP_GAP(S) (S)->gap
615 #define STMT_VINFO_GROUP_SAME_DR_STMT(S) (S)->same_dr_stmt
616 #define STMT_VINFO_GROUPED_ACCESS(S) ((S)->first_element != NULL && (S)->data_ref_info)
617 #define STMT_VINFO_LOOP_PHI_EVOLUTION_PART(S) (S)->loop_phi_evolution_part
619 #define GROUP_FIRST_ELEMENT(S) (S)->first_element
620 #define GROUP_NEXT_ELEMENT(S) (S)->next_element
621 #define GROUP_SIZE(S) (S)->size
622 #define GROUP_STORE_COUNT(S) (S)->store_count
623 #define GROUP_GAP(S) (S)->gap
624 #define GROUP_SAME_DR_STMT(S) (S)->same_dr_stmt
625 
626 #define STMT_VINFO_RELEVANT_P(S) ((S)->relevant != vect_unused_in_scope)
627 
628 #define HYBRID_SLP_STMT(S) ((S)->slp_type == hybrid)
629 #define PURE_SLP_STMT(S) ((S)->slp_type == pure_slp)
630 #define STMT_SLP_TYPE(S) (S)->slp_type
632 struct dataref_aux {
633  tree base_decl;
634  bool base_misaligned;
636 };
637 
638 #define VECT_MAX_COST 1000
639 
640 /* The maximum number of intermediate steps required in multi-step type
641  conversion. */
642 #define MAX_INTERM_CVT_STEPS 3
643 
644 /* The maximum vectorization factor supported by any target (V32QI). */
645 #define MAX_VECTORIZATION_FACTOR 32
647 /* Avoid GTY(()) on stmt_vec_info. */
648 typedef void *vec_void_p;
649 
651 
653 void free_stmt_vec_info_vec (void);
654 
655 /* Return a stmt_vec_info corresponding to STMT. */
657 static inline stmt_vec_info
658 vinfo_for_stmt (gimple stmt)
659 {
660  unsigned int uid = gimple_uid (stmt);
661  if (uid == 0)
662  return NULL;
663 
664  return (stmt_vec_info) stmt_vec_info_vec[uid - 1];
665 }
667 /* Set vectorizer information INFO for STMT. */
668 
669 static inline void
671 {
672  unsigned int uid = gimple_uid (stmt);
673  if (uid == 0)
674  {
676  uid = stmt_vec_info_vec.length () + 1;
677  gimple_set_uid (stmt, uid);
678  stmt_vec_info_vec.safe_push ((vec_void_p) info);
679  }
680  else
681  stmt_vec_info_vec[uid - 1] = (vec_void_p) info;
682 }
684 /* Return the earlier statement between STMT1 and STMT2. */
686 static inline gimple
688 {
689  unsigned int uid1, uid2;
691  if (stmt1 == NULL)
692  return stmt2;
694  if (stmt2 == NULL)
695  return stmt1;
697  uid1 = gimple_uid (stmt1);
698  uid2 = gimple_uid (stmt2);
700  if (uid1 == 0 || uid2 == 0)
701  return NULL;
703  gcc_checking_assert (uid1 <= stmt_vec_info_vec.length ()
704  && uid2 <= stmt_vec_info_vec.length ());
706  if (uid1 < uid2)
707  return stmt1;
708  else
709  return stmt2;
710 }
712 /* Return the later statement between STMT1 and STMT2. */
714 static inline gimple
715 get_later_stmt (gimple stmt1, gimple stmt2)
716 {
717  unsigned int uid1, uid2;
719  if (stmt1 == NULL)
720  return stmt2;
721 
722  if (stmt2 == NULL)
723  return stmt1;
725  uid1 = gimple_uid (stmt1);
726  uid2 = gimple_uid (stmt2);
727 
728  if (uid1 == 0 || uid2 == 0)
729  return NULL;
730 
731  gcc_assert (uid1 <= stmt_vec_info_vec.length ());
732  gcc_assert (uid2 <= stmt_vec_info_vec.length ());
734  if (uid1 > uid2)
735  return stmt1;
736  else
737  return stmt2;
738 }
739 
740 /* Return TRUE if a statement represented by STMT_INFO is a part of a
741  pattern. */
742 
743 static inline bool
745 {
746  gimple related_stmt;
747  stmt_vec_info related_stmt_info;
748 
749  related_stmt = STMT_VINFO_RELATED_STMT (stmt_info);
750  if (related_stmt
751  && (related_stmt_info = vinfo_for_stmt (related_stmt))
752  && STMT_VINFO_IN_PATTERN_P (related_stmt_info))
753  return true;
754 
755  return false;
756 }
757 
758 /* Return true if BB is a loop header. */
759 
760 static inline bool
762 {
763  if (bb == (bb->loop_father)->header)
764  return true;
766  return false;
767 }
768 
769 /* Return pow2 (X). */
770 
771 static inline int
772 vect_pow2 (int x)
773 {
774  int i, res = 1;
775 
776  for (i = 0; i < x; i++)
777  res *= 2;
778 
779  return res;
780 }
781 
782 /* Alias targetm.vectorize.builtin_vectorization_cost. */
784 static inline int
786  tree vectype, int misalign)
787 {
788  return targetm.vectorize.builtin_vectorization_cost (type_of_cost,
789  vectype, misalign);
790 }
791 
792 /* Get cost by calling cost target builtin. */
793 
794 static inline
795 int vect_get_stmt_cost (enum vect_cost_for_stmt type_of_cost)
796 {
797  return builtin_vectorization_cost (type_of_cost, NULL, 0);
798 }
799 
800 /* Alias targetm.vectorize.init_cost. */
801 
802 static inline void *
803 init_cost (struct loop *loop_info)
804 {
805  return targetm.vectorize.init_cost (loop_info);
806 }
807 
808 /* Alias targetm.vectorize.add_stmt_cost. */
809 
810 static inline unsigned
811 add_stmt_cost (void *data, int count, enum vect_cost_for_stmt kind,
812  stmt_vec_info stmt_info, int misalign,
813  enum vect_cost_model_location where)
814 {
815  return targetm.vectorize.add_stmt_cost (data, count, kind,
816  stmt_info, misalign, where);
817 }
818 
819 /* Alias targetm.vectorize.finish_cost. */
820 
821 static inline void
822 finish_cost (void *data, unsigned *prologue_cost,
823  unsigned *body_cost, unsigned *epilogue_cost)
824 {
825  targetm.vectorize.finish_cost (data, prologue_cost, body_cost, epilogue_cost);
826 }
827 
828 /* Alias targetm.vectorize.destroy_cost_data. */
829 
830 static inline void
831 destroy_cost_data (void *data)
832 {
833  targetm.vectorize.destroy_cost_data (data);
834 }
835 
836 
837 /*-----------------------------------------------------------------*/
838 /* Info on data references alignment. */
839 /*-----------------------------------------------------------------*/
840 inline void
841 set_dr_misalignment (struct data_reference *dr, int val)
842 {
843  dataref_aux *data_aux = (dataref_aux *) dr->aux;
844 
845  if (!data_aux)
846  {
847  data_aux = XCNEW (dataref_aux);
848  dr->aux = data_aux;
849  }
850 
851  data_aux->misalignment = val;
852 }
853 
854 inline int
855 dr_misalignment (struct data_reference *dr)
856 {
857  gcc_assert (dr->aux);
858  return ((dataref_aux *) dr->aux)->misalignment;
859 }
861 /* Reflects actual alignment of first access in the vectorized loop,
862  taking into account peeling/versioning if applied. */
863 #define DR_MISALIGNMENT(DR) dr_misalignment (DR)
864 #define SET_DR_MISALIGNMENT(DR, VAL) set_dr_misalignment (DR, VAL)
865 
866 /* Return TRUE if the data access is aligned, and FALSE otherwise. */
867 
868 static inline bool
869 aligned_access_p (struct data_reference *data_ref_info)
870 {
871  return (DR_MISALIGNMENT (data_ref_info) == 0);
872 }
873 
874 /* Return TRUE if the alignment of the data access is known, and FALSE
875  otherwise. */
876 
877 static inline bool
878 known_alignment_for_access_p (struct data_reference *data_ref_info)
879 {
880  return (DR_MISALIGNMENT (data_ref_info) != -1);
881 }
882 
883 
884 /* Return true if the vect cost model is unlimited. */
885 static inline bool
887 {
888  return flag_vect_cost_model == VECT_COST_MODEL_UNLIMITED;
889 }
890 
891 /* Source location */
892 extern LOC vect_location;
893 
894 /*-----------------------------------------------------------------*/
895 /* Function prototypes. */
896 /*-----------------------------------------------------------------*/
898 /* Simple loop peeling and versioning utilities for vectorizer's purposes -
899  in tree-vect-loop-manip.c. */
900 extern void slpeel_make_loop_iterate_ntimes (struct loop *, tree);
901 extern bool slpeel_can_duplicate_loop_p (const struct loop *, const_edge);
902 struct loop *slpeel_tree_duplicate_loop_to_edge_cfg (struct loop *, edge);
903 extern void vect_loop_versioning (loop_vec_info, unsigned int, bool);
905  unsigned int, bool);
906 extern void vect_do_peeling_for_alignment (loop_vec_info, unsigned int, bool);
907 extern LOC find_loop_location (struct loop *);
909 
910 /* In tree-vect-stmts.c. */
911 extern unsigned int current_vector_size;
916  tree *, enum vect_def_type *);
918  bb_vec_info, gimple *,
919  tree *, enum vect_def_type *, tree *);
921  enum tree_code *, enum tree_code *,
922  int *, vec<tree> *);
924  enum tree_code *,
925  int *, vec<tree> *);
928 extern void free_stmt_vec_info (gimple stmt);
930 extern void vect_model_simple_cost (stmt_vec_info, int, enum vect_def_type *,
933 extern void vect_model_store_cost (stmt_vec_info, int, bool,
934  enum vect_def_type, slp_tree,
937 extern void vect_model_load_cost (stmt_vec_info, int, bool, slp_tree,
940 extern unsigned record_stmt_cost (stmt_vector_for_cost *, int,
942  int, enum vect_cost_model_location);
951  bool *, slp_tree, slp_instance);
952 extern void vect_remove_stores (gimple);
953 extern bool vect_analyze_stmt (gimple, bool *, slp_tree);
955  tree, int, slp_tree);
956 extern void vect_get_load_cost (struct data_reference *, int, bool,
957  unsigned int *, unsigned int *,
959  stmt_vector_for_cost *, bool);
960 extern void vect_get_store_cost (struct data_reference *, int,
961  unsigned int *, stmt_vector_for_cost *);
963 extern void vect_get_vec_defs (tree, tree, gimple, vec<tree> *,
964  vec<tree> *, slp_tree, int);
965 extern tree vect_gen_perm_mask (tree, unsigned char *);
966 
967 /* In tree-vect-data-refs.c. */
968 extern bool vect_can_force_dr_alignment_p (const_tree, unsigned int);
970  (struct data_reference *, bool);
981  int *);
982 extern bool vect_analyze_data_refs (loop_vec_info, bb_vec_info, int *);
983 extern tree vect_create_data_ref_ptr (gimple, tree, struct loop *, tree,
985  gimple *, bool, bool *);
989 extern bool vect_store_lanes_supported (tree, unsigned HOST_WIDE_INT);
990 extern bool vect_grouped_load_supported (tree, unsigned HOST_WIDE_INT);
991 extern bool vect_load_lanes_supported (tree, unsigned HOST_WIDE_INT);
992 extern void vect_permute_store_chain (vec<tree> ,unsigned int, gimple,
996  struct loop **);
1000 extern tree vect_get_new_vect_var (tree, enum vect_var_kind, const char *);
1002  tree, struct loop *);
1003 
1004 /* In tree-vect-loop.c. */
1005 /* FORNOW: Used in tree-parloops.c. */
1006 extern void destroy_loop_vec_info (loop_vec_info, bool);
1007 extern gimple vect_force_simple_reduction (loop_vec_info, gimple, bool, bool *);
1008 /* Drive for loop analysis stage. */
1009 extern loop_vec_info vect_analyze_loop (struct loop *);
1010 /* Drive for loop transformation stage. */
1011 extern void vect_transform_loop (loop_vec_info);
1012 extern loop_vec_info vect_analyze_loop_form (struct loop *);
1014  gimple *);
1016  slp_tree);
1019 extern int vect_min_worthwhile_factor (enum tree_code);
1020 extern int vect_get_known_peeling_cost (loop_vec_info, int, int *, int,
1024 
1025 /* In tree-vect-slp.c. */
1026 extern void vect_free_slp_instance (slp_instance);
1027 extern bool vect_transform_slp_perm_load (slp_tree, vec<tree> ,
1028  gimple_stmt_iterator *, int,
1029  slp_instance, bool);
1035 extern void vect_get_slp_defs (vec<tree> , slp_tree,
1036  vec<vec<tree> > *, int);
1037 
1040 extern void vect_slp_transform_bb (basic_block);
1041 
1042 /* In tree-vect-patterns.c. */
1043 /* Pattern recognition functions.
1044  Additional pattern recognition functions can (and will) be added
1045  in the future. */
1046 typedef gimple (* vect_recog_func_ptr) (vec<gimple> *, tree *, tree *);
1047 #define NUM_PATTERNS 11
1049 
1050 /* In tree-vectorizer.c. */
1051 unsigned vectorize_loops (void);
1053 
1054 #endif /* GCC_TREE_VECTORIZER_H */