1//===---- CGLoopInfo.cpp - LLVM CodeGen for loop metadata -*- C++ -*-------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "CGLoopInfo.h"
10#include "clang/AST/ASTContext.h"
11#include "clang/AST/Attr.h"
12#include "clang/AST/Expr.h"
13#include "clang/Basic/CodeGenOptions.h"
14#include "llvm/IR/BasicBlock.h"
15#include "llvm/IR/CFG.h"
16#include "llvm/IR/Constants.h"
17#include "llvm/IR/InstrTypes.h"
18#include "llvm/IR/Instructions.h"
19#include "llvm/IR/Metadata.h"
20using namespace clang::CodeGen;
21using namespace llvm;
22
23MDNode *
24LoopInfo::createLoopPropertiesMetadata(ArrayRef<Metadata *> LoopProperties) {
25  LLVMContext &Ctx = Header->getContext();
26  SmallVector<Metadata *, 4> NewLoopProperties;
27  TempMDTuple TempNode = MDNode::getTemporary(Ctx, None);
28  NewLoopProperties.push_back(TempNode.get());
29  NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
30
31  MDNode *LoopID = MDNode::getDistinct(Ctx, NewLoopProperties);
32  LoopID->replaceOperandWith(0, LoopID);
33  return LoopID;
34}
35
36MDNode *LoopInfo::createPipeliningMetadata(const LoopAttributes &Attrs,
37                                           ArrayRef<Metadata *> LoopProperties,
38                                           bool &HasUserTransforms) {
39  LLVMContext &Ctx = Header->getContext();
40
41  Optional<bool> Enabled;
42  if (Attrs.PipelineDisabled)
43    Enabled = false;
44  else if (Attrs.PipelineInitiationInterval != 0)
45    Enabled = true;
46
47  if (Enabled != true) {
48    SmallVector<Metadata *, 4> NewLoopProperties;
49    if (Enabled == false) {
50      NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
51      NewLoopProperties.push_back(
52          MDNode::get(Ctx, {MDString::get(Ctx, "llvm.loop.pipeline.disable"),
53                            ConstantAsMetadata::get(ConstantInt::get(
54                                llvm::Type::getInt1Ty(Ctx), 1))}));
55      LoopProperties = NewLoopProperties;
56    }
57    return createLoopPropertiesMetadata(LoopProperties);
58  }
59
60  SmallVector<Metadata *, 4> Args;
61  TempMDTuple TempNode = MDNode::getTemporary(Ctx, None);
62  Args.push_back(TempNode.get());
63  Args.append(LoopProperties.begin(), LoopProperties.end());
64
65  if (Attrs.PipelineInitiationInterval > 0) {
66    Metadata *Vals[] = {
67        MDString::get(Ctx, "llvm.loop.pipeline.initiationinterval"),
68        ConstantAsMetadata::get(ConstantInt::get(
69            llvm::Type::getInt32Ty(Ctx), Attrs.PipelineInitiationInterval))};
70    Args.push_back(MDNode::get(Ctx, Vals));
71  }
72
73  // No follow-up: This is the last transformation.
74
75  MDNode *LoopID = MDNode::getDistinct(Ctx, Args);
76  LoopID->replaceOperandWith(0, LoopID);
77  HasUserTransforms = true;
78  return LoopID;
79}
80
81MDNode *
82LoopInfo::createPartialUnrollMetadata(const LoopAttributes &Attrs,
83                                      ArrayRef<Metadata *> LoopProperties,
84                                      bool &HasUserTransforms) {
85  LLVMContext &Ctx = Header->getContext();
86
87  Optional<bool> Enabled;
88  if (Attrs.UnrollEnable == LoopAttributes::Disable)
89    Enabled = false;
90  else if (Attrs.UnrollEnable == LoopAttributes::Full)
91    Enabled = None;
92  else if (Attrs.UnrollEnable != LoopAttributes::Unspecified ||
93           Attrs.UnrollCount != 0)
94    Enabled = true;
95
96  if (Enabled != true) {
97    // createFullUnrollMetadata will already have added llvm.loop.unroll.disable
98    // if unrolling is disabled.
99    return createPipeliningMetadata(Attrs, LoopProperties, HasUserTransforms);
100  }
101
102  SmallVector<Metadata *, 4> FollowupLoopProperties;
103
104  // Apply all loop properties to the unrolled loop.
105  FollowupLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
106
107  // Don't unroll an already unrolled loop.
108  FollowupLoopProperties.push_back(
109      MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.unroll.disable")));
110
111  bool FollowupHasTransforms = false;
112  MDNode *Followup = createPipeliningMetadata(Attrs, FollowupLoopProperties,
113                                              FollowupHasTransforms);
114
115  SmallVector<Metadata *, 4> Args;
116  TempMDTuple TempNode = MDNode::getTemporary(Ctx, None);
117  Args.push_back(TempNode.get());
118  Args.append(LoopProperties.begin(), LoopProperties.end());
119
120  // Setting unroll.count
121  if (Attrs.UnrollCount > 0) {
122    Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.unroll.count"),
123                        ConstantAsMetadata::get(ConstantInt::get(
124                            llvm::Type::getInt32Ty(Ctx), Attrs.UnrollCount))};
125    Args.push_back(MDNode::get(Ctx, Vals));
126  }
127
128  // Setting unroll.full or unroll.disable
129  if (Attrs.UnrollEnable == LoopAttributes::Enable) {
130    Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.unroll.enable")};
131    Args.push_back(MDNode::get(Ctx, Vals));
132  }
133
134  if (FollowupHasTransforms)
135    Args.push_back(MDNode::get(
136        Ctx, {MDString::get(Ctx, "llvm.loop.unroll.followup_all"), Followup}));
137
138  MDNode *LoopID = MDNode::getDistinct(Ctx, Args);
139  LoopID->replaceOperandWith(0, LoopID);
140  HasUserTransforms = true;
141  return LoopID;
142}
143
144MDNode *
145LoopInfo::createUnrollAndJamMetadata(const LoopAttributes &Attrs,
146                                     ArrayRef<Metadata *> LoopProperties,
147                                     bool &HasUserTransforms) {
148  LLVMContext &Ctx = Header->getContext();
149
150  Optional<bool> Enabled;
151  if (Attrs.UnrollAndJamEnable == LoopAttributes::Disable)
152    Enabled = false;
153  else if (Attrs.UnrollAndJamEnable == LoopAttributes::Enable ||
154           Attrs.UnrollAndJamCount != 0)
155    Enabled = true;
156
157  if (Enabled != true) {
158    SmallVector<Metadata *, 4> NewLoopProperties;
159    if (Enabled == false) {
160      NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
161      NewLoopProperties.push_back(MDNode::get(
162          Ctx, MDString::get(Ctx, "llvm.loop.unroll_and_jam.disable")));
163      LoopProperties = NewLoopProperties;
164    }
165    return createPartialUnrollMetadata(Attrs, LoopProperties,
166                                       HasUserTransforms);
167  }
168
169  SmallVector<Metadata *, 4> FollowupLoopProperties;
170  FollowupLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
171  FollowupLoopProperties.push_back(
172      MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.unroll_and_jam.disable")));
173
174  bool FollowupHasTransforms = false;
175  MDNode *Followup = createPartialUnrollMetadata(Attrs, FollowupLoopProperties,
176                                                 FollowupHasTransforms);
177
178  SmallVector<Metadata *, 4> Args;
179  TempMDTuple TempNode = MDNode::getTemporary(Ctx, None);
180  Args.push_back(TempNode.get());
181  Args.append(LoopProperties.begin(), LoopProperties.end());
182
183  // Setting unroll_and_jam.count
184  if (Attrs.UnrollAndJamCount > 0) {
185    Metadata *Vals[] = {
186        MDString::get(Ctx, "llvm.loop.unroll_and_jam.count"),
187        ConstantAsMetadata::get(ConstantInt::get(llvm::Type::getInt32Ty(Ctx),
188                                                 Attrs.UnrollAndJamCount))};
189    Args.push_back(MDNode::get(Ctx, Vals));
190  }
191
192  if (Attrs.UnrollAndJamEnable == LoopAttributes::Enable) {
193    Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.unroll_and_jam.enable")};
194    Args.push_back(MDNode::get(Ctx, Vals));
195  }
196
197  if (FollowupHasTransforms)
198    Args.push_back(MDNode::get(
199        Ctx, {MDString::get(Ctx, "llvm.loop.unroll_and_jam.followup_outer"),
200              Followup}));
201
202  if (UnrollAndJamInnerFollowup)
203    Args.push_back(MDNode::get(
204        Ctx, {MDString::get(Ctx, "llvm.loop.unroll_and_jam.followup_inner"),
205              UnrollAndJamInnerFollowup}));
206
207  MDNode *LoopID = MDNode::getDistinct(Ctx, Args);
208  LoopID->replaceOperandWith(0, LoopID);
209  HasUserTransforms = true;
210  return LoopID;
211}
212
213MDNode *
214LoopInfo::createLoopVectorizeMetadata(const LoopAttributes &Attrs,
215                                      ArrayRef<Metadata *> LoopProperties,
216                                      bool &HasUserTransforms) {
217  LLVMContext &Ctx = Header->getContext();
218
219  Optional<bool> Enabled;
220  if (Attrs.VectorizeEnable == LoopAttributes::Disable)
221    Enabled = false;
222  else if (Attrs.VectorizeEnable != LoopAttributes::Unspecified ||
223           Attrs.VectorizePredicateEnable != LoopAttributes::Unspecified ||
224           Attrs.InterleaveCount != 0 || Attrs.VectorizeWidth != 0)
225    Enabled = true;
226
227  if (Enabled != true) {
228    SmallVector<Metadata *, 4> NewLoopProperties;
229    if (Enabled == false) {
230      NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
231      NewLoopProperties.push_back(
232          MDNode::get(Ctx, {MDString::get(Ctx, "llvm.loop.vectorize.enable"),
233                            ConstantAsMetadata::get(ConstantInt::get(
234                                llvm::Type::getInt1Ty(Ctx), 0))}));
235      LoopProperties = NewLoopProperties;
236    }
237    return createUnrollAndJamMetadata(Attrs, LoopProperties, HasUserTransforms);
238  }
239
240  // Apply all loop properties to the vectorized loop.
241  SmallVector<Metadata *, 4> FollowupLoopProperties;
242  FollowupLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
243
244  // Don't vectorize an already vectorized loop.
245  FollowupLoopProperties.push_back(
246      MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.isvectorized")));
247
248  bool FollowupHasTransforms = false;
249  MDNode *Followup = createUnrollAndJamMetadata(Attrs, FollowupLoopProperties,
250                                                FollowupHasTransforms);
251
252  SmallVector<Metadata *, 4> Args;
253  TempMDTuple TempNode = MDNode::getTemporary(Ctx, None);
254  Args.push_back(TempNode.get());
255  Args.append(LoopProperties.begin(), LoopProperties.end());
256
257  // Setting vectorize.predicate
258  bool IsVectorPredicateEnabled = false;
259  if (Attrs.VectorizePredicateEnable != LoopAttributes::Unspecified &&
260      Attrs.VectorizeEnable != LoopAttributes::Disable &&
261      Attrs.VectorizeWidth < 1) {
262
263    IsVectorPredicateEnabled =
264        (Attrs.VectorizePredicateEnable == LoopAttributes::Enable);
265
266    Metadata *Vals[] = {
267        MDString::get(Ctx, "llvm.loop.vectorize.predicate.enable"),
268        ConstantAsMetadata::get(ConstantInt::get(llvm::Type::getInt1Ty(Ctx),
269                                                 IsVectorPredicateEnabled))};
270    Args.push_back(MDNode::get(Ctx, Vals));
271  }
272
273  // Setting vectorize.width
274  if (Attrs.VectorizeWidth > 0) {
275    Metadata *Vals[] = {
276        MDString::get(Ctx, "llvm.loop.vectorize.width"),
277        ConstantAsMetadata::get(ConstantInt::get(llvm::Type::getInt32Ty(Ctx),
278                                                 Attrs.VectorizeWidth))};
279    Args.push_back(MDNode::get(Ctx, Vals));
280  }
281
282  // Setting interleave.count
283  if (Attrs.InterleaveCount > 0) {
284    Metadata *Vals[] = {
285        MDString::get(Ctx, "llvm.loop.interleave.count"),
286        ConstantAsMetadata::get(ConstantInt::get(llvm::Type::getInt32Ty(Ctx),
287                                                 Attrs.InterleaveCount))};
288    Args.push_back(MDNode::get(Ctx, Vals));
289  }
290
291  // vectorize.enable is set if:
292  // 1) loop hint vectorize.enable is set, or
293  // 2) it is implied when vectorize.predicate is set, or
294  // 3) it is implied when vectorize.width is set.
295  if (Attrs.VectorizeEnable != LoopAttributes::Unspecified ||
296      IsVectorPredicateEnabled ||
297      Attrs.VectorizeWidth > 1 ) {
298    bool AttrVal = Attrs.VectorizeEnable != LoopAttributes::Disable;
299    Args.push_back(
300        MDNode::get(Ctx, {MDString::get(Ctx, "llvm.loop.vectorize.enable"),
301                          ConstantAsMetadata::get(ConstantInt::get(
302                              llvm::Type::getInt1Ty(Ctx), AttrVal))}));
303  }
304
305  if (FollowupHasTransforms)
306    Args.push_back(MDNode::get(
307        Ctx,
308        {MDString::get(Ctx, "llvm.loop.vectorize.followup_all"), Followup}));
309
310  MDNode *LoopID = MDNode::get(Ctx, Args);
311  LoopID->replaceOperandWith(0, LoopID);
312  HasUserTransforms = true;
313  return LoopID;
314}
315
316MDNode *
317LoopInfo::createLoopDistributeMetadata(const LoopAttributes &Attrs,
318                                       ArrayRef<Metadata *> LoopProperties,
319                                       bool &HasUserTransforms) {
320  LLVMContext &Ctx = Header->getContext();
321
322  Optional<bool> Enabled;
323  if (Attrs.DistributeEnable == LoopAttributes::Disable)
324    Enabled = false;
325  if (Attrs.DistributeEnable == LoopAttributes::Enable)
326    Enabled = true;
327
328  if (Enabled != true) {
329    SmallVector<Metadata *, 4> NewLoopProperties;
330    if (Enabled == false) {
331      NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
332      NewLoopProperties.push_back(
333          MDNode::get(Ctx, {MDString::get(Ctx, "llvm.loop.distribute.enable"),
334                            ConstantAsMetadata::get(ConstantInt::get(
335                                llvm::Type::getInt1Ty(Ctx), 0))}));
336      LoopProperties = NewLoopProperties;
337    }
338    return createLoopVectorizeMetadata(Attrs, LoopProperties,
339                                       HasUserTransforms);
340  }
341
342  bool FollowupHasTransforms = false;
343  MDNode *Followup =
344      createLoopVectorizeMetadata(Attrs, LoopProperties, FollowupHasTransforms);
345
346  SmallVector<Metadata *, 4> Args;
347  TempMDTuple TempNode = MDNode::getTemporary(Ctx, None);
348  Args.push_back(TempNode.get());
349  Args.append(LoopProperties.begin(), LoopProperties.end());
350
351  Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.distribute.enable"),
352                      ConstantAsMetadata::get(ConstantInt::get(
353                          llvm::Type::getInt1Ty(Ctx),
354                          (Attrs.DistributeEnable == LoopAttributes::Enable)))};
355  Args.push_back(MDNode::get(Ctx, Vals));
356
357  if (FollowupHasTransforms)
358    Args.push_back(MDNode::get(
359        Ctx,
360        {MDString::get(Ctx, "llvm.loop.distribute.followup_all"), Followup}));
361
362  MDNode *LoopID = MDNode::get(Ctx, Args);
363  LoopID->replaceOperandWith(0, LoopID);
364  HasUserTransforms = true;
365  return LoopID;
366}
367
368MDNode *LoopInfo::createFullUnrollMetadata(const LoopAttributes &Attrs,
369                                           ArrayRef<Metadata *> LoopProperties,
370                                           bool &HasUserTransforms) {
371  LLVMContext &Ctx = Header->getContext();
372
373  Optional<bool> Enabled;
374  if (Attrs.UnrollEnable == LoopAttributes::Disable)
375    Enabled = false;
376  else if (Attrs.UnrollEnable == LoopAttributes::Full)
377    Enabled = true;
378
379  if (Enabled != true) {
380    SmallVector<Metadata *, 4> NewLoopProperties;
381    if (Enabled == false) {
382      NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
383      NewLoopProperties.push_back(
384          MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.unroll.disable")));
385      LoopProperties = NewLoopProperties;
386    }
387    return createLoopDistributeMetadata(Attrs, LoopProperties,
388                                        HasUserTransforms);
389  }
390
391  SmallVector<Metadata *, 4> Args;
392  TempMDTuple TempNode = MDNode::getTemporary(Ctx, None);
393  Args.push_back(TempNode.get());
394  Args.append(LoopProperties.begin(), LoopProperties.end());
395  Args.push_back(MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.unroll.full")));
396
397  // No follow-up: there is no loop after full unrolling.
398  // TODO: Warn if there are transformations after full unrolling.
399
400  MDNode *LoopID = MDNode::getDistinct(Ctx, Args);
401  LoopID->replaceOperandWith(0, LoopID);
402  HasUserTransforms = true;
403  return LoopID;
404}
405
406MDNode *LoopInfo::createMetadata(
407    const LoopAttributes &Attrs,
408    llvm::ArrayRef<llvm::Metadata *> AdditionalLoopProperties,
409    bool &HasUserTransforms) {
410  SmallVector<Metadata *, 3> LoopProperties;
411
412  // If we have a valid start debug location for the loop, add it.
413  if (StartLoc) {
414    LoopProperties.push_back(StartLoc.getAsMDNode());
415
416    // If we also have a valid end debug location for the loop, add it.
417    if (EndLoc)
418      LoopProperties.push_back(EndLoc.getAsMDNode());
419  }
420
421  assert(!!AccGroup == Attrs.IsParallel &&
422         "There must be an access group iff the loop is parallel");
423  if (Attrs.IsParallel) {
424    LLVMContext &Ctx = Header->getContext();
425    LoopProperties.push_back(MDNode::get(
426        Ctx, {MDString::get(Ctx, "llvm.loop.parallel_accesses"), AccGroup}));
427  }
428
429  LoopProperties.insert(LoopProperties.end(), AdditionalLoopProperties.begin(),
430                        AdditionalLoopProperties.end());
431  return createFullUnrollMetadata(Attrs, LoopProperties, HasUserTransforms);
432}
433
434LoopAttributes::LoopAttributes(bool IsParallel)
435    : IsParallel(IsParallel), VectorizeEnable(LoopAttributes::Unspecified),
436      UnrollEnable(LoopAttributes::Unspecified),
437      UnrollAndJamEnable(LoopAttributes::Unspecified),
438      VectorizePredicateEnable(LoopAttributes::Unspecified), VectorizeWidth(0),
439      InterleaveCount(0), UnrollCount(0), UnrollAndJamCount(0),
440      DistributeEnable(LoopAttributes::Unspecified), PipelineDisabled(false),
441      PipelineInitiationInterval(0) {}
442
443void LoopAttributes::clear() {
444  IsParallel = false;
445  VectorizeWidth = 0;
446  InterleaveCount = 0;
447  UnrollCount = 0;
448  UnrollAndJamCount = 0;
449  VectorizeEnable = LoopAttributes::Unspecified;
450  UnrollEnable = LoopAttributes::Unspecified;
451  UnrollAndJamEnable = LoopAttributes::Unspecified;
452  VectorizePredicateEnable = LoopAttributes::Unspecified;
453  DistributeEnable = LoopAttributes::Unspecified;
454  PipelineDisabled = false;
455  PipelineInitiationInterval = 0;
456}
457
458LoopInfo::LoopInfo(BasicBlock *Header, const LoopAttributes &Attrs,
459                   const llvm::DebugLoc &StartLoc, const llvm::DebugLoc &EndLoc,
460                   LoopInfo *Parent)
461    : Header(Header), Attrs(Attrs), StartLoc(StartLoc), EndLoc(EndLoc),
462      Parent(Parent) {
463
464  if (Attrs.IsParallel) {
465    // Create an access group for this loop.
466    LLVMContext &Ctx = Header->getContext();
467    AccGroup = MDNode::getDistinct(Ctx, {});
468  }
469
470  if (!Attrs.IsParallel && Attrs.VectorizeWidth == 0 &&
471      Attrs.InterleaveCount == 0 && Attrs.UnrollCount == 0 &&
472      Attrs.UnrollAndJamCount == 0 && !Attrs.PipelineDisabled &&
473      Attrs.PipelineInitiationInterval == 0 &&
474      Attrs.VectorizePredicateEnable == LoopAttributes::Unspecified &&
475      Attrs.VectorizeEnable == LoopAttributes::Unspecified &&
476      Attrs.UnrollEnable == LoopAttributes::Unspecified &&
477      Attrs.UnrollAndJamEnable == LoopAttributes::Unspecified &&
478      Attrs.DistributeEnable == LoopAttributes::Unspecified && !StartLoc &&
479      !EndLoc)
480    return;
481
482  TempLoopID = MDNode::getTemporary(Header->getContext(), None);
483}
484
485void LoopInfo::finish() {
486  // We did not annotate the loop body instructions because there are no
487  // attributes for this loop.
488  if (!TempLoopID)
489    return;
490
491  MDNode *LoopID;
492  LoopAttributes CurLoopAttr = Attrs;
493  LLVMContext &Ctx = Header->getContext();
494
495  if (Parent && (Parent->Attrs.UnrollAndJamEnable ||
496                 Parent->Attrs.UnrollAndJamCount != 0)) {
497    // Parent unroll-and-jams this loop.
498    // Split the transformations in those that happens before the unroll-and-jam
499    // and those after.
500
501    LoopAttributes BeforeJam, AfterJam;
502
503    BeforeJam.IsParallel = AfterJam.IsParallel = Attrs.IsParallel;
504
505    BeforeJam.VectorizeWidth = Attrs.VectorizeWidth;
506    BeforeJam.InterleaveCount = Attrs.InterleaveCount;
507    BeforeJam.VectorizeEnable = Attrs.VectorizeEnable;
508    BeforeJam.DistributeEnable = Attrs.DistributeEnable;
509    BeforeJam.VectorizePredicateEnable = Attrs.VectorizePredicateEnable;
510
511    switch (Attrs.UnrollEnable) {
512    case LoopAttributes::Unspecified:
513    case LoopAttributes::Disable:
514      BeforeJam.UnrollEnable = Attrs.UnrollEnable;
515      AfterJam.UnrollEnable = Attrs.UnrollEnable;
516      break;
517    case LoopAttributes::Full:
518      BeforeJam.UnrollEnable = LoopAttributes::Full;
519      break;
520    case LoopAttributes::Enable:
521      AfterJam.UnrollEnable = LoopAttributes::Enable;
522      break;
523    }
524
525    AfterJam.VectorizePredicateEnable = Attrs.VectorizePredicateEnable;
526    AfterJam.UnrollCount = Attrs.UnrollCount;
527    AfterJam.PipelineDisabled = Attrs.PipelineDisabled;
528    AfterJam.PipelineInitiationInterval = Attrs.PipelineInitiationInterval;
529
530    // If this loop is subject of an unroll-and-jam by the parent loop, and has
531    // an unroll-and-jam annotation itself, we have to decide whether to first
532    // apply the parent's unroll-and-jam or this loop's unroll-and-jam. The
533    // UnrollAndJam pass processes loops from inner to outer, so we apply the
534    // inner first.
535    BeforeJam.UnrollAndJamCount = Attrs.UnrollAndJamCount;
536    BeforeJam.UnrollAndJamEnable = Attrs.UnrollAndJamEnable;
537
538    // Set the inner followup metadata to process by the outer loop. Only
539    // consider the first inner loop.
540    if (!Parent->UnrollAndJamInnerFollowup) {
541      // Splitting the attributes into a BeforeJam and an AfterJam part will
542      // stop 'llvm.loop.isvectorized' (generated by vectorization in BeforeJam)
543      // to be forwarded to the AfterJam part. We detect the situation here and
544      // add it manually.
545      SmallVector<Metadata *, 1> BeforeLoopProperties;
546      if (BeforeJam.VectorizeEnable != LoopAttributes::Unspecified ||
547          BeforeJam.VectorizePredicateEnable != LoopAttributes::Unspecified ||
548          BeforeJam.InterleaveCount != 0 || BeforeJam.VectorizeWidth != 0)
549        BeforeLoopProperties.push_back(
550            MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.isvectorized")));
551
552      bool InnerFollowupHasTransform = false;
553      MDNode *InnerFollowup = createMetadata(AfterJam, BeforeLoopProperties,
554                                             InnerFollowupHasTransform);
555      if (InnerFollowupHasTransform)
556        Parent->UnrollAndJamInnerFollowup = InnerFollowup;
557    }
558
559    CurLoopAttr = BeforeJam;
560  }
561
562  bool HasUserTransforms = false;
563  LoopID = createMetadata(CurLoopAttr, {}, HasUserTransforms);
564  TempLoopID->replaceAllUsesWith(LoopID);
565}
566
567void LoopInfoStack::push(BasicBlock *Header, const llvm::DebugLoc &StartLoc,
568                         const llvm::DebugLoc &EndLoc) {
569  Active.emplace_back(
570      new LoopInfo(Header, StagedAttrs, StartLoc, EndLoc,
571                   Active.empty() ? nullptr : Active.back().get()));
572  // Clear the attributes so nested loops do not inherit them.
573  StagedAttrs.clear();
574}
575
576void LoopInfoStack::push(BasicBlock *Header, clang::ASTContext &Ctx,
577                         const clang::CodeGenOptions &CGOpts,
578                         ArrayRef<const clang::Attr *> Attrs,
579                         const llvm::DebugLoc &StartLoc,
580                         const llvm::DebugLoc &EndLoc) {
581
582  // Identify loop hint attributes from Attrs.
583  for (const auto *Attr : Attrs) {
584    const LoopHintAttr *LH = dyn_cast<LoopHintAttr>(Attr);
585    const OpenCLUnrollHintAttr *OpenCLHint =
586        dyn_cast<OpenCLUnrollHintAttr>(Attr);
587
588    // Skip non loop hint attributes
589    if (!LH && !OpenCLHint) {
590      continue;
591    }
592
593    LoopHintAttr::OptionType Option = LoopHintAttr::Unroll;
594    LoopHintAttr::LoopHintState State = LoopHintAttr::Disable;
595    unsigned ValueInt = 1;
596    // Translate opencl_unroll_hint attribute argument to
597    // equivalent LoopHintAttr enums.
598    // OpenCL v2.0 s6.11.5:
599    // 0 - enable unroll (no argument).
600    // 1 - disable unroll.
601    // other positive integer n - unroll by n.
602    if (OpenCLHint) {
603      ValueInt = OpenCLHint->getUnrollHint();
604      if (ValueInt == 0) {
605        State = LoopHintAttr::Enable;
606      } else if (ValueInt != 1) {
607        Option = LoopHintAttr::UnrollCount;
608        State = LoopHintAttr::Numeric;
609      }
610    } else if (LH) {
611      auto *ValueExpr = LH->getValue();
612      if (ValueExpr) {
613        llvm::APSInt ValueAPS = ValueExpr->EvaluateKnownConstInt(Ctx);
614        ValueInt = ValueAPS.getSExtValue();
615      }
616
617      Option = LH->getOption();
618      State = LH->getState();
619    }
620    switch (State) {
621    case LoopHintAttr::Disable:
622      switch (Option) {
623      case LoopHintAttr::Vectorize:
624        // Disable vectorization by specifying a width of 1.
625        setVectorizeWidth(1);
626        break;
627      case LoopHintAttr::Interleave:
628        // Disable interleaving by speciyfing a count of 1.
629        setInterleaveCount(1);
630        break;
631      case LoopHintAttr::Unroll:
632        setUnrollState(LoopAttributes::Disable);
633        break;
634      case LoopHintAttr::UnrollAndJam:
635        setUnrollAndJamState(LoopAttributes::Disable);
636        break;
637      case LoopHintAttr::VectorizePredicate:
638        setVectorizePredicateState(LoopAttributes::Disable);
639        break;
640      case LoopHintAttr::Distribute:
641        setDistributeState(false);
642        break;
643      case LoopHintAttr::PipelineDisabled:
644        setPipelineDisabled(true);
645        break;
646      case LoopHintAttr::UnrollCount:
647      case LoopHintAttr::UnrollAndJamCount:
648      case LoopHintAttr::VectorizeWidth:
649      case LoopHintAttr::InterleaveCount:
650      case LoopHintAttr::PipelineInitiationInterval:
651        llvm_unreachable("Options cannot be disabled.");
652        break;
653      }
654      break;
655    case LoopHintAttr::Enable:
656      switch (Option) {
657      case LoopHintAttr::Vectorize:
658      case LoopHintAttr::Interleave:
659        setVectorizeEnable(true);
660        break;
661      case LoopHintAttr::Unroll:
662        setUnrollState(LoopAttributes::Enable);
663        break;
664      case LoopHintAttr::UnrollAndJam:
665        setUnrollAndJamState(LoopAttributes::Enable);
666        break;
667      case LoopHintAttr::VectorizePredicate:
668        setVectorizePredicateState(LoopAttributes::Enable);
669        break;
670      case LoopHintAttr::Distribute:
671        setDistributeState(true);
672        break;
673      case LoopHintAttr::UnrollCount:
674      case LoopHintAttr::UnrollAndJamCount:
675      case LoopHintAttr::VectorizeWidth:
676      case LoopHintAttr::InterleaveCount:
677      case LoopHintAttr::PipelineDisabled:
678      case LoopHintAttr::PipelineInitiationInterval:
679        llvm_unreachable("Options cannot enabled.");
680        break;
681      }
682      break;
683    case LoopHintAttr::AssumeSafety:
684      switch (Option) {
685      case LoopHintAttr::Vectorize:
686      case LoopHintAttr::Interleave:
687        // Apply "llvm.mem.parallel_loop_access" metadata to load/stores.
688        setParallel(true);
689        setVectorizeEnable(true);
690        break;
691      case LoopHintAttr::Unroll:
692      case LoopHintAttr::UnrollAndJam:
693      case LoopHintAttr::VectorizePredicate:
694      case LoopHintAttr::UnrollCount:
695      case LoopHintAttr::UnrollAndJamCount:
696      case LoopHintAttr::VectorizeWidth:
697      case LoopHintAttr::InterleaveCount:
698      case LoopHintAttr::Distribute:
699      case LoopHintAttr::PipelineDisabled:
700      case LoopHintAttr::PipelineInitiationInterval:
701        llvm_unreachable("Options cannot be used to assume mem safety.");
702        break;
703      }
704      break;
705    case LoopHintAttr::Full:
706      switch (Option) {
707      case LoopHintAttr::Unroll:
708        setUnrollState(LoopAttributes::Full);
709        break;
710      case LoopHintAttr::UnrollAndJam:
711        setUnrollAndJamState(LoopAttributes::Full);
712        break;
713      case LoopHintAttr::Vectorize:
714      case LoopHintAttr::Interleave:
715      case LoopHintAttr::UnrollCount:
716      case LoopHintAttr::UnrollAndJamCount:
717      case LoopHintAttr::VectorizeWidth:
718      case LoopHintAttr::InterleaveCount:
719      case LoopHintAttr::Distribute:
720      case LoopHintAttr::PipelineDisabled:
721      case LoopHintAttr::PipelineInitiationInterval:
722      case LoopHintAttr::VectorizePredicate:
723        llvm_unreachable("Options cannot be used with 'full' hint.");
724        break;
725      }
726      break;
727    case LoopHintAttr::Numeric:
728      switch (Option) {
729      case LoopHintAttr::VectorizeWidth:
730        setVectorizeWidth(ValueInt);
731        break;
732      case LoopHintAttr::InterleaveCount:
733        setInterleaveCount(ValueInt);
734        break;
735      case LoopHintAttr::UnrollCount:
736        setUnrollCount(ValueInt);
737        break;
738      case LoopHintAttr::UnrollAndJamCount:
739        setUnrollAndJamCount(ValueInt);
740        break;
741      case LoopHintAttr::PipelineInitiationInterval:
742        setPipelineInitiationInterval(ValueInt);
743        break;
744      case LoopHintAttr::Unroll:
745      case LoopHintAttr::UnrollAndJam:
746      case LoopHintAttr::VectorizePredicate:
747      case LoopHintAttr::Vectorize:
748      case LoopHintAttr::Interleave:
749      case LoopHintAttr::Distribute:
750      case LoopHintAttr::PipelineDisabled:
751        llvm_unreachable("Options cannot be assigned a value.");
752        break;
753      }
754      break;
755    }
756  }
757
758  if (CGOpts.OptimizationLevel > 0)
759    // Disable unrolling for the loop, if unrolling is disabled (via
760    // -fno-unroll-loops) and no pragmas override the decision.
761    if (!CGOpts.UnrollLoops &&
762        (StagedAttrs.UnrollEnable == LoopAttributes::Unspecified &&
763         StagedAttrs.UnrollCount == 0))
764      setUnrollState(LoopAttributes::Disable);
765
766  /// Stage the attributes.
767  push(Header, StartLoc, EndLoc);
768}
769
770void LoopInfoStack::pop() {
771  assert(!Active.empty() && "No active loops to pop");
772  Active.back()->finish();
773  Active.pop_back();
774}
775
776void LoopInfoStack::InsertHelper(Instruction *I) const {
777  if (I->mayReadOrWriteMemory()) {
778    SmallVector<Metadata *, 4> AccessGroups;
779    for (const auto &AL : Active) {
780      // Here we assume that every loop that has an access group is parallel.
781      if (MDNode *Group = AL->getAccessGroup())
782        AccessGroups.push_back(Group);
783    }
784    MDNode *UnionMD = nullptr;
785    if (AccessGroups.size() == 1)
786      UnionMD = cast<MDNode>(AccessGroups[0]);
787    else if (AccessGroups.size() >= 2)
788      UnionMD = MDNode::get(I->getContext(), AccessGroups);
789    I->setMetadata("llvm.access.group", UnionMD);
790  }
791
792  if (!hasInfo())
793    return;
794
795  const LoopInfo &L = getInfo();
796  if (!L.getLoopID())
797    return;
798
799  if (I->isTerminator()) {
800    for (BasicBlock *Succ : successors(I))
801      if (Succ == L.getHeader()) {
802        I->setMetadata(llvm::LLVMContext::MD_loop, L.getLoopID());
803        break;
804      }
805    return;
806  }
807}
808