1//===---- CGLoopInfo.cpp - LLVM CodeGen for loop metadata -*- C++ -*-------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "CGLoopInfo.h"
10#include "clang/AST/ASTContext.h"
11#include "clang/AST/Attr.h"
12#include "clang/AST/Expr.h"
13#include "clang/Basic/CodeGenOptions.h"
14#include "llvm/IR/BasicBlock.h"
15#include "llvm/IR/CFG.h"
16#include "llvm/IR/Constants.h"
17#include "llvm/IR/InstrTypes.h"
18#include "llvm/IR/Instructions.h"
19#include "llvm/IR/Metadata.h"
20#include <optional>
21using namespace clang::CodeGen;
22using namespace llvm;
23
24MDNode *
25LoopInfo::createLoopPropertiesMetadata(ArrayRef<Metadata *> LoopProperties) {
26  LLVMContext &Ctx = Header->getContext();
27  SmallVector<Metadata *, 4> NewLoopProperties;
28  NewLoopProperties.push_back(nullptr);
29  NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
30
31  MDNode *LoopID = MDNode::getDistinct(Ctx, NewLoopProperties);
32  LoopID->replaceOperandWith(0, LoopID);
33  return LoopID;
34}
35
36MDNode *LoopInfo::createPipeliningMetadata(const LoopAttributes &Attrs,
37                                           ArrayRef<Metadata *> LoopProperties,
38                                           bool &HasUserTransforms) {
39  LLVMContext &Ctx = Header->getContext();
40
41  std::optional<bool> Enabled;
42  if (Attrs.PipelineDisabled)
43    Enabled = false;
44  else if (Attrs.PipelineInitiationInterval != 0)
45    Enabled = true;
46
47  if (Enabled != true) {
48    SmallVector<Metadata *, 4> NewLoopProperties;
49    if (Enabled == false) {
50      NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
51      NewLoopProperties.push_back(
52          MDNode::get(Ctx, {MDString::get(Ctx, "llvm.loop.pipeline.disable"),
53                            ConstantAsMetadata::get(ConstantInt::get(
54                                llvm::Type::getInt1Ty(Ctx), 1))}));
55      LoopProperties = NewLoopProperties;
56    }
57    return createLoopPropertiesMetadata(LoopProperties);
58  }
59
60  SmallVector<Metadata *, 4> Args;
61  Args.push_back(nullptr);
62  Args.append(LoopProperties.begin(), LoopProperties.end());
63
64  if (Attrs.PipelineInitiationInterval > 0) {
65    Metadata *Vals[] = {
66        MDString::get(Ctx, "llvm.loop.pipeline.initiationinterval"),
67        ConstantAsMetadata::get(ConstantInt::get(
68            llvm::Type::getInt32Ty(Ctx), Attrs.PipelineInitiationInterval))};
69    Args.push_back(MDNode::get(Ctx, Vals));
70  }
71
72  // No follow-up: This is the last transformation.
73
74  MDNode *LoopID = MDNode::getDistinct(Ctx, Args);
75  LoopID->replaceOperandWith(0, LoopID);
76  HasUserTransforms = true;
77  return LoopID;
78}
79
80MDNode *
81LoopInfo::createPartialUnrollMetadata(const LoopAttributes &Attrs,
82                                      ArrayRef<Metadata *> LoopProperties,
83                                      bool &HasUserTransforms) {
84  LLVMContext &Ctx = Header->getContext();
85
86  std::optional<bool> Enabled;
87  if (Attrs.UnrollEnable == LoopAttributes::Disable)
88    Enabled = false;
89  else if (Attrs.UnrollEnable == LoopAttributes::Full)
90    Enabled = std::nullopt;
91  else if (Attrs.UnrollEnable != LoopAttributes::Unspecified ||
92           Attrs.UnrollCount != 0)
93    Enabled = true;
94
95  if (Enabled != true) {
96    // createFullUnrollMetadata will already have added llvm.loop.unroll.disable
97    // if unrolling is disabled.
98    return createPipeliningMetadata(Attrs, LoopProperties, HasUserTransforms);
99  }
100
101  SmallVector<Metadata *, 4> FollowupLoopProperties;
102
103  // Apply all loop properties to the unrolled loop.
104  FollowupLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
105
106  // Don't unroll an already unrolled loop.
107  FollowupLoopProperties.push_back(
108      MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.unroll.disable")));
109
110  bool FollowupHasTransforms = false;
111  MDNode *Followup = createPipeliningMetadata(Attrs, FollowupLoopProperties,
112                                              FollowupHasTransforms);
113
114  SmallVector<Metadata *, 4> Args;
115  Args.push_back(nullptr);
116  Args.append(LoopProperties.begin(), LoopProperties.end());
117
118  // Setting unroll.count
119  if (Attrs.UnrollCount > 0) {
120    Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.unroll.count"),
121                        ConstantAsMetadata::get(ConstantInt::get(
122                            llvm::Type::getInt32Ty(Ctx), Attrs.UnrollCount))};
123    Args.push_back(MDNode::get(Ctx, Vals));
124  }
125
126  // Setting unroll.full or unroll.disable
127  if (Attrs.UnrollEnable == LoopAttributes::Enable) {
128    Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.unroll.enable")};
129    Args.push_back(MDNode::get(Ctx, Vals));
130  }
131
132  if (FollowupHasTransforms)
133    Args.push_back(MDNode::get(
134        Ctx, {MDString::get(Ctx, "llvm.loop.unroll.followup_all"), Followup}));
135
136  MDNode *LoopID = MDNode::getDistinct(Ctx, Args);
137  LoopID->replaceOperandWith(0, LoopID);
138  HasUserTransforms = true;
139  return LoopID;
140}
141
142MDNode *
143LoopInfo::createUnrollAndJamMetadata(const LoopAttributes &Attrs,
144                                     ArrayRef<Metadata *> LoopProperties,
145                                     bool &HasUserTransforms) {
146  LLVMContext &Ctx = Header->getContext();
147
148  std::optional<bool> Enabled;
149  if (Attrs.UnrollAndJamEnable == LoopAttributes::Disable)
150    Enabled = false;
151  else if (Attrs.UnrollAndJamEnable == LoopAttributes::Enable ||
152           Attrs.UnrollAndJamCount != 0)
153    Enabled = true;
154
155  if (Enabled != true) {
156    SmallVector<Metadata *, 4> NewLoopProperties;
157    if (Enabled == false) {
158      NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
159      NewLoopProperties.push_back(MDNode::get(
160          Ctx, MDString::get(Ctx, "llvm.loop.unroll_and_jam.disable")));
161      LoopProperties = NewLoopProperties;
162    }
163    return createPartialUnrollMetadata(Attrs, LoopProperties,
164                                       HasUserTransforms);
165  }
166
167  SmallVector<Metadata *, 4> FollowupLoopProperties;
168  FollowupLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
169  FollowupLoopProperties.push_back(
170      MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.unroll_and_jam.disable")));
171
172  bool FollowupHasTransforms = false;
173  MDNode *Followup = createPartialUnrollMetadata(Attrs, FollowupLoopProperties,
174                                                 FollowupHasTransforms);
175
176  SmallVector<Metadata *, 4> Args;
177  Args.push_back(nullptr);
178  Args.append(LoopProperties.begin(), LoopProperties.end());
179
180  // Setting unroll_and_jam.count
181  if (Attrs.UnrollAndJamCount > 0) {
182    Metadata *Vals[] = {
183        MDString::get(Ctx, "llvm.loop.unroll_and_jam.count"),
184        ConstantAsMetadata::get(ConstantInt::get(llvm::Type::getInt32Ty(Ctx),
185                                                 Attrs.UnrollAndJamCount))};
186    Args.push_back(MDNode::get(Ctx, Vals));
187  }
188
189  if (Attrs.UnrollAndJamEnable == LoopAttributes::Enable) {
190    Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.unroll_and_jam.enable")};
191    Args.push_back(MDNode::get(Ctx, Vals));
192  }
193
194  if (FollowupHasTransforms)
195    Args.push_back(MDNode::get(
196        Ctx, {MDString::get(Ctx, "llvm.loop.unroll_and_jam.followup_outer"),
197              Followup}));
198
199  if (UnrollAndJamInnerFollowup)
200    Args.push_back(MDNode::get(
201        Ctx, {MDString::get(Ctx, "llvm.loop.unroll_and_jam.followup_inner"),
202              UnrollAndJamInnerFollowup}));
203
204  MDNode *LoopID = MDNode::getDistinct(Ctx, Args);
205  LoopID->replaceOperandWith(0, LoopID);
206  HasUserTransforms = true;
207  return LoopID;
208}
209
210MDNode *
211LoopInfo::createLoopVectorizeMetadata(const LoopAttributes &Attrs,
212                                      ArrayRef<Metadata *> LoopProperties,
213                                      bool &HasUserTransforms) {
214  LLVMContext &Ctx = Header->getContext();
215
216  std::optional<bool> Enabled;
217  if (Attrs.VectorizeEnable == LoopAttributes::Disable)
218    Enabled = false;
219  else if (Attrs.VectorizeEnable != LoopAttributes::Unspecified ||
220           Attrs.VectorizePredicateEnable != LoopAttributes::Unspecified ||
221           Attrs.InterleaveCount != 0 || Attrs.VectorizeWidth != 0 ||
222           Attrs.VectorizeScalable != LoopAttributes::Unspecified)
223    Enabled = true;
224
225  if (Enabled != true) {
226    SmallVector<Metadata *, 4> NewLoopProperties;
227    if (Enabled == false) {
228      NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
229      NewLoopProperties.push_back(
230          MDNode::get(Ctx, {MDString::get(Ctx, "llvm.loop.vectorize.enable"),
231                            ConstantAsMetadata::get(ConstantInt::get(
232                                llvm::Type::getInt1Ty(Ctx), 0))}));
233      LoopProperties = NewLoopProperties;
234    }
235    return createUnrollAndJamMetadata(Attrs, LoopProperties, HasUserTransforms);
236  }
237
238  // Apply all loop properties to the vectorized loop.
239  SmallVector<Metadata *, 4> FollowupLoopProperties;
240  FollowupLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
241
242  // Don't vectorize an already vectorized loop.
243  FollowupLoopProperties.push_back(
244      MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.isvectorized")));
245
246  bool FollowupHasTransforms = false;
247  MDNode *Followup = createUnrollAndJamMetadata(Attrs, FollowupLoopProperties,
248                                                FollowupHasTransforms);
249
250  SmallVector<Metadata *, 4> Args;
251  Args.push_back(nullptr);
252  Args.append(LoopProperties.begin(), LoopProperties.end());
253
254  // Setting vectorize.predicate when it has been specified and vectorization
255  // has not been disabled.
256  bool IsVectorPredicateEnabled = false;
257  if (Attrs.VectorizePredicateEnable != LoopAttributes::Unspecified) {
258    IsVectorPredicateEnabled =
259        (Attrs.VectorizePredicateEnable == LoopAttributes::Enable);
260
261    Metadata *Vals[] = {
262        MDString::get(Ctx, "llvm.loop.vectorize.predicate.enable"),
263        ConstantAsMetadata::get(ConstantInt::get(llvm::Type::getInt1Ty(Ctx),
264                                                 IsVectorPredicateEnabled))};
265    Args.push_back(MDNode::get(Ctx, Vals));
266  }
267
268  // Setting vectorize.width
269  if (Attrs.VectorizeWidth > 0) {
270    Metadata *Vals[] = {
271        MDString::get(Ctx, "llvm.loop.vectorize.width"),
272        ConstantAsMetadata::get(ConstantInt::get(llvm::Type::getInt32Ty(Ctx),
273                                                 Attrs.VectorizeWidth))};
274
275    Args.push_back(MDNode::get(Ctx, Vals));
276  }
277
278  if (Attrs.VectorizeScalable != LoopAttributes::Unspecified) {
279    bool IsScalable = Attrs.VectorizeScalable == LoopAttributes::Enable;
280    Metadata *Vals[] = {
281        MDString::get(Ctx, "llvm.loop.vectorize.scalable.enable"),
282        ConstantAsMetadata::get(
283            ConstantInt::get(llvm::Type::getInt1Ty(Ctx), IsScalable))};
284    Args.push_back(MDNode::get(Ctx, Vals));
285  }
286
287  // Setting interleave.count
288  if (Attrs.InterleaveCount > 0) {
289    Metadata *Vals[] = {
290        MDString::get(Ctx, "llvm.loop.interleave.count"),
291        ConstantAsMetadata::get(ConstantInt::get(llvm::Type::getInt32Ty(Ctx),
292                                                 Attrs.InterleaveCount))};
293    Args.push_back(MDNode::get(Ctx, Vals));
294  }
295
296  // vectorize.enable is set if:
297  // 1) loop hint vectorize.enable is set, or
298  // 2) it is implied when vectorize.predicate is set, or
299  // 3) it is implied when vectorize.width is set to a value > 1
300  // 4) it is implied when vectorize.scalable.enable is true
301  // 5) it is implied when vectorize.width is unset (0) and the user
302  //    explicitly requested fixed-width vectorization, i.e.
303  //    vectorize.scalable.enable is false.
304  if (Attrs.VectorizeEnable != LoopAttributes::Unspecified ||
305      (IsVectorPredicateEnabled && Attrs.VectorizeWidth != 1) ||
306      Attrs.VectorizeWidth > 1 ||
307      Attrs.VectorizeScalable == LoopAttributes::Enable ||
308      (Attrs.VectorizeScalable == LoopAttributes::Disable &&
309       Attrs.VectorizeWidth != 1)) {
310    bool AttrVal = Attrs.VectorizeEnable != LoopAttributes::Disable;
311    Args.push_back(
312        MDNode::get(Ctx, {MDString::get(Ctx, "llvm.loop.vectorize.enable"),
313                          ConstantAsMetadata::get(ConstantInt::get(
314                              llvm::Type::getInt1Ty(Ctx), AttrVal))}));
315  }
316
317  if (FollowupHasTransforms)
318    Args.push_back(MDNode::get(
319        Ctx,
320        {MDString::get(Ctx, "llvm.loop.vectorize.followup_all"), Followup}));
321
322  MDNode *LoopID = MDNode::getDistinct(Ctx, Args);
323  LoopID->replaceOperandWith(0, LoopID);
324  HasUserTransforms = true;
325  return LoopID;
326}
327
328MDNode *
329LoopInfo::createLoopDistributeMetadata(const LoopAttributes &Attrs,
330                                       ArrayRef<Metadata *> LoopProperties,
331                                       bool &HasUserTransforms) {
332  LLVMContext &Ctx = Header->getContext();
333
334  std::optional<bool> Enabled;
335  if (Attrs.DistributeEnable == LoopAttributes::Disable)
336    Enabled = false;
337  if (Attrs.DistributeEnable == LoopAttributes::Enable)
338    Enabled = true;
339
340  if (Enabled != true) {
341    SmallVector<Metadata *, 4> NewLoopProperties;
342    if (Enabled == false) {
343      NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
344      NewLoopProperties.push_back(
345          MDNode::get(Ctx, {MDString::get(Ctx, "llvm.loop.distribute.enable"),
346                            ConstantAsMetadata::get(ConstantInt::get(
347                                llvm::Type::getInt1Ty(Ctx), 0))}));
348      LoopProperties = NewLoopProperties;
349    }
350    return createLoopVectorizeMetadata(Attrs, LoopProperties,
351                                       HasUserTransforms);
352  }
353
354  bool FollowupHasTransforms = false;
355  MDNode *Followup =
356      createLoopVectorizeMetadata(Attrs, LoopProperties, FollowupHasTransforms);
357
358  SmallVector<Metadata *, 4> Args;
359  Args.push_back(nullptr);
360  Args.append(LoopProperties.begin(), LoopProperties.end());
361
362  Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.distribute.enable"),
363                      ConstantAsMetadata::get(ConstantInt::get(
364                          llvm::Type::getInt1Ty(Ctx),
365                          (Attrs.DistributeEnable == LoopAttributes::Enable)))};
366  Args.push_back(MDNode::get(Ctx, Vals));
367
368  if (FollowupHasTransforms)
369    Args.push_back(MDNode::get(
370        Ctx,
371        {MDString::get(Ctx, "llvm.loop.distribute.followup_all"), Followup}));
372
373  MDNode *LoopID = MDNode::getDistinct(Ctx, Args);
374  LoopID->replaceOperandWith(0, LoopID);
375  HasUserTransforms = true;
376  return LoopID;
377}
378
379MDNode *LoopInfo::createFullUnrollMetadata(const LoopAttributes &Attrs,
380                                           ArrayRef<Metadata *> LoopProperties,
381                                           bool &HasUserTransforms) {
382  LLVMContext &Ctx = Header->getContext();
383
384  std::optional<bool> Enabled;
385  if (Attrs.UnrollEnable == LoopAttributes::Disable)
386    Enabled = false;
387  else if (Attrs.UnrollEnable == LoopAttributes::Full)
388    Enabled = true;
389
390  if (Enabled != true) {
391    SmallVector<Metadata *, 4> NewLoopProperties;
392    if (Enabled == false) {
393      NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
394      NewLoopProperties.push_back(
395          MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.unroll.disable")));
396      LoopProperties = NewLoopProperties;
397    }
398    return createLoopDistributeMetadata(Attrs, LoopProperties,
399                                        HasUserTransforms);
400  }
401
402  SmallVector<Metadata *, 4> Args;
403  Args.push_back(nullptr);
404  Args.append(LoopProperties.begin(), LoopProperties.end());
405  Args.push_back(MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.unroll.full")));
406
407  // No follow-up: there is no loop after full unrolling.
408  // TODO: Warn if there are transformations after full unrolling.
409
410  MDNode *LoopID = MDNode::getDistinct(Ctx, Args);
411  LoopID->replaceOperandWith(0, LoopID);
412  HasUserTransforms = true;
413  return LoopID;
414}
415
416MDNode *LoopInfo::createMetadata(
417    const LoopAttributes &Attrs,
418    llvm::ArrayRef<llvm::Metadata *> AdditionalLoopProperties,
419    bool &HasUserTransforms) {
420  SmallVector<Metadata *, 3> LoopProperties;
421
422  // If we have a valid start debug location for the loop, add it.
423  if (StartLoc) {
424    LoopProperties.push_back(StartLoc.getAsMDNode());
425
426    // If we also have a valid end debug location for the loop, add it.
427    if (EndLoc)
428      LoopProperties.push_back(EndLoc.getAsMDNode());
429  }
430
431  LLVMContext &Ctx = Header->getContext();
432  if (Attrs.MustProgress)
433    LoopProperties.push_back(
434        MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.mustprogress")));
435
436  assert(!!AccGroup == Attrs.IsParallel &&
437         "There must be an access group iff the loop is parallel");
438  if (Attrs.IsParallel) {
439    LoopProperties.push_back(MDNode::get(
440        Ctx, {MDString::get(Ctx, "llvm.loop.parallel_accesses"), AccGroup}));
441  }
442
443  LoopProperties.insert(LoopProperties.end(), AdditionalLoopProperties.begin(),
444                        AdditionalLoopProperties.end());
445  return createFullUnrollMetadata(Attrs, LoopProperties, HasUserTransforms);
446}
447
448LoopAttributes::LoopAttributes(bool IsParallel)
449    : IsParallel(IsParallel), VectorizeEnable(LoopAttributes::Unspecified),
450      UnrollEnable(LoopAttributes::Unspecified),
451      UnrollAndJamEnable(LoopAttributes::Unspecified),
452      VectorizePredicateEnable(LoopAttributes::Unspecified), VectorizeWidth(0),
453      VectorizeScalable(LoopAttributes::Unspecified), InterleaveCount(0),
454      UnrollCount(0), UnrollAndJamCount(0),
455      DistributeEnable(LoopAttributes::Unspecified), PipelineDisabled(false),
456      PipelineInitiationInterval(0), MustProgress(false) {}
457
458void LoopAttributes::clear() {
459  IsParallel = false;
460  VectorizeWidth = 0;
461  VectorizeScalable = LoopAttributes::Unspecified;
462  InterleaveCount = 0;
463  UnrollCount = 0;
464  UnrollAndJamCount = 0;
465  VectorizeEnable = LoopAttributes::Unspecified;
466  UnrollEnable = LoopAttributes::Unspecified;
467  UnrollAndJamEnable = LoopAttributes::Unspecified;
468  VectorizePredicateEnable = LoopAttributes::Unspecified;
469  DistributeEnable = LoopAttributes::Unspecified;
470  PipelineDisabled = false;
471  PipelineInitiationInterval = 0;
472  MustProgress = false;
473}
474
475LoopInfo::LoopInfo(BasicBlock *Header, const LoopAttributes &Attrs,
476                   const llvm::DebugLoc &StartLoc, const llvm::DebugLoc &EndLoc,
477                   LoopInfo *Parent)
478    : Header(Header), Attrs(Attrs), StartLoc(StartLoc), EndLoc(EndLoc),
479      Parent(Parent) {
480
481  if (Attrs.IsParallel) {
482    // Create an access group for this loop.
483    LLVMContext &Ctx = Header->getContext();
484    AccGroup = MDNode::getDistinct(Ctx, {});
485  }
486
487  if (!Attrs.IsParallel && Attrs.VectorizeWidth == 0 &&
488      Attrs.VectorizeScalable == LoopAttributes::Unspecified &&
489      Attrs.InterleaveCount == 0 && Attrs.UnrollCount == 0 &&
490      Attrs.UnrollAndJamCount == 0 && !Attrs.PipelineDisabled &&
491      Attrs.PipelineInitiationInterval == 0 &&
492      Attrs.VectorizePredicateEnable == LoopAttributes::Unspecified &&
493      Attrs.VectorizeEnable == LoopAttributes::Unspecified &&
494      Attrs.UnrollEnable == LoopAttributes::Unspecified &&
495      Attrs.UnrollAndJamEnable == LoopAttributes::Unspecified &&
496      Attrs.DistributeEnable == LoopAttributes::Unspecified && !StartLoc &&
497      !EndLoc && !Attrs.MustProgress)
498    return;
499
500  TempLoopID = MDNode::getTemporary(Header->getContext(), std::nullopt);
501}
502
503void LoopInfo::finish() {
504  // We did not annotate the loop body instructions because there are no
505  // attributes for this loop.
506  if (!TempLoopID)
507    return;
508
509  MDNode *LoopID;
510  LoopAttributes CurLoopAttr = Attrs;
511  LLVMContext &Ctx = Header->getContext();
512
513  if (Parent && (Parent->Attrs.UnrollAndJamEnable ||
514                 Parent->Attrs.UnrollAndJamCount != 0)) {
515    // Parent unroll-and-jams this loop.
516    // Split the transformations in those that happens before the unroll-and-jam
517    // and those after.
518
519    LoopAttributes BeforeJam, AfterJam;
520
521    BeforeJam.IsParallel = AfterJam.IsParallel = Attrs.IsParallel;
522
523    BeforeJam.VectorizeWidth = Attrs.VectorizeWidth;
524    BeforeJam.VectorizeScalable = Attrs.VectorizeScalable;
525    BeforeJam.InterleaveCount = Attrs.InterleaveCount;
526    BeforeJam.VectorizeEnable = Attrs.VectorizeEnable;
527    BeforeJam.DistributeEnable = Attrs.DistributeEnable;
528    BeforeJam.VectorizePredicateEnable = Attrs.VectorizePredicateEnable;
529
530    switch (Attrs.UnrollEnable) {
531    case LoopAttributes::Unspecified:
532    case LoopAttributes::Disable:
533      BeforeJam.UnrollEnable = Attrs.UnrollEnable;
534      AfterJam.UnrollEnable = Attrs.UnrollEnable;
535      break;
536    case LoopAttributes::Full:
537      BeforeJam.UnrollEnable = LoopAttributes::Full;
538      break;
539    case LoopAttributes::Enable:
540      AfterJam.UnrollEnable = LoopAttributes::Enable;
541      break;
542    }
543
544    AfterJam.VectorizePredicateEnable = Attrs.VectorizePredicateEnable;
545    AfterJam.UnrollCount = Attrs.UnrollCount;
546    AfterJam.PipelineDisabled = Attrs.PipelineDisabled;
547    AfterJam.PipelineInitiationInterval = Attrs.PipelineInitiationInterval;
548
549    // If this loop is subject of an unroll-and-jam by the parent loop, and has
550    // an unroll-and-jam annotation itself, we have to decide whether to first
551    // apply the parent's unroll-and-jam or this loop's unroll-and-jam. The
552    // UnrollAndJam pass processes loops from inner to outer, so we apply the
553    // inner first.
554    BeforeJam.UnrollAndJamCount = Attrs.UnrollAndJamCount;
555    BeforeJam.UnrollAndJamEnable = Attrs.UnrollAndJamEnable;
556
557    // Set the inner followup metadata to process by the outer loop. Only
558    // consider the first inner loop.
559    if (!Parent->UnrollAndJamInnerFollowup) {
560      // Splitting the attributes into a BeforeJam and an AfterJam part will
561      // stop 'llvm.loop.isvectorized' (generated by vectorization in BeforeJam)
562      // to be forwarded to the AfterJam part. We detect the situation here and
563      // add it manually.
564      SmallVector<Metadata *, 1> BeforeLoopProperties;
565      if (BeforeJam.VectorizeEnable != LoopAttributes::Unspecified ||
566          BeforeJam.VectorizePredicateEnable != LoopAttributes::Unspecified ||
567          BeforeJam.InterleaveCount != 0 || BeforeJam.VectorizeWidth != 0 ||
568          BeforeJam.VectorizeScalable == LoopAttributes::Enable)
569        BeforeLoopProperties.push_back(
570            MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.isvectorized")));
571
572      bool InnerFollowupHasTransform = false;
573      MDNode *InnerFollowup = createMetadata(AfterJam, BeforeLoopProperties,
574                                             InnerFollowupHasTransform);
575      if (InnerFollowupHasTransform)
576        Parent->UnrollAndJamInnerFollowup = InnerFollowup;
577    }
578
579    CurLoopAttr = BeforeJam;
580  }
581
582  bool HasUserTransforms = false;
583  LoopID = createMetadata(CurLoopAttr, {}, HasUserTransforms);
584  TempLoopID->replaceAllUsesWith(LoopID);
585}
586
587void LoopInfoStack::push(BasicBlock *Header, const llvm::DebugLoc &StartLoc,
588                         const llvm::DebugLoc &EndLoc) {
589  Active.emplace_back(
590      new LoopInfo(Header, StagedAttrs, StartLoc, EndLoc,
591                   Active.empty() ? nullptr : Active.back().get()));
592  // Clear the attributes so nested loops do not inherit them.
593  StagedAttrs.clear();
594}
595
596void LoopInfoStack::push(BasicBlock *Header, clang::ASTContext &Ctx,
597                         const clang::CodeGenOptions &CGOpts,
598                         ArrayRef<const clang::Attr *> Attrs,
599                         const llvm::DebugLoc &StartLoc,
600                         const llvm::DebugLoc &EndLoc, bool MustProgress) {
601  // Identify loop hint attributes from Attrs.
602  for (const auto *Attr : Attrs) {
603    const LoopHintAttr *LH = dyn_cast<LoopHintAttr>(Attr);
604    const OpenCLUnrollHintAttr *OpenCLHint =
605        dyn_cast<OpenCLUnrollHintAttr>(Attr);
606
607    // Skip non loop hint attributes
608    if (!LH && !OpenCLHint) {
609      continue;
610    }
611
612    LoopHintAttr::OptionType Option = LoopHintAttr::Unroll;
613    LoopHintAttr::LoopHintState State = LoopHintAttr::Disable;
614    unsigned ValueInt = 1;
615    // Translate opencl_unroll_hint attribute argument to
616    // equivalent LoopHintAttr enums.
617    // OpenCL v2.0 s6.11.5:
618    // 0 - enable unroll (no argument).
619    // 1 - disable unroll.
620    // other positive integer n - unroll by n.
621    if (OpenCLHint) {
622      ValueInt = OpenCLHint->getUnrollHint();
623      if (ValueInt == 0) {
624        State = LoopHintAttr::Enable;
625      } else if (ValueInt != 1) {
626        Option = LoopHintAttr::UnrollCount;
627        State = LoopHintAttr::Numeric;
628      }
629    } else if (LH) {
630      auto *ValueExpr = LH->getValue();
631      if (ValueExpr) {
632        llvm::APSInt ValueAPS = ValueExpr->EvaluateKnownConstInt(Ctx);
633        ValueInt = ValueAPS.getSExtValue();
634      }
635
636      Option = LH->getOption();
637      State = LH->getState();
638    }
639    switch (State) {
640    case LoopHintAttr::Disable:
641      switch (Option) {
642      case LoopHintAttr::Vectorize:
643        // Disable vectorization by specifying a width of 1.
644        setVectorizeWidth(1);
645        setVectorizeScalable(LoopAttributes::Unspecified);
646        break;
647      case LoopHintAttr::Interleave:
648        // Disable interleaving by speciyfing a count of 1.
649        setInterleaveCount(1);
650        break;
651      case LoopHintAttr::Unroll:
652        setUnrollState(LoopAttributes::Disable);
653        break;
654      case LoopHintAttr::UnrollAndJam:
655        setUnrollAndJamState(LoopAttributes::Disable);
656        break;
657      case LoopHintAttr::VectorizePredicate:
658        setVectorizePredicateState(LoopAttributes::Disable);
659        break;
660      case LoopHintAttr::Distribute:
661        setDistributeState(false);
662        break;
663      case LoopHintAttr::PipelineDisabled:
664        setPipelineDisabled(true);
665        break;
666      case LoopHintAttr::UnrollCount:
667      case LoopHintAttr::UnrollAndJamCount:
668      case LoopHintAttr::VectorizeWidth:
669      case LoopHintAttr::InterleaveCount:
670      case LoopHintAttr::PipelineInitiationInterval:
671        llvm_unreachable("Options cannot be disabled.");
672        break;
673      }
674      break;
675    case LoopHintAttr::Enable:
676      switch (Option) {
677      case LoopHintAttr::Vectorize:
678      case LoopHintAttr::Interleave:
679        setVectorizeEnable(true);
680        break;
681      case LoopHintAttr::Unroll:
682        setUnrollState(LoopAttributes::Enable);
683        break;
684      case LoopHintAttr::UnrollAndJam:
685        setUnrollAndJamState(LoopAttributes::Enable);
686        break;
687      case LoopHintAttr::VectorizePredicate:
688        setVectorizePredicateState(LoopAttributes::Enable);
689        break;
690      case LoopHintAttr::Distribute:
691        setDistributeState(true);
692        break;
693      case LoopHintAttr::UnrollCount:
694      case LoopHintAttr::UnrollAndJamCount:
695      case LoopHintAttr::VectorizeWidth:
696      case LoopHintAttr::InterleaveCount:
697      case LoopHintAttr::PipelineDisabled:
698      case LoopHintAttr::PipelineInitiationInterval:
699        llvm_unreachable("Options cannot enabled.");
700        break;
701      }
702      break;
703    case LoopHintAttr::AssumeSafety:
704      switch (Option) {
705      case LoopHintAttr::Vectorize:
706      case LoopHintAttr::Interleave:
707        // Apply "llvm.mem.parallel_loop_access" metadata to load/stores.
708        setParallel(true);
709        setVectorizeEnable(true);
710        break;
711      case LoopHintAttr::Unroll:
712      case LoopHintAttr::UnrollAndJam:
713      case LoopHintAttr::VectorizePredicate:
714      case LoopHintAttr::UnrollCount:
715      case LoopHintAttr::UnrollAndJamCount:
716      case LoopHintAttr::VectorizeWidth:
717      case LoopHintAttr::InterleaveCount:
718      case LoopHintAttr::Distribute:
719      case LoopHintAttr::PipelineDisabled:
720      case LoopHintAttr::PipelineInitiationInterval:
721        llvm_unreachable("Options cannot be used to assume mem safety.");
722        break;
723      }
724      break;
725    case LoopHintAttr::Full:
726      switch (Option) {
727      case LoopHintAttr::Unroll:
728        setUnrollState(LoopAttributes::Full);
729        break;
730      case LoopHintAttr::UnrollAndJam:
731        setUnrollAndJamState(LoopAttributes::Full);
732        break;
733      case LoopHintAttr::Vectorize:
734      case LoopHintAttr::Interleave:
735      case LoopHintAttr::UnrollCount:
736      case LoopHintAttr::UnrollAndJamCount:
737      case LoopHintAttr::VectorizeWidth:
738      case LoopHintAttr::InterleaveCount:
739      case LoopHintAttr::Distribute:
740      case LoopHintAttr::PipelineDisabled:
741      case LoopHintAttr::PipelineInitiationInterval:
742      case LoopHintAttr::VectorizePredicate:
743        llvm_unreachable("Options cannot be used with 'full' hint.");
744        break;
745      }
746      break;
747    case LoopHintAttr::FixedWidth:
748    case LoopHintAttr::ScalableWidth:
749      switch (Option) {
750      case LoopHintAttr::VectorizeWidth:
751        setVectorizeScalable(State == LoopHintAttr::ScalableWidth
752                                 ? LoopAttributes::Enable
753                                 : LoopAttributes::Disable);
754        if (LH->getValue())
755          setVectorizeWidth(ValueInt);
756        break;
757      default:
758        llvm_unreachable("Options cannot be used with 'scalable' hint.");
759        break;
760      }
761      break;
762    case LoopHintAttr::Numeric:
763      switch (Option) {
764      case LoopHintAttr::InterleaveCount:
765        setInterleaveCount(ValueInt);
766        break;
767      case LoopHintAttr::UnrollCount:
768        setUnrollCount(ValueInt);
769        break;
770      case LoopHintAttr::UnrollAndJamCount:
771        setUnrollAndJamCount(ValueInt);
772        break;
773      case LoopHintAttr::PipelineInitiationInterval:
774        setPipelineInitiationInterval(ValueInt);
775        break;
776      case LoopHintAttr::Unroll:
777      case LoopHintAttr::UnrollAndJam:
778      case LoopHintAttr::VectorizePredicate:
779      case LoopHintAttr::Vectorize:
780      case LoopHintAttr::VectorizeWidth:
781      case LoopHintAttr::Interleave:
782      case LoopHintAttr::Distribute:
783      case LoopHintAttr::PipelineDisabled:
784        llvm_unreachable("Options cannot be assigned a value.");
785        break;
786      }
787      break;
788    }
789  }
790
791  setMustProgress(MustProgress);
792
793  if (CGOpts.OptimizationLevel > 0)
794    // Disable unrolling for the loop, if unrolling is disabled (via
795    // -fno-unroll-loops) and no pragmas override the decision.
796    if (!CGOpts.UnrollLoops &&
797        (StagedAttrs.UnrollEnable == LoopAttributes::Unspecified &&
798         StagedAttrs.UnrollCount == 0))
799      setUnrollState(LoopAttributes::Disable);
800
801  /// Stage the attributes.
802  push(Header, StartLoc, EndLoc);
803}
804
805void LoopInfoStack::pop() {
806  assert(!Active.empty() && "No active loops to pop");
807  Active.back()->finish();
808  Active.pop_back();
809}
810
811void LoopInfoStack::InsertHelper(Instruction *I) const {
812  if (I->mayReadOrWriteMemory()) {
813    SmallVector<Metadata *, 4> AccessGroups;
814    for (const auto &AL : Active) {
815      // Here we assume that every loop that has an access group is parallel.
816      if (MDNode *Group = AL->getAccessGroup())
817        AccessGroups.push_back(Group);
818    }
819    MDNode *UnionMD = nullptr;
820    if (AccessGroups.size() == 1)
821      UnionMD = cast<MDNode>(AccessGroups[0]);
822    else if (AccessGroups.size() >= 2)
823      UnionMD = MDNode::get(I->getContext(), AccessGroups);
824    I->setMetadata("llvm.access.group", UnionMD);
825  }
826
827  if (!hasInfo())
828    return;
829
830  const LoopInfo &L = getInfo();
831  if (!L.getLoopID())
832    return;
833
834  if (I->isTerminator()) {
835    for (BasicBlock *Succ : successors(I))
836      if (Succ == L.getHeader()) {
837        I->setMetadata(llvm::LLVMContext::MD_loop, L.getLoopID());
838        break;
839      }
840    return;
841  }
842}
843