1//===--- CGStmt.cpp - Emit LLVM Code from Statements ----------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This contains code to emit Stmt nodes as LLVM code.
10//
11//===----------------------------------------------------------------------===//
12
13#include "CGDebugInfo.h"
14#include "CGOpenMPRuntime.h"
15#include "CodeGenFunction.h"
16#include "CodeGenModule.h"
17#include "TargetInfo.h"
18#include "clang/AST/Attr.h"
19#include "clang/AST/Expr.h"
20#include "clang/AST/Stmt.h"
21#include "clang/AST/StmtVisitor.h"
22#include "clang/Basic/Builtins.h"
23#include "clang/Basic/DiagnosticSema.h"
24#include "clang/Basic/PrettyStackTrace.h"
25#include "clang/Basic/SourceManager.h"
26#include "clang/Basic/TargetInfo.h"
27#include "llvm/ADT/ArrayRef.h"
28#include "llvm/ADT/DenseMap.h"
29#include "llvm/ADT/SmallSet.h"
30#include "llvm/ADT/StringExtras.h"
31#include "llvm/IR/Assumptions.h"
32#include "llvm/IR/DataLayout.h"
33#include "llvm/IR/InlineAsm.h"
34#include "llvm/IR/Intrinsics.h"
35#include "llvm/IR/MDBuilder.h"
36#include "llvm/Support/SaveAndRestore.h"
37#include <optional>
38
39using namespace clang;
40using namespace CodeGen;
41
42//===----------------------------------------------------------------------===//
43//                              Statement Emission
44//===----------------------------------------------------------------------===//
45
46void CodeGenFunction::EmitStopPoint(const Stmt *S) {
47  if (CGDebugInfo *DI = getDebugInfo()) {
48    SourceLocation Loc;
49    Loc = S->getBeginLoc();
50    DI->EmitLocation(Builder, Loc);
51
52    LastStopPoint = Loc;
53  }
54}
55
56void CodeGenFunction::EmitStmt(const Stmt *S, ArrayRef<const Attr *> Attrs) {
57  assert(S && "Null statement?");
58  PGO.setCurrentStmt(S);
59
60  // These statements have their own debug info handling.
61  if (EmitSimpleStmt(S, Attrs))
62    return;
63
64  // Check if we are generating unreachable code.
65  if (!HaveInsertPoint()) {
66    // If so, and the statement doesn't contain a label, then we do not need to
67    // generate actual code. This is safe because (1) the current point is
68    // unreachable, so we don't need to execute the code, and (2) we've already
69    // handled the statements which update internal data structures (like the
70    // local variable map) which could be used by subsequent statements.
71    if (!ContainsLabel(S)) {
72      // Verify that any decl statements were handled as simple, they may be in
73      // scope of subsequent reachable statements.
74      assert(!isa<DeclStmt>(*S) && "Unexpected DeclStmt!");
75      return;
76    }
77
78    // Otherwise, make a new block to hold the code.
79    EnsureInsertPoint();
80  }
81
82  // Generate a stoppoint if we are emitting debug info.
83  EmitStopPoint(S);
84
85  // Ignore all OpenMP directives except for simd if OpenMP with Simd is
86  // enabled.
87  if (getLangOpts().OpenMP && getLangOpts().OpenMPSimd) {
88    if (const auto *D = dyn_cast<OMPExecutableDirective>(S)) {
89      EmitSimpleOMPExecutableDirective(*D);
90      return;
91    }
92  }
93
94  switch (S->getStmtClass()) {
95  case Stmt::NoStmtClass:
96  case Stmt::CXXCatchStmtClass:
97  case Stmt::SEHExceptStmtClass:
98  case Stmt::SEHFinallyStmtClass:
99  case Stmt::MSDependentExistsStmtClass:
100    llvm_unreachable("invalid statement class to emit generically");
101  case Stmt::NullStmtClass:
102  case Stmt::CompoundStmtClass:
103  case Stmt::DeclStmtClass:
104  case Stmt::LabelStmtClass:
105  case Stmt::AttributedStmtClass:
106  case Stmt::GotoStmtClass:
107  case Stmt::BreakStmtClass:
108  case Stmt::ContinueStmtClass:
109  case Stmt::DefaultStmtClass:
110  case Stmt::CaseStmtClass:
111  case Stmt::SEHLeaveStmtClass:
112    llvm_unreachable("should have emitted these statements as simple");
113
114#define STMT(Type, Base)
115#define ABSTRACT_STMT(Op)
116#define EXPR(Type, Base) \
117  case Stmt::Type##Class:
118#include "clang/AST/StmtNodes.inc"
119  {
120    // Remember the block we came in on.
121    llvm::BasicBlock *incoming = Builder.GetInsertBlock();
122    assert(incoming && "expression emission must have an insertion point");
123
124    EmitIgnoredExpr(cast<Expr>(S));
125
126    llvm::BasicBlock *outgoing = Builder.GetInsertBlock();
127    assert(outgoing && "expression emission cleared block!");
128
129    // The expression emitters assume (reasonably!) that the insertion
130    // point is always set.  To maintain that, the call-emission code
131    // for noreturn functions has to enter a new block with no
132    // predecessors.  We want to kill that block and mark the current
133    // insertion point unreachable in the common case of a call like
134    // "exit();".  Since expression emission doesn't otherwise create
135    // blocks with no predecessors, we can just test for that.
136    // However, we must be careful not to do this to our incoming
137    // block, because *statement* emission does sometimes create
138    // reachable blocks which will have no predecessors until later in
139    // the function.  This occurs with, e.g., labels that are not
140    // reachable by fallthrough.
141    if (incoming != outgoing && outgoing->use_empty()) {
142      outgoing->eraseFromParent();
143      Builder.ClearInsertionPoint();
144    }
145    break;
146  }
147
148  case Stmt::IndirectGotoStmtClass:
149    EmitIndirectGotoStmt(cast<IndirectGotoStmt>(*S)); break;
150
151  case Stmt::IfStmtClass:      EmitIfStmt(cast<IfStmt>(*S));              break;
152  case Stmt::WhileStmtClass:   EmitWhileStmt(cast<WhileStmt>(*S), Attrs); break;
153  case Stmt::DoStmtClass:      EmitDoStmt(cast<DoStmt>(*S), Attrs);       break;
154  case Stmt::ForStmtClass:     EmitForStmt(cast<ForStmt>(*S), Attrs);     break;
155
156  case Stmt::ReturnStmtClass:  EmitReturnStmt(cast<ReturnStmt>(*S));      break;
157
158  case Stmt::SwitchStmtClass:  EmitSwitchStmt(cast<SwitchStmt>(*S));      break;
159  case Stmt::GCCAsmStmtClass:  // Intentional fall-through.
160  case Stmt::MSAsmStmtClass:   EmitAsmStmt(cast<AsmStmt>(*S));            break;
161  case Stmt::CoroutineBodyStmtClass:
162    EmitCoroutineBody(cast<CoroutineBodyStmt>(*S));
163    break;
164  case Stmt::CoreturnStmtClass:
165    EmitCoreturnStmt(cast<CoreturnStmt>(*S));
166    break;
167  case Stmt::CapturedStmtClass: {
168    const CapturedStmt *CS = cast<CapturedStmt>(S);
169    EmitCapturedStmt(*CS, CS->getCapturedRegionKind());
170    }
171    break;
172  case Stmt::ObjCAtTryStmtClass:
173    EmitObjCAtTryStmt(cast<ObjCAtTryStmt>(*S));
174    break;
175  case Stmt::ObjCAtCatchStmtClass:
176    llvm_unreachable(
177                    "@catch statements should be handled by EmitObjCAtTryStmt");
178  case Stmt::ObjCAtFinallyStmtClass:
179    llvm_unreachable(
180                  "@finally statements should be handled by EmitObjCAtTryStmt");
181  case Stmt::ObjCAtThrowStmtClass:
182    EmitObjCAtThrowStmt(cast<ObjCAtThrowStmt>(*S));
183    break;
184  case Stmt::ObjCAtSynchronizedStmtClass:
185    EmitObjCAtSynchronizedStmt(cast<ObjCAtSynchronizedStmt>(*S));
186    break;
187  case Stmt::ObjCForCollectionStmtClass:
188    EmitObjCForCollectionStmt(cast<ObjCForCollectionStmt>(*S));
189    break;
190  case Stmt::ObjCAutoreleasePoolStmtClass:
191    EmitObjCAutoreleasePoolStmt(cast<ObjCAutoreleasePoolStmt>(*S));
192    break;
193
194  case Stmt::CXXTryStmtClass:
195    EmitCXXTryStmt(cast<CXXTryStmt>(*S));
196    break;
197  case Stmt::CXXForRangeStmtClass:
198    EmitCXXForRangeStmt(cast<CXXForRangeStmt>(*S), Attrs);
199    break;
200  case Stmt::SEHTryStmtClass:
201    EmitSEHTryStmt(cast<SEHTryStmt>(*S));
202    break;
203  case Stmt::OMPMetaDirectiveClass:
204    EmitOMPMetaDirective(cast<OMPMetaDirective>(*S));
205    break;
206  case Stmt::OMPCanonicalLoopClass:
207    EmitOMPCanonicalLoop(cast<OMPCanonicalLoop>(S));
208    break;
209  case Stmt::OMPParallelDirectiveClass:
210    EmitOMPParallelDirective(cast<OMPParallelDirective>(*S));
211    break;
212  case Stmt::OMPSimdDirectiveClass:
213    EmitOMPSimdDirective(cast<OMPSimdDirective>(*S));
214    break;
215  case Stmt::OMPTileDirectiveClass:
216    EmitOMPTileDirective(cast<OMPTileDirective>(*S));
217    break;
218  case Stmt::OMPUnrollDirectiveClass:
219    EmitOMPUnrollDirective(cast<OMPUnrollDirective>(*S));
220    break;
221  case Stmt::OMPForDirectiveClass:
222    EmitOMPForDirective(cast<OMPForDirective>(*S));
223    break;
224  case Stmt::OMPForSimdDirectiveClass:
225    EmitOMPForSimdDirective(cast<OMPForSimdDirective>(*S));
226    break;
227  case Stmt::OMPSectionsDirectiveClass:
228    EmitOMPSectionsDirective(cast<OMPSectionsDirective>(*S));
229    break;
230  case Stmt::OMPSectionDirectiveClass:
231    EmitOMPSectionDirective(cast<OMPSectionDirective>(*S));
232    break;
233  case Stmt::OMPSingleDirectiveClass:
234    EmitOMPSingleDirective(cast<OMPSingleDirective>(*S));
235    break;
236  case Stmt::OMPMasterDirectiveClass:
237    EmitOMPMasterDirective(cast<OMPMasterDirective>(*S));
238    break;
239  case Stmt::OMPCriticalDirectiveClass:
240    EmitOMPCriticalDirective(cast<OMPCriticalDirective>(*S));
241    break;
242  case Stmt::OMPParallelForDirectiveClass:
243    EmitOMPParallelForDirective(cast<OMPParallelForDirective>(*S));
244    break;
245  case Stmt::OMPParallelForSimdDirectiveClass:
246    EmitOMPParallelForSimdDirective(cast<OMPParallelForSimdDirective>(*S));
247    break;
248  case Stmt::OMPParallelMasterDirectiveClass:
249    EmitOMPParallelMasterDirective(cast<OMPParallelMasterDirective>(*S));
250    break;
251  case Stmt::OMPParallelSectionsDirectiveClass:
252    EmitOMPParallelSectionsDirective(cast<OMPParallelSectionsDirective>(*S));
253    break;
254  case Stmt::OMPTaskDirectiveClass:
255    EmitOMPTaskDirective(cast<OMPTaskDirective>(*S));
256    break;
257  case Stmt::OMPTaskyieldDirectiveClass:
258    EmitOMPTaskyieldDirective(cast<OMPTaskyieldDirective>(*S));
259    break;
260  case Stmt::OMPErrorDirectiveClass:
261    EmitOMPErrorDirective(cast<OMPErrorDirective>(*S));
262    break;
263  case Stmt::OMPBarrierDirectiveClass:
264    EmitOMPBarrierDirective(cast<OMPBarrierDirective>(*S));
265    break;
266  case Stmt::OMPTaskwaitDirectiveClass:
267    EmitOMPTaskwaitDirective(cast<OMPTaskwaitDirective>(*S));
268    break;
269  case Stmt::OMPTaskgroupDirectiveClass:
270    EmitOMPTaskgroupDirective(cast<OMPTaskgroupDirective>(*S));
271    break;
272  case Stmt::OMPFlushDirectiveClass:
273    EmitOMPFlushDirective(cast<OMPFlushDirective>(*S));
274    break;
275  case Stmt::OMPDepobjDirectiveClass:
276    EmitOMPDepobjDirective(cast<OMPDepobjDirective>(*S));
277    break;
278  case Stmt::OMPScanDirectiveClass:
279    EmitOMPScanDirective(cast<OMPScanDirective>(*S));
280    break;
281  case Stmt::OMPOrderedDirectiveClass:
282    EmitOMPOrderedDirective(cast<OMPOrderedDirective>(*S));
283    break;
284  case Stmt::OMPAtomicDirectiveClass:
285    EmitOMPAtomicDirective(cast<OMPAtomicDirective>(*S));
286    break;
287  case Stmt::OMPTargetDirectiveClass:
288    EmitOMPTargetDirective(cast<OMPTargetDirective>(*S));
289    break;
290  case Stmt::OMPTeamsDirectiveClass:
291    EmitOMPTeamsDirective(cast<OMPTeamsDirective>(*S));
292    break;
293  case Stmt::OMPCancellationPointDirectiveClass:
294    EmitOMPCancellationPointDirective(cast<OMPCancellationPointDirective>(*S));
295    break;
296  case Stmt::OMPCancelDirectiveClass:
297    EmitOMPCancelDirective(cast<OMPCancelDirective>(*S));
298    break;
299  case Stmt::OMPTargetDataDirectiveClass:
300    EmitOMPTargetDataDirective(cast<OMPTargetDataDirective>(*S));
301    break;
302  case Stmt::OMPTargetEnterDataDirectiveClass:
303    EmitOMPTargetEnterDataDirective(cast<OMPTargetEnterDataDirective>(*S));
304    break;
305  case Stmt::OMPTargetExitDataDirectiveClass:
306    EmitOMPTargetExitDataDirective(cast<OMPTargetExitDataDirective>(*S));
307    break;
308  case Stmt::OMPTargetParallelDirectiveClass:
309    EmitOMPTargetParallelDirective(cast<OMPTargetParallelDirective>(*S));
310    break;
311  case Stmt::OMPTargetParallelForDirectiveClass:
312    EmitOMPTargetParallelForDirective(cast<OMPTargetParallelForDirective>(*S));
313    break;
314  case Stmt::OMPTaskLoopDirectiveClass:
315    EmitOMPTaskLoopDirective(cast<OMPTaskLoopDirective>(*S));
316    break;
317  case Stmt::OMPTaskLoopSimdDirectiveClass:
318    EmitOMPTaskLoopSimdDirective(cast<OMPTaskLoopSimdDirective>(*S));
319    break;
320  case Stmt::OMPMasterTaskLoopDirectiveClass:
321    EmitOMPMasterTaskLoopDirective(cast<OMPMasterTaskLoopDirective>(*S));
322    break;
323  case Stmt::OMPMaskedTaskLoopDirectiveClass:
324    llvm_unreachable("masked taskloop directive not supported yet.");
325    break;
326  case Stmt::OMPMasterTaskLoopSimdDirectiveClass:
327    EmitOMPMasterTaskLoopSimdDirective(
328        cast<OMPMasterTaskLoopSimdDirective>(*S));
329    break;
330  case Stmt::OMPMaskedTaskLoopSimdDirectiveClass:
331    llvm_unreachable("masked taskloop simd directive not supported yet.");
332    break;
333  case Stmt::OMPParallelMasterTaskLoopDirectiveClass:
334    EmitOMPParallelMasterTaskLoopDirective(
335        cast<OMPParallelMasterTaskLoopDirective>(*S));
336    break;
337  case Stmt::OMPParallelMaskedTaskLoopDirectiveClass:
338    llvm_unreachable("parallel masked taskloop directive not supported yet.");
339    break;
340  case Stmt::OMPParallelMasterTaskLoopSimdDirectiveClass:
341    EmitOMPParallelMasterTaskLoopSimdDirective(
342        cast<OMPParallelMasterTaskLoopSimdDirective>(*S));
343    break;
344  case Stmt::OMPParallelMaskedTaskLoopSimdDirectiveClass:
345    llvm_unreachable(
346        "parallel masked taskloop simd directive not supported yet.");
347    break;
348  case Stmt::OMPDistributeDirectiveClass:
349    EmitOMPDistributeDirective(cast<OMPDistributeDirective>(*S));
350    break;
351  case Stmt::OMPTargetUpdateDirectiveClass:
352    EmitOMPTargetUpdateDirective(cast<OMPTargetUpdateDirective>(*S));
353    break;
354  case Stmt::OMPDistributeParallelForDirectiveClass:
355    EmitOMPDistributeParallelForDirective(
356        cast<OMPDistributeParallelForDirective>(*S));
357    break;
358  case Stmt::OMPDistributeParallelForSimdDirectiveClass:
359    EmitOMPDistributeParallelForSimdDirective(
360        cast<OMPDistributeParallelForSimdDirective>(*S));
361    break;
362  case Stmt::OMPDistributeSimdDirectiveClass:
363    EmitOMPDistributeSimdDirective(cast<OMPDistributeSimdDirective>(*S));
364    break;
365  case Stmt::OMPTargetParallelForSimdDirectiveClass:
366    EmitOMPTargetParallelForSimdDirective(
367        cast<OMPTargetParallelForSimdDirective>(*S));
368    break;
369  case Stmt::OMPTargetSimdDirectiveClass:
370    EmitOMPTargetSimdDirective(cast<OMPTargetSimdDirective>(*S));
371    break;
372  case Stmt::OMPTeamsDistributeDirectiveClass:
373    EmitOMPTeamsDistributeDirective(cast<OMPTeamsDistributeDirective>(*S));
374    break;
375  case Stmt::OMPTeamsDistributeSimdDirectiveClass:
376    EmitOMPTeamsDistributeSimdDirective(
377        cast<OMPTeamsDistributeSimdDirective>(*S));
378    break;
379  case Stmt::OMPTeamsDistributeParallelForSimdDirectiveClass:
380    EmitOMPTeamsDistributeParallelForSimdDirective(
381        cast<OMPTeamsDistributeParallelForSimdDirective>(*S));
382    break;
383  case Stmt::OMPTeamsDistributeParallelForDirectiveClass:
384    EmitOMPTeamsDistributeParallelForDirective(
385        cast<OMPTeamsDistributeParallelForDirective>(*S));
386    break;
387  case Stmt::OMPTargetTeamsDirectiveClass:
388    EmitOMPTargetTeamsDirective(cast<OMPTargetTeamsDirective>(*S));
389    break;
390  case Stmt::OMPTargetTeamsDistributeDirectiveClass:
391    EmitOMPTargetTeamsDistributeDirective(
392        cast<OMPTargetTeamsDistributeDirective>(*S));
393    break;
394  case Stmt::OMPTargetTeamsDistributeParallelForDirectiveClass:
395    EmitOMPTargetTeamsDistributeParallelForDirective(
396        cast<OMPTargetTeamsDistributeParallelForDirective>(*S));
397    break;
398  case Stmt::OMPTargetTeamsDistributeParallelForSimdDirectiveClass:
399    EmitOMPTargetTeamsDistributeParallelForSimdDirective(
400        cast<OMPTargetTeamsDistributeParallelForSimdDirective>(*S));
401    break;
402  case Stmt::OMPTargetTeamsDistributeSimdDirectiveClass:
403    EmitOMPTargetTeamsDistributeSimdDirective(
404        cast<OMPTargetTeamsDistributeSimdDirective>(*S));
405    break;
406  case Stmt::OMPInteropDirectiveClass:
407    EmitOMPInteropDirective(cast<OMPInteropDirective>(*S));
408    break;
409  case Stmt::OMPDispatchDirectiveClass:
410    CGM.ErrorUnsupported(S, "OpenMP dispatch directive");
411    break;
412  case Stmt::OMPScopeDirectiveClass:
413    llvm_unreachable("scope not supported with FE outlining");
414  case Stmt::OMPMaskedDirectiveClass:
415    EmitOMPMaskedDirective(cast<OMPMaskedDirective>(*S));
416    break;
417  case Stmt::OMPGenericLoopDirectiveClass:
418    EmitOMPGenericLoopDirective(cast<OMPGenericLoopDirective>(*S));
419    break;
420  case Stmt::OMPTeamsGenericLoopDirectiveClass:
421    EmitOMPTeamsGenericLoopDirective(cast<OMPTeamsGenericLoopDirective>(*S));
422    break;
423  case Stmt::OMPTargetTeamsGenericLoopDirectiveClass:
424    EmitOMPTargetTeamsGenericLoopDirective(
425        cast<OMPTargetTeamsGenericLoopDirective>(*S));
426    break;
427  case Stmt::OMPParallelGenericLoopDirectiveClass:
428    EmitOMPParallelGenericLoopDirective(
429        cast<OMPParallelGenericLoopDirective>(*S));
430    break;
431  case Stmt::OMPTargetParallelGenericLoopDirectiveClass:
432    EmitOMPTargetParallelGenericLoopDirective(
433        cast<OMPTargetParallelGenericLoopDirective>(*S));
434    break;
435  case Stmt::OMPParallelMaskedDirectiveClass:
436    EmitOMPParallelMaskedDirective(cast<OMPParallelMaskedDirective>(*S));
437    break;
438  }
439}
440
441bool CodeGenFunction::EmitSimpleStmt(const Stmt *S,
442                                     ArrayRef<const Attr *> Attrs) {
443  switch (S->getStmtClass()) {
444  default:
445    return false;
446  case Stmt::NullStmtClass:
447    break;
448  case Stmt::CompoundStmtClass:
449    EmitCompoundStmt(cast<CompoundStmt>(*S));
450    break;
451  case Stmt::DeclStmtClass:
452    EmitDeclStmt(cast<DeclStmt>(*S));
453    break;
454  case Stmt::LabelStmtClass:
455    EmitLabelStmt(cast<LabelStmt>(*S));
456    break;
457  case Stmt::AttributedStmtClass:
458    EmitAttributedStmt(cast<AttributedStmt>(*S));
459    break;
460  case Stmt::GotoStmtClass:
461    EmitGotoStmt(cast<GotoStmt>(*S));
462    break;
463  case Stmt::BreakStmtClass:
464    EmitBreakStmt(cast<BreakStmt>(*S));
465    break;
466  case Stmt::ContinueStmtClass:
467    EmitContinueStmt(cast<ContinueStmt>(*S));
468    break;
469  case Stmt::DefaultStmtClass:
470    EmitDefaultStmt(cast<DefaultStmt>(*S), Attrs);
471    break;
472  case Stmt::CaseStmtClass:
473    EmitCaseStmt(cast<CaseStmt>(*S), Attrs);
474    break;
475  case Stmt::SEHLeaveStmtClass:
476    EmitSEHLeaveStmt(cast<SEHLeaveStmt>(*S));
477    break;
478  }
479  return true;
480}
481
482/// EmitCompoundStmt - Emit a compound statement {..} node.  If GetLast is true,
483/// this captures the expression result of the last sub-statement and returns it
484/// (for use by the statement expression extension).
485Address CodeGenFunction::EmitCompoundStmt(const CompoundStmt &S, bool GetLast,
486                                          AggValueSlot AggSlot) {
487  PrettyStackTraceLoc CrashInfo(getContext().getSourceManager(),S.getLBracLoc(),
488                             "LLVM IR generation of compound statement ('{}')");
489
490  // Keep track of the current cleanup stack depth, including debug scopes.
491  LexicalScope Scope(*this, S.getSourceRange());
492
493  return EmitCompoundStmtWithoutScope(S, GetLast, AggSlot);
494}
495
496Address
497CodeGenFunction::EmitCompoundStmtWithoutScope(const CompoundStmt &S,
498                                              bool GetLast,
499                                              AggValueSlot AggSlot) {
500
501  const Stmt *ExprResult = S.getStmtExprResult();
502  assert((!GetLast || (GetLast && ExprResult)) &&
503         "If GetLast is true then the CompoundStmt must have a StmtExprResult");
504
505  Address RetAlloca = Address::invalid();
506
507  for (auto *CurStmt : S.body()) {
508    if (GetLast && ExprResult == CurStmt) {
509      // We have to special case labels here.  They are statements, but when put
510      // at the end of a statement expression, they yield the value of their
511      // subexpression.  Handle this by walking through all labels we encounter,
512      // emitting them before we evaluate the subexpr.
513      // Similar issues arise for attributed statements.
514      while (!isa<Expr>(ExprResult)) {
515        if (const auto *LS = dyn_cast<LabelStmt>(ExprResult)) {
516          EmitLabel(LS->getDecl());
517          ExprResult = LS->getSubStmt();
518        } else if (const auto *AS = dyn_cast<AttributedStmt>(ExprResult)) {
519          // FIXME: Update this if we ever have attributes that affect the
520          // semantics of an expression.
521          ExprResult = AS->getSubStmt();
522        } else {
523          llvm_unreachable("unknown value statement");
524        }
525      }
526
527      EnsureInsertPoint();
528
529      const Expr *E = cast<Expr>(ExprResult);
530      QualType ExprTy = E->getType();
531      if (hasAggregateEvaluationKind(ExprTy)) {
532        EmitAggExpr(E, AggSlot);
533      } else {
534        // We can't return an RValue here because there might be cleanups at
535        // the end of the StmtExpr.  Because of that, we have to emit the result
536        // here into a temporary alloca.
537        RetAlloca = CreateMemTemp(ExprTy);
538        EmitAnyExprToMem(E, RetAlloca, Qualifiers(),
539                         /*IsInit*/ false);
540      }
541    } else {
542      EmitStmt(CurStmt);
543    }
544  }
545
546  return RetAlloca;
547}
548
549void CodeGenFunction::SimplifyForwardingBlocks(llvm::BasicBlock *BB) {
550  llvm::BranchInst *BI = dyn_cast<llvm::BranchInst>(BB->getTerminator());
551
552  // If there is a cleanup stack, then we it isn't worth trying to
553  // simplify this block (we would need to remove it from the scope map
554  // and cleanup entry).
555  if (!EHStack.empty())
556    return;
557
558  // Can only simplify direct branches.
559  if (!BI || !BI->isUnconditional())
560    return;
561
562  // Can only simplify empty blocks.
563  if (BI->getIterator() != BB->begin())
564    return;
565
566  BB->replaceAllUsesWith(BI->getSuccessor(0));
567  BI->eraseFromParent();
568  BB->eraseFromParent();
569}
570
571void CodeGenFunction::EmitBlock(llvm::BasicBlock *BB, bool IsFinished) {
572  llvm::BasicBlock *CurBB = Builder.GetInsertBlock();
573
574  // Fall out of the current block (if necessary).
575  EmitBranch(BB);
576
577  if (IsFinished && BB->use_empty()) {
578    delete BB;
579    return;
580  }
581
582  // Place the block after the current block, if possible, or else at
583  // the end of the function.
584  if (CurBB && CurBB->getParent())
585    CurFn->insert(std::next(CurBB->getIterator()), BB);
586  else
587    CurFn->insert(CurFn->end(), BB);
588  Builder.SetInsertPoint(BB);
589}
590
591void CodeGenFunction::EmitBranch(llvm::BasicBlock *Target) {
592  // Emit a branch from the current block to the target one if this
593  // was a real block.  If this was just a fall-through block after a
594  // terminator, don't emit it.
595  llvm::BasicBlock *CurBB = Builder.GetInsertBlock();
596
597  if (!CurBB || CurBB->getTerminator()) {
598    // If there is no insert point or the previous block is already
599    // terminated, don't touch it.
600  } else {
601    // Otherwise, create a fall-through branch.
602    Builder.CreateBr(Target);
603  }
604
605  Builder.ClearInsertionPoint();
606}
607
608void CodeGenFunction::EmitBlockAfterUses(llvm::BasicBlock *block) {
609  bool inserted = false;
610  for (llvm::User *u : block->users()) {
611    if (llvm::Instruction *insn = dyn_cast<llvm::Instruction>(u)) {
612      CurFn->insert(std::next(insn->getParent()->getIterator()), block);
613      inserted = true;
614      break;
615    }
616  }
617
618  if (!inserted)
619    CurFn->insert(CurFn->end(), block);
620
621  Builder.SetInsertPoint(block);
622}
623
624CodeGenFunction::JumpDest
625CodeGenFunction::getJumpDestForLabel(const LabelDecl *D) {
626  JumpDest &Dest = LabelMap[D];
627  if (Dest.isValid()) return Dest;
628
629  // Create, but don't insert, the new block.
630  Dest = JumpDest(createBasicBlock(D->getName()),
631                  EHScopeStack::stable_iterator::invalid(),
632                  NextCleanupDestIndex++);
633  return Dest;
634}
635
636void CodeGenFunction::EmitLabel(const LabelDecl *D) {
637  // Add this label to the current lexical scope if we're within any
638  // normal cleanups.  Jumps "in" to this label --- when permitted by
639  // the language --- may need to be routed around such cleanups.
640  if (EHStack.hasNormalCleanups() && CurLexicalScope)
641    CurLexicalScope->addLabel(D);
642
643  JumpDest &Dest = LabelMap[D];
644
645  // If we didn't need a forward reference to this label, just go
646  // ahead and create a destination at the current scope.
647  if (!Dest.isValid()) {
648    Dest = getJumpDestInCurrentScope(D->getName());
649
650  // Otherwise, we need to give this label a target depth and remove
651  // it from the branch-fixups list.
652  } else {
653    assert(!Dest.getScopeDepth().isValid() && "already emitted label!");
654    Dest.setScopeDepth(EHStack.stable_begin());
655    ResolveBranchFixups(Dest.getBlock());
656  }
657
658  EmitBlock(Dest.getBlock());
659
660  // Emit debug info for labels.
661  if (CGDebugInfo *DI = getDebugInfo()) {
662    if (CGM.getCodeGenOpts().hasReducedDebugInfo()) {
663      DI->setLocation(D->getLocation());
664      DI->EmitLabel(D, Builder);
665    }
666  }
667
668  incrementProfileCounter(D->getStmt());
669}
670
671/// Change the cleanup scope of the labels in this lexical scope to
672/// match the scope of the enclosing context.
673void CodeGenFunction::LexicalScope::rescopeLabels() {
674  assert(!Labels.empty());
675  EHScopeStack::stable_iterator innermostScope
676    = CGF.EHStack.getInnermostNormalCleanup();
677
678  // Change the scope depth of all the labels.
679  for (SmallVectorImpl<const LabelDecl*>::const_iterator
680         i = Labels.begin(), e = Labels.end(); i != e; ++i) {
681    assert(CGF.LabelMap.count(*i));
682    JumpDest &dest = CGF.LabelMap.find(*i)->second;
683    assert(dest.getScopeDepth().isValid());
684    assert(innermostScope.encloses(dest.getScopeDepth()));
685    dest.setScopeDepth(innermostScope);
686  }
687
688  // Reparent the labels if the new scope also has cleanups.
689  if (innermostScope != EHScopeStack::stable_end() && ParentScope) {
690    ParentScope->Labels.append(Labels.begin(), Labels.end());
691  }
692}
693
694
695void CodeGenFunction::EmitLabelStmt(const LabelStmt &S) {
696  EmitLabel(S.getDecl());
697
698  // IsEHa - emit eha.scope.begin if it's a side entry of a scope
699  if (getLangOpts().EHAsynch && S.isSideEntry())
700    EmitSehCppScopeBegin();
701
702  EmitStmt(S.getSubStmt());
703}
704
705void CodeGenFunction::EmitAttributedStmt(const AttributedStmt &S) {
706  bool nomerge = false;
707  bool noinline = false;
708  bool alwaysinline = false;
709  const CallExpr *musttail = nullptr;
710
711  for (const auto *A : S.getAttrs()) {
712    switch (A->getKind()) {
713    default:
714      break;
715    case attr::NoMerge:
716      nomerge = true;
717      break;
718    case attr::NoInline:
719      noinline = true;
720      break;
721    case attr::AlwaysInline:
722      alwaysinline = true;
723      break;
724    case attr::MustTail:
725      const Stmt *Sub = S.getSubStmt();
726      const ReturnStmt *R = cast<ReturnStmt>(Sub);
727      musttail = cast<CallExpr>(R->getRetValue()->IgnoreParens());
728      break;
729    }
730  }
731  SaveAndRestore save_nomerge(InNoMergeAttributedStmt, nomerge);
732  SaveAndRestore save_noinline(InNoInlineAttributedStmt, noinline);
733  SaveAndRestore save_alwaysinline(InAlwaysInlineAttributedStmt, alwaysinline);
734  SaveAndRestore save_musttail(MustTailCall, musttail);
735  EmitStmt(S.getSubStmt(), S.getAttrs());
736}
737
738void CodeGenFunction::EmitGotoStmt(const GotoStmt &S) {
739  // If this code is reachable then emit a stop point (if generating
740  // debug info). We have to do this ourselves because we are on the
741  // "simple" statement path.
742  if (HaveInsertPoint())
743    EmitStopPoint(&S);
744
745  EmitBranchThroughCleanup(getJumpDestForLabel(S.getLabel()));
746}
747
748
749void CodeGenFunction::EmitIndirectGotoStmt(const IndirectGotoStmt &S) {
750  if (const LabelDecl *Target = S.getConstantTarget()) {
751    EmitBranchThroughCleanup(getJumpDestForLabel(Target));
752    return;
753  }
754
755  // Ensure that we have an i8* for our PHI node.
756  llvm::Value *V = Builder.CreateBitCast(EmitScalarExpr(S.getTarget()),
757                                         Int8PtrTy, "addr");
758  llvm::BasicBlock *CurBB = Builder.GetInsertBlock();
759
760  // Get the basic block for the indirect goto.
761  llvm::BasicBlock *IndGotoBB = GetIndirectGotoBlock();
762
763  // The first instruction in the block has to be the PHI for the switch dest,
764  // add an entry for this branch.
765  cast<llvm::PHINode>(IndGotoBB->begin())->addIncoming(V, CurBB);
766
767  EmitBranch(IndGotoBB);
768}
769
770void CodeGenFunction::EmitIfStmt(const IfStmt &S) {
771  // The else branch of a consteval if statement is always the only branch that
772  // can be runtime evaluated.
773  if (S.isConsteval()) {
774    const Stmt *Executed = S.isNegatedConsteval() ? S.getThen() : S.getElse();
775    if (Executed) {
776      RunCleanupsScope ExecutedScope(*this);
777      EmitStmt(Executed);
778    }
779    return;
780  }
781
782  // C99 6.8.4.1: The first substatement is executed if the expression compares
783  // unequal to 0.  The condition must be a scalar type.
784  LexicalScope ConditionScope(*this, S.getCond()->getSourceRange());
785
786  if (S.getInit())
787    EmitStmt(S.getInit());
788
789  if (S.getConditionVariable())
790    EmitDecl(*S.getConditionVariable());
791
792  // If the condition constant folds and can be elided, try to avoid emitting
793  // the condition and the dead arm of the if/else.
794  bool CondConstant;
795  if (ConstantFoldsToSimpleInteger(S.getCond(), CondConstant,
796                                   S.isConstexpr())) {
797    // Figure out which block (then or else) is executed.
798    const Stmt *Executed = S.getThen();
799    const Stmt *Skipped  = S.getElse();
800    if (!CondConstant)  // Condition false?
801      std::swap(Executed, Skipped);
802
803    // If the skipped block has no labels in it, just emit the executed block.
804    // This avoids emitting dead code and simplifies the CFG substantially.
805    if (S.isConstexpr() || !ContainsLabel(Skipped)) {
806      if (CondConstant)
807        incrementProfileCounter(&S);
808      if (Executed) {
809        RunCleanupsScope ExecutedScope(*this);
810        EmitStmt(Executed);
811      }
812      return;
813    }
814  }
815
816  // Otherwise, the condition did not fold, or we couldn't elide it.  Just emit
817  // the conditional branch.
818  llvm::BasicBlock *ThenBlock = createBasicBlock("if.then");
819  llvm::BasicBlock *ContBlock = createBasicBlock("if.end");
820  llvm::BasicBlock *ElseBlock = ContBlock;
821  if (S.getElse())
822    ElseBlock = createBasicBlock("if.else");
823
824  // Prefer the PGO based weights over the likelihood attribute.
825  // When the build isn't optimized the metadata isn't used, so don't generate
826  // it.
827  // Also, differentiate between disabled PGO and a never executed branch with
828  // PGO. Assuming PGO is in use:
829  // - we want to ignore the [[likely]] attribute if the branch is never
830  // executed,
831  // - assuming the profile is poor, preserving the attribute may still be
832  // beneficial.
833  // As an approximation, preserve the attribute only if both the branch and the
834  // parent context were not executed.
835  Stmt::Likelihood LH = Stmt::LH_None;
836  uint64_t ThenCount = getProfileCount(S.getThen());
837  if (!ThenCount && !getCurrentProfileCount() &&
838      CGM.getCodeGenOpts().OptimizationLevel)
839    LH = Stmt::getLikelihood(S.getThen(), S.getElse());
840
841  // When measuring MC/DC, always fully evaluate the condition up front using
842  // EvaluateExprAsBool() so that the test vector bitmap can be updated prior to
843  // executing the body of the if.then or if.else. This is useful for when
844  // there is a 'return' within the body, but this is particularly beneficial
845  // when one if-stmt is nested within another if-stmt so that all of the MC/DC
846  // updates are kept linear and consistent.
847  if (!CGM.getCodeGenOpts().MCDCCoverage)
848    EmitBranchOnBoolExpr(S.getCond(), ThenBlock, ElseBlock, ThenCount, LH);
849  else {
850    llvm::Value *BoolCondVal = EvaluateExprAsBool(S.getCond());
851    Builder.CreateCondBr(BoolCondVal, ThenBlock, ElseBlock);
852  }
853
854  // Emit the 'then' code.
855  EmitBlock(ThenBlock);
856  incrementProfileCounter(&S);
857  {
858    RunCleanupsScope ThenScope(*this);
859    EmitStmt(S.getThen());
860  }
861  EmitBranch(ContBlock);
862
863  // Emit the 'else' code if present.
864  if (const Stmt *Else = S.getElse()) {
865    {
866      // There is no need to emit line number for an unconditional branch.
867      auto NL = ApplyDebugLocation::CreateEmpty(*this);
868      EmitBlock(ElseBlock);
869    }
870    {
871      RunCleanupsScope ElseScope(*this);
872      EmitStmt(Else);
873    }
874    {
875      // There is no need to emit line number for an unconditional branch.
876      auto NL = ApplyDebugLocation::CreateEmpty(*this);
877      EmitBranch(ContBlock);
878    }
879  }
880
881  // Emit the continuation block for code after the if.
882  EmitBlock(ContBlock, true);
883}
884
885void CodeGenFunction::EmitWhileStmt(const WhileStmt &S,
886                                    ArrayRef<const Attr *> WhileAttrs) {
887  // Emit the header for the loop, which will also become
888  // the continue target.
889  JumpDest LoopHeader = getJumpDestInCurrentScope("while.cond");
890  EmitBlock(LoopHeader.getBlock());
891
892  // Create an exit block for when the condition fails, which will
893  // also become the break target.
894  JumpDest LoopExit = getJumpDestInCurrentScope("while.end");
895
896  // Store the blocks to use for break and continue.
897  BreakContinueStack.push_back(BreakContinue(LoopExit, LoopHeader));
898
899  // C++ [stmt.while]p2:
900  //   When the condition of a while statement is a declaration, the
901  //   scope of the variable that is declared extends from its point
902  //   of declaration (3.3.2) to the end of the while statement.
903  //   [...]
904  //   The object created in a condition is destroyed and created
905  //   with each iteration of the loop.
906  RunCleanupsScope ConditionScope(*this);
907
908  if (S.getConditionVariable())
909    EmitDecl(*S.getConditionVariable());
910
911  // Evaluate the conditional in the while header.  C99 6.8.5.1: The
912  // evaluation of the controlling expression takes place before each
913  // execution of the loop body.
914  llvm::Value *BoolCondVal = EvaluateExprAsBool(S.getCond());
915
916  // while(1) is common, avoid extra exit blocks.  Be sure
917  // to correctly handle break/continue though.
918  llvm::ConstantInt *C = dyn_cast<llvm::ConstantInt>(BoolCondVal);
919  bool CondIsConstInt = C != nullptr;
920  bool EmitBoolCondBranch = !CondIsConstInt || !C->isOne();
921  const SourceRange &R = S.getSourceRange();
922  LoopStack.push(LoopHeader.getBlock(), CGM.getContext(), CGM.getCodeGenOpts(),
923                 WhileAttrs, SourceLocToDebugLoc(R.getBegin()),
924                 SourceLocToDebugLoc(R.getEnd()),
925                 checkIfLoopMustProgress(CondIsConstInt));
926
927  // As long as the condition is true, go to the loop body.
928  llvm::BasicBlock *LoopBody = createBasicBlock("while.body");
929  if (EmitBoolCondBranch) {
930    llvm::BasicBlock *ExitBlock = LoopExit.getBlock();
931    if (ConditionScope.requiresCleanups())
932      ExitBlock = createBasicBlock("while.exit");
933    llvm::MDNode *Weights =
934        createProfileWeightsForLoop(S.getCond(), getProfileCount(S.getBody()));
935    if (!Weights && CGM.getCodeGenOpts().OptimizationLevel)
936      BoolCondVal = emitCondLikelihoodViaExpectIntrinsic(
937          BoolCondVal, Stmt::getLikelihood(S.getBody()));
938    Builder.CreateCondBr(BoolCondVal, LoopBody, ExitBlock, Weights);
939
940    if (ExitBlock != LoopExit.getBlock()) {
941      EmitBlock(ExitBlock);
942      EmitBranchThroughCleanup(LoopExit);
943    }
944  } else if (const Attr *A = Stmt::getLikelihoodAttr(S.getBody())) {
945    CGM.getDiags().Report(A->getLocation(),
946                          diag::warn_attribute_has_no_effect_on_infinite_loop)
947        << A << A->getRange();
948    CGM.getDiags().Report(
949        S.getWhileLoc(),
950        diag::note_attribute_has_no_effect_on_infinite_loop_here)
951        << SourceRange(S.getWhileLoc(), S.getRParenLoc());
952  }
953
954  // Emit the loop body.  We have to emit this in a cleanup scope
955  // because it might be a singleton DeclStmt.
956  {
957    RunCleanupsScope BodyScope(*this);
958    EmitBlock(LoopBody);
959    incrementProfileCounter(&S);
960    EmitStmt(S.getBody());
961  }
962
963  BreakContinueStack.pop_back();
964
965  // Immediately force cleanup.
966  ConditionScope.ForceCleanup();
967
968  EmitStopPoint(&S);
969  // Branch to the loop header again.
970  EmitBranch(LoopHeader.getBlock());
971
972  LoopStack.pop();
973
974  // Emit the exit block.
975  EmitBlock(LoopExit.getBlock(), true);
976
977  // The LoopHeader typically is just a branch if we skipped emitting
978  // a branch, try to erase it.
979  if (!EmitBoolCondBranch)
980    SimplifyForwardingBlocks(LoopHeader.getBlock());
981}
982
983void CodeGenFunction::EmitDoStmt(const DoStmt &S,
984                                 ArrayRef<const Attr *> DoAttrs) {
985  JumpDest LoopExit = getJumpDestInCurrentScope("do.end");
986  JumpDest LoopCond = getJumpDestInCurrentScope("do.cond");
987
988  uint64_t ParentCount = getCurrentProfileCount();
989
990  // Store the blocks to use for break and continue.
991  BreakContinueStack.push_back(BreakContinue(LoopExit, LoopCond));
992
993  // Emit the body of the loop.
994  llvm::BasicBlock *LoopBody = createBasicBlock("do.body");
995
996  EmitBlockWithFallThrough(LoopBody, &S);
997  {
998    RunCleanupsScope BodyScope(*this);
999    EmitStmt(S.getBody());
1000  }
1001
1002  EmitBlock(LoopCond.getBlock());
1003
1004  // C99 6.8.5.2: "The evaluation of the controlling expression takes place
1005  // after each execution of the loop body."
1006
1007  // Evaluate the conditional in the while header.
1008  // C99 6.8.5p2/p4: The first substatement is executed if the expression
1009  // compares unequal to 0.  The condition must be a scalar type.
1010  llvm::Value *BoolCondVal = EvaluateExprAsBool(S.getCond());
1011
1012  BreakContinueStack.pop_back();
1013
1014  // "do {} while (0)" is common in macros, avoid extra blocks.  Be sure
1015  // to correctly handle break/continue though.
1016  llvm::ConstantInt *C = dyn_cast<llvm::ConstantInt>(BoolCondVal);
1017  bool CondIsConstInt = C;
1018  bool EmitBoolCondBranch = !C || !C->isZero();
1019
1020  const SourceRange &R = S.getSourceRange();
1021  LoopStack.push(LoopBody, CGM.getContext(), CGM.getCodeGenOpts(), DoAttrs,
1022                 SourceLocToDebugLoc(R.getBegin()),
1023                 SourceLocToDebugLoc(R.getEnd()),
1024                 checkIfLoopMustProgress(CondIsConstInt));
1025
1026  // As long as the condition is true, iterate the loop.
1027  if (EmitBoolCondBranch) {
1028    uint64_t BackedgeCount = getProfileCount(S.getBody()) - ParentCount;
1029    Builder.CreateCondBr(
1030        BoolCondVal, LoopBody, LoopExit.getBlock(),
1031        createProfileWeightsForLoop(S.getCond(), BackedgeCount));
1032  }
1033
1034  LoopStack.pop();
1035
1036  // Emit the exit block.
1037  EmitBlock(LoopExit.getBlock());
1038
1039  // The DoCond block typically is just a branch if we skipped
1040  // emitting a branch, try to erase it.
1041  if (!EmitBoolCondBranch)
1042    SimplifyForwardingBlocks(LoopCond.getBlock());
1043}
1044
1045void CodeGenFunction::EmitForStmt(const ForStmt &S,
1046                                  ArrayRef<const Attr *> ForAttrs) {
1047  JumpDest LoopExit = getJumpDestInCurrentScope("for.end");
1048
1049  LexicalScope ForScope(*this, S.getSourceRange());
1050
1051  // Evaluate the first part before the loop.
1052  if (S.getInit())
1053    EmitStmt(S.getInit());
1054
1055  // Start the loop with a block that tests the condition.
1056  // If there's an increment, the continue scope will be overwritten
1057  // later.
1058  JumpDest CondDest = getJumpDestInCurrentScope("for.cond");
1059  llvm::BasicBlock *CondBlock = CondDest.getBlock();
1060  EmitBlock(CondBlock);
1061
1062  Expr::EvalResult Result;
1063  bool CondIsConstInt =
1064      !S.getCond() || S.getCond()->EvaluateAsInt(Result, getContext());
1065
1066  const SourceRange &R = S.getSourceRange();
1067  LoopStack.push(CondBlock, CGM.getContext(), CGM.getCodeGenOpts(), ForAttrs,
1068                 SourceLocToDebugLoc(R.getBegin()),
1069                 SourceLocToDebugLoc(R.getEnd()),
1070                 checkIfLoopMustProgress(CondIsConstInt));
1071
1072  // Create a cleanup scope for the condition variable cleanups.
1073  LexicalScope ConditionScope(*this, S.getSourceRange());
1074
1075  // If the for loop doesn't have an increment we can just use the condition as
1076  // the continue block. Otherwise, if there is no condition variable, we can
1077  // form the continue block now. If there is a condition variable, we can't
1078  // form the continue block until after we've emitted the condition, because
1079  // the condition is in scope in the increment, but Sema's jump diagnostics
1080  // ensure that there are no continues from the condition variable that jump
1081  // to the loop increment.
1082  JumpDest Continue;
1083  if (!S.getInc())
1084    Continue = CondDest;
1085  else if (!S.getConditionVariable())
1086    Continue = getJumpDestInCurrentScope("for.inc");
1087  BreakContinueStack.push_back(BreakContinue(LoopExit, Continue));
1088
1089  if (S.getCond()) {
1090    // If the for statement has a condition scope, emit the local variable
1091    // declaration.
1092    if (S.getConditionVariable()) {
1093      EmitDecl(*S.getConditionVariable());
1094
1095      // We have entered the condition variable's scope, so we're now able to
1096      // jump to the continue block.
1097      Continue = S.getInc() ? getJumpDestInCurrentScope("for.inc") : CondDest;
1098      BreakContinueStack.back().ContinueBlock = Continue;
1099    }
1100
1101    llvm::BasicBlock *ExitBlock = LoopExit.getBlock();
1102    // If there are any cleanups between here and the loop-exit scope,
1103    // create a block to stage a loop exit along.
1104    if (ForScope.requiresCleanups())
1105      ExitBlock = createBasicBlock("for.cond.cleanup");
1106
1107    // As long as the condition is true, iterate the loop.
1108    llvm::BasicBlock *ForBody = createBasicBlock("for.body");
1109
1110    // C99 6.8.5p2/p4: The first substatement is executed if the expression
1111    // compares unequal to 0.  The condition must be a scalar type.
1112    llvm::Value *BoolCondVal = EvaluateExprAsBool(S.getCond());
1113    llvm::MDNode *Weights =
1114        createProfileWeightsForLoop(S.getCond(), getProfileCount(S.getBody()));
1115    if (!Weights && CGM.getCodeGenOpts().OptimizationLevel)
1116      BoolCondVal = emitCondLikelihoodViaExpectIntrinsic(
1117          BoolCondVal, Stmt::getLikelihood(S.getBody()));
1118
1119    Builder.CreateCondBr(BoolCondVal, ForBody, ExitBlock, Weights);
1120
1121    if (ExitBlock != LoopExit.getBlock()) {
1122      EmitBlock(ExitBlock);
1123      EmitBranchThroughCleanup(LoopExit);
1124    }
1125
1126    EmitBlock(ForBody);
1127  } else {
1128    // Treat it as a non-zero constant.  Don't even create a new block for the
1129    // body, just fall into it.
1130  }
1131  incrementProfileCounter(&S);
1132
1133  {
1134    // Create a separate cleanup scope for the body, in case it is not
1135    // a compound statement.
1136    RunCleanupsScope BodyScope(*this);
1137    EmitStmt(S.getBody());
1138  }
1139
1140  // If there is an increment, emit it next.
1141  if (S.getInc()) {
1142    EmitBlock(Continue.getBlock());
1143    EmitStmt(S.getInc());
1144  }
1145
1146  BreakContinueStack.pop_back();
1147
1148  ConditionScope.ForceCleanup();
1149
1150  EmitStopPoint(&S);
1151  EmitBranch(CondBlock);
1152
1153  ForScope.ForceCleanup();
1154
1155  LoopStack.pop();
1156
1157  // Emit the fall-through block.
1158  EmitBlock(LoopExit.getBlock(), true);
1159}
1160
1161void
1162CodeGenFunction::EmitCXXForRangeStmt(const CXXForRangeStmt &S,
1163                                     ArrayRef<const Attr *> ForAttrs) {
1164  JumpDest LoopExit = getJumpDestInCurrentScope("for.end");
1165
1166  LexicalScope ForScope(*this, S.getSourceRange());
1167
1168  // Evaluate the first pieces before the loop.
1169  if (S.getInit())
1170    EmitStmt(S.getInit());
1171  EmitStmt(S.getRangeStmt());
1172  EmitStmt(S.getBeginStmt());
1173  EmitStmt(S.getEndStmt());
1174
1175  // Start the loop with a block that tests the condition.
1176  // If there's an increment, the continue scope will be overwritten
1177  // later.
1178  llvm::BasicBlock *CondBlock = createBasicBlock("for.cond");
1179  EmitBlock(CondBlock);
1180
1181  const SourceRange &R = S.getSourceRange();
1182  LoopStack.push(CondBlock, CGM.getContext(), CGM.getCodeGenOpts(), ForAttrs,
1183                 SourceLocToDebugLoc(R.getBegin()),
1184                 SourceLocToDebugLoc(R.getEnd()));
1185
1186  // If there are any cleanups between here and the loop-exit scope,
1187  // create a block to stage a loop exit along.
1188  llvm::BasicBlock *ExitBlock = LoopExit.getBlock();
1189  if (ForScope.requiresCleanups())
1190    ExitBlock = createBasicBlock("for.cond.cleanup");
1191
1192  // The loop body, consisting of the specified body and the loop variable.
1193  llvm::BasicBlock *ForBody = createBasicBlock("for.body");
1194
1195  // The body is executed if the expression, contextually converted
1196  // to bool, is true.
1197  llvm::Value *BoolCondVal = EvaluateExprAsBool(S.getCond());
1198  llvm::MDNode *Weights =
1199      createProfileWeightsForLoop(S.getCond(), getProfileCount(S.getBody()));
1200  if (!Weights && CGM.getCodeGenOpts().OptimizationLevel)
1201    BoolCondVal = emitCondLikelihoodViaExpectIntrinsic(
1202        BoolCondVal, Stmt::getLikelihood(S.getBody()));
1203  Builder.CreateCondBr(BoolCondVal, ForBody, ExitBlock, Weights);
1204
1205  if (ExitBlock != LoopExit.getBlock()) {
1206    EmitBlock(ExitBlock);
1207    EmitBranchThroughCleanup(LoopExit);
1208  }
1209
1210  EmitBlock(ForBody);
1211  incrementProfileCounter(&S);
1212
1213  // Create a block for the increment. In case of a 'continue', we jump there.
1214  JumpDest Continue = getJumpDestInCurrentScope("for.inc");
1215
1216  // Store the blocks to use for break and continue.
1217  BreakContinueStack.push_back(BreakContinue(LoopExit, Continue));
1218
1219  {
1220    // Create a separate cleanup scope for the loop variable and body.
1221    LexicalScope BodyScope(*this, S.getSourceRange());
1222    EmitStmt(S.getLoopVarStmt());
1223    EmitStmt(S.getBody());
1224  }
1225
1226  EmitStopPoint(&S);
1227  // If there is an increment, emit it next.
1228  EmitBlock(Continue.getBlock());
1229  EmitStmt(S.getInc());
1230
1231  BreakContinueStack.pop_back();
1232
1233  EmitBranch(CondBlock);
1234
1235  ForScope.ForceCleanup();
1236
1237  LoopStack.pop();
1238
1239  // Emit the fall-through block.
1240  EmitBlock(LoopExit.getBlock(), true);
1241}
1242
1243void CodeGenFunction::EmitReturnOfRValue(RValue RV, QualType Ty) {
1244  if (RV.isScalar()) {
1245    Builder.CreateStore(RV.getScalarVal(), ReturnValue);
1246  } else if (RV.isAggregate()) {
1247    LValue Dest = MakeAddrLValue(ReturnValue, Ty);
1248    LValue Src = MakeAddrLValue(RV.getAggregateAddress(), Ty);
1249    EmitAggregateCopy(Dest, Src, Ty, getOverlapForReturnValue());
1250  } else {
1251    EmitStoreOfComplex(RV.getComplexVal(), MakeAddrLValue(ReturnValue, Ty),
1252                       /*init*/ true);
1253  }
1254  EmitBranchThroughCleanup(ReturnBlock);
1255}
1256
1257namespace {
1258// RAII struct used to save and restore a return statment's result expression.
1259struct SaveRetExprRAII {
1260  SaveRetExprRAII(const Expr *RetExpr, CodeGenFunction &CGF)
1261      : OldRetExpr(CGF.RetExpr), CGF(CGF) {
1262    CGF.RetExpr = RetExpr;
1263  }
1264  ~SaveRetExprRAII() { CGF.RetExpr = OldRetExpr; }
1265  const Expr *OldRetExpr;
1266  CodeGenFunction &CGF;
1267};
1268} // namespace
1269
1270/// If we have 'return f(...);', where both caller and callee are SwiftAsync,
1271/// codegen it as 'tail call ...; ret void;'.
1272static void makeTailCallIfSwiftAsync(const CallExpr *CE, CGBuilderTy &Builder,
1273                                     const CGFunctionInfo *CurFnInfo) {
1274  auto calleeQualType = CE->getCallee()->getType();
1275  const FunctionType *calleeType = nullptr;
1276  if (calleeQualType->isFunctionPointerType() ||
1277      calleeQualType->isFunctionReferenceType() ||
1278      calleeQualType->isBlockPointerType() ||
1279      calleeQualType->isMemberFunctionPointerType()) {
1280    calleeType = calleeQualType->getPointeeType()->castAs<FunctionType>();
1281  } else if (auto *ty = dyn_cast<FunctionType>(calleeQualType)) {
1282    calleeType = ty;
1283  } else if (auto CMCE = dyn_cast<CXXMemberCallExpr>(CE)) {
1284    if (auto methodDecl = CMCE->getMethodDecl()) {
1285      // getMethodDecl() doesn't handle member pointers at the moment.
1286      calleeType = methodDecl->getType()->castAs<FunctionType>();
1287    } else {
1288      return;
1289    }
1290  } else {
1291    return;
1292  }
1293  if (calleeType->getCallConv() == CallingConv::CC_SwiftAsync &&
1294      (CurFnInfo->getASTCallingConvention() == CallingConv::CC_SwiftAsync)) {
1295    auto CI = cast<llvm::CallInst>(&Builder.GetInsertBlock()->back());
1296    CI->setTailCallKind(llvm::CallInst::TCK_MustTail);
1297    Builder.CreateRetVoid();
1298    Builder.ClearInsertionPoint();
1299  }
1300}
1301
1302/// EmitReturnStmt - Note that due to GCC extensions, this can have an operand
1303/// if the function returns void, or may be missing one if the function returns
1304/// non-void.  Fun stuff :).
1305void CodeGenFunction::EmitReturnStmt(const ReturnStmt &S) {
1306  if (requiresReturnValueCheck()) {
1307    llvm::Constant *SLoc = EmitCheckSourceLocation(S.getBeginLoc());
1308    auto *SLocPtr =
1309        new llvm::GlobalVariable(CGM.getModule(), SLoc->getType(), false,
1310                                 llvm::GlobalVariable::PrivateLinkage, SLoc);
1311    SLocPtr->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global);
1312    CGM.getSanitizerMetadata()->disableSanitizerForGlobal(SLocPtr);
1313    assert(ReturnLocation.isValid() && "No valid return location");
1314    Builder.CreateStore(SLocPtr, ReturnLocation);
1315  }
1316
1317  // Returning from an outlined SEH helper is UB, and we already warn on it.
1318  if (IsOutlinedSEHHelper) {
1319    Builder.CreateUnreachable();
1320    Builder.ClearInsertionPoint();
1321  }
1322
1323  // Emit the result value, even if unused, to evaluate the side effects.
1324  const Expr *RV = S.getRetValue();
1325
1326  // Record the result expression of the return statement. The recorded
1327  // expression is used to determine whether a block capture's lifetime should
1328  // end at the end of the full expression as opposed to the end of the scope
1329  // enclosing the block expression.
1330  //
1331  // This permits a small, easily-implemented exception to our over-conservative
1332  // rules about not jumping to statements following block literals with
1333  // non-trivial cleanups.
1334  SaveRetExprRAII SaveRetExpr(RV, *this);
1335
1336  RunCleanupsScope cleanupScope(*this);
1337  if (const auto *EWC = dyn_cast_or_null<ExprWithCleanups>(RV))
1338    RV = EWC->getSubExpr();
1339  // FIXME: Clean this up by using an LValue for ReturnTemp,
1340  // EmitStoreThroughLValue, and EmitAnyExpr.
1341  // Check if the NRVO candidate was not globalized in OpenMP mode.
1342  if (getLangOpts().ElideConstructors && S.getNRVOCandidate() &&
1343      S.getNRVOCandidate()->isNRVOVariable() &&
1344      (!getLangOpts().OpenMP ||
1345       !CGM.getOpenMPRuntime()
1346            .getAddressOfLocalVariable(*this, S.getNRVOCandidate())
1347            .isValid())) {
1348    // Apply the named return value optimization for this return statement,
1349    // which means doing nothing: the appropriate result has already been
1350    // constructed into the NRVO variable.
1351
1352    // If there is an NRVO flag for this variable, set it to 1 into indicate
1353    // that the cleanup code should not destroy the variable.
1354    if (llvm::Value *NRVOFlag = NRVOFlags[S.getNRVOCandidate()])
1355      Builder.CreateFlagStore(Builder.getTrue(), NRVOFlag);
1356  } else if (!ReturnValue.isValid() || (RV && RV->getType()->isVoidType())) {
1357    // Make sure not to return anything, but evaluate the expression
1358    // for side effects.
1359    if (RV) {
1360      EmitAnyExpr(RV);
1361      if (auto *CE = dyn_cast<CallExpr>(RV))
1362        makeTailCallIfSwiftAsync(CE, Builder, CurFnInfo);
1363    }
1364  } else if (!RV) {
1365    // Do nothing (return value is left uninitialized)
1366  } else if (FnRetTy->isReferenceType()) {
1367    // If this function returns a reference, take the address of the expression
1368    // rather than the value.
1369    RValue Result = EmitReferenceBindingToExpr(RV);
1370    Builder.CreateStore(Result.getScalarVal(), ReturnValue);
1371  } else {
1372    switch (getEvaluationKind(RV->getType())) {
1373    case TEK_Scalar:
1374      Builder.CreateStore(EmitScalarExpr(RV), ReturnValue);
1375      break;
1376    case TEK_Complex:
1377      EmitComplexExprIntoLValue(RV, MakeAddrLValue(ReturnValue, RV->getType()),
1378                                /*isInit*/ true);
1379      break;
1380    case TEK_Aggregate:
1381      EmitAggExpr(RV, AggValueSlot::forAddr(
1382                          ReturnValue, Qualifiers(),
1383                          AggValueSlot::IsDestructed,
1384                          AggValueSlot::DoesNotNeedGCBarriers,
1385                          AggValueSlot::IsNotAliased,
1386                          getOverlapForReturnValue()));
1387      break;
1388    }
1389  }
1390
1391  ++NumReturnExprs;
1392  if (!RV || RV->isEvaluatable(getContext()))
1393    ++NumSimpleReturnExprs;
1394
1395  cleanupScope.ForceCleanup();
1396  EmitBranchThroughCleanup(ReturnBlock);
1397}
1398
1399void CodeGenFunction::EmitDeclStmt(const DeclStmt &S) {
1400  // As long as debug info is modeled with instructions, we have to ensure we
1401  // have a place to insert here and write the stop point here.
1402  if (HaveInsertPoint())
1403    EmitStopPoint(&S);
1404
1405  for (const auto *I : S.decls())
1406    EmitDecl(*I);
1407}
1408
1409void CodeGenFunction::EmitBreakStmt(const BreakStmt &S) {
1410  assert(!BreakContinueStack.empty() && "break stmt not in a loop or switch!");
1411
1412  // If this code is reachable then emit a stop point (if generating
1413  // debug info). We have to do this ourselves because we are on the
1414  // "simple" statement path.
1415  if (HaveInsertPoint())
1416    EmitStopPoint(&S);
1417
1418  EmitBranchThroughCleanup(BreakContinueStack.back().BreakBlock);
1419}
1420
1421void CodeGenFunction::EmitContinueStmt(const ContinueStmt &S) {
1422  assert(!BreakContinueStack.empty() && "continue stmt not in a loop!");
1423
1424  // If this code is reachable then emit a stop point (if generating
1425  // debug info). We have to do this ourselves because we are on the
1426  // "simple" statement path.
1427  if (HaveInsertPoint())
1428    EmitStopPoint(&S);
1429
1430  EmitBranchThroughCleanup(BreakContinueStack.back().ContinueBlock);
1431}
1432
1433/// EmitCaseStmtRange - If case statement range is not too big then
1434/// add multiple cases to switch instruction, one for each value within
1435/// the range. If range is too big then emit "if" condition check.
1436void CodeGenFunction::EmitCaseStmtRange(const CaseStmt &S,
1437                                        ArrayRef<const Attr *> Attrs) {
1438  assert(S.getRHS() && "Expected RHS value in CaseStmt");
1439
1440  llvm::APSInt LHS = S.getLHS()->EvaluateKnownConstInt(getContext());
1441  llvm::APSInt RHS = S.getRHS()->EvaluateKnownConstInt(getContext());
1442
1443  // Emit the code for this case. We do this first to make sure it is
1444  // properly chained from our predecessor before generating the
1445  // switch machinery to enter this block.
1446  llvm::BasicBlock *CaseDest = createBasicBlock("sw.bb");
1447  EmitBlockWithFallThrough(CaseDest, &S);
1448  EmitStmt(S.getSubStmt());
1449
1450  // If range is empty, do nothing.
1451  if (LHS.isSigned() ? RHS.slt(LHS) : RHS.ult(LHS))
1452    return;
1453
1454  Stmt::Likelihood LH = Stmt::getLikelihood(Attrs);
1455  llvm::APInt Range = RHS - LHS;
1456  // FIXME: parameters such as this should not be hardcoded.
1457  if (Range.ult(llvm::APInt(Range.getBitWidth(), 64))) {
1458    // Range is small enough to add multiple switch instruction cases.
1459    uint64_t Total = getProfileCount(&S);
1460    unsigned NCases = Range.getZExtValue() + 1;
1461    // We only have one region counter for the entire set of cases here, so we
1462    // need to divide the weights evenly between the generated cases, ensuring
1463    // that the total weight is preserved. E.g., a weight of 5 over three cases
1464    // will be distributed as weights of 2, 2, and 1.
1465    uint64_t Weight = Total / NCases, Rem = Total % NCases;
1466    for (unsigned I = 0; I != NCases; ++I) {
1467      if (SwitchWeights)
1468        SwitchWeights->push_back(Weight + (Rem ? 1 : 0));
1469      else if (SwitchLikelihood)
1470        SwitchLikelihood->push_back(LH);
1471
1472      if (Rem)
1473        Rem--;
1474      SwitchInsn->addCase(Builder.getInt(LHS), CaseDest);
1475      ++LHS;
1476    }
1477    return;
1478  }
1479
1480  // The range is too big. Emit "if" condition into a new block,
1481  // making sure to save and restore the current insertion point.
1482  llvm::BasicBlock *RestoreBB = Builder.GetInsertBlock();
1483
1484  // Push this test onto the chain of range checks (which terminates
1485  // in the default basic block). The switch's default will be changed
1486  // to the top of this chain after switch emission is complete.
1487  llvm::BasicBlock *FalseDest = CaseRangeBlock;
1488  CaseRangeBlock = createBasicBlock("sw.caserange");
1489
1490  CurFn->insert(CurFn->end(), CaseRangeBlock);
1491  Builder.SetInsertPoint(CaseRangeBlock);
1492
1493  // Emit range check.
1494  llvm::Value *Diff =
1495    Builder.CreateSub(SwitchInsn->getCondition(), Builder.getInt(LHS));
1496  llvm::Value *Cond =
1497    Builder.CreateICmpULE(Diff, Builder.getInt(Range), "inbounds");
1498
1499  llvm::MDNode *Weights = nullptr;
1500  if (SwitchWeights) {
1501    uint64_t ThisCount = getProfileCount(&S);
1502    uint64_t DefaultCount = (*SwitchWeights)[0];
1503    Weights = createProfileWeights(ThisCount, DefaultCount);
1504
1505    // Since we're chaining the switch default through each large case range, we
1506    // need to update the weight for the default, ie, the first case, to include
1507    // this case.
1508    (*SwitchWeights)[0] += ThisCount;
1509  } else if (SwitchLikelihood)
1510    Cond = emitCondLikelihoodViaExpectIntrinsic(Cond, LH);
1511
1512  Builder.CreateCondBr(Cond, CaseDest, FalseDest, Weights);
1513
1514  // Restore the appropriate insertion point.
1515  if (RestoreBB)
1516    Builder.SetInsertPoint(RestoreBB);
1517  else
1518    Builder.ClearInsertionPoint();
1519}
1520
1521void CodeGenFunction::EmitCaseStmt(const CaseStmt &S,
1522                                   ArrayRef<const Attr *> Attrs) {
1523  // If there is no enclosing switch instance that we're aware of, then this
1524  // case statement and its block can be elided.  This situation only happens
1525  // when we've constant-folded the switch, are emitting the constant case,
1526  // and part of the constant case includes another case statement.  For
1527  // instance: switch (4) { case 4: do { case 5: } while (1); }
1528  if (!SwitchInsn) {
1529    EmitStmt(S.getSubStmt());
1530    return;
1531  }
1532
1533  // Handle case ranges.
1534  if (S.getRHS()) {
1535    EmitCaseStmtRange(S, Attrs);
1536    return;
1537  }
1538
1539  llvm::ConstantInt *CaseVal =
1540    Builder.getInt(S.getLHS()->EvaluateKnownConstInt(getContext()));
1541
1542  // Emit debuginfo for the case value if it is an enum value.
1543  const ConstantExpr *CE;
1544  if (auto ICE = dyn_cast<ImplicitCastExpr>(S.getLHS()))
1545    CE = dyn_cast<ConstantExpr>(ICE->getSubExpr());
1546  else
1547    CE = dyn_cast<ConstantExpr>(S.getLHS());
1548  if (CE) {
1549    if (auto DE = dyn_cast<DeclRefExpr>(CE->getSubExpr()))
1550      if (CGDebugInfo *Dbg = getDebugInfo())
1551        if (CGM.getCodeGenOpts().hasReducedDebugInfo())
1552          Dbg->EmitGlobalVariable(DE->getDecl(),
1553              APValue(llvm::APSInt(CaseVal->getValue())));
1554  }
1555
1556  if (SwitchLikelihood)
1557    SwitchLikelihood->push_back(Stmt::getLikelihood(Attrs));
1558
1559  // If the body of the case is just a 'break', try to not emit an empty block.
1560  // If we're profiling or we're not optimizing, leave the block in for better
1561  // debug and coverage analysis.
1562  if (!CGM.getCodeGenOpts().hasProfileClangInstr() &&
1563      CGM.getCodeGenOpts().OptimizationLevel > 0 &&
1564      isa<BreakStmt>(S.getSubStmt())) {
1565    JumpDest Block = BreakContinueStack.back().BreakBlock;
1566
1567    // Only do this optimization if there are no cleanups that need emitting.
1568    if (isObviouslyBranchWithoutCleanups(Block)) {
1569      if (SwitchWeights)
1570        SwitchWeights->push_back(getProfileCount(&S));
1571      SwitchInsn->addCase(CaseVal, Block.getBlock());
1572
1573      // If there was a fallthrough into this case, make sure to redirect it to
1574      // the end of the switch as well.
1575      if (Builder.GetInsertBlock()) {
1576        Builder.CreateBr(Block.getBlock());
1577        Builder.ClearInsertionPoint();
1578      }
1579      return;
1580    }
1581  }
1582
1583  llvm::BasicBlock *CaseDest = createBasicBlock("sw.bb");
1584  EmitBlockWithFallThrough(CaseDest, &S);
1585  if (SwitchWeights)
1586    SwitchWeights->push_back(getProfileCount(&S));
1587  SwitchInsn->addCase(CaseVal, CaseDest);
1588
1589  // Recursively emitting the statement is acceptable, but is not wonderful for
1590  // code where we have many case statements nested together, i.e.:
1591  //  case 1:
1592  //    case 2:
1593  //      case 3: etc.
1594  // Handling this recursively will create a new block for each case statement
1595  // that falls through to the next case which is IR intensive.  It also causes
1596  // deep recursion which can run into stack depth limitations.  Handle
1597  // sequential non-range case statements specially.
1598  //
1599  // TODO When the next case has a likelihood attribute the code returns to the
1600  // recursive algorithm. Maybe improve this case if it becomes common practice
1601  // to use a lot of attributes.
1602  const CaseStmt *CurCase = &S;
1603  const CaseStmt *NextCase = dyn_cast<CaseStmt>(S.getSubStmt());
1604
1605  // Otherwise, iteratively add consecutive cases to this switch stmt.
1606  while (NextCase && NextCase->getRHS() == nullptr) {
1607    CurCase = NextCase;
1608    llvm::ConstantInt *CaseVal =
1609      Builder.getInt(CurCase->getLHS()->EvaluateKnownConstInt(getContext()));
1610
1611    if (SwitchWeights)
1612      SwitchWeights->push_back(getProfileCount(NextCase));
1613    if (CGM.getCodeGenOpts().hasProfileClangInstr()) {
1614      CaseDest = createBasicBlock("sw.bb");
1615      EmitBlockWithFallThrough(CaseDest, CurCase);
1616    }
1617    // Since this loop is only executed when the CaseStmt has no attributes
1618    // use a hard-coded value.
1619    if (SwitchLikelihood)
1620      SwitchLikelihood->push_back(Stmt::LH_None);
1621
1622    SwitchInsn->addCase(CaseVal, CaseDest);
1623    NextCase = dyn_cast<CaseStmt>(CurCase->getSubStmt());
1624  }
1625
1626  // Generate a stop point for debug info if the case statement is
1627  // followed by a default statement. A fallthrough case before a
1628  // default case gets its own branch target.
1629  if (CurCase->getSubStmt()->getStmtClass() == Stmt::DefaultStmtClass)
1630    EmitStopPoint(CurCase);
1631
1632  // Normal default recursion for non-cases.
1633  EmitStmt(CurCase->getSubStmt());
1634}
1635
1636void CodeGenFunction::EmitDefaultStmt(const DefaultStmt &S,
1637                                      ArrayRef<const Attr *> Attrs) {
1638  // If there is no enclosing switch instance that we're aware of, then this
1639  // default statement can be elided. This situation only happens when we've
1640  // constant-folded the switch.
1641  if (!SwitchInsn) {
1642    EmitStmt(S.getSubStmt());
1643    return;
1644  }
1645
1646  llvm::BasicBlock *DefaultBlock = SwitchInsn->getDefaultDest();
1647  assert(DefaultBlock->empty() &&
1648         "EmitDefaultStmt: Default block already defined?");
1649
1650  if (SwitchLikelihood)
1651    SwitchLikelihood->front() = Stmt::getLikelihood(Attrs);
1652
1653  EmitBlockWithFallThrough(DefaultBlock, &S);
1654
1655  EmitStmt(S.getSubStmt());
1656}
1657
1658/// CollectStatementsForCase - Given the body of a 'switch' statement and a
1659/// constant value that is being switched on, see if we can dead code eliminate
1660/// the body of the switch to a simple series of statements to emit.  Basically,
1661/// on a switch (5) we want to find these statements:
1662///    case 5:
1663///      printf(...);    <--
1664///      ++i;            <--
1665///      break;
1666///
1667/// and add them to the ResultStmts vector.  If it is unsafe to do this
1668/// transformation (for example, one of the elided statements contains a label
1669/// that might be jumped to), return CSFC_Failure.  If we handled it and 'S'
1670/// should include statements after it (e.g. the printf() line is a substmt of
1671/// the case) then return CSFC_FallThrough.  If we handled it and found a break
1672/// statement, then return CSFC_Success.
1673///
1674/// If Case is non-null, then we are looking for the specified case, checking
1675/// that nothing we jump over contains labels.  If Case is null, then we found
1676/// the case and are looking for the break.
1677///
1678/// If the recursive walk actually finds our Case, then we set FoundCase to
1679/// true.
1680///
1681enum CSFC_Result { CSFC_Failure, CSFC_FallThrough, CSFC_Success };
1682static CSFC_Result CollectStatementsForCase(const Stmt *S,
1683                                            const SwitchCase *Case,
1684                                            bool &FoundCase,
1685                              SmallVectorImpl<const Stmt*> &ResultStmts) {
1686  // If this is a null statement, just succeed.
1687  if (!S)
1688    return Case ? CSFC_Success : CSFC_FallThrough;
1689
1690  // If this is the switchcase (case 4: or default) that we're looking for, then
1691  // we're in business.  Just add the substatement.
1692  if (const SwitchCase *SC = dyn_cast<SwitchCase>(S)) {
1693    if (S == Case) {
1694      FoundCase = true;
1695      return CollectStatementsForCase(SC->getSubStmt(), nullptr, FoundCase,
1696                                      ResultStmts);
1697    }
1698
1699    // Otherwise, this is some other case or default statement, just ignore it.
1700    return CollectStatementsForCase(SC->getSubStmt(), Case, FoundCase,
1701                                    ResultStmts);
1702  }
1703
1704  // If we are in the live part of the code and we found our break statement,
1705  // return a success!
1706  if (!Case && isa<BreakStmt>(S))
1707    return CSFC_Success;
1708
1709  // If this is a switch statement, then it might contain the SwitchCase, the
1710  // break, or neither.
1711  if (const CompoundStmt *CS = dyn_cast<CompoundStmt>(S)) {
1712    // Handle this as two cases: we might be looking for the SwitchCase (if so
1713    // the skipped statements must be skippable) or we might already have it.
1714    CompoundStmt::const_body_iterator I = CS->body_begin(), E = CS->body_end();
1715    bool StartedInLiveCode = FoundCase;
1716    unsigned StartSize = ResultStmts.size();
1717
1718    // If we've not found the case yet, scan through looking for it.
1719    if (Case) {
1720      // Keep track of whether we see a skipped declaration.  The code could be
1721      // using the declaration even if it is skipped, so we can't optimize out
1722      // the decl if the kept statements might refer to it.
1723      bool HadSkippedDecl = false;
1724
1725      // If we're looking for the case, just see if we can skip each of the
1726      // substatements.
1727      for (; Case && I != E; ++I) {
1728        HadSkippedDecl |= CodeGenFunction::mightAddDeclToScope(*I);
1729
1730        switch (CollectStatementsForCase(*I, Case, FoundCase, ResultStmts)) {
1731        case CSFC_Failure: return CSFC_Failure;
1732        case CSFC_Success:
1733          // A successful result means that either 1) that the statement doesn't
1734          // have the case and is skippable, or 2) does contain the case value
1735          // and also contains the break to exit the switch.  In the later case,
1736          // we just verify the rest of the statements are elidable.
1737          if (FoundCase) {
1738            // If we found the case and skipped declarations, we can't do the
1739            // optimization.
1740            if (HadSkippedDecl)
1741              return CSFC_Failure;
1742
1743            for (++I; I != E; ++I)
1744              if (CodeGenFunction::ContainsLabel(*I, true))
1745                return CSFC_Failure;
1746            return CSFC_Success;
1747          }
1748          break;
1749        case CSFC_FallThrough:
1750          // If we have a fallthrough condition, then we must have found the
1751          // case started to include statements.  Consider the rest of the
1752          // statements in the compound statement as candidates for inclusion.
1753          assert(FoundCase && "Didn't find case but returned fallthrough?");
1754          // We recursively found Case, so we're not looking for it anymore.
1755          Case = nullptr;
1756
1757          // If we found the case and skipped declarations, we can't do the
1758          // optimization.
1759          if (HadSkippedDecl)
1760            return CSFC_Failure;
1761          break;
1762        }
1763      }
1764
1765      if (!FoundCase)
1766        return CSFC_Success;
1767
1768      assert(!HadSkippedDecl && "fallthrough after skipping decl");
1769    }
1770
1771    // If we have statements in our range, then we know that the statements are
1772    // live and need to be added to the set of statements we're tracking.
1773    bool AnyDecls = false;
1774    for (; I != E; ++I) {
1775      AnyDecls |= CodeGenFunction::mightAddDeclToScope(*I);
1776
1777      switch (CollectStatementsForCase(*I, nullptr, FoundCase, ResultStmts)) {
1778      case CSFC_Failure: return CSFC_Failure;
1779      case CSFC_FallThrough:
1780        // A fallthrough result means that the statement was simple and just
1781        // included in ResultStmt, keep adding them afterwards.
1782        break;
1783      case CSFC_Success:
1784        // A successful result means that we found the break statement and
1785        // stopped statement inclusion.  We just ensure that any leftover stmts
1786        // are skippable and return success ourselves.
1787        for (++I; I != E; ++I)
1788          if (CodeGenFunction::ContainsLabel(*I, true))
1789            return CSFC_Failure;
1790        return CSFC_Success;
1791      }
1792    }
1793
1794    // If we're about to fall out of a scope without hitting a 'break;', we
1795    // can't perform the optimization if there were any decls in that scope
1796    // (we'd lose their end-of-lifetime).
1797    if (AnyDecls) {
1798      // If the entire compound statement was live, there's one more thing we
1799      // can try before giving up: emit the whole thing as a single statement.
1800      // We can do that unless the statement contains a 'break;'.
1801      // FIXME: Such a break must be at the end of a construct within this one.
1802      // We could emit this by just ignoring the BreakStmts entirely.
1803      if (StartedInLiveCode && !CodeGenFunction::containsBreak(S)) {
1804        ResultStmts.resize(StartSize);
1805        ResultStmts.push_back(S);
1806      } else {
1807        return CSFC_Failure;
1808      }
1809    }
1810
1811    return CSFC_FallThrough;
1812  }
1813
1814  // Okay, this is some other statement that we don't handle explicitly, like a
1815  // for statement or increment etc.  If we are skipping over this statement,
1816  // just verify it doesn't have labels, which would make it invalid to elide.
1817  if (Case) {
1818    if (CodeGenFunction::ContainsLabel(S, true))
1819      return CSFC_Failure;
1820    return CSFC_Success;
1821  }
1822
1823  // Otherwise, we want to include this statement.  Everything is cool with that
1824  // so long as it doesn't contain a break out of the switch we're in.
1825  if (CodeGenFunction::containsBreak(S)) return CSFC_Failure;
1826
1827  // Otherwise, everything is great.  Include the statement and tell the caller
1828  // that we fall through and include the next statement as well.
1829  ResultStmts.push_back(S);
1830  return CSFC_FallThrough;
1831}
1832
1833/// FindCaseStatementsForValue - Find the case statement being jumped to and
1834/// then invoke CollectStatementsForCase to find the list of statements to emit
1835/// for a switch on constant.  See the comment above CollectStatementsForCase
1836/// for more details.
1837static bool FindCaseStatementsForValue(const SwitchStmt &S,
1838                                       const llvm::APSInt &ConstantCondValue,
1839                                SmallVectorImpl<const Stmt*> &ResultStmts,
1840                                       ASTContext &C,
1841                                       const SwitchCase *&ResultCase) {
1842  // First step, find the switch case that is being branched to.  We can do this
1843  // efficiently by scanning the SwitchCase list.
1844  const SwitchCase *Case = S.getSwitchCaseList();
1845  const DefaultStmt *DefaultCase = nullptr;
1846
1847  for (; Case; Case = Case->getNextSwitchCase()) {
1848    // It's either a default or case.  Just remember the default statement in
1849    // case we're not jumping to any numbered cases.
1850    if (const DefaultStmt *DS = dyn_cast<DefaultStmt>(Case)) {
1851      DefaultCase = DS;
1852      continue;
1853    }
1854
1855    // Check to see if this case is the one we're looking for.
1856    const CaseStmt *CS = cast<CaseStmt>(Case);
1857    // Don't handle case ranges yet.
1858    if (CS->getRHS()) return false;
1859
1860    // If we found our case, remember it as 'case'.
1861    if (CS->getLHS()->EvaluateKnownConstInt(C) == ConstantCondValue)
1862      break;
1863  }
1864
1865  // If we didn't find a matching case, we use a default if it exists, or we
1866  // elide the whole switch body!
1867  if (!Case) {
1868    // It is safe to elide the body of the switch if it doesn't contain labels
1869    // etc.  If it is safe, return successfully with an empty ResultStmts list.
1870    if (!DefaultCase)
1871      return !CodeGenFunction::ContainsLabel(&S);
1872    Case = DefaultCase;
1873  }
1874
1875  // Ok, we know which case is being jumped to, try to collect all the
1876  // statements that follow it.  This can fail for a variety of reasons.  Also,
1877  // check to see that the recursive walk actually found our case statement.
1878  // Insane cases like this can fail to find it in the recursive walk since we
1879  // don't handle every stmt kind:
1880  // switch (4) {
1881  //   while (1) {
1882  //     case 4: ...
1883  bool FoundCase = false;
1884  ResultCase = Case;
1885  return CollectStatementsForCase(S.getBody(), Case, FoundCase,
1886                                  ResultStmts) != CSFC_Failure &&
1887         FoundCase;
1888}
1889
1890static std::optional<SmallVector<uint64_t, 16>>
1891getLikelihoodWeights(ArrayRef<Stmt::Likelihood> Likelihoods) {
1892  // Are there enough branches to weight them?
1893  if (Likelihoods.size() <= 1)
1894    return std::nullopt;
1895
1896  uint64_t NumUnlikely = 0;
1897  uint64_t NumNone = 0;
1898  uint64_t NumLikely = 0;
1899  for (const auto LH : Likelihoods) {
1900    switch (LH) {
1901    case Stmt::LH_Unlikely:
1902      ++NumUnlikely;
1903      break;
1904    case Stmt::LH_None:
1905      ++NumNone;
1906      break;
1907    case Stmt::LH_Likely:
1908      ++NumLikely;
1909      break;
1910    }
1911  }
1912
1913  // Is there a likelihood attribute used?
1914  if (NumUnlikely == 0 && NumLikely == 0)
1915    return std::nullopt;
1916
1917  // When multiple cases share the same code they can be combined during
1918  // optimization. In that case the weights of the branch will be the sum of
1919  // the individual weights. Make sure the combined sum of all neutral cases
1920  // doesn't exceed the value of a single likely attribute.
1921  // The additions both avoid divisions by 0 and make sure the weights of None
1922  // don't exceed the weight of Likely.
1923  const uint64_t Likely = INT32_MAX / (NumLikely + 2);
1924  const uint64_t None = Likely / (NumNone + 1);
1925  const uint64_t Unlikely = 0;
1926
1927  SmallVector<uint64_t, 16> Result;
1928  Result.reserve(Likelihoods.size());
1929  for (const auto LH : Likelihoods) {
1930    switch (LH) {
1931    case Stmt::LH_Unlikely:
1932      Result.push_back(Unlikely);
1933      break;
1934    case Stmt::LH_None:
1935      Result.push_back(None);
1936      break;
1937    case Stmt::LH_Likely:
1938      Result.push_back(Likely);
1939      break;
1940    }
1941  }
1942
1943  return Result;
1944}
1945
1946void CodeGenFunction::EmitSwitchStmt(const SwitchStmt &S) {
1947  // Handle nested switch statements.
1948  llvm::SwitchInst *SavedSwitchInsn = SwitchInsn;
1949  SmallVector<uint64_t, 16> *SavedSwitchWeights = SwitchWeights;
1950  SmallVector<Stmt::Likelihood, 16> *SavedSwitchLikelihood = SwitchLikelihood;
1951  llvm::BasicBlock *SavedCRBlock = CaseRangeBlock;
1952
1953  // See if we can constant fold the condition of the switch and therefore only
1954  // emit the live case statement (if any) of the switch.
1955  llvm::APSInt ConstantCondValue;
1956  if (ConstantFoldsToSimpleInteger(S.getCond(), ConstantCondValue)) {
1957    SmallVector<const Stmt*, 4> CaseStmts;
1958    const SwitchCase *Case = nullptr;
1959    if (FindCaseStatementsForValue(S, ConstantCondValue, CaseStmts,
1960                                   getContext(), Case)) {
1961      if (Case)
1962        incrementProfileCounter(Case);
1963      RunCleanupsScope ExecutedScope(*this);
1964
1965      if (S.getInit())
1966        EmitStmt(S.getInit());
1967
1968      // Emit the condition variable if needed inside the entire cleanup scope
1969      // used by this special case for constant folded switches.
1970      if (S.getConditionVariable())
1971        EmitDecl(*S.getConditionVariable());
1972
1973      // At this point, we are no longer "within" a switch instance, so
1974      // we can temporarily enforce this to ensure that any embedded case
1975      // statements are not emitted.
1976      SwitchInsn = nullptr;
1977
1978      // Okay, we can dead code eliminate everything except this case.  Emit the
1979      // specified series of statements and we're good.
1980      for (unsigned i = 0, e = CaseStmts.size(); i != e; ++i)
1981        EmitStmt(CaseStmts[i]);
1982      incrementProfileCounter(&S);
1983
1984      // Now we want to restore the saved switch instance so that nested
1985      // switches continue to function properly
1986      SwitchInsn = SavedSwitchInsn;
1987
1988      return;
1989    }
1990  }
1991
1992  JumpDest SwitchExit = getJumpDestInCurrentScope("sw.epilog");
1993
1994  RunCleanupsScope ConditionScope(*this);
1995
1996  if (S.getInit())
1997    EmitStmt(S.getInit());
1998
1999  if (S.getConditionVariable())
2000    EmitDecl(*S.getConditionVariable());
2001  llvm::Value *CondV = EmitScalarExpr(S.getCond());
2002
2003  // Create basic block to hold stuff that comes after switch
2004  // statement. We also need to create a default block now so that
2005  // explicit case ranges tests can have a place to jump to on
2006  // failure.
2007  llvm::BasicBlock *DefaultBlock = createBasicBlock("sw.default");
2008  SwitchInsn = Builder.CreateSwitch(CondV, DefaultBlock);
2009  if (PGO.haveRegionCounts()) {
2010    // Walk the SwitchCase list to find how many there are.
2011    uint64_t DefaultCount = 0;
2012    unsigned NumCases = 0;
2013    for (const SwitchCase *Case = S.getSwitchCaseList();
2014         Case;
2015         Case = Case->getNextSwitchCase()) {
2016      if (isa<DefaultStmt>(Case))
2017        DefaultCount = getProfileCount(Case);
2018      NumCases += 1;
2019    }
2020    SwitchWeights = new SmallVector<uint64_t, 16>();
2021    SwitchWeights->reserve(NumCases);
2022    // The default needs to be first. We store the edge count, so we already
2023    // know the right weight.
2024    SwitchWeights->push_back(DefaultCount);
2025  } else if (CGM.getCodeGenOpts().OptimizationLevel) {
2026    SwitchLikelihood = new SmallVector<Stmt::Likelihood, 16>();
2027    // Initialize the default case.
2028    SwitchLikelihood->push_back(Stmt::LH_None);
2029  }
2030
2031  CaseRangeBlock = DefaultBlock;
2032
2033  // Clear the insertion point to indicate we are in unreachable code.
2034  Builder.ClearInsertionPoint();
2035
2036  // All break statements jump to NextBlock. If BreakContinueStack is non-empty
2037  // then reuse last ContinueBlock.
2038  JumpDest OuterContinue;
2039  if (!BreakContinueStack.empty())
2040    OuterContinue = BreakContinueStack.back().ContinueBlock;
2041
2042  BreakContinueStack.push_back(BreakContinue(SwitchExit, OuterContinue));
2043
2044  // Emit switch body.
2045  EmitStmt(S.getBody());
2046
2047  BreakContinueStack.pop_back();
2048
2049  // Update the default block in case explicit case range tests have
2050  // been chained on top.
2051  SwitchInsn->setDefaultDest(CaseRangeBlock);
2052
2053  // If a default was never emitted:
2054  if (!DefaultBlock->getParent()) {
2055    // If we have cleanups, emit the default block so that there's a
2056    // place to jump through the cleanups from.
2057    if (ConditionScope.requiresCleanups()) {
2058      EmitBlock(DefaultBlock);
2059
2060    // Otherwise, just forward the default block to the switch end.
2061    } else {
2062      DefaultBlock->replaceAllUsesWith(SwitchExit.getBlock());
2063      delete DefaultBlock;
2064    }
2065  }
2066
2067  ConditionScope.ForceCleanup();
2068
2069  // Emit continuation.
2070  EmitBlock(SwitchExit.getBlock(), true);
2071  incrementProfileCounter(&S);
2072
2073  // If the switch has a condition wrapped by __builtin_unpredictable,
2074  // create metadata that specifies that the switch is unpredictable.
2075  // Don't bother if not optimizing because that metadata would not be used.
2076  auto *Call = dyn_cast<CallExpr>(S.getCond());
2077  if (Call && CGM.getCodeGenOpts().OptimizationLevel != 0) {
2078    auto *FD = dyn_cast_or_null<FunctionDecl>(Call->getCalleeDecl());
2079    if (FD && FD->getBuiltinID() == Builtin::BI__builtin_unpredictable) {
2080      llvm::MDBuilder MDHelper(getLLVMContext());
2081      SwitchInsn->setMetadata(llvm::LLVMContext::MD_unpredictable,
2082                              MDHelper.createUnpredictable());
2083    }
2084  }
2085
2086  if (SwitchWeights) {
2087    assert(SwitchWeights->size() == 1 + SwitchInsn->getNumCases() &&
2088           "switch weights do not match switch cases");
2089    // If there's only one jump destination there's no sense weighting it.
2090    if (SwitchWeights->size() > 1)
2091      SwitchInsn->setMetadata(llvm::LLVMContext::MD_prof,
2092                              createProfileWeights(*SwitchWeights));
2093    delete SwitchWeights;
2094  } else if (SwitchLikelihood) {
2095    assert(SwitchLikelihood->size() == 1 + SwitchInsn->getNumCases() &&
2096           "switch likelihoods do not match switch cases");
2097    std::optional<SmallVector<uint64_t, 16>> LHW =
2098        getLikelihoodWeights(*SwitchLikelihood);
2099    if (LHW) {
2100      llvm::MDBuilder MDHelper(CGM.getLLVMContext());
2101      SwitchInsn->setMetadata(llvm::LLVMContext::MD_prof,
2102                              createProfileWeights(*LHW));
2103    }
2104    delete SwitchLikelihood;
2105  }
2106  SwitchInsn = SavedSwitchInsn;
2107  SwitchWeights = SavedSwitchWeights;
2108  SwitchLikelihood = SavedSwitchLikelihood;
2109  CaseRangeBlock = SavedCRBlock;
2110}
2111
2112static std::string
2113SimplifyConstraint(const char *Constraint, const TargetInfo &Target,
2114                 SmallVectorImpl<TargetInfo::ConstraintInfo> *OutCons=nullptr) {
2115  std::string Result;
2116
2117  while (*Constraint) {
2118    switch (*Constraint) {
2119    default:
2120      Result += Target.convertConstraint(Constraint);
2121      break;
2122    // Ignore these
2123    case '*':
2124    case '?':
2125    case '!':
2126    case '=': // Will see this and the following in mult-alt constraints.
2127    case '+':
2128      break;
2129    case '#': // Ignore the rest of the constraint alternative.
2130      while (Constraint[1] && Constraint[1] != ',')
2131        Constraint++;
2132      break;
2133    case '&':
2134    case '%':
2135      Result += *Constraint;
2136      while (Constraint[1] && Constraint[1] == *Constraint)
2137        Constraint++;
2138      break;
2139    case ',':
2140      Result += "|";
2141      break;
2142    case 'g':
2143      Result += "imr";
2144      break;
2145    case '[': {
2146      assert(OutCons &&
2147             "Must pass output names to constraints with a symbolic name");
2148      unsigned Index;
2149      bool result = Target.resolveSymbolicName(Constraint, *OutCons, Index);
2150      assert(result && "Could not resolve symbolic name"); (void)result;
2151      Result += llvm::utostr(Index);
2152      break;
2153    }
2154    }
2155
2156    Constraint++;
2157  }
2158
2159  return Result;
2160}
2161
2162/// AddVariableConstraints - Look at AsmExpr and if it is a variable declared
2163/// as using a particular register add that as a constraint that will be used
2164/// in this asm stmt.
2165static std::string
2166AddVariableConstraints(const std::string &Constraint, const Expr &AsmExpr,
2167                       const TargetInfo &Target, CodeGenModule &CGM,
2168                       const AsmStmt &Stmt, const bool EarlyClobber,
2169                       std::string *GCCReg = nullptr) {
2170  const DeclRefExpr *AsmDeclRef = dyn_cast<DeclRefExpr>(&AsmExpr);
2171  if (!AsmDeclRef)
2172    return Constraint;
2173  const ValueDecl &Value = *AsmDeclRef->getDecl();
2174  const VarDecl *Variable = dyn_cast<VarDecl>(&Value);
2175  if (!Variable)
2176    return Constraint;
2177  if (Variable->getStorageClass() != SC_Register)
2178    return Constraint;
2179  AsmLabelAttr *Attr = Variable->getAttr<AsmLabelAttr>();
2180  if (!Attr)
2181    return Constraint;
2182  StringRef Register = Attr->getLabel();
2183  assert(Target.isValidGCCRegisterName(Register));
2184  // We're using validateOutputConstraint here because we only care if
2185  // this is a register constraint.
2186  TargetInfo::ConstraintInfo Info(Constraint, "");
2187  if (Target.validateOutputConstraint(Info) &&
2188      !Info.allowsRegister()) {
2189    CGM.ErrorUnsupported(&Stmt, "__asm__");
2190    return Constraint;
2191  }
2192  // Canonicalize the register here before returning it.
2193  Register = Target.getNormalizedGCCRegisterName(Register);
2194  if (GCCReg != nullptr)
2195    *GCCReg = Register.str();
2196  return (EarlyClobber ? "&{" : "{") + Register.str() + "}";
2197}
2198
2199std::pair<llvm::Value*, llvm::Type *> CodeGenFunction::EmitAsmInputLValue(
2200    const TargetInfo::ConstraintInfo &Info, LValue InputValue,
2201    QualType InputType, std::string &ConstraintStr, SourceLocation Loc) {
2202  if (Info.allowsRegister() || !Info.allowsMemory()) {
2203    if (CodeGenFunction::hasScalarEvaluationKind(InputType))
2204      return {EmitLoadOfLValue(InputValue, Loc).getScalarVal(), nullptr};
2205
2206    llvm::Type *Ty = ConvertType(InputType);
2207    uint64_t Size = CGM.getDataLayout().getTypeSizeInBits(Ty);
2208    if ((Size <= 64 && llvm::isPowerOf2_64(Size)) ||
2209        getTargetHooks().isScalarizableAsmOperand(*this, Ty)) {
2210      Ty = llvm::IntegerType::get(getLLVMContext(), Size);
2211
2212      return {
2213          Builder.CreateLoad(InputValue.getAddress(*this).withElementType(Ty)),
2214          nullptr};
2215    }
2216  }
2217
2218  Address Addr = InputValue.getAddress(*this);
2219  ConstraintStr += '*';
2220  return {Addr.getPointer(), Addr.getElementType()};
2221}
2222
2223std::pair<llvm::Value *, llvm::Type *>
2224CodeGenFunction::EmitAsmInput(const TargetInfo::ConstraintInfo &Info,
2225                              const Expr *InputExpr,
2226                              std::string &ConstraintStr) {
2227  // If this can't be a register or memory, i.e., has to be a constant
2228  // (immediate or symbolic), try to emit it as such.
2229  if (!Info.allowsRegister() && !Info.allowsMemory()) {
2230    if (Info.requiresImmediateConstant()) {
2231      Expr::EvalResult EVResult;
2232      InputExpr->EvaluateAsRValue(EVResult, getContext(), true);
2233
2234      llvm::APSInt IntResult;
2235      if (EVResult.Val.toIntegralConstant(IntResult, InputExpr->getType(),
2236                                          getContext()))
2237        return {llvm::ConstantInt::get(getLLVMContext(), IntResult), nullptr};
2238    }
2239
2240    Expr::EvalResult Result;
2241    if (InputExpr->EvaluateAsInt(Result, getContext()))
2242      return {llvm::ConstantInt::get(getLLVMContext(), Result.Val.getInt()),
2243              nullptr};
2244  }
2245
2246  if (Info.allowsRegister() || !Info.allowsMemory())
2247    if (CodeGenFunction::hasScalarEvaluationKind(InputExpr->getType()))
2248      return {EmitScalarExpr(InputExpr), nullptr};
2249  if (InputExpr->getStmtClass() == Expr::CXXThisExprClass)
2250    return {EmitScalarExpr(InputExpr), nullptr};
2251  InputExpr = InputExpr->IgnoreParenNoopCasts(getContext());
2252  LValue Dest = EmitLValue(InputExpr);
2253  return EmitAsmInputLValue(Info, Dest, InputExpr->getType(), ConstraintStr,
2254                            InputExpr->getExprLoc());
2255}
2256
2257/// getAsmSrcLocInfo - Return the !srcloc metadata node to attach to an inline
2258/// asm call instruction.  The !srcloc MDNode contains a list of constant
2259/// integers which are the source locations of the start of each line in the
2260/// asm.
2261static llvm::MDNode *getAsmSrcLocInfo(const StringLiteral *Str,
2262                                      CodeGenFunction &CGF) {
2263  SmallVector<llvm::Metadata *, 8> Locs;
2264  // Add the location of the first line to the MDNode.
2265  Locs.push_back(llvm::ConstantAsMetadata::get(llvm::ConstantInt::get(
2266      CGF.Int64Ty, Str->getBeginLoc().getRawEncoding())));
2267  StringRef StrVal = Str->getString();
2268  if (!StrVal.empty()) {
2269    const SourceManager &SM = CGF.CGM.getContext().getSourceManager();
2270    const LangOptions &LangOpts = CGF.CGM.getLangOpts();
2271    unsigned StartToken = 0;
2272    unsigned ByteOffset = 0;
2273
2274    // Add the location of the start of each subsequent line of the asm to the
2275    // MDNode.
2276    for (unsigned i = 0, e = StrVal.size() - 1; i != e; ++i) {
2277      if (StrVal[i] != '\n') continue;
2278      SourceLocation LineLoc = Str->getLocationOfByte(
2279          i + 1, SM, LangOpts, CGF.getTarget(), &StartToken, &ByteOffset);
2280      Locs.push_back(llvm::ConstantAsMetadata::get(
2281          llvm::ConstantInt::get(CGF.Int64Ty, LineLoc.getRawEncoding())));
2282    }
2283  }
2284
2285  return llvm::MDNode::get(CGF.getLLVMContext(), Locs);
2286}
2287
2288static void UpdateAsmCallInst(llvm::CallBase &Result, bool HasSideEffect,
2289                              bool HasUnwindClobber, bool ReadOnly,
2290                              bool ReadNone, bool NoMerge, const AsmStmt &S,
2291                              const std::vector<llvm::Type *> &ResultRegTypes,
2292                              const std::vector<llvm::Type *> &ArgElemTypes,
2293                              CodeGenFunction &CGF,
2294                              std::vector<llvm::Value *> &RegResults) {
2295  if (!HasUnwindClobber)
2296    Result.addFnAttr(llvm::Attribute::NoUnwind);
2297
2298  if (NoMerge)
2299    Result.addFnAttr(llvm::Attribute::NoMerge);
2300  // Attach readnone and readonly attributes.
2301  if (!HasSideEffect) {
2302    if (ReadNone)
2303      Result.setDoesNotAccessMemory();
2304    else if (ReadOnly)
2305      Result.setOnlyReadsMemory();
2306  }
2307
2308  // Add elementtype attribute for indirect constraints.
2309  for (auto Pair : llvm::enumerate(ArgElemTypes)) {
2310    if (Pair.value()) {
2311      auto Attr = llvm::Attribute::get(
2312          CGF.getLLVMContext(), llvm::Attribute::ElementType, Pair.value());
2313      Result.addParamAttr(Pair.index(), Attr);
2314    }
2315  }
2316
2317  // Slap the source location of the inline asm into a !srcloc metadata on the
2318  // call.
2319  if (const auto *gccAsmStmt = dyn_cast<GCCAsmStmt>(&S))
2320    Result.setMetadata("srcloc",
2321                       getAsmSrcLocInfo(gccAsmStmt->getAsmString(), CGF));
2322  else {
2323    // At least put the line number on MS inline asm blobs.
2324    llvm::Constant *Loc =
2325        llvm::ConstantInt::get(CGF.Int64Ty, S.getAsmLoc().getRawEncoding());
2326    Result.setMetadata("srcloc",
2327                       llvm::MDNode::get(CGF.getLLVMContext(),
2328                                         llvm::ConstantAsMetadata::get(Loc)));
2329  }
2330
2331  if (CGF.getLangOpts().assumeFunctionsAreConvergent())
2332    // Conservatively, mark all inline asm blocks in CUDA or OpenCL as
2333    // convergent (meaning, they may call an intrinsically convergent op, such
2334    // as bar.sync, and so can't have certain optimizations applied around
2335    // them).
2336    Result.addFnAttr(llvm::Attribute::Convergent);
2337  // Extract all of the register value results from the asm.
2338  if (ResultRegTypes.size() == 1) {
2339    RegResults.push_back(&Result);
2340  } else {
2341    for (unsigned i = 0, e = ResultRegTypes.size(); i != e; ++i) {
2342      llvm::Value *Tmp = CGF.Builder.CreateExtractValue(&Result, i, "asmresult");
2343      RegResults.push_back(Tmp);
2344    }
2345  }
2346}
2347
2348static void
2349EmitAsmStores(CodeGenFunction &CGF, const AsmStmt &S,
2350              const llvm::ArrayRef<llvm::Value *> RegResults,
2351              const llvm::ArrayRef<llvm::Type *> ResultRegTypes,
2352              const llvm::ArrayRef<llvm::Type *> ResultTruncRegTypes,
2353              const llvm::ArrayRef<LValue> ResultRegDests,
2354              const llvm::ArrayRef<QualType> ResultRegQualTys,
2355              const llvm::BitVector &ResultTypeRequiresCast,
2356              const llvm::BitVector &ResultRegIsFlagReg) {
2357  CGBuilderTy &Builder = CGF.Builder;
2358  CodeGenModule &CGM = CGF.CGM;
2359  llvm::LLVMContext &CTX = CGF.getLLVMContext();
2360
2361  assert(RegResults.size() == ResultRegTypes.size());
2362  assert(RegResults.size() == ResultTruncRegTypes.size());
2363  assert(RegResults.size() == ResultRegDests.size());
2364  // ResultRegDests can be also populated by addReturnRegisterOutputs() above,
2365  // in which case its size may grow.
2366  assert(ResultTypeRequiresCast.size() <= ResultRegDests.size());
2367  assert(ResultRegIsFlagReg.size() <= ResultRegDests.size());
2368
2369  for (unsigned i = 0, e = RegResults.size(); i != e; ++i) {
2370    llvm::Value *Tmp = RegResults[i];
2371    llvm::Type *TruncTy = ResultTruncRegTypes[i];
2372
2373    if ((i < ResultRegIsFlagReg.size()) && ResultRegIsFlagReg[i]) {
2374      // Target must guarantee the Value `Tmp` here is lowered to a boolean
2375      // value.
2376      llvm::Constant *Two = llvm::ConstantInt::get(Tmp->getType(), 2);
2377      llvm::Value *IsBooleanValue =
2378          Builder.CreateCmp(llvm::CmpInst::ICMP_ULT, Tmp, Two);
2379      llvm::Function *FnAssume = CGM.getIntrinsic(llvm::Intrinsic::assume);
2380      Builder.CreateCall(FnAssume, IsBooleanValue);
2381    }
2382
2383    // If the result type of the LLVM IR asm doesn't match the result type of
2384    // the expression, do the conversion.
2385    if (ResultRegTypes[i] != TruncTy) {
2386
2387      // Truncate the integer result to the right size, note that TruncTy can be
2388      // a pointer.
2389      if (TruncTy->isFloatingPointTy())
2390        Tmp = Builder.CreateFPTrunc(Tmp, TruncTy);
2391      else if (TruncTy->isPointerTy() && Tmp->getType()->isIntegerTy()) {
2392        uint64_t ResSize = CGM.getDataLayout().getTypeSizeInBits(TruncTy);
2393        Tmp = Builder.CreateTrunc(
2394            Tmp, llvm::IntegerType::get(CTX, (unsigned)ResSize));
2395        Tmp = Builder.CreateIntToPtr(Tmp, TruncTy);
2396      } else if (Tmp->getType()->isPointerTy() && TruncTy->isIntegerTy()) {
2397        uint64_t TmpSize =
2398            CGM.getDataLayout().getTypeSizeInBits(Tmp->getType());
2399        Tmp = Builder.CreatePtrToInt(
2400            Tmp, llvm::IntegerType::get(CTX, (unsigned)TmpSize));
2401        Tmp = Builder.CreateTrunc(Tmp, TruncTy);
2402      } else if (Tmp->getType()->isIntegerTy() && TruncTy->isIntegerTy()) {
2403        Tmp = Builder.CreateZExtOrTrunc(Tmp, TruncTy);
2404      } else if (Tmp->getType()->isVectorTy() || TruncTy->isVectorTy()) {
2405        Tmp = Builder.CreateBitCast(Tmp, TruncTy);
2406      }
2407    }
2408
2409    LValue Dest = ResultRegDests[i];
2410    // ResultTypeRequiresCast elements correspond to the first
2411    // ResultTypeRequiresCast.size() elements of RegResults.
2412    if ((i < ResultTypeRequiresCast.size()) && ResultTypeRequiresCast[i]) {
2413      unsigned Size = CGF.getContext().getTypeSize(ResultRegQualTys[i]);
2414      Address A = Dest.getAddress(CGF).withElementType(ResultRegTypes[i]);
2415      if (CGF.getTargetHooks().isScalarizableAsmOperand(CGF, TruncTy)) {
2416        Builder.CreateStore(Tmp, A);
2417        continue;
2418      }
2419
2420      QualType Ty =
2421          CGF.getContext().getIntTypeForBitwidth(Size, /*Signed=*/false);
2422      if (Ty.isNull()) {
2423        const Expr *OutExpr = S.getOutputExpr(i);
2424        CGM.getDiags().Report(OutExpr->getExprLoc(),
2425                              diag::err_store_value_to_reg);
2426        return;
2427      }
2428      Dest = CGF.MakeAddrLValue(A, Ty);
2429    }
2430    CGF.EmitStoreThroughLValue(RValue::get(Tmp), Dest);
2431  }
2432}
2433
2434static void EmitHipStdParUnsupportedAsm(CodeGenFunction *CGF,
2435                                        const AsmStmt &S) {
2436  constexpr auto Name = "__ASM__hipstdpar_unsupported";
2437
2438  StringRef Asm;
2439  if (auto GCCAsm = dyn_cast<GCCAsmStmt>(&S))
2440    Asm = GCCAsm->getAsmString()->getString();
2441
2442  auto &Ctx = CGF->CGM.getLLVMContext();
2443
2444  auto StrTy = llvm::ConstantDataArray::getString(Ctx, Asm);
2445  auto FnTy = llvm::FunctionType::get(llvm::Type::getVoidTy(Ctx),
2446                                      {StrTy->getType()}, false);
2447  auto UBF = CGF->CGM.getModule().getOrInsertFunction(Name, FnTy);
2448
2449  CGF->Builder.CreateCall(UBF, {StrTy});
2450}
2451
2452void CodeGenFunction::EmitAsmStmt(const AsmStmt &S) {
2453  // Pop all cleanup blocks at the end of the asm statement.
2454  CodeGenFunction::RunCleanupsScope Cleanups(*this);
2455
2456  // Assemble the final asm string.
2457  std::string AsmString = S.generateAsmString(getContext());
2458
2459  // Get all the output and input constraints together.
2460  SmallVector<TargetInfo::ConstraintInfo, 4> OutputConstraintInfos;
2461  SmallVector<TargetInfo::ConstraintInfo, 4> InputConstraintInfos;
2462
2463  bool IsHipStdPar = getLangOpts().HIPStdPar && getLangOpts().CUDAIsDevice;
2464  bool IsValidTargetAsm = true;
2465  for (unsigned i = 0, e = S.getNumOutputs(); i != e && IsValidTargetAsm; i++) {
2466    StringRef Name;
2467    if (const GCCAsmStmt *GAS = dyn_cast<GCCAsmStmt>(&S))
2468      Name = GAS->getOutputName(i);
2469    TargetInfo::ConstraintInfo Info(S.getOutputConstraint(i), Name);
2470    bool IsValid = getTarget().validateOutputConstraint(Info); (void)IsValid;
2471    if (IsHipStdPar && !IsValid)
2472      IsValidTargetAsm = false;
2473    else
2474      assert(IsValid && "Failed to parse output constraint");
2475    OutputConstraintInfos.push_back(Info);
2476  }
2477
2478  for (unsigned i = 0, e = S.getNumInputs(); i != e && IsValidTargetAsm; i++) {
2479    StringRef Name;
2480    if (const GCCAsmStmt *GAS = dyn_cast<GCCAsmStmt>(&S))
2481      Name = GAS->getInputName(i);
2482    TargetInfo::ConstraintInfo Info(S.getInputConstraint(i), Name);
2483    bool IsValid =
2484      getTarget().validateInputConstraint(OutputConstraintInfos, Info);
2485    if (IsHipStdPar && !IsValid)
2486      IsValidTargetAsm = false;
2487    else
2488      assert(IsValid && "Failed to parse input constraint");
2489    InputConstraintInfos.push_back(Info);
2490  }
2491
2492  if (!IsValidTargetAsm)
2493    return EmitHipStdParUnsupportedAsm(this, S);
2494
2495  std::string Constraints;
2496
2497  std::vector<LValue> ResultRegDests;
2498  std::vector<QualType> ResultRegQualTys;
2499  std::vector<llvm::Type *> ResultRegTypes;
2500  std::vector<llvm::Type *> ResultTruncRegTypes;
2501  std::vector<llvm::Type *> ArgTypes;
2502  std::vector<llvm::Type *> ArgElemTypes;
2503  std::vector<llvm::Value*> Args;
2504  llvm::BitVector ResultTypeRequiresCast;
2505  llvm::BitVector ResultRegIsFlagReg;
2506
2507  // Keep track of inout constraints.
2508  std::string InOutConstraints;
2509  std::vector<llvm::Value*> InOutArgs;
2510  std::vector<llvm::Type*> InOutArgTypes;
2511  std::vector<llvm::Type*> InOutArgElemTypes;
2512
2513  // Keep track of out constraints for tied input operand.
2514  std::vector<std::string> OutputConstraints;
2515
2516  // Keep track of defined physregs.
2517  llvm::SmallSet<std::string, 8> PhysRegOutputs;
2518
2519  // An inline asm can be marked readonly if it meets the following conditions:
2520  //  - it doesn't have any sideeffects
2521  //  - it doesn't clobber memory
2522  //  - it doesn't return a value by-reference
2523  // It can be marked readnone if it doesn't have any input memory constraints
2524  // in addition to meeting the conditions listed above.
2525  bool ReadOnly = true, ReadNone = true;
2526
2527  for (unsigned i = 0, e = S.getNumOutputs(); i != e; i++) {
2528    TargetInfo::ConstraintInfo &Info = OutputConstraintInfos[i];
2529
2530    // Simplify the output constraint.
2531    std::string OutputConstraint(S.getOutputConstraint(i));
2532    OutputConstraint = SimplifyConstraint(OutputConstraint.c_str() + 1,
2533                                          getTarget(), &OutputConstraintInfos);
2534
2535    const Expr *OutExpr = S.getOutputExpr(i);
2536    OutExpr = OutExpr->IgnoreParenNoopCasts(getContext());
2537
2538    std::string GCCReg;
2539    OutputConstraint = AddVariableConstraints(OutputConstraint, *OutExpr,
2540                                              getTarget(), CGM, S,
2541                                              Info.earlyClobber(),
2542                                              &GCCReg);
2543    // Give an error on multiple outputs to same physreg.
2544    if (!GCCReg.empty() && !PhysRegOutputs.insert(GCCReg).second)
2545      CGM.Error(S.getAsmLoc(), "multiple outputs to hard register: " + GCCReg);
2546
2547    OutputConstraints.push_back(OutputConstraint);
2548    LValue Dest = EmitLValue(OutExpr);
2549    if (!Constraints.empty())
2550      Constraints += ',';
2551
2552    // If this is a register output, then make the inline asm return it
2553    // by-value.  If this is a memory result, return the value by-reference.
2554    QualType QTy = OutExpr->getType();
2555    const bool IsScalarOrAggregate = hasScalarEvaluationKind(QTy) ||
2556                                     hasAggregateEvaluationKind(QTy);
2557    if (!Info.allowsMemory() && IsScalarOrAggregate) {
2558
2559      Constraints += "=" + OutputConstraint;
2560      ResultRegQualTys.push_back(QTy);
2561      ResultRegDests.push_back(Dest);
2562
2563      bool IsFlagReg = llvm::StringRef(OutputConstraint).starts_with("{@cc");
2564      ResultRegIsFlagReg.push_back(IsFlagReg);
2565
2566      llvm::Type *Ty = ConvertTypeForMem(QTy);
2567      const bool RequiresCast = Info.allowsRegister() &&
2568          (getTargetHooks().isScalarizableAsmOperand(*this, Ty) ||
2569           Ty->isAggregateType());
2570
2571      ResultTruncRegTypes.push_back(Ty);
2572      ResultTypeRequiresCast.push_back(RequiresCast);
2573
2574      if (RequiresCast) {
2575        unsigned Size = getContext().getTypeSize(QTy);
2576        Ty = llvm::IntegerType::get(getLLVMContext(), Size);
2577      }
2578      ResultRegTypes.push_back(Ty);
2579      // If this output is tied to an input, and if the input is larger, then
2580      // we need to set the actual result type of the inline asm node to be the
2581      // same as the input type.
2582      if (Info.hasMatchingInput()) {
2583        unsigned InputNo;
2584        for (InputNo = 0; InputNo != S.getNumInputs(); ++InputNo) {
2585          TargetInfo::ConstraintInfo &Input = InputConstraintInfos[InputNo];
2586          if (Input.hasTiedOperand() && Input.getTiedOperand() == i)
2587            break;
2588        }
2589        assert(InputNo != S.getNumInputs() && "Didn't find matching input!");
2590
2591        QualType InputTy = S.getInputExpr(InputNo)->getType();
2592        QualType OutputType = OutExpr->getType();
2593
2594        uint64_t InputSize = getContext().getTypeSize(InputTy);
2595        if (getContext().getTypeSize(OutputType) < InputSize) {
2596          // Form the asm to return the value as a larger integer or fp type.
2597          ResultRegTypes.back() = ConvertType(InputTy);
2598        }
2599      }
2600      if (llvm::Type* AdjTy =
2601            getTargetHooks().adjustInlineAsmType(*this, OutputConstraint,
2602                                                 ResultRegTypes.back()))
2603        ResultRegTypes.back() = AdjTy;
2604      else {
2605        CGM.getDiags().Report(S.getAsmLoc(),
2606                              diag::err_asm_invalid_type_in_input)
2607            << OutExpr->getType() << OutputConstraint;
2608      }
2609
2610      // Update largest vector width for any vector types.
2611      if (auto *VT = dyn_cast<llvm::VectorType>(ResultRegTypes.back()))
2612        LargestVectorWidth =
2613            std::max((uint64_t)LargestVectorWidth,
2614                     VT->getPrimitiveSizeInBits().getKnownMinValue());
2615    } else {
2616      Address DestAddr = Dest.getAddress(*this);
2617      // Matrix types in memory are represented by arrays, but accessed through
2618      // vector pointers, with the alignment specified on the access operation.
2619      // For inline assembly, update pointer arguments to use vector pointers.
2620      // Otherwise there will be a mis-match if the matrix is also an
2621      // input-argument which is represented as vector.
2622      if (isa<MatrixType>(OutExpr->getType().getCanonicalType()))
2623        DestAddr = DestAddr.withElementType(ConvertType(OutExpr->getType()));
2624
2625      ArgTypes.push_back(DestAddr.getType());
2626      ArgElemTypes.push_back(DestAddr.getElementType());
2627      Args.push_back(DestAddr.getPointer());
2628      Constraints += "=*";
2629      Constraints += OutputConstraint;
2630      ReadOnly = ReadNone = false;
2631    }
2632
2633    if (Info.isReadWrite()) {
2634      InOutConstraints += ',';
2635
2636      const Expr *InputExpr = S.getOutputExpr(i);
2637      llvm::Value *Arg;
2638      llvm::Type *ArgElemType;
2639      std::tie(Arg, ArgElemType) = EmitAsmInputLValue(
2640          Info, Dest, InputExpr->getType(), InOutConstraints,
2641          InputExpr->getExprLoc());
2642
2643      if (llvm::Type* AdjTy =
2644          getTargetHooks().adjustInlineAsmType(*this, OutputConstraint,
2645                                               Arg->getType()))
2646        Arg = Builder.CreateBitCast(Arg, AdjTy);
2647
2648      // Update largest vector width for any vector types.
2649      if (auto *VT = dyn_cast<llvm::VectorType>(Arg->getType()))
2650        LargestVectorWidth =
2651            std::max((uint64_t)LargestVectorWidth,
2652                     VT->getPrimitiveSizeInBits().getKnownMinValue());
2653      // Only tie earlyclobber physregs.
2654      if (Info.allowsRegister() && (GCCReg.empty() || Info.earlyClobber()))
2655        InOutConstraints += llvm::utostr(i);
2656      else
2657        InOutConstraints += OutputConstraint;
2658
2659      InOutArgTypes.push_back(Arg->getType());
2660      InOutArgElemTypes.push_back(ArgElemType);
2661      InOutArgs.push_back(Arg);
2662    }
2663  }
2664
2665  // If this is a Microsoft-style asm blob, store the return registers (EAX:EDX)
2666  // to the return value slot. Only do this when returning in registers.
2667  if (isa<MSAsmStmt>(&S)) {
2668    const ABIArgInfo &RetAI = CurFnInfo->getReturnInfo();
2669    if (RetAI.isDirect() || RetAI.isExtend()) {
2670      // Make a fake lvalue for the return value slot.
2671      LValue ReturnSlot = MakeAddrLValueWithoutTBAA(ReturnValue, FnRetTy);
2672      CGM.getTargetCodeGenInfo().addReturnRegisterOutputs(
2673          *this, ReturnSlot, Constraints, ResultRegTypes, ResultTruncRegTypes,
2674          ResultRegDests, AsmString, S.getNumOutputs());
2675      SawAsmBlock = true;
2676    }
2677  }
2678
2679  for (unsigned i = 0, e = S.getNumInputs(); i != e; i++) {
2680    const Expr *InputExpr = S.getInputExpr(i);
2681
2682    TargetInfo::ConstraintInfo &Info = InputConstraintInfos[i];
2683
2684    if (Info.allowsMemory())
2685      ReadNone = false;
2686
2687    if (!Constraints.empty())
2688      Constraints += ',';
2689
2690    // Simplify the input constraint.
2691    std::string InputConstraint(S.getInputConstraint(i));
2692    InputConstraint = SimplifyConstraint(InputConstraint.c_str(), getTarget(),
2693                                         &OutputConstraintInfos);
2694
2695    InputConstraint = AddVariableConstraints(
2696        InputConstraint, *InputExpr->IgnoreParenNoopCasts(getContext()),
2697        getTarget(), CGM, S, false /* No EarlyClobber */);
2698
2699    std::string ReplaceConstraint (InputConstraint);
2700    llvm::Value *Arg;
2701    llvm::Type *ArgElemType;
2702    std::tie(Arg, ArgElemType) = EmitAsmInput(Info, InputExpr, Constraints);
2703
2704    // If this input argument is tied to a larger output result, extend the
2705    // input to be the same size as the output.  The LLVM backend wants to see
2706    // the input and output of a matching constraint be the same size.  Note
2707    // that GCC does not define what the top bits are here.  We use zext because
2708    // that is usually cheaper, but LLVM IR should really get an anyext someday.
2709    if (Info.hasTiedOperand()) {
2710      unsigned Output = Info.getTiedOperand();
2711      QualType OutputType = S.getOutputExpr(Output)->getType();
2712      QualType InputTy = InputExpr->getType();
2713
2714      if (getContext().getTypeSize(OutputType) >
2715          getContext().getTypeSize(InputTy)) {
2716        // Use ptrtoint as appropriate so that we can do our extension.
2717        if (isa<llvm::PointerType>(Arg->getType()))
2718          Arg = Builder.CreatePtrToInt(Arg, IntPtrTy);
2719        llvm::Type *OutputTy = ConvertType(OutputType);
2720        if (isa<llvm::IntegerType>(OutputTy))
2721          Arg = Builder.CreateZExt(Arg, OutputTy);
2722        else if (isa<llvm::PointerType>(OutputTy))
2723          Arg = Builder.CreateZExt(Arg, IntPtrTy);
2724        else if (OutputTy->isFloatingPointTy())
2725          Arg = Builder.CreateFPExt(Arg, OutputTy);
2726      }
2727      // Deal with the tied operands' constraint code in adjustInlineAsmType.
2728      ReplaceConstraint = OutputConstraints[Output];
2729    }
2730    if (llvm::Type* AdjTy =
2731          getTargetHooks().adjustInlineAsmType(*this, ReplaceConstraint,
2732                                                   Arg->getType()))
2733      Arg = Builder.CreateBitCast(Arg, AdjTy);
2734    else
2735      CGM.getDiags().Report(S.getAsmLoc(), diag::err_asm_invalid_type_in_input)
2736          << InputExpr->getType() << InputConstraint;
2737
2738    // Update largest vector width for any vector types.
2739    if (auto *VT = dyn_cast<llvm::VectorType>(Arg->getType()))
2740      LargestVectorWidth =
2741          std::max((uint64_t)LargestVectorWidth,
2742                   VT->getPrimitiveSizeInBits().getKnownMinValue());
2743
2744    ArgTypes.push_back(Arg->getType());
2745    ArgElemTypes.push_back(ArgElemType);
2746    Args.push_back(Arg);
2747    Constraints += InputConstraint;
2748  }
2749
2750  // Append the "input" part of inout constraints.
2751  for (unsigned i = 0, e = InOutArgs.size(); i != e; i++) {
2752    ArgTypes.push_back(InOutArgTypes[i]);
2753    ArgElemTypes.push_back(InOutArgElemTypes[i]);
2754    Args.push_back(InOutArgs[i]);
2755  }
2756  Constraints += InOutConstraints;
2757
2758  // Labels
2759  SmallVector<llvm::BasicBlock *, 16> Transfer;
2760  llvm::BasicBlock *Fallthrough = nullptr;
2761  bool IsGCCAsmGoto = false;
2762  if (const auto *GS = dyn_cast<GCCAsmStmt>(&S)) {
2763    IsGCCAsmGoto = GS->isAsmGoto();
2764    if (IsGCCAsmGoto) {
2765      for (const auto *E : GS->labels()) {
2766        JumpDest Dest = getJumpDestForLabel(E->getLabel());
2767        Transfer.push_back(Dest.getBlock());
2768        if (!Constraints.empty())
2769          Constraints += ',';
2770        Constraints += "!i";
2771      }
2772      Fallthrough = createBasicBlock("asm.fallthrough");
2773    }
2774  }
2775
2776  bool HasUnwindClobber = false;
2777
2778  // Clobbers
2779  for (unsigned i = 0, e = S.getNumClobbers(); i != e; i++) {
2780    StringRef Clobber = S.getClobber(i);
2781
2782    if (Clobber == "memory")
2783      ReadOnly = ReadNone = false;
2784    else if (Clobber == "unwind") {
2785      HasUnwindClobber = true;
2786      continue;
2787    } else if (Clobber != "cc") {
2788      Clobber = getTarget().getNormalizedGCCRegisterName(Clobber);
2789      if (CGM.getCodeGenOpts().StackClashProtector &&
2790          getTarget().isSPRegName(Clobber)) {
2791        CGM.getDiags().Report(S.getAsmLoc(),
2792                              diag::warn_stack_clash_protection_inline_asm);
2793      }
2794    }
2795
2796    if (isa<MSAsmStmt>(&S)) {
2797      if (Clobber == "eax" || Clobber == "edx") {
2798        if (Constraints.find("=&A") != std::string::npos)
2799          continue;
2800        std::string::size_type position1 =
2801            Constraints.find("={" + Clobber.str() + "}");
2802        if (position1 != std::string::npos) {
2803          Constraints.insert(position1 + 1, "&");
2804          continue;
2805        }
2806        std::string::size_type position2 = Constraints.find("=A");
2807        if (position2 != std::string::npos) {
2808          Constraints.insert(position2 + 1, "&");
2809          continue;
2810        }
2811      }
2812    }
2813    if (!Constraints.empty())
2814      Constraints += ',';
2815
2816    Constraints += "~{";
2817    Constraints += Clobber;
2818    Constraints += '}';
2819  }
2820
2821  assert(!(HasUnwindClobber && IsGCCAsmGoto) &&
2822         "unwind clobber can't be used with asm goto");
2823
2824  // Add machine specific clobbers
2825  std::string_view MachineClobbers = getTarget().getClobbers();
2826  if (!MachineClobbers.empty()) {
2827    if (!Constraints.empty())
2828      Constraints += ',';
2829    Constraints += MachineClobbers;
2830  }
2831
2832  llvm::Type *ResultType;
2833  if (ResultRegTypes.empty())
2834    ResultType = VoidTy;
2835  else if (ResultRegTypes.size() == 1)
2836    ResultType = ResultRegTypes[0];
2837  else
2838    ResultType = llvm::StructType::get(getLLVMContext(), ResultRegTypes);
2839
2840  llvm::FunctionType *FTy =
2841    llvm::FunctionType::get(ResultType, ArgTypes, false);
2842
2843  bool HasSideEffect = S.isVolatile() || S.getNumOutputs() == 0;
2844
2845  llvm::InlineAsm::AsmDialect GnuAsmDialect =
2846      CGM.getCodeGenOpts().getInlineAsmDialect() == CodeGenOptions::IAD_ATT
2847          ? llvm::InlineAsm::AD_ATT
2848          : llvm::InlineAsm::AD_Intel;
2849  llvm::InlineAsm::AsmDialect AsmDialect = isa<MSAsmStmt>(&S) ?
2850    llvm::InlineAsm::AD_Intel : GnuAsmDialect;
2851
2852  llvm::InlineAsm *IA = llvm::InlineAsm::get(
2853      FTy, AsmString, Constraints, HasSideEffect,
2854      /* IsAlignStack */ false, AsmDialect, HasUnwindClobber);
2855  std::vector<llvm::Value*> RegResults;
2856  llvm::CallBrInst *CBR;
2857  llvm::DenseMap<llvm::BasicBlock *, SmallVector<llvm::Value *, 4>>
2858      CBRRegResults;
2859  if (IsGCCAsmGoto) {
2860    CBR = Builder.CreateCallBr(IA, Fallthrough, Transfer, Args);
2861    EmitBlock(Fallthrough);
2862    UpdateAsmCallInst(*CBR, HasSideEffect, false, ReadOnly, ReadNone,
2863                      InNoMergeAttributedStmt, S, ResultRegTypes, ArgElemTypes,
2864                      *this, RegResults);
2865    // Because we are emitting code top to bottom, we don't have enough
2866    // information at this point to know precisely whether we have a critical
2867    // edge. If we have outputs, split all indirect destinations.
2868    if (!RegResults.empty()) {
2869      unsigned i = 0;
2870      for (llvm::BasicBlock *Dest : CBR->getIndirectDests()) {
2871        llvm::Twine SynthName = Dest->getName() + ".split";
2872        llvm::BasicBlock *SynthBB = createBasicBlock(SynthName);
2873        llvm::IRBuilderBase::InsertPointGuard IPG(Builder);
2874        Builder.SetInsertPoint(SynthBB);
2875
2876        if (ResultRegTypes.size() == 1) {
2877          CBRRegResults[SynthBB].push_back(CBR);
2878        } else {
2879          for (unsigned j = 0, e = ResultRegTypes.size(); j != e; ++j) {
2880            llvm::Value *Tmp = Builder.CreateExtractValue(CBR, j, "asmresult");
2881            CBRRegResults[SynthBB].push_back(Tmp);
2882          }
2883        }
2884
2885        EmitBranch(Dest);
2886        EmitBlock(SynthBB);
2887        CBR->setIndirectDest(i++, SynthBB);
2888      }
2889    }
2890  } else if (HasUnwindClobber) {
2891    llvm::CallBase *Result = EmitCallOrInvoke(IA, Args, "");
2892    UpdateAsmCallInst(*Result, HasSideEffect, true, ReadOnly, ReadNone,
2893                      InNoMergeAttributedStmt, S, ResultRegTypes, ArgElemTypes,
2894                      *this, RegResults);
2895  } else {
2896    llvm::CallInst *Result =
2897        Builder.CreateCall(IA, Args, getBundlesForFunclet(IA));
2898    UpdateAsmCallInst(*Result, HasSideEffect, false, ReadOnly, ReadNone,
2899                      InNoMergeAttributedStmt, S, ResultRegTypes, ArgElemTypes,
2900                      *this, RegResults);
2901  }
2902
2903  EmitAsmStores(*this, S, RegResults, ResultRegTypes, ResultTruncRegTypes,
2904                ResultRegDests, ResultRegQualTys, ResultTypeRequiresCast,
2905                ResultRegIsFlagReg);
2906
2907  // If this is an asm goto with outputs, repeat EmitAsmStores, but with a
2908  // different insertion point; one for each indirect destination and with
2909  // CBRRegResults rather than RegResults.
2910  if (IsGCCAsmGoto && !CBRRegResults.empty()) {
2911    for (llvm::BasicBlock *Succ : CBR->getIndirectDests()) {
2912      llvm::IRBuilderBase::InsertPointGuard IPG(Builder);
2913      Builder.SetInsertPoint(Succ, --(Succ->end()));
2914      EmitAsmStores(*this, S, CBRRegResults[Succ], ResultRegTypes,
2915                    ResultTruncRegTypes, ResultRegDests, ResultRegQualTys,
2916                    ResultTypeRequiresCast, ResultRegIsFlagReg);
2917    }
2918  }
2919}
2920
2921LValue CodeGenFunction::InitCapturedStruct(const CapturedStmt &S) {
2922  const RecordDecl *RD = S.getCapturedRecordDecl();
2923  QualType RecordTy = getContext().getRecordType(RD);
2924
2925  // Initialize the captured struct.
2926  LValue SlotLV =
2927    MakeAddrLValue(CreateMemTemp(RecordTy, "agg.captured"), RecordTy);
2928
2929  RecordDecl::field_iterator CurField = RD->field_begin();
2930  for (CapturedStmt::const_capture_init_iterator I = S.capture_init_begin(),
2931                                                 E = S.capture_init_end();
2932       I != E; ++I, ++CurField) {
2933    LValue LV = EmitLValueForFieldInitialization(SlotLV, *CurField);
2934    if (CurField->hasCapturedVLAType()) {
2935      EmitLambdaVLACapture(CurField->getCapturedVLAType(), LV);
2936    } else {
2937      EmitInitializerForField(*CurField, LV, *I);
2938    }
2939  }
2940
2941  return SlotLV;
2942}
2943
2944/// Generate an outlined function for the body of a CapturedStmt, store any
2945/// captured variables into the captured struct, and call the outlined function.
2946llvm::Function *
2947CodeGenFunction::EmitCapturedStmt(const CapturedStmt &S, CapturedRegionKind K) {
2948  LValue CapStruct = InitCapturedStruct(S);
2949
2950  // Emit the CapturedDecl
2951  CodeGenFunction CGF(CGM, true);
2952  CGCapturedStmtRAII CapInfoRAII(CGF, new CGCapturedStmtInfo(S, K));
2953  llvm::Function *F = CGF.GenerateCapturedStmtFunction(S);
2954  delete CGF.CapturedStmtInfo;
2955
2956  // Emit call to the helper function.
2957  EmitCallOrInvoke(F, CapStruct.getPointer(*this));
2958
2959  return F;
2960}
2961
2962Address CodeGenFunction::GenerateCapturedStmtArgument(const CapturedStmt &S) {
2963  LValue CapStruct = InitCapturedStruct(S);
2964  return CapStruct.getAddress(*this);
2965}
2966
2967/// Creates the outlined function for a CapturedStmt.
2968llvm::Function *
2969CodeGenFunction::GenerateCapturedStmtFunction(const CapturedStmt &S) {
2970  assert(CapturedStmtInfo &&
2971    "CapturedStmtInfo should be set when generating the captured function");
2972  const CapturedDecl *CD = S.getCapturedDecl();
2973  const RecordDecl *RD = S.getCapturedRecordDecl();
2974  SourceLocation Loc = S.getBeginLoc();
2975  assert(CD->hasBody() && "missing CapturedDecl body");
2976
2977  // Build the argument list.
2978  ASTContext &Ctx = CGM.getContext();
2979  FunctionArgList Args;
2980  Args.append(CD->param_begin(), CD->param_end());
2981
2982  // Create the function declaration.
2983  const CGFunctionInfo &FuncInfo =
2984    CGM.getTypes().arrangeBuiltinFunctionDeclaration(Ctx.VoidTy, Args);
2985  llvm::FunctionType *FuncLLVMTy = CGM.getTypes().GetFunctionType(FuncInfo);
2986
2987  llvm::Function *F =
2988    llvm::Function::Create(FuncLLVMTy, llvm::GlobalValue::InternalLinkage,
2989                           CapturedStmtInfo->getHelperName(), &CGM.getModule());
2990  CGM.SetInternalFunctionAttributes(CD, F, FuncInfo);
2991  if (CD->isNothrow())
2992    F->addFnAttr(llvm::Attribute::NoUnwind);
2993
2994  // Generate the function.
2995  StartFunction(CD, Ctx.VoidTy, F, FuncInfo, Args, CD->getLocation(),
2996                CD->getBody()->getBeginLoc());
2997  // Set the context parameter in CapturedStmtInfo.
2998  Address DeclPtr = GetAddrOfLocalVar(CD->getContextParam());
2999  CapturedStmtInfo->setContextValue(Builder.CreateLoad(DeclPtr));
3000
3001  // Initialize variable-length arrays.
3002  LValue Base = MakeNaturalAlignAddrLValue(CapturedStmtInfo->getContextValue(),
3003                                           Ctx.getTagDeclType(RD));
3004  for (auto *FD : RD->fields()) {
3005    if (FD->hasCapturedVLAType()) {
3006      auto *ExprArg =
3007          EmitLoadOfLValue(EmitLValueForField(Base, FD), S.getBeginLoc())
3008              .getScalarVal();
3009      auto VAT = FD->getCapturedVLAType();
3010      VLASizeMap[VAT->getSizeExpr()] = ExprArg;
3011    }
3012  }
3013
3014  // If 'this' is captured, load it into CXXThisValue.
3015  if (CapturedStmtInfo->isCXXThisExprCaptured()) {
3016    FieldDecl *FD = CapturedStmtInfo->getThisFieldDecl();
3017    LValue ThisLValue = EmitLValueForField(Base, FD);
3018    CXXThisValue = EmitLoadOfLValue(ThisLValue, Loc).getScalarVal();
3019  }
3020
3021  PGO.assignRegionCounters(GlobalDecl(CD), F);
3022  CapturedStmtInfo->EmitBody(*this, CD->getBody());
3023  FinishFunction(CD->getBodyRBrace());
3024
3025  return F;
3026}
3027