CGExprAgg.cpp revision 353358
1193326Sed//===--- CGExprAgg.cpp - Emit LLVM Code from Aggregate Expressions --------===// 2193326Sed// 3353358Sdim// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4353358Sdim// See https://llvm.org/LICENSE.txt for license information. 5353358Sdim// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6193326Sed// 7193326Sed//===----------------------------------------------------------------------===// 8193326Sed// 9193326Sed// This contains code to emit Aggregate Expr nodes as LLVM code. 10193326Sed// 11193326Sed//===----------------------------------------------------------------------===// 12193326Sed 13193326Sed#include "CodeGenFunction.h" 14341825Sdim#include "CGCXXABI.h" 15249423Sdim#include "CGObjCRuntime.h" 16193326Sed#include "CodeGenModule.h" 17338697Sdim#include "ConstantEmitter.h" 18193326Sed#include "clang/AST/ASTContext.h" 19193326Sed#include "clang/AST/DeclCXX.h" 20234353Sdim#include "clang/AST/DeclTemplate.h" 21193326Sed#include "clang/AST/StmtVisitor.h" 22249423Sdim#include "llvm/IR/Constants.h" 23249423Sdim#include "llvm/IR/Function.h" 24249423Sdim#include "llvm/IR/GlobalVariable.h" 25249423Sdim#include "llvm/IR/Intrinsics.h" 26341825Sdim#include "llvm/IR/IntrinsicInst.h" 27193326Sedusing namespace clang; 28193326Sedusing namespace CodeGen; 29193326Sed 30193326Sed//===----------------------------------------------------------------------===// 31193326Sed// Aggregate Expression Emitter 32193326Sed//===----------------------------------------------------------------------===// 33193326Sed 34193326Sednamespace { 35199990Srdivackyclass AggExprEmitter : public StmtVisitor<AggExprEmitter> { 36193326Sed CodeGenFunction &CGF; 37193326Sed CGBuilderTy &Builder; 38218893Sdim AggValueSlot Dest; 39288943Sdim bool IsResultUnused; 40208600Srdivacky 41218893Sdim AggValueSlot EnsureSlot(QualType T) { 42218893Sdim if (!Dest.isIgnored()) return Dest; 43218893Sdim return CGF.CreateAggTemp(T, "agg.tmp.ensured"); 44218893Sdim } 45239462Sdim void EnsureDest(QualType T) { 46239462Sdim if (!Dest.isIgnored()) return; 47239462Sdim Dest = CGF.CreateAggTemp(T, "agg.tmp.ensured"); 48239462Sdim } 49218893Sdim 50341825Sdim // Calls `Fn` with a valid return value slot, potentially creating a temporary 51341825Sdim // to do so. If a temporary is created, an appropriate copy into `Dest` will 52341825Sdim // be emitted, as will lifetime markers. 53341825Sdim // 54341825Sdim // The given function should take a ReturnValueSlot, and return an RValue that 55341825Sdim // points to said slot. 56341825Sdim void withReturnValueSlot(const Expr *E, 57341825Sdim llvm::function_ref<RValue(ReturnValueSlot)> Fn); 58341825Sdim 59193326Sedpublic: 60288943Sdim AggExprEmitter(CodeGenFunction &cgf, AggValueSlot Dest, bool IsResultUnused) 61288943Sdim : CGF(cgf), Builder(CGF.Builder), Dest(Dest), 62288943Sdim IsResultUnused(IsResultUnused) { } 63193326Sed 64193326Sed //===--------------------------------------------------------------------===// 65193326Sed // Utilities 66193326Sed //===--------------------------------------------------------------------===// 67193326Sed 68193326Sed /// EmitAggLoadOfLValue - Given an expression with aggregate type that 69193326Sed /// represents a value lvalue, this method emits the address of the lvalue, 70193326Sed /// then loads the result into DestPtr. 71193326Sed void EmitAggLoadOfLValue(const Expr *E); 72193326Sed 73341825Sdim enum ExprValueKind { 74341825Sdim EVK_RValue, 75341825Sdim EVK_NonRValue 76341825Sdim }; 77341825Sdim 78193326Sed /// EmitFinalDestCopy - Perform the final copy to DestPtr, if desired. 79341825Sdim /// SrcIsRValue is true if source comes from an RValue. 80341825Sdim void EmitFinalDestCopy(QualType type, const LValue &src, 81341825Sdim ExprValueKind SrcValueKind = EVK_NonRValue); 82296417Sdim void EmitFinalDestCopy(QualType type, RValue src); 83239462Sdim void EmitCopy(QualType type, const AggValueSlot &dest, 84239462Sdim const AggValueSlot &src); 85193326Sed 86226633Sdim void EmitMoveFromReturnSlot(const Expr *E, RValue Src); 87208600Srdivacky 88296417Sdim void EmitArrayInit(Address DestPtr, llvm::ArrayType *AType, 89338697Sdim QualType ArrayQTy, InitListExpr *E); 90234353Sdim 91226633Sdim AggValueSlot::NeedsGCBarriers_t needsGC(QualType T) { 92234353Sdim if (CGF.getLangOpts().getGC() && TypeRequiresGCollection(T)) 93226633Sdim return AggValueSlot::NeedsGCBarriers; 94226633Sdim return AggValueSlot::DoesNotNeedGCBarriers; 95226633Sdim } 96226633Sdim 97208600Srdivacky bool TypeRequiresGCollection(QualType T); 98208600Srdivacky 99193326Sed //===--------------------------------------------------------------------===// 100193326Sed // Visitor Methods 101193326Sed //===--------------------------------------------------------------------===// 102198092Srdivacky 103288943Sdim void Visit(Expr *E) { 104288943Sdim ApplyDebugLocation DL(CGF, E); 105288943Sdim StmtVisitor<AggExprEmitter>::Visit(E); 106288943Sdim } 107288943Sdim 108193326Sed void VisitStmt(Stmt *S) { 109193326Sed CGF.ErrorUnsupported(S, "aggregate expression"); 110193326Sed } 111193326Sed void VisitParenExpr(ParenExpr *PE) { Visit(PE->getSubExpr()); } 112221345Sdim void VisitGenericSelectionExpr(GenericSelectionExpr *GE) { 113221345Sdim Visit(GE->getResultExpr()); 114221345Sdim } 115321369Sdim void VisitCoawaitExpr(CoawaitExpr *E) { 116321369Sdim CGF.EmitCoawaitExpr(*E, Dest, IsResultUnused); 117321369Sdim } 118321369Sdim void VisitCoyieldExpr(CoyieldExpr *E) { 119321369Sdim CGF.EmitCoyieldExpr(*E, Dest, IsResultUnused); 120321369Sdim } 121321369Sdim void VisitUnaryCoawait(UnaryOperator *E) { Visit(E->getSubExpr()); } 122193326Sed void VisitUnaryExtension(UnaryOperator *E) { Visit(E->getSubExpr()); } 123224145Sdim void VisitSubstNonTypeTemplateParmExpr(SubstNonTypeTemplateParmExpr *E) { 124224145Sdim return Visit(E->getReplacement()); 125224145Sdim } 126193326Sed 127344779Sdim void VisitConstantExpr(ConstantExpr *E) { 128344779Sdim return Visit(E->getSubExpr()); 129344779Sdim } 130344779Sdim 131193326Sed // l-values. 132327952Sdim void VisitDeclRefExpr(DeclRefExpr *E) { EmitAggLoadOfLValue(E); } 133193326Sed void VisitMemberExpr(MemberExpr *ME) { EmitAggLoadOfLValue(ME); } 134193326Sed void VisitUnaryDeref(UnaryOperator *E) { EmitAggLoadOfLValue(E); } 135193326Sed void VisitStringLiteral(StringLiteral *E) { EmitAggLoadOfLValue(E); } 136224145Sdim void VisitCompoundLiteralExpr(CompoundLiteralExpr *E); 137193326Sed void VisitArraySubscriptExpr(ArraySubscriptExpr *E) { 138193326Sed EmitAggLoadOfLValue(E); 139193326Sed } 140193326Sed void VisitPredefinedExpr(const PredefinedExpr *E) { 141198092Srdivacky EmitAggLoadOfLValue(E); 142193326Sed } 143198092Srdivacky 144193326Sed // Operators. 145198092Srdivacky void VisitCastExpr(CastExpr *E); 146193326Sed void VisitCallExpr(const CallExpr *E); 147193326Sed void VisitStmtExpr(const StmtExpr *E); 148193326Sed void VisitBinaryOperator(const BinaryOperator *BO); 149198398Srdivacky void VisitPointerToDataMemberBinaryOperator(const BinaryOperator *BO); 150193326Sed void VisitBinAssign(const BinaryOperator *E); 151193326Sed void VisitBinComma(const BinaryOperator *E); 152341825Sdim void VisitBinCmp(const BinaryOperator *E); 153193326Sed 154193326Sed void VisitObjCMessageExpr(ObjCMessageExpr *E); 155193326Sed void VisitObjCIvarRefExpr(ObjCIvarRefExpr *E) { 156193326Sed EmitAggLoadOfLValue(E); 157193326Sed } 158198092Srdivacky 159288943Sdim void VisitDesignatedInitUpdateExpr(DesignatedInitUpdateExpr *E); 160218893Sdim void VisitAbstractConditionalOperator(const AbstractConditionalOperator *CO); 161198092Srdivacky void VisitChooseExpr(const ChooseExpr *CE); 162193326Sed void VisitInitListExpr(InitListExpr *E); 163314564Sdim void VisitArrayInitLoopExpr(const ArrayInitLoopExpr *E, 164314564Sdim llvm::Value *outerBegin = nullptr); 165201361Srdivacky void VisitImplicitValueInitExpr(ImplicitValueInitExpr *E); 166288943Sdim void VisitNoInitExpr(NoInitExpr *E) { } // Do nothing. 167193326Sed void VisitCXXDefaultArgExpr(CXXDefaultArgExpr *DAE) { 168353358Sdim CodeGenFunction::CXXDefaultArgExprScope Scope(CGF, DAE); 169193326Sed Visit(DAE->getExpr()); 170193326Sed } 171251662Sdim void VisitCXXDefaultInitExpr(CXXDefaultInitExpr *DIE) { 172353358Sdim CodeGenFunction::CXXDefaultInitExprScope Scope(CGF, DIE); 173251662Sdim Visit(DIE->getExpr()); 174251662Sdim } 175193326Sed void VisitCXXBindTemporaryExpr(CXXBindTemporaryExpr *E); 176193326Sed void VisitCXXConstructExpr(const CXXConstructExpr *E); 177309124Sdim void VisitCXXInheritedCtorInitExpr(const CXXInheritedCtorInitExpr *E); 178234353Sdim void VisitLambdaExpr(LambdaExpr *E); 179261991Sdim void VisitCXXStdInitializerListExpr(CXXStdInitializerListExpr *E); 180218893Sdim void VisitExprWithCleanups(ExprWithCleanups *E); 181210299Sed void VisitCXXScalarValueInitExpr(CXXScalarValueInitExpr *E); 182199482Srdivacky void VisitCXXTypeidExpr(CXXTypeidExpr *E) { EmitAggLoadOfLValue(E); } 183224145Sdim void VisitMaterializeTemporaryExpr(MaterializeTemporaryExpr *E); 184218893Sdim void VisitOpaqueValueExpr(OpaqueValueExpr *E); 185218893Sdim 186234353Sdim void VisitPseudoObjectExpr(PseudoObjectExpr *E) { 187234353Sdim if (E->isGLValue()) { 188234353Sdim LValue LV = CGF.EmitPseudoObjectLValue(E); 189239462Sdim return EmitFinalDestCopy(E->getType(), LV); 190234353Sdim } 191234353Sdim 192234353Sdim CGF.EmitPseudoObjectRValue(E, EnsureSlot(E->getType())); 193234353Sdim } 194234353Sdim 195193326Sed void VisitVAArgExpr(VAArgExpr *E); 196193326Sed 197224145Sdim void EmitInitializationToLValue(Expr *E, LValue Address); 198224145Sdim void EmitNullInitializationToLValue(LValue Address); 199193326Sed // case Expr::ChooseExprClass: 200200583Srdivacky void VisitCXXThrowExpr(const CXXThrowExpr *E) { CGF.EmitCXXThrowExpr(E); } 201226633Sdim void VisitAtomicExpr(AtomicExpr *E) { 202296417Sdim RValue Res = CGF.EmitAtomicExpr(E); 203296417Sdim EmitFinalDestCopy(E->getType(), Res); 204226633Sdim } 205193326Sed}; 206193326Sed} // end anonymous namespace. 207193326Sed 208193326Sed//===----------------------------------------------------------------------===// 209193326Sed// Utilities 210193326Sed//===----------------------------------------------------------------------===// 211193326Sed 212193326Sed/// EmitAggLoadOfLValue - Given an expression with aggregate type that 213193326Sed/// represents a value lvalue, this method emits the address of the lvalue, 214193326Sed/// then loads the result into DestPtr. 215193326Sedvoid AggExprEmitter::EmitAggLoadOfLValue(const Expr *E) { 216193326Sed LValue LV = CGF.EmitLValue(E); 217249423Sdim 218249423Sdim // If the type of the l-value is atomic, then do an atomic load. 219288943Sdim if (LV.getType()->isAtomicType() || CGF.LValueIsSuitableForInlineAtomic(LV)) { 220261991Sdim CGF.EmitAtomicLoad(LV, E->getExprLoc(), Dest); 221249423Sdim return; 222249423Sdim } 223249423Sdim 224239462Sdim EmitFinalDestCopy(E->getType(), LV); 225193326Sed} 226193326Sed 227341825Sdim/// True if the given aggregate type requires special GC API calls. 228208600Srdivackybool AggExprEmitter::TypeRequiresGCollection(QualType T) { 229208600Srdivacky // Only record types have members that might require garbage collection. 230208600Srdivacky const RecordType *RecordTy = T->getAs<RecordType>(); 231208600Srdivacky if (!RecordTy) return false; 232208600Srdivacky 233208600Srdivacky // Don't mess with non-trivial C++ types. 234208600Srdivacky RecordDecl *Record = RecordTy->getDecl(); 235208600Srdivacky if (isa<CXXRecordDecl>(Record) && 236249423Sdim (cast<CXXRecordDecl>(Record)->hasNonTrivialCopyConstructor() || 237208600Srdivacky !cast<CXXRecordDecl>(Record)->hasTrivialDestructor())) 238208600Srdivacky return false; 239208600Srdivacky 240208600Srdivacky // Check whether the type has an object member. 241208600Srdivacky return Record->hasObjectMember(); 242208600Srdivacky} 243208600Srdivacky 244341825Sdimvoid AggExprEmitter::withReturnValueSlot( 245341825Sdim const Expr *E, llvm::function_ref<RValue(ReturnValueSlot)> EmitCall) { 246341825Sdim QualType RetTy = E->getType(); 247341825Sdim bool RequiresDestruction = 248341825Sdim Dest.isIgnored() && 249341825Sdim RetTy.isDestructedType() == QualType::DK_nontrivial_c_struct; 250341825Sdim 251341825Sdim // If it makes no observable difference, save a memcpy + temporary. 252341825Sdim // 253341825Sdim // We need to always provide our own temporary if destruction is required. 254341825Sdim // Otherwise, EmitCall will emit its own, notice that it's "unused", and end 255341825Sdim // its lifetime before we have the chance to emit a proper destructor call. 256341825Sdim bool UseTemp = Dest.isPotentiallyAliased() || Dest.requiresGCollection() || 257341825Sdim (RequiresDestruction && !Dest.getAddress().isValid()); 258341825Sdim 259341825Sdim Address RetAddr = Address::invalid(); 260341825Sdim Address RetAllocaAddr = Address::invalid(); 261341825Sdim 262341825Sdim EHScopeStack::stable_iterator LifetimeEndBlock; 263341825Sdim llvm::Value *LifetimeSizePtr = nullptr; 264341825Sdim llvm::IntrinsicInst *LifetimeStartInst = nullptr; 265341825Sdim if (!UseTemp) { 266341825Sdim RetAddr = Dest.getAddress(); 267341825Sdim } else { 268341825Sdim RetAddr = CGF.CreateMemTemp(RetTy, "tmp", &RetAllocaAddr); 269341825Sdim uint64_t Size = 270341825Sdim CGF.CGM.getDataLayout().getTypeAllocSize(CGF.ConvertTypeForMem(RetTy)); 271341825Sdim LifetimeSizePtr = CGF.EmitLifetimeStart(Size, RetAllocaAddr.getPointer()); 272341825Sdim if (LifetimeSizePtr) { 273341825Sdim LifetimeStartInst = 274341825Sdim cast<llvm::IntrinsicInst>(std::prev(Builder.GetInsertPoint())); 275341825Sdim assert(LifetimeStartInst->getIntrinsicID() == 276341825Sdim llvm::Intrinsic::lifetime_start && 277341825Sdim "Last insertion wasn't a lifetime.start?"); 278341825Sdim 279341825Sdim CGF.pushFullExprCleanup<CodeGenFunction::CallLifetimeEnd>( 280341825Sdim NormalEHLifetimeMarker, RetAllocaAddr, LifetimeSizePtr); 281341825Sdim LifetimeEndBlock = CGF.EHStack.stable_begin(); 282341825Sdim } 283341825Sdim } 284341825Sdim 285341825Sdim RValue Src = 286341825Sdim EmitCall(ReturnValueSlot(RetAddr, Dest.isVolatile(), IsResultUnused)); 287341825Sdim 288341825Sdim if (RequiresDestruction) 289341825Sdim CGF.pushDestroy(RetTy.isDestructedType(), Src.getAggregateAddress(), RetTy); 290341825Sdim 291341825Sdim if (!UseTemp) 292226633Sdim return; 293341825Sdim 294341825Sdim assert(Dest.getPointer() != Src.getAggregatePointer()); 295341825Sdim EmitFinalDestCopy(E->getType(), Src); 296341825Sdim 297341825Sdim if (!RequiresDestruction && LifetimeStartInst) { 298341825Sdim // If there's no dtor to run, the copy was the last use of our temporary. 299341825Sdim // Since we're not guaranteed to be in an ExprWithCleanups, clean up 300341825Sdim // eagerly. 301341825Sdim CGF.DeactivateCleanupBlock(LifetimeEndBlock, LifetimeStartInst); 302341825Sdim CGF.EmitLifetimeEnd(LifetimeSizePtr, RetAllocaAddr.getPointer()); 303210299Sed } 304208600Srdivacky} 305208600Srdivacky 306193326Sed/// EmitFinalDestCopy - Perform the final copy to DestPtr, if desired. 307296417Sdimvoid AggExprEmitter::EmitFinalDestCopy(QualType type, RValue src) { 308239462Sdim assert(src.isAggregate() && "value must be aggregate value!"); 309296417Sdim LValue srcLV = CGF.MakeAddrLValue(src.getAggregateAddress(), type); 310341825Sdim EmitFinalDestCopy(type, srcLV, EVK_RValue); 311239462Sdim} 312193326Sed 313239462Sdim/// EmitFinalDestCopy - Perform the final copy to DestPtr, if desired. 314341825Sdimvoid AggExprEmitter::EmitFinalDestCopy(QualType type, const LValue &src, 315341825Sdim ExprValueKind SrcValueKind) { 316218893Sdim // If Dest is ignored, then we're evaluating an aggregate expression 317239462Sdim // in a context that doesn't care about the result. Note that loads 318239462Sdim // from volatile l-values force the existence of a non-ignored 319239462Sdim // destination. 320239462Sdim if (Dest.isIgnored()) 321239462Sdim return; 322212904Sdim 323341825Sdim // Copy non-trivial C structs here. 324341825Sdim LValue DstLV = CGF.MakeAddrLValue( 325341825Sdim Dest.getAddress(), Dest.isVolatile() ? type.withVolatile() : type); 326341825Sdim 327341825Sdim if (SrcValueKind == EVK_RValue) { 328341825Sdim if (type.isNonTrivialToPrimitiveDestructiveMove() == QualType::PCK_Struct) { 329341825Sdim if (Dest.isPotentiallyAliased()) 330341825Sdim CGF.callCStructMoveAssignmentOperator(DstLV, src); 331341825Sdim else 332341825Sdim CGF.callCStructMoveConstructor(DstLV, src); 333341825Sdim return; 334341825Sdim } 335341825Sdim } else { 336341825Sdim if (type.isNonTrivialToPrimitiveCopy() == QualType::PCK_Struct) { 337341825Sdim if (Dest.isPotentiallyAliased()) 338341825Sdim CGF.callCStructCopyAssignmentOperator(DstLV, src); 339341825Sdim else 340341825Sdim CGF.callCStructCopyConstructor(DstLV, src); 341341825Sdim return; 342341825Sdim } 343341825Sdim } 344341825Sdim 345239462Sdim AggValueSlot srcAgg = 346239462Sdim AggValueSlot::forLValue(src, AggValueSlot::IsDestructed, 347341825Sdim needsGC(type), AggValueSlot::IsAliased, 348341825Sdim AggValueSlot::MayOverlap); 349239462Sdim EmitCopy(type, Dest, srcAgg); 350239462Sdim} 351193326Sed 352239462Sdim/// Perform a copy from the source into the destination. 353239462Sdim/// 354239462Sdim/// \param type - the type of the aggregate being copied; qualifiers are 355239462Sdim/// ignored 356239462Sdimvoid AggExprEmitter::EmitCopy(QualType type, const AggValueSlot &dest, 357239462Sdim const AggValueSlot &src) { 358239462Sdim if (dest.requiresGCollection()) { 359341825Sdim CharUnits sz = dest.getPreferredSize(CGF.getContext(), type); 360239462Sdim llvm::Value *size = llvm::ConstantInt::get(CGF.SizeTy, sz.getQuantity()); 361198092Srdivacky CGF.CGM.getObjCRuntime().EmitGCMemmoveCollectable(CGF, 362296417Sdim dest.getAddress(), 363296417Sdim src.getAddress(), 364239462Sdim size); 365198092Srdivacky return; 366198092Srdivacky } 367239462Sdim 368193326Sed // If the result of the assignment is used, copy the LHS there also. 369239462Sdim // It's volatile if either side is. Use the minimum alignment of 370239462Sdim // the two sides. 371341825Sdim LValue DestLV = CGF.MakeAddrLValue(dest.getAddress(), type); 372341825Sdim LValue SrcLV = CGF.MakeAddrLValue(src.getAddress(), type); 373341825Sdim CGF.EmitAggregateCopy(DestLV, SrcLV, type, dest.mayOverlap(), 374296417Sdim dest.isVolatile() || src.isVolatile()); 375193326Sed} 376193326Sed 377341825Sdim/// Emit the initializer for a std::initializer_list initialized with a 378234353Sdim/// real initializer list. 379261991Sdimvoid 380261991SdimAggExprEmitter::VisitCXXStdInitializerListExpr(CXXStdInitializerListExpr *E) { 381261991Sdim // Emit an array containing the elements. The array is externally destructed 382261991Sdim // if the std::initializer_list object is. 383261991Sdim ASTContext &Ctx = CGF.getContext(); 384261991Sdim LValue Array = CGF.EmitLValue(E->getSubExpr()); 385261991Sdim assert(Array.isSimple() && "initializer_list array not a simple lvalue"); 386296417Sdim Address ArrayPtr = Array.getAddress(); 387234353Sdim 388261991Sdim const ConstantArrayType *ArrayType = 389261991Sdim Ctx.getAsConstantArrayType(E->getSubExpr()->getType()); 390261991Sdim assert(ArrayType && "std::initializer_list constructed from non-array"); 391234353Sdim 392261991Sdim // FIXME: Perform the checks on the field types in SemaInit. 393261991Sdim RecordDecl *Record = E->getType()->castAs<RecordType>()->getDecl(); 394261991Sdim RecordDecl::field_iterator Field = Record->field_begin(); 395261991Sdim if (Field == Record->field_end()) { 396261991Sdim CGF.ErrorUnsupported(E, "weird std::initializer_list"); 397234353Sdim return; 398234353Sdim } 399234353Sdim 400234353Sdim // Start pointer. 401261991Sdim if (!Field->getType()->isPointerType() || 402261991Sdim !Ctx.hasSameType(Field->getType()->getPointeeType(), 403261991Sdim ArrayType->getElementType())) { 404261991Sdim CGF.ErrorUnsupported(E, "weird std::initializer_list"); 405234353Sdim return; 406234353Sdim } 407234353Sdim 408261991Sdim AggValueSlot Dest = EnsureSlot(E->getType()); 409296417Sdim LValue DestLV = CGF.MakeAddrLValue(Dest.getAddress(), E->getType()); 410261991Sdim LValue Start = CGF.EmitLValueForFieldInitialization(DestLV, *Field); 411261991Sdim llvm::Value *Zero = llvm::ConstantInt::get(CGF.PtrDiffTy, 0); 412261991Sdim llvm::Value *IdxStart[] = { Zero, Zero }; 413261991Sdim llvm::Value *ArrayStart = 414296417Sdim Builder.CreateInBoundsGEP(ArrayPtr.getPointer(), IdxStart, "arraystart"); 415261991Sdim CGF.EmitStoreThroughLValue(RValue::get(ArrayStart), Start); 416261991Sdim ++Field; 417261991Sdim 418261991Sdim if (Field == Record->field_end()) { 419261991Sdim CGF.ErrorUnsupported(E, "weird std::initializer_list"); 420234353Sdim return; 421234353Sdim } 422261991Sdim 423261991Sdim llvm::Value *Size = Builder.getInt(ArrayType->getSize()); 424261991Sdim LValue EndOrLength = CGF.EmitLValueForFieldInitialization(DestLV, *Field); 425261991Sdim if (Field->getType()->isPointerType() && 426261991Sdim Ctx.hasSameType(Field->getType()->getPointeeType(), 427261991Sdim ArrayType->getElementType())) { 428234353Sdim // End pointer. 429261991Sdim llvm::Value *IdxEnd[] = { Zero, Size }; 430261991Sdim llvm::Value *ArrayEnd = 431296417Sdim Builder.CreateInBoundsGEP(ArrayPtr.getPointer(), IdxEnd, "arrayend"); 432261991Sdim CGF.EmitStoreThroughLValue(RValue::get(ArrayEnd), EndOrLength); 433261991Sdim } else if (Ctx.hasSameType(Field->getType(), Ctx.getSizeType())) { 434234353Sdim // Length. 435261991Sdim CGF.EmitStoreThroughLValue(RValue::get(Size), EndOrLength); 436234353Sdim } else { 437261991Sdim CGF.ErrorUnsupported(E, "weird std::initializer_list"); 438234353Sdim return; 439234353Sdim } 440234353Sdim} 441234353Sdim 442341825Sdim/// Determine if E is a trivial array filler, that is, one that is 443276479Sdim/// equivalent to zero-initialization. 444276479Sdimstatic bool isTrivialFiller(Expr *E) { 445276479Sdim if (!E) 446276479Sdim return true; 447276479Sdim 448276479Sdim if (isa<ImplicitValueInitExpr>(E)) 449276479Sdim return true; 450276479Sdim 451276479Sdim if (auto *ILE = dyn_cast<InitListExpr>(E)) { 452276479Sdim if (ILE->getNumInits()) 453276479Sdim return false; 454276479Sdim return isTrivialFiller(ILE->getArrayFiller()); 455276479Sdim } 456276479Sdim 457276479Sdim if (auto *Cons = dyn_cast_or_null<CXXConstructExpr>(E)) 458276479Sdim return Cons->getConstructor()->isDefaultConstructor() && 459276479Sdim Cons->getConstructor()->isTrivial(); 460276479Sdim 461276479Sdim // FIXME: Are there other cases where we can avoid emitting an initializer? 462276479Sdim return false; 463276479Sdim} 464276479Sdim 465341825Sdim/// Emit initialization of an array from an initializer list. 466296417Sdimvoid AggExprEmitter::EmitArrayInit(Address DestPtr, llvm::ArrayType *AType, 467338697Sdim QualType ArrayQTy, InitListExpr *E) { 468234353Sdim uint64_t NumInitElements = E->getNumInits(); 469234353Sdim 470234353Sdim uint64_t NumArrayElements = AType->getNumElements(); 471234353Sdim assert(NumInitElements <= NumArrayElements); 472234353Sdim 473338697Sdim QualType elementType = 474338697Sdim CGF.getContext().getAsArrayType(ArrayQTy)->getElementType(); 475338697Sdim 476234353Sdim // DestPtr is an array*. Construct an elementType* by drilling 477234353Sdim // down a level. 478234353Sdim llvm::Value *zero = llvm::ConstantInt::get(CGF.SizeTy, 0); 479234353Sdim llvm::Value *indices[] = { zero, zero }; 480234353Sdim llvm::Value *begin = 481296417Sdim Builder.CreateInBoundsGEP(DestPtr.getPointer(), indices, "arrayinit.begin"); 482234353Sdim 483296417Sdim CharUnits elementSize = CGF.getContext().getTypeSizeInChars(elementType); 484296417Sdim CharUnits elementAlign = 485296417Sdim DestPtr.getAlignment().alignmentOfArrayElement(elementSize); 486296417Sdim 487338697Sdim // Consider initializing the array by copying from a global. For this to be 488338697Sdim // more efficient than per-element initialization, the size of the elements 489338697Sdim // with explicit initializers should be large enough. 490338697Sdim if (NumInitElements * elementSize.getQuantity() > 16 && 491338697Sdim elementType.isTriviallyCopyableType(CGF.getContext())) { 492338697Sdim CodeGen::CodeGenModule &CGM = CGF.CGM; 493338697Sdim ConstantEmitter Emitter(CGM); 494338697Sdim LangAS AS = ArrayQTy.getAddressSpace(); 495338697Sdim if (llvm::Constant *C = Emitter.tryEmitForInitializer(E, AS, ArrayQTy)) { 496338697Sdim auto GV = new llvm::GlobalVariable( 497338697Sdim CGM.getModule(), C->getType(), 498338697Sdim CGM.isTypeConstant(ArrayQTy, /* ExcludeCtorDtor= */ true), 499338697Sdim llvm::GlobalValue::PrivateLinkage, C, "constinit", 500338697Sdim /* InsertBefore= */ nullptr, llvm::GlobalVariable::NotThreadLocal, 501338697Sdim CGM.getContext().getTargetAddressSpace(AS)); 502338697Sdim Emitter.finalize(GV); 503338697Sdim CharUnits Align = CGM.getContext().getTypeAlignInChars(ArrayQTy); 504338697Sdim GV->setAlignment(Align.getQuantity()); 505338697Sdim EmitFinalDestCopy(ArrayQTy, CGF.MakeAddrLValue(GV, ArrayQTy, Align)); 506338697Sdim return; 507338697Sdim } 508338697Sdim } 509338697Sdim 510234353Sdim // Exception safety requires us to destroy all the 511234353Sdim // already-constructed members if an initializer throws. 512234353Sdim // For that, we'll need an EH cleanup. 513234353Sdim QualType::DestructionKind dtorKind = elementType.isDestructedType(); 514296417Sdim Address endOfInit = Address::invalid(); 515234353Sdim EHScopeStack::stable_iterator cleanup; 516276479Sdim llvm::Instruction *cleanupDominator = nullptr; 517234353Sdim if (CGF.needsEHCleanup(dtorKind)) { 518234353Sdim // In principle we could tell the cleanup where we are more 519234353Sdim // directly, but the control flow can get so varied here that it 520234353Sdim // would actually be quite complex. Therefore we go through an 521234353Sdim // alloca. 522296417Sdim endOfInit = CGF.CreateTempAlloca(begin->getType(), CGF.getPointerAlign(), 523234353Sdim "arrayinit.endOfInit"); 524234353Sdim cleanupDominator = Builder.CreateStore(begin, endOfInit); 525234353Sdim CGF.pushIrregularPartialArrayCleanup(begin, endOfInit, elementType, 526296417Sdim elementAlign, 527234353Sdim CGF.getDestroyer(dtorKind)); 528234353Sdim cleanup = CGF.EHStack.stable_begin(); 529234353Sdim 530234353Sdim // Otherwise, remember that we didn't need a cleanup. 531234353Sdim } else { 532234353Sdim dtorKind = QualType::DK_none; 533234353Sdim } 534234353Sdim 535234353Sdim llvm::Value *one = llvm::ConstantInt::get(CGF.SizeTy, 1); 536234353Sdim 537234353Sdim // The 'current element to initialize'. The invariants on this 538234353Sdim // variable are complicated. Essentially, after each iteration of 539234353Sdim // the loop, it points to the last initialized element, except 540234353Sdim // that it points to the beginning of the array before any 541234353Sdim // elements have been initialized. 542234353Sdim llvm::Value *element = begin; 543234353Sdim 544234353Sdim // Emit the explicit initializers. 545234353Sdim for (uint64_t i = 0; i != NumInitElements; ++i) { 546234353Sdim // Advance to the next element. 547234353Sdim if (i > 0) { 548234353Sdim element = Builder.CreateInBoundsGEP(element, one, "arrayinit.element"); 549234353Sdim 550234353Sdim // Tell the cleanup that it needs to destroy up to this 551234353Sdim // element. TODO: some of these stores can be trivially 552234353Sdim // observed to be unnecessary. 553296417Sdim if (endOfInit.isValid()) Builder.CreateStore(element, endOfInit); 554234353Sdim } 555234353Sdim 556296417Sdim LValue elementLV = 557296417Sdim CGF.MakeAddrLValue(Address(element, elementAlign), elementType); 558261991Sdim EmitInitializationToLValue(E->getInit(i), elementLV); 559234353Sdim } 560234353Sdim 561234353Sdim // Check whether there's a non-trivial array-fill expression. 562234353Sdim Expr *filler = E->getArrayFiller(); 563276479Sdim bool hasTrivialFiller = isTrivialFiller(filler); 564234353Sdim 565234353Sdim // Any remaining elements need to be zero-initialized, possibly 566234353Sdim // using the filler expression. We can skip this if the we're 567234353Sdim // emitting to zeroed memory. 568234353Sdim if (NumInitElements != NumArrayElements && 569234353Sdim !(Dest.isZeroed() && hasTrivialFiller && 570234353Sdim CGF.getTypes().isZeroInitializable(elementType))) { 571234353Sdim 572234353Sdim // Use an actual loop. This is basically 573234353Sdim // do { *array++ = filler; } while (array != end); 574234353Sdim 575234353Sdim // Advance to the start of the rest of the array. 576234353Sdim if (NumInitElements) { 577234353Sdim element = Builder.CreateInBoundsGEP(element, one, "arrayinit.start"); 578296417Sdim if (endOfInit.isValid()) Builder.CreateStore(element, endOfInit); 579234353Sdim } 580234353Sdim 581234353Sdim // Compute the end of the array. 582234353Sdim llvm::Value *end = Builder.CreateInBoundsGEP(begin, 583234353Sdim llvm::ConstantInt::get(CGF.SizeTy, NumArrayElements), 584234353Sdim "arrayinit.end"); 585234353Sdim 586234353Sdim llvm::BasicBlock *entryBB = Builder.GetInsertBlock(); 587234353Sdim llvm::BasicBlock *bodyBB = CGF.createBasicBlock("arrayinit.body"); 588234353Sdim 589234353Sdim // Jump into the body. 590234353Sdim CGF.EmitBlock(bodyBB); 591234353Sdim llvm::PHINode *currentElement = 592234353Sdim Builder.CreatePHI(element->getType(), 2, "arrayinit.cur"); 593234353Sdim currentElement->addIncoming(element, entryBB); 594234353Sdim 595234353Sdim // Emit the actual filler expression. 596321369Sdim { 597321369Sdim // C++1z [class.temporary]p5: 598321369Sdim // when a default constructor is called to initialize an element of 599321369Sdim // an array with no corresponding initializer [...] the destruction of 600321369Sdim // every temporary created in a default argument is sequenced before 601321369Sdim // the construction of the next array element, if any 602321369Sdim CodeGenFunction::RunCleanupsScope CleanupsScope(CGF); 603321369Sdim LValue elementLV = 604321369Sdim CGF.MakeAddrLValue(Address(currentElement, elementAlign), elementType); 605321369Sdim if (filler) 606321369Sdim EmitInitializationToLValue(filler, elementLV); 607321369Sdim else 608321369Sdim EmitNullInitializationToLValue(elementLV); 609321369Sdim } 610234353Sdim 611234353Sdim // Move on to the next element. 612234353Sdim llvm::Value *nextElement = 613234353Sdim Builder.CreateInBoundsGEP(currentElement, one, "arrayinit.next"); 614234353Sdim 615234353Sdim // Tell the EH cleanup that we finished with the last element. 616296417Sdim if (endOfInit.isValid()) Builder.CreateStore(nextElement, endOfInit); 617234353Sdim 618234353Sdim // Leave the loop if we're done. 619234353Sdim llvm::Value *done = Builder.CreateICmpEQ(nextElement, end, 620234353Sdim "arrayinit.done"); 621234353Sdim llvm::BasicBlock *endBB = CGF.createBasicBlock("arrayinit.end"); 622234353Sdim Builder.CreateCondBr(done, endBB, bodyBB); 623234353Sdim currentElement->addIncoming(nextElement, Builder.GetInsertBlock()); 624234353Sdim 625234353Sdim CGF.EmitBlock(endBB); 626234353Sdim } 627234353Sdim 628234353Sdim // Leave the partial-array cleanup if we entered one. 629234353Sdim if (dtorKind) CGF.DeactivateCleanupBlock(cleanup, cleanupDominator); 630234353Sdim} 631234353Sdim 632193326Sed//===----------------------------------------------------------------------===// 633193326Sed// Visitor Methods 634193326Sed//===----------------------------------------------------------------------===// 635193326Sed 636224145Sdimvoid AggExprEmitter::VisitMaterializeTemporaryExpr(MaterializeTemporaryExpr *E){ 637224145Sdim Visit(E->GetTemporaryExpr()); 638224145Sdim} 639224145Sdim 640218893Sdimvoid AggExprEmitter::VisitOpaqueValueExpr(OpaqueValueExpr *e) { 641341825Sdim // If this is a unique OVE, just visit its source expression. 642341825Sdim if (e->isUnique()) 643341825Sdim Visit(e->getSourceExpr()); 644341825Sdim else 645341825Sdim EmitFinalDestCopy(e->getType(), CGF.getOrCreateOpaqueLValueMapping(e)); 646218893Sdim} 647218893Sdim 648224145Sdimvoid 649224145SdimAggExprEmitter::VisitCompoundLiteralExpr(CompoundLiteralExpr *E) { 650249423Sdim if (Dest.isPotentiallyAliased() && 651249423Sdim E->getType().isPODType(CGF.getContext())) { 652224145Sdim // For a POD type, just emit a load of the lvalue + a copy, because our 653224145Sdim // compound literal might alias the destination. 654224145Sdim EmitAggLoadOfLValue(E); 655224145Sdim return; 656224145Sdim } 657341825Sdim 658224145Sdim AggValueSlot Slot = EnsureSlot(E->getType()); 659224145Sdim CGF.EmitAggExpr(E->getInitializer(), Slot); 660224145Sdim} 661224145Sdim 662249423Sdim/// Attempt to look through various unimportant expressions to find a 663249423Sdim/// cast of the given kind. 664249423Sdimstatic Expr *findPeephole(Expr *op, CastKind kind) { 665249423Sdim while (true) { 666249423Sdim op = op->IgnoreParens(); 667249423Sdim if (CastExpr *castE = dyn_cast<CastExpr>(op)) { 668249423Sdim if (castE->getCastKind() == kind) 669249423Sdim return castE->getSubExpr(); 670249423Sdim if (castE->getCastKind() == CK_NoOp) 671249423Sdim continue; 672249423Sdim } 673276479Sdim return nullptr; 674249423Sdim } 675249423Sdim} 676224145Sdim 677198092Srdivackyvoid AggExprEmitter::VisitCastExpr(CastExpr *E) { 678296417Sdim if (const auto *ECE = dyn_cast<ExplicitCastExpr>(E)) 679296417Sdim CGF.CGM.EmitExplicitCastExprType(ECE, &CGF); 680198092Srdivacky switch (E->getCastKind()) { 681212904Sdim case CK_Dynamic: { 682243830Sdim // FIXME: Can this actually happen? We have no test coverage for it. 683208600Srdivacky assert(isa<CXXDynamicCastExpr>(E) && "CK_Dynamic without a dynamic_cast?"); 684243830Sdim LValue LV = CGF.EmitCheckedLValue(E->getSubExpr(), 685243830Sdim CodeGenFunction::TCK_Load); 686208600Srdivacky // FIXME: Do we also need to handle property references here? 687208600Srdivacky if (LV.isSimple()) 688208600Srdivacky CGF.EmitDynamicCast(LV.getAddress(), cast<CXXDynamicCastExpr>(E)); 689208600Srdivacky else 690208600Srdivacky CGF.CGM.ErrorUnsupported(E, "non-simple lvalue dynamic_cast"); 691341825Sdim 692218893Sdim if (!Dest.isIgnored()) 693218893Sdim CGF.CGM.ErrorUnsupported(E, "lvalue dynamic_cast with a destination"); 694208600Srdivacky break; 695208600Srdivacky } 696341825Sdim 697212904Sdim case CK_ToUnion: { 698288943Sdim // Evaluate even if the destination is ignored. 699288943Sdim if (Dest.isIgnored()) { 700288943Sdim CGF.EmitAnyExpr(E->getSubExpr(), AggValueSlot::ignored(), 701288943Sdim /*ignoreResult=*/true); 702288943Sdim break; 703288943Sdim } 704221345Sdim 705198092Srdivacky // GCC union extension 706212904Sdim QualType Ty = E->getSubExpr()->getType(); 707296417Sdim Address CastPtr = 708296417Sdim Builder.CreateElementBitCast(Dest.getAddress(), CGF.ConvertType(Ty)); 709224145Sdim EmitInitializationToLValue(E->getSubExpr(), 710224145Sdim CGF.MakeAddrLValue(CastPtr, Ty)); 711198092Srdivacky break; 712193326Sed } 713193326Sed 714353358Sdim case CK_LValueToRValueBitCast: { 715353358Sdim if (Dest.isIgnored()) { 716353358Sdim CGF.EmitAnyExpr(E->getSubExpr(), AggValueSlot::ignored(), 717353358Sdim /*ignoreResult=*/true); 718353358Sdim break; 719353358Sdim } 720353358Sdim 721353358Sdim LValue SourceLV = CGF.EmitLValue(E->getSubExpr()); 722353358Sdim Address SourceAddress = 723353358Sdim Builder.CreateElementBitCast(SourceLV.getAddress(), CGF.Int8Ty); 724353358Sdim Address DestAddress = 725353358Sdim Builder.CreateElementBitCast(Dest.getAddress(), CGF.Int8Ty); 726353358Sdim llvm::Value *SizeVal = llvm::ConstantInt::get( 727353358Sdim CGF.SizeTy, 728353358Sdim CGF.getContext().getTypeSizeInChars(E->getType()).getQuantity()); 729353358Sdim Builder.CreateMemCpy(DestAddress, SourceAddress, SizeVal); 730353358Sdim break; 731353358Sdim } 732353358Sdim 733212904Sdim case CK_DerivedToBase: 734212904Sdim case CK_BaseToDerived: 735212904Sdim case CK_UncheckedDerivedToBase: { 736226633Sdim llvm_unreachable("cannot perform hierarchy conversion in EmitAggExpr: " 737208600Srdivacky "should have been unpacked before we got here"); 738208600Srdivacky } 739208600Srdivacky 740249423Sdim case CK_NonAtomicToAtomic: 741249423Sdim case CK_AtomicToNonAtomic: { 742249423Sdim bool isToAtomic = (E->getCastKind() == CK_NonAtomicToAtomic); 743249423Sdim 744249423Sdim // Determine the atomic and value types. 745249423Sdim QualType atomicType = E->getSubExpr()->getType(); 746249423Sdim QualType valueType = E->getType(); 747249423Sdim if (isToAtomic) std::swap(atomicType, valueType); 748249423Sdim 749249423Sdim assert(atomicType->isAtomicType()); 750249423Sdim assert(CGF.getContext().hasSameUnqualifiedType(valueType, 751249423Sdim atomicType->castAs<AtomicType>()->getValueType())); 752249423Sdim 753249423Sdim // Just recurse normally if we're ignoring the result or the 754249423Sdim // atomic type doesn't change representation. 755249423Sdim if (Dest.isIgnored() || !CGF.CGM.isPaddedAtomicType(atomicType)) { 756249423Sdim return Visit(E->getSubExpr()); 757249423Sdim } 758249423Sdim 759249423Sdim CastKind peepholeTarget = 760249423Sdim (isToAtomic ? CK_AtomicToNonAtomic : CK_NonAtomicToAtomic); 761249423Sdim 762249423Sdim // These two cases are reverses of each other; try to peephole them. 763249423Sdim if (Expr *op = findPeephole(E->getSubExpr(), peepholeTarget)) { 764249423Sdim assert(CGF.getContext().hasSameUnqualifiedType(op->getType(), 765249423Sdim E->getType()) && 766249423Sdim "peephole significantly changed types?"); 767249423Sdim return Visit(op); 768249423Sdim } 769249423Sdim 770249423Sdim // If we're converting an r-value of non-atomic type to an r-value 771261991Sdim // of atomic type, just emit directly into the relevant sub-object. 772249423Sdim if (isToAtomic) { 773261991Sdim AggValueSlot valueDest = Dest; 774261991Sdim if (!valueDest.isIgnored() && CGF.CGM.isPaddedAtomicType(atomicType)) { 775341825Sdim // Zero-initialize. (Strictly speaking, we only need to initialize 776261991Sdim // the padding at the end, but this is simpler.) 777261991Sdim if (!Dest.isZeroed()) 778296417Sdim CGF.EmitNullInitialization(Dest.getAddress(), atomicType); 779261991Sdim 780261991Sdim // Build a GEP to refer to the subobject. 781296417Sdim Address valueAddr = 782353358Sdim CGF.Builder.CreateStructGEP(valueDest.getAddress(), 0); 783261991Sdim valueDest = AggValueSlot::forAddr(valueAddr, 784261991Sdim valueDest.getQualifiers(), 785261991Sdim valueDest.isExternallyDestructed(), 786261991Sdim valueDest.requiresGCollection(), 787261991Sdim valueDest.isPotentiallyAliased(), 788341825Sdim AggValueSlot::DoesNotOverlap, 789261991Sdim AggValueSlot::IsZeroed); 790261991Sdim } 791341825Sdim 792261991Sdim CGF.EmitAggExpr(E->getSubExpr(), valueDest); 793249423Sdim return; 794249423Sdim } 795249423Sdim 796249423Sdim // Otherwise, we're converting an atomic type to a non-atomic type. 797261991Sdim // Make an atomic temporary, emit into that, and then copy the value out. 798249423Sdim AggValueSlot atomicSlot = 799249423Sdim CGF.CreateAggTemp(atomicType, "atomic-to-nonatomic.temp"); 800249423Sdim CGF.EmitAggExpr(E->getSubExpr(), atomicSlot); 801249423Sdim 802353358Sdim Address valueAddr = Builder.CreateStructGEP(atomicSlot.getAddress(), 0); 803249423Sdim RValue rvalue = RValue::getAggregate(valueAddr, atomicSlot.isVolatile()); 804249423Sdim return EmitFinalDestCopy(valueType, rvalue); 805249423Sdim } 806353358Sdim case CK_AddressSpaceConversion: 807353358Sdim return Visit(E->getSubExpr()); 808249423Sdim 809239462Sdim case CK_LValueToRValue: 810239462Sdim // If we're loading from a volatile type, force the destination 811239462Sdim // into existence. 812239462Sdim if (E->getSubExpr()->getType().isVolatileQualified()) { 813239462Sdim EnsureDest(E->getType()); 814239462Sdim return Visit(E->getSubExpr()); 815239462Sdim } 816249423Sdim 817327952Sdim LLVM_FALLTHROUGH; 818239462Sdim 819353358Sdim 820212904Sdim case CK_NoOp: 821212904Sdim case CK_UserDefinedConversion: 822212904Sdim case CK_ConstructorConversion: 823198092Srdivacky assert(CGF.getContext().hasSameUnqualifiedType(E->getSubExpr()->getType(), 824198092Srdivacky E->getType()) && 825198092Srdivacky "Implicit cast types must be compatible"); 826198092Srdivacky Visit(E->getSubExpr()); 827198092Srdivacky break; 828341825Sdim 829212904Sdim case CK_LValueBitCast: 830218893Sdim llvm_unreachable("should not be emitting lvalue bitcast as rvalue"); 831221345Sdim 832218893Sdim case CK_Dependent: 833218893Sdim case CK_BitCast: 834218893Sdim case CK_ArrayToPointerDecay: 835218893Sdim case CK_FunctionToPointerDecay: 836218893Sdim case CK_NullToPointer: 837218893Sdim case CK_NullToMemberPointer: 838218893Sdim case CK_BaseToDerivedMemberPointer: 839218893Sdim case CK_DerivedToBaseMemberPointer: 840218893Sdim case CK_MemberPointerToBoolean: 841234353Sdim case CK_ReinterpretMemberPointer: 842218893Sdim case CK_IntegralToPointer: 843218893Sdim case CK_PointerToIntegral: 844218893Sdim case CK_PointerToBoolean: 845218893Sdim case CK_ToVoid: 846218893Sdim case CK_VectorSplat: 847218893Sdim case CK_IntegralCast: 848296417Sdim case CK_BooleanToSignedIntegral: 849218893Sdim case CK_IntegralToBoolean: 850218893Sdim case CK_IntegralToFloating: 851218893Sdim case CK_FloatingToIntegral: 852218893Sdim case CK_FloatingToBoolean: 853218893Sdim case CK_FloatingCast: 854226633Sdim case CK_CPointerToObjCPointerCast: 855226633Sdim case CK_BlockPointerToObjCPointerCast: 856218893Sdim case CK_AnyPointerToBlockPointerCast: 857218893Sdim case CK_ObjCObjectLValueCast: 858218893Sdim case CK_FloatingRealToComplex: 859218893Sdim case CK_FloatingComplexToReal: 860218893Sdim case CK_FloatingComplexToBoolean: 861218893Sdim case CK_FloatingComplexCast: 862218893Sdim case CK_FloatingComplexToIntegralComplex: 863218893Sdim case CK_IntegralRealToComplex: 864218893Sdim case CK_IntegralComplexToReal: 865218893Sdim case CK_IntegralComplexToBoolean: 866218893Sdim case CK_IntegralComplexCast: 867218893Sdim case CK_IntegralComplexToFloatingComplex: 868226633Sdim case CK_ARCProduceObject: 869226633Sdim case CK_ARCConsumeObject: 870226633Sdim case CK_ARCReclaimReturnedObject: 871226633Sdim case CK_ARCExtendBlockObject: 872234353Sdim case CK_CopyAndAutoreleaseBlockObject: 873243830Sdim case CK_BuiltinFnToFnPtr: 874344779Sdim case CK_ZeroToOCLOpaqueType: 875353358Sdim 876314564Sdim case CK_IntToOCLSampler: 877344779Sdim case CK_FixedPointCast: 878344779Sdim case CK_FixedPointToBoolean: 879353358Sdim case CK_FixedPointToIntegral: 880353358Sdim case CK_IntegralToFixedPoint: 881218893Sdim llvm_unreachable("cast kind invalid for aggregate types"); 882198398Srdivacky } 883193326Sed} 884193326Sed 885193326Sedvoid AggExprEmitter::VisitCallExpr(const CallExpr *E) { 886288943Sdim if (E->getCallReturnType(CGF.getContext())->isReferenceType()) { 887193326Sed EmitAggLoadOfLValue(E); 888193326Sed return; 889193326Sed } 890198092Srdivacky 891341825Sdim withReturnValueSlot(E, [&](ReturnValueSlot Slot) { 892341825Sdim return CGF.EmitCallExpr(E, Slot); 893341825Sdim }); 894193326Sed} 895193326Sed 896193326Sedvoid AggExprEmitter::VisitObjCMessageExpr(ObjCMessageExpr *E) { 897341825Sdim withReturnValueSlot(E, [&](ReturnValueSlot Slot) { 898341825Sdim return CGF.EmitObjCMessageExpr(E, Slot); 899341825Sdim }); 900193326Sed} 901193326Sed 902193326Sedvoid AggExprEmitter::VisitBinComma(const BinaryOperator *E) { 903218893Sdim CGF.EmitIgnoredExpr(E->getLHS()); 904218893Sdim Visit(E->getRHS()); 905193326Sed} 906193326Sed 907193326Sedvoid AggExprEmitter::VisitStmtExpr(const StmtExpr *E) { 908218893Sdim CodeGenFunction::StmtExprEvaluation eval(CGF); 909218893Sdim CGF.EmitCompoundStmt(*E->getSubStmt(), true, Dest); 910193326Sed} 911193326Sed 912341825Sdimenum CompareKind { 913341825Sdim CK_Less, 914341825Sdim CK_Greater, 915341825Sdim CK_Equal, 916341825Sdim}; 917341825Sdim 918341825Sdimstatic llvm::Value *EmitCompare(CGBuilderTy &Builder, CodeGenFunction &CGF, 919341825Sdim const BinaryOperator *E, llvm::Value *LHS, 920341825Sdim llvm::Value *RHS, CompareKind Kind, 921341825Sdim const char *NameSuffix = "") { 922341825Sdim QualType ArgTy = E->getLHS()->getType(); 923341825Sdim if (const ComplexType *CT = ArgTy->getAs<ComplexType>()) 924341825Sdim ArgTy = CT->getElementType(); 925341825Sdim 926341825Sdim if (const auto *MPT = ArgTy->getAs<MemberPointerType>()) { 927341825Sdim assert(Kind == CK_Equal && 928341825Sdim "member pointers may only be compared for equality"); 929341825Sdim return CGF.CGM.getCXXABI().EmitMemberPointerComparison( 930341825Sdim CGF, LHS, RHS, MPT, /*IsInequality*/ false); 931341825Sdim } 932341825Sdim 933341825Sdim // Compute the comparison instructions for the specified comparison kind. 934341825Sdim struct CmpInstInfo { 935341825Sdim const char *Name; 936341825Sdim llvm::CmpInst::Predicate FCmp; 937341825Sdim llvm::CmpInst::Predicate SCmp; 938341825Sdim llvm::CmpInst::Predicate UCmp; 939341825Sdim }; 940341825Sdim CmpInstInfo InstInfo = [&]() -> CmpInstInfo { 941341825Sdim using FI = llvm::FCmpInst; 942341825Sdim using II = llvm::ICmpInst; 943341825Sdim switch (Kind) { 944341825Sdim case CK_Less: 945341825Sdim return {"cmp.lt", FI::FCMP_OLT, II::ICMP_SLT, II::ICMP_ULT}; 946341825Sdim case CK_Greater: 947341825Sdim return {"cmp.gt", FI::FCMP_OGT, II::ICMP_SGT, II::ICMP_UGT}; 948341825Sdim case CK_Equal: 949341825Sdim return {"cmp.eq", FI::FCMP_OEQ, II::ICMP_EQ, II::ICMP_EQ}; 950341825Sdim } 951341825Sdim llvm_unreachable("Unrecognised CompareKind enum"); 952341825Sdim }(); 953341825Sdim 954341825Sdim if (ArgTy->hasFloatingRepresentation()) 955341825Sdim return Builder.CreateFCmp(InstInfo.FCmp, LHS, RHS, 956341825Sdim llvm::Twine(InstInfo.Name) + NameSuffix); 957341825Sdim if (ArgTy->isIntegralOrEnumerationType() || ArgTy->isPointerType()) { 958341825Sdim auto Inst = 959341825Sdim ArgTy->hasSignedIntegerRepresentation() ? InstInfo.SCmp : InstInfo.UCmp; 960341825Sdim return Builder.CreateICmp(Inst, LHS, RHS, 961341825Sdim llvm::Twine(InstInfo.Name) + NameSuffix); 962341825Sdim } 963341825Sdim 964341825Sdim llvm_unreachable("unsupported aggregate binary expression should have " 965341825Sdim "already been handled"); 966341825Sdim} 967341825Sdim 968341825Sdimvoid AggExprEmitter::VisitBinCmp(const BinaryOperator *E) { 969341825Sdim using llvm::BasicBlock; 970341825Sdim using llvm::PHINode; 971341825Sdim using llvm::Value; 972341825Sdim assert(CGF.getContext().hasSameType(E->getLHS()->getType(), 973341825Sdim E->getRHS()->getType())); 974341825Sdim const ComparisonCategoryInfo &CmpInfo = 975341825Sdim CGF.getContext().CompCategories.getInfoForType(E->getType()); 976341825Sdim assert(CmpInfo.Record->isTriviallyCopyable() && 977341825Sdim "cannot copy non-trivially copyable aggregate"); 978341825Sdim 979341825Sdim QualType ArgTy = E->getLHS()->getType(); 980341825Sdim 981341825Sdim // TODO: Handle comparing these types. 982341825Sdim if (ArgTy->isVectorType()) 983341825Sdim return CGF.ErrorUnsupported( 984341825Sdim E, "aggregate three-way comparison with vector arguments"); 985341825Sdim if (!ArgTy->isIntegralOrEnumerationType() && !ArgTy->isRealFloatingType() && 986341825Sdim !ArgTy->isNullPtrType() && !ArgTy->isPointerType() && 987341825Sdim !ArgTy->isMemberPointerType() && !ArgTy->isAnyComplexType()) { 988341825Sdim return CGF.ErrorUnsupported(E, "aggregate three-way comparison"); 989341825Sdim } 990341825Sdim bool IsComplex = ArgTy->isAnyComplexType(); 991341825Sdim 992341825Sdim // Evaluate the operands to the expression and extract their values. 993341825Sdim auto EmitOperand = [&](Expr *E) -> std::pair<Value *, Value *> { 994341825Sdim RValue RV = CGF.EmitAnyExpr(E); 995341825Sdim if (RV.isScalar()) 996341825Sdim return {RV.getScalarVal(), nullptr}; 997341825Sdim if (RV.isAggregate()) 998341825Sdim return {RV.getAggregatePointer(), nullptr}; 999341825Sdim assert(RV.isComplex()); 1000341825Sdim return RV.getComplexVal(); 1001341825Sdim }; 1002341825Sdim auto LHSValues = EmitOperand(E->getLHS()), 1003341825Sdim RHSValues = EmitOperand(E->getRHS()); 1004341825Sdim 1005341825Sdim auto EmitCmp = [&](CompareKind K) { 1006341825Sdim Value *Cmp = EmitCompare(Builder, CGF, E, LHSValues.first, RHSValues.first, 1007341825Sdim K, IsComplex ? ".r" : ""); 1008341825Sdim if (!IsComplex) 1009341825Sdim return Cmp; 1010341825Sdim assert(K == CompareKind::CK_Equal); 1011341825Sdim Value *CmpImag = EmitCompare(Builder, CGF, E, LHSValues.second, 1012341825Sdim RHSValues.second, K, ".i"); 1013341825Sdim return Builder.CreateAnd(Cmp, CmpImag, "and.eq"); 1014341825Sdim }; 1015341825Sdim auto EmitCmpRes = [&](const ComparisonCategoryInfo::ValueInfo *VInfo) { 1016341825Sdim return Builder.getInt(VInfo->getIntValue()); 1017341825Sdim }; 1018341825Sdim 1019341825Sdim Value *Select; 1020341825Sdim if (ArgTy->isNullPtrType()) { 1021341825Sdim Select = EmitCmpRes(CmpInfo.getEqualOrEquiv()); 1022341825Sdim } else if (CmpInfo.isEquality()) { 1023341825Sdim Select = Builder.CreateSelect( 1024341825Sdim EmitCmp(CK_Equal), EmitCmpRes(CmpInfo.getEqualOrEquiv()), 1025341825Sdim EmitCmpRes(CmpInfo.getNonequalOrNonequiv()), "sel.eq"); 1026341825Sdim } else if (!CmpInfo.isPartial()) { 1027341825Sdim Value *SelectOne = 1028341825Sdim Builder.CreateSelect(EmitCmp(CK_Less), EmitCmpRes(CmpInfo.getLess()), 1029341825Sdim EmitCmpRes(CmpInfo.getGreater()), "sel.lt"); 1030341825Sdim Select = Builder.CreateSelect(EmitCmp(CK_Equal), 1031341825Sdim EmitCmpRes(CmpInfo.getEqualOrEquiv()), 1032341825Sdim SelectOne, "sel.eq"); 1033341825Sdim } else { 1034341825Sdim Value *SelectEq = Builder.CreateSelect( 1035341825Sdim EmitCmp(CK_Equal), EmitCmpRes(CmpInfo.getEqualOrEquiv()), 1036341825Sdim EmitCmpRes(CmpInfo.getUnordered()), "sel.eq"); 1037341825Sdim Value *SelectGT = Builder.CreateSelect(EmitCmp(CK_Greater), 1038341825Sdim EmitCmpRes(CmpInfo.getGreater()), 1039341825Sdim SelectEq, "sel.gt"); 1040341825Sdim Select = Builder.CreateSelect( 1041341825Sdim EmitCmp(CK_Less), EmitCmpRes(CmpInfo.getLess()), SelectGT, "sel.lt"); 1042341825Sdim } 1043341825Sdim // Create the return value in the destination slot. 1044341825Sdim EnsureDest(E->getType()); 1045341825Sdim LValue DestLV = CGF.MakeAddrLValue(Dest.getAddress(), E->getType()); 1046341825Sdim 1047341825Sdim // Emit the address of the first (and only) field in the comparison category 1048341825Sdim // type, and initialize it from the constant integer value selected above. 1049341825Sdim LValue FieldLV = CGF.EmitLValueForFieldInitialization( 1050341825Sdim DestLV, *CmpInfo.Record->field_begin()); 1051341825Sdim CGF.EmitStoreThroughLValue(RValue::get(Select), FieldLV, /*IsInit*/ true); 1052341825Sdim 1053341825Sdim // All done! The result is in the Dest slot. 1054341825Sdim} 1055341825Sdim 1056193326Sedvoid AggExprEmitter::VisitBinaryOperator(const BinaryOperator *E) { 1057212904Sdim if (E->getOpcode() == BO_PtrMemD || E->getOpcode() == BO_PtrMemI) 1058198398Srdivacky VisitPointerToDataMemberBinaryOperator(E); 1059198398Srdivacky else 1060198398Srdivacky CGF.ErrorUnsupported(E, "aggregate binary expression"); 1061193326Sed} 1062193326Sed 1063198398Srdivackyvoid AggExprEmitter::VisitPointerToDataMemberBinaryOperator( 1064198398Srdivacky const BinaryOperator *E) { 1065198398Srdivacky LValue LV = CGF.EmitPointerToDataMemberBinaryExpr(E); 1066239462Sdim EmitFinalDestCopy(E->getType(), LV); 1067198398Srdivacky} 1068198398Srdivacky 1069239462Sdim/// Is the value of the given expression possibly a reference to or 1070239462Sdim/// into a __block variable? 1071239462Sdimstatic bool isBlockVarRef(const Expr *E) { 1072239462Sdim // Make sure we look through parens. 1073239462Sdim E = E->IgnoreParens(); 1074239462Sdim 1075239462Sdim // Check for a direct reference to a __block variable. 1076239462Sdim if (const DeclRefExpr *DRE = dyn_cast<DeclRefExpr>(E)) { 1077239462Sdim const VarDecl *var = dyn_cast<VarDecl>(DRE->getDecl()); 1078239462Sdim return (var && var->hasAttr<BlocksAttr>()); 1079239462Sdim } 1080239462Sdim 1081239462Sdim // More complicated stuff. 1082239462Sdim 1083239462Sdim // Binary operators. 1084239462Sdim if (const BinaryOperator *op = dyn_cast<BinaryOperator>(E)) { 1085239462Sdim // For an assignment or pointer-to-member operation, just care 1086239462Sdim // about the LHS. 1087239462Sdim if (op->isAssignmentOp() || op->isPtrMemOp()) 1088239462Sdim return isBlockVarRef(op->getLHS()); 1089239462Sdim 1090239462Sdim // For a comma, just care about the RHS. 1091239462Sdim if (op->getOpcode() == BO_Comma) 1092239462Sdim return isBlockVarRef(op->getRHS()); 1093239462Sdim 1094239462Sdim // FIXME: pointer arithmetic? 1095239462Sdim return false; 1096239462Sdim 1097239462Sdim // Check both sides of a conditional operator. 1098239462Sdim } else if (const AbstractConditionalOperator *op 1099239462Sdim = dyn_cast<AbstractConditionalOperator>(E)) { 1100239462Sdim return isBlockVarRef(op->getTrueExpr()) 1101239462Sdim || isBlockVarRef(op->getFalseExpr()); 1102239462Sdim 1103239462Sdim // OVEs are required to support BinaryConditionalOperators. 1104239462Sdim } else if (const OpaqueValueExpr *op 1105239462Sdim = dyn_cast<OpaqueValueExpr>(E)) { 1106239462Sdim if (const Expr *src = op->getSourceExpr()) 1107239462Sdim return isBlockVarRef(src); 1108239462Sdim 1109239462Sdim // Casts are necessary to get things like (*(int*)&var) = foo(). 1110239462Sdim // We don't really care about the kind of cast here, except 1111239462Sdim // we don't want to look through l2r casts, because it's okay 1112239462Sdim // to get the *value* in a __block variable. 1113239462Sdim } else if (const CastExpr *cast = dyn_cast<CastExpr>(E)) { 1114239462Sdim if (cast->getCastKind() == CK_LValueToRValue) 1115239462Sdim return false; 1116239462Sdim return isBlockVarRef(cast->getSubExpr()); 1117239462Sdim 1118239462Sdim // Handle unary operators. Again, just aggressively look through 1119239462Sdim // it, ignoring the operation. 1120239462Sdim } else if (const UnaryOperator *uop = dyn_cast<UnaryOperator>(E)) { 1121239462Sdim return isBlockVarRef(uop->getSubExpr()); 1122239462Sdim 1123239462Sdim // Look into the base of a field access. 1124239462Sdim } else if (const MemberExpr *mem = dyn_cast<MemberExpr>(E)) { 1125239462Sdim return isBlockVarRef(mem->getBase()); 1126239462Sdim 1127239462Sdim // Look into the base of a subscript. 1128239462Sdim } else if (const ArraySubscriptExpr *sub = dyn_cast<ArraySubscriptExpr>(E)) { 1129239462Sdim return isBlockVarRef(sub->getBase()); 1130239462Sdim } 1131239462Sdim 1132239462Sdim return false; 1133239462Sdim} 1134239462Sdim 1135193326Sedvoid AggExprEmitter::VisitBinAssign(const BinaryOperator *E) { 1136193326Sed // For an assignment to work, the value on the right has 1137193326Sed // to be compatible with the value on the left. 1138193326Sed assert(CGF.getContext().hasSameUnqualifiedType(E->getLHS()->getType(), 1139193326Sed E->getRHS()->getType()) 1140193326Sed && "Invalid assignment"); 1141218893Sdim 1142239462Sdim // If the LHS might be a __block variable, and the RHS can 1143239462Sdim // potentially cause a block copy, we need to evaluate the RHS first 1144239462Sdim // so that the assignment goes the right place. 1145239462Sdim // This is pretty semantically fragile. 1146239462Sdim if (isBlockVarRef(E->getLHS()) && 1147239462Sdim E->getRHS()->HasSideEffects(CGF.getContext())) { 1148239462Sdim // Ensure that we have a destination, and evaluate the RHS into that. 1149239462Sdim EnsureDest(E->getRHS()->getType()); 1150239462Sdim Visit(E->getRHS()); 1151239462Sdim 1152239462Sdim // Now emit the LHS and copy into it. 1153243830Sdim LValue LHS = CGF.EmitCheckedLValue(E->getLHS(), CodeGenFunction::TCK_Store); 1154239462Sdim 1155249423Sdim // That copy is an atomic copy if the LHS is atomic. 1156288943Sdim if (LHS.getType()->isAtomicType() || 1157288943Sdim CGF.LValueIsSuitableForInlineAtomic(LHS)) { 1158249423Sdim CGF.EmitAtomicStore(Dest.asRValue(), LHS, /*isInit*/ false); 1159249423Sdim return; 1160249423Sdim } 1161249423Sdim 1162239462Sdim EmitCopy(E->getLHS()->getType(), 1163239462Sdim AggValueSlot::forLValue(LHS, AggValueSlot::IsDestructed, 1164239462Sdim needsGC(E->getLHS()->getType()), 1165341825Sdim AggValueSlot::IsAliased, 1166341825Sdim AggValueSlot::MayOverlap), 1167239462Sdim Dest); 1168239462Sdim return; 1169239462Sdim } 1170341825Sdim 1171193326Sed LValue LHS = CGF.EmitLValue(E->getLHS()); 1172193326Sed 1173249423Sdim // If we have an atomic type, evaluate into the destination and then 1174249423Sdim // do an atomic copy. 1175288943Sdim if (LHS.getType()->isAtomicType() || 1176288943Sdim CGF.LValueIsSuitableForInlineAtomic(LHS)) { 1177249423Sdim EnsureDest(E->getRHS()->getType()); 1178249423Sdim Visit(E->getRHS()); 1179249423Sdim CGF.EmitAtomicStore(Dest.asRValue(), LHS, /*isInit*/ false); 1180249423Sdim return; 1181249423Sdim } 1182249423Sdim 1183234353Sdim // Codegen the RHS so that it stores directly into the LHS. 1184234353Sdim AggValueSlot LHSSlot = 1185341825Sdim AggValueSlot::forLValue(LHS, AggValueSlot::IsDestructed, 1186234353Sdim needsGC(E->getLHS()->getType()), 1187341825Sdim AggValueSlot::IsAliased, 1188341825Sdim AggValueSlot::MayOverlap); 1189249423Sdim // A non-volatile aggregate destination might have volatile member. 1190249423Sdim if (!LHSSlot.isVolatile() && 1191249423Sdim CGF.hasVolatileMember(E->getLHS()->getType())) 1192249423Sdim LHSSlot.setVolatile(true); 1193341825Sdim 1194239462Sdim CGF.EmitAggExpr(E->getRHS(), LHSSlot); 1195239462Sdim 1196239462Sdim // Copy into the destination if the assignment isn't ignored. 1197239462Sdim EmitFinalDestCopy(E->getType(), LHS); 1198193326Sed} 1199193326Sed 1200218893Sdimvoid AggExprEmitter:: 1201218893SdimVisitAbstractConditionalOperator(const AbstractConditionalOperator *E) { 1202193326Sed llvm::BasicBlock *LHSBlock = CGF.createBasicBlock("cond.true"); 1203193326Sed llvm::BasicBlock *RHSBlock = CGF.createBasicBlock("cond.false"); 1204193326Sed llvm::BasicBlock *ContBlock = CGF.createBasicBlock("cond.end"); 1205198092Srdivacky 1206218893Sdim // Bind the common expression if necessary. 1207218893Sdim CodeGenFunction::OpaqueValueMapping binding(CGF, E); 1208218893Sdim 1209218893Sdim CodeGenFunction::ConditionalEvaluation eval(CGF); 1210288943Sdim CGF.EmitBranchOnBoolExpr(E->getCond(), LHSBlock, RHSBlock, 1211288943Sdim CGF.getProfileCount(E)); 1212198092Srdivacky 1213218893Sdim // Save whether the destination's lifetime is externally managed. 1214226633Sdim bool isExternallyDestructed = Dest.isExternallyDestructed(); 1215218893Sdim 1216218893Sdim eval.begin(CGF); 1217193326Sed CGF.EmitBlock(LHSBlock); 1218288943Sdim CGF.incrementProfileCounter(E); 1219218893Sdim Visit(E->getTrueExpr()); 1220218893Sdim eval.end(CGF); 1221198092Srdivacky 1222218893Sdim assert(CGF.HaveInsertPoint() && "expression evaluation ended with no IP!"); 1223218893Sdim CGF.Builder.CreateBr(ContBlock); 1224193326Sed 1225218893Sdim // If the result of an agg expression is unused, then the emission 1226218893Sdim // of the LHS might need to create a destination slot. That's fine 1227218893Sdim // with us, and we can safely emit the RHS into the same slot, but 1228226633Sdim // we shouldn't claim that it's already being destructed. 1229226633Sdim Dest.setExternallyDestructed(isExternallyDestructed); 1230198092Srdivacky 1231218893Sdim eval.begin(CGF); 1232193326Sed CGF.EmitBlock(RHSBlock); 1233218893Sdim Visit(E->getFalseExpr()); 1234218893Sdim eval.end(CGF); 1235198092Srdivacky 1236193326Sed CGF.EmitBlock(ContBlock); 1237193326Sed} 1238193326Sed 1239198092Srdivackyvoid AggExprEmitter::VisitChooseExpr(const ChooseExpr *CE) { 1240261991Sdim Visit(CE->getChosenSubExpr()); 1241198092Srdivacky} 1242198092Srdivacky 1243193326Sedvoid AggExprEmitter::VisitVAArgExpr(VAArgExpr *VE) { 1244296417Sdim Address ArgValue = Address::invalid(); 1245296417Sdim Address ArgPtr = CGF.EmitVAArg(VE, ArgValue); 1246193326Sed 1247309124Sdim // If EmitVAArg fails, emit an error. 1248296417Sdim if (!ArgPtr.isValid()) { 1249309124Sdim CGF.ErrorUnsupported(VE, "aggregate va_arg expression"); 1250193326Sed return; 1251193326Sed } 1252193326Sed 1253239462Sdim EmitFinalDestCopy(VE->getType(), CGF.MakeAddrLValue(ArgPtr, VE->getType())); 1254193326Sed} 1255193326Sed 1256193326Sedvoid AggExprEmitter::VisitCXXBindTemporaryExpr(CXXBindTemporaryExpr *E) { 1257218893Sdim // Ensure that we have a slot, but if we already do, remember 1258226633Sdim // whether it was externally destructed. 1259226633Sdim bool wasExternallyDestructed = Dest.isExternallyDestructed(); 1260239462Sdim EnsureDest(E->getType()); 1261198092Srdivacky 1262226633Sdim // We're going to push a destructor if there isn't already one. 1263226633Sdim Dest.setExternallyDestructed(); 1264226633Sdim 1265218893Sdim Visit(E->getSubExpr()); 1266193326Sed 1267226633Sdim // Push that destructor we promised. 1268226633Sdim if (!wasExternallyDestructed) 1269296417Sdim CGF.EmitCXXTemporary(E->getTemporary(), E->getType(), Dest.getAddress()); 1270193326Sed} 1271193326Sed 1272193326Sedvoid 1273193326SedAggExprEmitter::VisitCXXConstructExpr(const CXXConstructExpr *E) { 1274218893Sdim AggValueSlot Slot = EnsureSlot(E->getType()); 1275218893Sdim CGF.EmitCXXConstructExpr(E, Slot); 1276193326Sed} 1277193326Sed 1278309124Sdimvoid AggExprEmitter::VisitCXXInheritedCtorInitExpr( 1279309124Sdim const CXXInheritedCtorInitExpr *E) { 1280309124Sdim AggValueSlot Slot = EnsureSlot(E->getType()); 1281309124Sdim CGF.EmitInheritedCXXConstructorCall( 1282309124Sdim E->getConstructor(), E->constructsVBase(), Slot.getAddress(), 1283309124Sdim E->inheritedFromVBase(), E); 1284309124Sdim} 1285309124Sdim 1286234353Sdimvoid 1287234353SdimAggExprEmitter::VisitLambdaExpr(LambdaExpr *E) { 1288234353Sdim AggValueSlot Slot = EnsureSlot(E->getType()); 1289353358Sdim LValue SlotLV = CGF.MakeAddrLValue(Slot.getAddress(), E->getType()); 1290353358Sdim 1291353358Sdim // We'll need to enter cleanup scopes in case any of the element 1292353358Sdim // initializers throws an exception. 1293353358Sdim SmallVector<EHScopeStack::stable_iterator, 16> Cleanups; 1294353358Sdim llvm::Instruction *CleanupDominator = nullptr; 1295353358Sdim 1296353358Sdim CXXRecordDecl::field_iterator CurField = E->getLambdaClass()->field_begin(); 1297353358Sdim for (LambdaExpr::const_capture_init_iterator i = E->capture_init_begin(), 1298353358Sdim e = E->capture_init_end(); 1299353358Sdim i != e; ++i, ++CurField) { 1300353358Sdim // Emit initialization 1301353358Sdim LValue LV = CGF.EmitLValueForFieldInitialization(SlotLV, *CurField); 1302353358Sdim if (CurField->hasCapturedVLAType()) { 1303353358Sdim CGF.EmitLambdaVLACapture(CurField->getCapturedVLAType(), LV); 1304353358Sdim continue; 1305353358Sdim } 1306353358Sdim 1307353358Sdim EmitInitializationToLValue(*i, LV); 1308353358Sdim 1309353358Sdim // Push a destructor if necessary. 1310353358Sdim if (QualType::DestructionKind DtorKind = 1311353358Sdim CurField->getType().isDestructedType()) { 1312353358Sdim assert(LV.isSimple()); 1313353358Sdim if (CGF.needsEHCleanup(DtorKind)) { 1314353358Sdim if (!CleanupDominator) 1315353358Sdim CleanupDominator = CGF.Builder.CreateAlignedLoad( 1316353358Sdim CGF.Int8Ty, 1317353358Sdim llvm::Constant::getNullValue(CGF.Int8PtrTy), 1318353358Sdim CharUnits::One()); // placeholder 1319353358Sdim 1320353358Sdim CGF.pushDestroy(EHCleanup, LV.getAddress(), CurField->getType(), 1321353358Sdim CGF.getDestroyer(DtorKind), false); 1322353358Sdim Cleanups.push_back(CGF.EHStack.stable_begin()); 1323353358Sdim } 1324353358Sdim } 1325353358Sdim } 1326353358Sdim 1327353358Sdim // Deactivate all the partial cleanups in reverse order, which 1328353358Sdim // generally means popping them. 1329353358Sdim for (unsigned i = Cleanups.size(); i != 0; --i) 1330353358Sdim CGF.DeactivateCleanupBlock(Cleanups[i-1], CleanupDominator); 1331353358Sdim 1332353358Sdim // Destroy the placeholder if we made one. 1333353358Sdim if (CleanupDominator) 1334353358Sdim CleanupDominator->eraseFromParent(); 1335234353Sdim} 1336234353Sdim 1337218893Sdimvoid AggExprEmitter::VisitExprWithCleanups(ExprWithCleanups *E) { 1338234353Sdim CGF.enterFullExpression(E); 1339234353Sdim CodeGenFunction::RunCleanupsScope cleanups(CGF); 1340234353Sdim Visit(E->getSubExpr()); 1341193326Sed} 1342193326Sed 1343210299Sedvoid AggExprEmitter::VisitCXXScalarValueInitExpr(CXXScalarValueInitExpr *E) { 1344218893Sdim QualType T = E->getType(); 1345218893Sdim AggValueSlot Slot = EnsureSlot(T); 1346296417Sdim EmitNullInitializationToLValue(CGF.MakeAddrLValue(Slot.getAddress(), T)); 1347198398Srdivacky} 1348198398Srdivacky 1349201361Srdivackyvoid AggExprEmitter::VisitImplicitValueInitExpr(ImplicitValueInitExpr *E) { 1350218893Sdim QualType T = E->getType(); 1351218893Sdim AggValueSlot Slot = EnsureSlot(T); 1352296417Sdim EmitNullInitializationToLValue(CGF.MakeAddrLValue(Slot.getAddress(), T)); 1353218893Sdim} 1354201361Srdivacky 1355218893Sdim/// isSimpleZero - If emitting this value will obviously just cause a store of 1356218893Sdim/// zero to memory, return true. This can return false if uncertain, so it just 1357218893Sdim/// handles simple cases. 1358218893Sdimstatic bool isSimpleZero(const Expr *E, CodeGenFunction &CGF) { 1359221345Sdim E = E->IgnoreParens(); 1360221345Sdim 1361218893Sdim // 0 1362218893Sdim if (const IntegerLiteral *IL = dyn_cast<IntegerLiteral>(E)) 1363218893Sdim return IL->getValue() == 0; 1364218893Sdim // +0.0 1365218893Sdim if (const FloatingLiteral *FL = dyn_cast<FloatingLiteral>(E)) 1366218893Sdim return FL->getValue().isPosZero(); 1367218893Sdim // int() 1368218893Sdim if ((isa<ImplicitValueInitExpr>(E) || isa<CXXScalarValueInitExpr>(E)) && 1369218893Sdim CGF.getTypes().isZeroInitializable(E->getType())) 1370218893Sdim return true; 1371218893Sdim // (int*)0 - Null pointer expressions. 1372218893Sdim if (const CastExpr *ICE = dyn_cast<CastExpr>(E)) 1373314564Sdim return ICE->getCastKind() == CK_NullToPointer && 1374353358Sdim CGF.getTypes().isPointerZeroInitializable(E->getType()) && 1375353358Sdim !E->HasSideEffects(CGF.getContext()); 1376218893Sdim // '\0' 1377218893Sdim if (const CharacterLiteral *CL = dyn_cast<CharacterLiteral>(E)) 1378218893Sdim return CL->getValue() == 0; 1379341825Sdim 1380218893Sdim // Otherwise, hard case: conservatively return false. 1381218893Sdim return false; 1382201361Srdivacky} 1383201361Srdivacky 1384218893Sdim 1385341825Sdimvoid 1386261991SdimAggExprEmitter::EmitInitializationToLValue(Expr *E, LValue LV) { 1387224145Sdim QualType type = LV.getType(); 1388193326Sed // FIXME: Ignore result? 1389193326Sed // FIXME: Are initializers affected by volatile? 1390218893Sdim if (Dest.isZeroed() && isSimpleZero(E, CGF)) { 1391218893Sdim // Storing "i32 0" to a zero'd memory location is a noop. 1392249423Sdim return; 1393249423Sdim } else if (isa<ImplicitValueInitExpr>(E) || isa<CXXScalarValueInitExpr>(E)) { 1394249423Sdim return EmitNullInitializationToLValue(LV); 1395288943Sdim } else if (isa<NoInitExpr>(E)) { 1396288943Sdim // Do nothing. 1397288943Sdim return; 1398224145Sdim } else if (type->isReferenceType()) { 1399261991Sdim RValue RV = CGF.EmitReferenceBindingToExpr(E); 1400249423Sdim return CGF.EmitStoreThroughLValue(RV, LV); 1401249423Sdim } 1402341825Sdim 1403249423Sdim switch (CGF.getEvaluationKind(type)) { 1404249423Sdim case TEK_Complex: 1405249423Sdim CGF.EmitComplexExprIntoLValue(E, LV, /*isInit*/ true); 1406249423Sdim return; 1407249423Sdim case TEK_Aggregate: 1408226633Sdim CGF.EmitAggExpr(E, AggValueSlot::forLValue(LV, 1409226633Sdim AggValueSlot::IsDestructed, 1410226633Sdim AggValueSlot::DoesNotNeedGCBarriers, 1411226633Sdim AggValueSlot::IsNotAliased, 1412341825Sdim AggValueSlot::MayOverlap, 1413224145Sdim Dest.isZeroed())); 1414249423Sdim return; 1415249423Sdim case TEK_Scalar: 1416249423Sdim if (LV.isSimple()) { 1417276479Sdim CGF.EmitScalarInit(E, /*D=*/nullptr, LV, /*Captured=*/false); 1418249423Sdim } else { 1419249423Sdim CGF.EmitStoreThroughLValue(RValue::get(CGF.EmitScalarExpr(E)), LV); 1420249423Sdim } 1421249423Sdim return; 1422193326Sed } 1423249423Sdim llvm_unreachable("bad evaluation kind"); 1424193326Sed} 1425193326Sed 1426224145Sdimvoid AggExprEmitter::EmitNullInitializationToLValue(LValue lv) { 1427224145Sdim QualType type = lv.getType(); 1428224145Sdim 1429218893Sdim // If the destination slot is already zeroed out before the aggregate is 1430218893Sdim // copied into it, we don't have to emit any zeros here. 1431224145Sdim if (Dest.isZeroed() && CGF.getTypes().isZeroInitializable(type)) 1432218893Sdim return; 1433341825Sdim 1434249423Sdim if (CGF.hasScalarEvaluationKind(type)) { 1435249423Sdim // For non-aggregates, we can store the appropriate null constant. 1436249423Sdim llvm::Value *null = CGF.CGM.EmitNullConstant(type); 1437234353Sdim // Note that the following is not equivalent to 1438234353Sdim // EmitStoreThroughBitfieldLValue for ARC types. 1439234353Sdim if (lv.isBitField()) { 1440234353Sdim CGF.EmitStoreThroughBitfieldLValue(RValue::get(null), lv); 1441234353Sdim } else { 1442234353Sdim assert(lv.isSimple()); 1443234353Sdim CGF.EmitStoreOfScalar(null, lv, /* isInitialization */ true); 1444234353Sdim } 1445193326Sed } else { 1446193326Sed // There's a potential optimization opportunity in combining 1447193326Sed // memsets; that would be easy for arrays, but relatively 1448193326Sed // difficult for structures with the current code. 1449224145Sdim CGF.EmitNullInitialization(lv.getAddress(), lv.getType()); 1450193326Sed } 1451193326Sed} 1452193326Sed 1453193326Sedvoid AggExprEmitter::VisitInitListExpr(InitListExpr *E) { 1454193326Sed#if 0 1455200583Srdivacky // FIXME: Assess perf here? Figure out what cases are worth optimizing here 1456200583Srdivacky // (Length of globals? Chunks of zeroed-out space?). 1457193326Sed // 1458193326Sed // If we can, prefer a copy from a global; this is a lot less code for long 1459193326Sed // globals, and it's easier for the current optimizers to analyze. 1460200583Srdivacky if (llvm::Constant* C = CGF.CGM.EmitConstantExpr(E, E->getType(), &CGF)) { 1461193326Sed llvm::GlobalVariable* GV = 1462200583Srdivacky new llvm::GlobalVariable(CGF.CGM.getModule(), C->getType(), true, 1463200583Srdivacky llvm::GlobalValue::InternalLinkage, C, ""); 1464239462Sdim EmitFinalDestCopy(E->getType(), CGF.MakeAddrLValue(GV, E->getType())); 1465193326Sed return; 1466193326Sed } 1467193326Sed#endif 1468218893Sdim if (E->hadArrayRangeDesignator()) 1469193326Sed CGF.ErrorUnsupported(E, "GNU array range designator extension"); 1470193326Sed 1471314564Sdim if (E->isTransparent()) 1472314564Sdim return Visit(E->getInit(0)); 1473314564Sdim 1474261991Sdim AggValueSlot Dest = EnsureSlot(E->getType()); 1475218893Sdim 1476296417Sdim LValue DestLV = CGF.MakeAddrLValue(Dest.getAddress(), E->getType()); 1477234353Sdim 1478193326Sed // Handle initialization of an array. 1479193326Sed if (E->getType()->isArrayType()) { 1480296417Sdim auto AType = cast<llvm::ArrayType>(Dest.getAddress().getElementType()); 1481338697Sdim EmitArrayInit(Dest.getAddress(), AType, E->getType(), E); 1482193326Sed return; 1483193326Sed } 1484198092Srdivacky 1485193326Sed assert(E->getType()->isRecordType() && "Only support structs/unions here!"); 1486198092Srdivacky 1487193326Sed // Do struct initialization; this code just sets each individual member 1488193326Sed // to the approprate value. This makes bitfield support automatic; 1489193326Sed // the disadvantage is that the generated code is more difficult for 1490193326Sed // the optimizer, especially with bitfields. 1491193326Sed unsigned NumInitElements = E->getNumInits(); 1492224145Sdim RecordDecl *record = E->getType()->castAs<RecordType>()->getDecl(); 1493251662Sdim 1494309124Sdim // We'll need to enter cleanup scopes in case any of the element 1495309124Sdim // initializers throws an exception. 1496309124Sdim SmallVector<EHScopeStack::stable_iterator, 16> cleanups; 1497309124Sdim llvm::Instruction *cleanupDominator = nullptr; 1498353358Sdim auto addCleanup = [&](const EHScopeStack::stable_iterator &cleanup) { 1499353358Sdim cleanups.push_back(cleanup); 1500353358Sdim if (!cleanupDominator) // create placeholder once needed 1501353358Sdim cleanupDominator = CGF.Builder.CreateAlignedLoad( 1502353358Sdim CGF.Int8Ty, llvm::Constant::getNullValue(CGF.Int8PtrTy), 1503353358Sdim CharUnits::One()); 1504353358Sdim }; 1505309124Sdim 1506309124Sdim unsigned curInitIndex = 0; 1507309124Sdim 1508309124Sdim // Emit initialization of base classes. 1509309124Sdim if (auto *CXXRD = dyn_cast<CXXRecordDecl>(record)) { 1510309124Sdim assert(E->getNumInits() >= CXXRD->getNumBases() && 1511309124Sdim "missing initializer for base class"); 1512309124Sdim for (auto &Base : CXXRD->bases()) { 1513309124Sdim assert(!Base.isVirtual() && "should not see vbases here"); 1514309124Sdim auto *BaseRD = Base.getType()->getAsCXXRecordDecl(); 1515309124Sdim Address V = CGF.GetAddressOfDirectBaseInCompleteClass( 1516309124Sdim Dest.getAddress(), CXXRD, BaseRD, 1517309124Sdim /*isBaseVirtual*/ false); 1518341825Sdim AggValueSlot AggSlot = AggValueSlot::forAddr( 1519341825Sdim V, Qualifiers(), 1520341825Sdim AggValueSlot::IsDestructed, 1521341825Sdim AggValueSlot::DoesNotNeedGCBarriers, 1522341825Sdim AggValueSlot::IsNotAliased, 1523353358Sdim CGF.getOverlapForBaseInit(CXXRD, BaseRD, Base.isVirtual())); 1524309124Sdim CGF.EmitAggExpr(E->getInit(curInitIndex++), AggSlot); 1525309124Sdim 1526309124Sdim if (QualType::DestructionKind dtorKind = 1527309124Sdim Base.getType().isDestructedType()) { 1528309124Sdim CGF.pushDestroy(dtorKind, V, Base.getType()); 1529353358Sdim addCleanup(CGF.EHStack.stable_begin()); 1530309124Sdim } 1531309124Sdim } 1532309124Sdim } 1533309124Sdim 1534251662Sdim // Prepare a 'this' for CXXDefaultInitExprs. 1535296417Sdim CodeGenFunction::FieldConstructionScope FCS(CGF, Dest.getAddress()); 1536251662Sdim 1537224145Sdim if (record->isUnion()) { 1538193326Sed // Only initialize one field of a union. The field itself is 1539193326Sed // specified by the initializer list. 1540193326Sed if (!E->getInitializedFieldInUnion()) { 1541193326Sed // Empty union; we have nothing to do. 1542198092Srdivacky 1543193326Sed#ifndef NDEBUG 1544193326Sed // Make sure that it's really an empty and not a failure of 1545193326Sed // semantic analysis. 1546276479Sdim for (const auto *Field : record->fields()) 1547193326Sed assert(Field->isUnnamedBitfield() && "Only unnamed bitfields allowed"); 1548193326Sed#endif 1549193326Sed return; 1550193326Sed } 1551193326Sed 1552193326Sed // FIXME: volatility 1553193326Sed FieldDecl *Field = E->getInitializedFieldInUnion(); 1554218893Sdim 1555234982Sdim LValue FieldLoc = CGF.EmitLValueForFieldInitialization(DestLV, Field); 1556193326Sed if (NumInitElements) { 1557193326Sed // Store the initializer into the field 1558224145Sdim EmitInitializationToLValue(E->getInit(0), FieldLoc); 1559193326Sed } else { 1560218893Sdim // Default-initialize to null. 1561224145Sdim EmitNullInitializationToLValue(FieldLoc); 1562193326Sed } 1563193326Sed 1564193326Sed return; 1565193326Sed } 1566198092Srdivacky 1567193326Sed // Here we iterate over the fields; this makes it simpler to both 1568193326Sed // default-initialize fields and skip over unnamed fields. 1569276479Sdim for (const auto *field : record->fields()) { 1570224145Sdim // We're done once we hit the flexible array member. 1571224145Sdim if (field->getType()->isIncompleteArrayType()) 1572193326Sed break; 1573193326Sed 1574224145Sdim // Always skip anonymous bitfields. 1575224145Sdim if (field->isUnnamedBitfield()) 1576193326Sed continue; 1577193326Sed 1578224145Sdim // We're done if we reach the end of the explicit initializers, we 1579224145Sdim // have a zeroed object, and the rest of the fields are 1580224145Sdim // zero-initializable. 1581224145Sdim if (curInitIndex == NumInitElements && Dest.isZeroed() && 1582218893Sdim CGF.getTypes().isZeroInitializable(E->getType())) 1583218893Sdim break; 1584234982Sdim 1585341825Sdim 1586276479Sdim LValue LV = CGF.EmitLValueForFieldInitialization(DestLV, field); 1587193326Sed // We never generate write-barries for initialized fields. 1588224145Sdim LV.setNonGC(true); 1589341825Sdim 1590224145Sdim if (curInitIndex < NumInitElements) { 1591204962Srdivacky // Store the initializer into the field. 1592224145Sdim EmitInitializationToLValue(E->getInit(curInitIndex++), LV); 1593193326Sed } else { 1594321369Sdim // We're out of initializers; default-initialize to null 1595224145Sdim EmitNullInitializationToLValue(LV); 1596193326Sed } 1597224145Sdim 1598224145Sdim // Push a destructor if necessary. 1599224145Sdim // FIXME: if we have an array of structures, all explicitly 1600224145Sdim // initialized, we can end up pushing a linear number of cleanups. 1601224145Sdim bool pushedCleanup = false; 1602224145Sdim if (QualType::DestructionKind dtorKind 1603224145Sdim = field->getType().isDestructedType()) { 1604224145Sdim assert(LV.isSimple()); 1605224145Sdim if (CGF.needsEHCleanup(dtorKind)) { 1606224145Sdim CGF.pushDestroy(EHCleanup, LV.getAddress(), field->getType(), 1607224145Sdim CGF.getDestroyer(dtorKind), false); 1608353358Sdim addCleanup(CGF.EHStack.stable_begin()); 1609224145Sdim pushedCleanup = true; 1610224145Sdim } 1611224145Sdim } 1612341825Sdim 1613218893Sdim // If the GEP didn't get used because of a dead zero init or something 1614218893Sdim // else, clean it up for -O0 builds and general tidiness. 1615341825Sdim if (!pushedCleanup && LV.isSimple()) 1616218893Sdim if (llvm::GetElementPtrInst *GEP = 1617296417Sdim dyn_cast<llvm::GetElementPtrInst>(LV.getPointer())) 1618218893Sdim if (GEP->use_empty()) 1619218893Sdim GEP->eraseFromParent(); 1620193326Sed } 1621224145Sdim 1622224145Sdim // Deactivate all the partial cleanups in reverse order, which 1623224145Sdim // generally means popping them. 1624353358Sdim assert((cleanupDominator || cleanups.empty()) && 1625353358Sdim "Missing cleanupDominator before deactivating cleanup blocks"); 1626224145Sdim for (unsigned i = cleanups.size(); i != 0; --i) 1627234353Sdim CGF.DeactivateCleanupBlock(cleanups[i-1], cleanupDominator); 1628234353Sdim 1629234353Sdim // Destroy the placeholder if we made one. 1630234353Sdim if (cleanupDominator) 1631234353Sdim cleanupDominator->eraseFromParent(); 1632193326Sed} 1633193326Sed 1634314564Sdimvoid AggExprEmitter::VisitArrayInitLoopExpr(const ArrayInitLoopExpr *E, 1635314564Sdim llvm::Value *outerBegin) { 1636314564Sdim // Emit the common subexpression. 1637314564Sdim CodeGenFunction::OpaqueValueMapping binding(CGF, E->getCommonExpr()); 1638314564Sdim 1639314564Sdim Address destPtr = EnsureSlot(E->getType()).getAddress(); 1640314564Sdim uint64_t numElements = E->getArraySize().getZExtValue(); 1641314564Sdim 1642314564Sdim if (!numElements) 1643314564Sdim return; 1644314564Sdim 1645314564Sdim // destPtr is an array*. Construct an elementType* by drilling down a level. 1646314564Sdim llvm::Value *zero = llvm::ConstantInt::get(CGF.SizeTy, 0); 1647314564Sdim llvm::Value *indices[] = {zero, zero}; 1648314564Sdim llvm::Value *begin = Builder.CreateInBoundsGEP(destPtr.getPointer(), indices, 1649314564Sdim "arrayinit.begin"); 1650314564Sdim 1651314564Sdim // Prepare to special-case multidimensional array initialization: we avoid 1652314564Sdim // emitting multiple destructor loops in that case. 1653314564Sdim if (!outerBegin) 1654314564Sdim outerBegin = begin; 1655314564Sdim ArrayInitLoopExpr *InnerLoop = dyn_cast<ArrayInitLoopExpr>(E->getSubExpr()); 1656314564Sdim 1657314564Sdim QualType elementType = 1658314564Sdim CGF.getContext().getAsArrayType(E->getType())->getElementType(); 1659314564Sdim CharUnits elementSize = CGF.getContext().getTypeSizeInChars(elementType); 1660314564Sdim CharUnits elementAlign = 1661314564Sdim destPtr.getAlignment().alignmentOfArrayElement(elementSize); 1662314564Sdim 1663314564Sdim llvm::BasicBlock *entryBB = Builder.GetInsertBlock(); 1664314564Sdim llvm::BasicBlock *bodyBB = CGF.createBasicBlock("arrayinit.body"); 1665314564Sdim 1666314564Sdim // Jump into the body. 1667314564Sdim CGF.EmitBlock(bodyBB); 1668314564Sdim llvm::PHINode *index = 1669314564Sdim Builder.CreatePHI(zero->getType(), 2, "arrayinit.index"); 1670314564Sdim index->addIncoming(zero, entryBB); 1671314564Sdim llvm::Value *element = Builder.CreateInBoundsGEP(begin, index); 1672314564Sdim 1673314564Sdim // Prepare for a cleanup. 1674314564Sdim QualType::DestructionKind dtorKind = elementType.isDestructedType(); 1675314564Sdim EHScopeStack::stable_iterator cleanup; 1676314564Sdim if (CGF.needsEHCleanup(dtorKind) && !InnerLoop) { 1677314564Sdim if (outerBegin->getType() != element->getType()) 1678314564Sdim outerBegin = Builder.CreateBitCast(outerBegin, element->getType()); 1679314564Sdim CGF.pushRegularPartialArrayCleanup(outerBegin, element, elementType, 1680314564Sdim elementAlign, 1681314564Sdim CGF.getDestroyer(dtorKind)); 1682314564Sdim cleanup = CGF.EHStack.stable_begin(); 1683314564Sdim } else { 1684314564Sdim dtorKind = QualType::DK_none; 1685314564Sdim } 1686314564Sdim 1687314564Sdim // Emit the actual filler expression. 1688314564Sdim { 1689314564Sdim // Temporaries created in an array initialization loop are destroyed 1690314564Sdim // at the end of each iteration. 1691314564Sdim CodeGenFunction::RunCleanupsScope CleanupsScope(CGF); 1692314564Sdim CodeGenFunction::ArrayInitLoopExprScope Scope(CGF, index); 1693314564Sdim LValue elementLV = 1694314564Sdim CGF.MakeAddrLValue(Address(element, elementAlign), elementType); 1695314564Sdim 1696314564Sdim if (InnerLoop) { 1697314564Sdim // If the subexpression is an ArrayInitLoopExpr, share its cleanup. 1698314564Sdim auto elementSlot = AggValueSlot::forLValue( 1699314564Sdim elementLV, AggValueSlot::IsDestructed, 1700341825Sdim AggValueSlot::DoesNotNeedGCBarriers, 1701341825Sdim AggValueSlot::IsNotAliased, 1702341825Sdim AggValueSlot::DoesNotOverlap); 1703314564Sdim AggExprEmitter(CGF, elementSlot, false) 1704314564Sdim .VisitArrayInitLoopExpr(InnerLoop, outerBegin); 1705314564Sdim } else 1706314564Sdim EmitInitializationToLValue(E->getSubExpr(), elementLV); 1707314564Sdim } 1708314564Sdim 1709314564Sdim // Move on to the next element. 1710314564Sdim llvm::Value *nextIndex = Builder.CreateNUWAdd( 1711314564Sdim index, llvm::ConstantInt::get(CGF.SizeTy, 1), "arrayinit.next"); 1712314564Sdim index->addIncoming(nextIndex, Builder.GetInsertBlock()); 1713314564Sdim 1714314564Sdim // Leave the loop if we're done. 1715314564Sdim llvm::Value *done = Builder.CreateICmpEQ( 1716314564Sdim nextIndex, llvm::ConstantInt::get(CGF.SizeTy, numElements), 1717314564Sdim "arrayinit.done"); 1718314564Sdim llvm::BasicBlock *endBB = CGF.createBasicBlock("arrayinit.end"); 1719314564Sdim Builder.CreateCondBr(done, endBB, bodyBB); 1720314564Sdim 1721314564Sdim CGF.EmitBlock(endBB); 1722314564Sdim 1723314564Sdim // Leave the partial-array cleanup if we entered one. 1724314564Sdim if (dtorKind) 1725314564Sdim CGF.DeactivateCleanupBlock(cleanup, index); 1726314564Sdim} 1727314564Sdim 1728288943Sdimvoid AggExprEmitter::VisitDesignatedInitUpdateExpr(DesignatedInitUpdateExpr *E) { 1729288943Sdim AggValueSlot Dest = EnsureSlot(E->getType()); 1730288943Sdim 1731296417Sdim LValue DestLV = CGF.MakeAddrLValue(Dest.getAddress(), E->getType()); 1732288943Sdim EmitInitializationToLValue(E->getBase(), DestLV); 1733288943Sdim VisitInitListExpr(E->getUpdater()); 1734288943Sdim} 1735288943Sdim 1736193326Sed//===----------------------------------------------------------------------===// 1737193326Sed// Entry Points into this File 1738193326Sed//===----------------------------------------------------------------------===// 1739193326Sed 1740218893Sdim/// GetNumNonZeroBytesInInit - Get an approximate count of the number of 1741218893Sdim/// non-zero bytes that will be stored when outputting the initializer for the 1742218893Sdim/// specified initializer expression. 1743221345Sdimstatic CharUnits GetNumNonZeroBytesInInit(const Expr *E, CodeGenFunction &CGF) { 1744221345Sdim E = E->IgnoreParens(); 1745218893Sdim 1746218893Sdim // 0 and 0.0 won't require any non-zero stores! 1747221345Sdim if (isSimpleZero(E, CGF)) return CharUnits::Zero(); 1748218893Sdim 1749218893Sdim // If this is an initlist expr, sum up the size of sizes of the (present) 1750218893Sdim // elements. If this is something weird, assume the whole thing is non-zero. 1751218893Sdim const InitListExpr *ILE = dyn_cast<InitListExpr>(E); 1752341825Sdim while (ILE && ILE->isTransparent()) 1753341825Sdim ILE = dyn_cast<InitListExpr>(ILE->getInit(0)); 1754276479Sdim if (!ILE || !CGF.getTypes().isZeroInitializable(ILE->getType())) 1755221345Sdim return CGF.getContext().getTypeSizeInChars(E->getType()); 1756341825Sdim 1757218893Sdim // InitListExprs for structs have to be handled carefully. If there are 1758218893Sdim // reference members, we need to consider the size of the reference, not the 1759218893Sdim // referencee. InitListExprs for unions and arrays can't have references. 1760218893Sdim if (const RecordType *RT = E->getType()->getAs<RecordType>()) { 1761218893Sdim if (!RT->isUnionType()) { 1762218893Sdim RecordDecl *SD = E->getType()->getAs<RecordType>()->getDecl(); 1763221345Sdim CharUnits NumNonZeroBytes = CharUnits::Zero(); 1764341825Sdim 1765218893Sdim unsigned ILEElement = 0; 1766309124Sdim if (auto *CXXRD = dyn_cast<CXXRecordDecl>(SD)) 1767309124Sdim while (ILEElement != CXXRD->getNumBases()) 1768309124Sdim NumNonZeroBytes += 1769309124Sdim GetNumNonZeroBytesInInit(ILE->getInit(ILEElement++), CGF); 1770276479Sdim for (const auto *Field : SD->fields()) { 1771218893Sdim // We're done once we hit the flexible array member or run out of 1772218893Sdim // InitListExpr elements. 1773218893Sdim if (Field->getType()->isIncompleteArrayType() || 1774218893Sdim ILEElement == ILE->getNumInits()) 1775218893Sdim break; 1776218893Sdim if (Field->isUnnamedBitfield()) 1777218893Sdim continue; 1778218893Sdim 1779218893Sdim const Expr *E = ILE->getInit(ILEElement++); 1780341825Sdim 1781218893Sdim // Reference values are always non-null and have the width of a pointer. 1782218893Sdim if (Field->getType()->isReferenceType()) 1783221345Sdim NumNonZeroBytes += CGF.getContext().toCharUnitsFromBits( 1784251662Sdim CGF.getTarget().getPointerWidth(0)); 1785218893Sdim else 1786218893Sdim NumNonZeroBytes += GetNumNonZeroBytesInInit(E, CGF); 1787218893Sdim } 1788341825Sdim 1789218893Sdim return NumNonZeroBytes; 1790218893Sdim } 1791218893Sdim } 1792341825Sdim 1793341825Sdim 1794221345Sdim CharUnits NumNonZeroBytes = CharUnits::Zero(); 1795218893Sdim for (unsigned i = 0, e = ILE->getNumInits(); i != e; ++i) 1796218893Sdim NumNonZeroBytes += GetNumNonZeroBytesInInit(ILE->getInit(i), CGF); 1797218893Sdim return NumNonZeroBytes; 1798218893Sdim} 1799218893Sdim 1800218893Sdim/// CheckAggExprForMemSetUse - If the initializer is large and has a lot of 1801218893Sdim/// zeros in it, emit a memset and avoid storing the individual zeros. 1802218893Sdim/// 1803218893Sdimstatic void CheckAggExprForMemSetUse(AggValueSlot &Slot, const Expr *E, 1804218893Sdim CodeGenFunction &CGF) { 1805218893Sdim // If the slot is already known to be zeroed, nothing to do. Don't mess with 1806218893Sdim // volatile stores. 1807296417Sdim if (Slot.isZeroed() || Slot.isVolatile() || !Slot.getAddress().isValid()) 1808276479Sdim return; 1809221345Sdim 1810221345Sdim // C++ objects with a user-declared constructor don't need zero'ing. 1811243830Sdim if (CGF.getLangOpts().CPlusPlus) 1812221345Sdim if (const RecordType *RT = CGF.getContext() 1813221345Sdim .getBaseElementType(E->getType())->getAs<RecordType>()) { 1814221345Sdim const CXXRecordDecl *RD = cast<CXXRecordDecl>(RT->getDecl()); 1815221345Sdim if (RD->hasUserDeclaredConstructor()) 1816221345Sdim return; 1817221345Sdim } 1818221345Sdim 1819218893Sdim // If the type is 16-bytes or smaller, prefer individual stores over memset. 1820341825Sdim CharUnits Size = Slot.getPreferredSize(CGF.getContext(), E->getType()); 1821296417Sdim if (Size <= CharUnits::fromQuantity(16)) 1822218893Sdim return; 1823218893Sdim 1824218893Sdim // Check to see if over 3/4 of the initializer are known to be zero. If so, 1825218893Sdim // we prefer to emit memset + individual stores for the rest. 1826221345Sdim CharUnits NumNonZeroBytes = GetNumNonZeroBytesInInit(E, CGF); 1827296417Sdim if (NumNonZeroBytes*4 > Size) 1828218893Sdim return; 1829341825Sdim 1830218893Sdim // Okay, it seems like a good idea to use an initial memset, emit the call. 1831296417Sdim llvm::Constant *SizeVal = CGF.Builder.getInt64(Size.getQuantity()); 1832218893Sdim 1833341825Sdim Address Loc = Slot.getAddress(); 1834296417Sdim Loc = CGF.Builder.CreateElementBitCast(Loc, CGF.Int8Ty); 1835296417Sdim CGF.Builder.CreateMemSet(Loc, CGF.Builder.getInt8(0), SizeVal, false); 1836341825Sdim 1837218893Sdim // Tell the AggExprEmitter that the slot is known zero. 1838218893Sdim Slot.setZeroed(); 1839218893Sdim} 1840218893Sdim 1841218893Sdim 1842218893Sdim 1843218893Sdim 1844193326Sed/// EmitAggExpr - Emit the computation of the specified expression of aggregate 1845193326Sed/// type. The result is computed into DestPtr. Note that if DestPtr is null, 1846193326Sed/// the value of the aggregate expression is not needed. If VolatileDest is 1847193326Sed/// true, DestPtr cannot be 0. 1848239462Sdimvoid CodeGenFunction::EmitAggExpr(const Expr *E, AggValueSlot Slot) { 1849249423Sdim assert(E && hasAggregateEvaluationKind(E->getType()) && 1850193326Sed "Invalid aggregate expression to emit"); 1851296417Sdim assert((Slot.getAddress().isValid() || Slot.isIgnored()) && 1852218893Sdim "slot has bits but no address"); 1853198092Srdivacky 1854218893Sdim // Optimize the slot if possible. 1855218893Sdim CheckAggExprForMemSetUse(Slot, E, *this); 1856341825Sdim 1857288943Sdim AggExprEmitter(*this, Slot, Slot.isIgnored()).Visit(const_cast<Expr*>(E)); 1858193326Sed} 1859193326Sed 1860203955SrdivackyLValue CodeGenFunction::EmitAggExprToLValue(const Expr *E) { 1861249423Sdim assert(hasAggregateEvaluationKind(E->getType()) && "Invalid argument!"); 1862296417Sdim Address Temp = CreateMemTemp(E->getType()); 1863212904Sdim LValue LV = MakeAddrLValue(Temp, E->getType()); 1864226633Sdim EmitAggExpr(E, AggValueSlot::forLValue(LV, AggValueSlot::IsNotDestructed, 1865226633Sdim AggValueSlot::DoesNotNeedGCBarriers, 1866341825Sdim AggValueSlot::IsNotAliased, 1867341825Sdim AggValueSlot::DoesNotOverlap)); 1868212904Sdim return LV; 1869203955Srdivacky} 1870203955Srdivacky 1871353358SdimAggValueSlot::Overlap_t 1872353358SdimCodeGenFunction::getOverlapForFieldInit(const FieldDecl *FD) { 1873353358Sdim if (!FD->hasAttr<NoUniqueAddressAttr>() || !FD->getType()->isRecordType()) 1874353358Sdim return AggValueSlot::DoesNotOverlap; 1875353358Sdim 1876353358Sdim // If the field lies entirely within the enclosing class's nvsize, its tail 1877353358Sdim // padding cannot overlap any already-initialized object. (The only subobjects 1878353358Sdim // with greater addresses that might already be initialized are vbases.) 1879353358Sdim const RecordDecl *ClassRD = FD->getParent(); 1880353358Sdim const ASTRecordLayout &Layout = getContext().getASTRecordLayout(ClassRD); 1881353358Sdim if (Layout.getFieldOffset(FD->getFieldIndex()) + 1882353358Sdim getContext().getTypeSize(FD->getType()) <= 1883353358Sdim (uint64_t)getContext().toBits(Layout.getNonVirtualSize())) 1884353358Sdim return AggValueSlot::DoesNotOverlap; 1885353358Sdim 1886353358Sdim // The tail padding may contain values we need to preserve. 1887353358Sdim return AggValueSlot::MayOverlap; 1888353358Sdim} 1889353358Sdim 1890353358SdimAggValueSlot::Overlap_t CodeGenFunction::getOverlapForBaseInit( 1891341825Sdim const CXXRecordDecl *RD, const CXXRecordDecl *BaseRD, bool IsVirtual) { 1892353358Sdim // If the most-derived object is a field declared with [[no_unique_address]], 1893353358Sdim // the tail padding of any virtual base could be reused for other subobjects 1894353358Sdim // of that field's class. 1895341825Sdim if (IsVirtual) 1896353358Sdim return AggValueSlot::MayOverlap; 1897341825Sdim 1898341825Sdim // If the base class is laid out entirely within the nvsize of the derived 1899341825Sdim // class, its tail padding cannot yet be initialized, so we can issue 1900341825Sdim // stores at the full width of the base class. 1901341825Sdim const ASTRecordLayout &Layout = getContext().getASTRecordLayout(RD); 1902341825Sdim if (Layout.getBaseClassOffset(BaseRD) + 1903341825Sdim getContext().getASTRecordLayout(BaseRD).getSize() <= 1904341825Sdim Layout.getNonVirtualSize()) 1905341825Sdim return AggValueSlot::DoesNotOverlap; 1906341825Sdim 1907341825Sdim // The tail padding may contain values we need to preserve. 1908341825Sdim return AggValueSlot::MayOverlap; 1909341825Sdim} 1910341825Sdim 1911341825Sdimvoid CodeGenFunction::EmitAggregateCopy(LValue Dest, LValue Src, QualType Ty, 1912341825Sdim AggValueSlot::Overlap_t MayOverlap, 1913341825Sdim bool isVolatile) { 1914193326Sed assert(!Ty->isAnyComplexType() && "Shouldn't happen for complex"); 1915198092Srdivacky 1916341825Sdim Address DestPtr = Dest.getAddress(); 1917341825Sdim Address SrcPtr = Src.getAddress(); 1918341825Sdim 1919243830Sdim if (getLangOpts().CPlusPlus) { 1920207619Srdivacky if (const RecordType *RT = Ty->getAs<RecordType>()) { 1921208600Srdivacky CXXRecordDecl *Record = cast<CXXRecordDecl>(RT->getDecl()); 1922341825Sdim assert((Record->hasTrivialCopyConstructor() || 1923226633Sdim Record->hasTrivialCopyAssignment() || 1924226633Sdim Record->hasTrivialMoveConstructor() || 1925288943Sdim Record->hasTrivialMoveAssignment() || 1926288943Sdim Record->isUnion()) && 1927249423Sdim "Trying to aggregate-copy a type without a trivial copy/move " 1928208600Srdivacky "constructor or assignment operator"); 1929208600Srdivacky // Ignore empty classes in C++. 1930208600Srdivacky if (Record->isEmpty()) 1931207619Srdivacky return; 1932207619Srdivacky } 1933207619Srdivacky } 1934341825Sdim 1935193326Sed // Aggregate assignment turns into llvm.memcpy. This is almost valid per 1936193326Sed // C99 6.5.16.1p3, which states "If the value being stored in an object is 1937193326Sed // read from another object that overlaps in anyway the storage of the first 1938193326Sed // object, then the overlap shall be exact and the two objects shall have 1939193326Sed // qualified or unqualified versions of a compatible type." 1940193326Sed // 1941193326Sed // memcpy is not defined if the source and destination pointers are exactly 1942193326Sed // equal, but other compilers do this optimization, and almost every memcpy 1943193326Sed // implementation handles this case safely. If there is a libc that does not 1944193326Sed // safely handle this, we can add a target hook. 1945198092Srdivacky 1946341825Sdim // Get data size info for this aggregate. Don't copy the tail padding if this 1947341825Sdim // might be a potentially-overlapping subobject, since the tail padding might 1948341825Sdim // be occupied by a different object. Otherwise, copying it is fine. 1949243830Sdim std::pair<CharUnits, CharUnits> TypeInfo; 1950341825Sdim if (MayOverlap) 1951243830Sdim TypeInfo = getContext().getTypeInfoDataSizeInChars(Ty); 1952243830Sdim else 1953243830Sdim TypeInfo = getContext().getTypeInfoInChars(Ty); 1954198092Srdivacky 1955288943Sdim llvm::Value *SizeVal = nullptr; 1956288943Sdim if (TypeInfo.first.isZero()) { 1957288943Sdim // But note that getTypeInfo returns 0 for a VLA. 1958288943Sdim if (auto *VAT = dyn_cast_or_null<VariableArrayType>( 1959288943Sdim getContext().getAsArrayType(Ty))) { 1960288943Sdim QualType BaseEltTy; 1961288943Sdim SizeVal = emitArrayLength(VAT, BaseEltTy, DestPtr); 1962341825Sdim TypeInfo = getContext().getTypeInfoInChars(BaseEltTy); 1963288943Sdim assert(!TypeInfo.first.isZero()); 1964288943Sdim SizeVal = Builder.CreateNUWMul( 1965288943Sdim SizeVal, 1966288943Sdim llvm::ConstantInt::get(SizeTy, TypeInfo.first.getQuantity())); 1967288943Sdim } 1968288943Sdim } 1969288943Sdim if (!SizeVal) { 1970288943Sdim SizeVal = llvm::ConstantInt::get(SizeTy, TypeInfo.first.getQuantity()); 1971288943Sdim } 1972198092Srdivacky 1973193326Sed // FIXME: If we have a volatile struct, the optimizer can remove what might 1974193326Sed // appear to be `extra' memory ops: 1975193326Sed // 1976193326Sed // volatile struct { int i; } a, b; 1977193326Sed // 1978193326Sed // int main() { 1979193326Sed // a = b; 1980193326Sed // a = b; 1981193326Sed // } 1982193326Sed // 1983206275Srdivacky // we need to use a different call here. We use isVolatile to indicate when 1984193326Sed // either the source or the destination is volatile. 1985206275Srdivacky 1986296417Sdim DestPtr = Builder.CreateElementBitCast(DestPtr, Int8Ty); 1987296417Sdim SrcPtr = Builder.CreateElementBitCast(SrcPtr, Int8Ty); 1988206275Srdivacky 1989224145Sdim // Don't do any of the memmove_collectable tests if GC isn't set. 1990234353Sdim if (CGM.getLangOpts().getGC() == LangOptions::NonGC) { 1991224145Sdim // fall through 1992224145Sdim } else if (const RecordType *RecordTy = Ty->getAs<RecordType>()) { 1993210299Sed RecordDecl *Record = RecordTy->getDecl(); 1994210299Sed if (Record->hasObjectMember()) { 1995341825Sdim CGM.getObjCRuntime().EmitGCMemmoveCollectable(*this, DestPtr, SrcPtr, 1996210299Sed SizeVal); 1997210299Sed return; 1998210299Sed } 1999224145Sdim } else if (Ty->isArrayType()) { 2000210299Sed QualType BaseType = getContext().getBaseElementType(Ty); 2001210299Sed if (const RecordType *RecordTy = BaseType->getAs<RecordType>()) { 2002210299Sed if (RecordTy->getDecl()->hasObjectMember()) { 2003341825Sdim CGM.getObjCRuntime().EmitGCMemmoveCollectable(*this, DestPtr, SrcPtr, 2004210299Sed SizeVal); 2005210299Sed return; 2006210299Sed } 2007210299Sed } 2008210299Sed } 2009243830Sdim 2010296417Sdim auto Inst = Builder.CreateMemCpy(DestPtr, SrcPtr, SizeVal, isVolatile); 2011296417Sdim 2012243830Sdim // Determine the metadata to describe the position of any padding in this 2013243830Sdim // memcpy, as well as the TBAA tags for the members of the struct, in case 2014243830Sdim // the optimizer wishes to expand it in to scalar memory operations. 2015296417Sdim if (llvm::MDNode *TBAAStructTag = CGM.getTBAAStructInfo(Ty)) 2016296417Sdim Inst->setMetadata(llvm::LLVMContext::MD_tbaa_struct, TBAAStructTag); 2017341825Sdim 2018341825Sdim if (CGM.getCodeGenOpts().NewStructPathTBAA) { 2019341825Sdim TBAAAccessInfo TBAAInfo = CGM.mergeTBAAInfoForMemoryTransfer( 2020341825Sdim Dest.getTBAAInfo(), Src.getTBAAInfo()); 2021341825Sdim CGM.DecorateInstructionWithTBAA(Inst, TBAAInfo); 2022341825Sdim } 2023193326Sed} 2024