CGExprAgg.cpp revision 344779
1//===--- CGExprAgg.cpp - Emit LLVM Code from Aggregate Expressions --------===// 2// 3// The LLVM Compiler Infrastructure 4// 5// This file is distributed under the University of Illinois Open Source 6// License. See LICENSE.TXT for details. 7// 8//===----------------------------------------------------------------------===// 9// 10// This contains code to emit Aggregate Expr nodes as LLVM code. 11// 12//===----------------------------------------------------------------------===// 13 14#include "CodeGenFunction.h" 15#include "CGCXXABI.h" 16#include "CGObjCRuntime.h" 17#include "CodeGenModule.h" 18#include "ConstantEmitter.h" 19#include "clang/AST/ASTContext.h" 20#include "clang/AST/DeclCXX.h" 21#include "clang/AST/DeclTemplate.h" 22#include "clang/AST/StmtVisitor.h" 23#include "llvm/IR/Constants.h" 24#include "llvm/IR/Function.h" 25#include "llvm/IR/GlobalVariable.h" 26#include "llvm/IR/Intrinsics.h" 27#include "llvm/IR/IntrinsicInst.h" 28using namespace clang; 29using namespace CodeGen; 30 31//===----------------------------------------------------------------------===// 32// Aggregate Expression Emitter 33//===----------------------------------------------------------------------===// 34 35namespace { 36class AggExprEmitter : public StmtVisitor<AggExprEmitter> { 37 CodeGenFunction &CGF; 38 CGBuilderTy &Builder; 39 AggValueSlot Dest; 40 bool IsResultUnused; 41 42 AggValueSlot EnsureSlot(QualType T) { 43 if (!Dest.isIgnored()) return Dest; 44 return CGF.CreateAggTemp(T, "agg.tmp.ensured"); 45 } 46 void EnsureDest(QualType T) { 47 if (!Dest.isIgnored()) return; 48 Dest = CGF.CreateAggTemp(T, "agg.tmp.ensured"); 49 } 50 51 // Calls `Fn` with a valid return value slot, potentially creating a temporary 52 // to do so. If a temporary is created, an appropriate copy into `Dest` will 53 // be emitted, as will lifetime markers. 54 // 55 // The given function should take a ReturnValueSlot, and return an RValue that 56 // points to said slot. 57 void withReturnValueSlot(const Expr *E, 58 llvm::function_ref<RValue(ReturnValueSlot)> Fn); 59 60public: 61 AggExprEmitter(CodeGenFunction &cgf, AggValueSlot Dest, bool IsResultUnused) 62 : CGF(cgf), Builder(CGF.Builder), Dest(Dest), 63 IsResultUnused(IsResultUnused) { } 64 65 //===--------------------------------------------------------------------===// 66 // Utilities 67 //===--------------------------------------------------------------------===// 68 69 /// EmitAggLoadOfLValue - Given an expression with aggregate type that 70 /// represents a value lvalue, this method emits the address of the lvalue, 71 /// then loads the result into DestPtr. 72 void EmitAggLoadOfLValue(const Expr *E); 73 74 enum ExprValueKind { 75 EVK_RValue, 76 EVK_NonRValue 77 }; 78 79 /// EmitFinalDestCopy - Perform the final copy to DestPtr, if desired. 80 /// SrcIsRValue is true if source comes from an RValue. 81 void EmitFinalDestCopy(QualType type, const LValue &src, 82 ExprValueKind SrcValueKind = EVK_NonRValue); 83 void EmitFinalDestCopy(QualType type, RValue src); 84 void EmitCopy(QualType type, const AggValueSlot &dest, 85 const AggValueSlot &src); 86 87 void EmitMoveFromReturnSlot(const Expr *E, RValue Src); 88 89 void EmitArrayInit(Address DestPtr, llvm::ArrayType *AType, 90 QualType ArrayQTy, InitListExpr *E); 91 92 AggValueSlot::NeedsGCBarriers_t needsGC(QualType T) { 93 if (CGF.getLangOpts().getGC() && TypeRequiresGCollection(T)) 94 return AggValueSlot::NeedsGCBarriers; 95 return AggValueSlot::DoesNotNeedGCBarriers; 96 } 97 98 bool TypeRequiresGCollection(QualType T); 99 100 //===--------------------------------------------------------------------===// 101 // Visitor Methods 102 //===--------------------------------------------------------------------===// 103 104 void Visit(Expr *E) { 105 ApplyDebugLocation DL(CGF, E); 106 StmtVisitor<AggExprEmitter>::Visit(E); 107 } 108 109 void VisitStmt(Stmt *S) { 110 CGF.ErrorUnsupported(S, "aggregate expression"); 111 } 112 void VisitParenExpr(ParenExpr *PE) { Visit(PE->getSubExpr()); } 113 void VisitGenericSelectionExpr(GenericSelectionExpr *GE) { 114 Visit(GE->getResultExpr()); 115 } 116 void VisitCoawaitExpr(CoawaitExpr *E) { 117 CGF.EmitCoawaitExpr(*E, Dest, IsResultUnused); 118 } 119 void VisitCoyieldExpr(CoyieldExpr *E) { 120 CGF.EmitCoyieldExpr(*E, Dest, IsResultUnused); 121 } 122 void VisitUnaryCoawait(UnaryOperator *E) { Visit(E->getSubExpr()); } 123 void VisitUnaryExtension(UnaryOperator *E) { Visit(E->getSubExpr()); } 124 void VisitSubstNonTypeTemplateParmExpr(SubstNonTypeTemplateParmExpr *E) { 125 return Visit(E->getReplacement()); 126 } 127 128 void VisitConstantExpr(ConstantExpr *E) { 129 return Visit(E->getSubExpr()); 130 } 131 132 // l-values. 133 void VisitDeclRefExpr(DeclRefExpr *E) { EmitAggLoadOfLValue(E); } 134 void VisitMemberExpr(MemberExpr *ME) { EmitAggLoadOfLValue(ME); } 135 void VisitUnaryDeref(UnaryOperator *E) { EmitAggLoadOfLValue(E); } 136 void VisitStringLiteral(StringLiteral *E) { EmitAggLoadOfLValue(E); } 137 void VisitCompoundLiteralExpr(CompoundLiteralExpr *E); 138 void VisitArraySubscriptExpr(ArraySubscriptExpr *E) { 139 EmitAggLoadOfLValue(E); 140 } 141 void VisitPredefinedExpr(const PredefinedExpr *E) { 142 EmitAggLoadOfLValue(E); 143 } 144 145 // Operators. 146 void VisitCastExpr(CastExpr *E); 147 void VisitCallExpr(const CallExpr *E); 148 void VisitStmtExpr(const StmtExpr *E); 149 void VisitBinaryOperator(const BinaryOperator *BO); 150 void VisitPointerToDataMemberBinaryOperator(const BinaryOperator *BO); 151 void VisitBinAssign(const BinaryOperator *E); 152 void VisitBinComma(const BinaryOperator *E); 153 void VisitBinCmp(const BinaryOperator *E); 154 155 void VisitObjCMessageExpr(ObjCMessageExpr *E); 156 void VisitObjCIvarRefExpr(ObjCIvarRefExpr *E) { 157 EmitAggLoadOfLValue(E); 158 } 159 160 void VisitDesignatedInitUpdateExpr(DesignatedInitUpdateExpr *E); 161 void VisitAbstractConditionalOperator(const AbstractConditionalOperator *CO); 162 void VisitChooseExpr(const ChooseExpr *CE); 163 void VisitInitListExpr(InitListExpr *E); 164 void VisitArrayInitLoopExpr(const ArrayInitLoopExpr *E, 165 llvm::Value *outerBegin = nullptr); 166 void VisitImplicitValueInitExpr(ImplicitValueInitExpr *E); 167 void VisitNoInitExpr(NoInitExpr *E) { } // Do nothing. 168 void VisitCXXDefaultArgExpr(CXXDefaultArgExpr *DAE) { 169 Visit(DAE->getExpr()); 170 } 171 void VisitCXXDefaultInitExpr(CXXDefaultInitExpr *DIE) { 172 CodeGenFunction::CXXDefaultInitExprScope Scope(CGF); 173 Visit(DIE->getExpr()); 174 } 175 void VisitCXXBindTemporaryExpr(CXXBindTemporaryExpr *E); 176 void VisitCXXConstructExpr(const CXXConstructExpr *E); 177 void VisitCXXInheritedCtorInitExpr(const CXXInheritedCtorInitExpr *E); 178 void VisitLambdaExpr(LambdaExpr *E); 179 void VisitCXXStdInitializerListExpr(CXXStdInitializerListExpr *E); 180 void VisitExprWithCleanups(ExprWithCleanups *E); 181 void VisitCXXScalarValueInitExpr(CXXScalarValueInitExpr *E); 182 void VisitCXXTypeidExpr(CXXTypeidExpr *E) { EmitAggLoadOfLValue(E); } 183 void VisitMaterializeTemporaryExpr(MaterializeTemporaryExpr *E); 184 void VisitOpaqueValueExpr(OpaqueValueExpr *E); 185 186 void VisitPseudoObjectExpr(PseudoObjectExpr *E) { 187 if (E->isGLValue()) { 188 LValue LV = CGF.EmitPseudoObjectLValue(E); 189 return EmitFinalDestCopy(E->getType(), LV); 190 } 191 192 CGF.EmitPseudoObjectRValue(E, EnsureSlot(E->getType())); 193 } 194 195 void VisitVAArgExpr(VAArgExpr *E); 196 197 void EmitInitializationToLValue(Expr *E, LValue Address); 198 void EmitNullInitializationToLValue(LValue Address); 199 // case Expr::ChooseExprClass: 200 void VisitCXXThrowExpr(const CXXThrowExpr *E) { CGF.EmitCXXThrowExpr(E); } 201 void VisitAtomicExpr(AtomicExpr *E) { 202 RValue Res = CGF.EmitAtomicExpr(E); 203 EmitFinalDestCopy(E->getType(), Res); 204 } 205}; 206} // end anonymous namespace. 207 208//===----------------------------------------------------------------------===// 209// Utilities 210//===----------------------------------------------------------------------===// 211 212/// EmitAggLoadOfLValue - Given an expression with aggregate type that 213/// represents a value lvalue, this method emits the address of the lvalue, 214/// then loads the result into DestPtr. 215void AggExprEmitter::EmitAggLoadOfLValue(const Expr *E) { 216 LValue LV = CGF.EmitLValue(E); 217 218 // If the type of the l-value is atomic, then do an atomic load. 219 if (LV.getType()->isAtomicType() || CGF.LValueIsSuitableForInlineAtomic(LV)) { 220 CGF.EmitAtomicLoad(LV, E->getExprLoc(), Dest); 221 return; 222 } 223 224 EmitFinalDestCopy(E->getType(), LV); 225} 226 227/// True if the given aggregate type requires special GC API calls. 228bool AggExprEmitter::TypeRequiresGCollection(QualType T) { 229 // Only record types have members that might require garbage collection. 230 const RecordType *RecordTy = T->getAs<RecordType>(); 231 if (!RecordTy) return false; 232 233 // Don't mess with non-trivial C++ types. 234 RecordDecl *Record = RecordTy->getDecl(); 235 if (isa<CXXRecordDecl>(Record) && 236 (cast<CXXRecordDecl>(Record)->hasNonTrivialCopyConstructor() || 237 !cast<CXXRecordDecl>(Record)->hasTrivialDestructor())) 238 return false; 239 240 // Check whether the type has an object member. 241 return Record->hasObjectMember(); 242} 243 244void AggExprEmitter::withReturnValueSlot( 245 const Expr *E, llvm::function_ref<RValue(ReturnValueSlot)> EmitCall) { 246 QualType RetTy = E->getType(); 247 bool RequiresDestruction = 248 Dest.isIgnored() && 249 RetTy.isDestructedType() == QualType::DK_nontrivial_c_struct; 250 251 // If it makes no observable difference, save a memcpy + temporary. 252 // 253 // We need to always provide our own temporary if destruction is required. 254 // Otherwise, EmitCall will emit its own, notice that it's "unused", and end 255 // its lifetime before we have the chance to emit a proper destructor call. 256 bool UseTemp = Dest.isPotentiallyAliased() || Dest.requiresGCollection() || 257 (RequiresDestruction && !Dest.getAddress().isValid()); 258 259 Address RetAddr = Address::invalid(); 260 Address RetAllocaAddr = Address::invalid(); 261 262 EHScopeStack::stable_iterator LifetimeEndBlock; 263 llvm::Value *LifetimeSizePtr = nullptr; 264 llvm::IntrinsicInst *LifetimeStartInst = nullptr; 265 if (!UseTemp) { 266 RetAddr = Dest.getAddress(); 267 } else { 268 RetAddr = CGF.CreateMemTemp(RetTy, "tmp", &RetAllocaAddr); 269 uint64_t Size = 270 CGF.CGM.getDataLayout().getTypeAllocSize(CGF.ConvertTypeForMem(RetTy)); 271 LifetimeSizePtr = CGF.EmitLifetimeStart(Size, RetAllocaAddr.getPointer()); 272 if (LifetimeSizePtr) { 273 LifetimeStartInst = 274 cast<llvm::IntrinsicInst>(std::prev(Builder.GetInsertPoint())); 275 assert(LifetimeStartInst->getIntrinsicID() == 276 llvm::Intrinsic::lifetime_start && 277 "Last insertion wasn't a lifetime.start?"); 278 279 CGF.pushFullExprCleanup<CodeGenFunction::CallLifetimeEnd>( 280 NormalEHLifetimeMarker, RetAllocaAddr, LifetimeSizePtr); 281 LifetimeEndBlock = CGF.EHStack.stable_begin(); 282 } 283 } 284 285 RValue Src = 286 EmitCall(ReturnValueSlot(RetAddr, Dest.isVolatile(), IsResultUnused)); 287 288 if (RequiresDestruction) 289 CGF.pushDestroy(RetTy.isDestructedType(), Src.getAggregateAddress(), RetTy); 290 291 if (!UseTemp) 292 return; 293 294 assert(Dest.getPointer() != Src.getAggregatePointer()); 295 EmitFinalDestCopy(E->getType(), Src); 296 297 if (!RequiresDestruction && LifetimeStartInst) { 298 // If there's no dtor to run, the copy was the last use of our temporary. 299 // Since we're not guaranteed to be in an ExprWithCleanups, clean up 300 // eagerly. 301 CGF.DeactivateCleanupBlock(LifetimeEndBlock, LifetimeStartInst); 302 CGF.EmitLifetimeEnd(LifetimeSizePtr, RetAllocaAddr.getPointer()); 303 } 304} 305 306/// EmitFinalDestCopy - Perform the final copy to DestPtr, if desired. 307void AggExprEmitter::EmitFinalDestCopy(QualType type, RValue src) { 308 assert(src.isAggregate() && "value must be aggregate value!"); 309 LValue srcLV = CGF.MakeAddrLValue(src.getAggregateAddress(), type); 310 EmitFinalDestCopy(type, srcLV, EVK_RValue); 311} 312 313/// EmitFinalDestCopy - Perform the final copy to DestPtr, if desired. 314void AggExprEmitter::EmitFinalDestCopy(QualType type, const LValue &src, 315 ExprValueKind SrcValueKind) { 316 // If Dest is ignored, then we're evaluating an aggregate expression 317 // in a context that doesn't care about the result. Note that loads 318 // from volatile l-values force the existence of a non-ignored 319 // destination. 320 if (Dest.isIgnored()) 321 return; 322 323 // Copy non-trivial C structs here. 324 LValue DstLV = CGF.MakeAddrLValue( 325 Dest.getAddress(), Dest.isVolatile() ? type.withVolatile() : type); 326 327 if (SrcValueKind == EVK_RValue) { 328 if (type.isNonTrivialToPrimitiveDestructiveMove() == QualType::PCK_Struct) { 329 if (Dest.isPotentiallyAliased()) 330 CGF.callCStructMoveAssignmentOperator(DstLV, src); 331 else 332 CGF.callCStructMoveConstructor(DstLV, src); 333 return; 334 } 335 } else { 336 if (type.isNonTrivialToPrimitiveCopy() == QualType::PCK_Struct) { 337 if (Dest.isPotentiallyAliased()) 338 CGF.callCStructCopyAssignmentOperator(DstLV, src); 339 else 340 CGF.callCStructCopyConstructor(DstLV, src); 341 return; 342 } 343 } 344 345 AggValueSlot srcAgg = 346 AggValueSlot::forLValue(src, AggValueSlot::IsDestructed, 347 needsGC(type), AggValueSlot::IsAliased, 348 AggValueSlot::MayOverlap); 349 EmitCopy(type, Dest, srcAgg); 350} 351 352/// Perform a copy from the source into the destination. 353/// 354/// \param type - the type of the aggregate being copied; qualifiers are 355/// ignored 356void AggExprEmitter::EmitCopy(QualType type, const AggValueSlot &dest, 357 const AggValueSlot &src) { 358 if (dest.requiresGCollection()) { 359 CharUnits sz = dest.getPreferredSize(CGF.getContext(), type); 360 llvm::Value *size = llvm::ConstantInt::get(CGF.SizeTy, sz.getQuantity()); 361 CGF.CGM.getObjCRuntime().EmitGCMemmoveCollectable(CGF, 362 dest.getAddress(), 363 src.getAddress(), 364 size); 365 return; 366 } 367 368 // If the result of the assignment is used, copy the LHS there also. 369 // It's volatile if either side is. Use the minimum alignment of 370 // the two sides. 371 LValue DestLV = CGF.MakeAddrLValue(dest.getAddress(), type); 372 LValue SrcLV = CGF.MakeAddrLValue(src.getAddress(), type); 373 CGF.EmitAggregateCopy(DestLV, SrcLV, type, dest.mayOverlap(), 374 dest.isVolatile() || src.isVolatile()); 375} 376 377/// Emit the initializer for a std::initializer_list initialized with a 378/// real initializer list. 379void 380AggExprEmitter::VisitCXXStdInitializerListExpr(CXXStdInitializerListExpr *E) { 381 // Emit an array containing the elements. The array is externally destructed 382 // if the std::initializer_list object is. 383 ASTContext &Ctx = CGF.getContext(); 384 LValue Array = CGF.EmitLValue(E->getSubExpr()); 385 assert(Array.isSimple() && "initializer_list array not a simple lvalue"); 386 Address ArrayPtr = Array.getAddress(); 387 388 const ConstantArrayType *ArrayType = 389 Ctx.getAsConstantArrayType(E->getSubExpr()->getType()); 390 assert(ArrayType && "std::initializer_list constructed from non-array"); 391 392 // FIXME: Perform the checks on the field types in SemaInit. 393 RecordDecl *Record = E->getType()->castAs<RecordType>()->getDecl(); 394 RecordDecl::field_iterator Field = Record->field_begin(); 395 if (Field == Record->field_end()) { 396 CGF.ErrorUnsupported(E, "weird std::initializer_list"); 397 return; 398 } 399 400 // Start pointer. 401 if (!Field->getType()->isPointerType() || 402 !Ctx.hasSameType(Field->getType()->getPointeeType(), 403 ArrayType->getElementType())) { 404 CGF.ErrorUnsupported(E, "weird std::initializer_list"); 405 return; 406 } 407 408 AggValueSlot Dest = EnsureSlot(E->getType()); 409 LValue DestLV = CGF.MakeAddrLValue(Dest.getAddress(), E->getType()); 410 LValue Start = CGF.EmitLValueForFieldInitialization(DestLV, *Field); 411 llvm::Value *Zero = llvm::ConstantInt::get(CGF.PtrDiffTy, 0); 412 llvm::Value *IdxStart[] = { Zero, Zero }; 413 llvm::Value *ArrayStart = 414 Builder.CreateInBoundsGEP(ArrayPtr.getPointer(), IdxStart, "arraystart"); 415 CGF.EmitStoreThroughLValue(RValue::get(ArrayStart), Start); 416 ++Field; 417 418 if (Field == Record->field_end()) { 419 CGF.ErrorUnsupported(E, "weird std::initializer_list"); 420 return; 421 } 422 423 llvm::Value *Size = Builder.getInt(ArrayType->getSize()); 424 LValue EndOrLength = CGF.EmitLValueForFieldInitialization(DestLV, *Field); 425 if (Field->getType()->isPointerType() && 426 Ctx.hasSameType(Field->getType()->getPointeeType(), 427 ArrayType->getElementType())) { 428 // End pointer. 429 llvm::Value *IdxEnd[] = { Zero, Size }; 430 llvm::Value *ArrayEnd = 431 Builder.CreateInBoundsGEP(ArrayPtr.getPointer(), IdxEnd, "arrayend"); 432 CGF.EmitStoreThroughLValue(RValue::get(ArrayEnd), EndOrLength); 433 } else if (Ctx.hasSameType(Field->getType(), Ctx.getSizeType())) { 434 // Length. 435 CGF.EmitStoreThroughLValue(RValue::get(Size), EndOrLength); 436 } else { 437 CGF.ErrorUnsupported(E, "weird std::initializer_list"); 438 return; 439 } 440} 441 442/// Determine if E is a trivial array filler, that is, one that is 443/// equivalent to zero-initialization. 444static bool isTrivialFiller(Expr *E) { 445 if (!E) 446 return true; 447 448 if (isa<ImplicitValueInitExpr>(E)) 449 return true; 450 451 if (auto *ILE = dyn_cast<InitListExpr>(E)) { 452 if (ILE->getNumInits()) 453 return false; 454 return isTrivialFiller(ILE->getArrayFiller()); 455 } 456 457 if (auto *Cons = dyn_cast_or_null<CXXConstructExpr>(E)) 458 return Cons->getConstructor()->isDefaultConstructor() && 459 Cons->getConstructor()->isTrivial(); 460 461 // FIXME: Are there other cases where we can avoid emitting an initializer? 462 return false; 463} 464 465/// Emit initialization of an array from an initializer list. 466void AggExprEmitter::EmitArrayInit(Address DestPtr, llvm::ArrayType *AType, 467 QualType ArrayQTy, InitListExpr *E) { 468 uint64_t NumInitElements = E->getNumInits(); 469 470 uint64_t NumArrayElements = AType->getNumElements(); 471 assert(NumInitElements <= NumArrayElements); 472 473 QualType elementType = 474 CGF.getContext().getAsArrayType(ArrayQTy)->getElementType(); 475 476 // DestPtr is an array*. Construct an elementType* by drilling 477 // down a level. 478 llvm::Value *zero = llvm::ConstantInt::get(CGF.SizeTy, 0); 479 llvm::Value *indices[] = { zero, zero }; 480 llvm::Value *begin = 481 Builder.CreateInBoundsGEP(DestPtr.getPointer(), indices, "arrayinit.begin"); 482 483 CharUnits elementSize = CGF.getContext().getTypeSizeInChars(elementType); 484 CharUnits elementAlign = 485 DestPtr.getAlignment().alignmentOfArrayElement(elementSize); 486 487 // Consider initializing the array by copying from a global. For this to be 488 // more efficient than per-element initialization, the size of the elements 489 // with explicit initializers should be large enough. 490 if (NumInitElements * elementSize.getQuantity() > 16 && 491 elementType.isTriviallyCopyableType(CGF.getContext())) { 492 CodeGen::CodeGenModule &CGM = CGF.CGM; 493 ConstantEmitter Emitter(CGM); 494 LangAS AS = ArrayQTy.getAddressSpace(); 495 if (llvm::Constant *C = Emitter.tryEmitForInitializer(E, AS, ArrayQTy)) { 496 auto GV = new llvm::GlobalVariable( 497 CGM.getModule(), C->getType(), 498 CGM.isTypeConstant(ArrayQTy, /* ExcludeCtorDtor= */ true), 499 llvm::GlobalValue::PrivateLinkage, C, "constinit", 500 /* InsertBefore= */ nullptr, llvm::GlobalVariable::NotThreadLocal, 501 CGM.getContext().getTargetAddressSpace(AS)); 502 Emitter.finalize(GV); 503 CharUnits Align = CGM.getContext().getTypeAlignInChars(ArrayQTy); 504 GV->setAlignment(Align.getQuantity()); 505 EmitFinalDestCopy(ArrayQTy, CGF.MakeAddrLValue(GV, ArrayQTy, Align)); 506 return; 507 } 508 } 509 510 // Exception safety requires us to destroy all the 511 // already-constructed members if an initializer throws. 512 // For that, we'll need an EH cleanup. 513 QualType::DestructionKind dtorKind = elementType.isDestructedType(); 514 Address endOfInit = Address::invalid(); 515 EHScopeStack::stable_iterator cleanup; 516 llvm::Instruction *cleanupDominator = nullptr; 517 if (CGF.needsEHCleanup(dtorKind)) { 518 // In principle we could tell the cleanup where we are more 519 // directly, but the control flow can get so varied here that it 520 // would actually be quite complex. Therefore we go through an 521 // alloca. 522 endOfInit = CGF.CreateTempAlloca(begin->getType(), CGF.getPointerAlign(), 523 "arrayinit.endOfInit"); 524 cleanupDominator = Builder.CreateStore(begin, endOfInit); 525 CGF.pushIrregularPartialArrayCleanup(begin, endOfInit, elementType, 526 elementAlign, 527 CGF.getDestroyer(dtorKind)); 528 cleanup = CGF.EHStack.stable_begin(); 529 530 // Otherwise, remember that we didn't need a cleanup. 531 } else { 532 dtorKind = QualType::DK_none; 533 } 534 535 llvm::Value *one = llvm::ConstantInt::get(CGF.SizeTy, 1); 536 537 // The 'current element to initialize'. The invariants on this 538 // variable are complicated. Essentially, after each iteration of 539 // the loop, it points to the last initialized element, except 540 // that it points to the beginning of the array before any 541 // elements have been initialized. 542 llvm::Value *element = begin; 543 544 // Emit the explicit initializers. 545 for (uint64_t i = 0; i != NumInitElements; ++i) { 546 // Advance to the next element. 547 if (i > 0) { 548 element = Builder.CreateInBoundsGEP(element, one, "arrayinit.element"); 549 550 // Tell the cleanup that it needs to destroy up to this 551 // element. TODO: some of these stores can be trivially 552 // observed to be unnecessary. 553 if (endOfInit.isValid()) Builder.CreateStore(element, endOfInit); 554 } 555 556 LValue elementLV = 557 CGF.MakeAddrLValue(Address(element, elementAlign), elementType); 558 EmitInitializationToLValue(E->getInit(i), elementLV); 559 } 560 561 // Check whether there's a non-trivial array-fill expression. 562 Expr *filler = E->getArrayFiller(); 563 bool hasTrivialFiller = isTrivialFiller(filler); 564 565 // Any remaining elements need to be zero-initialized, possibly 566 // using the filler expression. We can skip this if the we're 567 // emitting to zeroed memory. 568 if (NumInitElements != NumArrayElements && 569 !(Dest.isZeroed() && hasTrivialFiller && 570 CGF.getTypes().isZeroInitializable(elementType))) { 571 572 // Use an actual loop. This is basically 573 // do { *array++ = filler; } while (array != end); 574 575 // Advance to the start of the rest of the array. 576 if (NumInitElements) { 577 element = Builder.CreateInBoundsGEP(element, one, "arrayinit.start"); 578 if (endOfInit.isValid()) Builder.CreateStore(element, endOfInit); 579 } 580 581 // Compute the end of the array. 582 llvm::Value *end = Builder.CreateInBoundsGEP(begin, 583 llvm::ConstantInt::get(CGF.SizeTy, NumArrayElements), 584 "arrayinit.end"); 585 586 llvm::BasicBlock *entryBB = Builder.GetInsertBlock(); 587 llvm::BasicBlock *bodyBB = CGF.createBasicBlock("arrayinit.body"); 588 589 // Jump into the body. 590 CGF.EmitBlock(bodyBB); 591 llvm::PHINode *currentElement = 592 Builder.CreatePHI(element->getType(), 2, "arrayinit.cur"); 593 currentElement->addIncoming(element, entryBB); 594 595 // Emit the actual filler expression. 596 { 597 // C++1z [class.temporary]p5: 598 // when a default constructor is called to initialize an element of 599 // an array with no corresponding initializer [...] the destruction of 600 // every temporary created in a default argument is sequenced before 601 // the construction of the next array element, if any 602 CodeGenFunction::RunCleanupsScope CleanupsScope(CGF); 603 LValue elementLV = 604 CGF.MakeAddrLValue(Address(currentElement, elementAlign), elementType); 605 if (filler) 606 EmitInitializationToLValue(filler, elementLV); 607 else 608 EmitNullInitializationToLValue(elementLV); 609 } 610 611 // Move on to the next element. 612 llvm::Value *nextElement = 613 Builder.CreateInBoundsGEP(currentElement, one, "arrayinit.next"); 614 615 // Tell the EH cleanup that we finished with the last element. 616 if (endOfInit.isValid()) Builder.CreateStore(nextElement, endOfInit); 617 618 // Leave the loop if we're done. 619 llvm::Value *done = Builder.CreateICmpEQ(nextElement, end, 620 "arrayinit.done"); 621 llvm::BasicBlock *endBB = CGF.createBasicBlock("arrayinit.end"); 622 Builder.CreateCondBr(done, endBB, bodyBB); 623 currentElement->addIncoming(nextElement, Builder.GetInsertBlock()); 624 625 CGF.EmitBlock(endBB); 626 } 627 628 // Leave the partial-array cleanup if we entered one. 629 if (dtorKind) CGF.DeactivateCleanupBlock(cleanup, cleanupDominator); 630} 631 632//===----------------------------------------------------------------------===// 633// Visitor Methods 634//===----------------------------------------------------------------------===// 635 636void AggExprEmitter::VisitMaterializeTemporaryExpr(MaterializeTemporaryExpr *E){ 637 Visit(E->GetTemporaryExpr()); 638} 639 640void AggExprEmitter::VisitOpaqueValueExpr(OpaqueValueExpr *e) { 641 // If this is a unique OVE, just visit its source expression. 642 if (e->isUnique()) 643 Visit(e->getSourceExpr()); 644 else 645 EmitFinalDestCopy(e->getType(), CGF.getOrCreateOpaqueLValueMapping(e)); 646} 647 648void 649AggExprEmitter::VisitCompoundLiteralExpr(CompoundLiteralExpr *E) { 650 if (Dest.isPotentiallyAliased() && 651 E->getType().isPODType(CGF.getContext())) { 652 // For a POD type, just emit a load of the lvalue + a copy, because our 653 // compound literal might alias the destination. 654 EmitAggLoadOfLValue(E); 655 return; 656 } 657 658 AggValueSlot Slot = EnsureSlot(E->getType()); 659 CGF.EmitAggExpr(E->getInitializer(), Slot); 660} 661 662/// Attempt to look through various unimportant expressions to find a 663/// cast of the given kind. 664static Expr *findPeephole(Expr *op, CastKind kind) { 665 while (true) { 666 op = op->IgnoreParens(); 667 if (CastExpr *castE = dyn_cast<CastExpr>(op)) { 668 if (castE->getCastKind() == kind) 669 return castE->getSubExpr(); 670 if (castE->getCastKind() == CK_NoOp) 671 continue; 672 } 673 return nullptr; 674 } 675} 676 677void AggExprEmitter::VisitCastExpr(CastExpr *E) { 678 if (const auto *ECE = dyn_cast<ExplicitCastExpr>(E)) 679 CGF.CGM.EmitExplicitCastExprType(ECE, &CGF); 680 switch (E->getCastKind()) { 681 case CK_Dynamic: { 682 // FIXME: Can this actually happen? We have no test coverage for it. 683 assert(isa<CXXDynamicCastExpr>(E) && "CK_Dynamic without a dynamic_cast?"); 684 LValue LV = CGF.EmitCheckedLValue(E->getSubExpr(), 685 CodeGenFunction::TCK_Load); 686 // FIXME: Do we also need to handle property references here? 687 if (LV.isSimple()) 688 CGF.EmitDynamicCast(LV.getAddress(), cast<CXXDynamicCastExpr>(E)); 689 else 690 CGF.CGM.ErrorUnsupported(E, "non-simple lvalue dynamic_cast"); 691 692 if (!Dest.isIgnored()) 693 CGF.CGM.ErrorUnsupported(E, "lvalue dynamic_cast with a destination"); 694 break; 695 } 696 697 case CK_ToUnion: { 698 // Evaluate even if the destination is ignored. 699 if (Dest.isIgnored()) { 700 CGF.EmitAnyExpr(E->getSubExpr(), AggValueSlot::ignored(), 701 /*ignoreResult=*/true); 702 break; 703 } 704 705 // GCC union extension 706 QualType Ty = E->getSubExpr()->getType(); 707 Address CastPtr = 708 Builder.CreateElementBitCast(Dest.getAddress(), CGF.ConvertType(Ty)); 709 EmitInitializationToLValue(E->getSubExpr(), 710 CGF.MakeAddrLValue(CastPtr, Ty)); 711 break; 712 } 713 714 case CK_DerivedToBase: 715 case CK_BaseToDerived: 716 case CK_UncheckedDerivedToBase: { 717 llvm_unreachable("cannot perform hierarchy conversion in EmitAggExpr: " 718 "should have been unpacked before we got here"); 719 } 720 721 case CK_NonAtomicToAtomic: 722 case CK_AtomicToNonAtomic: { 723 bool isToAtomic = (E->getCastKind() == CK_NonAtomicToAtomic); 724 725 // Determine the atomic and value types. 726 QualType atomicType = E->getSubExpr()->getType(); 727 QualType valueType = E->getType(); 728 if (isToAtomic) std::swap(atomicType, valueType); 729 730 assert(atomicType->isAtomicType()); 731 assert(CGF.getContext().hasSameUnqualifiedType(valueType, 732 atomicType->castAs<AtomicType>()->getValueType())); 733 734 // Just recurse normally if we're ignoring the result or the 735 // atomic type doesn't change representation. 736 if (Dest.isIgnored() || !CGF.CGM.isPaddedAtomicType(atomicType)) { 737 return Visit(E->getSubExpr()); 738 } 739 740 CastKind peepholeTarget = 741 (isToAtomic ? CK_AtomicToNonAtomic : CK_NonAtomicToAtomic); 742 743 // These two cases are reverses of each other; try to peephole them. 744 if (Expr *op = findPeephole(E->getSubExpr(), peepholeTarget)) { 745 assert(CGF.getContext().hasSameUnqualifiedType(op->getType(), 746 E->getType()) && 747 "peephole significantly changed types?"); 748 return Visit(op); 749 } 750 751 // If we're converting an r-value of non-atomic type to an r-value 752 // of atomic type, just emit directly into the relevant sub-object. 753 if (isToAtomic) { 754 AggValueSlot valueDest = Dest; 755 if (!valueDest.isIgnored() && CGF.CGM.isPaddedAtomicType(atomicType)) { 756 // Zero-initialize. (Strictly speaking, we only need to initialize 757 // the padding at the end, but this is simpler.) 758 if (!Dest.isZeroed()) 759 CGF.EmitNullInitialization(Dest.getAddress(), atomicType); 760 761 // Build a GEP to refer to the subobject. 762 Address valueAddr = 763 CGF.Builder.CreateStructGEP(valueDest.getAddress(), 0, 764 CharUnits()); 765 valueDest = AggValueSlot::forAddr(valueAddr, 766 valueDest.getQualifiers(), 767 valueDest.isExternallyDestructed(), 768 valueDest.requiresGCollection(), 769 valueDest.isPotentiallyAliased(), 770 AggValueSlot::DoesNotOverlap, 771 AggValueSlot::IsZeroed); 772 } 773 774 CGF.EmitAggExpr(E->getSubExpr(), valueDest); 775 return; 776 } 777 778 // Otherwise, we're converting an atomic type to a non-atomic type. 779 // Make an atomic temporary, emit into that, and then copy the value out. 780 AggValueSlot atomicSlot = 781 CGF.CreateAggTemp(atomicType, "atomic-to-nonatomic.temp"); 782 CGF.EmitAggExpr(E->getSubExpr(), atomicSlot); 783 784 Address valueAddr = 785 Builder.CreateStructGEP(atomicSlot.getAddress(), 0, CharUnits()); 786 RValue rvalue = RValue::getAggregate(valueAddr, atomicSlot.isVolatile()); 787 return EmitFinalDestCopy(valueType, rvalue); 788 } 789 790 case CK_LValueToRValue: 791 // If we're loading from a volatile type, force the destination 792 // into existence. 793 if (E->getSubExpr()->getType().isVolatileQualified()) { 794 EnsureDest(E->getType()); 795 return Visit(E->getSubExpr()); 796 } 797 798 LLVM_FALLTHROUGH; 799 800 case CK_NoOp: 801 case CK_UserDefinedConversion: 802 case CK_ConstructorConversion: 803 assert(CGF.getContext().hasSameUnqualifiedType(E->getSubExpr()->getType(), 804 E->getType()) && 805 "Implicit cast types must be compatible"); 806 Visit(E->getSubExpr()); 807 break; 808 809 case CK_LValueBitCast: 810 llvm_unreachable("should not be emitting lvalue bitcast as rvalue"); 811 812 case CK_Dependent: 813 case CK_BitCast: 814 case CK_ArrayToPointerDecay: 815 case CK_FunctionToPointerDecay: 816 case CK_NullToPointer: 817 case CK_NullToMemberPointer: 818 case CK_BaseToDerivedMemberPointer: 819 case CK_DerivedToBaseMemberPointer: 820 case CK_MemberPointerToBoolean: 821 case CK_ReinterpretMemberPointer: 822 case CK_IntegralToPointer: 823 case CK_PointerToIntegral: 824 case CK_PointerToBoolean: 825 case CK_ToVoid: 826 case CK_VectorSplat: 827 case CK_IntegralCast: 828 case CK_BooleanToSignedIntegral: 829 case CK_IntegralToBoolean: 830 case CK_IntegralToFloating: 831 case CK_FloatingToIntegral: 832 case CK_FloatingToBoolean: 833 case CK_FloatingCast: 834 case CK_CPointerToObjCPointerCast: 835 case CK_BlockPointerToObjCPointerCast: 836 case CK_AnyPointerToBlockPointerCast: 837 case CK_ObjCObjectLValueCast: 838 case CK_FloatingRealToComplex: 839 case CK_FloatingComplexToReal: 840 case CK_FloatingComplexToBoolean: 841 case CK_FloatingComplexCast: 842 case CK_FloatingComplexToIntegralComplex: 843 case CK_IntegralRealToComplex: 844 case CK_IntegralComplexToReal: 845 case CK_IntegralComplexToBoolean: 846 case CK_IntegralComplexCast: 847 case CK_IntegralComplexToFloatingComplex: 848 case CK_ARCProduceObject: 849 case CK_ARCConsumeObject: 850 case CK_ARCReclaimReturnedObject: 851 case CK_ARCExtendBlockObject: 852 case CK_CopyAndAutoreleaseBlockObject: 853 case CK_BuiltinFnToFnPtr: 854 case CK_ZeroToOCLOpaqueType: 855 case CK_AddressSpaceConversion: 856 case CK_IntToOCLSampler: 857 case CK_FixedPointCast: 858 case CK_FixedPointToBoolean: 859 llvm_unreachable("cast kind invalid for aggregate types"); 860 } 861} 862 863void AggExprEmitter::VisitCallExpr(const CallExpr *E) { 864 if (E->getCallReturnType(CGF.getContext())->isReferenceType()) { 865 EmitAggLoadOfLValue(E); 866 return; 867 } 868 869 withReturnValueSlot(E, [&](ReturnValueSlot Slot) { 870 return CGF.EmitCallExpr(E, Slot); 871 }); 872} 873 874void AggExprEmitter::VisitObjCMessageExpr(ObjCMessageExpr *E) { 875 withReturnValueSlot(E, [&](ReturnValueSlot Slot) { 876 return CGF.EmitObjCMessageExpr(E, Slot); 877 }); 878} 879 880void AggExprEmitter::VisitBinComma(const BinaryOperator *E) { 881 CGF.EmitIgnoredExpr(E->getLHS()); 882 Visit(E->getRHS()); 883} 884 885void AggExprEmitter::VisitStmtExpr(const StmtExpr *E) { 886 CodeGenFunction::StmtExprEvaluation eval(CGF); 887 CGF.EmitCompoundStmt(*E->getSubStmt(), true, Dest); 888} 889 890enum CompareKind { 891 CK_Less, 892 CK_Greater, 893 CK_Equal, 894}; 895 896static llvm::Value *EmitCompare(CGBuilderTy &Builder, CodeGenFunction &CGF, 897 const BinaryOperator *E, llvm::Value *LHS, 898 llvm::Value *RHS, CompareKind Kind, 899 const char *NameSuffix = "") { 900 QualType ArgTy = E->getLHS()->getType(); 901 if (const ComplexType *CT = ArgTy->getAs<ComplexType>()) 902 ArgTy = CT->getElementType(); 903 904 if (const auto *MPT = ArgTy->getAs<MemberPointerType>()) { 905 assert(Kind == CK_Equal && 906 "member pointers may only be compared for equality"); 907 return CGF.CGM.getCXXABI().EmitMemberPointerComparison( 908 CGF, LHS, RHS, MPT, /*IsInequality*/ false); 909 } 910 911 // Compute the comparison instructions for the specified comparison kind. 912 struct CmpInstInfo { 913 const char *Name; 914 llvm::CmpInst::Predicate FCmp; 915 llvm::CmpInst::Predicate SCmp; 916 llvm::CmpInst::Predicate UCmp; 917 }; 918 CmpInstInfo InstInfo = [&]() -> CmpInstInfo { 919 using FI = llvm::FCmpInst; 920 using II = llvm::ICmpInst; 921 switch (Kind) { 922 case CK_Less: 923 return {"cmp.lt", FI::FCMP_OLT, II::ICMP_SLT, II::ICMP_ULT}; 924 case CK_Greater: 925 return {"cmp.gt", FI::FCMP_OGT, II::ICMP_SGT, II::ICMP_UGT}; 926 case CK_Equal: 927 return {"cmp.eq", FI::FCMP_OEQ, II::ICMP_EQ, II::ICMP_EQ}; 928 } 929 llvm_unreachable("Unrecognised CompareKind enum"); 930 }(); 931 932 if (ArgTy->hasFloatingRepresentation()) 933 return Builder.CreateFCmp(InstInfo.FCmp, LHS, RHS, 934 llvm::Twine(InstInfo.Name) + NameSuffix); 935 if (ArgTy->isIntegralOrEnumerationType() || ArgTy->isPointerType()) { 936 auto Inst = 937 ArgTy->hasSignedIntegerRepresentation() ? InstInfo.SCmp : InstInfo.UCmp; 938 return Builder.CreateICmp(Inst, LHS, RHS, 939 llvm::Twine(InstInfo.Name) + NameSuffix); 940 } 941 942 llvm_unreachable("unsupported aggregate binary expression should have " 943 "already been handled"); 944} 945 946void AggExprEmitter::VisitBinCmp(const BinaryOperator *E) { 947 using llvm::BasicBlock; 948 using llvm::PHINode; 949 using llvm::Value; 950 assert(CGF.getContext().hasSameType(E->getLHS()->getType(), 951 E->getRHS()->getType())); 952 const ComparisonCategoryInfo &CmpInfo = 953 CGF.getContext().CompCategories.getInfoForType(E->getType()); 954 assert(CmpInfo.Record->isTriviallyCopyable() && 955 "cannot copy non-trivially copyable aggregate"); 956 957 QualType ArgTy = E->getLHS()->getType(); 958 959 // TODO: Handle comparing these types. 960 if (ArgTy->isVectorType()) 961 return CGF.ErrorUnsupported( 962 E, "aggregate three-way comparison with vector arguments"); 963 if (!ArgTy->isIntegralOrEnumerationType() && !ArgTy->isRealFloatingType() && 964 !ArgTy->isNullPtrType() && !ArgTy->isPointerType() && 965 !ArgTy->isMemberPointerType() && !ArgTy->isAnyComplexType()) { 966 return CGF.ErrorUnsupported(E, "aggregate three-way comparison"); 967 } 968 bool IsComplex = ArgTy->isAnyComplexType(); 969 970 // Evaluate the operands to the expression and extract their values. 971 auto EmitOperand = [&](Expr *E) -> std::pair<Value *, Value *> { 972 RValue RV = CGF.EmitAnyExpr(E); 973 if (RV.isScalar()) 974 return {RV.getScalarVal(), nullptr}; 975 if (RV.isAggregate()) 976 return {RV.getAggregatePointer(), nullptr}; 977 assert(RV.isComplex()); 978 return RV.getComplexVal(); 979 }; 980 auto LHSValues = EmitOperand(E->getLHS()), 981 RHSValues = EmitOperand(E->getRHS()); 982 983 auto EmitCmp = [&](CompareKind K) { 984 Value *Cmp = EmitCompare(Builder, CGF, E, LHSValues.first, RHSValues.first, 985 K, IsComplex ? ".r" : ""); 986 if (!IsComplex) 987 return Cmp; 988 assert(K == CompareKind::CK_Equal); 989 Value *CmpImag = EmitCompare(Builder, CGF, E, LHSValues.second, 990 RHSValues.second, K, ".i"); 991 return Builder.CreateAnd(Cmp, CmpImag, "and.eq"); 992 }; 993 auto EmitCmpRes = [&](const ComparisonCategoryInfo::ValueInfo *VInfo) { 994 return Builder.getInt(VInfo->getIntValue()); 995 }; 996 997 Value *Select; 998 if (ArgTy->isNullPtrType()) { 999 Select = EmitCmpRes(CmpInfo.getEqualOrEquiv()); 1000 } else if (CmpInfo.isEquality()) { 1001 Select = Builder.CreateSelect( 1002 EmitCmp(CK_Equal), EmitCmpRes(CmpInfo.getEqualOrEquiv()), 1003 EmitCmpRes(CmpInfo.getNonequalOrNonequiv()), "sel.eq"); 1004 } else if (!CmpInfo.isPartial()) { 1005 Value *SelectOne = 1006 Builder.CreateSelect(EmitCmp(CK_Less), EmitCmpRes(CmpInfo.getLess()), 1007 EmitCmpRes(CmpInfo.getGreater()), "sel.lt"); 1008 Select = Builder.CreateSelect(EmitCmp(CK_Equal), 1009 EmitCmpRes(CmpInfo.getEqualOrEquiv()), 1010 SelectOne, "sel.eq"); 1011 } else { 1012 Value *SelectEq = Builder.CreateSelect( 1013 EmitCmp(CK_Equal), EmitCmpRes(CmpInfo.getEqualOrEquiv()), 1014 EmitCmpRes(CmpInfo.getUnordered()), "sel.eq"); 1015 Value *SelectGT = Builder.CreateSelect(EmitCmp(CK_Greater), 1016 EmitCmpRes(CmpInfo.getGreater()), 1017 SelectEq, "sel.gt"); 1018 Select = Builder.CreateSelect( 1019 EmitCmp(CK_Less), EmitCmpRes(CmpInfo.getLess()), SelectGT, "sel.lt"); 1020 } 1021 // Create the return value in the destination slot. 1022 EnsureDest(E->getType()); 1023 LValue DestLV = CGF.MakeAddrLValue(Dest.getAddress(), E->getType()); 1024 1025 // Emit the address of the first (and only) field in the comparison category 1026 // type, and initialize it from the constant integer value selected above. 1027 LValue FieldLV = CGF.EmitLValueForFieldInitialization( 1028 DestLV, *CmpInfo.Record->field_begin()); 1029 CGF.EmitStoreThroughLValue(RValue::get(Select), FieldLV, /*IsInit*/ true); 1030 1031 // All done! The result is in the Dest slot. 1032} 1033 1034void AggExprEmitter::VisitBinaryOperator(const BinaryOperator *E) { 1035 if (E->getOpcode() == BO_PtrMemD || E->getOpcode() == BO_PtrMemI) 1036 VisitPointerToDataMemberBinaryOperator(E); 1037 else 1038 CGF.ErrorUnsupported(E, "aggregate binary expression"); 1039} 1040 1041void AggExprEmitter::VisitPointerToDataMemberBinaryOperator( 1042 const BinaryOperator *E) { 1043 LValue LV = CGF.EmitPointerToDataMemberBinaryExpr(E); 1044 EmitFinalDestCopy(E->getType(), LV); 1045} 1046 1047/// Is the value of the given expression possibly a reference to or 1048/// into a __block variable? 1049static bool isBlockVarRef(const Expr *E) { 1050 // Make sure we look through parens. 1051 E = E->IgnoreParens(); 1052 1053 // Check for a direct reference to a __block variable. 1054 if (const DeclRefExpr *DRE = dyn_cast<DeclRefExpr>(E)) { 1055 const VarDecl *var = dyn_cast<VarDecl>(DRE->getDecl()); 1056 return (var && var->hasAttr<BlocksAttr>()); 1057 } 1058 1059 // More complicated stuff. 1060 1061 // Binary operators. 1062 if (const BinaryOperator *op = dyn_cast<BinaryOperator>(E)) { 1063 // For an assignment or pointer-to-member operation, just care 1064 // about the LHS. 1065 if (op->isAssignmentOp() || op->isPtrMemOp()) 1066 return isBlockVarRef(op->getLHS()); 1067 1068 // For a comma, just care about the RHS. 1069 if (op->getOpcode() == BO_Comma) 1070 return isBlockVarRef(op->getRHS()); 1071 1072 // FIXME: pointer arithmetic? 1073 return false; 1074 1075 // Check both sides of a conditional operator. 1076 } else if (const AbstractConditionalOperator *op 1077 = dyn_cast<AbstractConditionalOperator>(E)) { 1078 return isBlockVarRef(op->getTrueExpr()) 1079 || isBlockVarRef(op->getFalseExpr()); 1080 1081 // OVEs are required to support BinaryConditionalOperators. 1082 } else if (const OpaqueValueExpr *op 1083 = dyn_cast<OpaqueValueExpr>(E)) { 1084 if (const Expr *src = op->getSourceExpr()) 1085 return isBlockVarRef(src); 1086 1087 // Casts are necessary to get things like (*(int*)&var) = foo(). 1088 // We don't really care about the kind of cast here, except 1089 // we don't want to look through l2r casts, because it's okay 1090 // to get the *value* in a __block variable. 1091 } else if (const CastExpr *cast = dyn_cast<CastExpr>(E)) { 1092 if (cast->getCastKind() == CK_LValueToRValue) 1093 return false; 1094 return isBlockVarRef(cast->getSubExpr()); 1095 1096 // Handle unary operators. Again, just aggressively look through 1097 // it, ignoring the operation. 1098 } else if (const UnaryOperator *uop = dyn_cast<UnaryOperator>(E)) { 1099 return isBlockVarRef(uop->getSubExpr()); 1100 1101 // Look into the base of a field access. 1102 } else if (const MemberExpr *mem = dyn_cast<MemberExpr>(E)) { 1103 return isBlockVarRef(mem->getBase()); 1104 1105 // Look into the base of a subscript. 1106 } else if (const ArraySubscriptExpr *sub = dyn_cast<ArraySubscriptExpr>(E)) { 1107 return isBlockVarRef(sub->getBase()); 1108 } 1109 1110 return false; 1111} 1112 1113void AggExprEmitter::VisitBinAssign(const BinaryOperator *E) { 1114 // For an assignment to work, the value on the right has 1115 // to be compatible with the value on the left. 1116 assert(CGF.getContext().hasSameUnqualifiedType(E->getLHS()->getType(), 1117 E->getRHS()->getType()) 1118 && "Invalid assignment"); 1119 1120 // If the LHS might be a __block variable, and the RHS can 1121 // potentially cause a block copy, we need to evaluate the RHS first 1122 // so that the assignment goes the right place. 1123 // This is pretty semantically fragile. 1124 if (isBlockVarRef(E->getLHS()) && 1125 E->getRHS()->HasSideEffects(CGF.getContext())) { 1126 // Ensure that we have a destination, and evaluate the RHS into that. 1127 EnsureDest(E->getRHS()->getType()); 1128 Visit(E->getRHS()); 1129 1130 // Now emit the LHS and copy into it. 1131 LValue LHS = CGF.EmitCheckedLValue(E->getLHS(), CodeGenFunction::TCK_Store); 1132 1133 // That copy is an atomic copy if the LHS is atomic. 1134 if (LHS.getType()->isAtomicType() || 1135 CGF.LValueIsSuitableForInlineAtomic(LHS)) { 1136 CGF.EmitAtomicStore(Dest.asRValue(), LHS, /*isInit*/ false); 1137 return; 1138 } 1139 1140 EmitCopy(E->getLHS()->getType(), 1141 AggValueSlot::forLValue(LHS, AggValueSlot::IsDestructed, 1142 needsGC(E->getLHS()->getType()), 1143 AggValueSlot::IsAliased, 1144 AggValueSlot::MayOverlap), 1145 Dest); 1146 return; 1147 } 1148 1149 LValue LHS = CGF.EmitLValue(E->getLHS()); 1150 1151 // If we have an atomic type, evaluate into the destination and then 1152 // do an atomic copy. 1153 if (LHS.getType()->isAtomicType() || 1154 CGF.LValueIsSuitableForInlineAtomic(LHS)) { 1155 EnsureDest(E->getRHS()->getType()); 1156 Visit(E->getRHS()); 1157 CGF.EmitAtomicStore(Dest.asRValue(), LHS, /*isInit*/ false); 1158 return; 1159 } 1160 1161 // Codegen the RHS so that it stores directly into the LHS. 1162 AggValueSlot LHSSlot = 1163 AggValueSlot::forLValue(LHS, AggValueSlot::IsDestructed, 1164 needsGC(E->getLHS()->getType()), 1165 AggValueSlot::IsAliased, 1166 AggValueSlot::MayOverlap); 1167 // A non-volatile aggregate destination might have volatile member. 1168 if (!LHSSlot.isVolatile() && 1169 CGF.hasVolatileMember(E->getLHS()->getType())) 1170 LHSSlot.setVolatile(true); 1171 1172 CGF.EmitAggExpr(E->getRHS(), LHSSlot); 1173 1174 // Copy into the destination if the assignment isn't ignored. 1175 EmitFinalDestCopy(E->getType(), LHS); 1176} 1177 1178void AggExprEmitter:: 1179VisitAbstractConditionalOperator(const AbstractConditionalOperator *E) { 1180 llvm::BasicBlock *LHSBlock = CGF.createBasicBlock("cond.true"); 1181 llvm::BasicBlock *RHSBlock = CGF.createBasicBlock("cond.false"); 1182 llvm::BasicBlock *ContBlock = CGF.createBasicBlock("cond.end"); 1183 1184 // Bind the common expression if necessary. 1185 CodeGenFunction::OpaqueValueMapping binding(CGF, E); 1186 1187 CodeGenFunction::ConditionalEvaluation eval(CGF); 1188 CGF.EmitBranchOnBoolExpr(E->getCond(), LHSBlock, RHSBlock, 1189 CGF.getProfileCount(E)); 1190 1191 // Save whether the destination's lifetime is externally managed. 1192 bool isExternallyDestructed = Dest.isExternallyDestructed(); 1193 1194 eval.begin(CGF); 1195 CGF.EmitBlock(LHSBlock); 1196 CGF.incrementProfileCounter(E); 1197 Visit(E->getTrueExpr()); 1198 eval.end(CGF); 1199 1200 assert(CGF.HaveInsertPoint() && "expression evaluation ended with no IP!"); 1201 CGF.Builder.CreateBr(ContBlock); 1202 1203 // If the result of an agg expression is unused, then the emission 1204 // of the LHS might need to create a destination slot. That's fine 1205 // with us, and we can safely emit the RHS into the same slot, but 1206 // we shouldn't claim that it's already being destructed. 1207 Dest.setExternallyDestructed(isExternallyDestructed); 1208 1209 eval.begin(CGF); 1210 CGF.EmitBlock(RHSBlock); 1211 Visit(E->getFalseExpr()); 1212 eval.end(CGF); 1213 1214 CGF.EmitBlock(ContBlock); 1215} 1216 1217void AggExprEmitter::VisitChooseExpr(const ChooseExpr *CE) { 1218 Visit(CE->getChosenSubExpr()); 1219} 1220 1221void AggExprEmitter::VisitVAArgExpr(VAArgExpr *VE) { 1222 Address ArgValue = Address::invalid(); 1223 Address ArgPtr = CGF.EmitVAArg(VE, ArgValue); 1224 1225 // If EmitVAArg fails, emit an error. 1226 if (!ArgPtr.isValid()) { 1227 CGF.ErrorUnsupported(VE, "aggregate va_arg expression"); 1228 return; 1229 } 1230 1231 EmitFinalDestCopy(VE->getType(), CGF.MakeAddrLValue(ArgPtr, VE->getType())); 1232} 1233 1234void AggExprEmitter::VisitCXXBindTemporaryExpr(CXXBindTemporaryExpr *E) { 1235 // Ensure that we have a slot, but if we already do, remember 1236 // whether it was externally destructed. 1237 bool wasExternallyDestructed = Dest.isExternallyDestructed(); 1238 EnsureDest(E->getType()); 1239 1240 // We're going to push a destructor if there isn't already one. 1241 Dest.setExternallyDestructed(); 1242 1243 Visit(E->getSubExpr()); 1244 1245 // Push that destructor we promised. 1246 if (!wasExternallyDestructed) 1247 CGF.EmitCXXTemporary(E->getTemporary(), E->getType(), Dest.getAddress()); 1248} 1249 1250void 1251AggExprEmitter::VisitCXXConstructExpr(const CXXConstructExpr *E) { 1252 AggValueSlot Slot = EnsureSlot(E->getType()); 1253 CGF.EmitCXXConstructExpr(E, Slot); 1254} 1255 1256void AggExprEmitter::VisitCXXInheritedCtorInitExpr( 1257 const CXXInheritedCtorInitExpr *E) { 1258 AggValueSlot Slot = EnsureSlot(E->getType()); 1259 CGF.EmitInheritedCXXConstructorCall( 1260 E->getConstructor(), E->constructsVBase(), Slot.getAddress(), 1261 E->inheritedFromVBase(), E); 1262} 1263 1264void 1265AggExprEmitter::VisitLambdaExpr(LambdaExpr *E) { 1266 AggValueSlot Slot = EnsureSlot(E->getType()); 1267 CGF.EmitLambdaExpr(E, Slot); 1268} 1269 1270void AggExprEmitter::VisitExprWithCleanups(ExprWithCleanups *E) { 1271 CGF.enterFullExpression(E); 1272 CodeGenFunction::RunCleanupsScope cleanups(CGF); 1273 Visit(E->getSubExpr()); 1274} 1275 1276void AggExprEmitter::VisitCXXScalarValueInitExpr(CXXScalarValueInitExpr *E) { 1277 QualType T = E->getType(); 1278 AggValueSlot Slot = EnsureSlot(T); 1279 EmitNullInitializationToLValue(CGF.MakeAddrLValue(Slot.getAddress(), T)); 1280} 1281 1282void AggExprEmitter::VisitImplicitValueInitExpr(ImplicitValueInitExpr *E) { 1283 QualType T = E->getType(); 1284 AggValueSlot Slot = EnsureSlot(T); 1285 EmitNullInitializationToLValue(CGF.MakeAddrLValue(Slot.getAddress(), T)); 1286} 1287 1288/// isSimpleZero - If emitting this value will obviously just cause a store of 1289/// zero to memory, return true. This can return false if uncertain, so it just 1290/// handles simple cases. 1291static bool isSimpleZero(const Expr *E, CodeGenFunction &CGF) { 1292 E = E->IgnoreParens(); 1293 1294 // 0 1295 if (const IntegerLiteral *IL = dyn_cast<IntegerLiteral>(E)) 1296 return IL->getValue() == 0; 1297 // +0.0 1298 if (const FloatingLiteral *FL = dyn_cast<FloatingLiteral>(E)) 1299 return FL->getValue().isPosZero(); 1300 // int() 1301 if ((isa<ImplicitValueInitExpr>(E) || isa<CXXScalarValueInitExpr>(E)) && 1302 CGF.getTypes().isZeroInitializable(E->getType())) 1303 return true; 1304 // (int*)0 - Null pointer expressions. 1305 if (const CastExpr *ICE = dyn_cast<CastExpr>(E)) 1306 return ICE->getCastKind() == CK_NullToPointer && 1307 CGF.getTypes().isPointerZeroInitializable(E->getType()); 1308 // '\0' 1309 if (const CharacterLiteral *CL = dyn_cast<CharacterLiteral>(E)) 1310 return CL->getValue() == 0; 1311 1312 // Otherwise, hard case: conservatively return false. 1313 return false; 1314} 1315 1316 1317void 1318AggExprEmitter::EmitInitializationToLValue(Expr *E, LValue LV) { 1319 QualType type = LV.getType(); 1320 // FIXME: Ignore result? 1321 // FIXME: Are initializers affected by volatile? 1322 if (Dest.isZeroed() && isSimpleZero(E, CGF)) { 1323 // Storing "i32 0" to a zero'd memory location is a noop. 1324 return; 1325 } else if (isa<ImplicitValueInitExpr>(E) || isa<CXXScalarValueInitExpr>(E)) { 1326 return EmitNullInitializationToLValue(LV); 1327 } else if (isa<NoInitExpr>(E)) { 1328 // Do nothing. 1329 return; 1330 } else if (type->isReferenceType()) { 1331 RValue RV = CGF.EmitReferenceBindingToExpr(E); 1332 return CGF.EmitStoreThroughLValue(RV, LV); 1333 } 1334 1335 switch (CGF.getEvaluationKind(type)) { 1336 case TEK_Complex: 1337 CGF.EmitComplexExprIntoLValue(E, LV, /*isInit*/ true); 1338 return; 1339 case TEK_Aggregate: 1340 CGF.EmitAggExpr(E, AggValueSlot::forLValue(LV, 1341 AggValueSlot::IsDestructed, 1342 AggValueSlot::DoesNotNeedGCBarriers, 1343 AggValueSlot::IsNotAliased, 1344 AggValueSlot::MayOverlap, 1345 Dest.isZeroed())); 1346 return; 1347 case TEK_Scalar: 1348 if (LV.isSimple()) { 1349 CGF.EmitScalarInit(E, /*D=*/nullptr, LV, /*Captured=*/false); 1350 } else { 1351 CGF.EmitStoreThroughLValue(RValue::get(CGF.EmitScalarExpr(E)), LV); 1352 } 1353 return; 1354 } 1355 llvm_unreachable("bad evaluation kind"); 1356} 1357 1358void AggExprEmitter::EmitNullInitializationToLValue(LValue lv) { 1359 QualType type = lv.getType(); 1360 1361 // If the destination slot is already zeroed out before the aggregate is 1362 // copied into it, we don't have to emit any zeros here. 1363 if (Dest.isZeroed() && CGF.getTypes().isZeroInitializable(type)) 1364 return; 1365 1366 if (CGF.hasScalarEvaluationKind(type)) { 1367 // For non-aggregates, we can store the appropriate null constant. 1368 llvm::Value *null = CGF.CGM.EmitNullConstant(type); 1369 // Note that the following is not equivalent to 1370 // EmitStoreThroughBitfieldLValue for ARC types. 1371 if (lv.isBitField()) { 1372 CGF.EmitStoreThroughBitfieldLValue(RValue::get(null), lv); 1373 } else { 1374 assert(lv.isSimple()); 1375 CGF.EmitStoreOfScalar(null, lv, /* isInitialization */ true); 1376 } 1377 } else { 1378 // There's a potential optimization opportunity in combining 1379 // memsets; that would be easy for arrays, but relatively 1380 // difficult for structures with the current code. 1381 CGF.EmitNullInitialization(lv.getAddress(), lv.getType()); 1382 } 1383} 1384 1385void AggExprEmitter::VisitInitListExpr(InitListExpr *E) { 1386#if 0 1387 // FIXME: Assess perf here? Figure out what cases are worth optimizing here 1388 // (Length of globals? Chunks of zeroed-out space?). 1389 // 1390 // If we can, prefer a copy from a global; this is a lot less code for long 1391 // globals, and it's easier for the current optimizers to analyze. 1392 if (llvm::Constant* C = CGF.CGM.EmitConstantExpr(E, E->getType(), &CGF)) { 1393 llvm::GlobalVariable* GV = 1394 new llvm::GlobalVariable(CGF.CGM.getModule(), C->getType(), true, 1395 llvm::GlobalValue::InternalLinkage, C, ""); 1396 EmitFinalDestCopy(E->getType(), CGF.MakeAddrLValue(GV, E->getType())); 1397 return; 1398 } 1399#endif 1400 if (E->hadArrayRangeDesignator()) 1401 CGF.ErrorUnsupported(E, "GNU array range designator extension"); 1402 1403 if (E->isTransparent()) 1404 return Visit(E->getInit(0)); 1405 1406 AggValueSlot Dest = EnsureSlot(E->getType()); 1407 1408 LValue DestLV = CGF.MakeAddrLValue(Dest.getAddress(), E->getType()); 1409 1410 // Handle initialization of an array. 1411 if (E->getType()->isArrayType()) { 1412 auto AType = cast<llvm::ArrayType>(Dest.getAddress().getElementType()); 1413 EmitArrayInit(Dest.getAddress(), AType, E->getType(), E); 1414 return; 1415 } 1416 1417 assert(E->getType()->isRecordType() && "Only support structs/unions here!"); 1418 1419 // Do struct initialization; this code just sets each individual member 1420 // to the approprate value. This makes bitfield support automatic; 1421 // the disadvantage is that the generated code is more difficult for 1422 // the optimizer, especially with bitfields. 1423 unsigned NumInitElements = E->getNumInits(); 1424 RecordDecl *record = E->getType()->castAs<RecordType>()->getDecl(); 1425 1426 // We'll need to enter cleanup scopes in case any of the element 1427 // initializers throws an exception. 1428 SmallVector<EHScopeStack::stable_iterator, 16> cleanups; 1429 llvm::Instruction *cleanupDominator = nullptr; 1430 1431 unsigned curInitIndex = 0; 1432 1433 // Emit initialization of base classes. 1434 if (auto *CXXRD = dyn_cast<CXXRecordDecl>(record)) { 1435 assert(E->getNumInits() >= CXXRD->getNumBases() && 1436 "missing initializer for base class"); 1437 for (auto &Base : CXXRD->bases()) { 1438 assert(!Base.isVirtual() && "should not see vbases here"); 1439 auto *BaseRD = Base.getType()->getAsCXXRecordDecl(); 1440 Address V = CGF.GetAddressOfDirectBaseInCompleteClass( 1441 Dest.getAddress(), CXXRD, BaseRD, 1442 /*isBaseVirtual*/ false); 1443 AggValueSlot AggSlot = AggValueSlot::forAddr( 1444 V, Qualifiers(), 1445 AggValueSlot::IsDestructed, 1446 AggValueSlot::DoesNotNeedGCBarriers, 1447 AggValueSlot::IsNotAliased, 1448 CGF.overlapForBaseInit(CXXRD, BaseRD, Base.isVirtual())); 1449 CGF.EmitAggExpr(E->getInit(curInitIndex++), AggSlot); 1450 1451 if (QualType::DestructionKind dtorKind = 1452 Base.getType().isDestructedType()) { 1453 CGF.pushDestroy(dtorKind, V, Base.getType()); 1454 cleanups.push_back(CGF.EHStack.stable_begin()); 1455 } 1456 } 1457 } 1458 1459 // Prepare a 'this' for CXXDefaultInitExprs. 1460 CodeGenFunction::FieldConstructionScope FCS(CGF, Dest.getAddress()); 1461 1462 if (record->isUnion()) { 1463 // Only initialize one field of a union. The field itself is 1464 // specified by the initializer list. 1465 if (!E->getInitializedFieldInUnion()) { 1466 // Empty union; we have nothing to do. 1467 1468#ifndef NDEBUG 1469 // Make sure that it's really an empty and not a failure of 1470 // semantic analysis. 1471 for (const auto *Field : record->fields()) 1472 assert(Field->isUnnamedBitfield() && "Only unnamed bitfields allowed"); 1473#endif 1474 return; 1475 } 1476 1477 // FIXME: volatility 1478 FieldDecl *Field = E->getInitializedFieldInUnion(); 1479 1480 LValue FieldLoc = CGF.EmitLValueForFieldInitialization(DestLV, Field); 1481 if (NumInitElements) { 1482 // Store the initializer into the field 1483 EmitInitializationToLValue(E->getInit(0), FieldLoc); 1484 } else { 1485 // Default-initialize to null. 1486 EmitNullInitializationToLValue(FieldLoc); 1487 } 1488 1489 return; 1490 } 1491 1492 // Here we iterate over the fields; this makes it simpler to both 1493 // default-initialize fields and skip over unnamed fields. 1494 for (const auto *field : record->fields()) { 1495 // We're done once we hit the flexible array member. 1496 if (field->getType()->isIncompleteArrayType()) 1497 break; 1498 1499 // Always skip anonymous bitfields. 1500 if (field->isUnnamedBitfield()) 1501 continue; 1502 1503 // We're done if we reach the end of the explicit initializers, we 1504 // have a zeroed object, and the rest of the fields are 1505 // zero-initializable. 1506 if (curInitIndex == NumInitElements && Dest.isZeroed() && 1507 CGF.getTypes().isZeroInitializable(E->getType())) 1508 break; 1509 1510 1511 LValue LV = CGF.EmitLValueForFieldInitialization(DestLV, field); 1512 // We never generate write-barries for initialized fields. 1513 LV.setNonGC(true); 1514 1515 if (curInitIndex < NumInitElements) { 1516 // Store the initializer into the field. 1517 EmitInitializationToLValue(E->getInit(curInitIndex++), LV); 1518 } else { 1519 // We're out of initializers; default-initialize to null 1520 EmitNullInitializationToLValue(LV); 1521 } 1522 1523 // Push a destructor if necessary. 1524 // FIXME: if we have an array of structures, all explicitly 1525 // initialized, we can end up pushing a linear number of cleanups. 1526 bool pushedCleanup = false; 1527 if (QualType::DestructionKind dtorKind 1528 = field->getType().isDestructedType()) { 1529 assert(LV.isSimple()); 1530 if (CGF.needsEHCleanup(dtorKind)) { 1531 if (!cleanupDominator) 1532 cleanupDominator = CGF.Builder.CreateAlignedLoad( 1533 CGF.Int8Ty, 1534 llvm::Constant::getNullValue(CGF.Int8PtrTy), 1535 CharUnits::One()); // placeholder 1536 1537 CGF.pushDestroy(EHCleanup, LV.getAddress(), field->getType(), 1538 CGF.getDestroyer(dtorKind), false); 1539 cleanups.push_back(CGF.EHStack.stable_begin()); 1540 pushedCleanup = true; 1541 } 1542 } 1543 1544 // If the GEP didn't get used because of a dead zero init or something 1545 // else, clean it up for -O0 builds and general tidiness. 1546 if (!pushedCleanup && LV.isSimple()) 1547 if (llvm::GetElementPtrInst *GEP = 1548 dyn_cast<llvm::GetElementPtrInst>(LV.getPointer())) 1549 if (GEP->use_empty()) 1550 GEP->eraseFromParent(); 1551 } 1552 1553 // Deactivate all the partial cleanups in reverse order, which 1554 // generally means popping them. 1555 for (unsigned i = cleanups.size(); i != 0; --i) 1556 CGF.DeactivateCleanupBlock(cleanups[i-1], cleanupDominator); 1557 1558 // Destroy the placeholder if we made one. 1559 if (cleanupDominator) 1560 cleanupDominator->eraseFromParent(); 1561} 1562 1563void AggExprEmitter::VisitArrayInitLoopExpr(const ArrayInitLoopExpr *E, 1564 llvm::Value *outerBegin) { 1565 // Emit the common subexpression. 1566 CodeGenFunction::OpaqueValueMapping binding(CGF, E->getCommonExpr()); 1567 1568 Address destPtr = EnsureSlot(E->getType()).getAddress(); 1569 uint64_t numElements = E->getArraySize().getZExtValue(); 1570 1571 if (!numElements) 1572 return; 1573 1574 // destPtr is an array*. Construct an elementType* by drilling down a level. 1575 llvm::Value *zero = llvm::ConstantInt::get(CGF.SizeTy, 0); 1576 llvm::Value *indices[] = {zero, zero}; 1577 llvm::Value *begin = Builder.CreateInBoundsGEP(destPtr.getPointer(), indices, 1578 "arrayinit.begin"); 1579 1580 // Prepare to special-case multidimensional array initialization: we avoid 1581 // emitting multiple destructor loops in that case. 1582 if (!outerBegin) 1583 outerBegin = begin; 1584 ArrayInitLoopExpr *InnerLoop = dyn_cast<ArrayInitLoopExpr>(E->getSubExpr()); 1585 1586 QualType elementType = 1587 CGF.getContext().getAsArrayType(E->getType())->getElementType(); 1588 CharUnits elementSize = CGF.getContext().getTypeSizeInChars(elementType); 1589 CharUnits elementAlign = 1590 destPtr.getAlignment().alignmentOfArrayElement(elementSize); 1591 1592 llvm::BasicBlock *entryBB = Builder.GetInsertBlock(); 1593 llvm::BasicBlock *bodyBB = CGF.createBasicBlock("arrayinit.body"); 1594 1595 // Jump into the body. 1596 CGF.EmitBlock(bodyBB); 1597 llvm::PHINode *index = 1598 Builder.CreatePHI(zero->getType(), 2, "arrayinit.index"); 1599 index->addIncoming(zero, entryBB); 1600 llvm::Value *element = Builder.CreateInBoundsGEP(begin, index); 1601 1602 // Prepare for a cleanup. 1603 QualType::DestructionKind dtorKind = elementType.isDestructedType(); 1604 EHScopeStack::stable_iterator cleanup; 1605 if (CGF.needsEHCleanup(dtorKind) && !InnerLoop) { 1606 if (outerBegin->getType() != element->getType()) 1607 outerBegin = Builder.CreateBitCast(outerBegin, element->getType()); 1608 CGF.pushRegularPartialArrayCleanup(outerBegin, element, elementType, 1609 elementAlign, 1610 CGF.getDestroyer(dtorKind)); 1611 cleanup = CGF.EHStack.stable_begin(); 1612 } else { 1613 dtorKind = QualType::DK_none; 1614 } 1615 1616 // Emit the actual filler expression. 1617 { 1618 // Temporaries created in an array initialization loop are destroyed 1619 // at the end of each iteration. 1620 CodeGenFunction::RunCleanupsScope CleanupsScope(CGF); 1621 CodeGenFunction::ArrayInitLoopExprScope Scope(CGF, index); 1622 LValue elementLV = 1623 CGF.MakeAddrLValue(Address(element, elementAlign), elementType); 1624 1625 if (InnerLoop) { 1626 // If the subexpression is an ArrayInitLoopExpr, share its cleanup. 1627 auto elementSlot = AggValueSlot::forLValue( 1628 elementLV, AggValueSlot::IsDestructed, 1629 AggValueSlot::DoesNotNeedGCBarriers, 1630 AggValueSlot::IsNotAliased, 1631 AggValueSlot::DoesNotOverlap); 1632 AggExprEmitter(CGF, elementSlot, false) 1633 .VisitArrayInitLoopExpr(InnerLoop, outerBegin); 1634 } else 1635 EmitInitializationToLValue(E->getSubExpr(), elementLV); 1636 } 1637 1638 // Move on to the next element. 1639 llvm::Value *nextIndex = Builder.CreateNUWAdd( 1640 index, llvm::ConstantInt::get(CGF.SizeTy, 1), "arrayinit.next"); 1641 index->addIncoming(nextIndex, Builder.GetInsertBlock()); 1642 1643 // Leave the loop if we're done. 1644 llvm::Value *done = Builder.CreateICmpEQ( 1645 nextIndex, llvm::ConstantInt::get(CGF.SizeTy, numElements), 1646 "arrayinit.done"); 1647 llvm::BasicBlock *endBB = CGF.createBasicBlock("arrayinit.end"); 1648 Builder.CreateCondBr(done, endBB, bodyBB); 1649 1650 CGF.EmitBlock(endBB); 1651 1652 // Leave the partial-array cleanup if we entered one. 1653 if (dtorKind) 1654 CGF.DeactivateCleanupBlock(cleanup, index); 1655} 1656 1657void AggExprEmitter::VisitDesignatedInitUpdateExpr(DesignatedInitUpdateExpr *E) { 1658 AggValueSlot Dest = EnsureSlot(E->getType()); 1659 1660 LValue DestLV = CGF.MakeAddrLValue(Dest.getAddress(), E->getType()); 1661 EmitInitializationToLValue(E->getBase(), DestLV); 1662 VisitInitListExpr(E->getUpdater()); 1663} 1664 1665//===----------------------------------------------------------------------===// 1666// Entry Points into this File 1667//===----------------------------------------------------------------------===// 1668 1669/// GetNumNonZeroBytesInInit - Get an approximate count of the number of 1670/// non-zero bytes that will be stored when outputting the initializer for the 1671/// specified initializer expression. 1672static CharUnits GetNumNonZeroBytesInInit(const Expr *E, CodeGenFunction &CGF) { 1673 E = E->IgnoreParens(); 1674 1675 // 0 and 0.0 won't require any non-zero stores! 1676 if (isSimpleZero(E, CGF)) return CharUnits::Zero(); 1677 1678 // If this is an initlist expr, sum up the size of sizes of the (present) 1679 // elements. If this is something weird, assume the whole thing is non-zero. 1680 const InitListExpr *ILE = dyn_cast<InitListExpr>(E); 1681 while (ILE && ILE->isTransparent()) 1682 ILE = dyn_cast<InitListExpr>(ILE->getInit(0)); 1683 if (!ILE || !CGF.getTypes().isZeroInitializable(ILE->getType())) 1684 return CGF.getContext().getTypeSizeInChars(E->getType()); 1685 1686 // InitListExprs for structs have to be handled carefully. If there are 1687 // reference members, we need to consider the size of the reference, not the 1688 // referencee. InitListExprs for unions and arrays can't have references. 1689 if (const RecordType *RT = E->getType()->getAs<RecordType>()) { 1690 if (!RT->isUnionType()) { 1691 RecordDecl *SD = E->getType()->getAs<RecordType>()->getDecl(); 1692 CharUnits NumNonZeroBytes = CharUnits::Zero(); 1693 1694 unsigned ILEElement = 0; 1695 if (auto *CXXRD = dyn_cast<CXXRecordDecl>(SD)) 1696 while (ILEElement != CXXRD->getNumBases()) 1697 NumNonZeroBytes += 1698 GetNumNonZeroBytesInInit(ILE->getInit(ILEElement++), CGF); 1699 for (const auto *Field : SD->fields()) { 1700 // We're done once we hit the flexible array member or run out of 1701 // InitListExpr elements. 1702 if (Field->getType()->isIncompleteArrayType() || 1703 ILEElement == ILE->getNumInits()) 1704 break; 1705 if (Field->isUnnamedBitfield()) 1706 continue; 1707 1708 const Expr *E = ILE->getInit(ILEElement++); 1709 1710 // Reference values are always non-null and have the width of a pointer. 1711 if (Field->getType()->isReferenceType()) 1712 NumNonZeroBytes += CGF.getContext().toCharUnitsFromBits( 1713 CGF.getTarget().getPointerWidth(0)); 1714 else 1715 NumNonZeroBytes += GetNumNonZeroBytesInInit(E, CGF); 1716 } 1717 1718 return NumNonZeroBytes; 1719 } 1720 } 1721 1722 1723 CharUnits NumNonZeroBytes = CharUnits::Zero(); 1724 for (unsigned i = 0, e = ILE->getNumInits(); i != e; ++i) 1725 NumNonZeroBytes += GetNumNonZeroBytesInInit(ILE->getInit(i), CGF); 1726 return NumNonZeroBytes; 1727} 1728 1729/// CheckAggExprForMemSetUse - If the initializer is large and has a lot of 1730/// zeros in it, emit a memset and avoid storing the individual zeros. 1731/// 1732static void CheckAggExprForMemSetUse(AggValueSlot &Slot, const Expr *E, 1733 CodeGenFunction &CGF) { 1734 // If the slot is already known to be zeroed, nothing to do. Don't mess with 1735 // volatile stores. 1736 if (Slot.isZeroed() || Slot.isVolatile() || !Slot.getAddress().isValid()) 1737 return; 1738 1739 // C++ objects with a user-declared constructor don't need zero'ing. 1740 if (CGF.getLangOpts().CPlusPlus) 1741 if (const RecordType *RT = CGF.getContext() 1742 .getBaseElementType(E->getType())->getAs<RecordType>()) { 1743 const CXXRecordDecl *RD = cast<CXXRecordDecl>(RT->getDecl()); 1744 if (RD->hasUserDeclaredConstructor()) 1745 return; 1746 } 1747 1748 // If the type is 16-bytes or smaller, prefer individual stores over memset. 1749 CharUnits Size = Slot.getPreferredSize(CGF.getContext(), E->getType()); 1750 if (Size <= CharUnits::fromQuantity(16)) 1751 return; 1752 1753 // Check to see if over 3/4 of the initializer are known to be zero. If so, 1754 // we prefer to emit memset + individual stores for the rest. 1755 CharUnits NumNonZeroBytes = GetNumNonZeroBytesInInit(E, CGF); 1756 if (NumNonZeroBytes*4 > Size) 1757 return; 1758 1759 // Okay, it seems like a good idea to use an initial memset, emit the call. 1760 llvm::Constant *SizeVal = CGF.Builder.getInt64(Size.getQuantity()); 1761 1762 Address Loc = Slot.getAddress(); 1763 Loc = CGF.Builder.CreateElementBitCast(Loc, CGF.Int8Ty); 1764 CGF.Builder.CreateMemSet(Loc, CGF.Builder.getInt8(0), SizeVal, false); 1765 1766 // Tell the AggExprEmitter that the slot is known zero. 1767 Slot.setZeroed(); 1768} 1769 1770 1771 1772 1773/// EmitAggExpr - Emit the computation of the specified expression of aggregate 1774/// type. The result is computed into DestPtr. Note that if DestPtr is null, 1775/// the value of the aggregate expression is not needed. If VolatileDest is 1776/// true, DestPtr cannot be 0. 1777void CodeGenFunction::EmitAggExpr(const Expr *E, AggValueSlot Slot) { 1778 assert(E && hasAggregateEvaluationKind(E->getType()) && 1779 "Invalid aggregate expression to emit"); 1780 assert((Slot.getAddress().isValid() || Slot.isIgnored()) && 1781 "slot has bits but no address"); 1782 1783 // Optimize the slot if possible. 1784 CheckAggExprForMemSetUse(Slot, E, *this); 1785 1786 AggExprEmitter(*this, Slot, Slot.isIgnored()).Visit(const_cast<Expr*>(E)); 1787} 1788 1789LValue CodeGenFunction::EmitAggExprToLValue(const Expr *E) { 1790 assert(hasAggregateEvaluationKind(E->getType()) && "Invalid argument!"); 1791 Address Temp = CreateMemTemp(E->getType()); 1792 LValue LV = MakeAddrLValue(Temp, E->getType()); 1793 EmitAggExpr(E, AggValueSlot::forLValue(LV, AggValueSlot::IsNotDestructed, 1794 AggValueSlot::DoesNotNeedGCBarriers, 1795 AggValueSlot::IsNotAliased, 1796 AggValueSlot::DoesNotOverlap)); 1797 return LV; 1798} 1799 1800AggValueSlot::Overlap_t CodeGenFunction::overlapForBaseInit( 1801 const CXXRecordDecl *RD, const CXXRecordDecl *BaseRD, bool IsVirtual) { 1802 // Virtual bases are initialized first, in address order, so there's never 1803 // any overlap during their initialization. 1804 // 1805 // FIXME: Under P0840, this is no longer true: the tail padding of a vbase 1806 // of a field could be reused by a vbase of a containing class. 1807 if (IsVirtual) 1808 return AggValueSlot::DoesNotOverlap; 1809 1810 // If the base class is laid out entirely within the nvsize of the derived 1811 // class, its tail padding cannot yet be initialized, so we can issue 1812 // stores at the full width of the base class. 1813 const ASTRecordLayout &Layout = getContext().getASTRecordLayout(RD); 1814 if (Layout.getBaseClassOffset(BaseRD) + 1815 getContext().getASTRecordLayout(BaseRD).getSize() <= 1816 Layout.getNonVirtualSize()) 1817 return AggValueSlot::DoesNotOverlap; 1818 1819 // The tail padding may contain values we need to preserve. 1820 return AggValueSlot::MayOverlap; 1821} 1822 1823void CodeGenFunction::EmitAggregateCopy(LValue Dest, LValue Src, QualType Ty, 1824 AggValueSlot::Overlap_t MayOverlap, 1825 bool isVolatile) { 1826 assert(!Ty->isAnyComplexType() && "Shouldn't happen for complex"); 1827 1828 Address DestPtr = Dest.getAddress(); 1829 Address SrcPtr = Src.getAddress(); 1830 1831 if (getLangOpts().CPlusPlus) { 1832 if (const RecordType *RT = Ty->getAs<RecordType>()) { 1833 CXXRecordDecl *Record = cast<CXXRecordDecl>(RT->getDecl()); 1834 assert((Record->hasTrivialCopyConstructor() || 1835 Record->hasTrivialCopyAssignment() || 1836 Record->hasTrivialMoveConstructor() || 1837 Record->hasTrivialMoveAssignment() || 1838 Record->isUnion()) && 1839 "Trying to aggregate-copy a type without a trivial copy/move " 1840 "constructor or assignment operator"); 1841 // Ignore empty classes in C++. 1842 if (Record->isEmpty()) 1843 return; 1844 } 1845 } 1846 1847 // Aggregate assignment turns into llvm.memcpy. This is almost valid per 1848 // C99 6.5.16.1p3, which states "If the value being stored in an object is 1849 // read from another object that overlaps in anyway the storage of the first 1850 // object, then the overlap shall be exact and the two objects shall have 1851 // qualified or unqualified versions of a compatible type." 1852 // 1853 // memcpy is not defined if the source and destination pointers are exactly 1854 // equal, but other compilers do this optimization, and almost every memcpy 1855 // implementation handles this case safely. If there is a libc that does not 1856 // safely handle this, we can add a target hook. 1857 1858 // Get data size info for this aggregate. Don't copy the tail padding if this 1859 // might be a potentially-overlapping subobject, since the tail padding might 1860 // be occupied by a different object. Otherwise, copying it is fine. 1861 std::pair<CharUnits, CharUnits> TypeInfo; 1862 if (MayOverlap) 1863 TypeInfo = getContext().getTypeInfoDataSizeInChars(Ty); 1864 else 1865 TypeInfo = getContext().getTypeInfoInChars(Ty); 1866 1867 llvm::Value *SizeVal = nullptr; 1868 if (TypeInfo.first.isZero()) { 1869 // But note that getTypeInfo returns 0 for a VLA. 1870 if (auto *VAT = dyn_cast_or_null<VariableArrayType>( 1871 getContext().getAsArrayType(Ty))) { 1872 QualType BaseEltTy; 1873 SizeVal = emitArrayLength(VAT, BaseEltTy, DestPtr); 1874 TypeInfo = getContext().getTypeInfoInChars(BaseEltTy); 1875 assert(!TypeInfo.first.isZero()); 1876 SizeVal = Builder.CreateNUWMul( 1877 SizeVal, 1878 llvm::ConstantInt::get(SizeTy, TypeInfo.first.getQuantity())); 1879 } 1880 } 1881 if (!SizeVal) { 1882 SizeVal = llvm::ConstantInt::get(SizeTy, TypeInfo.first.getQuantity()); 1883 } 1884 1885 // FIXME: If we have a volatile struct, the optimizer can remove what might 1886 // appear to be `extra' memory ops: 1887 // 1888 // volatile struct { int i; } a, b; 1889 // 1890 // int main() { 1891 // a = b; 1892 // a = b; 1893 // } 1894 // 1895 // we need to use a different call here. We use isVolatile to indicate when 1896 // either the source or the destination is volatile. 1897 1898 DestPtr = Builder.CreateElementBitCast(DestPtr, Int8Ty); 1899 SrcPtr = Builder.CreateElementBitCast(SrcPtr, Int8Ty); 1900 1901 // Don't do any of the memmove_collectable tests if GC isn't set. 1902 if (CGM.getLangOpts().getGC() == LangOptions::NonGC) { 1903 // fall through 1904 } else if (const RecordType *RecordTy = Ty->getAs<RecordType>()) { 1905 RecordDecl *Record = RecordTy->getDecl(); 1906 if (Record->hasObjectMember()) { 1907 CGM.getObjCRuntime().EmitGCMemmoveCollectable(*this, DestPtr, SrcPtr, 1908 SizeVal); 1909 return; 1910 } 1911 } else if (Ty->isArrayType()) { 1912 QualType BaseType = getContext().getBaseElementType(Ty); 1913 if (const RecordType *RecordTy = BaseType->getAs<RecordType>()) { 1914 if (RecordTy->getDecl()->hasObjectMember()) { 1915 CGM.getObjCRuntime().EmitGCMemmoveCollectable(*this, DestPtr, SrcPtr, 1916 SizeVal); 1917 return; 1918 } 1919 } 1920 } 1921 1922 auto Inst = Builder.CreateMemCpy(DestPtr, SrcPtr, SizeVal, isVolatile); 1923 1924 // Determine the metadata to describe the position of any padding in this 1925 // memcpy, as well as the TBAA tags for the members of the struct, in case 1926 // the optimizer wishes to expand it in to scalar memory operations. 1927 if (llvm::MDNode *TBAAStructTag = CGM.getTBAAStructInfo(Ty)) 1928 Inst->setMetadata(llvm::LLVMContext::MD_tbaa_struct, TBAAStructTag); 1929 1930 if (CGM.getCodeGenOpts().NewStructPathTBAA) { 1931 TBAAAccessInfo TBAAInfo = CGM.mergeTBAAInfoForMemoryTransfer( 1932 Dest.getTBAAInfo(), Src.getTBAAInfo()); 1933 CGM.DecorateInstructionWithTBAA(Inst, TBAAInfo); 1934 } 1935} 1936