Lines Matching refs:CGF
30 CodeGenFunction &CGF;
43 AtomicInfo(CodeGenFunction &CGF, LValue &lvalue)
44 : CGF(CGF), AtomicSizeInBits(0), ValueSizeInBits(0),
47 ASTContext &C = CGF.getContext();
54 EvaluationKind = CGF.getEvaluationKind(ValueTy);
83 auto VoidPtrAddr = CGF.EmitCastToVoidPtr(lvalue.getBitFieldPointer());
87 VoidPtrAddr = CGF.Builder.CreateConstGEP1_64(
89 auto Addr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
91 CGF.Builder.getIntNTy(AtomicSizeInBits)->getPointerTo(),
121 AtomicTy = ValueTy = CGF.getContext().getExtVectorType(
172 CharUnits size = CGF.getContext().toCharUnitsFromBits(AtomicSizeInBits);
173 return CGF.CGM.getSize(size);
204 addr = CGF.Builder.CreateStructGEP(addr, 0, CharUnits());
206 return LValue::MakeAddr(addr, getValueType(), CGF.getContext(),
309 Address TempAlloca = CGF.CreateMemTemp(
316 return CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
321 static RValue emitAtomicLibcall(CodeGenFunction &CGF,
326 CGF.CGM.getTypes().arrangeFreeFunctionCall(resultType, args,
328 llvm::FunctionType *fnTy = CGF.CGM.getTypes().GetFunctionType(fnInfo);
329 llvm::Constant *fn = CGF.CGM.CreateRuntimeFunction(fnTy, fnName);
330 return CGF.EmitCall(fnInfo, fn, ReturnValueSlot(), args);
351 return !isFullSizeType(CGF.CGM, type, AtomicSizeInBits);
353 return !isFullSizeType(CGF.CGM, type->getStructElementType(0),
369 CGF.Builder.CreateMemSet(
370 addr, llvm::ConstantInt::get(CGF.Int8Ty, 0),
371 CGF.getContext().toCharUnitsFromBits(AtomicSizeInBits).getQuantity(),
376 static void emitAtomicCmpXchg(CodeGenFunction &CGF, AtomicExpr *E, bool IsWeak,
383 llvm::Value *Expected = CGF.Builder.CreateLoad(Val1);
384 llvm::Value *Desired = CGF.Builder.CreateLoad(Val2);
386 llvm::AtomicCmpXchgInst *Pair = CGF.Builder.CreateAtomicCmpXchg(
393 llvm::Value *Old = CGF.Builder.CreateExtractValue(Pair, 0);
394 llvm::Value *Cmp = CGF.Builder.CreateExtractValue(Pair, 1);
399 CGF.createBasicBlock("cmpxchg.store_expected", CGF.CurFn);
404 CGF.createBasicBlock("cmpxchg.continue", CGF.CurFn);
408 CGF.Builder.CreateCondBr(Cmp, ContinueBB, StoreExpectedBB);
410 CGF.Builder.SetInsertPoint(StoreExpectedBB);
412 CGF.Builder.CreateStore(Old, Val1);
414 CGF.Builder.CreateBr(ContinueBB);
416 CGF.Builder.SetInsertPoint(ContinueBB);
418 CGF.EmitStoreOfScalar(Cmp, CGF.MakeAddrLValue(Dest, E->getType()));
424 static void emitAtomicCmpXchgFailureSet(CodeGenFunction &CGF, AtomicExpr *E,
450 emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2, Size,
458 MonotonicBB = CGF.createBasicBlock("monotonic_fail", CGF.CurFn);
460 AcquireBB = CGF.createBasicBlock("acquire_fail", CGF.CurFn);
462 SeqCstBB = CGF.createBasicBlock("seqcst_fail", CGF.CurFn);
464 llvm::BasicBlock *ContBB = CGF.createBasicBlock("atomic.continue", CGF.CurFn);
466 llvm::SwitchInst *SI = CGF.Builder.CreateSwitch(FailureOrderVal, MonotonicBB);
473 CGF.Builder.SetInsertPoint(MonotonicBB);
474 emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2,
476 CGF.Builder.CreateBr(ContBB);
479 CGF.Builder.SetInsertPoint(AcquireBB);
480 emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2,
482 CGF.Builder.CreateBr(ContBB);
483 SI->addCase(CGF.Builder.getInt32(AtomicExpr::AO_ABI_memory_order_consume),
485 SI->addCase(CGF.Builder.getInt32(AtomicExpr::AO_ABI_memory_order_acquire),
489 CGF.Builder.SetInsertPoint(SeqCstBB);
490 emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2,
492 CGF.Builder.CreateBr(ContBB);
493 SI->addCase(CGF.Builder.getInt32(AtomicExpr::AO_ABI_memory_order_seq_cst),
497 CGF.Builder.SetInsertPoint(ContBB);
500 static void EmitAtomicOp(CodeGenFunction &CGF, AtomicExpr *E, Address Dest,
512 emitAtomicCmpXchgFailureSet(CGF, E, false, Dest, Ptr, Val1, Val2,
516 emitAtomicCmpXchgFailureSet(CGF, E, true, Dest, Ptr, Val1, Val2,
522 emitAtomicCmpXchgFailureSet(CGF, E, IsWeakC->getZExtValue(), Dest, Ptr,
527 CGF.createBasicBlock("cmpxchg.strong", CGF.CurFn);
528 llvm::BasicBlock *WeakBB = CGF.createBasicBlock("cmxchg.weak", CGF.CurFn);
530 CGF.createBasicBlock("cmpxchg.continue", CGF.CurFn);
532 llvm::SwitchInst *SI = CGF.Builder.CreateSwitch(IsWeak, WeakBB);
533 SI->addCase(CGF.Builder.getInt1(false), StrongBB);
535 CGF.Builder.SetInsertPoint(StrongBB);
536 emitAtomicCmpXchgFailureSet(CGF, E, false, Dest, Ptr, Val1, Val2,
538 CGF.Builder.CreateBr(ContBB);
540 CGF.Builder.SetInsertPoint(WeakBB);
541 emitAtomicCmpXchgFailureSet(CGF, E, true, Dest, Ptr, Val1, Val2,
543 CGF.Builder.CreateBr(ContBB);
545 CGF.Builder.SetInsertPoint(ContBB);
552 llvm::LoadInst *Load = CGF.Builder.CreateLoad(Ptr);
555 CGF.Builder.CreateStore(Load, Dest);
562 llvm::Value *LoadVal1 = CGF.Builder.CreateLoad(Val1);
563 llvm::StoreInst *Store = CGF.Builder.CreateStore(LoadVal1, Ptr);
623 llvm::Value *LoadVal1 = CGF.Builder.CreateLoad(Val1);
625 CGF.Builder.CreateAtomicRMW(Op, Ptr.getPointer(), LoadVal1, Order);
632 Result = CGF.Builder.CreateBinOp(PostOp, RMWI, LoadVal1);
634 Result = CGF.Builder.CreateNot(Result);
635 CGF.Builder.CreateStore(Result, Dest);
641 EmitValToTemp(CodeGenFunction &CGF, Expr *E) {
642 Address DeclPtr = CGF.CreateMemTemp(E->getType(), ".atomictmp");
643 CGF.EmitAnyExprToMem(E, DeclPtr, E->getType().getQualifiers(),
649 AddDirectArgument(CodeGenFunction &CGF, CallArgList &Args,
654 CharUnits Align = CGF.getContext().getTypeAlignInChars(ValTy);
655 int64_t SizeInBits = CGF.getContext().toBits(SizeInChars);
657 CGF.getContext().getIntTypeForBitwidth(SizeInBits, /*Signed=*/false);
658 llvm::Type *IPtrTy = llvm::IntegerType::get(CGF.getLLVMContext(),
660 Address Ptr = Address(CGF.Builder.CreateBitCast(Val, IPtrTy), Align);
661 Val = CGF.EmitLoadOfScalar(Ptr, false,
662 CGF.getContext().getPointerType(ValTy),
668 Args.add(RValue::get(CGF.EmitCastToVoidPtr(Val)),
669 CGF.getContext().VoidPtrTy);
1158 llvm::IntegerType::get(CGF.getLLVMContext(), AtomicSizeInBits);
1159 return CGF.Builder.CreateBitCast(addr, ty->getPointerTo(addrspace));
1164 uint64_t SourceSizeInBits = CGF.CGM.getDataLayout().getTypeSizeInBits(Ty);
1167 CGF.Builder.CreateMemCpy(Tmp, Addr,
1185 addr = CGF.Builder.CreateStructGEP(addr, 0, CharUnits());
1189 return CGF.convertTempToRValue(addr, getValueType(), loc);
1193 return RValue::get(CGF.Builder.CreateLoad(addr));
1195 return CGF.EmitLoadOfBitfieldLValue(
1199 return CGF.EmitLoadOfLValue(
1203 return CGF.EmitLoadOfExtVectorElementLValue(LValue::MakeExtVectorElt(
1220 ? CGF.ConvertTypeForMem(ValueTy)
1224 return RValue::get(CGF.EmitFromMemory(IntVal, ValueTy));
1226 return RValue::get(CGF.Builder.CreateIntToPtr(IntVal, ValTy));
1228 return RValue::get(CGF.Builder.CreateBitCast(IntVal, ValTy));
1245 CGF.Builder.CreateStore(IntVal, CastTemp)
1255 Args.add(RValue::get(getAtomicSizeValue()), CGF.getContext().getSizeType());
1256 Args.add(RValue::get(CGF.EmitCastToVoidPtr(getAtomicPointer())),
1257 CGF.getContext().VoidPtrTy);
1258 Args.add(RValue::get(CGF.EmitCastToVoidPtr(AddForLoaded)),
1259 CGF.getContext().VoidPtrTy);
1261 llvm::ConstantInt::get(CGF.IntTy, translateAtomicOrdering(AO))),
1262 CGF.getContext().IntTy);
1263 emitAtomicLibcall(CGF, "__atomic_load", CGF.getContext().VoidTy, Args);
1270 llvm::LoadInst *Load = CGF.Builder.CreateLoad(Addr, "atomic-load");
1277 CGF.CGM.DecorateInstructionWithTBAA(Load, LVal.getTBAAInfo());
1366 CGF.EmitAggregateCopy(getAtomicAddress(),
1384 CGF.EmitStoreOfScalar(rvalue.getScalarVal(), TempLVal, /*init*/ true);
1386 CGF.EmitStoreOfComplex(rvalue.getComplexVal(), TempLVal, /*init*/ true);
1400 LValue TempLV = CGF.MakeAddrLValue(CreateTempAlloca(), getAtomicType());
1401 AtomicInfo Atomics(CGF, TempLV);
1412 return CGF.EmitToMemory(Value, ValueTy);
1415 CGF.getLLVMContext(),
1418 return CGF.Builder.CreatePtrToInt(Value, InputIntTy);
1420 return CGF.Builder.CreateBitCast(Value, InputIntTy);
1429 return CGF.Builder.CreateLoad(Addr);
1437 auto *Inst = CGF.Builder.CreateAtomicCmpXchg(Addr.getPointer(),
1445 auto *PreviousVal = CGF.Builder.CreateExtractValue(Inst, /*Idxs=*/0);
1446 auto *SuccessFailureVal = CGF.Builder.CreateExtractValue(Inst, /*Idxs=*/1);
1458 Args.add(RValue::get(getAtomicSizeValue()), CGF.getContext().getSizeType());
1459 Args.add(RValue::get(CGF.EmitCastToVoidPtr(getAtomicPointer())),
1460 CGF.getContext().VoidPtrTy);
1461 Args.add(RValue::get(CGF.EmitCastToVoidPtr(ExpectedAddr)),
1462 CGF.getContext().VoidPtrTy);
1463 Args.add(RValue::get(CGF.EmitCastToVoidPtr(DesiredAddr)),
1464 CGF.getContext().VoidPtrTy);
1466 CGF.IntTy, translateAtomicOrdering(Success))),
1467 CGF.getContext().IntTy);
1469 CGF.IntTy, translateAtomicOrdering(Failure))),
1470 CGF.getContext().IntTy);
1471 auto SuccessFailureRVal = emitAtomicLibcall(CGF, "__atomic_compare_exchange",
1472 CGF.getContext().BoolTy, Args);
1511 EmitAtomicUpdateValue(CodeGenFunction &CGF, AtomicInfo &Atomics, RValue OldRVal,
1519 DesiredLVal = CGF.MakeAddrLValue(DesiredAddr, AtomicLVal.getType());
1551 UpRVal = CGF.EmitLoadOfLValue(UpdateLVal, SourceLocation());
1556 CGF.EmitStoreThroughLValue(NewRVal, DesiredLVal);
1559 CGF.EmitStoreOfComplex(NewRVal.getComplexVal(), DesiredLVal,
1572 auto *ContBB = CGF.createBasicBlock("atomic_cont");
1573 auto *ExitBB = CGF.createBasicBlock("atomic_exit");
1574 CGF.EmitBlock(ContBB);
1578 auto *OldVal = CGF.Builder.CreateLoad(ExpectedAddr);
1579 CGF.Builder.CreateStore(OldVal, DesiredAddr);
1584 EmitAtomicUpdateValue(CGF, *this, OldRVal, UpdateOp, DesiredAddr);
1589 CGF.Builder.CreateCondBr(Res, ExitBB, ContBB);
1590 CGF.EmitBlock(ExitBB, /*IsFinished=*/true);
1601 auto *ContBB = CGF.createBasicBlock("atomic_cont");
1602 auto *ExitBB = CGF.createBasicBlock("atomic_exit");
1603 auto *CurBB = CGF.Builder.GetInsertBlock();
1604 CGF.EmitBlock(ContBB);
1605 llvm::PHINode *PHI = CGF.Builder.CreatePHI(OldVal->getType(),
1612 CGF.Builder.CreateStore(PHI, NewAtomicIntAddr);
1616 EmitAtomicUpdateValue(CGF, *this, OldRVal, UpdateOp, NewAtomicAddr);
1617 auto *DesiredVal = CGF.Builder.CreateLoad(NewAtomicIntAddr);
1620 PHI->addIncoming(Res.first, CGF.Builder.GetInsertBlock());
1621 CGF.Builder.CreateCondBr(Res.second, ExitBB, ContBB);
1622 CGF.EmitBlock(ExitBB, /*IsFinished=*/true);
1625 static void EmitAtomicUpdateValue(CodeGenFunction &CGF, AtomicInfo &Atomics,
1649 CGF.EmitStoreThroughLValue(UpdateRVal, DesiredLVal);
1659 auto *ContBB = CGF.createBasicBlock("atomic_cont");
1660 auto *ExitBB = CGF.createBasicBlock("atomic_exit");
1661 CGF.EmitBlock(ContBB);
1665 auto *OldVal = CGF.Builder.CreateLoad(ExpectedAddr);
1666 CGF.Builder.CreateStore(OldVal, DesiredAddr);
1668 EmitAtomicUpdateValue(CGF, *this, UpdateRVal, DesiredAddr);
1673 CGF.Builder.CreateCondBr(Res, ExitBB, ContBB);
1674 CGF.EmitBlock(ExitBB, /*IsFinished=*/true);
1684 auto *ContBB = CGF.createBasicBlock("atomic_cont");
1685 auto *ExitBB = CGF.createBasicBlock("atomic_exit");
1686 auto *CurBB = CGF.Builder.GetInsertBlock();
1687 CGF.EmitBlock(ContBB);
1688 llvm::PHINode *PHI = CGF.Builder.CreatePHI(OldVal->getType(),
1695 CGF.Builder.CreateStore(PHI, NewAtomicIntAddr);
1697 EmitAtomicUpdateValue(CGF, *this, UpdateRVal, NewAtomicAddr);
1698 auto *DesiredVal = CGF.Builder.CreateLoad(NewAtomicIntAddr);
1701 PHI->addIncoming(Res.first, CGF.Builder.GetInsertBlock());
1702 CGF.Builder.CreateCondBr(Res.second, ExitBB, ContBB);
1703 CGF.EmitBlock(ExitBB, /*IsFinished=*/true);