CGAtomic.cpp revision 1.1.1.4 
1//===--- CGAtomic.cpp - Emit LLVM IR for atomic operations ----------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file contains the code for emitting atomic operations.
10//
11//===----------------------------------------------------------------------===//
12
13#include "CGCall.h"
14#include "CGRecordLayout.h"
15#include "CodeGenFunction.h"
16#include "CodeGenModule.h"
17#include "TargetInfo.h"
18#include "clang/AST/ASTContext.h"
19#include "clang/CodeGen/CGFunctionInfo.h"
20#include "clang/Frontend/FrontendDiagnostic.h"
21#include "llvm/ADT/DenseMap.h"
22#include "llvm/IR/DataLayout.h"
23#include "llvm/IR/Intrinsics.h"
24#include "llvm/IR/Operator.h"
25
26using namespace clang;
27using namespace CodeGen;
28
29namespace {
30  class AtomicInfo {
31    CodeGenFunction &CGF;
32    QualType AtomicTy;
33    QualType ValueTy;
34    uint64_t AtomicSizeInBits;
35    uint64_t ValueSizeInBits;
36    CharUnits AtomicAlign;
37    CharUnits ValueAlign;
38    TypeEvaluationKind EvaluationKind;
39    bool UseLibcall;
40    LValue LVal;
41    CGBitFieldInfo BFI;
42  public:
43    AtomicInfo(CodeGenFunction &CGF, LValue &lvalue)
44        : CGF(CGF), AtomicSizeInBits(0), ValueSizeInBits(0),
45          EvaluationKind(TEK_Scalar), UseLibcall(true) {
46      assert(!lvalue.isGlobalReg());
47      ASTContext &C = CGF.getContext();
48      if (lvalue.isSimple()) {
49        AtomicTy = lvalue.getType();
50        if (auto *ATy = AtomicTy->getAs<AtomicType>())
51          ValueTy = ATy->getValueType();
52        else
53          ValueTy = AtomicTy;
54        EvaluationKind = CGF.getEvaluationKind(ValueTy);
55
56        uint64_t ValueAlignInBits;
57        uint64_t AtomicAlignInBits;
58        TypeInfo ValueTI = C.getTypeInfo(ValueTy);
59        ValueSizeInBits = ValueTI.Width;
60        ValueAlignInBits = ValueTI.Align;
61
62        TypeInfo AtomicTI = C.getTypeInfo(AtomicTy);
63        AtomicSizeInBits = AtomicTI.Width;
64        AtomicAlignInBits = AtomicTI.Align;
65
66        assert(ValueSizeInBits <= AtomicSizeInBits);
67        assert(ValueAlignInBits <= AtomicAlignInBits);
68
69        AtomicAlign = C.toCharUnitsFromBits(AtomicAlignInBits);
70        ValueAlign = C.toCharUnitsFromBits(ValueAlignInBits);
71        if (lvalue.getAlignment().isZero())
72          lvalue.setAlignment(AtomicAlign);
73
74        LVal = lvalue;
75      } else if (lvalue.isBitField()) {
76        ValueTy = lvalue.getType();
77        ValueSizeInBits = C.getTypeSize(ValueTy);
78        auto &OrigBFI = lvalue.getBitFieldInfo();
79        auto Offset = OrigBFI.Offset % C.toBits(lvalue.getAlignment());
80        AtomicSizeInBits = C.toBits(
81            C.toCharUnitsFromBits(Offset + OrigBFI.Size + C.getCharWidth() - 1)
82                .alignTo(lvalue.getAlignment()));
83        auto VoidPtrAddr = CGF.EmitCastToVoidPtr(lvalue.getBitFieldPointer());
84        auto OffsetInChars =
85            (C.toCharUnitsFromBits(OrigBFI.Offset) / lvalue.getAlignment()) *
86            lvalue.getAlignment();
87        VoidPtrAddr = CGF.Builder.CreateConstGEP1_64(
88            CGF.Int8Ty, VoidPtrAddr, OffsetInChars.getQuantity());
89        llvm::Type *IntTy = CGF.Builder.getIntNTy(AtomicSizeInBits);
90        auto Addr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
91            VoidPtrAddr, IntTy->getPointerTo(), "atomic_bitfield_base");
92        BFI = OrigBFI;
93        BFI.Offset = Offset;
94        BFI.StorageSize = AtomicSizeInBits;
95        BFI.StorageOffset += OffsetInChars;
96        LVal = LValue::MakeBitfield(Address(Addr, IntTy, lvalue.getAlignment()),
97                                    BFI, lvalue.getType(), lvalue.getBaseInfo(),
98                                    lvalue.getTBAAInfo());
99        AtomicTy = C.getIntTypeForBitwidth(AtomicSizeInBits, OrigBFI.IsSigned);
100        if (AtomicTy.isNull()) {
101          llvm::APInt Size(
102              /*numBits=*/32,
103              C.toCharUnitsFromBits(AtomicSizeInBits).getQuantity());
104          AtomicTy =
105              C.getConstantArrayType(C.CharTy, Size, nullptr, ArrayType::Normal,
106                                     /*IndexTypeQuals=*/0);
107        }
108        AtomicAlign = ValueAlign = lvalue.getAlignment();
109      } else if (lvalue.isVectorElt()) {
110        ValueTy = lvalue.getType()->castAs<VectorType>()->getElementType();
111        ValueSizeInBits = C.getTypeSize(ValueTy);
112        AtomicTy = lvalue.getType();
113        AtomicSizeInBits = C.getTypeSize(AtomicTy);
114        AtomicAlign = ValueAlign = lvalue.getAlignment();
115        LVal = lvalue;
116      } else {
117        assert(lvalue.isExtVectorElt());
118        ValueTy = lvalue.getType();
119        ValueSizeInBits = C.getTypeSize(ValueTy);
120        AtomicTy = ValueTy = CGF.getContext().getExtVectorType(
121            lvalue.getType(), cast<llvm::FixedVectorType>(
122                                  lvalue.getExtVectorAddress().getElementType())
123                                  ->getNumElements());
124        AtomicSizeInBits = C.getTypeSize(AtomicTy);
125        AtomicAlign = ValueAlign = lvalue.getAlignment();
126        LVal = lvalue;
127      }
128      UseLibcall = !C.getTargetInfo().hasBuiltinAtomic(
129          AtomicSizeInBits, C.toBits(lvalue.getAlignment()));
130    }
131
132    QualType getAtomicType() const { return AtomicTy; }
133    QualType getValueType() const { return ValueTy; }
134    CharUnits getAtomicAlignment() const { return AtomicAlign; }
135    uint64_t getAtomicSizeInBits() const { return AtomicSizeInBits; }
136    uint64_t getValueSizeInBits() const { return ValueSizeInBits; }
137    TypeEvaluationKind getEvaluationKind() const { return EvaluationKind; }
138    bool shouldUseLibcall() const { return UseLibcall; }
139    const LValue &getAtomicLValue() const { return LVal; }
140    llvm::Value *getAtomicPointer() const {
141      if (LVal.isSimple())
142        return LVal.getPointer(CGF);
143      else if (LVal.isBitField())
144        return LVal.getBitFieldPointer();
145      else if (LVal.isVectorElt())
146        return LVal.getVectorPointer();
147      assert(LVal.isExtVectorElt());
148      return LVal.getExtVectorPointer();
149    }
150    Address getAtomicAddress() const {
151      llvm::Type *ElTy;
152      if (LVal.isSimple())
153        ElTy = LVal.getAddress(CGF).getElementType();
154      else if (LVal.isBitField())
155        ElTy = LVal.getBitFieldAddress().getElementType();
156      else if (LVal.isVectorElt())
157        ElTy = LVal.getVectorAddress().getElementType();
158      else
159        ElTy = LVal.getExtVectorAddress().getElementType();
160      return Address(getAtomicPointer(), ElTy, getAtomicAlignment());
161    }
162
163    Address getAtomicAddressAsAtomicIntPointer() const {
164      return emitCastToAtomicIntPointer(getAtomicAddress());
165    }
166
167    /// Is the atomic size larger than the underlying value type?
168    ///
169    /// Note that the absence of padding does not mean that atomic
170    /// objects are completely interchangeable with non-atomic
171    /// objects: we might have promoted the alignment of a type
172    /// without making it bigger.
173    bool hasPadding() const {
174      return (ValueSizeInBits != AtomicSizeInBits);
175    }
176
177    bool emitMemSetZeroIfNecessary() const;
178
179    llvm::Value *getAtomicSizeValue() const {
180      CharUnits size = CGF.getContext().toCharUnitsFromBits(AtomicSizeInBits);
181      return CGF.CGM.getSize(size);
182    }
183
184    /// Cast the given pointer to an integer pointer suitable for atomic
185    /// operations if the source.
186    Address emitCastToAtomicIntPointer(Address Addr) const;
187
188    /// If Addr is compatible with the iN that will be used for an atomic
189    /// operation, bitcast it. Otherwise, create a temporary that is suitable
190    /// and copy the value across.
191    Address convertToAtomicIntPointer(Address Addr) const;
192
193    /// Turn an atomic-layout object into an r-value.
194    RValue convertAtomicTempToRValue(Address addr, AggValueSlot resultSlot,
195                                     SourceLocation loc, bool AsValue) const;
196
197    /// Converts a rvalue to integer value.
198    llvm::Value *convertRValueToInt(RValue RVal) const;
199
200    RValue ConvertIntToValueOrAtomic(llvm::Value *IntVal,
201                                     AggValueSlot ResultSlot,
202                                     SourceLocation Loc, bool AsValue) const;
203
204    /// Copy an atomic r-value into atomic-layout memory.
205    void emitCopyIntoMemory(RValue rvalue) const;
206
207    /// Project an l-value down to the value field.
208    LValue projectValue() const {
209      assert(LVal.isSimple());
210      Address addr = getAtomicAddress();
211      if (hasPadding())
212        addr = CGF.Builder.CreateStructGEP(addr, 0);
213
214      return LValue::MakeAddr(addr, getValueType(), CGF.getContext(),
215                              LVal.getBaseInfo(), LVal.getTBAAInfo());
216    }
217
218    /// Emits atomic load.
219    /// \returns Loaded value.
220    RValue EmitAtomicLoad(AggValueSlot ResultSlot, SourceLocation Loc,
221                          bool AsValue, llvm::AtomicOrdering AO,
222                          bool IsVolatile);
223
224    /// Emits atomic compare-and-exchange sequence.
225    /// \param Expected Expected value.
226    /// \param Desired Desired value.
227    /// \param Success Atomic ordering for success operation.
228    /// \param Failure Atomic ordering for failed operation.
229    /// \param IsWeak true if atomic operation is weak, false otherwise.
230    /// \returns Pair of values: previous value from storage (value type) and
231    /// boolean flag (i1 type) with true if success and false otherwise.
232    std::pair<RValue, llvm::Value *>
233    EmitAtomicCompareExchange(RValue Expected, RValue Desired,
234                              llvm::AtomicOrdering Success =
235                                  llvm::AtomicOrdering::SequentiallyConsistent,
236                              llvm::AtomicOrdering Failure =
237                                  llvm::AtomicOrdering::SequentiallyConsistent,
238                              bool IsWeak = false);
239
240    /// Emits atomic update.
241    /// \param AO Atomic ordering.
242    /// \param UpdateOp Update operation for the current lvalue.
243    void EmitAtomicUpdate(llvm::AtomicOrdering AO,
244                          const llvm::function_ref<RValue(RValue)> &UpdateOp,
245                          bool IsVolatile);
246    /// Emits atomic update.
247    /// \param AO Atomic ordering.
248    void EmitAtomicUpdate(llvm::AtomicOrdering AO, RValue UpdateRVal,
249                          bool IsVolatile);
250
251    /// Materialize an atomic r-value in atomic-layout memory.
252    Address materializeRValue(RValue rvalue) const;
253
254    /// Creates temp alloca for intermediate operations on atomic value.
255    Address CreateTempAlloca() const;
256  private:
257    bool requiresMemSetZero(llvm::Type *type) const;
258
259
260    /// Emits atomic load as a libcall.
261    void EmitAtomicLoadLibcall(llvm::Value *AddForLoaded,
262                               llvm::AtomicOrdering AO, bool IsVolatile);
263    /// Emits atomic load as LLVM instruction.
264    llvm::Value *EmitAtomicLoadOp(llvm::AtomicOrdering AO, bool IsVolatile);
265    /// Emits atomic compare-and-exchange op as a libcall.
266    llvm::Value *EmitAtomicCompareExchangeLibcall(
267        llvm::Value *ExpectedAddr, llvm::Value *DesiredAddr,
268        llvm::AtomicOrdering Success =
269            llvm::AtomicOrdering::SequentiallyConsistent,
270        llvm::AtomicOrdering Failure =
271            llvm::AtomicOrdering::SequentiallyConsistent);
272    /// Emits atomic compare-and-exchange op as LLVM instruction.
273    std::pair<llvm::Value *, llvm::Value *> EmitAtomicCompareExchangeOp(
274        llvm::Value *ExpectedVal, llvm::Value *DesiredVal,
275        llvm::AtomicOrdering Success =
276            llvm::AtomicOrdering::SequentiallyConsistent,
277        llvm::AtomicOrdering Failure =
278            llvm::AtomicOrdering::SequentiallyConsistent,
279        bool IsWeak = false);
280    /// Emit atomic update as libcalls.
281    void
282    EmitAtomicUpdateLibcall(llvm::AtomicOrdering AO,
283                            const llvm::function_ref<RValue(RValue)> &UpdateOp,
284                            bool IsVolatile);
285    /// Emit atomic update as LLVM instructions.
286    void EmitAtomicUpdateOp(llvm::AtomicOrdering AO,
287                            const llvm::function_ref<RValue(RValue)> &UpdateOp,
288                            bool IsVolatile);
289    /// Emit atomic update as libcalls.
290    void EmitAtomicUpdateLibcall(llvm::AtomicOrdering AO, RValue UpdateRVal,
291                                 bool IsVolatile);
292    /// Emit atomic update as LLVM instructions.
293    void EmitAtomicUpdateOp(llvm::AtomicOrdering AO, RValue UpdateRal,
294                            bool IsVolatile);
295  };
296}
297
298Address AtomicInfo::CreateTempAlloca() const {
299  Address TempAlloca = CGF.CreateMemTemp(
300      (LVal.isBitField() && ValueSizeInBits > AtomicSizeInBits) ? ValueTy
301                                                                : AtomicTy,
302      getAtomicAlignment(),
303      "atomic-temp");
304  // Cast to pointer to value type for bitfields.
305  if (LVal.isBitField())
306    return CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
307        TempAlloca, getAtomicAddress().getType(),
308        getAtomicAddress().getElementType());
309  return TempAlloca;
310}
311
312static RValue emitAtomicLibcall(CodeGenFunction &CGF,
313                                StringRef fnName,
314                                QualType resultType,
315                                CallArgList &args) {
316  const CGFunctionInfo &fnInfo =
317    CGF.CGM.getTypes().arrangeBuiltinFunctionCall(resultType, args);
318  llvm::FunctionType *fnTy = CGF.CGM.getTypes().GetFunctionType(fnInfo);
319  llvm::AttrBuilder fnAttrB(CGF.getLLVMContext());
320  fnAttrB.addAttribute(llvm::Attribute::NoUnwind);
321  fnAttrB.addAttribute(llvm::Attribute::WillReturn);
322  llvm::AttributeList fnAttrs = llvm::AttributeList::get(
323      CGF.getLLVMContext(), llvm::AttributeList::FunctionIndex, fnAttrB);
324
325  llvm::FunctionCallee fn =
326      CGF.CGM.CreateRuntimeFunction(fnTy, fnName, fnAttrs);
327  auto callee = CGCallee::forDirect(fn);
328  return CGF.EmitCall(fnInfo, callee, ReturnValueSlot(), args);
329}
330
331/// Does a store of the given IR type modify the full expected width?
332static bool isFullSizeType(CodeGenModule &CGM, llvm::Type *type,
333                           uint64_t expectedSize) {
334  return (CGM.getDataLayout().getTypeStoreSize(type) * 8 == expectedSize);
335}
336
337/// Does the atomic type require memsetting to zero before initialization?
338///
339/// The IR type is provided as a way of making certain queries faster.
340bool AtomicInfo::requiresMemSetZero(llvm::Type *type) const {
341  // If the atomic type has size padding, we definitely need a memset.
342  if (hasPadding()) return true;
343
344  // Otherwise, do some simple heuristics to try to avoid it:
345  switch (getEvaluationKind()) {
346  // For scalars and complexes, check whether the store size of the
347  // type uses the full size.
348  case TEK_Scalar:
349    return !isFullSizeType(CGF.CGM, type, AtomicSizeInBits);
350  case TEK_Complex:
351    return !isFullSizeType(CGF.CGM, type->getStructElementType(0),
352                           AtomicSizeInBits / 2);
353
354  // Padding in structs has an undefined bit pattern.  User beware.
355  case TEK_Aggregate:
356    return false;
357  }
358  llvm_unreachable("bad evaluation kind");
359}
360
361bool AtomicInfo::emitMemSetZeroIfNecessary() const {
362  assert(LVal.isSimple());
363  Address addr = LVal.getAddress(CGF);
364  if (!requiresMemSetZero(addr.getElementType()))
365    return false;
366
367  CGF.Builder.CreateMemSet(
368      addr.getPointer(), llvm::ConstantInt::get(CGF.Int8Ty, 0),
369      CGF.getContext().toCharUnitsFromBits(AtomicSizeInBits).getQuantity(),
370      LVal.getAlignment().getAsAlign());
371  return true;
372}
373
374static void emitAtomicCmpXchg(CodeGenFunction &CGF, AtomicExpr *E, bool IsWeak,
375                              Address Dest, Address Ptr,
376                              Address Val1, Address Val2,
377                              uint64_t Size,
378                              llvm::AtomicOrdering SuccessOrder,
379                              llvm::AtomicOrdering FailureOrder,
380                              llvm::SyncScope::ID Scope) {
381  // Note that cmpxchg doesn't support weak cmpxchg, at least at the moment.
382  llvm::Value *Expected = CGF.Builder.CreateLoad(Val1);
383  llvm::Value *Desired = CGF.Builder.CreateLoad(Val2);
384
385  llvm::AtomicCmpXchgInst *Pair = CGF.Builder.CreateAtomicCmpXchg(
386      Ptr.getPointer(), Expected, Desired, SuccessOrder, FailureOrder,
387      Scope);
388  Pair->setVolatile(E->isVolatile());
389  Pair->setWeak(IsWeak);
390
391  // Cmp holds the result of the compare-exchange operation: true on success,
392  // false on failure.
393  llvm::Value *Old = CGF.Builder.CreateExtractValue(Pair, 0);
394  llvm::Value *Cmp = CGF.Builder.CreateExtractValue(Pair, 1);
395
396  // This basic block is used to hold the store instruction if the operation
397  // failed.
398  llvm::BasicBlock *StoreExpectedBB =
399      CGF.createBasicBlock("cmpxchg.store_expected", CGF.CurFn);
400
401  // This basic block is the exit point of the operation, we should end up
402  // here regardless of whether or not the operation succeeded.
403  llvm::BasicBlock *ContinueBB =
404      CGF.createBasicBlock("cmpxchg.continue", CGF.CurFn);
405
406  // Update Expected if Expected isn't equal to Old, otherwise branch to the
407  // exit point.
408  CGF.Builder.CreateCondBr(Cmp, ContinueBB, StoreExpectedBB);
409
410  CGF.Builder.SetInsertPoint(StoreExpectedBB);
411  // Update the memory at Expected with Old's value.
412  CGF.Builder.CreateStore(Old, Val1);
413  // Finally, branch to the exit point.
414  CGF.Builder.CreateBr(ContinueBB);
415
416  CGF.Builder.SetInsertPoint(ContinueBB);
417  // Update the memory at Dest with Cmp's value.
418  CGF.EmitStoreOfScalar(Cmp, CGF.MakeAddrLValue(Dest, E->getType()));
419}
420
421/// Given an ordering required on success, emit all possible cmpxchg
422/// instructions to cope with the provided (but possibly only dynamically known)
423/// FailureOrder.
424static void emitAtomicCmpXchgFailureSet(CodeGenFunction &CGF, AtomicExpr *E,
425                                        bool IsWeak, Address Dest, Address Ptr,
426                                        Address Val1, Address Val2,
427                                        llvm::Value *FailureOrderVal,
428                                        uint64_t Size,
429                                        llvm::AtomicOrdering SuccessOrder,
430                                        llvm::SyncScope::ID Scope) {
431  llvm::AtomicOrdering FailureOrder;
432  if (llvm::ConstantInt *FO = dyn_cast<llvm::ConstantInt>(FailureOrderVal)) {
433    auto FOS = FO->getSExtValue();
434    if (!llvm::isValidAtomicOrderingCABI(FOS))
435      FailureOrder = llvm::AtomicOrdering::Monotonic;
436    else
437      switch ((llvm::AtomicOrderingCABI)FOS) {
438      case llvm::AtomicOrderingCABI::relaxed:
439      // 31.7.2.18: "The failure argument shall not be memory_order_release
440      // nor memory_order_acq_rel". Fallback to monotonic.
441      case llvm::AtomicOrderingCABI::release:
442      case llvm::AtomicOrderingCABI::acq_rel:
443        FailureOrder = llvm::AtomicOrdering::Monotonic;
444        break;
445      case llvm::AtomicOrderingCABI::consume:
446      case llvm::AtomicOrderingCABI::acquire:
447        FailureOrder = llvm::AtomicOrdering::Acquire;
448        break;
449      case llvm::AtomicOrderingCABI::seq_cst:
450        FailureOrder = llvm::AtomicOrdering::SequentiallyConsistent;
451        break;
452      }
453    // Prior to c++17, "the failure argument shall be no stronger than the
454    // success argument". This condition has been lifted and the only
455    // precondition is 31.7.2.18. Effectively treat this as a DR and skip
456    // language version checks.
457    emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2, Size, SuccessOrder,
458                      FailureOrder, Scope);
459    return;
460  }
461
462  // Create all the relevant BB's
463  auto *MonotonicBB = CGF.createBasicBlock("monotonic_fail", CGF.CurFn);
464  auto *AcquireBB = CGF.createBasicBlock("acquire_fail", CGF.CurFn);
465  auto *SeqCstBB = CGF.createBasicBlock("seqcst_fail", CGF.CurFn);
466  auto *ContBB = CGF.createBasicBlock("atomic.continue", CGF.CurFn);
467
468  // MonotonicBB is arbitrarily chosen as the default case; in practice, this
469  // doesn't matter unless someone is crazy enough to use something that
470  // doesn't fold to a constant for the ordering.
471  llvm::SwitchInst *SI = CGF.Builder.CreateSwitch(FailureOrderVal, MonotonicBB);
472  // Implemented as acquire, since it's the closest in LLVM.
473  SI->addCase(CGF.Builder.getInt32((int)llvm::AtomicOrderingCABI::consume),
474              AcquireBB);
475  SI->addCase(CGF.Builder.getInt32((int)llvm::AtomicOrderingCABI::acquire),
476              AcquireBB);
477  SI->addCase(CGF.Builder.getInt32((int)llvm::AtomicOrderingCABI::seq_cst),
478              SeqCstBB);
479
480  // Emit all the different atomics
481  CGF.Builder.SetInsertPoint(MonotonicBB);
482  emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2,
483                    Size, SuccessOrder, llvm::AtomicOrdering::Monotonic, Scope);
484  CGF.Builder.CreateBr(ContBB);
485
486  CGF.Builder.SetInsertPoint(AcquireBB);
487  emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2, Size, SuccessOrder,
488                    llvm::AtomicOrdering::Acquire, Scope);
489  CGF.Builder.CreateBr(ContBB);
490
491  CGF.Builder.SetInsertPoint(SeqCstBB);
492  emitAtomicCmpXchg(CGF, E, IsWeak, Dest, Ptr, Val1, Val2, Size, SuccessOrder,
493                    llvm::AtomicOrdering::SequentiallyConsistent, Scope);
494  CGF.Builder.CreateBr(ContBB);
495
496  CGF.Builder.SetInsertPoint(ContBB);
497}
498
499/// Duplicate the atomic min/max operation in conventional IR for the builtin
500/// variants that return the new rather than the original value.
501static llvm::Value *EmitPostAtomicMinMax(CGBuilderTy &Builder,
502                                         AtomicExpr::AtomicOp Op,
503                                         bool IsSigned,
504                                         llvm::Value *OldVal,
505                                         llvm::Value *RHS) {
506  llvm::CmpInst::Predicate Pred;
507  switch (Op) {
508  default:
509    llvm_unreachable("Unexpected min/max operation");
510  case AtomicExpr::AO__atomic_max_fetch:
511    Pred = IsSigned ? llvm::CmpInst::ICMP_SGT : llvm::CmpInst::ICMP_UGT;
512    break;
513  case AtomicExpr::AO__atomic_min_fetch:
514    Pred = IsSigned ? llvm::CmpInst::ICMP_SLT : llvm::CmpInst::ICMP_ULT;
515    break;
516  }
517  llvm::Value *Cmp = Builder.CreateICmp(Pred, OldVal, RHS, "tst");
518  return Builder.CreateSelect(Cmp, OldVal, RHS, "newval");
519}
520
521static void EmitAtomicOp(CodeGenFunction &CGF, AtomicExpr *E, Address Dest,
522                         Address Ptr, Address Val1, Address Val2,
523                         llvm::Value *IsWeak, llvm::Value *FailureOrder,
524                         uint64_t Size, llvm::AtomicOrdering Order,
525                         llvm::SyncScope::ID Scope) {
526  llvm::AtomicRMWInst::BinOp Op = llvm::AtomicRMWInst::Add;
527  bool PostOpMinMax = false;
528  unsigned PostOp = 0;
529
530  switch (E->getOp()) {
531  case AtomicExpr::AO__c11_atomic_init:
532  case AtomicExpr::AO__opencl_atomic_init:
533    llvm_unreachable("Already handled!");
534
535  case AtomicExpr::AO__c11_atomic_compare_exchange_strong:
536  case AtomicExpr::AO__hip_atomic_compare_exchange_strong:
537  case AtomicExpr::AO__opencl_atomic_compare_exchange_strong:
538    emitAtomicCmpXchgFailureSet(CGF, E, false, Dest, Ptr, Val1, Val2,
539                                FailureOrder, Size, Order, Scope);
540    return;
541  case AtomicExpr::AO__c11_atomic_compare_exchange_weak:
542  case AtomicExpr::AO__opencl_atomic_compare_exchange_weak:
543  case AtomicExpr::AO__hip_atomic_compare_exchange_weak:
544    emitAtomicCmpXchgFailureSet(CGF, E, true, Dest, Ptr, Val1, Val2,
545                                FailureOrder, Size, Order, Scope);
546    return;
547  case AtomicExpr::AO__atomic_compare_exchange:
548  case AtomicExpr::AO__atomic_compare_exchange_n: {
549    if (llvm::ConstantInt *IsWeakC = dyn_cast<llvm::ConstantInt>(IsWeak)) {
550      emitAtomicCmpXchgFailureSet(CGF, E, IsWeakC->getZExtValue(), Dest, Ptr,
551                                  Val1, Val2, FailureOrder, Size, Order, Scope);
552    } else {
553      // Create all the relevant BB's
554      llvm::BasicBlock *StrongBB =
555          CGF.createBasicBlock("cmpxchg.strong", CGF.CurFn);
556      llvm::BasicBlock *WeakBB = CGF.createBasicBlock("cmxchg.weak", CGF.CurFn);
557      llvm::BasicBlock *ContBB =
558          CGF.createBasicBlock("cmpxchg.continue", CGF.CurFn);
559
560      llvm::SwitchInst *SI = CGF.Builder.CreateSwitch(IsWeak, WeakBB);
561      SI->addCase(CGF.Builder.getInt1(false), StrongBB);
562
563      CGF.Builder.SetInsertPoint(StrongBB);
564      emitAtomicCmpXchgFailureSet(CGF, E, false, Dest, Ptr, Val1, Val2,
565                                  FailureOrder, Size, Order, Scope);
566      CGF.Builder.CreateBr(ContBB);
567
568      CGF.Builder.SetInsertPoint(WeakBB);
569      emitAtomicCmpXchgFailureSet(CGF, E, true, Dest, Ptr, Val1, Val2,
570                                  FailureOrder, Size, Order, Scope);
571      CGF.Builder.CreateBr(ContBB);
572
573      CGF.Builder.SetInsertPoint(ContBB);
574    }
575    return;
576  }
577  case AtomicExpr::AO__c11_atomic_load:
578  case AtomicExpr::AO__opencl_atomic_load:
579  case AtomicExpr::AO__hip_atomic_load:
580  case AtomicExpr::AO__atomic_load_n:
581  case AtomicExpr::AO__atomic_load: {
582    llvm::LoadInst *Load = CGF.Builder.CreateLoad(Ptr);
583    Load->setAtomic(Order, Scope);
584    Load->setVolatile(E->isVolatile());
585    CGF.Builder.CreateStore(Load, Dest);
586    return;
587  }
588
589  case AtomicExpr::AO__c11_atomic_store:
590  case AtomicExpr::AO__opencl_atomic_store:
591  case AtomicExpr::AO__hip_atomic_store:
592  case AtomicExpr::AO__atomic_store:
593  case AtomicExpr::AO__atomic_store_n: {
594    llvm::Value *LoadVal1 = CGF.Builder.CreateLoad(Val1);
595    llvm::StoreInst *Store = CGF.Builder.CreateStore(LoadVal1, Ptr);
596    Store->setAtomic(Order, Scope);
597    Store->setVolatile(E->isVolatile());
598    return;
599  }
600
601  case AtomicExpr::AO__c11_atomic_exchange:
602  case AtomicExpr::AO__hip_atomic_exchange:
603  case AtomicExpr::AO__opencl_atomic_exchange:
604  case AtomicExpr::AO__atomic_exchange_n:
605  case AtomicExpr::AO__atomic_exchange:
606    Op = llvm::AtomicRMWInst::Xchg;
607    break;
608
609  case AtomicExpr::AO__atomic_add_fetch:
610    PostOp = E->getValueType()->isFloatingType() ? llvm::Instruction::FAdd
611                                                 : llvm::Instruction::Add;
612    [[fallthrough]];
613  case AtomicExpr::AO__c11_atomic_fetch_add:
614  case AtomicExpr::AO__hip_atomic_fetch_add:
615  case AtomicExpr::AO__opencl_atomic_fetch_add:
616  case AtomicExpr::AO__atomic_fetch_add:
617    Op = E->getValueType()->isFloatingType() ? llvm::AtomicRMWInst::FAdd
618                                             : llvm::AtomicRMWInst::Add;
619    break;
620
621  case AtomicExpr::AO__atomic_sub_fetch:
622    PostOp = E->getValueType()->isFloatingType() ? llvm::Instruction::FSub
623                                                 : llvm::Instruction::Sub;
624    [[fallthrough]];
625  case AtomicExpr::AO__c11_atomic_fetch_sub:
626  case AtomicExpr::AO__opencl_atomic_fetch_sub:
627  case AtomicExpr::AO__atomic_fetch_sub:
628    Op = E->getValueType()->isFloatingType() ? llvm::AtomicRMWInst::FSub
629                                             : llvm::AtomicRMWInst::Sub;
630    break;
631
632  case AtomicExpr::AO__atomic_min_fetch:
633    PostOpMinMax = true;
634    [[fallthrough]];
635  case AtomicExpr::AO__c11_atomic_fetch_min:
636  case AtomicExpr::AO__hip_atomic_fetch_min:
637  case AtomicExpr::AO__opencl_atomic_fetch_min:
638  case AtomicExpr::AO__atomic_fetch_min:
639    Op = E->getValueType()->isSignedIntegerType() ? llvm::AtomicRMWInst::Min
640                                                  : llvm::AtomicRMWInst::UMin;
641    break;
642
643  case AtomicExpr::AO__atomic_max_fetch:
644    PostOpMinMax = true;
645    [[fallthrough]];
646  case AtomicExpr::AO__c11_atomic_fetch_max:
647  case AtomicExpr::AO__hip_atomic_fetch_max:
648  case AtomicExpr::AO__opencl_atomic_fetch_max:
649  case AtomicExpr::AO__atomic_fetch_max:
650    Op = E->getValueType()->isSignedIntegerType() ? llvm::AtomicRMWInst::Max
651                                                  : llvm::AtomicRMWInst::UMax;
652    break;
653
654  case AtomicExpr::AO__atomic_and_fetch:
655    PostOp = llvm::Instruction::And;
656    [[fallthrough]];
657  case AtomicExpr::AO__c11_atomic_fetch_and:
658  case AtomicExpr::AO__hip_atomic_fetch_and:
659  case AtomicExpr::AO__opencl_atomic_fetch_and:
660  case AtomicExpr::AO__atomic_fetch_and:
661    Op = llvm::AtomicRMWInst::And;
662    break;
663
664  case AtomicExpr::AO__atomic_or_fetch:
665    PostOp = llvm::Instruction::Or;
666    [[fallthrough]];
667  case AtomicExpr::AO__c11_atomic_fetch_or:
668  case AtomicExpr::AO__hip_atomic_fetch_or:
669  case AtomicExpr::AO__opencl_atomic_fetch_or:
670  case AtomicExpr::AO__atomic_fetch_or:
671    Op = llvm::AtomicRMWInst::Or;
672    break;
673
674  case AtomicExpr::AO__atomic_xor_fetch:
675    PostOp = llvm::Instruction::Xor;
676    [[fallthrough]];
677  case AtomicExpr::AO__c11_atomic_fetch_xor:
678  case AtomicExpr::AO__hip_atomic_fetch_xor:
679  case AtomicExpr::AO__opencl_atomic_fetch_xor:
680  case AtomicExpr::AO__atomic_fetch_xor:
681    Op = llvm::AtomicRMWInst::Xor;
682    break;
683
684  case AtomicExpr::AO__atomic_nand_fetch:
685    PostOp = llvm::Instruction::And; // the NOT is special cased below
686    [[fallthrough]];
687  case AtomicExpr::AO__c11_atomic_fetch_nand:
688  case AtomicExpr::AO__atomic_fetch_nand:
689    Op = llvm::AtomicRMWInst::Nand;
690    break;
691  }
692
693  llvm::Value *LoadVal1 = CGF.Builder.CreateLoad(Val1);
694  llvm::AtomicRMWInst *RMWI =
695      CGF.Builder.CreateAtomicRMW(Op, Ptr.getPointer(), LoadVal1, Order, Scope);
696  RMWI->setVolatile(E->isVolatile());
697
698  // For __atomic_*_fetch operations, perform the operation again to
699  // determine the value which was written.
700  llvm::Value *Result = RMWI;
701  if (PostOpMinMax)
702    Result = EmitPostAtomicMinMax(CGF.Builder, E->getOp(),
703                                  E->getValueType()->isSignedIntegerType(),
704                                  RMWI, LoadVal1);
705  else if (PostOp)
706    Result = CGF.Builder.CreateBinOp((llvm::Instruction::BinaryOps)PostOp, RMWI,
707                                     LoadVal1);
708  if (E->getOp() == AtomicExpr::AO__atomic_nand_fetch)
709    Result = CGF.Builder.CreateNot(Result);
710  CGF.Builder.CreateStore(Result, Dest);
711}
712
713// This function emits any expression (scalar, complex, or aggregate)
714// into a temporary alloca.
715static Address
716EmitValToTemp(CodeGenFunction &CGF, Expr *E) {
717  Address DeclPtr = CGF.CreateMemTemp(E->getType(), ".atomictmp");
718  CGF.EmitAnyExprToMem(E, DeclPtr, E->getType().getQualifiers(),
719                       /*Init*/ true);
720  return DeclPtr;
721}
722
723static void EmitAtomicOp(CodeGenFunction &CGF, AtomicExpr *Expr, Address Dest,
724                         Address Ptr, Address Val1, Address Val2,
725                         llvm::Value *IsWeak, llvm::Value *FailureOrder,
726                         uint64_t Size, llvm::AtomicOrdering Order,
727                         llvm::Value *Scope) {
728  auto ScopeModel = Expr->getScopeModel();
729
730  // LLVM atomic instructions always have synch scope. If clang atomic
731  // expression has no scope operand, use default LLVM synch scope.
732  if (!ScopeModel) {
733    EmitAtomicOp(CGF, Expr, Dest, Ptr, Val1, Val2, IsWeak, FailureOrder, Size,
734                 Order, CGF.CGM.getLLVMContext().getOrInsertSyncScopeID(""));
735    return;
736  }
737
738  // Handle constant scope.
739  if (auto SC = dyn_cast<llvm::ConstantInt>(Scope)) {
740    auto SCID = CGF.getTargetHooks().getLLVMSyncScopeID(
741        CGF.CGM.getLangOpts(), ScopeModel->map(SC->getZExtValue()),
742        Order, CGF.CGM.getLLVMContext());
743    EmitAtomicOp(CGF, Expr, Dest, Ptr, Val1, Val2, IsWeak, FailureOrder, Size,
744                 Order, SCID);
745    return;
746  }
747
748  // Handle non-constant scope.
749  auto &Builder = CGF.Builder;
750  auto Scopes = ScopeModel->getRuntimeValues();
751  llvm::DenseMap<unsigned, llvm::BasicBlock *> BB;
752  for (auto S : Scopes)
753    BB[S] = CGF.createBasicBlock(getAsString(ScopeModel->map(S)), CGF.CurFn);
754
755  llvm::BasicBlock *ContBB =
756      CGF.createBasicBlock("atomic.scope.continue", CGF.CurFn);
757
758  auto *SC = Builder.CreateIntCast(Scope, Builder.getInt32Ty(), false);
759  // If unsupported synch scope is encountered at run time, assume a fallback
760  // synch scope value.
761  auto FallBack = ScopeModel->getFallBackValue();
762  llvm::SwitchInst *SI = Builder.CreateSwitch(SC, BB[FallBack]);
763  for (auto S : Scopes) {
764    auto *B = BB[S];
765    if (S != FallBack)
766      SI->addCase(Builder.getInt32(S), B);
767
768    Builder.SetInsertPoint(B);
769    EmitAtomicOp(CGF, Expr, Dest, Ptr, Val1, Val2, IsWeak, FailureOrder, Size,
770                 Order,
771                 CGF.getTargetHooks().getLLVMSyncScopeID(CGF.CGM.getLangOpts(),
772                                                         ScopeModel->map(S),
773                                                         Order,
774                                                         CGF.getLLVMContext()));
775    Builder.CreateBr(ContBB);
776  }
777
778  Builder.SetInsertPoint(ContBB);
779}
780
781static void
782AddDirectArgument(CodeGenFunction &CGF, CallArgList &Args,
783                  bool UseOptimizedLibcall, llvm::Value *Val, QualType ValTy,
784                  SourceLocation Loc, CharUnits SizeInChars) {
785  if (UseOptimizedLibcall) {
786    // Load value and pass it to the function directly.
787    CharUnits Align = CGF.getContext().getTypeAlignInChars(ValTy);
788    int64_t SizeInBits = CGF.getContext().toBits(SizeInChars);
789    ValTy =
790        CGF.getContext().getIntTypeForBitwidth(SizeInBits, /*Signed=*/false);
791    llvm::Type *ITy = llvm::IntegerType::get(CGF.getLLVMContext(), SizeInBits);
792    Address Ptr = Address(CGF.Builder.CreateBitCast(Val, ITy->getPointerTo()),
793                          ITy, Align);
794    Val = CGF.EmitLoadOfScalar(Ptr, false,
795                               CGF.getContext().getPointerType(ValTy),
796                               Loc);
797    // Coerce the value into an appropriately sized integer type.
798    Args.add(RValue::get(Val), ValTy);
799  } else {
800    // Non-optimized functions always take a reference.
801    Args.add(RValue::get(CGF.EmitCastToVoidPtr(Val)),
802                         CGF.getContext().VoidPtrTy);
803  }
804}
805
806RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E) {
807  QualType AtomicTy = E->getPtr()->getType()->getPointeeType();
808  QualType MemTy = AtomicTy;
809  if (const AtomicType *AT = AtomicTy->getAs<AtomicType>())
810    MemTy = AT->getValueType();
811  llvm::Value *IsWeak = nullptr, *OrderFail = nullptr;
812
813  Address Val1 = Address::invalid();
814  Address Val2 = Address::invalid();
815  Address Dest = Address::invalid();
816  Address Ptr = EmitPointerWithAlignment(E->getPtr());
817
818  if (E->getOp() == AtomicExpr::AO__c11_atomic_init ||
819      E->getOp() == AtomicExpr::AO__opencl_atomic_init) {
820    LValue lvalue = MakeAddrLValue(Ptr, AtomicTy);
821    EmitAtomicInit(E->getVal1(), lvalue);
822    return RValue::get(nullptr);
823  }
824
825  auto TInfo = getContext().getTypeInfoInChars(AtomicTy);
826  uint64_t Size = TInfo.Width.getQuantity();
827  unsigned MaxInlineWidthInBits = getTarget().getMaxAtomicInlineWidth();
828
829  bool Oversized = getContext().toBits(TInfo.Width) > MaxInlineWidthInBits;
830  bool Misaligned = (Ptr.getAlignment() % TInfo.Width) != 0;
831  bool UseLibcall = Misaligned | Oversized;
832  bool ShouldCastToIntPtrTy = true;
833
834  CharUnits MaxInlineWidth =
835      getContext().toCharUnitsFromBits(MaxInlineWidthInBits);
836
837  DiagnosticsEngine &Diags = CGM.getDiags();
838
839  if (Misaligned) {
840    Diags.Report(E->getBeginLoc(), diag::warn_atomic_op_misaligned)
841        << (int)TInfo.Width.getQuantity()
842        << (int)Ptr.getAlignment().getQuantity();
843  }
844
845  if (Oversized) {
846    Diags.Report(E->getBeginLoc(), diag::warn_atomic_op_oversized)
847        << (int)TInfo.Width.getQuantity() << (int)MaxInlineWidth.getQuantity();
848  }
849
850  llvm::Value *Order = EmitScalarExpr(E->getOrder());
851  llvm::Value *Scope =
852      E->getScopeModel() ? EmitScalarExpr(E->getScope()) : nullptr;
853
854  switch (E->getOp()) {
855  case AtomicExpr::AO__c11_atomic_init:
856  case AtomicExpr::AO__opencl_atomic_init:
857    llvm_unreachable("Already handled above with EmitAtomicInit!");
858
859  case AtomicExpr::AO__c11_atomic_load:
860  case AtomicExpr::AO__opencl_atomic_load:
861  case AtomicExpr::AO__hip_atomic_load:
862  case AtomicExpr::AO__atomic_load_n:
863    break;
864
865  case AtomicExpr::AO__atomic_load:
866    Dest = EmitPointerWithAlignment(E->getVal1());
867    break;
868
869  case AtomicExpr::AO__atomic_store:
870    Val1 = EmitPointerWithAlignment(E->getVal1());
871    break;
872
873  case AtomicExpr::AO__atomic_exchange:
874    Val1 = EmitPointerWithAlignment(E->getVal1());
875    Dest = EmitPointerWithAlignment(E->getVal2());
876    break;
877
878  case AtomicExpr::AO__c11_atomic_compare_exchange_strong:
879  case AtomicExpr::AO__c11_atomic_compare_exchange_weak:
880  case AtomicExpr::AO__opencl_atomic_compare_exchange_strong:
881  case AtomicExpr::AO__hip_atomic_compare_exchange_strong:
882  case AtomicExpr::AO__opencl_atomic_compare_exchange_weak:
883  case AtomicExpr::AO__hip_atomic_compare_exchange_weak:
884  case AtomicExpr::AO__atomic_compare_exchange_n:
885  case AtomicExpr::AO__atomic_compare_exchange:
886    Val1 = EmitPointerWithAlignment(E->getVal1());
887    if (E->getOp() == AtomicExpr::AO__atomic_compare_exchange)
888      Val2 = EmitPointerWithAlignment(E->getVal2());
889    else
890      Val2 = EmitValToTemp(*this, E->getVal2());
891    OrderFail = EmitScalarExpr(E->getOrderFail());
892    if (E->getOp() == AtomicExpr::AO__atomic_compare_exchange_n ||
893        E->getOp() == AtomicExpr::AO__atomic_compare_exchange)
894      IsWeak = EmitScalarExpr(E->getWeak());
895    break;
896
897  case AtomicExpr::AO__c11_atomic_fetch_add:
898  case AtomicExpr::AO__c11_atomic_fetch_sub:
899  case AtomicExpr::AO__hip_atomic_fetch_add:
900  case AtomicExpr::AO__opencl_atomic_fetch_add:
901  case AtomicExpr::AO__opencl_atomic_fetch_sub:
902    if (MemTy->isPointerType()) {
903      // For pointer arithmetic, we're required to do a bit of math:
904      // adding 1 to an int* is not the same as adding 1 to a uintptr_t.
905      // ... but only for the C11 builtins. The GNU builtins expect the
906      // user to multiply by sizeof(T).
907      QualType Val1Ty = E->getVal1()->getType();
908      llvm::Value *Val1Scalar = EmitScalarExpr(E->getVal1());
909      CharUnits PointeeIncAmt =
910          getContext().getTypeSizeInChars(MemTy->getPointeeType());
911      Val1Scalar = Builder.CreateMul(Val1Scalar, CGM.getSize(PointeeIncAmt));
912      auto Temp = CreateMemTemp(Val1Ty, ".atomictmp");
913      Val1 = Temp;
914      EmitStoreOfScalar(Val1Scalar, MakeAddrLValue(Temp, Val1Ty));
915      break;
916    }
917    [[fallthrough]];
918  case AtomicExpr::AO__atomic_fetch_add:
919  case AtomicExpr::AO__atomic_fetch_sub:
920  case AtomicExpr::AO__atomic_add_fetch:
921  case AtomicExpr::AO__atomic_sub_fetch:
922    ShouldCastToIntPtrTy = !MemTy->isFloatingType();
923    [[fallthrough]];
924
925  case AtomicExpr::AO__c11_atomic_store:
926  case AtomicExpr::AO__c11_atomic_exchange:
927  case AtomicExpr::AO__opencl_atomic_store:
928  case AtomicExpr::AO__hip_atomic_store:
929  case AtomicExpr::AO__opencl_atomic_exchange:
930  case AtomicExpr::AO__hip_atomic_exchange:
931  case AtomicExpr::AO__atomic_store_n:
932  case AtomicExpr::AO__atomic_exchange_n:
933  case AtomicExpr::AO__c11_atomic_fetch_and:
934  case AtomicExpr::AO__c11_atomic_fetch_or:
935  case AtomicExpr::AO__c11_atomic_fetch_xor:
936  case AtomicExpr::AO__c11_atomic_fetch_nand:
937  case AtomicExpr::AO__c11_atomic_fetch_max:
938  case AtomicExpr::AO__c11_atomic_fetch_min:
939  case AtomicExpr::AO__opencl_atomic_fetch_and:
940  case AtomicExpr::AO__opencl_atomic_fetch_or:
941  case AtomicExpr::AO__opencl_atomic_fetch_xor:
942  case AtomicExpr::AO__opencl_atomic_fetch_min:
943  case AtomicExpr::AO__opencl_atomic_fetch_max:
944  case AtomicExpr::AO__atomic_fetch_and:
945  case AtomicExpr::AO__hip_atomic_fetch_and:
946  case AtomicExpr::AO__atomic_fetch_or:
947  case AtomicExpr::AO__hip_atomic_fetch_or:
948  case AtomicExpr::AO__atomic_fetch_xor:
949  case AtomicExpr::AO__hip_atomic_fetch_xor:
950  case AtomicExpr::AO__atomic_fetch_nand:
951  case AtomicExpr::AO__atomic_and_fetch:
952  case AtomicExpr::AO__atomic_or_fetch:
953  case AtomicExpr::AO__atomic_xor_fetch:
954  case AtomicExpr::AO__atomic_nand_fetch:
955  case AtomicExpr::AO__atomic_max_fetch:
956  case AtomicExpr::AO__atomic_min_fetch:
957  case AtomicExpr::AO__atomic_fetch_max:
958  case AtomicExpr::AO__hip_atomic_fetch_max:
959  case AtomicExpr::AO__atomic_fetch_min:
960  case AtomicExpr::AO__hip_atomic_fetch_min:
961    Val1 = EmitValToTemp(*this, E->getVal1());
962    break;
963  }
964
965  QualType RValTy = E->getType().getUnqualifiedType();
966
967  // The inlined atomics only function on iN types, where N is a power of 2. We
968  // need to make sure (via temporaries if necessary) that all incoming values
969  // are compatible.
970  LValue AtomicVal = MakeAddrLValue(Ptr, AtomicTy);
971  AtomicInfo Atomics(*this, AtomicVal);
972
973  if (ShouldCastToIntPtrTy) {
974    Ptr = Atomics.emitCastToAtomicIntPointer(Ptr);
975    if (Val1.isValid())
976      Val1 = Atomics.convertToAtomicIntPointer(Val1);
977    if (Val2.isValid())
978      Val2 = Atomics.convertToAtomicIntPointer(Val2);
979  }
980  if (Dest.isValid()) {
981    if (ShouldCastToIntPtrTy)
982      Dest = Atomics.emitCastToAtomicIntPointer(Dest);
983  } else if (E->isCmpXChg())
984    Dest = CreateMemTemp(RValTy, "cmpxchg.bool");
985  else if (!RValTy->isVoidType()) {
986    Dest = Atomics.CreateTempAlloca();
987    if (ShouldCastToIntPtrTy)
988      Dest = Atomics.emitCastToAtomicIntPointer(Dest);
989  }
990
991  // Use a library call.  See: http://gcc.gnu.org/wiki/Atomic/GCCMM/LIbrary .
992  if (UseLibcall) {
993    bool UseOptimizedLibcall = false;
994    switch (E->getOp()) {
995    case AtomicExpr::AO__c11_atomic_init:
996    case AtomicExpr::AO__opencl_atomic_init:
997      llvm_unreachable("Already handled above with EmitAtomicInit!");
998
999    case AtomicExpr::AO__c11_atomic_fetch_add:
1000    case AtomicExpr::AO__opencl_atomic_fetch_add:
1001    case AtomicExpr::AO__atomic_fetch_add:
1002    case AtomicExpr::AO__hip_atomic_fetch_add:
1003    case AtomicExpr::AO__c11_atomic_fetch_and:
1004    case AtomicExpr::AO__opencl_atomic_fetch_and:
1005    case AtomicExpr::AO__hip_atomic_fetch_and:
1006    case AtomicExpr::AO__atomic_fetch_and:
1007    case AtomicExpr::AO__c11_atomic_fetch_or:
1008    case AtomicExpr::AO__opencl_atomic_fetch_or:
1009    case AtomicExpr::AO__hip_atomic_fetch_or:
1010    case AtomicExpr::AO__atomic_fetch_or:
1011    case AtomicExpr::AO__c11_atomic_fetch_nand:
1012    case AtomicExpr::AO__atomic_fetch_nand:
1013    case AtomicExpr::AO__c11_atomic_fetch_sub:
1014    case AtomicExpr::AO__opencl_atomic_fetch_sub:
1015    case AtomicExpr::AO__atomic_fetch_sub:
1016    case AtomicExpr::AO__c11_atomic_fetch_xor:
1017    case AtomicExpr::AO__opencl_atomic_fetch_xor:
1018    case AtomicExpr::AO__opencl_atomic_fetch_min:
1019    case AtomicExpr::AO__opencl_atomic_fetch_max:
1020    case AtomicExpr::AO__atomic_fetch_xor:
1021    case AtomicExpr::AO__hip_atomic_fetch_xor:
1022    case AtomicExpr::AO__c11_atomic_fetch_max:
1023    case AtomicExpr::AO__c11_atomic_fetch_min:
1024    case AtomicExpr::AO__atomic_add_fetch:
1025    case AtomicExpr::AO__atomic_and_fetch:
1026    case AtomicExpr::AO__atomic_nand_fetch:
1027    case AtomicExpr::AO__atomic_or_fetch:
1028    case AtomicExpr::AO__atomic_sub_fetch:
1029    case AtomicExpr::AO__atomic_xor_fetch:
1030    case AtomicExpr::AO__atomic_fetch_max:
1031    case AtomicExpr::AO__hip_atomic_fetch_max:
1032    case AtomicExpr::AO__atomic_fetch_min:
1033    case AtomicExpr::AO__hip_atomic_fetch_min:
1034    case AtomicExpr::AO__atomic_max_fetch:
1035    case AtomicExpr::AO__atomic_min_fetch:
1036      // For these, only library calls for certain sizes exist.
1037      UseOptimizedLibcall = true;
1038      break;
1039
1040    case AtomicExpr::AO__atomic_load:
1041    case AtomicExpr::AO__atomic_store:
1042    case AtomicExpr::AO__atomic_exchange:
1043    case AtomicExpr::AO__atomic_compare_exchange:
1044      // Use the generic version if we don't know that the operand will be
1045      // suitably aligned for the optimized version.
1046      if (Misaligned)
1047        break;
1048      [[fallthrough]];
1049    case AtomicExpr::AO__c11_atomic_load:
1050    case AtomicExpr::AO__c11_atomic_store:
1051    case AtomicExpr::AO__c11_atomic_exchange:
1052    case AtomicExpr::AO__c11_atomic_compare_exchange_weak:
1053    case AtomicExpr::AO__c11_atomic_compare_exchange_strong:
1054    case AtomicExpr::AO__hip_atomic_compare_exchange_strong:
1055    case AtomicExpr::AO__opencl_atomic_load:
1056    case AtomicExpr::AO__hip_atomic_load:
1057    case AtomicExpr::AO__opencl_atomic_store:
1058    case AtomicExpr::AO__hip_atomic_store:
1059    case AtomicExpr::AO__opencl_atomic_exchange:
1060    case AtomicExpr::AO__hip_atomic_exchange:
1061    case AtomicExpr::AO__opencl_atomic_compare_exchange_weak:
1062    case AtomicExpr::AO__hip_atomic_compare_exchange_weak:
1063    case AtomicExpr::AO__opencl_atomic_compare_exchange_strong:
1064    case AtomicExpr::AO__atomic_load_n:
1065    case AtomicExpr::AO__atomic_store_n:
1066    case AtomicExpr::AO__atomic_exchange_n:
1067    case AtomicExpr::AO__atomic_compare_exchange_n:
1068      // Only use optimized library calls for sizes for which they exist.
1069      // FIXME: Size == 16 optimized library functions exist too.
1070      if (Size == 1 || Size == 2 || Size == 4 || Size == 8)
1071        UseOptimizedLibcall = true;
1072      break;
1073    }
1074
1075    CallArgList Args;
1076    if (!UseOptimizedLibcall) {
1077      // For non-optimized library calls, the size is the first parameter
1078      Args.add(RValue::get(llvm::ConstantInt::get(SizeTy, Size)),
1079               getContext().getSizeType());
1080    }
1081    // Atomic address is the first or second parameter
1082    // The OpenCL atomic library functions only accept pointer arguments to
1083    // generic address space.
1084    auto CastToGenericAddrSpace = [&](llvm::Value *V, QualType PT) {
1085      if (!E->isOpenCL())
1086        return V;
1087      auto AS = PT->castAs<PointerType>()->getPointeeType().getAddressSpace();
1088      if (AS == LangAS::opencl_generic)
1089        return V;
1090      auto DestAS = getContext().getTargetAddressSpace(LangAS::opencl_generic);
1091      auto T = llvm::cast<llvm::PointerType>(V->getType());
1092      auto *DestType = llvm::PointerType::getWithSamePointeeType(T, DestAS);
1093
1094      return getTargetHooks().performAddrSpaceCast(
1095          *this, V, AS, LangAS::opencl_generic, DestType, false);
1096    };
1097
1098    Args.add(RValue::get(CastToGenericAddrSpace(
1099                 EmitCastToVoidPtr(Ptr.getPointer()), E->getPtr()->getType())),
1100             getContext().VoidPtrTy);
1101
1102    std::string LibCallName;
1103    QualType LoweredMemTy =
1104      MemTy->isPointerType() ? getContext().getIntPtrType() : MemTy;
1105    QualType RetTy;
1106    bool HaveRetTy = false;
1107    llvm::Instruction::BinaryOps PostOp = (llvm::Instruction::BinaryOps)0;
1108    bool PostOpMinMax = false;
1109    switch (E->getOp()) {
1110    case AtomicExpr::AO__c11_atomic_init:
1111    case AtomicExpr::AO__opencl_atomic_init:
1112      llvm_unreachable("Already handled!");
1113
1114    // There is only one libcall for compare an exchange, because there is no
1115    // optimisation benefit possible from a libcall version of a weak compare
1116    // and exchange.
1117    // bool __atomic_compare_exchange(size_t size, void *mem, void *expected,
1118    //                                void *desired, int success, int failure)
1119    // bool __atomic_compare_exchange_N(T *mem, T *expected, T desired,
1120    //                                  int success, int failure)
1121    case AtomicExpr::AO__c11_atomic_compare_exchange_weak:
1122    case AtomicExpr::AO__c11_atomic_compare_exchange_strong:
1123    case AtomicExpr::AO__opencl_atomic_compare_exchange_weak:
1124    case AtomicExpr::AO__hip_atomic_compare_exchange_weak:
1125    case AtomicExpr::AO__opencl_atomic_compare_exchange_strong:
1126    case AtomicExpr::AO__hip_atomic_compare_exchange_strong:
1127    case AtomicExpr::AO__atomic_compare_exchange:
1128    case AtomicExpr::AO__atomic_compare_exchange_n:
1129      LibCallName = "__atomic_compare_exchange";
1130      RetTy = getContext().BoolTy;
1131      HaveRetTy = true;
1132      Args.add(
1133          RValue::get(CastToGenericAddrSpace(
1134              EmitCastToVoidPtr(Val1.getPointer()), E->getVal1()->getType())),
1135          getContext().VoidPtrTy);
1136      AddDirectArgument(*this, Args, UseOptimizedLibcall, Val2.getPointer(),
1137                        MemTy, E->getExprLoc(), TInfo.Width);
1138      Args.add(RValue::get(Order), getContext().IntTy);
1139      Order = OrderFail;
1140      break;
1141    // void __atomic_exchange(size_t size, void *mem, void *val, void *return,
1142    //                        int order)
1143    // T __atomic_exchange_N(T *mem, T val, int order)
1144    case AtomicExpr::AO__c11_atomic_exchange:
1145    case AtomicExpr::AO__opencl_atomic_exchange:
1146    case AtomicExpr::AO__atomic_exchange_n:
1147    case AtomicExpr::AO__atomic_exchange:
1148    case AtomicExpr::AO__hip_atomic_exchange:
1149      LibCallName = "__atomic_exchange";
1150      AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
1151                        MemTy, E->getExprLoc(), TInfo.Width);
1152      break;
1153    // void __atomic_store(size_t size, void *mem, void *val, int order)
1154    // void __atomic_store_N(T *mem, T val, int order)
1155    case AtomicExpr::AO__c11_atomic_store:
1156    case AtomicExpr::AO__opencl_atomic_store:
1157    case AtomicExpr::AO__hip_atomic_store:
1158    case AtomicExpr::AO__atomic_store:
1159    case AtomicExpr::AO__atomic_store_n:
1160      LibCallName = "__atomic_store";
1161      RetTy = getContext().VoidTy;
1162      HaveRetTy = true;
1163      AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
1164                        MemTy, E->getExprLoc(), TInfo.Width);
1165      break;
1166    // void __atomic_load(size_t size, void *mem, void *return, int order)
1167    // T __atomic_load_N(T *mem, int order)
1168    case AtomicExpr::AO__c11_atomic_load:
1169    case AtomicExpr::AO__opencl_atomic_load:
1170    case AtomicExpr::AO__hip_atomic_load:
1171    case AtomicExpr::AO__atomic_load:
1172    case AtomicExpr::AO__atomic_load_n:
1173      LibCallName = "__atomic_load";
1174      break;
1175    // T __atomic_add_fetch_N(T *mem, T val, int order)
1176    // T __atomic_fetch_add_N(T *mem, T val, int order)
1177    case AtomicExpr::AO__atomic_add_fetch:
1178      PostOp = llvm::Instruction::Add;
1179      [[fallthrough]];
1180    case AtomicExpr::AO__c11_atomic_fetch_add:
1181    case AtomicExpr::AO__opencl_atomic_fetch_add:
1182    case AtomicExpr::AO__atomic_fetch_add:
1183    case AtomicExpr::AO__hip_atomic_fetch_add:
1184      LibCallName = "__atomic_fetch_add";
1185      AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
1186                        LoweredMemTy, E->getExprLoc(), TInfo.Width);
1187      break;
1188    // T __atomic_and_fetch_N(T *mem, T val, int order)
1189    // T __atomic_fetch_and_N(T *mem, T val, int order)
1190    case AtomicExpr::AO__atomic_and_fetch:
1191      PostOp = llvm::Instruction::And;
1192      [[fallthrough]];
1193    case AtomicExpr::AO__c11_atomic_fetch_and:
1194    case AtomicExpr::AO__opencl_atomic_fetch_and:
1195    case AtomicExpr::AO__hip_atomic_fetch_and:
1196    case AtomicExpr::AO__atomic_fetch_and:
1197      LibCallName = "__atomic_fetch_and";
1198      AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
1199                        MemTy, E->getExprLoc(), TInfo.Width);
1200      break;
1201    // T __atomic_or_fetch_N(T *mem, T val, int order)
1202    // T __atomic_fetch_or_N(T *mem, T val, int order)
1203    case AtomicExpr::AO__atomic_or_fetch:
1204      PostOp = llvm::Instruction::Or;
1205      [[fallthrough]];
1206    case AtomicExpr::AO__c11_atomic_fetch_or:
1207    case AtomicExpr::AO__opencl_atomic_fetch_or:
1208    case AtomicExpr::AO__hip_atomic_fetch_or:
1209    case AtomicExpr::AO__atomic_fetch_or:
1210      LibCallName = "__atomic_fetch_or";
1211      AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
1212                        MemTy, E->getExprLoc(), TInfo.Width);
1213      break;
1214    // T __atomic_sub_fetch_N(T *mem, T val, int order)
1215    // T __atomic_fetch_sub_N(T *mem, T val, int order)
1216    case AtomicExpr::AO__atomic_sub_fetch:
1217      PostOp = llvm::Instruction::Sub;
1218      [[fallthrough]];
1219    case AtomicExpr::AO__c11_atomic_fetch_sub:
1220    case AtomicExpr::AO__opencl_atomic_fetch_sub:
1221    case AtomicExpr::AO__atomic_fetch_sub:
1222      LibCallName = "__atomic_fetch_sub";
1223      AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
1224                        LoweredMemTy, E->getExprLoc(), TInfo.Width);
1225      break;
1226    // T __atomic_xor_fetch_N(T *mem, T val, int order)
1227    // T __atomic_fetch_xor_N(T *mem, T val, int order)
1228    case AtomicExpr::AO__atomic_xor_fetch:
1229      PostOp = llvm::Instruction::Xor;
1230      [[fallthrough]];
1231    case AtomicExpr::AO__c11_atomic_fetch_xor:
1232    case AtomicExpr::AO__opencl_atomic_fetch_xor:
1233    case AtomicExpr::AO__hip_atomic_fetch_xor:
1234    case AtomicExpr::AO__atomic_fetch_xor:
1235      LibCallName = "__atomic_fetch_xor";
1236      AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
1237                        MemTy, E->getExprLoc(), TInfo.Width);
1238      break;
1239    case AtomicExpr::AO__atomic_min_fetch:
1240      PostOpMinMax = true;
1241      [[fallthrough]];
1242    case AtomicExpr::AO__c11_atomic_fetch_min:
1243    case AtomicExpr::AO__atomic_fetch_min:
1244    case AtomicExpr::AO__hip_atomic_fetch_min:
1245    case AtomicExpr::AO__opencl_atomic_fetch_min:
1246      LibCallName = E->getValueType()->isSignedIntegerType()
1247                        ? "__atomic_fetch_min"
1248                        : "__atomic_fetch_umin";
1249      AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
1250                        LoweredMemTy, E->getExprLoc(), TInfo.Width);
1251      break;
1252    case AtomicExpr::AO__atomic_max_fetch:
1253      PostOpMinMax = true;
1254      [[fallthrough]];
1255    case AtomicExpr::AO__c11_atomic_fetch_max:
1256    case AtomicExpr::AO__atomic_fetch_max:
1257    case AtomicExpr::AO__hip_atomic_fetch_max:
1258    case AtomicExpr::AO__opencl_atomic_fetch_max:
1259      LibCallName = E->getValueType()->isSignedIntegerType()
1260                        ? "__atomic_fetch_max"
1261                        : "__atomic_fetch_umax";
1262      AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
1263                        LoweredMemTy, E->getExprLoc(), TInfo.Width);
1264      break;
1265    // T __atomic_nand_fetch_N(T *mem, T val, int order)
1266    // T __atomic_fetch_nand_N(T *mem, T val, int order)
1267    case AtomicExpr::AO__atomic_nand_fetch:
1268      PostOp = llvm::Instruction::And; // the NOT is special cased below
1269      [[fallthrough]];
1270    case AtomicExpr::AO__c11_atomic_fetch_nand:
1271    case AtomicExpr::AO__atomic_fetch_nand:
1272      LibCallName = "__atomic_fetch_nand";
1273      AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(),
1274                        MemTy, E->getExprLoc(), TInfo.Width);
1275      break;
1276    }
1277
1278    if (E->isOpenCL()) {
1279      LibCallName = std::string("__opencl") +
1280          StringRef(LibCallName).drop_front(1).str();
1281
1282    }
1283    // Optimized functions have the size in their name.
1284    if (UseOptimizedLibcall)
1285      LibCallName += "_" + llvm::utostr(Size);
1286    // By default, assume we return a value of the atomic type.
1287    if (!HaveRetTy) {
1288      if (UseOptimizedLibcall) {
1289        // Value is returned directly.
1290        // The function returns an appropriately sized integer type.
1291        RetTy = getContext().getIntTypeForBitwidth(
1292            getContext().toBits(TInfo.Width), /*Signed=*/false);
1293      } else {
1294        // Value is returned through parameter before the order.
1295        RetTy = getContext().VoidTy;
1296        Args.add(RValue::get(EmitCastToVoidPtr(Dest.getPointer())),
1297                 getContext().VoidPtrTy);
1298      }
1299    }
1300    // order is always the last parameter
1301    Args.add(RValue::get(Order),
1302             getContext().IntTy);
1303    if (E->isOpenCL())
1304      Args.add(RValue::get(Scope), getContext().IntTy);
1305
1306    // PostOp is only needed for the atomic_*_fetch operations, and
1307    // thus is only needed for and implemented in the
1308    // UseOptimizedLibcall codepath.
1309    assert(UseOptimizedLibcall || (!PostOp && !PostOpMinMax));
1310
1311    RValue Res = emitAtomicLibcall(*this, LibCallName, RetTy, Args);
1312    // The value is returned directly from the libcall.
1313    if (E->isCmpXChg())
1314      return Res;
1315
1316    // The value is returned directly for optimized libcalls but the expr
1317    // provided an out-param.
1318    if (UseOptimizedLibcall && Res.getScalarVal()) {
1319      llvm::Value *ResVal = Res.getScalarVal();
1320      if (PostOpMinMax) {
1321        llvm::Value *LoadVal1 = Args[1].getRValue(*this).getScalarVal();
1322        ResVal = EmitPostAtomicMinMax(Builder, E->getOp(),
1323                                      E->getValueType()->isSignedIntegerType(),
1324                                      ResVal, LoadVal1);
1325      } else if (PostOp) {
1326        llvm::Value *LoadVal1 = Args[1].getRValue(*this).getScalarVal();
1327        ResVal = Builder.CreateBinOp(PostOp, ResVal, LoadVal1);
1328      }
1329      if (E->getOp() == AtomicExpr::AO__atomic_nand_fetch)
1330        ResVal = Builder.CreateNot(ResVal);
1331
1332      Builder.CreateStore(
1333          ResVal, Builder.CreateElementBitCast(Dest, ResVal->getType()));
1334    }
1335
1336    if (RValTy->isVoidType())
1337      return RValue::get(nullptr);
1338
1339    return convertTempToRValue(
1340        Builder.CreateElementBitCast(Dest, ConvertTypeForMem(RValTy)),
1341        RValTy, E->getExprLoc());
1342  }
1343
1344  bool IsStore = E->getOp() == AtomicExpr::AO__c11_atomic_store ||
1345                 E->getOp() == AtomicExpr::AO__opencl_atomic_store ||
1346                 E->getOp() == AtomicExpr::AO__hip_atomic_store ||
1347                 E->getOp() == AtomicExpr::AO__atomic_store ||
1348                 E->getOp() == AtomicExpr::AO__atomic_store_n;
1349  bool IsLoad = E->getOp() == AtomicExpr::AO__c11_atomic_load ||
1350                E->getOp() == AtomicExpr::AO__opencl_atomic_load ||
1351                E->getOp() == AtomicExpr::AO__hip_atomic_load ||
1352                E->getOp() == AtomicExpr::AO__atomic_load ||
1353                E->getOp() == AtomicExpr::AO__atomic_load_n;
1354
1355  if (isa<llvm::ConstantInt>(Order)) {
1356    auto ord = cast<llvm::ConstantInt>(Order)->getZExtValue();
1357    // We should not ever get to a case where the ordering isn't a valid C ABI
1358    // value, but it's hard to enforce that in general.
1359    if (llvm::isValidAtomicOrderingCABI(ord))
1360      switch ((llvm::AtomicOrderingCABI)ord) {
1361      case llvm::AtomicOrderingCABI::relaxed:
1362        EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1363                     llvm::AtomicOrdering::Monotonic, Scope);
1364        break;
1365      case llvm::AtomicOrderingCABI::consume:
1366      case llvm::AtomicOrderingCABI::acquire:
1367        if (IsStore)
1368          break; // Avoid crashing on code with undefined behavior
1369        EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1370                     llvm::AtomicOrdering::Acquire, Scope);
1371        break;
1372      case llvm::AtomicOrderingCABI::release:
1373        if (IsLoad)
1374          break; // Avoid crashing on code with undefined behavior
1375        EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1376                     llvm::AtomicOrdering::Release, Scope);
1377        break;
1378      case llvm::AtomicOrderingCABI::acq_rel:
1379        if (IsLoad || IsStore)
1380          break; // Avoid crashing on code with undefined behavior
1381        EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1382                     llvm::AtomicOrdering::AcquireRelease, Scope);
1383        break;
1384      case llvm::AtomicOrderingCABI::seq_cst:
1385        EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1386                     llvm::AtomicOrdering::SequentiallyConsistent, Scope);
1387        break;
1388      }
1389    if (RValTy->isVoidType())
1390      return RValue::get(nullptr);
1391
1392    return convertTempToRValue(
1393        Builder.CreateElementBitCast(Dest, ConvertTypeForMem(RValTy)),
1394        RValTy, E->getExprLoc());
1395  }
1396
1397  // Long case, when Order isn't obviously constant.
1398
1399  // Create all the relevant BB's
1400  llvm::BasicBlock *MonotonicBB = nullptr, *AcquireBB = nullptr,
1401                   *ReleaseBB = nullptr, *AcqRelBB = nullptr,
1402                   *SeqCstBB = nullptr;
1403  MonotonicBB = createBasicBlock("monotonic", CurFn);
1404  if (!IsStore)
1405    AcquireBB = createBasicBlock("acquire", CurFn);
1406  if (!IsLoad)
1407    ReleaseBB = createBasicBlock("release", CurFn);
1408  if (!IsLoad && !IsStore)
1409    AcqRelBB = createBasicBlock("acqrel", CurFn);
1410  SeqCstBB = createBasicBlock("seqcst", CurFn);
1411  llvm::BasicBlock *ContBB = createBasicBlock("atomic.continue", CurFn);
1412
1413  // Create the switch for the split
1414  // MonotonicBB is arbitrarily chosen as the default case; in practice, this
1415  // doesn't matter unless someone is crazy enough to use something that
1416  // doesn't fold to a constant for the ordering.
1417  Order = Builder.CreateIntCast(Order, Builder.getInt32Ty(), false);
1418  llvm::SwitchInst *SI = Builder.CreateSwitch(Order, MonotonicBB);
1419
1420  // Emit all the different atomics
1421  Builder.SetInsertPoint(MonotonicBB);
1422  EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1423               llvm::AtomicOrdering::Monotonic, Scope);
1424  Builder.CreateBr(ContBB);
1425  if (!IsStore) {
1426    Builder.SetInsertPoint(AcquireBB);
1427    EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1428                 llvm::AtomicOrdering::Acquire, Scope);
1429    Builder.CreateBr(ContBB);
1430    SI->addCase(Builder.getInt32((int)llvm::AtomicOrderingCABI::consume),
1431                AcquireBB);
1432    SI->addCase(Builder.getInt32((int)llvm::AtomicOrderingCABI::acquire),
1433                AcquireBB);
1434  }
1435  if (!IsLoad) {
1436    Builder.SetInsertPoint(ReleaseBB);
1437    EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1438                 llvm::AtomicOrdering::Release, Scope);
1439    Builder.CreateBr(ContBB);
1440    SI->addCase(Builder.getInt32((int)llvm::AtomicOrderingCABI::release),
1441                ReleaseBB);
1442  }
1443  if (!IsLoad && !IsStore) {
1444    Builder.SetInsertPoint(AcqRelBB);
1445    EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1446                 llvm::AtomicOrdering::AcquireRelease, Scope);
1447    Builder.CreateBr(ContBB);
1448    SI->addCase(Builder.getInt32((int)llvm::AtomicOrderingCABI::acq_rel),
1449                AcqRelBB);
1450  }
1451  Builder.SetInsertPoint(SeqCstBB);
1452  EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, IsWeak, OrderFail, Size,
1453               llvm::AtomicOrdering::SequentiallyConsistent, Scope);
1454  Builder.CreateBr(ContBB);
1455  SI->addCase(Builder.getInt32((int)llvm::AtomicOrderingCABI::seq_cst),
1456              SeqCstBB);
1457
1458  // Cleanup and return
1459  Builder.SetInsertPoint(ContBB);
1460  if (RValTy->isVoidType())
1461    return RValue::get(nullptr);
1462
1463  assert(Atomics.getValueSizeInBits() <= Atomics.getAtomicSizeInBits());
1464  return convertTempToRValue(
1465      Builder.CreateElementBitCast(Dest, ConvertTypeForMem(RValTy)),
1466      RValTy, E->getExprLoc());
1467}
1468
1469Address AtomicInfo::emitCastToAtomicIntPointer(Address addr) const {
1470  llvm::IntegerType *ty =
1471    llvm::IntegerType::get(CGF.getLLVMContext(), AtomicSizeInBits);
1472  return CGF.Builder.CreateElementBitCast(addr, ty);
1473}
1474
1475Address AtomicInfo::convertToAtomicIntPointer(Address Addr) const {
1476  llvm::Type *Ty = Addr.getElementType();
1477  uint64_t SourceSizeInBits = CGF.CGM.getDataLayout().getTypeSizeInBits(Ty);
1478  if (SourceSizeInBits != AtomicSizeInBits) {
1479    Address Tmp = CreateTempAlloca();
1480    CGF.Builder.CreateMemCpy(Tmp, Addr,
1481                             std::min(AtomicSizeInBits, SourceSizeInBits) / 8);
1482    Addr = Tmp;
1483  }
1484
1485  return emitCastToAtomicIntPointer(Addr);
1486}
1487
1488RValue AtomicInfo::convertAtomicTempToRValue(Address addr,
1489                                             AggValueSlot resultSlot,
1490                                             SourceLocation loc,
1491                                             bool asValue) const {
1492  if (LVal.isSimple()) {
1493    if (EvaluationKind == TEK_Aggregate)
1494      return resultSlot.asRValue();
1495
1496    // Drill into the padding structure if we have one.
1497    if (hasPadding())
1498      addr = CGF.Builder.CreateStructGEP(addr, 0);
1499
1500    // Otherwise, just convert the temporary to an r-value using the
1501    // normal conversion routine.
1502    return CGF.convertTempToRValue(addr, getValueType(), loc);
1503  }
1504  if (!asValue)
1505    // Get RValue from temp memory as atomic for non-simple lvalues
1506    return RValue::get(CGF.Builder.CreateLoad(addr));
1507  if (LVal.isBitField())
1508    return CGF.EmitLoadOfBitfieldLValue(
1509        LValue::MakeBitfield(addr, LVal.getBitFieldInfo(), LVal.getType(),
1510                             LVal.getBaseInfo(), TBAAAccessInfo()), loc);
1511  if (LVal.isVectorElt())
1512    return CGF.EmitLoadOfLValue(
1513        LValue::MakeVectorElt(addr, LVal.getVectorIdx(), LVal.getType(),
1514                              LVal.getBaseInfo(), TBAAAccessInfo()), loc);
1515  assert(LVal.isExtVectorElt());
1516  return CGF.EmitLoadOfExtVectorElementLValue(LValue::MakeExtVectorElt(
1517      addr, LVal.getExtVectorElts(), LVal.getType(),
1518      LVal.getBaseInfo(), TBAAAccessInfo()));
1519}
1520
1521RValue AtomicInfo::ConvertIntToValueOrAtomic(llvm::Value *IntVal,
1522                                             AggValueSlot ResultSlot,
1523                                             SourceLocation Loc,
1524                                             bool AsValue) const {
1525  // Try not to in some easy cases.
1526  assert(IntVal->getType()->isIntegerTy() && "Expected integer value");
1527  if (getEvaluationKind() == TEK_Scalar &&
1528      (((!LVal.isBitField() ||
1529         LVal.getBitFieldInfo().Size == ValueSizeInBits) &&
1530        !hasPadding()) ||
1531       !AsValue)) {
1532    auto *ValTy = AsValue
1533                      ? CGF.ConvertTypeForMem(ValueTy)
1534                      : getAtomicAddress().getElementType();
1535    if (ValTy->isIntegerTy()) {
1536      assert(IntVal->getType() == ValTy && "Different integer types.");
1537      return RValue::get(CGF.EmitFromMemory(IntVal, ValueTy));
1538    } else if (ValTy->isPointerTy())
1539      return RValue::get(CGF.Builder.CreateIntToPtr(IntVal, ValTy));
1540    else if (llvm::CastInst::isBitCastable(IntVal->getType(), ValTy))
1541      return RValue::get(CGF.Builder.CreateBitCast(IntVal, ValTy));
1542  }
1543
1544  // Create a temporary.  This needs to be big enough to hold the
1545  // atomic integer.
1546  Address Temp = Address::invalid();
1547  bool TempIsVolatile = false;
1548  if (AsValue && getEvaluationKind() == TEK_Aggregate) {
1549    assert(!ResultSlot.isIgnored());
1550    Temp = ResultSlot.getAddress();
1551    TempIsVolatile = ResultSlot.isVolatile();
1552  } else {
1553    Temp = CreateTempAlloca();
1554  }
1555
1556  // Slam the integer into the temporary.
1557  Address CastTemp = emitCastToAtomicIntPointer(Temp);
1558  CGF.Builder.CreateStore(IntVal, CastTemp)
1559      ->setVolatile(TempIsVolatile);
1560
1561  return convertAtomicTempToRValue(Temp, ResultSlot, Loc, AsValue);
1562}
1563
1564void AtomicInfo::EmitAtomicLoadLibcall(llvm::Value *AddForLoaded,
1565                                       llvm::AtomicOrdering AO, bool) {
1566  // void __atomic_load(size_t size, void *mem, void *return, int order);
1567  CallArgList Args;
1568  Args.add(RValue::get(getAtomicSizeValue()), CGF.getContext().getSizeType());
1569  Args.add(RValue::get(CGF.EmitCastToVoidPtr(getAtomicPointer())),
1570           CGF.getContext().VoidPtrTy);
1571  Args.add(RValue::get(CGF.EmitCastToVoidPtr(AddForLoaded)),
1572           CGF.getContext().VoidPtrTy);
1573  Args.add(
1574      RValue::get(llvm::ConstantInt::get(CGF.IntTy, (int)llvm::toCABI(AO))),
1575      CGF.getContext().IntTy);
1576  emitAtomicLibcall(CGF, "__atomic_load", CGF.getContext().VoidTy, Args);
1577}
1578
1579llvm::Value *AtomicInfo::EmitAtomicLoadOp(llvm::AtomicOrdering AO,
1580                                          bool IsVolatile) {
1581  // Okay, we're doing this natively.
1582  Address Addr = getAtomicAddressAsAtomicIntPointer();
1583  llvm::LoadInst *Load = CGF.Builder.CreateLoad(Addr, "atomic-load");
1584  Load->setAtomic(AO);
1585
1586  // Other decoration.
1587  if (IsVolatile)
1588    Load->setVolatile(true);
1589  CGF.CGM.DecorateInstructionWithTBAA(Load, LVal.getTBAAInfo());
1590  return Load;
1591}
1592
1593/// An LValue is a candidate for having its loads and stores be made atomic if
1594/// we are operating under /volatile:ms *and* the LValue itself is volatile and
1595/// performing such an operation can be performed without a libcall.
1596bool CodeGenFunction::LValueIsSuitableForInlineAtomic(LValue LV) {
1597  if (!CGM.getLangOpts().MSVolatile) return false;
1598  AtomicInfo AI(*this, LV);
1599  bool IsVolatile = LV.isVolatile() || hasVolatileMember(LV.getType());
1600  // An atomic is inline if we don't need to use a libcall.
1601  bool AtomicIsInline = !AI.shouldUseLibcall();
1602  // MSVC doesn't seem to do this for types wider than a pointer.
1603  if (getContext().getTypeSize(LV.getType()) >
1604      getContext().getTypeSize(getContext().getIntPtrType()))
1605    return false;
1606  return IsVolatile && AtomicIsInline;
1607}
1608
1609RValue CodeGenFunction::EmitAtomicLoad(LValue LV, SourceLocation SL,
1610                                       AggValueSlot Slot) {
1611  llvm::AtomicOrdering AO;
1612  bool IsVolatile = LV.isVolatileQualified();
1613  if (LV.getType()->isAtomicType()) {
1614    AO = llvm::AtomicOrdering::SequentiallyConsistent;
1615  } else {
1616    AO = llvm::AtomicOrdering::Acquire;
1617    IsVolatile = true;
1618  }
1619  return EmitAtomicLoad(LV, SL, AO, IsVolatile, Slot);
1620}
1621
1622RValue AtomicInfo::EmitAtomicLoad(AggValueSlot ResultSlot, SourceLocation Loc,
1623                                  bool AsValue, llvm::AtomicOrdering AO,
1624                                  bool IsVolatile) {
1625  // Check whether we should use a library call.
1626  if (shouldUseLibcall()) {
1627    Address TempAddr = Address::invalid();
1628    if (LVal.isSimple() && !ResultSlot.isIgnored()) {
1629      assert(getEvaluationKind() == TEK_Aggregate);
1630      TempAddr = ResultSlot.getAddress();
1631    } else
1632      TempAddr = CreateTempAlloca();
1633
1634    EmitAtomicLoadLibcall(TempAddr.getPointer(), AO, IsVolatile);
1635
1636    // Okay, turn that back into the original value or whole atomic (for
1637    // non-simple lvalues) type.
1638    return convertAtomicTempToRValue(TempAddr, ResultSlot, Loc, AsValue);
1639  }
1640
1641  // Okay, we're doing this natively.
1642  auto *Load = EmitAtomicLoadOp(AO, IsVolatile);
1643
1644  // If we're ignoring an aggregate return, don't do anything.
1645  if (getEvaluationKind() == TEK_Aggregate && ResultSlot.isIgnored())
1646    return RValue::getAggregate(Address::invalid(), false);
1647
1648  // Okay, turn that back into the original value or atomic (for non-simple
1649  // lvalues) type.
1650  return ConvertIntToValueOrAtomic(Load, ResultSlot, Loc, AsValue);
1651}
1652
1653/// Emit a load from an l-value of atomic type.  Note that the r-value
1654/// we produce is an r-value of the atomic *value* type.
1655RValue CodeGenFunction::EmitAtomicLoad(LValue src, SourceLocation loc,
1656                                       llvm::AtomicOrdering AO, bool IsVolatile,
1657                                       AggValueSlot resultSlot) {
1658  AtomicInfo Atomics(*this, src);
1659  return Atomics.EmitAtomicLoad(resultSlot, loc, /*AsValue=*/true, AO,
1660                                IsVolatile);
1661}
1662
1663/// Copy an r-value into memory as part of storing to an atomic type.
1664/// This needs to create a bit-pattern suitable for atomic operations.
1665void AtomicInfo::emitCopyIntoMemory(RValue rvalue) const {
1666  assert(LVal.isSimple());
1667  // If we have an r-value, the rvalue should be of the atomic type,
1668  // which means that the caller is responsible for having zeroed
1669  // any padding.  Just do an aggregate copy of that type.
1670  if (rvalue.isAggregate()) {
1671    LValue Dest = CGF.MakeAddrLValue(getAtomicAddress(), getAtomicType());
1672    LValue Src = CGF.MakeAddrLValue(rvalue.getAggregateAddress(),
1673                                    getAtomicType());
1674    bool IsVolatile = rvalue.isVolatileQualified() ||
1675                      LVal.isVolatileQualified();
1676    CGF.EmitAggregateCopy(Dest, Src, getAtomicType(),
1677                          AggValueSlot::DoesNotOverlap, IsVolatile);
1678    return;
1679  }
1680
1681  // Okay, otherwise we're copying stuff.
1682
1683  // Zero out the buffer if necessary.
1684  emitMemSetZeroIfNecessary();
1685
1686  // Drill past the padding if present.
1687  LValue TempLVal = projectValue();
1688
1689  // Okay, store the rvalue in.
1690  if (rvalue.isScalar()) {
1691    CGF.EmitStoreOfScalar(rvalue.getScalarVal(), TempLVal, /*init*/ true);
1692  } else {
1693    CGF.EmitStoreOfComplex(rvalue.getComplexVal(), TempLVal, /*init*/ true);
1694  }
1695}
1696
1697
1698/// Materialize an r-value into memory for the purposes of storing it
1699/// to an atomic type.
1700Address AtomicInfo::materializeRValue(RValue rvalue) const {
1701  // Aggregate r-values are already in memory, and EmitAtomicStore
1702  // requires them to be values of the atomic type.
1703  if (rvalue.isAggregate())
1704    return rvalue.getAggregateAddress();
1705
1706  // Otherwise, make a temporary and materialize into it.
1707  LValue TempLV = CGF.MakeAddrLValue(CreateTempAlloca(), getAtomicType());
1708  AtomicInfo Atomics(CGF, TempLV);
1709  Atomics.emitCopyIntoMemory(rvalue);
1710  return TempLV.getAddress(CGF);
1711}
1712
1713llvm::Value *AtomicInfo::convertRValueToInt(RValue RVal) const {
1714  // If we've got a scalar value of the right size, try to avoid going
1715  // through memory.
1716  if (RVal.isScalar() && (!hasPadding() || !LVal.isSimple())) {
1717    llvm::Value *Value = RVal.getScalarVal();
1718    if (isa<llvm::IntegerType>(Value->getType()))
1719      return CGF.EmitToMemory(Value, ValueTy);
1720    else {
1721      llvm::IntegerType *InputIntTy = llvm::IntegerType::get(
1722          CGF.getLLVMContext(),
1723          LVal.isSimple() ? getValueSizeInBits() : getAtomicSizeInBits());
1724      if (isa<llvm::PointerType>(Value->getType()))
1725        return CGF.Builder.CreatePtrToInt(Value, InputIntTy);
1726      else if (llvm::BitCastInst::isBitCastable(Value->getType(), InputIntTy))
1727        return CGF.Builder.CreateBitCast(Value, InputIntTy);
1728    }
1729  }
1730  // Otherwise, we need to go through memory.
1731  // Put the r-value in memory.
1732  Address Addr = materializeRValue(RVal);
1733
1734  // Cast the temporary to the atomic int type and pull a value out.
1735  Addr = emitCastToAtomicIntPointer(Addr);
1736  return CGF.Builder.CreateLoad(Addr);
1737}
1738
1739std::pair<llvm::Value *, llvm::Value *> AtomicInfo::EmitAtomicCompareExchangeOp(
1740    llvm::Value *ExpectedVal, llvm::Value *DesiredVal,
1741    llvm::AtomicOrdering Success, llvm::AtomicOrdering Failure, bool IsWeak) {
1742  // Do the atomic store.
1743  Address Addr = getAtomicAddressAsAtomicIntPointer();
1744  auto *Inst = CGF.Builder.CreateAtomicCmpXchg(Addr.getPointer(),
1745                                               ExpectedVal, DesiredVal,
1746                                               Success, Failure);
1747  // Other decoration.
1748  Inst->setVolatile(LVal.isVolatileQualified());
1749  Inst->setWeak(IsWeak);
1750
1751  // Okay, turn that back into the original value type.
1752  auto *PreviousVal = CGF.Builder.CreateExtractValue(Inst, /*Idxs=*/0);
1753  auto *SuccessFailureVal = CGF.Builder.CreateExtractValue(Inst, /*Idxs=*/1);
1754  return std::make_pair(PreviousVal, SuccessFailureVal);
1755}
1756
1757llvm::Value *
1758AtomicInfo::EmitAtomicCompareExchangeLibcall(llvm::Value *ExpectedAddr,
1759                                             llvm::Value *DesiredAddr,
1760                                             llvm::AtomicOrdering Success,
1761                                             llvm::AtomicOrdering Failure) {
1762  // bool __atomic_compare_exchange(size_t size, void *obj, void *expected,
1763  // void *desired, int success, int failure);
1764  CallArgList Args;
1765  Args.add(RValue::get(getAtomicSizeValue()), CGF.getContext().getSizeType());
1766  Args.add(RValue::get(CGF.EmitCastToVoidPtr(getAtomicPointer())),
1767           CGF.getContext().VoidPtrTy);
1768  Args.add(RValue::get(CGF.EmitCastToVoidPtr(ExpectedAddr)),
1769           CGF.getContext().VoidPtrTy);
1770  Args.add(RValue::get(CGF.EmitCastToVoidPtr(DesiredAddr)),
1771           CGF.getContext().VoidPtrTy);
1772  Args.add(RValue::get(
1773               llvm::ConstantInt::get(CGF.IntTy, (int)llvm::toCABI(Success))),
1774           CGF.getContext().IntTy);
1775  Args.add(RValue::get(
1776               llvm::ConstantInt::get(CGF.IntTy, (int)llvm::toCABI(Failure))),
1777           CGF.getContext().IntTy);
1778  auto SuccessFailureRVal = emitAtomicLibcall(CGF, "__atomic_compare_exchange",
1779                                              CGF.getContext().BoolTy, Args);
1780
1781  return SuccessFailureRVal.getScalarVal();
1782}
1783
1784std::pair<RValue, llvm::Value *> AtomicInfo::EmitAtomicCompareExchange(
1785    RValue Expected, RValue Desired, llvm::AtomicOrdering Success,
1786    llvm::AtomicOrdering Failure, bool IsWeak) {
1787  // Check whether we should use a library call.
1788  if (shouldUseLibcall()) {
1789    // Produce a source address.
1790    Address ExpectedAddr = materializeRValue(Expected);
1791    Address DesiredAddr = materializeRValue(Desired);
1792    auto *Res = EmitAtomicCompareExchangeLibcall(ExpectedAddr.getPointer(),
1793                                                 DesiredAddr.getPointer(),
1794                                                 Success, Failure);
1795    return std::make_pair(
1796        convertAtomicTempToRValue(ExpectedAddr, AggValueSlot::ignored(),
1797                                  SourceLocation(), /*AsValue=*/false),
1798        Res);
1799  }
1800
1801  // If we've got a scalar value of the right size, try to avoid going
1802  // through memory.
1803  auto *ExpectedVal = convertRValueToInt(Expected);
1804  auto *DesiredVal = convertRValueToInt(Desired);
1805  auto Res = EmitAtomicCompareExchangeOp(ExpectedVal, DesiredVal, Success,
1806                                         Failure, IsWeak);
1807  return std::make_pair(
1808      ConvertIntToValueOrAtomic(Res.first, AggValueSlot::ignored(),
1809                                SourceLocation(), /*AsValue=*/false),
1810      Res.second);
1811}
1812
1813static void
1814EmitAtomicUpdateValue(CodeGenFunction &CGF, AtomicInfo &Atomics, RValue OldRVal,
1815                      const llvm::function_ref<RValue(RValue)> &UpdateOp,
1816                      Address DesiredAddr) {
1817  RValue UpRVal;
1818  LValue AtomicLVal = Atomics.getAtomicLValue();
1819  LValue DesiredLVal;
1820  if (AtomicLVal.isSimple()) {
1821    UpRVal = OldRVal;
1822    DesiredLVal = CGF.MakeAddrLValue(DesiredAddr, AtomicLVal.getType());
1823  } else {
1824    // Build new lvalue for temp address.
1825    Address Ptr = Atomics.materializeRValue(OldRVal);
1826    LValue UpdateLVal;
1827    if (AtomicLVal.isBitField()) {
1828      UpdateLVal =
1829          LValue::MakeBitfield(Ptr, AtomicLVal.getBitFieldInfo(),
1830                               AtomicLVal.getType(),
1831                               AtomicLVal.getBaseInfo(),
1832                               AtomicLVal.getTBAAInfo());
1833      DesiredLVal =
1834          LValue::MakeBitfield(DesiredAddr, AtomicLVal.getBitFieldInfo(),
1835                               AtomicLVal.getType(), AtomicLVal.getBaseInfo(),
1836                               AtomicLVal.getTBAAInfo());
1837    } else if (AtomicLVal.isVectorElt()) {
1838      UpdateLVal = LValue::MakeVectorElt(Ptr, AtomicLVal.getVectorIdx(),
1839                                         AtomicLVal.getType(),
1840                                         AtomicLVal.getBaseInfo(),
1841                                         AtomicLVal.getTBAAInfo());
1842      DesiredLVal = LValue::MakeVectorElt(
1843          DesiredAddr, AtomicLVal.getVectorIdx(), AtomicLVal.getType(),
1844          AtomicLVal.getBaseInfo(), AtomicLVal.getTBAAInfo());
1845    } else {
1846      assert(AtomicLVal.isExtVectorElt());
1847      UpdateLVal = LValue::MakeExtVectorElt(Ptr, AtomicLVal.getExtVectorElts(),
1848                                            AtomicLVal.getType(),
1849                                            AtomicLVal.getBaseInfo(),
1850                                            AtomicLVal.getTBAAInfo());
1851      DesiredLVal = LValue::MakeExtVectorElt(
1852          DesiredAddr, AtomicLVal.getExtVectorElts(), AtomicLVal.getType(),
1853          AtomicLVal.getBaseInfo(), AtomicLVal.getTBAAInfo());
1854    }
1855    UpRVal = CGF.EmitLoadOfLValue(UpdateLVal, SourceLocation());
1856  }
1857  // Store new value in the corresponding memory area.
1858  RValue NewRVal = UpdateOp(UpRVal);
1859  if (NewRVal.isScalar()) {
1860    CGF.EmitStoreThroughLValue(NewRVal, DesiredLVal);
1861  } else {
1862    assert(NewRVal.isComplex());
1863    CGF.EmitStoreOfComplex(NewRVal.getComplexVal(), DesiredLVal,
1864                           /*isInit=*/false);
1865  }
1866}
1867
1868void AtomicInfo::EmitAtomicUpdateLibcall(
1869    llvm::AtomicOrdering AO, const llvm::function_ref<RValue(RValue)> &UpdateOp,
1870    bool IsVolatile) {
1871  auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO);
1872
1873  Address ExpectedAddr = CreateTempAlloca();
1874
1875  EmitAtomicLoadLibcall(ExpectedAddr.getPointer(), AO, IsVolatile);
1876  auto *ContBB = CGF.createBasicBlock("atomic_cont");
1877  auto *ExitBB = CGF.createBasicBlock("atomic_exit");
1878  CGF.EmitBlock(ContBB);
1879  Address DesiredAddr = CreateTempAlloca();
1880  if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) ||
1881      requiresMemSetZero(getAtomicAddress().getElementType())) {
1882    auto *OldVal = CGF.Builder.CreateLoad(ExpectedAddr);
1883    CGF.Builder.CreateStore(OldVal, DesiredAddr);
1884  }
1885  auto OldRVal = convertAtomicTempToRValue(ExpectedAddr,
1886                                           AggValueSlot::ignored(),
1887                                           SourceLocation(), /*AsValue=*/false);
1888  EmitAtomicUpdateValue(CGF, *this, OldRVal, UpdateOp, DesiredAddr);
1889  auto *Res =
1890      EmitAtomicCompareExchangeLibcall(ExpectedAddr.getPointer(),
1891                                       DesiredAddr.getPointer(),
1892                                       AO, Failure);
1893  CGF.Builder.CreateCondBr(Res, ExitBB, ContBB);
1894  CGF.EmitBlock(ExitBB, /*IsFinished=*/true);
1895}
1896
1897void AtomicInfo::EmitAtomicUpdateOp(
1898    llvm::AtomicOrdering AO, const llvm::function_ref<RValue(RValue)> &UpdateOp,
1899    bool IsVolatile) {
1900  auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO);
1901
1902  // Do the atomic load.
1903  auto *OldVal = EmitAtomicLoadOp(Failure, IsVolatile);
1904  // For non-simple lvalues perform compare-and-swap procedure.
1905  auto *ContBB = CGF.createBasicBlock("atomic_cont");
1906  auto *ExitBB = CGF.createBasicBlock("atomic_exit");
1907  auto *CurBB = CGF.Builder.GetInsertBlock();
1908  CGF.EmitBlock(ContBB);
1909  llvm::PHINode *PHI = CGF.Builder.CreatePHI(OldVal->getType(),
1910                                             /*NumReservedValues=*/2);
1911  PHI->addIncoming(OldVal, CurBB);
1912  Address NewAtomicAddr = CreateTempAlloca();
1913  Address NewAtomicIntAddr = emitCastToAtomicIntPointer(NewAtomicAddr);
1914  if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) ||
1915      requiresMemSetZero(getAtomicAddress().getElementType())) {
1916    CGF.Builder.CreateStore(PHI, NewAtomicIntAddr);
1917  }
1918  auto OldRVal = ConvertIntToValueOrAtomic(PHI, AggValueSlot::ignored(),
1919                                           SourceLocation(), /*AsValue=*/false);
1920  EmitAtomicUpdateValue(CGF, *this, OldRVal, UpdateOp, NewAtomicAddr);
1921  auto *DesiredVal = CGF.Builder.CreateLoad(NewAtomicIntAddr);
1922  // Try to write new value using cmpxchg operation.
1923  auto Res = EmitAtomicCompareExchangeOp(PHI, DesiredVal, AO, Failure);
1924  PHI->addIncoming(Res.first, CGF.Builder.GetInsertBlock());
1925  CGF.Builder.CreateCondBr(Res.second, ExitBB, ContBB);
1926  CGF.EmitBlock(ExitBB, /*IsFinished=*/true);
1927}
1928
1929static void EmitAtomicUpdateValue(CodeGenFunction &CGF, AtomicInfo &Atomics,
1930                                  RValue UpdateRVal, Address DesiredAddr) {
1931  LValue AtomicLVal = Atomics.getAtomicLValue();
1932  LValue DesiredLVal;
1933  // Build new lvalue for temp address.
1934  if (AtomicLVal.isBitField()) {
1935    DesiredLVal =
1936        LValue::MakeBitfield(DesiredAddr, AtomicLVal.getBitFieldInfo(),
1937                             AtomicLVal.getType(), AtomicLVal.getBaseInfo(),
1938                             AtomicLVal.getTBAAInfo());
1939  } else if (AtomicLVal.isVectorElt()) {
1940    DesiredLVal =
1941        LValue::MakeVectorElt(DesiredAddr, AtomicLVal.getVectorIdx(),
1942                              AtomicLVal.getType(), AtomicLVal.getBaseInfo(),
1943                              AtomicLVal.getTBAAInfo());
1944  } else {
1945    assert(AtomicLVal.isExtVectorElt());
1946    DesiredLVal = LValue::MakeExtVectorElt(
1947        DesiredAddr, AtomicLVal.getExtVectorElts(), AtomicLVal.getType(),
1948        AtomicLVal.getBaseInfo(), AtomicLVal.getTBAAInfo());
1949  }
1950  // Store new value in the corresponding memory area.
1951  assert(UpdateRVal.isScalar());
1952  CGF.EmitStoreThroughLValue(UpdateRVal, DesiredLVal);
1953}
1954
1955void AtomicInfo::EmitAtomicUpdateLibcall(llvm::AtomicOrdering AO,
1956                                         RValue UpdateRVal, bool IsVolatile) {
1957  auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO);
1958
1959  Address ExpectedAddr = CreateTempAlloca();
1960
1961  EmitAtomicLoadLibcall(ExpectedAddr.getPointer(), AO, IsVolatile);
1962  auto *ContBB = CGF.createBasicBlock("atomic_cont");
1963  auto *ExitBB = CGF.createBasicBlock("atomic_exit");
1964  CGF.EmitBlock(ContBB);
1965  Address DesiredAddr = CreateTempAlloca();
1966  if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) ||
1967      requiresMemSetZero(getAtomicAddress().getElementType())) {
1968    auto *OldVal = CGF.Builder.CreateLoad(ExpectedAddr);
1969    CGF.Builder.CreateStore(OldVal, DesiredAddr);
1970  }
1971  EmitAtomicUpdateValue(CGF, *this, UpdateRVal, DesiredAddr);
1972  auto *Res =
1973      EmitAtomicCompareExchangeLibcall(ExpectedAddr.getPointer(),
1974                                       DesiredAddr.getPointer(),
1975                                       AO, Failure);
1976  CGF.Builder.CreateCondBr(Res, ExitBB, ContBB);
1977  CGF.EmitBlock(ExitBB, /*IsFinished=*/true);
1978}
1979
1980void AtomicInfo::EmitAtomicUpdateOp(llvm::AtomicOrdering AO, RValue UpdateRVal,
1981                                    bool IsVolatile) {
1982  auto Failure = llvm::AtomicCmpXchgInst::getStrongestFailureOrdering(AO);
1983
1984  // Do the atomic load.
1985  auto *OldVal = EmitAtomicLoadOp(Failure, IsVolatile);
1986  // For non-simple lvalues perform compare-and-swap procedure.
1987  auto *ContBB = CGF.createBasicBlock("atomic_cont");
1988  auto *ExitBB = CGF.createBasicBlock("atomic_exit");
1989  auto *CurBB = CGF.Builder.GetInsertBlock();
1990  CGF.EmitBlock(ContBB);
1991  llvm::PHINode *PHI = CGF.Builder.CreatePHI(OldVal->getType(),
1992                                             /*NumReservedValues=*/2);
1993  PHI->addIncoming(OldVal, CurBB);
1994  Address NewAtomicAddr = CreateTempAlloca();
1995  Address NewAtomicIntAddr = emitCastToAtomicIntPointer(NewAtomicAddr);
1996  if ((LVal.isBitField() && BFI.Size != ValueSizeInBits) ||
1997      requiresMemSetZero(getAtomicAddress().getElementType())) {
1998    CGF.Builder.CreateStore(PHI, NewAtomicIntAddr);
1999  }
2000  EmitAtomicUpdateValue(CGF, *this, UpdateRVal, NewAtomicAddr);
2001  auto *DesiredVal = CGF.Builder.CreateLoad(NewAtomicIntAddr);
2002  // Try to write new value using cmpxchg operation.
2003  auto Res = EmitAtomicCompareExchangeOp(PHI, DesiredVal, AO, Failure);
2004  PHI->addIncoming(Res.first, CGF.Builder.GetInsertBlock());
2005  CGF.Builder.CreateCondBr(Res.second, ExitBB, ContBB);
2006  CGF.EmitBlock(ExitBB, /*IsFinished=*/true);
2007}
2008
2009void AtomicInfo::EmitAtomicUpdate(
2010    llvm::AtomicOrdering AO, const llvm::function_ref<RValue(RValue)> &UpdateOp,
2011    bool IsVolatile) {
2012  if (shouldUseLibcall()) {
2013    EmitAtomicUpdateLibcall(AO, UpdateOp, IsVolatile);
2014  } else {
2015    EmitAtomicUpdateOp(AO, UpdateOp, IsVolatile);
2016  }
2017}
2018
2019void AtomicInfo::EmitAtomicUpdate(llvm::AtomicOrdering AO, RValue UpdateRVal,
2020                                  bool IsVolatile) {
2021  if (shouldUseLibcall()) {
2022    EmitAtomicUpdateLibcall(AO, UpdateRVal, IsVolatile);
2023  } else {
2024    EmitAtomicUpdateOp(AO, UpdateRVal, IsVolatile);
2025  }
2026}
2027
2028void CodeGenFunction::EmitAtomicStore(RValue rvalue, LValue lvalue,
2029                                      bool isInit) {
2030  bool IsVolatile = lvalue.isVolatileQualified();
2031  llvm::AtomicOrdering AO;
2032  if (lvalue.getType()->isAtomicType()) {
2033    AO = llvm::AtomicOrdering::SequentiallyConsistent;
2034  } else {
2035    AO = llvm::AtomicOrdering::Release;
2036    IsVolatile = true;
2037  }
2038  return EmitAtomicStore(rvalue, lvalue, AO, IsVolatile, isInit);
2039}
2040
2041/// Emit a store to an l-value of atomic type.
2042///
2043/// Note that the r-value is expected to be an r-value *of the atomic
2044/// type*; this means that for aggregate r-values, it should include
2045/// storage for any padding that was necessary.
2046void CodeGenFunction::EmitAtomicStore(RValue rvalue, LValue dest,
2047                                      llvm::AtomicOrdering AO, bool IsVolatile,
2048                                      bool isInit) {
2049  // If this is an aggregate r-value, it should agree in type except
2050  // maybe for address-space qualification.
2051  assert(!rvalue.isAggregate() ||
2052         rvalue.getAggregateAddress().getElementType() ==
2053             dest.getAddress(*this).getElementType());
2054
2055  AtomicInfo atomics(*this, dest);
2056  LValue LVal = atomics.getAtomicLValue();
2057
2058  // If this is an initialization, just put the value there normally.
2059  if (LVal.isSimple()) {
2060    if (isInit) {
2061      atomics.emitCopyIntoMemory(rvalue);
2062      return;
2063    }
2064
2065    // Check whether we should use a library call.
2066    if (atomics.shouldUseLibcall()) {
2067      // Produce a source address.
2068      Address srcAddr = atomics.materializeRValue(rvalue);
2069
2070      // void __atomic_store(size_t size, void *mem, void *val, int order)
2071      CallArgList args;
2072      args.add(RValue::get(atomics.getAtomicSizeValue()),
2073               getContext().getSizeType());
2074      args.add(RValue::get(EmitCastToVoidPtr(atomics.getAtomicPointer())),
2075               getContext().VoidPtrTy);
2076      args.add(RValue::get(EmitCastToVoidPtr(srcAddr.getPointer())),
2077               getContext().VoidPtrTy);
2078      args.add(
2079          RValue::get(llvm::ConstantInt::get(IntTy, (int)llvm::toCABI(AO))),
2080          getContext().IntTy);
2081      emitAtomicLibcall(*this, "__atomic_store", getContext().VoidTy, args);
2082      return;
2083    }
2084
2085    // Okay, we're doing this natively.
2086    llvm::Value *intValue = atomics.convertRValueToInt(rvalue);
2087
2088    // Do the atomic store.
2089    Address addr =
2090        atomics.emitCastToAtomicIntPointer(atomics.getAtomicAddress());
2091    intValue = Builder.CreateIntCast(
2092        intValue, addr.getElementType(), /*isSigned=*/false);
2093    llvm::StoreInst *store = Builder.CreateStore(intValue, addr);
2094
2095    if (AO == llvm::AtomicOrdering::Acquire)
2096      AO = llvm::AtomicOrdering::Monotonic;
2097    else if (AO == llvm::AtomicOrdering::AcquireRelease)
2098      AO = llvm::AtomicOrdering::Release;
2099    // Initializations don't need to be atomic.
2100    if (!isInit)
2101      store->setAtomic(AO);
2102
2103    // Other decoration.
2104    if (IsVolatile)
2105      store->setVolatile(true);
2106    CGM.DecorateInstructionWithTBAA(store, dest.getTBAAInfo());
2107    return;
2108  }
2109
2110  // Emit simple atomic update operation.
2111  atomics.EmitAtomicUpdate(AO, rvalue, IsVolatile);
2112}
2113
2114/// Emit a compare-and-exchange op for atomic type.
2115///
2116std::pair<RValue, llvm::Value *> CodeGenFunction::EmitAtomicCompareExchange(
2117    LValue Obj, RValue Expected, RValue Desired, SourceLocation Loc,
2118    llvm::AtomicOrdering Success, llvm::AtomicOrdering Failure, bool IsWeak,
2119    AggValueSlot Slot) {
2120  // If this is an aggregate r-value, it should agree in type except
2121  // maybe for address-space qualification.
2122  assert(!Expected.isAggregate() ||
2123         Expected.getAggregateAddress().getElementType() ==
2124             Obj.getAddress(*this).getElementType());
2125  assert(!Desired.isAggregate() ||
2126         Desired.getAggregateAddress().getElementType() ==
2127             Obj.getAddress(*this).getElementType());
2128  AtomicInfo Atomics(*this, Obj);
2129
2130  return Atomics.EmitAtomicCompareExchange(Expected, Desired, Success, Failure,
2131                                           IsWeak);
2132}
2133
2134void CodeGenFunction::EmitAtomicUpdate(
2135    LValue LVal, llvm::AtomicOrdering AO,
2136    const llvm::function_ref<RValue(RValue)> &UpdateOp, bool IsVolatile) {
2137  AtomicInfo Atomics(*this, LVal);
2138  Atomics.EmitAtomicUpdate(AO, UpdateOp, IsVolatile);
2139}
2140
2141void CodeGenFunction::EmitAtomicInit(Expr *init, LValue dest) {
2142  AtomicInfo atomics(*this, dest);
2143
2144  switch (atomics.getEvaluationKind()) {
2145  case TEK_Scalar: {
2146    llvm::Value *value = EmitScalarExpr(init);
2147    atomics.emitCopyIntoMemory(RValue::get(value));
2148    return;
2149  }
2150
2151  case TEK_Complex: {
2152    ComplexPairTy value = EmitComplexExpr(init);
2153    atomics.emitCopyIntoMemory(RValue::getComplex(value));
2154    return;
2155  }
2156
2157  case TEK_Aggregate: {
2158    // Fix up the destination if the initializer isn't an expression
2159    // of atomic type.
2160    bool Zeroed = false;
2161    if (!init->getType()->isAtomicType()) {
2162      Zeroed = atomics.emitMemSetZeroIfNecessary();
2163      dest = atomics.projectValue();
2164    }
2165
2166    // Evaluate the expression directly into the destination.
2167    AggValueSlot slot = AggValueSlot::forLValue(
2168        dest, *this, AggValueSlot::IsNotDestructed,
2169        AggValueSlot::DoesNotNeedGCBarriers, AggValueSlot::IsNotAliased,
2170        AggValueSlot::DoesNotOverlap,
2171        Zeroed ? AggValueSlot::IsZeroed : AggValueSlot::IsNotZeroed);
2172
2173    EmitAggExpr(init, slot);
2174    return;
2175  }
2176  }
2177  llvm_unreachable("bad evaluation kind");
2178}
2179