1//===- LowerAtomic.cpp - Lower atomic intrinsics --------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This pass lowers atomic intrinsics to non-atomic form for use in a known
10// non-preemptible environment.
11//
12//===----------------------------------------------------------------------===//
13
14#include "llvm/Transforms/Scalar/LowerAtomic.h"
15#include "llvm/IR/Function.h"
16#include "llvm/IR/IRBuilder.h"
17#include "llvm/InitializePasses.h"
18#include "llvm/Pass.h"
19#include "llvm/Transforms/Scalar.h"
20using namespace llvm;
21
22#define DEBUG_TYPE "loweratomic"
23
24static bool LowerAtomicCmpXchgInst(AtomicCmpXchgInst *CXI) {
25  IRBuilder<> Builder(CXI);
26  Value *Ptr = CXI->getPointerOperand();
27  Value *Cmp = CXI->getCompareOperand();
28  Value *Val = CXI->getNewValOperand();
29
30  LoadInst *Orig = Builder.CreateLoad(Val->getType(), Ptr);
31  Value *Equal = Builder.CreateICmpEQ(Orig, Cmp);
32  Value *Res = Builder.CreateSelect(Equal, Val, Orig);
33  Builder.CreateStore(Res, Ptr);
34
35  Res = Builder.CreateInsertValue(UndefValue::get(CXI->getType()), Orig, 0);
36  Res = Builder.CreateInsertValue(Res, Equal, 1);
37
38  CXI->replaceAllUsesWith(Res);
39  CXI->eraseFromParent();
40  return true;
41}
42
43static bool LowerAtomicRMWInst(AtomicRMWInst *RMWI) {
44  IRBuilder<> Builder(RMWI);
45  Value *Ptr = RMWI->getPointerOperand();
46  Value *Val = RMWI->getValOperand();
47
48  LoadInst *Orig = Builder.CreateLoad(Val->getType(), Ptr);
49  Value *Res = nullptr;
50
51  switch (RMWI->getOperation()) {
52  default: llvm_unreachable("Unexpected RMW operation");
53  case AtomicRMWInst::Xchg:
54    Res = Val;
55    break;
56  case AtomicRMWInst::Add:
57    Res = Builder.CreateAdd(Orig, Val);
58    break;
59  case AtomicRMWInst::Sub:
60    Res = Builder.CreateSub(Orig, Val);
61    break;
62  case AtomicRMWInst::And:
63    Res = Builder.CreateAnd(Orig, Val);
64    break;
65  case AtomicRMWInst::Nand:
66    Res = Builder.CreateNot(Builder.CreateAnd(Orig, Val));
67    break;
68  case AtomicRMWInst::Or:
69    Res = Builder.CreateOr(Orig, Val);
70    break;
71  case AtomicRMWInst::Xor:
72    Res = Builder.CreateXor(Orig, Val);
73    break;
74  case AtomicRMWInst::Max:
75    Res = Builder.CreateSelect(Builder.CreateICmpSLT(Orig, Val),
76                               Val, Orig);
77    break;
78  case AtomicRMWInst::Min:
79    Res = Builder.CreateSelect(Builder.CreateICmpSLT(Orig, Val),
80                               Orig, Val);
81    break;
82  case AtomicRMWInst::UMax:
83    Res = Builder.CreateSelect(Builder.CreateICmpULT(Orig, Val),
84                               Val, Orig);
85    break;
86  case AtomicRMWInst::UMin:
87    Res = Builder.CreateSelect(Builder.CreateICmpULT(Orig, Val),
88                               Orig, Val);
89    break;
90  case AtomicRMWInst::FAdd:
91    Res = Builder.CreateFAdd(Orig, Val);
92    break;
93  case AtomicRMWInst::FSub:
94    Res = Builder.CreateFSub(Orig, Val);
95    break;
96  }
97  Builder.CreateStore(Res, Ptr);
98  RMWI->replaceAllUsesWith(Orig);
99  RMWI->eraseFromParent();
100  return true;
101}
102
103static bool LowerFenceInst(FenceInst *FI) {
104  FI->eraseFromParent();
105  return true;
106}
107
108static bool LowerLoadInst(LoadInst *LI) {
109  LI->setAtomic(AtomicOrdering::NotAtomic);
110  return true;
111}
112
113static bool LowerStoreInst(StoreInst *SI) {
114  SI->setAtomic(AtomicOrdering::NotAtomic);
115  return true;
116}
117
118static bool runOnBasicBlock(BasicBlock &BB) {
119  bool Changed = false;
120  for (Instruction &Inst : make_early_inc_range(BB)) {
121    if (FenceInst *FI = dyn_cast<FenceInst>(&Inst))
122      Changed |= LowerFenceInst(FI);
123    else if (AtomicCmpXchgInst *CXI = dyn_cast<AtomicCmpXchgInst>(&Inst))
124      Changed |= LowerAtomicCmpXchgInst(CXI);
125    else if (AtomicRMWInst *RMWI = dyn_cast<AtomicRMWInst>(&Inst))
126      Changed |= LowerAtomicRMWInst(RMWI);
127    else if (LoadInst *LI = dyn_cast<LoadInst>(&Inst)) {
128      if (LI->isAtomic())
129        LowerLoadInst(LI);
130    } else if (StoreInst *SI = dyn_cast<StoreInst>(&Inst)) {
131      if (SI->isAtomic())
132        LowerStoreInst(SI);
133    }
134  }
135  return Changed;
136}
137
138static bool lowerAtomics(Function &F) {
139  bool Changed = false;
140  for (BasicBlock &BB : F) {
141    Changed |= runOnBasicBlock(BB);
142  }
143  return Changed;
144}
145
146PreservedAnalyses LowerAtomicPass::run(Function &F, FunctionAnalysisManager &) {
147  if (lowerAtomics(F))
148    return PreservedAnalyses::none();
149  return PreservedAnalyses::all();
150}
151
152namespace {
153class LowerAtomicLegacyPass : public FunctionPass {
154public:
155  static char ID;
156
157  LowerAtomicLegacyPass() : FunctionPass(ID) {
158    initializeLowerAtomicLegacyPassPass(*PassRegistry::getPassRegistry());
159  }
160
161  bool runOnFunction(Function &F) override {
162    // Don't skip optnone functions; atomics still need to be lowered.
163    FunctionAnalysisManager DummyFAM;
164    auto PA = Impl.run(F, DummyFAM);
165    return !PA.areAllPreserved();
166  }
167
168private:
169  LowerAtomicPass Impl;
170  };
171}
172
173char LowerAtomicLegacyPass::ID = 0;
174INITIALIZE_PASS(LowerAtomicLegacyPass, "loweratomic",
175                "Lower atomic intrinsics to non-atomic form", false, false)
176
177Pass *llvm::createLowerAtomicPass() { return new LowerAtomicLegacyPass(); }
178