1//===- LowerAtomic.cpp - Lower atomic intrinsics --------------------------===// 2// 3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4// See https://llvm.org/LICENSE.txt for license information. 5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6// 7//===----------------------------------------------------------------------===// 8// 9// This pass lowers atomic intrinsics to non-atomic form for use in a known 10// non-preemptible environment. 11// 12//===----------------------------------------------------------------------===// 13 14#include "llvm/Transforms/Scalar/LowerAtomic.h" 15#include "llvm/IR/Function.h" 16#include "llvm/IR/IRBuilder.h" 17#include "llvm/InitializePasses.h" 18#include "llvm/Pass.h" 19#include "llvm/Transforms/Scalar.h" 20using namespace llvm; 21 22#define DEBUG_TYPE "loweratomic" 23 24static bool LowerAtomicCmpXchgInst(AtomicCmpXchgInst *CXI) { 25 IRBuilder<> Builder(CXI); 26 Value *Ptr = CXI->getPointerOperand(); 27 Value *Cmp = CXI->getCompareOperand(); 28 Value *Val = CXI->getNewValOperand(); 29 30 LoadInst *Orig = Builder.CreateLoad(Val->getType(), Ptr); 31 Value *Equal = Builder.CreateICmpEQ(Orig, Cmp); 32 Value *Res = Builder.CreateSelect(Equal, Val, Orig); 33 Builder.CreateStore(Res, Ptr); 34 35 Res = Builder.CreateInsertValue(UndefValue::get(CXI->getType()), Orig, 0); 36 Res = Builder.CreateInsertValue(Res, Equal, 1); 37 38 CXI->replaceAllUsesWith(Res); 39 CXI->eraseFromParent(); 40 return true; 41} 42 43static bool LowerAtomicRMWInst(AtomicRMWInst *RMWI) { 44 IRBuilder<> Builder(RMWI); 45 Value *Ptr = RMWI->getPointerOperand(); 46 Value *Val = RMWI->getValOperand(); 47 48 LoadInst *Orig = Builder.CreateLoad(Val->getType(), Ptr); 49 Value *Res = nullptr; 50 51 switch (RMWI->getOperation()) { 52 default: llvm_unreachable("Unexpected RMW operation"); 53 case AtomicRMWInst::Xchg: 54 Res = Val; 55 break; 56 case AtomicRMWInst::Add: 57 Res = Builder.CreateAdd(Orig, Val); 58 break; 59 case AtomicRMWInst::Sub: 60 Res = Builder.CreateSub(Orig, Val); 61 break; 62 case AtomicRMWInst::And: 63 Res = Builder.CreateAnd(Orig, Val); 64 break; 65 case AtomicRMWInst::Nand: 66 Res = Builder.CreateNot(Builder.CreateAnd(Orig, Val)); 67 break; 68 case AtomicRMWInst::Or: 69 Res = Builder.CreateOr(Orig, Val); 70 break; 71 case AtomicRMWInst::Xor: 72 Res = Builder.CreateXor(Orig, Val); 73 break; 74 case AtomicRMWInst::Max: 75 Res = Builder.CreateSelect(Builder.CreateICmpSLT(Orig, Val), 76 Val, Orig); 77 break; 78 case AtomicRMWInst::Min: 79 Res = Builder.CreateSelect(Builder.CreateICmpSLT(Orig, Val), 80 Orig, Val); 81 break; 82 case AtomicRMWInst::UMax: 83 Res = Builder.CreateSelect(Builder.CreateICmpULT(Orig, Val), 84 Val, Orig); 85 break; 86 case AtomicRMWInst::UMin: 87 Res = Builder.CreateSelect(Builder.CreateICmpULT(Orig, Val), 88 Orig, Val); 89 break; 90 case AtomicRMWInst::FAdd: 91 Res = Builder.CreateFAdd(Orig, Val); 92 break; 93 case AtomicRMWInst::FSub: 94 Res = Builder.CreateFSub(Orig, Val); 95 break; 96 } 97 Builder.CreateStore(Res, Ptr); 98 RMWI->replaceAllUsesWith(Orig); 99 RMWI->eraseFromParent(); 100 return true; 101} 102 103static bool LowerFenceInst(FenceInst *FI) { 104 FI->eraseFromParent(); 105 return true; 106} 107 108static bool LowerLoadInst(LoadInst *LI) { 109 LI->setAtomic(AtomicOrdering::NotAtomic); 110 return true; 111} 112 113static bool LowerStoreInst(StoreInst *SI) { 114 SI->setAtomic(AtomicOrdering::NotAtomic); 115 return true; 116} 117 118static bool runOnBasicBlock(BasicBlock &BB) { 119 bool Changed = false; 120 for (Instruction &Inst : make_early_inc_range(BB)) { 121 if (FenceInst *FI = dyn_cast<FenceInst>(&Inst)) 122 Changed |= LowerFenceInst(FI); 123 else if (AtomicCmpXchgInst *CXI = dyn_cast<AtomicCmpXchgInst>(&Inst)) 124 Changed |= LowerAtomicCmpXchgInst(CXI); 125 else if (AtomicRMWInst *RMWI = dyn_cast<AtomicRMWInst>(&Inst)) 126 Changed |= LowerAtomicRMWInst(RMWI); 127 else if (LoadInst *LI = dyn_cast<LoadInst>(&Inst)) { 128 if (LI->isAtomic()) 129 LowerLoadInst(LI); 130 } else if (StoreInst *SI = dyn_cast<StoreInst>(&Inst)) { 131 if (SI->isAtomic()) 132 LowerStoreInst(SI); 133 } 134 } 135 return Changed; 136} 137 138static bool lowerAtomics(Function &F) { 139 bool Changed = false; 140 for (BasicBlock &BB : F) { 141 Changed |= runOnBasicBlock(BB); 142 } 143 return Changed; 144} 145 146PreservedAnalyses LowerAtomicPass::run(Function &F, FunctionAnalysisManager &) { 147 if (lowerAtomics(F)) 148 return PreservedAnalyses::none(); 149 return PreservedAnalyses::all(); 150} 151 152namespace { 153class LowerAtomicLegacyPass : public FunctionPass { 154public: 155 static char ID; 156 157 LowerAtomicLegacyPass() : FunctionPass(ID) { 158 initializeLowerAtomicLegacyPassPass(*PassRegistry::getPassRegistry()); 159 } 160 161 bool runOnFunction(Function &F) override { 162 // Don't skip optnone functions; atomics still need to be lowered. 163 FunctionAnalysisManager DummyFAM; 164 auto PA = Impl.run(F, DummyFAM); 165 return !PA.areAllPreserved(); 166 } 167 168private: 169 LowerAtomicPass Impl; 170 }; 171} 172 173char LowerAtomicLegacyPass::ID = 0; 174INITIALIZE_PASS(LowerAtomicLegacyPass, "loweratomic", 175 "Lower atomic intrinsics to non-atomic form", false, false) 176 177Pass *llvm::createLowerAtomicPass() { return new LowerAtomicLegacyPass(); } 178