1351278Sdim//===- InstCombineAtomicRMW.cpp -------------------------------------------===// 2351278Sdim// 3351278Sdim// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4351278Sdim// See https://llvm.org/LICENSE.txt for license information. 5351278Sdim// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6351278Sdim// 7351278Sdim//===----------------------------------------------------------------------===// 8351278Sdim// 9351278Sdim// This file implements the visit functions for atomic rmw instructions. 10351278Sdim// 11351278Sdim//===----------------------------------------------------------------------===// 12351278Sdim#include "InstCombineInternal.h" 13351278Sdim#include "llvm/IR/Instructions.h" 14351278Sdim 15351278Sdimusing namespace llvm; 16351278Sdim 17351278Sdimnamespace { 18351278Sdim/// Return true if and only if the given instruction does not modify the memory 19351278Sdim/// location referenced. Note that an idemptent atomicrmw may still have 20351278Sdim/// ordering effects on nearby instructions, or be volatile. 21351278Sdim/// TODO: Common w/ the version in AtomicExpandPass, and change the term used. 22351278Sdim/// Idemptotent is confusing in this context. 23351278Sdimbool isIdempotentRMW(AtomicRMWInst& RMWI) { 24351278Sdim if (auto CF = dyn_cast<ConstantFP>(RMWI.getValOperand())) 25351278Sdim switch(RMWI.getOperation()) { 26351278Sdim case AtomicRMWInst::FAdd: // -0.0 27351278Sdim return CF->isZero() && CF->isNegative(); 28351278Sdim case AtomicRMWInst::FSub: // +0.0 29351278Sdim return CF->isZero() && !CF->isNegative(); 30351278Sdim default: 31351278Sdim return false; 32351278Sdim }; 33351278Sdim 34351278Sdim auto C = dyn_cast<ConstantInt>(RMWI.getValOperand()); 35351278Sdim if(!C) 36351278Sdim return false; 37351278Sdim 38351278Sdim switch(RMWI.getOperation()) { 39351278Sdim case AtomicRMWInst::Add: 40351278Sdim case AtomicRMWInst::Sub: 41351278Sdim case AtomicRMWInst::Or: 42351278Sdim case AtomicRMWInst::Xor: 43351278Sdim return C->isZero(); 44351278Sdim case AtomicRMWInst::And: 45351278Sdim return C->isMinusOne(); 46351278Sdim case AtomicRMWInst::Min: 47351278Sdim return C->isMaxValue(true); 48351278Sdim case AtomicRMWInst::Max: 49351278Sdim return C->isMinValue(true); 50351278Sdim case AtomicRMWInst::UMin: 51351278Sdim return C->isMaxValue(false); 52351278Sdim case AtomicRMWInst::UMax: 53351278Sdim return C->isMinValue(false); 54351278Sdim default: 55351278Sdim return false; 56351278Sdim } 57351278Sdim} 58351278Sdim 59351278Sdim/// Return true if the given instruction always produces a value in memory 60351278Sdim/// equivalent to its value operand. 61351278Sdimbool isSaturating(AtomicRMWInst& RMWI) { 62351278Sdim if (auto CF = dyn_cast<ConstantFP>(RMWI.getValOperand())) 63351278Sdim switch(RMWI.getOperation()) { 64351278Sdim case AtomicRMWInst::FAdd: 65351278Sdim case AtomicRMWInst::FSub: 66351278Sdim return CF->isNaN(); 67351278Sdim default: 68351278Sdim return false; 69351278Sdim }; 70351278Sdim 71351278Sdim auto C = dyn_cast<ConstantInt>(RMWI.getValOperand()); 72351278Sdim if(!C) 73351278Sdim return false; 74351278Sdim 75351278Sdim switch(RMWI.getOperation()) { 76351278Sdim default: 77351278Sdim return false; 78351278Sdim case AtomicRMWInst::Xchg: 79351278Sdim return true; 80351278Sdim case AtomicRMWInst::Or: 81351278Sdim return C->isAllOnesValue(); 82351278Sdim case AtomicRMWInst::And: 83351278Sdim return C->isZero(); 84351278Sdim case AtomicRMWInst::Min: 85351278Sdim return C->isMinValue(true); 86351278Sdim case AtomicRMWInst::Max: 87351278Sdim return C->isMaxValue(true); 88351278Sdim case AtomicRMWInst::UMin: 89351278Sdim return C->isMinValue(false); 90351278Sdim case AtomicRMWInst::UMax: 91351278Sdim return C->isMaxValue(false); 92351278Sdim }; 93351278Sdim} 94351278Sdim} 95351278Sdim 96351278SdimInstruction *InstCombiner::visitAtomicRMWInst(AtomicRMWInst &RMWI) { 97351278Sdim 98351278Sdim // Volatile RMWs perform a load and a store, we cannot replace this by just a 99351278Sdim // load or just a store. We chose not to canonicalize out of general paranoia 100351278Sdim // about user expectations around volatile. 101351278Sdim if (RMWI.isVolatile()) 102351278Sdim return nullptr; 103351278Sdim 104351278Sdim // Any atomicrmw op which produces a known result in memory can be 105351278Sdim // replaced w/an atomicrmw xchg. 106351278Sdim if (isSaturating(RMWI) && 107351278Sdim RMWI.getOperation() != AtomicRMWInst::Xchg) { 108351278Sdim RMWI.setOperation(AtomicRMWInst::Xchg); 109351278Sdim return &RMWI; 110351278Sdim } 111351278Sdim 112351278Sdim AtomicOrdering Ordering = RMWI.getOrdering(); 113351278Sdim assert(Ordering != AtomicOrdering::NotAtomic && 114351278Sdim Ordering != AtomicOrdering::Unordered && 115351278Sdim "AtomicRMWs don't make sense with Unordered or NotAtomic"); 116351278Sdim 117351278Sdim // Any atomicrmw xchg with no uses can be converted to a atomic store if the 118351278Sdim // ordering is compatible. 119351278Sdim if (RMWI.getOperation() == AtomicRMWInst::Xchg && 120351278Sdim RMWI.use_empty()) { 121351278Sdim if (Ordering != AtomicOrdering::Release && 122351278Sdim Ordering != AtomicOrdering::Monotonic) 123351278Sdim return nullptr; 124351278Sdim auto *SI = new StoreInst(RMWI.getValOperand(), 125351278Sdim RMWI.getPointerOperand(), &RMWI); 126351278Sdim SI->setAtomic(Ordering, RMWI.getSyncScopeID()); 127360784Sdim SI->setAlignment(MaybeAlign(DL.getABITypeAlignment(RMWI.getType()))); 128351278Sdim return eraseInstFromFunction(RMWI); 129351278Sdim } 130351278Sdim 131351278Sdim if (!isIdempotentRMW(RMWI)) 132351278Sdim return nullptr; 133351278Sdim 134351278Sdim // We chose to canonicalize all idempotent operations to an single 135351278Sdim // operation code and constant. This makes it easier for the rest of the 136351278Sdim // optimizer to match easily. The choices of or w/0 and fadd w/-0.0 are 137351278Sdim // arbitrary. 138351278Sdim if (RMWI.getType()->isIntegerTy() && 139351278Sdim RMWI.getOperation() != AtomicRMWInst::Or) { 140351278Sdim RMWI.setOperation(AtomicRMWInst::Or); 141351278Sdim RMWI.setOperand(1, ConstantInt::get(RMWI.getType(), 0)); 142351278Sdim return &RMWI; 143351278Sdim } else if (RMWI.getType()->isFloatingPointTy() && 144351278Sdim RMWI.getOperation() != AtomicRMWInst::FAdd) { 145351278Sdim RMWI.setOperation(AtomicRMWInst::FAdd); 146351278Sdim RMWI.setOperand(1, ConstantFP::getNegativeZero(RMWI.getType())); 147351278Sdim return &RMWI; 148351278Sdim } 149351278Sdim 150351278Sdim // Check if the required ordering is compatible with an atomic load. 151351278Sdim if (Ordering != AtomicOrdering::Acquire && 152351278Sdim Ordering != AtomicOrdering::Monotonic) 153351278Sdim return nullptr; 154351278Sdim 155351278Sdim LoadInst *Load = new LoadInst(RMWI.getType(), RMWI.getPointerOperand()); 156351278Sdim Load->setAtomic(Ordering, RMWI.getSyncScopeID()); 157360784Sdim Load->setAlignment(MaybeAlign(DL.getABITypeAlignment(RMWI.getType()))); 158351278Sdim return Load; 159351278Sdim} 160