1//===- InstCombineAtomicRMW.cpp -------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the visit functions for atomic rmw instructions.
10//
11//===----------------------------------------------------------------------===//
12
13#include "InstCombineInternal.h"
14#include "llvm/IR/Instructions.h"
15
16using namespace llvm;
17
18namespace {
19/// Return true if and only if the given instruction does not modify the memory
20/// location referenced.  Note that an idemptent atomicrmw may still have
21/// ordering effects on nearby instructions, or be volatile.
22/// TODO: Common w/ the version in AtomicExpandPass, and change the term used.
23/// Idemptotent is confusing in this context.
24bool isIdempotentRMW(AtomicRMWInst& RMWI) {
25  if (auto CF = dyn_cast<ConstantFP>(RMWI.getValOperand()))
26    switch(RMWI.getOperation()) {
27    case AtomicRMWInst::FAdd: // -0.0
28      return CF->isZero() && CF->isNegative();
29    case AtomicRMWInst::FSub: // +0.0
30      return CF->isZero() && !CF->isNegative();
31    default:
32      return false;
33    };
34
35  auto C = dyn_cast<ConstantInt>(RMWI.getValOperand());
36  if(!C)
37    return false;
38
39  switch(RMWI.getOperation()) {
40    case AtomicRMWInst::Add:
41    case AtomicRMWInst::Sub:
42    case AtomicRMWInst::Or:
43    case AtomicRMWInst::Xor:
44      return C->isZero();
45    case AtomicRMWInst::And:
46      return C->isMinusOne();
47    case AtomicRMWInst::Min:
48      return C->isMaxValue(true);
49    case AtomicRMWInst::Max:
50      return C->isMinValue(true);
51    case AtomicRMWInst::UMin:
52      return C->isMaxValue(false);
53    case AtomicRMWInst::UMax:
54      return C->isMinValue(false);
55    default:
56      return false;
57  }
58}
59
60/// Return true if the given instruction always produces a value in memory
61/// equivalent to its value operand.
62bool isSaturating(AtomicRMWInst& RMWI) {
63  if (auto CF = dyn_cast<ConstantFP>(RMWI.getValOperand()))
64    switch (RMWI.getOperation()) {
65    case AtomicRMWInst::FMax:
66      // maxnum(x, +inf) -> +inf
67      return !CF->isNegative() && CF->isInfinity();
68    case AtomicRMWInst::FMin:
69      // minnum(x, -inf) -> +inf
70      return CF->isNegative() && CF->isInfinity();
71    case AtomicRMWInst::FAdd:
72    case AtomicRMWInst::FSub:
73      return CF->isNaN();
74    default:
75      return false;
76    };
77
78  auto C = dyn_cast<ConstantInt>(RMWI.getValOperand());
79  if(!C)
80    return false;
81
82  switch(RMWI.getOperation()) {
83  default:
84    return false;
85  case AtomicRMWInst::Xchg:
86    return true;
87  case AtomicRMWInst::Or:
88    return C->isAllOnesValue();
89  case AtomicRMWInst::And:
90    return C->isZero();
91  case AtomicRMWInst::Min:
92    return C->isMinValue(true);
93  case AtomicRMWInst::Max:
94    return C->isMaxValue(true);
95  case AtomicRMWInst::UMin:
96    return C->isMinValue(false);
97  case AtomicRMWInst::UMax:
98    return C->isMaxValue(false);
99  };
100}
101} // namespace
102
103Instruction *InstCombinerImpl::visitAtomicRMWInst(AtomicRMWInst &RMWI) {
104
105  // Volatile RMWs perform a load and a store, we cannot replace this by just a
106  // load or just a store. We chose not to canonicalize out of general paranoia
107  // about user expectations around volatile.
108  if (RMWI.isVolatile())
109    return nullptr;
110
111  // Any atomicrmw op which produces a known result in memory can be
112  // replaced w/an atomicrmw xchg.
113  if (isSaturating(RMWI) &&
114      RMWI.getOperation() != AtomicRMWInst::Xchg) {
115    RMWI.setOperation(AtomicRMWInst::Xchg);
116    return &RMWI;
117  }
118
119  AtomicOrdering Ordering = RMWI.getOrdering();
120  assert(Ordering != AtomicOrdering::NotAtomic &&
121         Ordering != AtomicOrdering::Unordered &&
122         "AtomicRMWs don't make sense with Unordered or NotAtomic");
123
124  // Any atomicrmw xchg with no uses can be converted to a atomic store if the
125  // ordering is compatible.
126  if (RMWI.getOperation() == AtomicRMWInst::Xchg &&
127      RMWI.use_empty()) {
128    if (Ordering != AtomicOrdering::Release &&
129        Ordering != AtomicOrdering::Monotonic)
130      return nullptr;
131    new StoreInst(RMWI.getValOperand(), RMWI.getPointerOperand(),
132                  /*isVolatile*/ false, RMWI.getAlign(), Ordering,
133                  RMWI.getSyncScopeID(), &RMWI);
134    return eraseInstFromFunction(RMWI);
135  }
136
137  if (!isIdempotentRMW(RMWI))
138    return nullptr;
139
140  // We chose to canonicalize all idempotent operations to an single
141  // operation code and constant.  This makes it easier for the rest of the
142  // optimizer to match easily.  The choices of or w/0 and fadd w/-0.0 are
143  // arbitrary.
144  if (RMWI.getType()->isIntegerTy() &&
145      RMWI.getOperation() != AtomicRMWInst::Or) {
146    RMWI.setOperation(AtomicRMWInst::Or);
147    return replaceOperand(RMWI, 1, ConstantInt::get(RMWI.getType(), 0));
148  } else if (RMWI.getType()->isFloatingPointTy() &&
149             RMWI.getOperation() != AtomicRMWInst::FAdd) {
150    RMWI.setOperation(AtomicRMWInst::FAdd);
151    return replaceOperand(RMWI, 1, ConstantFP::getNegativeZero(RMWI.getType()));
152  }
153
154  return nullptr;
155}
156