1//===- X86VZeroUpper.cpp - AVX vzeroupper instruction inserter ------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file defines the pass which inserts x86 AVX vzeroupper instructions
10// before calls to SSE encoded functions. This avoids transition latency
11// penalty when transferring control between AVX encoded instructions and old
12// SSE encoding mode.
13//
14//===----------------------------------------------------------------------===//
15
16#include "X86.h"
17#include "X86InstrInfo.h"
18#include "X86Subtarget.h"
19#include "llvm/ADT/SmallVector.h"
20#include "llvm/ADT/Statistic.h"
21#include "llvm/CodeGen/MachineBasicBlock.h"
22#include "llvm/CodeGen/MachineFunction.h"
23#include "llvm/CodeGen/MachineFunctionPass.h"
24#include "llvm/CodeGen/MachineInstr.h"
25#include "llvm/CodeGen/MachineInstrBuilder.h"
26#include "llvm/CodeGen/MachineOperand.h"
27#include "llvm/CodeGen/MachineRegisterInfo.h"
28#include "llvm/CodeGen/TargetInstrInfo.h"
29#include "llvm/CodeGen/TargetRegisterInfo.h"
30#include "llvm/IR/CallingConv.h"
31#include "llvm/IR/DebugLoc.h"
32#include "llvm/IR/Function.h"
33#include "llvm/Support/Debug.h"
34#include "llvm/Support/ErrorHandling.h"
35#include "llvm/Support/raw_ostream.h"
36#include <cassert>
37
38using namespace llvm;
39
40#define DEBUG_TYPE "x86-vzeroupper"
41
42STATISTIC(NumVZU, "Number of vzeroupper instructions inserted");
43
44namespace {
45
46  class VZeroUpperInserter : public MachineFunctionPass {
47  public:
48    VZeroUpperInserter() : MachineFunctionPass(ID) {}
49
50    bool runOnMachineFunction(MachineFunction &MF) override;
51
52    MachineFunctionProperties getRequiredProperties() const override {
53      return MachineFunctionProperties().set(
54          MachineFunctionProperties::Property::NoVRegs);
55    }
56
57    StringRef getPassName() const override { return "X86 vzeroupper inserter"; }
58
59  private:
60    void processBasicBlock(MachineBasicBlock &MBB);
61    void insertVZeroUpper(MachineBasicBlock::iterator I,
62                          MachineBasicBlock &MBB);
63    void addDirtySuccessor(MachineBasicBlock &MBB);
64
65    using BlockExitState = enum { PASS_THROUGH, EXITS_CLEAN, EXITS_DIRTY };
66
67    static const char* getBlockExitStateName(BlockExitState ST);
68
69    // Core algorithm state:
70    // BlockState - Each block is either:
71    //   - PASS_THROUGH: There are neither YMM/ZMM dirtying instructions nor
72    //                   vzeroupper instructions in this block.
73    //   - EXITS_CLEAN: There is (or will be) a vzeroupper instruction in this
74    //                  block that will ensure that YMM/ZMM is clean on exit.
75    //   - EXITS_DIRTY: An instruction in the block dirties YMM/ZMM and no
76    //                  subsequent vzeroupper in the block clears it.
77    //
78    // AddedToDirtySuccessors - This flag is raised when a block is added to the
79    //                          DirtySuccessors list to ensure that it's not
80    //                          added multiple times.
81    //
82    // FirstUnguardedCall - Records the location of the first unguarded call in
83    //                      each basic block that may need to be guarded by a
84    //                      vzeroupper. We won't know whether it actually needs
85    //                      to be guarded until we discover a predecessor that
86    //                      is DIRTY_OUT.
87    struct BlockState {
88      BlockExitState ExitState = PASS_THROUGH;
89      bool AddedToDirtySuccessors = false;
90      MachineBasicBlock::iterator FirstUnguardedCall;
91
92      BlockState() = default;
93    };
94
95    using BlockStateMap = SmallVector<BlockState, 8>;
96    using DirtySuccessorsWorkList = SmallVector<MachineBasicBlock *, 8>;
97
98    BlockStateMap BlockStates;
99    DirtySuccessorsWorkList DirtySuccessors;
100    bool EverMadeChange;
101    bool IsX86INTR;
102    const TargetInstrInfo *TII;
103
104    static char ID;
105  };
106
107} // end anonymous namespace
108
109char VZeroUpperInserter::ID = 0;
110
111FunctionPass *llvm::createX86IssueVZeroUpperPass() {
112  return new VZeroUpperInserter();
113}
114
115#ifndef NDEBUG
116const char* VZeroUpperInserter::getBlockExitStateName(BlockExitState ST) {
117  switch (ST) {
118    case PASS_THROUGH: return "Pass-through";
119    case EXITS_DIRTY: return "Exits-dirty";
120    case EXITS_CLEAN: return "Exits-clean";
121  }
122  llvm_unreachable("Invalid block exit state.");
123}
124#endif
125
126/// VZEROUPPER cleans state that is related to Y/ZMM0-15 only.
127/// Thus, there is no need to check for Y/ZMM16 and above.
128static bool isYmmOrZmmReg(unsigned Reg) {
129  return (Reg >= X86::YMM0 && Reg <= X86::YMM15) ||
130         (Reg >= X86::ZMM0 && Reg <= X86::ZMM15);
131}
132
133static bool checkFnHasLiveInYmmOrZmm(MachineRegisterInfo &MRI) {
134  for (std::pair<unsigned, unsigned> LI : MRI.liveins())
135    if (isYmmOrZmmReg(LI.first))
136      return true;
137
138  return false;
139}
140
141static bool clobbersAllYmmAndZmmRegs(const MachineOperand &MO) {
142  for (unsigned reg = X86::YMM0; reg <= X86::YMM15; ++reg) {
143    if (!MO.clobbersPhysReg(reg))
144      return false;
145  }
146  for (unsigned reg = X86::ZMM0; reg <= X86::ZMM15; ++reg) {
147    if (!MO.clobbersPhysReg(reg))
148      return false;
149  }
150  return true;
151}
152
153static bool hasYmmOrZmmReg(MachineInstr &MI) {
154  for (const MachineOperand &MO : MI.operands()) {
155    if (MI.isCall() && MO.isRegMask() && !clobbersAllYmmAndZmmRegs(MO))
156      return true;
157    if (!MO.isReg())
158      continue;
159    if (MO.isDebug())
160      continue;
161    if (isYmmOrZmmReg(MO.getReg()))
162      return true;
163  }
164  return false;
165}
166
167/// Check if given call instruction has a RegMask operand.
168static bool callHasRegMask(MachineInstr &MI) {
169  assert(MI.isCall() && "Can only be called on call instructions.");
170  for (const MachineOperand &MO : MI.operands()) {
171    if (MO.isRegMask())
172      return true;
173  }
174  return false;
175}
176
177/// Insert a vzeroupper instruction before I.
178void VZeroUpperInserter::insertVZeroUpper(MachineBasicBlock::iterator I,
179                                          MachineBasicBlock &MBB) {
180  DebugLoc dl = I->getDebugLoc();
181  BuildMI(MBB, I, dl, TII->get(X86::VZEROUPPER));
182  ++NumVZU;
183  EverMadeChange = true;
184}
185
186/// Add MBB to the DirtySuccessors list if it hasn't already been added.
187void VZeroUpperInserter::addDirtySuccessor(MachineBasicBlock &MBB) {
188  if (!BlockStates[MBB.getNumber()].AddedToDirtySuccessors) {
189    DirtySuccessors.push_back(&MBB);
190    BlockStates[MBB.getNumber()].AddedToDirtySuccessors = true;
191  }
192}
193
194/// Loop over all of the instructions in the basic block, inserting vzeroupper
195/// instructions before function calls.
196void VZeroUpperInserter::processBasicBlock(MachineBasicBlock &MBB) {
197  // Start by assuming that the block is PASS_THROUGH which implies no unguarded
198  // calls.
199  BlockExitState CurState = PASS_THROUGH;
200  BlockStates[MBB.getNumber()].FirstUnguardedCall = MBB.end();
201
202  for (MachineInstr &MI : MBB) {
203    bool IsCall = MI.isCall();
204    bool IsReturn = MI.isReturn();
205    bool IsControlFlow = IsCall || IsReturn;
206
207    // No need for vzeroupper before iret in interrupt handler function,
208    // epilogue will restore YMM/ZMM registers if needed.
209    if (IsX86INTR && IsReturn)
210      continue;
211
212    // An existing VZERO* instruction resets the state.
213    if (MI.getOpcode() == X86::VZEROALL || MI.getOpcode() == X86::VZEROUPPER) {
214      CurState = EXITS_CLEAN;
215      continue;
216    }
217
218    // Shortcut: don't need to check regular instructions in dirty state.
219    if (!IsControlFlow && CurState == EXITS_DIRTY)
220      continue;
221
222    if (hasYmmOrZmmReg(MI)) {
223      // We found a ymm/zmm-using instruction; this could be an AVX/AVX512
224      // instruction, or it could be control flow.
225      CurState = EXITS_DIRTY;
226      continue;
227    }
228
229    // Check for control-flow out of the current function (which might
230    // indirectly execute SSE instructions).
231    if (!IsControlFlow)
232      continue;
233
234    // If the call has no RegMask, skip it as well. It usually happens on
235    // helper function calls (such as '_chkstk', '_ftol2') where standard
236    // calling convention is not used (RegMask is not used to mark register
237    // clobbered and register usage (def/implicit-def/use) is well-defined and
238    // explicitly specified.
239    if (IsCall && !callHasRegMask(MI))
240      continue;
241
242    // The VZEROUPPER instruction resets the upper 128 bits of YMM0-YMM15
243    // registers. In addition, the processor changes back to Clean state, after
244    // which execution of SSE instructions or AVX instructions has no transition
245    // penalty. Add the VZEROUPPER instruction before any function call/return
246    // that might execute SSE code.
247    // FIXME: In some cases, we may want to move the VZEROUPPER into a
248    // predecessor block.
249    if (CurState == EXITS_DIRTY) {
250      // After the inserted VZEROUPPER the state becomes clean again, but
251      // other YMM/ZMM may appear before other subsequent calls or even before
252      // the end of the BB.
253      insertVZeroUpper(MI, MBB);
254      CurState = EXITS_CLEAN;
255    } else if (CurState == PASS_THROUGH) {
256      // If this block is currently in pass-through state and we encounter a
257      // call then whether we need a vzeroupper or not depends on whether this
258      // block has successors that exit dirty. Record the location of the call,
259      // and set the state to EXITS_CLEAN, but do not insert the vzeroupper yet.
260      // It will be inserted later if necessary.
261      BlockStates[MBB.getNumber()].FirstUnguardedCall = MI;
262      CurState = EXITS_CLEAN;
263    }
264  }
265
266  LLVM_DEBUG(dbgs() << "MBB #" << MBB.getNumber() << " exit state: "
267                    << getBlockExitStateName(CurState) << '\n');
268
269  if (CurState == EXITS_DIRTY)
270    for (MachineBasicBlock::succ_iterator SI = MBB.succ_begin(),
271                                          SE = MBB.succ_end();
272         SI != SE; ++SI)
273      addDirtySuccessor(**SI);
274
275  BlockStates[MBB.getNumber()].ExitState = CurState;
276}
277
278/// Loop over all of the basic blocks, inserting vzeroupper instructions before
279/// function calls.
280bool VZeroUpperInserter::runOnMachineFunction(MachineFunction &MF) {
281  const X86Subtarget &ST = MF.getSubtarget<X86Subtarget>();
282  if (!ST.hasAVX() || !ST.insertVZEROUPPER())
283    return false;
284  TII = ST.getInstrInfo();
285  MachineRegisterInfo &MRI = MF.getRegInfo();
286  EverMadeChange = false;
287  IsX86INTR = MF.getFunction().getCallingConv() == CallingConv::X86_INTR;
288
289  bool FnHasLiveInYmmOrZmm = checkFnHasLiveInYmmOrZmm(MRI);
290
291  // Fast check: if the function doesn't use any ymm/zmm registers, we don't
292  // need to insert any VZEROUPPER instructions.  This is constant-time, so it
293  // is cheap in the common case of no ymm/zmm use.
294  bool YmmOrZmmUsed = FnHasLiveInYmmOrZmm;
295  for (auto *RC : {&X86::VR256RegClass, &X86::VR512_0_15RegClass}) {
296    if (!YmmOrZmmUsed) {
297      for (TargetRegisterClass::iterator i = RC->begin(), e = RC->end(); i != e;
298           i++) {
299        if (!MRI.reg_nodbg_empty(*i)) {
300          YmmOrZmmUsed = true;
301          break;
302        }
303      }
304    }
305  }
306  if (!YmmOrZmmUsed)
307    return false;
308
309  assert(BlockStates.empty() && DirtySuccessors.empty() &&
310         "X86VZeroUpper state should be clear");
311  BlockStates.resize(MF.getNumBlockIDs());
312
313  // Process all blocks. This will compute block exit states, record the first
314  // unguarded call in each block, and add successors of dirty blocks to the
315  // DirtySuccessors list.
316  for (MachineBasicBlock &MBB : MF)
317    processBasicBlock(MBB);
318
319  // If any YMM/ZMM regs are live-in to this function, add the entry block to
320  // the DirtySuccessors list
321  if (FnHasLiveInYmmOrZmm)
322    addDirtySuccessor(MF.front());
323
324  // Re-visit all blocks that are successors of EXITS_DIRTY blocks. Add
325  // vzeroupper instructions to unguarded calls, and propagate EXITS_DIRTY
326  // through PASS_THROUGH blocks.
327  while (!DirtySuccessors.empty()) {
328    MachineBasicBlock &MBB = *DirtySuccessors.back();
329    DirtySuccessors.pop_back();
330    BlockState &BBState = BlockStates[MBB.getNumber()];
331
332    // MBB is a successor of a dirty block, so its first call needs to be
333    // guarded.
334    if (BBState.FirstUnguardedCall != MBB.end())
335      insertVZeroUpper(BBState.FirstUnguardedCall, MBB);
336
337    // If this successor was a pass-through block, then it is now dirty. Its
338    // successors need to be added to the worklist (if they haven't been
339    // already).
340    if (BBState.ExitState == PASS_THROUGH) {
341      LLVM_DEBUG(dbgs() << "MBB #" << MBB.getNumber()
342                        << " was Pass-through, is now Dirty-out.\n");
343      for (MachineBasicBlock *Succ : MBB.successors())
344        addDirtySuccessor(*Succ);
345    }
346  }
347
348  BlockStates.clear();
349  return EverMadeChange;
350}
351