1//===- X86VZeroUpper.cpp - AVX vzeroupper instruction inserter ------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file defines the pass which inserts x86 AVX vzeroupper instructions
10// before calls to SSE encoded functions. This avoids transition latency
11// penalty when transferring control between AVX encoded instructions and old
12// SSE encoding mode.
13//
14//===----------------------------------------------------------------------===//
15
16#include "X86.h"
17#include "X86InstrInfo.h"
18#include "X86Subtarget.h"
19#include "llvm/ADT/SmallVector.h"
20#include "llvm/ADT/Statistic.h"
21#include "llvm/CodeGen/MachineBasicBlock.h"
22#include "llvm/CodeGen/MachineFunction.h"
23#include "llvm/CodeGen/MachineFunctionPass.h"
24#include "llvm/CodeGen/MachineInstr.h"
25#include "llvm/CodeGen/MachineInstrBuilder.h"
26#include "llvm/CodeGen/MachineOperand.h"
27#include "llvm/CodeGen/MachineRegisterInfo.h"
28#include "llvm/CodeGen/TargetInstrInfo.h"
29#include "llvm/CodeGen/TargetRegisterInfo.h"
30#include "llvm/IR/CallingConv.h"
31#include "llvm/IR/DebugLoc.h"
32#include "llvm/IR/Function.h"
33#include "llvm/Support/Debug.h"
34#include "llvm/Support/ErrorHandling.h"
35#include "llvm/Support/raw_ostream.h"
36#include <cassert>
37
38using namespace llvm;
39
40#define DEBUG_TYPE "x86-vzeroupper"
41
42static cl::opt<bool>
43UseVZeroUpper("x86-use-vzeroupper", cl::Hidden,
44  cl::desc("Minimize AVX to SSE transition penalty"),
45  cl::init(true));
46
47STATISTIC(NumVZU, "Number of vzeroupper instructions inserted");
48
49namespace {
50
51  class VZeroUpperInserter : public MachineFunctionPass {
52  public:
53    VZeroUpperInserter() : MachineFunctionPass(ID) {}
54
55    bool runOnMachineFunction(MachineFunction &MF) override;
56
57    MachineFunctionProperties getRequiredProperties() const override {
58      return MachineFunctionProperties().set(
59          MachineFunctionProperties::Property::NoVRegs);
60    }
61
62    StringRef getPassName() const override { return "X86 vzeroupper inserter"; }
63
64  private:
65    void processBasicBlock(MachineBasicBlock &MBB);
66    void insertVZeroUpper(MachineBasicBlock::iterator I,
67                          MachineBasicBlock &MBB);
68    void addDirtySuccessor(MachineBasicBlock &MBB);
69
70    using BlockExitState = enum { PASS_THROUGH, EXITS_CLEAN, EXITS_DIRTY };
71
72    static const char* getBlockExitStateName(BlockExitState ST);
73
74    // Core algorithm state:
75    // BlockState - Each block is either:
76    //   - PASS_THROUGH: There are neither YMM/ZMM dirtying instructions nor
77    //                   vzeroupper instructions in this block.
78    //   - EXITS_CLEAN: There is (or will be) a vzeroupper instruction in this
79    //                  block that will ensure that YMM/ZMM is clean on exit.
80    //   - EXITS_DIRTY: An instruction in the block dirties YMM/ZMM and no
81    //                  subsequent vzeroupper in the block clears it.
82    //
83    // AddedToDirtySuccessors - This flag is raised when a block is added to the
84    //                          DirtySuccessors list to ensure that it's not
85    //                          added multiple times.
86    //
87    // FirstUnguardedCall - Records the location of the first unguarded call in
88    //                      each basic block that may need to be guarded by a
89    //                      vzeroupper. We won't know whether it actually needs
90    //                      to be guarded until we discover a predecessor that
91    //                      is DIRTY_OUT.
92    struct BlockState {
93      BlockExitState ExitState = PASS_THROUGH;
94      bool AddedToDirtySuccessors = false;
95      MachineBasicBlock::iterator FirstUnguardedCall;
96
97      BlockState() = default;
98    };
99
100    using BlockStateMap = SmallVector<BlockState, 8>;
101    using DirtySuccessorsWorkList = SmallVector<MachineBasicBlock *, 8>;
102
103    BlockStateMap BlockStates;
104    DirtySuccessorsWorkList DirtySuccessors;
105    bool EverMadeChange;
106    bool IsX86INTR;
107    const TargetInstrInfo *TII;
108
109    static char ID;
110  };
111
112} // end anonymous namespace
113
114char VZeroUpperInserter::ID = 0;
115
116FunctionPass *llvm::createX86IssueVZeroUpperPass() {
117  return new VZeroUpperInserter();
118}
119
120#ifndef NDEBUG
121const char* VZeroUpperInserter::getBlockExitStateName(BlockExitState ST) {
122  switch (ST) {
123    case PASS_THROUGH: return "Pass-through";
124    case EXITS_DIRTY: return "Exits-dirty";
125    case EXITS_CLEAN: return "Exits-clean";
126  }
127  llvm_unreachable("Invalid block exit state.");
128}
129#endif
130
131/// VZEROUPPER cleans state that is related to Y/ZMM0-15 only.
132/// Thus, there is no need to check for Y/ZMM16 and above.
133static bool isYmmOrZmmReg(unsigned Reg) {
134  return (Reg >= X86::YMM0 && Reg <= X86::YMM15) ||
135         (Reg >= X86::ZMM0 && Reg <= X86::ZMM15);
136}
137
138static bool checkFnHasLiveInYmmOrZmm(MachineRegisterInfo &MRI) {
139  for (std::pair<unsigned, unsigned> LI : MRI.liveins())
140    if (isYmmOrZmmReg(LI.first))
141      return true;
142
143  return false;
144}
145
146static bool clobbersAllYmmAndZmmRegs(const MachineOperand &MO) {
147  for (unsigned reg = X86::YMM0; reg <= X86::YMM15; ++reg) {
148    if (!MO.clobbersPhysReg(reg))
149      return false;
150  }
151  for (unsigned reg = X86::ZMM0; reg <= X86::ZMM15; ++reg) {
152    if (!MO.clobbersPhysReg(reg))
153      return false;
154  }
155  return true;
156}
157
158static bool hasYmmOrZmmReg(MachineInstr &MI) {
159  for (const MachineOperand &MO : MI.operands()) {
160    if (MI.isCall() && MO.isRegMask() && !clobbersAllYmmAndZmmRegs(MO))
161      return true;
162    if (!MO.isReg())
163      continue;
164    if (MO.isDebug())
165      continue;
166    if (isYmmOrZmmReg(MO.getReg()))
167      return true;
168  }
169  return false;
170}
171
172/// Check if given call instruction has a RegMask operand.
173static bool callHasRegMask(MachineInstr &MI) {
174  assert(MI.isCall() && "Can only be called on call instructions.");
175  for (const MachineOperand &MO : MI.operands()) {
176    if (MO.isRegMask())
177      return true;
178  }
179  return false;
180}
181
182/// Insert a vzeroupper instruction before I.
183void VZeroUpperInserter::insertVZeroUpper(MachineBasicBlock::iterator I,
184                                          MachineBasicBlock &MBB) {
185  BuildMI(MBB, I, I->getDebugLoc(), TII->get(X86::VZEROUPPER));
186  ++NumVZU;
187  EverMadeChange = true;
188}
189
190/// Add MBB to the DirtySuccessors list if it hasn't already been added.
191void VZeroUpperInserter::addDirtySuccessor(MachineBasicBlock &MBB) {
192  if (!BlockStates[MBB.getNumber()].AddedToDirtySuccessors) {
193    DirtySuccessors.push_back(&MBB);
194    BlockStates[MBB.getNumber()].AddedToDirtySuccessors = true;
195  }
196}
197
198/// Loop over all of the instructions in the basic block, inserting vzeroupper
199/// instructions before function calls.
200void VZeroUpperInserter::processBasicBlock(MachineBasicBlock &MBB) {
201  // Start by assuming that the block is PASS_THROUGH which implies no unguarded
202  // calls.
203  BlockExitState CurState = PASS_THROUGH;
204  BlockStates[MBB.getNumber()].FirstUnguardedCall = MBB.end();
205
206  for (MachineInstr &MI : MBB) {
207    bool IsCall = MI.isCall();
208    bool IsReturn = MI.isReturn();
209    bool IsControlFlow = IsCall || IsReturn;
210
211    // No need for vzeroupper before iret in interrupt handler function,
212    // epilogue will restore YMM/ZMM registers if needed.
213    if (IsX86INTR && IsReturn)
214      continue;
215
216    // An existing VZERO* instruction resets the state.
217    if (MI.getOpcode() == X86::VZEROALL || MI.getOpcode() == X86::VZEROUPPER) {
218      CurState = EXITS_CLEAN;
219      continue;
220    }
221
222    // Shortcut: don't need to check regular instructions in dirty state.
223    if (!IsControlFlow && CurState == EXITS_DIRTY)
224      continue;
225
226    if (hasYmmOrZmmReg(MI)) {
227      // We found a ymm/zmm-using instruction; this could be an AVX/AVX512
228      // instruction, or it could be control flow.
229      CurState = EXITS_DIRTY;
230      continue;
231    }
232
233    // Check for control-flow out of the current function (which might
234    // indirectly execute SSE instructions).
235    if (!IsControlFlow)
236      continue;
237
238    // If the call has no RegMask, skip it as well. It usually happens on
239    // helper function calls (such as '_chkstk', '_ftol2') where standard
240    // calling convention is not used (RegMask is not used to mark register
241    // clobbered and register usage (def/implicit-def/use) is well-defined and
242    // explicitly specified.
243    if (IsCall && !callHasRegMask(MI))
244      continue;
245
246    // The VZEROUPPER instruction resets the upper 128 bits of YMM0-YMM15
247    // registers. In addition, the processor changes back to Clean state, after
248    // which execution of SSE instructions or AVX instructions has no transition
249    // penalty. Add the VZEROUPPER instruction before any function call/return
250    // that might execute SSE code.
251    // FIXME: In some cases, we may want to move the VZEROUPPER into a
252    // predecessor block.
253    if (CurState == EXITS_DIRTY) {
254      // After the inserted VZEROUPPER the state becomes clean again, but
255      // other YMM/ZMM may appear before other subsequent calls or even before
256      // the end of the BB.
257      insertVZeroUpper(MI, MBB);
258      CurState = EXITS_CLEAN;
259    } else if (CurState == PASS_THROUGH) {
260      // If this block is currently in pass-through state and we encounter a
261      // call then whether we need a vzeroupper or not depends on whether this
262      // block has successors that exit dirty. Record the location of the call,
263      // and set the state to EXITS_CLEAN, but do not insert the vzeroupper yet.
264      // It will be inserted later if necessary.
265      BlockStates[MBB.getNumber()].FirstUnguardedCall = MI;
266      CurState = EXITS_CLEAN;
267    }
268  }
269
270  LLVM_DEBUG(dbgs() << "MBB #" << MBB.getNumber() << " exit state: "
271                    << getBlockExitStateName(CurState) << '\n');
272
273  if (CurState == EXITS_DIRTY)
274    for (MachineBasicBlock *Succ : MBB.successors())
275      addDirtySuccessor(*Succ);
276
277  BlockStates[MBB.getNumber()].ExitState = CurState;
278}
279
280/// Loop over all of the basic blocks, inserting vzeroupper instructions before
281/// function calls.
282bool VZeroUpperInserter::runOnMachineFunction(MachineFunction &MF) {
283  if (!UseVZeroUpper)
284    return false;
285
286  const X86Subtarget &ST = MF.getSubtarget<X86Subtarget>();
287  if (!ST.hasAVX() || !ST.insertVZEROUPPER())
288    return false;
289  TII = ST.getInstrInfo();
290  MachineRegisterInfo &MRI = MF.getRegInfo();
291  EverMadeChange = false;
292  IsX86INTR = MF.getFunction().getCallingConv() == CallingConv::X86_INTR;
293
294  bool FnHasLiveInYmmOrZmm = checkFnHasLiveInYmmOrZmm(MRI);
295
296  // Fast check: if the function doesn't use any ymm/zmm registers, we don't
297  // need to insert any VZEROUPPER instructions.  This is constant-time, so it
298  // is cheap in the common case of no ymm/zmm use.
299  bool YmmOrZmmUsed = FnHasLiveInYmmOrZmm;
300  for (const auto *RC : {&X86::VR256RegClass, &X86::VR512_0_15RegClass}) {
301    if (!YmmOrZmmUsed) {
302      for (MCPhysReg R : *RC) {
303        if (!MRI.reg_nodbg_empty(R)) {
304          YmmOrZmmUsed = true;
305          break;
306        }
307      }
308    }
309  }
310  if (!YmmOrZmmUsed)
311    return false;
312
313  assert(BlockStates.empty() && DirtySuccessors.empty() &&
314         "X86VZeroUpper state should be clear");
315  BlockStates.resize(MF.getNumBlockIDs());
316
317  // Process all blocks. This will compute block exit states, record the first
318  // unguarded call in each block, and add successors of dirty blocks to the
319  // DirtySuccessors list.
320  for (MachineBasicBlock &MBB : MF)
321    processBasicBlock(MBB);
322
323  // If any YMM/ZMM regs are live-in to this function, add the entry block to
324  // the DirtySuccessors list
325  if (FnHasLiveInYmmOrZmm)
326    addDirtySuccessor(MF.front());
327
328  // Re-visit all blocks that are successors of EXITS_DIRTY blocks. Add
329  // vzeroupper instructions to unguarded calls, and propagate EXITS_DIRTY
330  // through PASS_THROUGH blocks.
331  while (!DirtySuccessors.empty()) {
332    MachineBasicBlock &MBB = *DirtySuccessors.back();
333    DirtySuccessors.pop_back();
334    BlockState &BBState = BlockStates[MBB.getNumber()];
335
336    // MBB is a successor of a dirty block, so its first call needs to be
337    // guarded.
338    if (BBState.FirstUnguardedCall != MBB.end())
339      insertVZeroUpper(BBState.FirstUnguardedCall, MBB);
340
341    // If this successor was a pass-through block, then it is now dirty. Its
342    // successors need to be added to the worklist (if they haven't been
343    // already).
344    if (BBState.ExitState == PASS_THROUGH) {
345      LLVM_DEBUG(dbgs() << "MBB #" << MBB.getNumber()
346                        << " was Pass-through, is now Dirty-out.\n");
347      for (MachineBasicBlock *Succ : MBB.successors())
348        addDirtySuccessor(*Succ);
349    }
350  }
351
352  BlockStates.clear();
353  return EverMadeChange;
354}
355