1//===- ARMSLSHardening.cpp - Harden Straight Line Missspeculation ---------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file contains a pass to insert code to mitigate against side channel
10// vulnerabilities that may happen under straight line miss-speculation.
11//
12//===----------------------------------------------------------------------===//
13
14#include "ARM.h"
15#include "ARMInstrInfo.h"
16#include "ARMSubtarget.h"
17#include "llvm/CodeGen/IndirectThunks.h"
18#include "llvm/CodeGen/MachineBasicBlock.h"
19#include "llvm/CodeGen/MachineFunction.h"
20#include "llvm/CodeGen/MachineFunctionPass.h"
21#include "llvm/CodeGen/MachineInstr.h"
22#include "llvm/CodeGen/MachineInstrBuilder.h"
23#include "llvm/CodeGen/MachineOperand.h"
24#include "llvm/IR/DebugLoc.h"
25#include <cassert>
26
27using namespace llvm;
28
29#define DEBUG_TYPE "arm-sls-hardening"
30
31#define ARM_SLS_HARDENING_NAME "ARM sls hardening pass"
32
33namespace {
34
35class ARMSLSHardening : public MachineFunctionPass {
36public:
37  const TargetInstrInfo *TII;
38  const ARMSubtarget *ST;
39
40  static char ID;
41
42  ARMSLSHardening() : MachineFunctionPass(ID) {
43    initializeARMSLSHardeningPass(*PassRegistry::getPassRegistry());
44  }
45
46  bool runOnMachineFunction(MachineFunction &Fn) override;
47
48  StringRef getPassName() const override { return ARM_SLS_HARDENING_NAME; }
49
50  void getAnalysisUsage(AnalysisUsage &AU) const override {
51    AU.setPreservesCFG();
52    MachineFunctionPass::getAnalysisUsage(AU);
53  }
54
55private:
56  bool hardenReturnsAndBRs(MachineBasicBlock &MBB) const;
57  bool hardenIndirectCalls(MachineBasicBlock &MBB) const;
58  MachineBasicBlock &
59  ConvertIndirectCallToIndirectJump(MachineBasicBlock &MBB,
60                                    MachineBasicBlock::iterator) const;
61};
62
63} // end anonymous namespace
64
65char ARMSLSHardening::ID = 0;
66
67INITIALIZE_PASS(ARMSLSHardening, "arm-sls-hardening",
68                ARM_SLS_HARDENING_NAME, false, false)
69
70static void insertSpeculationBarrier(const ARMSubtarget *ST,
71                                     MachineBasicBlock &MBB,
72                                     MachineBasicBlock::iterator MBBI,
73                                     DebugLoc DL,
74                                     bool AlwaysUseISBDSB = false) {
75  assert(MBBI != MBB.begin() &&
76         "Must not insert SpeculationBarrierEndBB as only instruction in MBB.");
77  assert(std::prev(MBBI)->isBarrier() &&
78         "SpeculationBarrierEndBB must only follow unconditional control flow "
79         "instructions.");
80  assert(std::prev(MBBI)->isTerminator() &&
81         "SpeculationBarrierEndBB must only follow terminators.");
82  const TargetInstrInfo *TII = ST->getInstrInfo();
83  assert(ST->hasDataBarrier() || ST->hasSB());
84  bool ProduceSB = ST->hasSB() && !AlwaysUseISBDSB;
85  unsigned BarrierOpc =
86      ProduceSB ? (ST->isThumb() ? ARM::t2SpeculationBarrierSBEndBB
87                                 : ARM::SpeculationBarrierSBEndBB)
88                : (ST->isThumb() ? ARM::t2SpeculationBarrierISBDSBEndBB
89                                 : ARM::SpeculationBarrierISBDSBEndBB);
90  if (MBBI == MBB.end() || !isSpeculationBarrierEndBBOpcode(MBBI->getOpcode()))
91    BuildMI(MBB, MBBI, DL, TII->get(BarrierOpc));
92}
93
94bool ARMSLSHardening::runOnMachineFunction(MachineFunction &MF) {
95  ST = &MF.getSubtarget<ARMSubtarget>();
96  TII = MF.getSubtarget().getInstrInfo();
97
98  bool Modified = false;
99  for (auto &MBB : MF) {
100    Modified |= hardenReturnsAndBRs(MBB);
101    Modified |= hardenIndirectCalls(MBB);
102  }
103
104  return Modified;
105}
106
107bool ARMSLSHardening::hardenReturnsAndBRs(MachineBasicBlock &MBB) const {
108  if (!ST->hardenSlsRetBr())
109    return false;
110  assert(!ST->isThumb1Only());
111  bool Modified = false;
112  MachineBasicBlock::iterator MBBI = MBB.getFirstTerminator(), E = MBB.end();
113  MachineBasicBlock::iterator NextMBBI;
114  for (; MBBI != E; MBBI = NextMBBI) {
115    MachineInstr &MI = *MBBI;
116    NextMBBI = std::next(MBBI);
117    if (isIndirectControlFlowNotComingBack(MI)) {
118      assert(MI.isTerminator());
119      assert(!TII->isPredicated(MI));
120      insertSpeculationBarrier(ST, MBB, std::next(MBBI), MI.getDebugLoc());
121      Modified = true;
122    }
123  }
124  return Modified;
125}
126
127static const char SLSBLRNamePrefix[] = "__llvm_slsblr_thunk_";
128
129static const struct ThunkNameRegMode {
130  const char* Name;
131  Register Reg;
132  bool isThumb;
133} SLSBLRThunks[] = {
134    {"__llvm_slsblr_thunk_arm_r0", ARM::R0, false},
135    {"__llvm_slsblr_thunk_arm_r1", ARM::R1, false},
136    {"__llvm_slsblr_thunk_arm_r2", ARM::R2, false},
137    {"__llvm_slsblr_thunk_arm_r3", ARM::R3, false},
138    {"__llvm_slsblr_thunk_arm_r4", ARM::R4, false},
139    {"__llvm_slsblr_thunk_arm_r5", ARM::R5, false},
140    {"__llvm_slsblr_thunk_arm_r6", ARM::R6, false},
141    {"__llvm_slsblr_thunk_arm_r7", ARM::R7, false},
142    {"__llvm_slsblr_thunk_arm_r8", ARM::R8, false},
143    {"__llvm_slsblr_thunk_arm_r9", ARM::R9, false},
144    {"__llvm_slsblr_thunk_arm_r10", ARM::R10, false},
145    {"__llvm_slsblr_thunk_arm_r11", ARM::R11, false},
146    {"__llvm_slsblr_thunk_arm_sp", ARM::SP, false},
147    {"__llvm_slsblr_thunk_arm_pc", ARM::PC, false},
148    {"__llvm_slsblr_thunk_thumb_r0", ARM::R0, true},
149    {"__llvm_slsblr_thunk_thumb_r1", ARM::R1, true},
150    {"__llvm_slsblr_thunk_thumb_r2", ARM::R2, true},
151    {"__llvm_slsblr_thunk_thumb_r3", ARM::R3, true},
152    {"__llvm_slsblr_thunk_thumb_r4", ARM::R4, true},
153    {"__llvm_slsblr_thunk_thumb_r5", ARM::R5, true},
154    {"__llvm_slsblr_thunk_thumb_r6", ARM::R6, true},
155    {"__llvm_slsblr_thunk_thumb_r7", ARM::R7, true},
156    {"__llvm_slsblr_thunk_thumb_r8", ARM::R8, true},
157    {"__llvm_slsblr_thunk_thumb_r9", ARM::R9, true},
158    {"__llvm_slsblr_thunk_thumb_r10", ARM::R10, true},
159    {"__llvm_slsblr_thunk_thumb_r11", ARM::R11, true},
160    {"__llvm_slsblr_thunk_thumb_sp", ARM::SP, true},
161    {"__llvm_slsblr_thunk_thumb_pc", ARM::PC, true},
162};
163
164// An enum for tracking whether Arm and Thumb thunks have been inserted into the
165// current module so far.
166enum ArmInsertedThunks { ArmThunk = 1, ThumbThunk = 2 };
167
168inline ArmInsertedThunks &operator|=(ArmInsertedThunks &X,
169                                     ArmInsertedThunks Y) {
170  return X = static_cast<ArmInsertedThunks>(X | Y);
171}
172
173namespace {
174struct SLSBLRThunkInserter
175    : ThunkInserter<SLSBLRThunkInserter, ArmInsertedThunks> {
176  const char *getThunkPrefix() { return SLSBLRNamePrefix; }
177  bool mayUseThunk(const MachineFunction &MF,
178                   ArmInsertedThunks InsertedThunks) {
179    if ((InsertedThunks & ArmThunk &&
180         !MF.getSubtarget<ARMSubtarget>().isThumb()) ||
181        (InsertedThunks & ThumbThunk &&
182         MF.getSubtarget<ARMSubtarget>().isThumb()))
183      return false;
184    ComdatThunks &= !MF.getSubtarget<ARMSubtarget>().hardenSlsNoComdat();
185    // FIXME: This could also check if there are any indirect calls in the
186    // function to more accurately reflect if a thunk will be needed.
187    return MF.getSubtarget<ARMSubtarget>().hardenSlsBlr();
188  }
189  ArmInsertedThunks insertThunks(MachineModuleInfo &MMI, MachineFunction &MF);
190  void populateThunk(MachineFunction &MF);
191
192private:
193  bool ComdatThunks = true;
194};
195} // namespace
196
197ArmInsertedThunks SLSBLRThunkInserter::insertThunks(MachineModuleInfo &MMI,
198                                                    MachineFunction &MF) {
199  // FIXME: It probably would be possible to filter which thunks to produce
200  // based on which registers are actually used in indirect calls in this
201  // function. But would that be a worthwhile optimization?
202  const ARMSubtarget *ST = &MF.getSubtarget<ARMSubtarget>();
203  for (auto T : SLSBLRThunks)
204    if (ST->isThumb() == T.isThumb)
205      createThunkFunction(MMI, T.Name, ComdatThunks,
206                          T.isThumb ? "+thumb-mode" : "");
207  return ST->isThumb() ? ThumbThunk : ArmThunk;
208}
209
210void SLSBLRThunkInserter::populateThunk(MachineFunction &MF) {
211  // FIXME: How to better communicate Register number, rather than through
212  // name and lookup table?
213  assert(MF.getName().starts_with(getThunkPrefix()));
214  auto ThunkIt = llvm::find_if(
215      SLSBLRThunks, [&MF](auto T) { return T.Name == MF.getName(); });
216  assert(ThunkIt != std::end(SLSBLRThunks));
217  Register ThunkReg = ThunkIt->Reg;
218  bool isThumb = ThunkIt->isThumb;
219
220  const TargetInstrInfo *TII = MF.getSubtarget<ARMSubtarget>().getInstrInfo();
221  MachineBasicBlock *Entry = &MF.front();
222  Entry->clear();
223
224  //  These thunks need to consist of the following instructions:
225  //  __llvm_slsblr_thunk_(arm/thumb)_rN:
226  //      bx  rN
227  //      barrierInsts
228  Entry->addLiveIn(ThunkReg);
229  if (isThumb)
230    BuildMI(Entry, DebugLoc(), TII->get(ARM::tBX))
231        .addReg(ThunkReg)
232        .add(predOps(ARMCC::AL));
233  else
234    BuildMI(Entry, DebugLoc(), TII->get(ARM::BX))
235        .addReg(ThunkReg);
236
237  // Make sure the thunks do not make use of the SB extension in case there is
238  // a function somewhere that will call to it that for some reason disabled
239  // the SB extension locally on that function, even though it's enabled for
240  // the module otherwise. Therefore set AlwaysUseISBSDB to true.
241  insertSpeculationBarrier(&MF.getSubtarget<ARMSubtarget>(), *Entry,
242                           Entry->end(), DebugLoc(), true /*AlwaysUseISBDSB*/);
243}
244
245MachineBasicBlock &ARMSLSHardening::ConvertIndirectCallToIndirectJump(
246    MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI) const {
247  // Transform an indirect call to an indirect jump as follows:
248  // Before:
249  //   |-----------------------------|
250  //   |      ...                    |
251  //   |  instI                      |
252  //   |  BLX rN                     |
253  //   |  instJ                      |
254  //   |      ...                    |
255  //   |-----------------------------|
256  //
257  // After:
258  //   |----------   -------------------------|
259  //   |      ...                             |
260  //   |  instI                               |
261  //   |  *call* __llvm_slsblr_thunk_mode_xN  |
262  //   |  instJ                               |
263  //   |      ...                             |
264  //   |--------------------------------------|
265  //
266  //   __llvm_slsblr_thunk_mode_xN:
267  //   |-----------------------------|
268  //   |  BX rN                      |
269  //   |  barrierInsts               |
270  //   |-----------------------------|
271  //
272  // The __llvm_slsblr_thunk_mode_xN thunks are created by the
273  // SLSBLRThunkInserter.
274  // This function merely needs to transform an indirect call to a direct call
275  // to __llvm_slsblr_thunk_xN.
276  MachineInstr &IndirectCall = *MBBI;
277  assert(isIndirectCall(IndirectCall) && !IndirectCall.isReturn());
278  int RegOpIdxOnIndirectCall = -1;
279  bool isThumb;
280  switch (IndirectCall.getOpcode()) {
281  case ARM::BLX:   // !isThumb2
282  case ARM::BLX_noip:   // !isThumb2
283    isThumb = false;
284    RegOpIdxOnIndirectCall = 0;
285    break;
286  case ARM::tBLXr:      // isThumb2
287  case ARM::tBLXr_noip: // isThumb2
288    isThumb = true;
289    RegOpIdxOnIndirectCall = 2;
290    break;
291  default:
292    llvm_unreachable("unhandled Indirect Call");
293  }
294
295  Register Reg = IndirectCall.getOperand(RegOpIdxOnIndirectCall).getReg();
296  // Since linkers are allowed to clobber R12 on function calls, the above
297  // mitigation only works if the original indirect call instruction was not
298  // using R12. Code generation before must make sure that no indirect call
299  // using R12 was produced if the mitigation is enabled.
300  // Also, the transformation is incorrect if the indirect call uses LR, so
301  // also have to avoid that.
302  assert(Reg != ARM::R12 && Reg != ARM::LR);
303  bool RegIsKilled = IndirectCall.getOperand(RegOpIdxOnIndirectCall).isKill();
304
305  DebugLoc DL = IndirectCall.getDebugLoc();
306
307  MachineFunction &MF = *MBBI->getMF();
308  auto ThunkIt = llvm::find_if(SLSBLRThunks, [Reg, isThumb](auto T) {
309    return T.Reg == Reg && T.isThumb == isThumb;
310  });
311  assert(ThunkIt != std::end(SLSBLRThunks));
312  Module *M = MF.getFunction().getParent();
313  const GlobalValue *GV = cast<GlobalValue>(M->getNamedValue(ThunkIt->Name));
314
315  MachineInstr *BL =
316      isThumb ? BuildMI(MBB, MBBI, DL, TII->get(ARM::tBL))
317                    .addImm(IndirectCall.getOperand(0).getImm())
318                    .addReg(IndirectCall.getOperand(1).getReg())
319                    .addGlobalAddress(GV)
320              : BuildMI(MBB, MBBI, DL, TII->get(ARM::BL)).addGlobalAddress(GV);
321
322  // Now copy the implicit operands from IndirectCall to BL and copy other
323  // necessary info.
324  // However, both IndirectCall and BL instructions implictly use SP and
325  // implicitly define LR. Blindly copying implicit operands would result in SP
326  // and LR operands to be present multiple times. While this may not be too
327  // much of an issue, let's avoid that for cleanliness, by removing those
328  // implicit operands from the BL created above before we copy over all
329  // implicit operands from the IndirectCall.
330  int ImpLROpIdx = -1;
331  int ImpSPOpIdx = -1;
332  for (unsigned OpIdx = BL->getNumExplicitOperands();
333       OpIdx < BL->getNumOperands(); OpIdx++) {
334    MachineOperand Op = BL->getOperand(OpIdx);
335    if (!Op.isReg())
336      continue;
337    if (Op.getReg() == ARM::LR && Op.isDef())
338      ImpLROpIdx = OpIdx;
339    if (Op.getReg() == ARM::SP && !Op.isDef())
340      ImpSPOpIdx = OpIdx;
341  }
342  assert(ImpLROpIdx != -1);
343  assert(ImpSPOpIdx != -1);
344  int FirstOpIdxToRemove = std::max(ImpLROpIdx, ImpSPOpIdx);
345  int SecondOpIdxToRemove = std::min(ImpLROpIdx, ImpSPOpIdx);
346  BL->removeOperand(FirstOpIdxToRemove);
347  BL->removeOperand(SecondOpIdxToRemove);
348  // Now copy over the implicit operands from the original IndirectCall
349  BL->copyImplicitOps(MF, IndirectCall);
350  MF.moveCallSiteInfo(&IndirectCall, BL);
351  // Also add the register called in the IndirectCall as being used in the
352  // called thunk.
353  BL->addOperand(MachineOperand::CreateReg(Reg, false /*isDef*/, true /*isImp*/,
354                                           RegIsKilled /*isKill*/));
355  // Remove IndirectCallinstruction
356  MBB.erase(MBBI);
357  return MBB;
358}
359
360bool ARMSLSHardening::hardenIndirectCalls(MachineBasicBlock &MBB) const {
361  if (!ST->hardenSlsBlr())
362    return false;
363  bool Modified = false;
364  MachineBasicBlock::iterator MBBI = MBB.begin(), E = MBB.end();
365  MachineBasicBlock::iterator NextMBBI;
366  for (; MBBI != E; MBBI = NextMBBI) {
367    MachineInstr &MI = *MBBI;
368    NextMBBI = std::next(MBBI);
369    // Tail calls are both indirect calls and "returns".
370    // They are also indirect jumps, so should be handled by sls-harden-retbr,
371    // rather than sls-harden-blr.
372    if (isIndirectCall(MI) && !MI.isReturn()) {
373      ConvertIndirectCallToIndirectJump(MBB, MBBI);
374      Modified = true;
375    }
376  }
377  return Modified;
378}
379
380
381
382FunctionPass *llvm::createARMSLSHardeningPass() {
383  return new ARMSLSHardening();
384}
385
386namespace {
387class ARMIndirectThunks : public MachineFunctionPass {
388public:
389  static char ID;
390
391  ARMIndirectThunks() : MachineFunctionPass(ID) {}
392
393  StringRef getPassName() const override { return "ARM Indirect Thunks"; }
394
395  bool doInitialization(Module &M) override;
396  bool runOnMachineFunction(MachineFunction &MF) override;
397
398  void getAnalysisUsage(AnalysisUsage &AU) const override {
399    MachineFunctionPass::getAnalysisUsage(AU);
400    AU.addRequired<MachineModuleInfoWrapperPass>();
401    AU.addPreserved<MachineModuleInfoWrapperPass>();
402  }
403
404private:
405  std::tuple<SLSBLRThunkInserter> TIs;
406
407  // FIXME: When LLVM moves to C++17, these can become folds
408  template <typename... ThunkInserterT>
409  static void initTIs(Module &M,
410                      std::tuple<ThunkInserterT...> &ThunkInserters) {
411    (void)std::initializer_list<int>{
412        (std::get<ThunkInserterT>(ThunkInserters).init(M), 0)...};
413  }
414  template <typename... ThunkInserterT>
415  static bool runTIs(MachineModuleInfo &MMI, MachineFunction &MF,
416                     std::tuple<ThunkInserterT...> &ThunkInserters) {
417    bool Modified = false;
418    (void)std::initializer_list<int>{
419        Modified |= std::get<ThunkInserterT>(ThunkInserters).run(MMI, MF)...};
420    return Modified;
421  }
422};
423
424} // end anonymous namespace
425
426char ARMIndirectThunks::ID = 0;
427
428FunctionPass *llvm::createARMIndirectThunks() {
429  return new ARMIndirectThunks();
430}
431
432bool ARMIndirectThunks::doInitialization(Module &M) {
433  initTIs(M, TIs);
434  return false;
435}
436
437bool ARMIndirectThunks::runOnMachineFunction(MachineFunction &MF) {
438  LLVM_DEBUG(dbgs() << getPassName() << '\n');
439  auto &MMI = getAnalysis<MachineModuleInfoWrapperPass>().getMMI();
440  return runTIs(MMI, MF, TIs);
441}
442