1//===- AArch64StackTagging.cpp - Stack tagging in IR --===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//===----------------------------------------------------------------------===//
9
10#include "AArch64.h"
11#include "AArch64InstrInfo.h"
12#include "AArch64Subtarget.h"
13#include "AArch64TargetMachine.h"
14#include "llvm/ADT/DenseMap.h"
15#include "llvm/ADT/DepthFirstIterator.h"
16#include "llvm/ADT/MapVector.h"
17#include "llvm/ADT/SmallVector.h"
18#include "llvm/ADT/Statistic.h"
19#include "llvm/Analysis/AliasAnalysis.h"
20#include "llvm/Analysis/CFG.h"
21#include "llvm/Analysis/LoopInfo.h"
22#include "llvm/Analysis/PostDominators.h"
23#include "llvm/Analysis/ScalarEvolution.h"
24#include "llvm/Analysis/ScalarEvolutionExpressions.h"
25#include "llvm/Analysis/StackSafetyAnalysis.h"
26#include "llvm/Analysis/ValueTracking.h"
27#include "llvm/CodeGen/LiveRegUnits.h"
28#include "llvm/CodeGen/MachineBasicBlock.h"
29#include "llvm/CodeGen/MachineFunction.h"
30#include "llvm/CodeGen/MachineFunctionPass.h"
31#include "llvm/CodeGen/MachineInstr.h"
32#include "llvm/CodeGen/MachineInstrBuilder.h"
33#include "llvm/CodeGen/MachineLoopInfo.h"
34#include "llvm/CodeGen/MachineOperand.h"
35#include "llvm/CodeGen/MachineRegisterInfo.h"
36#include "llvm/CodeGen/TargetPassConfig.h"
37#include "llvm/CodeGen/TargetRegisterInfo.h"
38#include "llvm/IR/DebugLoc.h"
39#include "llvm/IR/Dominators.h"
40#include "llvm/IR/Function.h"
41#include "llvm/IR/GetElementPtrTypeIterator.h"
42#include "llvm/IR/IRBuilder.h"
43#include "llvm/IR/InstIterator.h"
44#include "llvm/IR/Instruction.h"
45#include "llvm/IR/Instructions.h"
46#include "llvm/IR/IntrinsicInst.h"
47#include "llvm/IR/IntrinsicsAArch64.h"
48#include "llvm/IR/Metadata.h"
49#include "llvm/IR/ValueHandle.h"
50#include "llvm/InitializePasses.h"
51#include "llvm/Pass.h"
52#include "llvm/Support/Casting.h"
53#include "llvm/Support/Debug.h"
54#include "llvm/Support/raw_ostream.h"
55#include "llvm/Transforms/Utils/Local.h"
56#include "llvm/Transforms/Utils/MemoryTaggingSupport.h"
57#include <cassert>
58#include <iterator>
59#include <memory>
60#include <utility>
61
62using namespace llvm;
63
64#define DEBUG_TYPE "aarch64-stack-tagging"
65
66static cl::opt<bool> ClMergeInit(
67    "stack-tagging-merge-init", cl::Hidden, cl::init(true),
68    cl::desc("merge stack variable initializers with tagging when possible"));
69
70static cl::opt<bool>
71    ClUseStackSafety("stack-tagging-use-stack-safety", cl::Hidden,
72                     cl::init(true),
73                     cl::desc("Use Stack Safety analysis results"));
74
75static cl::opt<unsigned> ClScanLimit("stack-tagging-merge-init-scan-limit",
76                                     cl::init(40), cl::Hidden);
77
78static cl::opt<unsigned>
79    ClMergeInitSizeLimit("stack-tagging-merge-init-size-limit", cl::init(272),
80                         cl::Hidden);
81
82static cl::opt<size_t> ClMaxLifetimes(
83    "stack-tagging-max-lifetimes-for-alloca", cl::Hidden, cl::init(3),
84    cl::ReallyHidden,
85    cl::desc("How many lifetime ends to handle for a single alloca."),
86    cl::Optional);
87
88static const Align kTagGranuleSize = Align(16);
89
90namespace {
91
92class InitializerBuilder {
93  uint64_t Size;
94  const DataLayout *DL;
95  Value *BasePtr;
96  Function *SetTagFn;
97  Function *SetTagZeroFn;
98  Function *StgpFn;
99
100  // List of initializers sorted by start offset.
101  struct Range {
102    uint64_t Start, End;
103    Instruction *Inst;
104  };
105  SmallVector<Range, 4> Ranges;
106  // 8-aligned offset => 8-byte initializer
107  // Missing keys are zero initialized.
108  std::map<uint64_t, Value *> Out;
109
110public:
111  InitializerBuilder(uint64_t Size, const DataLayout *DL, Value *BasePtr,
112                     Function *SetTagFn, Function *SetTagZeroFn,
113                     Function *StgpFn)
114      : Size(Size), DL(DL), BasePtr(BasePtr), SetTagFn(SetTagFn),
115        SetTagZeroFn(SetTagZeroFn), StgpFn(StgpFn) {}
116
117  bool addRange(uint64_t Start, uint64_t End, Instruction *Inst) {
118    auto I =
119        llvm::lower_bound(Ranges, Start, [](const Range &LHS, uint64_t RHS) {
120          return LHS.End <= RHS;
121        });
122    if (I != Ranges.end() && End > I->Start) {
123      // Overlap - bail.
124      return false;
125    }
126    Ranges.insert(I, {Start, End, Inst});
127    return true;
128  }
129
130  bool addStore(uint64_t Offset, StoreInst *SI, const DataLayout *DL) {
131    int64_t StoreSize = DL->getTypeStoreSize(SI->getOperand(0)->getType());
132    if (!addRange(Offset, Offset + StoreSize, SI))
133      return false;
134    IRBuilder<> IRB(SI);
135    applyStore(IRB, Offset, Offset + StoreSize, SI->getOperand(0));
136    return true;
137  }
138
139  bool addMemSet(uint64_t Offset, MemSetInst *MSI) {
140    uint64_t StoreSize = cast<ConstantInt>(MSI->getLength())->getZExtValue();
141    if (!addRange(Offset, Offset + StoreSize, MSI))
142      return false;
143    IRBuilder<> IRB(MSI);
144    applyMemSet(IRB, Offset, Offset + StoreSize,
145                cast<ConstantInt>(MSI->getValue()));
146    return true;
147  }
148
149  void applyMemSet(IRBuilder<> &IRB, int64_t Start, int64_t End,
150                   ConstantInt *V) {
151    // Out[] does not distinguish between zero and undef, and we already know
152    // that this memset does not overlap with any other initializer. Nothing to
153    // do for memset(0).
154    if (V->isZero())
155      return;
156    for (int64_t Offset = Start - Start % 8; Offset < End; Offset += 8) {
157      uint64_t Cst = 0x0101010101010101UL;
158      int LowBits = Offset < Start ? (Start - Offset) * 8 : 0;
159      if (LowBits)
160        Cst = (Cst >> LowBits) << LowBits;
161      int HighBits = End - Offset < 8 ? (8 - (End - Offset)) * 8 : 0;
162      if (HighBits)
163        Cst = (Cst << HighBits) >> HighBits;
164      ConstantInt *C =
165          ConstantInt::get(IRB.getInt64Ty(), Cst * V->getZExtValue());
166
167      Value *&CurrentV = Out[Offset];
168      if (!CurrentV) {
169        CurrentV = C;
170      } else {
171        CurrentV = IRB.CreateOr(CurrentV, C);
172      }
173    }
174  }
175
176  // Take a 64-bit slice of the value starting at the given offset (in bytes).
177  // Offset can be negative. Pad with zeroes on both sides when necessary.
178  Value *sliceValue(IRBuilder<> &IRB, Value *V, int64_t Offset) {
179    if (Offset > 0) {
180      V = IRB.CreateLShr(V, Offset * 8);
181      V = IRB.CreateZExtOrTrunc(V, IRB.getInt64Ty());
182    } else if (Offset < 0) {
183      V = IRB.CreateZExtOrTrunc(V, IRB.getInt64Ty());
184      V = IRB.CreateShl(V, -Offset * 8);
185    } else {
186      V = IRB.CreateZExtOrTrunc(V, IRB.getInt64Ty());
187    }
188    return V;
189  }
190
191  void applyStore(IRBuilder<> &IRB, int64_t Start, int64_t End,
192                  Value *StoredValue) {
193    StoredValue = flatten(IRB, StoredValue);
194    for (int64_t Offset = Start - Start % 8; Offset < End; Offset += 8) {
195      Value *V = sliceValue(IRB, StoredValue, Offset - Start);
196      Value *&CurrentV = Out[Offset];
197      if (!CurrentV) {
198        CurrentV = V;
199      } else {
200        CurrentV = IRB.CreateOr(CurrentV, V);
201      }
202    }
203  }
204
205  void generate(IRBuilder<> &IRB) {
206    LLVM_DEBUG(dbgs() << "Combined initializer\n");
207    // No initializers => the entire allocation is undef.
208    if (Ranges.empty()) {
209      emitUndef(IRB, 0, Size);
210      return;
211    }
212
213    // Look through 8-byte initializer list 16 bytes at a time;
214    // If one of the two 8-byte halfs is non-zero non-undef, emit STGP.
215    // Otherwise, emit zeroes up to next available item.
216    uint64_t LastOffset = 0;
217    for (uint64_t Offset = 0; Offset < Size; Offset += 16) {
218      auto I1 = Out.find(Offset);
219      auto I2 = Out.find(Offset + 8);
220      if (I1 == Out.end() && I2 == Out.end())
221        continue;
222
223      if (Offset > LastOffset)
224        emitZeroes(IRB, LastOffset, Offset - LastOffset);
225
226      Value *Store1 = I1 == Out.end() ? Constant::getNullValue(IRB.getInt64Ty())
227                                      : I1->second;
228      Value *Store2 = I2 == Out.end() ? Constant::getNullValue(IRB.getInt64Ty())
229                                      : I2->second;
230      emitPair(IRB, Offset, Store1, Store2);
231      LastOffset = Offset + 16;
232    }
233
234    // memset(0) does not update Out[], therefore the tail can be either undef
235    // or zero.
236    if (LastOffset < Size)
237      emitZeroes(IRB, LastOffset, Size - LastOffset);
238
239    for (const auto &R : Ranges) {
240      R.Inst->eraseFromParent();
241    }
242  }
243
244  void emitZeroes(IRBuilder<> &IRB, uint64_t Offset, uint64_t Size) {
245    LLVM_DEBUG(dbgs() << "  [" << Offset << ", " << Offset + Size
246                      << ") zero\n");
247    Value *Ptr = BasePtr;
248    if (Offset)
249      Ptr = IRB.CreateConstGEP1_32(IRB.getInt8Ty(), Ptr, Offset);
250    IRB.CreateCall(SetTagZeroFn,
251                   {Ptr, ConstantInt::get(IRB.getInt64Ty(), Size)});
252  }
253
254  void emitUndef(IRBuilder<> &IRB, uint64_t Offset, uint64_t Size) {
255    LLVM_DEBUG(dbgs() << "  [" << Offset << ", " << Offset + Size
256                      << ") undef\n");
257    Value *Ptr = BasePtr;
258    if (Offset)
259      Ptr = IRB.CreateConstGEP1_32(IRB.getInt8Ty(), Ptr, Offset);
260    IRB.CreateCall(SetTagFn, {Ptr, ConstantInt::get(IRB.getInt64Ty(), Size)});
261  }
262
263  void emitPair(IRBuilder<> &IRB, uint64_t Offset, Value *A, Value *B) {
264    LLVM_DEBUG(dbgs() << "  [" << Offset << ", " << Offset + 16 << "):\n");
265    LLVM_DEBUG(dbgs() << "    " << *A << "\n    " << *B << "\n");
266    Value *Ptr = BasePtr;
267    if (Offset)
268      Ptr = IRB.CreateConstGEP1_32(IRB.getInt8Ty(), Ptr, Offset);
269    IRB.CreateCall(StgpFn, {Ptr, A, B});
270  }
271
272  Value *flatten(IRBuilder<> &IRB, Value *V) {
273    if (V->getType()->isIntegerTy())
274      return V;
275    // vector of pointers -> vector of ints
276    if (VectorType *VecTy = dyn_cast<VectorType>(V->getType())) {
277      LLVMContext &Ctx = IRB.getContext();
278      Type *EltTy = VecTy->getElementType();
279      if (EltTy->isPointerTy()) {
280        uint32_t EltSize = DL->getTypeSizeInBits(EltTy);
281        auto *NewTy = FixedVectorType::get(
282            IntegerType::get(Ctx, EltSize),
283            cast<FixedVectorType>(VecTy)->getNumElements());
284        V = IRB.CreatePointerCast(V, NewTy);
285      }
286    }
287    return IRB.CreateBitOrPointerCast(
288        V, IRB.getIntNTy(DL->getTypeStoreSize(V->getType()) * 8));
289  }
290};
291
292class AArch64StackTagging : public FunctionPass {
293  const bool MergeInit;
294  const bool UseStackSafety;
295
296public:
297  static char ID; // Pass ID, replacement for typeid
298
299  AArch64StackTagging(bool IsOptNone = false)
300      : FunctionPass(ID),
301        MergeInit(ClMergeInit.getNumOccurrences() ? ClMergeInit : !IsOptNone),
302        UseStackSafety(ClUseStackSafety.getNumOccurrences() ? ClUseStackSafety
303                                                            : !IsOptNone) {
304    initializeAArch64StackTaggingPass(*PassRegistry::getPassRegistry());
305  }
306
307  void tagAlloca(AllocaInst *AI, Instruction *InsertBefore, Value *Ptr,
308                 uint64_t Size);
309  void untagAlloca(AllocaInst *AI, Instruction *InsertBefore, uint64_t Size);
310
311  Instruction *collectInitializers(Instruction *StartInst, Value *StartPtr,
312                                   uint64_t Size, InitializerBuilder &IB);
313
314  Instruction *insertBaseTaggedPointer(
315      const MapVector<AllocaInst *, memtag::AllocaInfo> &Allocas,
316      const DominatorTree *DT);
317  bool runOnFunction(Function &F) override;
318
319  StringRef getPassName() const override { return "AArch64 Stack Tagging"; }
320
321private:
322  Function *F = nullptr;
323  Function *SetTagFunc = nullptr;
324  const DataLayout *DL = nullptr;
325  AAResults *AA = nullptr;
326  const StackSafetyGlobalInfo *SSI = nullptr;
327
328  void getAnalysisUsage(AnalysisUsage &AU) const override {
329    AU.setPreservesCFG();
330    if (UseStackSafety)
331      AU.addRequired<StackSafetyGlobalInfoWrapperPass>();
332    if (MergeInit)
333      AU.addRequired<AAResultsWrapperPass>();
334  }
335};
336
337} // end anonymous namespace
338
339char AArch64StackTagging::ID = 0;
340
341INITIALIZE_PASS_BEGIN(AArch64StackTagging, DEBUG_TYPE, "AArch64 Stack Tagging",
342                      false, false)
343INITIALIZE_PASS_DEPENDENCY(AAResultsWrapperPass)
344INITIALIZE_PASS_DEPENDENCY(StackSafetyGlobalInfoWrapperPass)
345INITIALIZE_PASS_END(AArch64StackTagging, DEBUG_TYPE, "AArch64 Stack Tagging",
346                    false, false)
347
348FunctionPass *llvm::createAArch64StackTaggingPass(bool IsOptNone) {
349  return new AArch64StackTagging(IsOptNone);
350}
351
352Instruction *AArch64StackTagging::collectInitializers(Instruction *StartInst,
353                                                      Value *StartPtr,
354                                                      uint64_t Size,
355                                                      InitializerBuilder &IB) {
356  MemoryLocation AllocaLoc{StartPtr, Size};
357  Instruction *LastInst = StartInst;
358  BasicBlock::iterator BI(StartInst);
359
360  unsigned Count = 0;
361  for (; Count < ClScanLimit && !BI->isTerminator(); ++BI) {
362    if (!isa<DbgInfoIntrinsic>(*BI))
363      ++Count;
364
365    if (isNoModRef(AA->getModRefInfo(&*BI, AllocaLoc)))
366      continue;
367
368    if (!isa<StoreInst>(BI) && !isa<MemSetInst>(BI)) {
369      // If the instruction is readnone, ignore it, otherwise bail out.  We
370      // don't even allow readonly here because we don't want something like:
371      // A[1] = 2; strlen(A); A[2] = 2; -> memcpy(A, ...); strlen(A).
372      if (BI->mayWriteToMemory() || BI->mayReadFromMemory())
373        break;
374      continue;
375    }
376
377    if (StoreInst *NextStore = dyn_cast<StoreInst>(BI)) {
378      if (!NextStore->isSimple())
379        break;
380
381      // Check to see if this store is to a constant offset from the start ptr.
382      std::optional<int64_t> Offset =
383          isPointerOffset(StartPtr, NextStore->getPointerOperand(), *DL);
384      if (!Offset)
385        break;
386
387      if (!IB.addStore(*Offset, NextStore, DL))
388        break;
389      LastInst = NextStore;
390    } else {
391      MemSetInst *MSI = cast<MemSetInst>(BI);
392
393      if (MSI->isVolatile() || !isa<ConstantInt>(MSI->getLength()))
394        break;
395
396      if (!isa<ConstantInt>(MSI->getValue()))
397        break;
398
399      // Check to see if this store is to a constant offset from the start ptr.
400      std::optional<int64_t> Offset =
401          isPointerOffset(StartPtr, MSI->getDest(), *DL);
402      if (!Offset)
403        break;
404
405      if (!IB.addMemSet(*Offset, MSI))
406        break;
407      LastInst = MSI;
408    }
409  }
410  return LastInst;
411}
412
413void AArch64StackTagging::tagAlloca(AllocaInst *AI, Instruction *InsertBefore,
414                                    Value *Ptr, uint64_t Size) {
415  auto SetTagZeroFunc =
416      Intrinsic::getDeclaration(F->getParent(), Intrinsic::aarch64_settag_zero);
417  auto StgpFunc =
418      Intrinsic::getDeclaration(F->getParent(), Intrinsic::aarch64_stgp);
419
420  InitializerBuilder IB(Size, DL, Ptr, SetTagFunc, SetTagZeroFunc, StgpFunc);
421  bool LittleEndian =
422      Triple(AI->getModule()->getTargetTriple()).isLittleEndian();
423  // Current implementation of initializer merging assumes little endianness.
424  if (MergeInit && !F->hasOptNone() && LittleEndian &&
425      Size < ClMergeInitSizeLimit) {
426    LLVM_DEBUG(dbgs() << "collecting initializers for " << *AI
427                      << ", size = " << Size << "\n");
428    InsertBefore = collectInitializers(InsertBefore, Ptr, Size, IB);
429  }
430
431  IRBuilder<> IRB(InsertBefore);
432  IB.generate(IRB);
433}
434
435void AArch64StackTagging::untagAlloca(AllocaInst *AI, Instruction *InsertBefore,
436                                      uint64_t Size) {
437  IRBuilder<> IRB(InsertBefore);
438  IRB.CreateCall(SetTagFunc, {IRB.CreatePointerCast(AI, IRB.getInt8PtrTy()),
439                              ConstantInt::get(IRB.getInt64Ty(), Size)});
440}
441
442Instruction *AArch64StackTagging::insertBaseTaggedPointer(
443    const MapVector<AllocaInst *, memtag::AllocaInfo> &AllocasToInstrument,
444    const DominatorTree *DT) {
445  BasicBlock *PrologueBB = nullptr;
446  // Try sinking IRG as deep as possible to avoid hurting shrink wrap.
447  for (auto &I : AllocasToInstrument) {
448    const memtag::AllocaInfo &Info = I.second;
449    AllocaInst *AI = Info.AI;
450    if (!PrologueBB) {
451      PrologueBB = AI->getParent();
452      continue;
453    }
454    PrologueBB = DT->findNearestCommonDominator(PrologueBB, AI->getParent());
455  }
456  assert(PrologueBB);
457
458  IRBuilder<> IRB(&PrologueBB->front());
459  Function *IRG_SP =
460      Intrinsic::getDeclaration(F->getParent(), Intrinsic::aarch64_irg_sp);
461  Instruction *Base =
462      IRB.CreateCall(IRG_SP, {Constant::getNullValue(IRB.getInt64Ty())});
463  Base->setName("basetag");
464  return Base;
465}
466
467// FIXME: check for MTE extension
468bool AArch64StackTagging::runOnFunction(Function &Fn) {
469  if (!Fn.hasFnAttribute(Attribute::SanitizeMemTag))
470    return false;
471
472  if (UseStackSafety)
473    SSI = &getAnalysis<StackSafetyGlobalInfoWrapperPass>().getResult();
474  F = &Fn;
475  DL = &Fn.getParent()->getDataLayout();
476  if (MergeInit)
477    AA = &getAnalysis<AAResultsWrapperPass>().getAAResults();
478
479  memtag::StackInfoBuilder SIB(SSI);
480  for (Instruction &I : instructions(F))
481    SIB.visit(I);
482  memtag::StackInfo &SInfo = SIB.get();
483
484  if (SInfo.AllocasToInstrument.empty())
485    return false;
486
487  std::unique_ptr<DominatorTree> DeleteDT;
488  DominatorTree *DT = nullptr;
489  if (auto *P = getAnalysisIfAvailable<DominatorTreeWrapperPass>())
490    DT = &P->getDomTree();
491
492  if (DT == nullptr) {
493    DeleteDT = std::make_unique<DominatorTree>(*F);
494    DT = DeleteDT.get();
495  }
496
497  std::unique_ptr<PostDominatorTree> DeletePDT;
498  PostDominatorTree *PDT = nullptr;
499  if (auto *P = getAnalysisIfAvailable<PostDominatorTreeWrapperPass>())
500    PDT = &P->getPostDomTree();
501
502  if (PDT == nullptr) {
503    DeletePDT = std::make_unique<PostDominatorTree>(*F);
504    PDT = DeletePDT.get();
505  }
506
507  std::unique_ptr<LoopInfo> DeleteLI;
508  LoopInfo *LI = nullptr;
509  if (auto *LIWP = getAnalysisIfAvailable<LoopInfoWrapperPass>()) {
510    LI = &LIWP->getLoopInfo();
511  } else {
512    DeleteLI = std::make_unique<LoopInfo>(*DT);
513    LI = DeleteLI.get();
514  }
515
516  SetTagFunc =
517      Intrinsic::getDeclaration(F->getParent(), Intrinsic::aarch64_settag);
518
519  Instruction *Base = insertBaseTaggedPointer(SInfo.AllocasToInstrument, DT);
520
521  int NextTag = 0;
522  for (auto &I : SInfo.AllocasToInstrument) {
523    memtag::AllocaInfo &Info = I.second;
524    assert(Info.AI && SIB.isInterestingAlloca(*Info.AI));
525    TrackingVH<Instruction> OldAI = Info.AI;
526    memtag::alignAndPadAlloca(Info, kTagGranuleSize);
527    AllocaInst *AI = Info.AI;
528    int Tag = NextTag;
529    NextTag = (NextTag + 1) % 16;
530    // Replace alloca with tagp(alloca).
531    IRBuilder<> IRB(Info.AI->getNextNode());
532    Function *TagP = Intrinsic::getDeclaration(
533        F->getParent(), Intrinsic::aarch64_tagp, {Info.AI->getType()});
534    Instruction *TagPCall =
535        IRB.CreateCall(TagP, {Constant::getNullValue(Info.AI->getType()), Base,
536                              ConstantInt::get(IRB.getInt64Ty(), Tag)});
537    if (Info.AI->hasName())
538      TagPCall->setName(Info.AI->getName() + ".tag");
539    Info.AI->replaceAllUsesWith(TagPCall);
540    TagPCall->setOperand(0, Info.AI);
541
542    // Calls to functions that may return twice (e.g. setjmp) confuse the
543    // postdominator analysis, and will leave us to keep memory tagged after
544    // function return. Work around this by always untagging at every return
545    // statement if return_twice functions are called.
546    bool StandardLifetime =
547        SInfo.UnrecognizedLifetimes.empty() &&
548        memtag::isStandardLifetime(Info.LifetimeStart, Info.LifetimeEnd, DT, LI,
549                                   ClMaxLifetimes) &&
550        !SInfo.CallsReturnTwice;
551    if (StandardLifetime) {
552      IntrinsicInst *Start = Info.LifetimeStart[0];
553      uint64_t Size =
554          cast<ConstantInt>(Start->getArgOperand(0))->getZExtValue();
555      Size = alignTo(Size, kTagGranuleSize);
556      tagAlloca(AI, Start->getNextNode(), Start->getArgOperand(1), Size);
557
558      auto TagEnd = [&](Instruction *Node) { untagAlloca(AI, Node, Size); };
559      if (!DT || !PDT ||
560          !memtag::forAllReachableExits(*DT, *PDT, *LI, Start, Info.LifetimeEnd,
561                                        SInfo.RetVec, TagEnd)) {
562        for (auto *End : Info.LifetimeEnd)
563          End->eraseFromParent();
564      }
565    } else {
566      uint64_t Size = *Info.AI->getAllocationSize(*DL);
567      Value *Ptr = IRB.CreatePointerCast(TagPCall, IRB.getInt8PtrTy());
568      tagAlloca(AI, &*IRB.GetInsertPoint(), Ptr, Size);
569      for (auto *RI : SInfo.RetVec) {
570        untagAlloca(AI, RI, Size);
571      }
572      // We may have inserted tag/untag outside of any lifetime interval.
573      // Remove all lifetime intrinsics for this alloca.
574      for (auto *II : Info.LifetimeStart)
575        II->eraseFromParent();
576      for (auto *II : Info.LifetimeEnd)
577        II->eraseFromParent();
578    }
579
580    // Fixup debug intrinsics to point to the new alloca.
581    for (auto *DVI : Info.DbgVariableIntrinsics)
582      DVI->replaceVariableLocationOp(OldAI, Info.AI);
583  }
584
585  // If we have instrumented at least one alloca, all unrecognized lifetime
586  // intrinsics have to go.
587  for (auto *I : SInfo.UnrecognizedLifetimes)
588    I->eraseFromParent();
589
590  return true;
591}
592