1//===- AArch64StackTagging.cpp - Stack tagging in IR --===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//===----------------------------------------------------------------------===//
9
10#include "AArch64.h"
11#include "AArch64InstrInfo.h"
12#include "AArch64Subtarget.h"
13#include "AArch64TargetMachine.h"
14#include "llvm/ADT/MapVector.h"
15#include "llvm/ADT/SmallVector.h"
16#include "llvm/ADT/Statistic.h"
17#include "llvm/Analysis/AliasAnalysis.h"
18#include "llvm/Analysis/CFG.h"
19#include "llvm/Analysis/LoopInfo.h"
20#include "llvm/Analysis/PostDominators.h"
21#include "llvm/Analysis/ScalarEvolution.h"
22#include "llvm/Analysis/ScalarEvolutionExpressions.h"
23#include "llvm/Analysis/StackSafetyAnalysis.h"
24#include "llvm/Analysis/ValueTracking.h"
25#include "llvm/CodeGen/LiveRegUnits.h"
26#include "llvm/CodeGen/MachineBasicBlock.h"
27#include "llvm/CodeGen/MachineFunction.h"
28#include "llvm/CodeGen/MachineFunctionPass.h"
29#include "llvm/CodeGen/MachineInstr.h"
30#include "llvm/CodeGen/MachineInstrBuilder.h"
31#include "llvm/CodeGen/MachineLoopInfo.h"
32#include "llvm/CodeGen/MachineOperand.h"
33#include "llvm/CodeGen/MachineRegisterInfo.h"
34#include "llvm/CodeGen/TargetPassConfig.h"
35#include "llvm/CodeGen/TargetRegisterInfo.h"
36#include "llvm/IR/DebugLoc.h"
37#include "llvm/IR/Dominators.h"
38#include "llvm/IR/Function.h"
39#include "llvm/IR/GetElementPtrTypeIterator.h"
40#include "llvm/IR/IRBuilder.h"
41#include "llvm/IR/InstIterator.h"
42#include "llvm/IR/Instruction.h"
43#include "llvm/IR/Instructions.h"
44#include "llvm/IR/IntrinsicInst.h"
45#include "llvm/IR/IntrinsicsAArch64.h"
46#include "llvm/IR/Metadata.h"
47#include "llvm/IR/ValueHandle.h"
48#include "llvm/InitializePasses.h"
49#include "llvm/Pass.h"
50#include "llvm/Support/Casting.h"
51#include "llvm/Support/Debug.h"
52#include "llvm/Support/raw_ostream.h"
53#include "llvm/Transforms/Utils/Local.h"
54#include "llvm/Transforms/Utils/MemoryTaggingSupport.h"
55#include <cassert>
56#include <iterator>
57#include <memory>
58#include <utility>
59
60using namespace llvm;
61
62#define DEBUG_TYPE "aarch64-stack-tagging"
63
64static cl::opt<bool> ClMergeInit(
65    "stack-tagging-merge-init", cl::Hidden, cl::init(true),
66    cl::desc("merge stack variable initializers with tagging when possible"));
67
68static cl::opt<bool>
69    ClUseStackSafety("stack-tagging-use-stack-safety", cl::Hidden,
70                     cl::init(true),
71                     cl::desc("Use Stack Safety analysis results"));
72
73static cl::opt<unsigned> ClScanLimit("stack-tagging-merge-init-scan-limit",
74                                     cl::init(40), cl::Hidden);
75
76static cl::opt<unsigned>
77    ClMergeInitSizeLimit("stack-tagging-merge-init-size-limit", cl::init(272),
78                         cl::Hidden);
79
80static cl::opt<size_t> ClMaxLifetimes(
81    "stack-tagging-max-lifetimes-for-alloca", cl::Hidden, cl::init(3),
82    cl::ReallyHidden,
83    cl::desc("How many lifetime ends to handle for a single alloca."),
84    cl::Optional);
85
86static const Align kTagGranuleSize = Align(16);
87
88namespace {
89
90class InitializerBuilder {
91  uint64_t Size;
92  const DataLayout *DL;
93  Value *BasePtr;
94  Function *SetTagFn;
95  Function *SetTagZeroFn;
96  Function *StgpFn;
97
98  // List of initializers sorted by start offset.
99  struct Range {
100    uint64_t Start, End;
101    Instruction *Inst;
102  };
103  SmallVector<Range, 4> Ranges;
104  // 8-aligned offset => 8-byte initializer
105  // Missing keys are zero initialized.
106  std::map<uint64_t, Value *> Out;
107
108public:
109  InitializerBuilder(uint64_t Size, const DataLayout *DL, Value *BasePtr,
110                     Function *SetTagFn, Function *SetTagZeroFn,
111                     Function *StgpFn)
112      : Size(Size), DL(DL), BasePtr(BasePtr), SetTagFn(SetTagFn),
113        SetTagZeroFn(SetTagZeroFn), StgpFn(StgpFn) {}
114
115  bool addRange(uint64_t Start, uint64_t End, Instruction *Inst) {
116    auto I =
117        llvm::lower_bound(Ranges, Start, [](const Range &LHS, uint64_t RHS) {
118          return LHS.End <= RHS;
119        });
120    if (I != Ranges.end() && End > I->Start) {
121      // Overlap - bail.
122      return false;
123    }
124    Ranges.insert(I, {Start, End, Inst});
125    return true;
126  }
127
128  bool addStore(uint64_t Offset, StoreInst *SI, const DataLayout *DL) {
129    int64_t StoreSize = DL->getTypeStoreSize(SI->getOperand(0)->getType());
130    if (!addRange(Offset, Offset + StoreSize, SI))
131      return false;
132    IRBuilder<> IRB(SI);
133    applyStore(IRB, Offset, Offset + StoreSize, SI->getOperand(0));
134    return true;
135  }
136
137  bool addMemSet(uint64_t Offset, MemSetInst *MSI) {
138    uint64_t StoreSize = cast<ConstantInt>(MSI->getLength())->getZExtValue();
139    if (!addRange(Offset, Offset + StoreSize, MSI))
140      return false;
141    IRBuilder<> IRB(MSI);
142    applyMemSet(IRB, Offset, Offset + StoreSize,
143                cast<ConstantInt>(MSI->getValue()));
144    return true;
145  }
146
147  void applyMemSet(IRBuilder<> &IRB, int64_t Start, int64_t End,
148                   ConstantInt *V) {
149    // Out[] does not distinguish between zero and undef, and we already know
150    // that this memset does not overlap with any other initializer. Nothing to
151    // do for memset(0).
152    if (V->isZero())
153      return;
154    for (int64_t Offset = Start - Start % 8; Offset < End; Offset += 8) {
155      uint64_t Cst = 0x0101010101010101UL;
156      int LowBits = Offset < Start ? (Start - Offset) * 8 : 0;
157      if (LowBits)
158        Cst = (Cst >> LowBits) << LowBits;
159      int HighBits = End - Offset < 8 ? (8 - (End - Offset)) * 8 : 0;
160      if (HighBits)
161        Cst = (Cst << HighBits) >> HighBits;
162      ConstantInt *C =
163          ConstantInt::get(IRB.getInt64Ty(), Cst * V->getZExtValue());
164
165      Value *&CurrentV = Out[Offset];
166      if (!CurrentV) {
167        CurrentV = C;
168      } else {
169        CurrentV = IRB.CreateOr(CurrentV, C);
170      }
171    }
172  }
173
174  // Take a 64-bit slice of the value starting at the given offset (in bytes).
175  // Offset can be negative. Pad with zeroes on both sides when necessary.
176  Value *sliceValue(IRBuilder<> &IRB, Value *V, int64_t Offset) {
177    if (Offset > 0) {
178      V = IRB.CreateLShr(V, Offset * 8);
179      V = IRB.CreateZExtOrTrunc(V, IRB.getInt64Ty());
180    } else if (Offset < 0) {
181      V = IRB.CreateZExtOrTrunc(V, IRB.getInt64Ty());
182      V = IRB.CreateShl(V, -Offset * 8);
183    } else {
184      V = IRB.CreateZExtOrTrunc(V, IRB.getInt64Ty());
185    }
186    return V;
187  }
188
189  void applyStore(IRBuilder<> &IRB, int64_t Start, int64_t End,
190                  Value *StoredValue) {
191    StoredValue = flatten(IRB, StoredValue);
192    for (int64_t Offset = Start - Start % 8; Offset < End; Offset += 8) {
193      Value *V = sliceValue(IRB, StoredValue, Offset - Start);
194      Value *&CurrentV = Out[Offset];
195      if (!CurrentV) {
196        CurrentV = V;
197      } else {
198        CurrentV = IRB.CreateOr(CurrentV, V);
199      }
200    }
201  }
202
203  void generate(IRBuilder<> &IRB) {
204    LLVM_DEBUG(dbgs() << "Combined initializer\n");
205    // No initializers => the entire allocation is undef.
206    if (Ranges.empty()) {
207      emitUndef(IRB, 0, Size);
208      return;
209    }
210
211    // Look through 8-byte initializer list 16 bytes at a time;
212    // If one of the two 8-byte halfs is non-zero non-undef, emit STGP.
213    // Otherwise, emit zeroes up to next available item.
214    uint64_t LastOffset = 0;
215    for (uint64_t Offset = 0; Offset < Size; Offset += 16) {
216      auto I1 = Out.find(Offset);
217      auto I2 = Out.find(Offset + 8);
218      if (I1 == Out.end() && I2 == Out.end())
219        continue;
220
221      if (Offset > LastOffset)
222        emitZeroes(IRB, LastOffset, Offset - LastOffset);
223
224      Value *Store1 = I1 == Out.end() ? Constant::getNullValue(IRB.getInt64Ty())
225                                      : I1->second;
226      Value *Store2 = I2 == Out.end() ? Constant::getNullValue(IRB.getInt64Ty())
227                                      : I2->second;
228      emitPair(IRB, Offset, Store1, Store2);
229      LastOffset = Offset + 16;
230    }
231
232    // memset(0) does not update Out[], therefore the tail can be either undef
233    // or zero.
234    if (LastOffset < Size)
235      emitZeroes(IRB, LastOffset, Size - LastOffset);
236
237    for (const auto &R : Ranges) {
238      R.Inst->eraseFromParent();
239    }
240  }
241
242  void emitZeroes(IRBuilder<> &IRB, uint64_t Offset, uint64_t Size) {
243    LLVM_DEBUG(dbgs() << "  [" << Offset << ", " << Offset + Size
244                      << ") zero\n");
245    Value *Ptr = BasePtr;
246    if (Offset)
247      Ptr = IRB.CreateConstGEP1_32(IRB.getInt8Ty(), Ptr, Offset);
248    IRB.CreateCall(SetTagZeroFn,
249                   {Ptr, ConstantInt::get(IRB.getInt64Ty(), Size)});
250  }
251
252  void emitUndef(IRBuilder<> &IRB, uint64_t Offset, uint64_t Size) {
253    LLVM_DEBUG(dbgs() << "  [" << Offset << ", " << Offset + Size
254                      << ") undef\n");
255    Value *Ptr = BasePtr;
256    if (Offset)
257      Ptr = IRB.CreateConstGEP1_32(IRB.getInt8Ty(), Ptr, Offset);
258    IRB.CreateCall(SetTagFn, {Ptr, ConstantInt::get(IRB.getInt64Ty(), Size)});
259  }
260
261  void emitPair(IRBuilder<> &IRB, uint64_t Offset, Value *A, Value *B) {
262    LLVM_DEBUG(dbgs() << "  [" << Offset << ", " << Offset + 16 << "):\n");
263    LLVM_DEBUG(dbgs() << "    " << *A << "\n    " << *B << "\n");
264    Value *Ptr = BasePtr;
265    if (Offset)
266      Ptr = IRB.CreateConstGEP1_32(IRB.getInt8Ty(), Ptr, Offset);
267    IRB.CreateCall(StgpFn, {Ptr, A, B});
268  }
269
270  Value *flatten(IRBuilder<> &IRB, Value *V) {
271    if (V->getType()->isIntegerTy())
272      return V;
273    // vector of pointers -> vector of ints
274    if (VectorType *VecTy = dyn_cast<VectorType>(V->getType())) {
275      LLVMContext &Ctx = IRB.getContext();
276      Type *EltTy = VecTy->getElementType();
277      if (EltTy->isPointerTy()) {
278        uint32_t EltSize = DL->getTypeSizeInBits(EltTy);
279        auto *NewTy = FixedVectorType::get(
280            IntegerType::get(Ctx, EltSize),
281            cast<FixedVectorType>(VecTy)->getNumElements());
282        V = IRB.CreatePointerCast(V, NewTy);
283      }
284    }
285    return IRB.CreateBitOrPointerCast(
286        V, IRB.getIntNTy(DL->getTypeStoreSize(V->getType()) * 8));
287  }
288};
289
290class AArch64StackTagging : public FunctionPass {
291  const bool MergeInit;
292  const bool UseStackSafety;
293
294public:
295  static char ID; // Pass ID, replacement for typeid
296
297  AArch64StackTagging(bool IsOptNone = false)
298      : FunctionPass(ID),
299        MergeInit(ClMergeInit.getNumOccurrences() ? ClMergeInit : !IsOptNone),
300        UseStackSafety(ClUseStackSafety.getNumOccurrences() ? ClUseStackSafety
301                                                            : !IsOptNone) {
302    initializeAArch64StackTaggingPass(*PassRegistry::getPassRegistry());
303  }
304
305  void tagAlloca(AllocaInst *AI, Instruction *InsertBefore, Value *Ptr,
306                 uint64_t Size);
307  void untagAlloca(AllocaInst *AI, Instruction *InsertBefore, uint64_t Size);
308
309  Instruction *collectInitializers(Instruction *StartInst, Value *StartPtr,
310                                   uint64_t Size, InitializerBuilder &IB);
311
312  Instruction *insertBaseTaggedPointer(
313      const MapVector<AllocaInst *, memtag::AllocaInfo> &Allocas,
314      const DominatorTree *DT);
315  bool runOnFunction(Function &F) override;
316
317  StringRef getPassName() const override { return "AArch64 Stack Tagging"; }
318
319private:
320  Function *F = nullptr;
321  Function *SetTagFunc = nullptr;
322  const DataLayout *DL = nullptr;
323  AAResults *AA = nullptr;
324  const StackSafetyGlobalInfo *SSI = nullptr;
325
326  void getAnalysisUsage(AnalysisUsage &AU) const override {
327    AU.setPreservesCFG();
328    if (UseStackSafety)
329      AU.addRequired<StackSafetyGlobalInfoWrapperPass>();
330    if (MergeInit)
331      AU.addRequired<AAResultsWrapperPass>();
332  }
333};
334
335} // end anonymous namespace
336
337char AArch64StackTagging::ID = 0;
338
339INITIALIZE_PASS_BEGIN(AArch64StackTagging, DEBUG_TYPE, "AArch64 Stack Tagging",
340                      false, false)
341INITIALIZE_PASS_DEPENDENCY(AAResultsWrapperPass)
342INITIALIZE_PASS_DEPENDENCY(StackSafetyGlobalInfoWrapperPass)
343INITIALIZE_PASS_END(AArch64StackTagging, DEBUG_TYPE, "AArch64 Stack Tagging",
344                    false, false)
345
346FunctionPass *llvm::createAArch64StackTaggingPass(bool IsOptNone) {
347  return new AArch64StackTagging(IsOptNone);
348}
349
350Instruction *AArch64StackTagging::collectInitializers(Instruction *StartInst,
351                                                      Value *StartPtr,
352                                                      uint64_t Size,
353                                                      InitializerBuilder &IB) {
354  MemoryLocation AllocaLoc{StartPtr, Size};
355  Instruction *LastInst = StartInst;
356  BasicBlock::iterator BI(StartInst);
357
358  unsigned Count = 0;
359  for (; Count < ClScanLimit && !BI->isTerminator(); ++BI) {
360    if (!isa<DbgInfoIntrinsic>(*BI))
361      ++Count;
362
363    if (isNoModRef(AA->getModRefInfo(&*BI, AllocaLoc)))
364      continue;
365
366    if (!isa<StoreInst>(BI) && !isa<MemSetInst>(BI)) {
367      // If the instruction is readnone, ignore it, otherwise bail out.  We
368      // don't even allow readonly here because we don't want something like:
369      // A[1] = 2; strlen(A); A[2] = 2; -> memcpy(A, ...); strlen(A).
370      if (BI->mayWriteToMemory() || BI->mayReadFromMemory())
371        break;
372      continue;
373    }
374
375    if (StoreInst *NextStore = dyn_cast<StoreInst>(BI)) {
376      if (!NextStore->isSimple())
377        break;
378
379      // Check to see if this store is to a constant offset from the start ptr.
380      std::optional<int64_t> Offset =
381          NextStore->getPointerOperand()->getPointerOffsetFrom(StartPtr, *DL);
382      if (!Offset)
383        break;
384
385      if (!IB.addStore(*Offset, NextStore, DL))
386        break;
387      LastInst = NextStore;
388    } else {
389      MemSetInst *MSI = cast<MemSetInst>(BI);
390
391      if (MSI->isVolatile() || !isa<ConstantInt>(MSI->getLength()))
392        break;
393
394      if (!isa<ConstantInt>(MSI->getValue()))
395        break;
396
397      // Check to see if this store is to a constant offset from the start ptr.
398      std::optional<int64_t> Offset =
399          MSI->getDest()->getPointerOffsetFrom(StartPtr, *DL);
400      if (!Offset)
401        break;
402
403      if (!IB.addMemSet(*Offset, MSI))
404        break;
405      LastInst = MSI;
406    }
407  }
408  return LastInst;
409}
410
411void AArch64StackTagging::tagAlloca(AllocaInst *AI, Instruction *InsertBefore,
412                                    Value *Ptr, uint64_t Size) {
413  auto SetTagZeroFunc =
414      Intrinsic::getDeclaration(F->getParent(), Intrinsic::aarch64_settag_zero);
415  auto StgpFunc =
416      Intrinsic::getDeclaration(F->getParent(), Intrinsic::aarch64_stgp);
417
418  InitializerBuilder IB(Size, DL, Ptr, SetTagFunc, SetTagZeroFunc, StgpFunc);
419  bool LittleEndian =
420      Triple(AI->getModule()->getTargetTriple()).isLittleEndian();
421  // Current implementation of initializer merging assumes little endianness.
422  if (MergeInit && !F->hasOptNone() && LittleEndian &&
423      Size < ClMergeInitSizeLimit) {
424    LLVM_DEBUG(dbgs() << "collecting initializers for " << *AI
425                      << ", size = " << Size << "\n");
426    InsertBefore = collectInitializers(InsertBefore, Ptr, Size, IB);
427  }
428
429  IRBuilder<> IRB(InsertBefore);
430  IB.generate(IRB);
431}
432
433void AArch64StackTagging::untagAlloca(AllocaInst *AI, Instruction *InsertBefore,
434                                      uint64_t Size) {
435  IRBuilder<> IRB(InsertBefore);
436  IRB.CreateCall(SetTagFunc, {IRB.CreatePointerCast(AI, IRB.getPtrTy()),
437                              ConstantInt::get(IRB.getInt64Ty(), Size)});
438}
439
440Instruction *AArch64StackTagging::insertBaseTaggedPointer(
441    const MapVector<AllocaInst *, memtag::AllocaInfo> &AllocasToInstrument,
442    const DominatorTree *DT) {
443  BasicBlock *PrologueBB = nullptr;
444  // Try sinking IRG as deep as possible to avoid hurting shrink wrap.
445  for (auto &I : AllocasToInstrument) {
446    const memtag::AllocaInfo &Info = I.second;
447    AllocaInst *AI = Info.AI;
448    if (!PrologueBB) {
449      PrologueBB = AI->getParent();
450      continue;
451    }
452    PrologueBB = DT->findNearestCommonDominator(PrologueBB, AI->getParent());
453  }
454  assert(PrologueBB);
455
456  IRBuilder<> IRB(&PrologueBB->front());
457  Function *IRG_SP =
458      Intrinsic::getDeclaration(F->getParent(), Intrinsic::aarch64_irg_sp);
459  Instruction *Base =
460      IRB.CreateCall(IRG_SP, {Constant::getNullValue(IRB.getInt64Ty())});
461  Base->setName("basetag");
462  return Base;
463}
464
465// FIXME: check for MTE extension
466bool AArch64StackTagging::runOnFunction(Function &Fn) {
467  if (!Fn.hasFnAttribute(Attribute::SanitizeMemTag))
468    return false;
469
470  if (UseStackSafety)
471    SSI = &getAnalysis<StackSafetyGlobalInfoWrapperPass>().getResult();
472  F = &Fn;
473  DL = &Fn.getParent()->getDataLayout();
474  if (MergeInit)
475    AA = &getAnalysis<AAResultsWrapperPass>().getAAResults();
476
477  memtag::StackInfoBuilder SIB(SSI);
478  for (Instruction &I : instructions(F))
479    SIB.visit(I);
480  memtag::StackInfo &SInfo = SIB.get();
481
482  if (SInfo.AllocasToInstrument.empty())
483    return false;
484
485  std::unique_ptr<DominatorTree> DeleteDT;
486  DominatorTree *DT = nullptr;
487  if (auto *P = getAnalysisIfAvailable<DominatorTreeWrapperPass>())
488    DT = &P->getDomTree();
489
490  if (DT == nullptr) {
491    DeleteDT = std::make_unique<DominatorTree>(*F);
492    DT = DeleteDT.get();
493  }
494
495  std::unique_ptr<PostDominatorTree> DeletePDT;
496  PostDominatorTree *PDT = nullptr;
497  if (auto *P = getAnalysisIfAvailable<PostDominatorTreeWrapperPass>())
498    PDT = &P->getPostDomTree();
499
500  if (PDT == nullptr) {
501    DeletePDT = std::make_unique<PostDominatorTree>(*F);
502    PDT = DeletePDT.get();
503  }
504
505  std::unique_ptr<LoopInfo> DeleteLI;
506  LoopInfo *LI = nullptr;
507  if (auto *LIWP = getAnalysisIfAvailable<LoopInfoWrapperPass>()) {
508    LI = &LIWP->getLoopInfo();
509  } else {
510    DeleteLI = std::make_unique<LoopInfo>(*DT);
511    LI = DeleteLI.get();
512  }
513
514  SetTagFunc =
515      Intrinsic::getDeclaration(F->getParent(), Intrinsic::aarch64_settag);
516
517  Instruction *Base = insertBaseTaggedPointer(SInfo.AllocasToInstrument, DT);
518
519  int NextTag = 0;
520  for (auto &I : SInfo.AllocasToInstrument) {
521    memtag::AllocaInfo &Info = I.second;
522    assert(Info.AI && SIB.isInterestingAlloca(*Info.AI));
523    TrackingVH<Instruction> OldAI = Info.AI;
524    memtag::alignAndPadAlloca(Info, kTagGranuleSize);
525    AllocaInst *AI = Info.AI;
526    int Tag = NextTag;
527    NextTag = (NextTag + 1) % 16;
528    // Replace alloca with tagp(alloca).
529    IRBuilder<> IRB(Info.AI->getNextNode());
530    Function *TagP = Intrinsic::getDeclaration(
531        F->getParent(), Intrinsic::aarch64_tagp, {Info.AI->getType()});
532    Instruction *TagPCall =
533        IRB.CreateCall(TagP, {Constant::getNullValue(Info.AI->getType()), Base,
534                              ConstantInt::get(IRB.getInt64Ty(), Tag)});
535    if (Info.AI->hasName())
536      TagPCall->setName(Info.AI->getName() + ".tag");
537    Info.AI->replaceAllUsesWith(TagPCall);
538    TagPCall->setOperand(0, Info.AI);
539
540    // Calls to functions that may return twice (e.g. setjmp) confuse the
541    // postdominator analysis, and will leave us to keep memory tagged after
542    // function return. Work around this by always untagging at every return
543    // statement if return_twice functions are called.
544    bool StandardLifetime =
545        SInfo.UnrecognizedLifetimes.empty() &&
546        memtag::isStandardLifetime(Info.LifetimeStart, Info.LifetimeEnd, DT, LI,
547                                   ClMaxLifetimes) &&
548        !SInfo.CallsReturnTwice;
549    if (StandardLifetime) {
550      IntrinsicInst *Start = Info.LifetimeStart[0];
551      uint64_t Size =
552          cast<ConstantInt>(Start->getArgOperand(0))->getZExtValue();
553      Size = alignTo(Size, kTagGranuleSize);
554      tagAlloca(AI, Start->getNextNode(), Start->getArgOperand(1), Size);
555
556      auto TagEnd = [&](Instruction *Node) { untagAlloca(AI, Node, Size); };
557      if (!DT || !PDT ||
558          !memtag::forAllReachableExits(*DT, *PDT, *LI, Start, Info.LifetimeEnd,
559                                        SInfo.RetVec, TagEnd)) {
560        for (auto *End : Info.LifetimeEnd)
561          End->eraseFromParent();
562      }
563    } else {
564      uint64_t Size = *Info.AI->getAllocationSize(*DL);
565      Value *Ptr = IRB.CreatePointerCast(TagPCall, IRB.getPtrTy());
566      tagAlloca(AI, &*IRB.GetInsertPoint(), Ptr, Size);
567      for (auto *RI : SInfo.RetVec) {
568        untagAlloca(AI, RI, Size);
569      }
570      // We may have inserted tag/untag outside of any lifetime interval.
571      // Remove all lifetime intrinsics for this alloca.
572      for (auto *II : Info.LifetimeStart)
573        II->eraseFromParent();
574      for (auto *II : Info.LifetimeEnd)
575        II->eraseFromParent();
576    }
577
578    // Fixup debug intrinsics to point to the new alloca.
579    for (auto *DVI : Info.DbgVariableIntrinsics)
580      DVI->replaceVariableLocationOp(OldAI, Info.AI);
581  }
582
583  // If we have instrumented at least one alloca, all unrecognized lifetime
584  // intrinsics have to go.
585  for (auto *I : SInfo.UnrecognizedLifetimes)
586    I->eraseFromParent();
587
588  return true;
589}
590