1//===-- AMDGPUCtorDtorLowering.cpp - Handle global ctors and dtors --------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8///
9/// \file
10/// This pass creates a unified init and fini kernel with the required metadata
11//===----------------------------------------------------------------------===//
12
13#include "AMDGPUCtorDtorLowering.h"
14#include "AMDGPU.h"
15#include "llvm/IR/Constants.h"
16#include "llvm/IR/Function.h"
17#include "llvm/IR/GlobalVariable.h"
18#include "llvm/IR/IRBuilder.h"
19#include "llvm/IR/Module.h"
20#include "llvm/IR/Value.h"
21#include "llvm/Pass.h"
22#include "llvm/Transforms/Utils/ModuleUtils.h"
23
24using namespace llvm;
25
26#define DEBUG_TYPE "amdgpu-lower-ctor-dtor"
27
28namespace {
29
30static Function *createInitOrFiniKernelFunction(Module &M, bool IsCtor) {
31  StringRef InitOrFiniKernelName = "amdgcn.device.init";
32  if (!IsCtor)
33    InitOrFiniKernelName = "amdgcn.device.fini";
34  if (M.getFunction(InitOrFiniKernelName))
35    return nullptr;
36
37  Function *InitOrFiniKernel = Function::createWithDefaultAttr(
38      FunctionType::get(Type::getVoidTy(M.getContext()), false),
39      GlobalValue::WeakODRLinkage, 0, InitOrFiniKernelName, &M);
40  InitOrFiniKernel->setCallingConv(CallingConv::AMDGPU_KERNEL);
41  InitOrFiniKernel->addFnAttr("amdgpu-flat-work-group-size", "1,1");
42  if (IsCtor)
43    InitOrFiniKernel->addFnAttr("device-init");
44  else
45    InitOrFiniKernel->addFnAttr("device-fini");
46  return InitOrFiniKernel;
47}
48
49// The linker will provide the associated symbols to allow us to traverse the
50// global constructors / destructors in priority order. We create the IR
51// required to call each callback in this section. This is equivalent to the
52// following code.
53//
54// extern "C" void * __init_array_start[];
55// extern "C" void * __init_array_end[];
56// extern "C" void * __fini_array_start[];
57// extern "C" void * __fini_array_end[];
58//
59// using InitCallback = void();
60// using FiniCallback = void(void);
61//
62// void call_init_array_callbacks() {
63//   for (auto start = __init_array_start; start != __init_array_end; ++start)
64//     reinterpret_cast<InitCallback *>(*start)();
65// }
66//
67// void call_fini_array_callbacks() {
68//  size_t fini_array_size = __fini_array_end - __fini_array_start;
69//  for (size_t i = fini_array_size; i > 0; --i)
70//    reinterpret_cast<FiniCallback *>(__fini_array_start[i - 1])();
71// }
72static void createInitOrFiniCalls(Function &F, bool IsCtor) {
73  Module &M = *F.getParent();
74  LLVMContext &C = M.getContext();
75
76  IRBuilder<> IRB(BasicBlock::Create(C, "entry", &F));
77  auto *LoopBB = BasicBlock::Create(C, "while.entry", &F);
78  auto *ExitBB = BasicBlock::Create(C, "while.end", &F);
79  Type *PtrTy = IRB.getPtrTy(AMDGPUAS::GLOBAL_ADDRESS);
80
81  auto *Begin = M.getOrInsertGlobal(
82      IsCtor ? "__init_array_start" : "__fini_array_start",
83      ArrayType::get(PtrTy, 0), [&]() {
84        return new GlobalVariable(
85            M, ArrayType::get(PtrTy, 0),
86            /*isConstant=*/true, GlobalValue::ExternalLinkage,
87            /*Initializer=*/nullptr,
88            IsCtor ? "__init_array_start" : "__fini_array_start",
89            /*InsertBefore=*/nullptr, GlobalVariable::NotThreadLocal,
90            /*AddressSpace=*/1);
91      });
92  auto *End = M.getOrInsertGlobal(
93      IsCtor ? "__init_array_end" : "__fini_array_end",
94      ArrayType::get(PtrTy, 0), [&]() {
95        return new GlobalVariable(
96            M, ArrayType::get(PtrTy, 0),
97            /*isConstant=*/true, GlobalValue::ExternalLinkage,
98            /*Initializer=*/nullptr,
99            IsCtor ? "__init_array_end" : "__fini_array_end",
100            /*InsertBefore=*/nullptr, GlobalVariable::NotThreadLocal,
101            /*AddressSpace=*/1);
102      });
103
104  // The constructor type is suppoed to allow using the argument vectors, but
105  // for now we just call them with no arguments.
106  auto *CallBackTy = FunctionType::get(IRB.getVoidTy(), {});
107
108  Value *Start = Begin;
109  Value *Stop = End;
110  // The destructor array must be called in reverse order. Get a constant
111  // expression to the end of the array and iterate backwards instead.
112  if (!IsCtor) {
113    Type *Int64Ty = IntegerType::getInt64Ty(C);
114    auto *EndPtr = IRB.CreatePtrToInt(End, Int64Ty);
115    auto *BeginPtr = IRB.CreatePtrToInt(Begin, Int64Ty);
116    auto *ByteSize = IRB.CreateSub(EndPtr, BeginPtr);
117    auto *Size = IRB.CreateAShr(ByteSize, ConstantInt::get(Int64Ty, 3));
118    auto *Offset = IRB.CreateSub(Size, ConstantInt::get(Int64Ty, 1));
119    Start = IRB.CreateInBoundsGEP(
120        ArrayType::get(IRB.getPtrTy(), 0), Begin,
121        ArrayRef<Value *>({ConstantInt::get(Int64Ty, 0), Offset}));
122    Stop = Begin;
123  }
124
125  IRB.CreateCondBr(
126      IRB.CreateCmp(IsCtor ? ICmpInst::ICMP_NE : ICmpInst::ICMP_UGE, Start,
127                    Stop),
128      LoopBB, ExitBB);
129  IRB.SetInsertPoint(LoopBB);
130  auto *CallBackPHI = IRB.CreatePHI(PtrTy, 2, "ptr");
131  auto *CallBack = IRB.CreateLoad(IRB.getPtrTy(F.getAddressSpace()),
132                                  CallBackPHI, "callback");
133  IRB.CreateCall(CallBackTy, CallBack);
134  auto *NewCallBack =
135      IRB.CreateConstGEP1_64(PtrTy, CallBackPHI, IsCtor ? 1 : -1, "next");
136  auto *EndCmp = IRB.CreateCmp(IsCtor ? ICmpInst::ICMP_EQ : ICmpInst::ICMP_ULT,
137                               NewCallBack, Stop, "end");
138  CallBackPHI->addIncoming(Start, &F.getEntryBlock());
139  CallBackPHI->addIncoming(NewCallBack, LoopBB);
140  IRB.CreateCondBr(EndCmp, ExitBB, LoopBB);
141  IRB.SetInsertPoint(ExitBB);
142  IRB.CreateRetVoid();
143}
144
145static bool createInitOrFiniKernel(Module &M, StringRef GlobalName,
146                                   bool IsCtor) {
147  GlobalVariable *GV = M.getGlobalVariable(GlobalName);
148  if (!GV || !GV->hasInitializer())
149    return false;
150  ConstantArray *GA = dyn_cast<ConstantArray>(GV->getInitializer());
151  if (!GA || GA->getNumOperands() == 0)
152    return false;
153
154  Function *InitOrFiniKernel = createInitOrFiniKernelFunction(M, IsCtor);
155  if (!InitOrFiniKernel)
156    return false;
157
158  createInitOrFiniCalls(*InitOrFiniKernel, IsCtor);
159
160  appendToUsed(M, {InitOrFiniKernel});
161  return true;
162}
163
164static bool lowerCtorsAndDtors(Module &M) {
165  bool Modified = false;
166  Modified |= createInitOrFiniKernel(M, "llvm.global_ctors", /*IsCtor =*/true);
167  Modified |= createInitOrFiniKernel(M, "llvm.global_dtors", /*IsCtor =*/false);
168  return Modified;
169}
170
171class AMDGPUCtorDtorLoweringLegacy final : public ModulePass {
172public:
173  static char ID;
174  AMDGPUCtorDtorLoweringLegacy() : ModulePass(ID) {}
175  bool runOnModule(Module &M) override { return lowerCtorsAndDtors(M); }
176};
177
178} // End anonymous namespace
179
180PreservedAnalyses AMDGPUCtorDtorLoweringPass::run(Module &M,
181                                                  ModuleAnalysisManager &AM) {
182  return lowerCtorsAndDtors(M) ? PreservedAnalyses::none()
183                               : PreservedAnalyses::all();
184}
185
186char AMDGPUCtorDtorLoweringLegacy::ID = 0;
187char &llvm::AMDGPUCtorDtorLoweringLegacyPassID =
188    AMDGPUCtorDtorLoweringLegacy::ID;
189INITIALIZE_PASS(AMDGPUCtorDtorLoweringLegacy, DEBUG_TYPE,
190                "Lower ctors and dtors for AMDGPU", false, false)
191
192ModulePass *llvm::createAMDGPUCtorDtorLoweringLegacyPass() {
193  return new AMDGPUCtorDtorLoweringLegacy();
194}
195