1//===-- AMDGPU.h - MachineFunction passes hw codegen --------------*- C++ -*-=//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7/// \file
8//===----------------------------------------------------------------------===//
9
10#ifndef LLVM_LIB_TARGET_AMDGPU_AMDGPU_H
11#define LLVM_LIB_TARGET_AMDGPU_AMDGPU_H
12
13#include "llvm/IR/IntrinsicsR600.h" // TODO: Sink this.
14#include "llvm/IR/IntrinsicsAMDGPU.h" // TODO: Sink this.
15#include "llvm/Support/CodeGen.h"
16
17namespace llvm {
18
19class AMDGPUTargetMachine;
20class FunctionPass;
21class GCNTargetMachine;
22class ImmutablePass;
23class ModulePass;
24class Pass;
25class Target;
26class TargetMachine;
27class TargetOptions;
28class PassRegistry;
29class Module;
30
31// GlobalISel passes
32void initializeAMDGPUPreLegalizerCombinerPass(PassRegistry &);
33FunctionPass *createAMDGPUPreLegalizeCombiner(bool IsOptNone);
34void initializeAMDGPUPostLegalizerCombinerPass(PassRegistry &);
35FunctionPass *createAMDGPUPostLegalizeCombiner(bool IsOptNone);
36FunctionPass *createAMDGPURegBankCombiner(bool IsOptNone);
37void initializeAMDGPURegBankCombinerPass(PassRegistry &);
38
39// R600 Passes
40FunctionPass *createR600VectorRegMerger();
41FunctionPass *createR600ExpandSpecialInstrsPass();
42FunctionPass *createR600EmitClauseMarkers();
43FunctionPass *createR600ClauseMergePass();
44FunctionPass *createR600Packetizer();
45FunctionPass *createR600ControlFlowFinalizer();
46FunctionPass *createAMDGPUCFGStructurizerPass();
47FunctionPass *createR600ISelDag(TargetMachine *TM, CodeGenOpt::Level OptLevel);
48
49// SI Passes
50FunctionPass *createGCNDPPCombinePass();
51FunctionPass *createSIAnnotateControlFlowPass();
52FunctionPass *createSIFoldOperandsPass();
53FunctionPass *createSIPeepholeSDWAPass();
54FunctionPass *createSILowerI1CopiesPass();
55FunctionPass *createSIFixupVectorISelPass();
56FunctionPass *createSIAddIMGInitPass();
57FunctionPass *createSIShrinkInstructionsPass();
58FunctionPass *createSILoadStoreOptimizerPass();
59FunctionPass *createSIWholeQuadModePass();
60FunctionPass *createSIFixControlFlowLiveIntervalsPass();
61FunctionPass *createSIOptimizeExecMaskingPreRAPass();
62FunctionPass *createSIFixSGPRCopiesPass();
63FunctionPass *createSIMemoryLegalizerPass();
64FunctionPass *createSIInsertWaitcntsPass();
65FunctionPass *createSIPreAllocateWWMRegsPass();
66FunctionPass *createSIFormMemoryClausesPass();
67
68FunctionPass *createSIPostRABundlerPass();
69FunctionPass *createAMDGPUSimplifyLibCallsPass(const TargetMachine *);
70FunctionPass *createAMDGPUUseNativeCallsPass();
71FunctionPass *createAMDGPUCodeGenPreparePass();
72FunctionPass *createAMDGPUMachineCFGStructurizerPass();
73FunctionPass *createAMDGPUPropagateAttributesEarlyPass(const TargetMachine *);
74ModulePass *createAMDGPUPropagateAttributesLatePass(const TargetMachine *);
75FunctionPass *createAMDGPURewriteOutArgumentsPass();
76FunctionPass *createSIModeRegisterPass();
77
78void initializeAMDGPUDAGToDAGISelPass(PassRegistry&);
79
80void initializeAMDGPUMachineCFGStructurizerPass(PassRegistry&);
81extern char &AMDGPUMachineCFGStructurizerID;
82
83void initializeAMDGPUAlwaysInlinePass(PassRegistry&);
84
85Pass *createAMDGPUAnnotateKernelFeaturesPass();
86void initializeAMDGPUAnnotateKernelFeaturesPass(PassRegistry &);
87extern char &AMDGPUAnnotateKernelFeaturesID;
88
89FunctionPass *createAMDGPUAtomicOptimizerPass();
90void initializeAMDGPUAtomicOptimizerPass(PassRegistry &);
91extern char &AMDGPUAtomicOptimizerID;
92
93ModulePass *createAMDGPULowerIntrinsicsPass();
94void initializeAMDGPULowerIntrinsicsPass(PassRegistry &);
95extern char &AMDGPULowerIntrinsicsID;
96
97ModulePass *createAMDGPUFixFunctionBitcastsPass();
98void initializeAMDGPUFixFunctionBitcastsPass(PassRegistry &);
99extern char &AMDGPUFixFunctionBitcastsID;
100
101FunctionPass *createAMDGPULowerKernelArgumentsPass();
102void initializeAMDGPULowerKernelArgumentsPass(PassRegistry &);
103extern char &AMDGPULowerKernelArgumentsID;
104
105ModulePass *createAMDGPULowerKernelAttributesPass();
106void initializeAMDGPULowerKernelAttributesPass(PassRegistry &);
107extern char &AMDGPULowerKernelAttributesID;
108
109void initializeAMDGPUPropagateAttributesEarlyPass(PassRegistry &);
110extern char &AMDGPUPropagateAttributesEarlyID;
111
112void initializeAMDGPUPropagateAttributesLatePass(PassRegistry &);
113extern char &AMDGPUPropagateAttributesLateID;
114
115void initializeAMDGPURewriteOutArgumentsPass(PassRegistry &);
116extern char &AMDGPURewriteOutArgumentsID;
117
118void initializeGCNDPPCombinePass(PassRegistry &);
119extern char &GCNDPPCombineID;
120
121void initializeR600ClauseMergePassPass(PassRegistry &);
122extern char &R600ClauseMergePassID;
123
124void initializeR600ControlFlowFinalizerPass(PassRegistry &);
125extern char &R600ControlFlowFinalizerID;
126
127void initializeR600ExpandSpecialInstrsPassPass(PassRegistry &);
128extern char &R600ExpandSpecialInstrsPassID;
129
130void initializeR600VectorRegMergerPass(PassRegistry &);
131extern char &R600VectorRegMergerID;
132
133void initializeR600PacketizerPass(PassRegistry &);
134extern char &R600PacketizerID;
135
136void initializeSIFoldOperandsPass(PassRegistry &);
137extern char &SIFoldOperandsID;
138
139void initializeSIPeepholeSDWAPass(PassRegistry &);
140extern char &SIPeepholeSDWAID;
141
142void initializeSIShrinkInstructionsPass(PassRegistry&);
143extern char &SIShrinkInstructionsID;
144
145void initializeSIFixSGPRCopiesPass(PassRegistry &);
146extern char &SIFixSGPRCopiesID;
147
148void initializeSIFixVGPRCopiesPass(PassRegistry &);
149extern char &SIFixVGPRCopiesID;
150
151void initializeSIFixupVectorISelPass(PassRegistry &);
152extern char &SIFixupVectorISelID;
153
154void initializeSILowerI1CopiesPass(PassRegistry &);
155extern char &SILowerI1CopiesID;
156
157void initializeSILowerSGPRSpillsPass(PassRegistry &);
158extern char &SILowerSGPRSpillsID;
159
160void initializeSILoadStoreOptimizerPass(PassRegistry &);
161extern char &SILoadStoreOptimizerID;
162
163void initializeSIWholeQuadModePass(PassRegistry &);
164extern char &SIWholeQuadModeID;
165
166void initializeSILowerControlFlowPass(PassRegistry &);
167extern char &SILowerControlFlowID;
168
169void initializeSIRemoveShortExecBranchesPass(PassRegistry &);
170extern char &SIRemoveShortExecBranchesID;
171
172void initializeSIPreEmitPeepholePass(PassRegistry &);
173extern char &SIPreEmitPeepholeID;
174
175void initializeSIInsertSkipsPass(PassRegistry &);
176extern char &SIInsertSkipsPassID;
177
178void initializeSIOptimizeExecMaskingPass(PassRegistry &);
179extern char &SIOptimizeExecMaskingID;
180
181void initializeSIPreAllocateWWMRegsPass(PassRegistry &);
182extern char &SIPreAllocateWWMRegsID;
183
184void initializeAMDGPUSimplifyLibCallsPass(PassRegistry &);
185extern char &AMDGPUSimplifyLibCallsID;
186
187void initializeAMDGPUUseNativeCallsPass(PassRegistry &);
188extern char &AMDGPUUseNativeCallsID;
189
190void initializeSIAddIMGInitPass(PassRegistry &);
191extern char &SIAddIMGInitID;
192
193void initializeAMDGPUPerfHintAnalysisPass(PassRegistry &);
194extern char &AMDGPUPerfHintAnalysisID;
195
196// Passes common to R600 and SI
197FunctionPass *createAMDGPUPromoteAlloca();
198void initializeAMDGPUPromoteAllocaPass(PassRegistry&);
199extern char &AMDGPUPromoteAllocaID;
200
201FunctionPass *createAMDGPUPromoteAllocaToVector();
202void initializeAMDGPUPromoteAllocaToVectorPass(PassRegistry&);
203extern char &AMDGPUPromoteAllocaToVectorID;
204
205Pass *createAMDGPUStructurizeCFGPass();
206FunctionPass *createAMDGPUISelDag(
207  TargetMachine *TM = nullptr,
208  CodeGenOpt::Level OptLevel = CodeGenOpt::Default);
209ModulePass *createAMDGPUAlwaysInlinePass(bool GlobalOpt = true);
210ModulePass *createR600OpenCLImageTypeLoweringPass();
211FunctionPass *createAMDGPUAnnotateUniformValues();
212
213ModulePass *createAMDGPUPrintfRuntimeBinding();
214void initializeAMDGPUPrintfRuntimeBindingPass(PassRegistry&);
215extern char &AMDGPUPrintfRuntimeBindingID;
216
217ModulePass* createAMDGPUUnifyMetadataPass();
218void initializeAMDGPUUnifyMetadataPass(PassRegistry&);
219extern char &AMDGPUUnifyMetadataID;
220
221void initializeSIOptimizeExecMaskingPreRAPass(PassRegistry&);
222extern char &SIOptimizeExecMaskingPreRAID;
223
224void initializeAMDGPUAnnotateUniformValuesPass(PassRegistry&);
225extern char &AMDGPUAnnotateUniformValuesPassID;
226
227void initializeAMDGPUCodeGenPreparePass(PassRegistry&);
228extern char &AMDGPUCodeGenPrepareID;
229
230void initializeSIAnnotateControlFlowPass(PassRegistry&);
231extern char &SIAnnotateControlFlowPassID;
232
233void initializeSIMemoryLegalizerPass(PassRegistry&);
234extern char &SIMemoryLegalizerID;
235
236void initializeSIModeRegisterPass(PassRegistry&);
237extern char &SIModeRegisterID;
238
239void initializeSIInsertHardClausesPass(PassRegistry &);
240extern char &SIInsertHardClausesID;
241
242void initializeSIInsertWaitcntsPass(PassRegistry&);
243extern char &SIInsertWaitcntsID;
244
245void initializeSIFormMemoryClausesPass(PassRegistry&);
246extern char &SIFormMemoryClausesID;
247
248void initializeSIPostRABundlerPass(PassRegistry&);
249extern char &SIPostRABundlerID;
250
251void initializeAMDGPUUnifyDivergentExitNodesPass(PassRegistry&);
252extern char &AMDGPUUnifyDivergentExitNodesID;
253
254ImmutablePass *createAMDGPUAAWrapperPass();
255void initializeAMDGPUAAWrapperPassPass(PassRegistry&);
256ImmutablePass *createAMDGPUExternalAAWrapperPass();
257void initializeAMDGPUExternalAAWrapperPass(PassRegistry&);
258
259void initializeAMDGPUArgumentUsageInfoPass(PassRegistry &);
260
261Pass *createAMDGPUFunctionInliningPass();
262void initializeAMDGPUInlinerPass(PassRegistry&);
263
264ModulePass *createAMDGPUOpenCLEnqueuedBlockLoweringPass();
265void initializeAMDGPUOpenCLEnqueuedBlockLoweringPass(PassRegistry &);
266extern char &AMDGPUOpenCLEnqueuedBlockLoweringID;
267
268void initializeGCNRegBankReassignPass(PassRegistry &);
269extern char &GCNRegBankReassignID;
270
271void initializeGCNNSAReassignPass(PassRegistry &);
272extern char &GCNNSAReassignID;
273
274namespace AMDGPU {
275enum TargetIndex {
276  TI_CONSTDATA_START,
277  TI_SCRATCH_RSRC_DWORD0,
278  TI_SCRATCH_RSRC_DWORD1,
279  TI_SCRATCH_RSRC_DWORD2,
280  TI_SCRATCH_RSRC_DWORD3
281};
282}
283
284} // End namespace llvm
285
286/// OpenCL uses address spaces to differentiate between
287/// various memory regions on the hardware. On the CPU
288/// all of the address spaces point to the same memory,
289/// however on the GPU, each address space points to
290/// a separate piece of memory that is unique from other
291/// memory locations.
292namespace AMDGPUAS {
293  enum : unsigned {
294    // The maximum value for flat, generic, local, private, constant and region.
295    MAX_AMDGPU_ADDRESS = 7,
296
297    FLAT_ADDRESS = 0,     ///< Address space for flat memory.
298    GLOBAL_ADDRESS = 1,   ///< Address space for global memory (RAT0, VTX0).
299    REGION_ADDRESS = 2,   ///< Address space for region memory. (GDS)
300
301    CONSTANT_ADDRESS = 4, ///< Address space for constant memory (VTX2).
302    LOCAL_ADDRESS = 3,    ///< Address space for local memory.
303    PRIVATE_ADDRESS = 5,  ///< Address space for private memory.
304
305    CONSTANT_ADDRESS_32BIT = 6, ///< Address space for 32-bit constant memory.
306
307    BUFFER_FAT_POINTER = 7, ///< Address space for 160-bit buffer fat pointers.
308
309    /// Address space for direct addressible parameter memory (CONST0).
310    PARAM_D_ADDRESS = 6,
311    /// Address space for indirect addressible parameter memory (VTX1).
312    PARAM_I_ADDRESS = 7,
313
314    // Do not re-order the CONSTANT_BUFFER_* enums.  Several places depend on
315    // this order to be able to dynamically index a constant buffer, for
316    // example:
317    //
318    // ConstantBufferAS = CONSTANT_BUFFER_0 + CBIdx
319
320    CONSTANT_BUFFER_0 = 8,
321    CONSTANT_BUFFER_1 = 9,
322    CONSTANT_BUFFER_2 = 10,
323    CONSTANT_BUFFER_3 = 11,
324    CONSTANT_BUFFER_4 = 12,
325    CONSTANT_BUFFER_5 = 13,
326    CONSTANT_BUFFER_6 = 14,
327    CONSTANT_BUFFER_7 = 15,
328    CONSTANT_BUFFER_8 = 16,
329    CONSTANT_BUFFER_9 = 17,
330    CONSTANT_BUFFER_10 = 18,
331    CONSTANT_BUFFER_11 = 19,
332    CONSTANT_BUFFER_12 = 20,
333    CONSTANT_BUFFER_13 = 21,
334    CONSTANT_BUFFER_14 = 22,
335    CONSTANT_BUFFER_15 = 23,
336
337    // Some places use this if the address space can't be determined.
338    UNKNOWN_ADDRESS_SPACE = ~0u,
339  };
340}
341
342#endif
343