1//===----- CGOpenMPRuntimeNVPTX.h - Interface to OpenMP NVPTX Runtimes ----===// 2// 3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4// See https://llvm.org/LICENSE.txt for license information. 5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6// 7//===----------------------------------------------------------------------===// 8// 9// This provides a class for OpenMP runtime code generation specialized to NVPTX 10// targets. 11// 12//===----------------------------------------------------------------------===// 13 14#ifndef LLVM_CLANG_LIB_CODEGEN_CGOPENMPRUNTIMENVPTX_H 15#define LLVM_CLANG_LIB_CODEGEN_CGOPENMPRUNTIMENVPTX_H 16 17#include "CGOpenMPRuntime.h" 18#include "CodeGenFunction.h" 19#include "clang/AST/StmtOpenMP.h" 20 21namespace clang { 22namespace CodeGen { 23 24class CGOpenMPRuntimeNVPTX : public CGOpenMPRuntime { 25public: 26 /// Defines the execution mode. 27 enum ExecutionMode { 28 /// SPMD execution mode (all threads are worker threads). 29 EM_SPMD, 30 /// Non-SPMD execution mode (1 master thread, others are workers). 31 EM_NonSPMD, 32 /// Unknown execution mode (orphaned directive). 33 EM_Unknown, 34 }; 35private: 36 /// Parallel outlined function work for workers to execute. 37 llvm::SmallVector<llvm::Function *, 16> Work; 38 39 struct EntryFunctionState { 40 llvm::BasicBlock *ExitBB = nullptr; 41 }; 42 43 class WorkerFunctionState { 44 public: 45 llvm::Function *WorkerFn; 46 const CGFunctionInfo &CGFI; 47 SourceLocation Loc; 48 49 WorkerFunctionState(CodeGenModule &CGM, SourceLocation Loc); 50 51 private: 52 void createWorkerFunction(CodeGenModule &CGM); 53 }; 54 55 ExecutionMode getExecutionMode() const; 56 57 bool requiresFullRuntime() const { return RequiresFullRuntime; } 58 59 /// Get barrier to synchronize all threads in a block. 60 void syncCTAThreads(CodeGenFunction &CGF); 61 62 /// Emit the worker function for the current target region. 63 void emitWorkerFunction(WorkerFunctionState &WST); 64 65 /// Helper for worker function. Emit body of worker loop. 66 void emitWorkerLoop(CodeGenFunction &CGF, WorkerFunctionState &WST); 67 68 /// Helper for non-SPMD target entry function. Guide the master and 69 /// worker threads to their respective locations. 70 void emitNonSPMDEntryHeader(CodeGenFunction &CGF, EntryFunctionState &EST, 71 WorkerFunctionState &WST); 72 73 /// Signal termination of OMP execution for non-SPMD target entry 74 /// function. 75 void emitNonSPMDEntryFooter(CodeGenFunction &CGF, EntryFunctionState &EST); 76 77 /// Helper for generic variables globalization prolog. 78 void emitGenericVarsProlog(CodeGenFunction &CGF, SourceLocation Loc, 79 bool WithSPMDCheck = false); 80 81 /// Helper for generic variables globalization epilog. 82 void emitGenericVarsEpilog(CodeGenFunction &CGF, bool WithSPMDCheck = false); 83 84 /// Helper for SPMD mode target directive's entry function. 85 void emitSPMDEntryHeader(CodeGenFunction &CGF, EntryFunctionState &EST, 86 const OMPExecutableDirective &D); 87 88 /// Signal termination of SPMD mode execution. 89 void emitSPMDEntryFooter(CodeGenFunction &CGF, EntryFunctionState &EST); 90 91 // 92 // Base class overrides. 93 // 94 95 /// Creates offloading entry for the provided entry ID \a ID, 96 /// address \a Addr, size \a Size, and flags \a Flags. 97 void createOffloadEntry(llvm::Constant *ID, llvm::Constant *Addr, 98 uint64_t Size, int32_t Flags, 99 llvm::GlobalValue::LinkageTypes Linkage) override; 100 101 /// Emit outlined function specialized for the Fork-Join 102 /// programming model for applicable target directives on the NVPTX device. 103 /// \param D Directive to emit. 104 /// \param ParentName Name of the function that encloses the target region. 105 /// \param OutlinedFn Outlined function value to be defined by this call. 106 /// \param OutlinedFnID Outlined function ID value to be defined by this call. 107 /// \param IsOffloadEntry True if the outlined function is an offload entry. 108 /// An outlined function may not be an entry if, e.g. the if clause always 109 /// evaluates to false. 110 void emitNonSPMDKernel(const OMPExecutableDirective &D, StringRef ParentName, 111 llvm::Function *&OutlinedFn, 112 llvm::Constant *&OutlinedFnID, bool IsOffloadEntry, 113 const RegionCodeGenTy &CodeGen); 114 115 /// Emit outlined function specialized for the Single Program 116 /// Multiple Data programming model for applicable target directives on the 117 /// NVPTX device. 118 /// \param D Directive to emit. 119 /// \param ParentName Name of the function that encloses the target region. 120 /// \param OutlinedFn Outlined function value to be defined by this call. 121 /// \param OutlinedFnID Outlined function ID value to be defined by this call. 122 /// \param IsOffloadEntry True if the outlined function is an offload entry. 123 /// \param CodeGen Object containing the target statements. 124 /// An outlined function may not be an entry if, e.g. the if clause always 125 /// evaluates to false. 126 void emitSPMDKernel(const OMPExecutableDirective &D, StringRef ParentName, 127 llvm::Function *&OutlinedFn, 128 llvm::Constant *&OutlinedFnID, bool IsOffloadEntry, 129 const RegionCodeGenTy &CodeGen); 130 131 /// Emit outlined function for 'target' directive on the NVPTX 132 /// device. 133 /// \param D Directive to emit. 134 /// \param ParentName Name of the function that encloses the target region. 135 /// \param OutlinedFn Outlined function value to be defined by this call. 136 /// \param OutlinedFnID Outlined function ID value to be defined by this call. 137 /// \param IsOffloadEntry True if the outlined function is an offload entry. 138 /// An outlined function may not be an entry if, e.g. the if clause always 139 /// evaluates to false. 140 void emitTargetOutlinedFunction(const OMPExecutableDirective &D, 141 StringRef ParentName, 142 llvm::Function *&OutlinedFn, 143 llvm::Constant *&OutlinedFnID, 144 bool IsOffloadEntry, 145 const RegionCodeGenTy &CodeGen) override; 146 147 /// Emits code for parallel or serial call of the \a OutlinedFn with 148 /// variables captured in a record which address is stored in \a 149 /// CapturedStruct. 150 /// This call is for the Non-SPMD Execution Mode. 151 /// \param OutlinedFn Outlined function to be run in parallel threads. Type of 152 /// this function is void(*)(kmp_int32 *, kmp_int32, struct context_vars*). 153 /// \param CapturedVars A pointer to the record with the references to 154 /// variables used in \a OutlinedFn function. 155 /// \param IfCond Condition in the associated 'if' clause, if it was 156 /// specified, nullptr otherwise. 157 void emitNonSPMDParallelCall(CodeGenFunction &CGF, SourceLocation Loc, 158 llvm::Value *OutlinedFn, 159 ArrayRef<llvm::Value *> CapturedVars, 160 const Expr *IfCond); 161 162 /// Emits code for parallel or serial call of the \a OutlinedFn with 163 /// variables captured in a record which address is stored in \a 164 /// CapturedStruct. 165 /// This call is for a parallel directive within an SPMD target directive. 166 /// \param OutlinedFn Outlined function to be run in parallel threads. Type of 167 /// this function is void(*)(kmp_int32 *, kmp_int32, struct context_vars*). 168 /// \param CapturedVars A pointer to the record with the references to 169 /// variables used in \a OutlinedFn function. 170 /// \param IfCond Condition in the associated 'if' clause, if it was 171 /// specified, nullptr otherwise. 172 /// 173 void emitSPMDParallelCall(CodeGenFunction &CGF, SourceLocation Loc, 174 llvm::Function *OutlinedFn, 175 ArrayRef<llvm::Value *> CapturedVars, 176 const Expr *IfCond); 177 178protected: 179 /// Get the function name of an outlined region. 180 // The name can be customized depending on the target. 181 // 182 StringRef getOutlinedHelperName() const override { 183 return "__omp_outlined__"; 184 } 185 186 /// Check if the default location must be constant. 187 /// Constant for NVPTX for better optimization. 188 bool isDefaultLocationConstant() const override { return true; } 189 190 /// Returns additional flags that can be stored in reserved_2 field of the 191 /// default location. 192 /// For NVPTX target contains data about SPMD/Non-SPMD execution mode + 193 /// Full/Lightweight runtime mode. Used for better optimization. 194 unsigned getDefaultLocationReserved2Flags() const override; 195 196 /// Tries to emit declare variant function for \p OldGD from \p NewGD. 197 /// \param OrigAddr LLVM IR value for \p OldGD. 198 /// \param IsForDefinition true, if requested emission for the definition of 199 /// \p OldGD. 200 /// \returns true, was able to emit a definition function for \p OldGD, which 201 /// points to \p NewGD. 202 /// NVPTX backend does not support global aliases, so just use the function, 203 /// emitted for \p NewGD instead of \p OldGD. 204 bool tryEmitDeclareVariant(const GlobalDecl &NewGD, const GlobalDecl &OldGD, 205 llvm::GlobalValue *OrigAddr, 206 bool IsForDefinition) override; 207 208public: 209 explicit CGOpenMPRuntimeNVPTX(CodeGenModule &CGM); 210 void clear() override; 211 212 /// Emit call to void __kmpc_push_proc_bind(ident_t *loc, kmp_int32 213 /// global_tid, int proc_bind) to generate code for 'proc_bind' clause. 214 virtual void emitProcBindClause(CodeGenFunction &CGF, 215 llvm::omp::ProcBindKind ProcBind, 216 SourceLocation Loc) override; 217 218 /// Emits call to void __kmpc_push_num_threads(ident_t *loc, kmp_int32 219 /// global_tid, kmp_int32 num_threads) to generate code for 'num_threads' 220 /// clause. 221 /// \param NumThreads An integer value of threads. 222 virtual void emitNumThreadsClause(CodeGenFunction &CGF, 223 llvm::Value *NumThreads, 224 SourceLocation Loc) override; 225 226 /// This function ought to emit, in the general case, a call to 227 // the openmp runtime kmpc_push_num_teams. In NVPTX backend it is not needed 228 // as these numbers are obtained through the PTX grid and block configuration. 229 /// \param NumTeams An integer expression of teams. 230 /// \param ThreadLimit An integer expression of threads. 231 void emitNumTeamsClause(CodeGenFunction &CGF, const Expr *NumTeams, 232 const Expr *ThreadLimit, SourceLocation Loc) override; 233 234 /// Emits inlined function for the specified OpenMP parallel 235 // directive. 236 /// \a D. This outlined function has type void(*)(kmp_int32 *ThreadID, 237 /// kmp_int32 BoundID, struct context_vars*). 238 /// \param D OpenMP directive. 239 /// \param ThreadIDVar Variable for thread id in the current OpenMP region. 240 /// \param InnermostKind Kind of innermost directive (for simple directives it 241 /// is a directive itself, for combined - its innermost directive). 242 /// \param CodeGen Code generation sequence for the \a D directive. 243 llvm::Function * 244 emitParallelOutlinedFunction(const OMPExecutableDirective &D, 245 const VarDecl *ThreadIDVar, 246 OpenMPDirectiveKind InnermostKind, 247 const RegionCodeGenTy &CodeGen) override; 248 249 /// Emits inlined function for the specified OpenMP teams 250 // directive. 251 /// \a D. This outlined function has type void(*)(kmp_int32 *ThreadID, 252 /// kmp_int32 BoundID, struct context_vars*). 253 /// \param D OpenMP directive. 254 /// \param ThreadIDVar Variable for thread id in the current OpenMP region. 255 /// \param InnermostKind Kind of innermost directive (for simple directives it 256 /// is a directive itself, for combined - its innermost directive). 257 /// \param CodeGen Code generation sequence for the \a D directive. 258 llvm::Function * 259 emitTeamsOutlinedFunction(const OMPExecutableDirective &D, 260 const VarDecl *ThreadIDVar, 261 OpenMPDirectiveKind InnermostKind, 262 const RegionCodeGenTy &CodeGen) override; 263 264 /// Emits code for teams call of the \a OutlinedFn with 265 /// variables captured in a record which address is stored in \a 266 /// CapturedStruct. 267 /// \param OutlinedFn Outlined function to be run by team masters. Type of 268 /// this function is void(*)(kmp_int32 *, kmp_int32, struct context_vars*). 269 /// \param CapturedVars A pointer to the record with the references to 270 /// variables used in \a OutlinedFn function. 271 /// 272 void emitTeamsCall(CodeGenFunction &CGF, const OMPExecutableDirective &D, 273 SourceLocation Loc, llvm::Function *OutlinedFn, 274 ArrayRef<llvm::Value *> CapturedVars) override; 275 276 /// Emits code for parallel or serial call of the \a OutlinedFn with 277 /// variables captured in a record which address is stored in \a 278 /// CapturedStruct. 279 /// \param OutlinedFn Outlined function to be run in parallel threads. Type of 280 /// this function is void(*)(kmp_int32 *, kmp_int32, struct context_vars*). 281 /// \param CapturedVars A pointer to the record with the references to 282 /// variables used in \a OutlinedFn function. 283 /// \param IfCond Condition in the associated 'if' clause, if it was 284 /// specified, nullptr otherwise. 285 void emitParallelCall(CodeGenFunction &CGF, SourceLocation Loc, 286 llvm::Function *OutlinedFn, 287 ArrayRef<llvm::Value *> CapturedVars, 288 const Expr *IfCond) override; 289 290 /// Emit an implicit/explicit barrier for OpenMP threads. 291 /// \param Kind Directive for which this implicit barrier call must be 292 /// generated. Must be OMPD_barrier for explicit barrier generation. 293 /// \param EmitChecks true if need to emit checks for cancellation barriers. 294 /// \param ForceSimpleCall true simple barrier call must be emitted, false if 295 /// runtime class decides which one to emit (simple or with cancellation 296 /// checks). 297 /// 298 void emitBarrierCall(CodeGenFunction &CGF, SourceLocation Loc, 299 OpenMPDirectiveKind Kind, bool EmitChecks = true, 300 bool ForceSimpleCall = false) override; 301 302 /// Emits a critical region. 303 /// \param CriticalName Name of the critical region. 304 /// \param CriticalOpGen Generator for the statement associated with the given 305 /// critical region. 306 /// \param Hint Value of the 'hint' clause (optional). 307 void emitCriticalRegion(CodeGenFunction &CGF, StringRef CriticalName, 308 const RegionCodeGenTy &CriticalOpGen, 309 SourceLocation Loc, 310 const Expr *Hint = nullptr) override; 311 312 /// Emit a code for reduction clause. 313 /// 314 /// \param Privates List of private copies for original reduction arguments. 315 /// \param LHSExprs List of LHS in \a ReductionOps reduction operations. 316 /// \param RHSExprs List of RHS in \a ReductionOps reduction operations. 317 /// \param ReductionOps List of reduction operations in form 'LHS binop RHS' 318 /// or 'operator binop(LHS, RHS)'. 319 /// \param Options List of options for reduction codegen: 320 /// WithNowait true if parent directive has also nowait clause, false 321 /// otherwise. 322 /// SimpleReduction Emit reduction operation only. Used for omp simd 323 /// directive on the host. 324 /// ReductionKind The kind of reduction to perform. 325 virtual void emitReduction(CodeGenFunction &CGF, SourceLocation Loc, 326 ArrayRef<const Expr *> Privates, 327 ArrayRef<const Expr *> LHSExprs, 328 ArrayRef<const Expr *> RHSExprs, 329 ArrayRef<const Expr *> ReductionOps, 330 ReductionOptionsTy Options) override; 331 332 /// Returns specified OpenMP runtime function for the current OpenMP 333 /// implementation. Specialized for the NVPTX device. 334 /// \param Function OpenMP runtime function. 335 /// \return Specified function. 336 llvm::FunctionCallee createNVPTXRuntimeFunction(unsigned Function); 337 338 /// Translates the native parameter of outlined function if this is required 339 /// for target. 340 /// \param FD Field decl from captured record for the parameter. 341 /// \param NativeParam Parameter itself. 342 const VarDecl *translateParameter(const FieldDecl *FD, 343 const VarDecl *NativeParam) const override; 344 345 /// Gets the address of the native argument basing on the address of the 346 /// target-specific parameter. 347 /// \param NativeParam Parameter itself. 348 /// \param TargetParam Corresponding target-specific parameter. 349 Address getParameterAddress(CodeGenFunction &CGF, const VarDecl *NativeParam, 350 const VarDecl *TargetParam) const override; 351 352 /// Emits call of the outlined function with the provided arguments, 353 /// translating these arguments to correct target-specific arguments. 354 void emitOutlinedFunctionCall( 355 CodeGenFunction &CGF, SourceLocation Loc, llvm::FunctionCallee OutlinedFn, 356 ArrayRef<llvm::Value *> Args = llvm::None) const override; 357 358 /// Emits OpenMP-specific function prolog. 359 /// Required for device constructs. 360 void emitFunctionProlog(CodeGenFunction &CGF, const Decl *D) override; 361 362 /// Gets the OpenMP-specific address of the local variable. 363 Address getAddressOfLocalVariable(CodeGenFunction &CGF, 364 const VarDecl *VD) override; 365 366 /// Target codegen is specialized based on two data-sharing modes: CUDA, in 367 /// which the local variables are actually global threadlocal, and Generic, in 368 /// which the local variables are placed in global memory if they may escape 369 /// their declaration context. 370 enum DataSharingMode { 371 /// CUDA data sharing mode. 372 CUDA, 373 /// Generic data-sharing mode. 374 Generic, 375 }; 376 377 /// Cleans up references to the objects in finished function. 378 /// 379 void functionFinished(CodeGenFunction &CGF) override; 380 381 /// Choose a default value for the dist_schedule clause. 382 void getDefaultDistScheduleAndChunk(CodeGenFunction &CGF, 383 const OMPLoopDirective &S, OpenMPDistScheduleClauseKind &ScheduleKind, 384 llvm::Value *&Chunk) const override; 385 386 /// Choose a default value for the schedule clause. 387 void getDefaultScheduleAndChunk(CodeGenFunction &CGF, 388 const OMPLoopDirective &S, OpenMPScheduleClauseKind &ScheduleKind, 389 const Expr *&ChunkExpr) const override; 390 391 /// Adjust some parameters for the target-based directives, like addresses of 392 /// the variables captured by reference in lambdas. 393 void adjustTargetSpecificDataForLambdas( 394 CodeGenFunction &CGF, const OMPExecutableDirective &D) const override; 395 396 /// Perform check on requires decl to ensure that target architecture 397 /// supports unified addressing 398 void checkArchForUnifiedAddressing(const OMPRequiresDecl *D) override; 399 400 /// Returns default address space for the constant firstprivates, __constant__ 401 /// address space by default. 402 unsigned getDefaultFirstprivateAddressSpace() const override; 403 404 /// Checks if the variable has associated OMPAllocateDeclAttr attribute with 405 /// the predefined allocator and translates it into the corresponding address 406 /// space. 407 bool hasAllocateAttributeForGlobalVar(const VarDecl *VD, LangAS &AS) override; 408 409private: 410 /// Track the execution mode when codegening directives within a target 411 /// region. The appropriate mode (SPMD/NON-SPMD) is set on entry to the 412 /// target region and used by containing directives such as 'parallel' 413 /// to emit optimized code. 414 ExecutionMode CurrentExecutionMode = EM_Unknown; 415 416 /// Check if the full runtime is required (default - yes). 417 bool RequiresFullRuntime = true; 418 419 /// true if we're emitting the code for the target region and next parallel 420 /// region is L0 for sure. 421 bool IsInTargetMasterThreadRegion = false; 422 /// true if currently emitting code for target/teams/distribute region, false 423 /// - otherwise. 424 bool IsInTTDRegion = false; 425 /// true if we're definitely in the parallel region. 426 bool IsInParallelRegion = false; 427 428 /// Map between an outlined function and its wrapper. 429 llvm::DenseMap<llvm::Function *, llvm::Function *> WrapperFunctionsMap; 430 431 /// Emit function which wraps the outline parallel region 432 /// and controls the parameters which are passed to this function. 433 /// The wrapper ensures that the outlined function is called 434 /// with the correct arguments when data is shared. 435 llvm::Function *createParallelDataSharingWrapper( 436 llvm::Function *OutlinedParallelFn, const OMPExecutableDirective &D); 437 438 /// The data for the single globalized variable. 439 struct MappedVarData { 440 /// Corresponding field in the global record. 441 const FieldDecl *FD = nullptr; 442 /// Corresponding address. 443 Address PrivateAddr = Address::invalid(); 444 /// true, if only one element is required (for latprivates in SPMD mode), 445 /// false, if need to create based on the warp-size. 446 bool IsOnePerTeam = false; 447 MappedVarData() = delete; 448 MappedVarData(const FieldDecl *FD, bool IsOnePerTeam = false) 449 : FD(FD), IsOnePerTeam(IsOnePerTeam) {} 450 }; 451 /// The map of local variables to their addresses in the global memory. 452 using DeclToAddrMapTy = llvm::MapVector<const Decl *, MappedVarData>; 453 /// Set of the parameters passed by value escaping OpenMP context. 454 using EscapedParamsTy = llvm::SmallPtrSet<const Decl *, 4>; 455 struct FunctionData { 456 DeclToAddrMapTy LocalVarData; 457 llvm::Optional<DeclToAddrMapTy> SecondaryLocalVarData = llvm::None; 458 EscapedParamsTy EscapedParameters; 459 llvm::SmallVector<const ValueDecl*, 4> EscapedVariableLengthDecls; 460 llvm::SmallVector<llvm::Value *, 4> EscapedVariableLengthDeclsAddrs; 461 const RecordDecl *GlobalRecord = nullptr; 462 llvm::Optional<const RecordDecl *> SecondaryGlobalRecord = llvm::None; 463 llvm::Value *GlobalRecordAddr = nullptr; 464 llvm::Value *IsInSPMDModeFlag = nullptr; 465 std::unique_ptr<CodeGenFunction::OMPMapVars> MappedParams; 466 }; 467 /// Maps the function to the list of the globalized variables with their 468 /// addresses. 469 llvm::SmallDenseMap<llvm::Function *, FunctionData> FunctionGlobalizedDecls; 470 /// List of records for the globalized variables in target/teams/distribute 471 /// contexts. Inner records are going to be joined into the single record, 472 /// while those resulting records are going to be joined into the single 473 /// union. This resulting union (one per CU) is the entry point for the static 474 /// memory management runtime functions. 475 struct GlobalPtrSizeRecsTy { 476 llvm::GlobalVariable *UseSharedMemory = nullptr; 477 llvm::GlobalVariable *RecSize = nullptr; 478 llvm::GlobalVariable *Buffer = nullptr; 479 SourceLocation Loc; 480 llvm::SmallVector<const RecordDecl *, 2> Records; 481 unsigned RegionCounter = 0; 482 }; 483 llvm::SmallVector<GlobalPtrSizeRecsTy, 8> GlobalizedRecords; 484 llvm::GlobalVariable *KernelTeamsReductionPtr = nullptr; 485 /// List of the records with the list of fields for the reductions across the 486 /// teams. Used to build the intermediate buffer for the fast teams 487 /// reductions. 488 /// All the records are gathered into a union `union.type` is created. 489 llvm::SmallVector<const RecordDecl *, 4> TeamsReductions; 490 /// Shared pointer for the global memory in the global memory buffer used for 491 /// the given kernel. 492 llvm::GlobalVariable *KernelStaticGlobalized = nullptr; 493 /// Pair of the Non-SPMD team and all reductions variables in this team 494 /// region. 495 std::pair<const Decl *, llvm::SmallVector<const ValueDecl *, 4>> 496 TeamAndReductions; 497}; 498 499} // CodeGen namespace. 500} // clang namespace. 501 502#endif // LLVM_CLANG_LIB_CODEGEN_CGOPENMPRUNTIMENVPTX_H 503