libgomp/plugin/plugin-gcn.c

200 /* Structure describing the run-time and grid properties of an HSA kernel
213 /* Collection of information needed for a dispatch of a kernel from a
214    kernel.  */
219   /* Pointer to a command queue associated with a kernel dispatch agent.  */
221   /* Pointer to a memory space used for kernel arguments passing.  */
239   /* Leave space for the real kernel arguments.
276   struct kernel_info *kernel;
362 /* Mkoffload uses this structure to describe a kernel.
364    OpenMP kernel dimensions are passed at runtime.
365    OpenACC kernel dimensions are passed at compile time, here.  */
435   /* The HSA memory region from which to allocate kernel arguments.  */
447      locked for reading while a kernel is being run, and for writing if the
451   /* The module associated with this kernel.  */
465 /* Information required to identify, finalize and run any given kernel.  */
471   /* Name of the kernel, required to locate it within the GCN object-code
474   /* The specific agent the kernel has been or will be finalized for and run
477   /* The specific module where the kernel takes place.  */
479   /* Information provided by mkoffload associated with the kernel.  */
481   /* Mutex enforcing that at most once thread ever initializes a kernel for
485   /* Flag indicating whether the kernel has been initialized and all fields
488   /* Flag indicating that the kernel has a problem that blocks an execution.  */
492   /* Required size of kernel arguments.  */
519   /* An array of kernel_info structures describing each kernel in this
536      The mutex is locked until the kernel exits.  */
1024 /* Dump kernel DISPATCH data structure and indent it by INDENT spaces.  */
1152    This depends on the kernel's resource usage levels.  */
1697    The arena is released, but not deallocated, when the kernel exits.  */
1750     hsa_fatal ("Could not allocate memory for GCN kernel arena", status);
1810      kernargs for kernels that have no inputs or outputs (the kernel may
1864 /* Create kernel dispatch data structure for given KERNEL, along with
1868 create_kernel_dispatch (struct kernel_info *kernel, int num_teams)
1870   struct agent_info *agent = kernel->agent;
1874   shadow->agent = kernel->agent;
1875   shadow->object = kernel->object;
1883   shadow->private_segment_size = kernel->private_segment_size;
1884   shadow->group_segment_size = kernel->group_segment_size;
1889   if (kernel->kernarg_segment_size > 8)
1899     hsa_fatal ("Could not allocate memory for GCN kernel arguments", status);
1913   kernargs->heap_ptr = (int64_t)kernel->module->heap;
1916   if (kernel->kind == KIND_OPENMP)
1927 /* Output any data written to console output from the kernel.  It is expected
1928    that this function is polled during kernel execution.
1939 console_output (struct kernel_info *kernel, struct kernargs *kernargs,
1980 /* Release data structure created for a kernel dispatch in SHADOW argument,
1986   GCN_DEBUG ("Released kernel dispatch: %p\n", shadow);
2002 /* Extract the properties from a kernel binary.  */
2005 init_kernel_properties (struct kernel_info *kernel)
2008   struct agent_info *agent = kernel->agent;
2010   char *buf = alloca (strlen (kernel->name) + 4);
2011   sprintf (buf, "%s.kd", kernel->name);
2017       hsa_warn ("Could not find symbol for kernel in the code object", status);
2022   GCN_DEBUG ("Located kernel %s\n", kernel->name);
2024     (kernel_symbol, HSA_EXECUTABLE_SYMBOL_INFO_KERNEL_OBJECT, &kernel->object);
2026     hsa_fatal ("Could not extract a kernel object from its symbol", status);
2029      &kernel->kernarg_segment_size);
2031     hsa_fatal ("Could not get info about kernel argument size", status);
2034      &kernel->group_segment_size);
2036     hsa_fatal ("Could not get info about kernel group segment size", status);
2039      &kernel->private_segment_size);
2041     hsa_fatal ("Could not get info about kernel private segment size",
2044   /* The kernel type is not known until something tries to launch it.  */
2045   kernel->kind = KIND_UNKNOWN;
2048 	     "following segment sizes: \n", kernel->name);
2050 	     (unsigned) kernel->group_segment_size);
2052 	     (unsigned) kernel->private_segment_size);
2054 	     (unsigned) kernel->kernarg_segment_size);
2058   kernel->initialization_failed = true;
2066 init_kernel (struct kernel_info *kernel)
2068   if (pthread_mutex_lock (&kernel->init_mutex))
2069     GOMP_PLUGIN_fatal ("Could not lock a GCN kernel initialization mutex");
2070   if (kernel->initialized)
2072       if (pthread_mutex_unlock (&kernel->init_mutex))
2073 	GOMP_PLUGIN_fatal ("Could not unlock a GCN kernel initialization "
2079   init_kernel_properties (kernel);
2081   if (!kernel->initialization_failed)
2085       kernel->initialized = true;
2087   if (pthread_mutex_unlock (&kernel->init_mutex))
2088     GOMP_PLUGIN_fatal ("Could not unlock a GCN kernel initialization "
2100 run_kernel (struct kernel_info *kernel, void *vars,
2104   GCN_DEBUG ("SGPRs: %d, VGPRs: %d\n", kernel->description->sgpr_count,
2105 	     kernel->description->vpgr_count);
2109   if (kla->ndim == 3 && kernel->description->vpgr_count > 0)
2111       int granulated_vgprs = (kernel->description->vpgr_count + 3) & ~3;
2122   GCN_DEBUG ("GCN launch on queue: %d:%d\n", kernel->agent->device_id,
2149   struct agent_info *agent = kernel->agent;
2156   if (!kernel->initialized)
2157     GOMP_PLUGIN_fatal ("Called kernel must be initialized");
2159   hsa_queue_t *command_q = (aq ? aq->hsa_queue : kernel->agent->sync_queue);
2172   int override_x = kernel->kind == KIND_UNKNOWN ? 0 : override_x_dim;
2173   int override_z = kernel->kind == KIND_UNKNOWN ? 0 : override_z_dim;
2221     = create_kernel_dispatch (kernel, packet->grid_size_x);
2230   packet->private_segment_size = kernel->private_segment_size;
2231   packet->group_segment_size = kernel->group_segment_size;
2232   packet->kernel_object = kernel->object;
2240   GCN_DEBUG ("Copying kernel runtime pointer to kernarg_address\n");
2247   GCN_DEBUG ("Going to dispatch kernel %s on device %d\n", kernel->name,
2264       console_output (kernel, shadow->kernarg_address, false);
2266   console_output (kernel, shadow->kernarg_address, true);
2283       GOMP_PLUGIN_error ("Possible kernel exit value corruption, 2 most"
2315 init_basic_kernel_info (struct kernel_info *kernel,
2320   kernel->agent = agent;
2321   kernel->module = module;
2322   kernel->name = d->name;
2323   kernel->description = d;
2324   if (pthread_mutex_init (&kernel->init_mutex, NULL))
2326       GOMP_PLUGIN_error ("Failed to initialize a GCN kernel mutex");
2530 	GOMP_PLUGIN_error ("Failed to destroy a GCN kernel initialization "
2554       run_kernel (entry->u.launch.kernel,
2712 /* Request an asynchronous kernel launch on the specified queue.  This
2714    kernel to run.  */
2717 queue_push_launch (struct goacc_asyncqueue *aq, struct kernel_info *kernel,
2720   assert (aq->agent == kernel->agent);
2733   aq->queue[queue_last].u.launch.kernel = kernel;
3022 /* Execute an OpenACC kernel, synchronously or asynchronously.  */
3025 gcn_exec (struct kernel_info *kernel, size_t mapnum, void **hostaddrs,
3029   if (!GOMP_OFFLOAD_can_run (kernel))
3032   /* If we get here then this must be an OpenACC kernel.  */
3033   kernel->kind = KIND_OPENACC;
3036   void **ind_da = alloc_by_agent (kernel->agent, sizeof (void*) * mapnum);
3043   for (unsigned i = 0; i < kernel->module->image_desc->kernel_count; i++)
3046 	= &kernel->module->image_desc->kernel_infos[i];
3047       if (d->name == kernel->name)
3055      hsa_kernel_desc->oacc_dims[] or dimensions passed to this offload kernel
3076       dims[0] = get_cu_count (kernel->agent) * 4; /* Gangs.  */
3082       dims[0] = get_cu_count (kernel->agent) * (32 / dims[1]);
3087       dims[1] = get_cu_count (kernel->agent) * 32 / dims[0];
3141 	= (char *) kernel->name;
3154       run_kernel (kernel, ind_da, &kla, NULL, false);
3159       queue_push_launch (aq, kernel, ind_da, &kla);
3307       GOMP_PLUGIN_error ("Could not find suitable memory region for kernel "
3311   GCN_DEBUG ("Selected kernel arguments memory region:\n");
3337    TARGET_DATA and return references to kernel descriptors in TARGET_TABLE.
3356   struct kernel_info *kernel;
3388   kernel = &module->kernels[0];
3390   /* Allocate memory for kernel dependencies.  */
3394       if (!init_basic_kernel_info (kernel, d, agent, module))
3397 	module->init_array_func = kernel;
3399         module->fini_array_func = kernel;
3402 	  pair->start = (uintptr_t) kernel;
3403 	  pair->end = (uintptr_t) (kernel + 1);
3406       kernel++;
3627   struct kernel_info *kernel = (struct kernel_info *) fn_ptr;
3629   init_kernel (kernel);
3630   if (kernel->initialization_failed)
3747 /* Run a synchronous OpenMP kernel on DEVICE and pass it an array of pointers
3748    in VARS as a parameter.  The kernel is identified by FN_PTR which must point
3756   struct kernel_info *kernel = (struct kernel_info *) fn_ptr;
3759   assert (agent == kernel->agent);
3761   /* If we get here then the kernel must be OpenMP.  */
3762   kernel->kind = KIND_OPENMP;
3766       GCN_WARNING ("Will not run GCN kernel because the grid size is zero\n");
3769   run_kernel (kernel, vars, kla, NULL, false);
3772 /* Run an asynchronous OpenMP kernel on DEVICE.  This is similar to
3782   struct kernel_info *kernel = (struct kernel_info *) tgt_fn;
3785   assert (agent == kernel->agent);
3787   /* If we get here then the kernel must be OpenMP.  */
3788   kernel->kind = KIND_OPENMP;
3792       GCN_WARNING ("Will not run GCN kernel because the grid size is zero\n");
3797   queue_push_launch (agent->omp_async_queue, kernel, tgt_vars, kla);
3805 /* Run a synchronous OpenACC kernel.  The device number is inferred from the
3813   struct kernel_info *kernel = (struct kernel_info *) fn_ptr;
3815   gcn_exec (kernel, mapnum, hostaddrs, devaddrs, dims, targ_mem_desc, false,
3819 /* Run an asynchronous OpenACC kernel on the specified queue.  */
3827   struct kernel_info *kernel = (struct kernel_info *) fn_ptr;
3829   gcn_exec (kernel, mapnum, hostaddrs, devaddrs, dims, targ_mem_desc, true,