1/*
2 * Copyright (c) 2014 ETH Zurich.
3 * All rights reserved.
4 *
5 * This file is distributed under the terms in the attached LICENSE file.
6 * If you do not find this file, copies can be found by writing to:
7 * ETH Zurich D-INFK, Universitaetsstrasse 64, CH-8092 Zurich. Attn: Systems Group.
8 */
9
10#ifndef __LIBBOMP_INTERNAL_H
11#define __LIBBOMP_INTERNAL_H
12
13#include <barrelfish/barrelfish.h>
14#include <flounder/flounder_txqueue.h>
15
16#include <numa.h>
17#include <omp.h>
18
19#include <omp_abi.h>
20#include <omp_icv.h>
21#include <omp_environment.h>
22
23#include <bomp_debug.h>
24
25
26#include <if/bomp_defs.h>
27
28
29///< size of the control channel in bytes
30#define BOMP_CHANNEL_SIZE 2048
31
32///< bomp thread id
33typedef uint32_t bomp_tid_t;
34
35/// the maximum size of a execution node
36#define BOMP_NODE_SIZE_MAX -1
37
38
39///< BOMP node types
40typedef enum bomp_node_type {
41    BOMP_NODE_INVALID,  ///< the node has an invalid type
42    BOMP_NODE_MASTER,   ///< this is the program master node
43    BOMP_NODE_LOCAL,    ///< the node is a address space local node
44    BOMP_NODE_REMOTE    ///< the node is on a different address space
45} bomp_node_type_t;
46
47typedef enum bomp_thread_role {
48    BOMP_THREAD_ROLE_INVALID, ///< Invalid thread type
49    BOMP_THREAD_ROLE_WORKER,  ///< Normal worker thread
50    BOMP_THREAD_ROLE_NODE,    ///< Node coordinator thread
51    BOMP_THREAD_ROLE_MASTER   ///< Program master thread (initial thread)
52} bomp_thread_role_t;
53
54///< type of the execute function of BOMP
55typedef void (*bomp_thread_fn_t)(void *);
56
57/**
58 * state of a normal BOMP thread
59 */
60struct bomp_thread {
61    coreid_t id;                 ///< id of the thread
62    coreid_t coreid;             ///< id of the core the thread runs on
63    struct omp_icv_task *icvt;   ///<
64    struct bomp_tls *tls;
65    size_t stack_size;           ///< size of the stack
66    void *msgbuf;                ///< message buffer for this frame
67    struct capref msgframe;      ///< backing frame for the message buffer
68    struct bomp_node *node;      ///< the node this threads belongs to
69    errval_t thread_err;         ///< stores the error in case of failure
70    struct bomp_binding *ctrl;   ///< control channel
71    struct tx_queue txq;         ///< Flounder TX queue
72};
73
74/**
75 * state of a BOMP node coordinator
76 */
77struct bomp_node {
78    nodeid_t id;                  ///< the id of the execution node
79    nodeid_t numa_node;           ///< numa node id
80    bomp_node_type_t type;        ///< type of this node
81    struct bomp_tls *tls;         ///< pointer to the thread local storage
82    coreid_t threads_max;         ///< the number of threads of this node
83    coreid_t threads_active;      ///< the number of active threads on this node
84    size_t stack_size;           ///< size of the stack
85    struct bomp_thread *threads;  ///< pointer to the local threads array
86    struct bomp_binding *ctrl;    ///< control channel
87    void *msgbuf;                 ///< message buffer for this frame
88    struct capref msgframe;       ///< backing frame for the message buffer
89    errval_t node_err;            ///< error of the code
90    struct tx_queue txq;          ///< Flounder TX queue
91};
92
93
94/**
95 * \brief stores the state of the BOMP master thread
96 *
97 * The BOMP master thread is also a node coordinator of its own node.
98 */
99struct bomp_master {
100    nodeid_t num_nodes;          ///< the number of nodes in the system
101    struct bomp_node *nodes;     ///< array of nodes to other
102    coreid_t threads_max;        ///< the maximum number of threads in the system
103    coreid_t nodes_active;       ///< the number of active threads in the system
104    struct bomp_node local;      ///< the local node
105};
106
107struct bomp_binding *ctrl;   ///< control channel
108    struct tx_queue txq;         ///< Flounder TX queue
109
110
111struct bomp_work {
112    coreid_t thread_id;
113};
114
115struct bomp_tls {
116    struct thread *self;        ///< pointer ot the struct thread
117    struct omp_icv icv;             ///< pointer holding the environment variables
118    coreid_t thread_id;
119    bomp_thread_role_t role;     ///< identifies the role of the thread
120    union {
121        struct bomp_master master;
122        struct bomp_node   node;
123        struct bomp_thread thread;
124    } r;
125
126};
127
128
129errval_t bomp_node_init(bomp_node_type_t type, nodeid_t numanode, nodeid_t nodeid, coreid_t nthreads,
130                        size_t stack_size, struct bomp_node *node);
131coreid_t bomp_node_exec(struct bomp_node *node, void *fn, void *arg, coreid_t tid_start, coreid_t nthreads);
132
133errval_t bomp_thread_init(coreid_t core, size_t stack_size, struct bomp_thread *thread);
134
135errval_t bomp_thread_exec(struct bomp_thread *thread,
136                          bomp_thread_fn_t fn, void *arg, uint32_t tid);
137
138
139void bomp_start_processing(void (*fn)(void *),
140                           void *data,
141                           coreid_t tid_start,
142                           coreid_t nthreads);
143void bomp_end_processing(void);
144
145/**
146 * \brief obtaining a pointer to the control variables
147 *
148 * \return pointe to the ICV struct
149 */
150static inline struct omp_icv *bomp_icv_get(void)
151{
152    struct bomp_tls *tls = thread_get_tls();
153    return &tls->icv;
154}
155
156static inline void bomp_icv_set_task(struct omp_icv_task *task)
157{
158    struct bomp_tls *tls = thread_get_tls();
159    tls->icv.task = task;
160}
161
162static inline struct omp_icv_task *bomp_icv_get_task(void)
163{
164    struct bomp_tls *tls = thread_get_tls();
165    return tls->icv.task;
166}
167
168
169
170#if 0
171/**
172 * \brief this struct stores thread local data such as the team / task
173 *        of this thread
174 */
175struct bomp_thread
176{
177    bomp_thread_fn_t fn;
178    void *arg;
179
180    struct bomp_task *task;
181
182};
183
184
185struct bomp_work {
186    void (*fn)(void *);
187    void *data;
188    unsigned thread_id;
189    unsigned num_threads;
190    unsigned num_vtreads;
191    struct bomp_barrier *barrier;
192};
193
194
195struct bomp_thread_local_data {
196    void *thr; // thread reference
197    struct bomp_work *work;
198    struct bomp_icv_data *icv;
199};
200#endif
201
202
203
204
205#endif/* _LIBBOMP_H */
206