1/**
2 * \file
3 * \brief Implementation of the backend functions for domain exclusive address
4 *        spaces
5 *
6 * This backend of libbomp deals with worker domains instead of threads.
7 */
8
9/*
10 * Copyright (c) 2007, 2008, 2009, 2010, ETH Zurich.
11 * All rights reserved.
12 *
13 * This file is distributed under the terms in the attached LICENSE file.
14 * If you do not find this file, copies can be found by writing to:
15 * ETH Zurich D-INFK, Haldeneggsteig 4, CH-8092 Zurich. Attn: Systems Group.
16 */
17
18#include <barrelfish/barrelfish.h>
19#include <barrelfish/waitset.h>
20#include <stdio.h>
21#include <barrelfish/sys_debug.h>
22
23#include <bomp_internal.h>
24#include <xomp/xomp.h>
25#include <xomp_debug.h>
26
27static struct bomp_state *xomp_st;
28
29/*
30 * ----------------------------------------------------------------------------
31 * XOMP barriers
32 * ----------------------------------------------------------------------------
33 */
34/**
35 * \brief enters the barrier for the master thread and waits until all
36 *        workers also entered the barrier
37 *
38 * \param barrier   The barrier to enter
39 */
40static inline void xomp_barrier_enter(struct bomp_barrier *barrier)
41{
42    errval_t err;
43    int cycle = barrier->cycle;
44
45    if (__sync_fetch_and_add(&barrier->counter, 1) == (barrier->max - 1)) {
46        barrier->counter = 0;
47        barrier->cycle = !barrier->cycle;
48        return;
49    }
50
51    while (cycle == barrier->cycle) {
52        /* dispatch the events such that we recv the done messages */
53        err = event_dispatch_non_block(get_default_waitset());
54        switch (err_no(err)) {
55            case SYS_ERR_OK:
56                break;
57            case LIB_ERR_NO_EVENT:
58                thread_yield();
59                break;
60            default:
61                USER_PANIC_ERR(err, "in event dispatch");
62        }
63    }
64}
65
66
67static void set_numa(unsigned id)
68{
69    /* nop */
70}
71
72static void xomp_synchronize(void)
73{
74    /* todo */
75}
76
77static void xomp_start_processing(void (*fn)(void *),
78                                  void *data,
79                                  unsigned nthreads)
80{
81    XMP_DEBUG("start processing fn:%p, data:%p, threads:%u\n", fn, data, nthreads);
82
83    errval_t err;
84
85    /* Create Threads and ask them to process the function specified */
86    /* Let them die as soon as they are done */
87
88    struct bomp_work *xdata;
89
90    struct bomp_barrier *barrier;
91
92    g_bomp_state->num_threads = nthreads;
93
94    char *memory = calloc(1, sizeof(void *) + sizeof(*barrier) + sizeof(*xdata));
95    assert(memory != NULL);
96
97    g_bomp_state->tld = (struct bomp_thread_local_data **) memory;
98
99    memory += sizeof(struct bomp_thread_local_data *);
100
101    /* Create a barier for the work that will be carried out by the threads */
102
103    barrier = (struct bomp_barrier *) memory;
104
105    memory += sizeof(struct bomp_barrier);
106    bomp_barrier_init(barrier, nthreads);
107
108    /* For main thread */
109
110    xdata = (struct bomp_work *) memory;
111
112    memory += sizeof(struct bomp_work);
113
114    xdata->fn = fn;
115    xdata->data = data;
116    xdata->thread_id = 0;
117    xdata->barrier = barrier;
118    bomp_set_tls(xdata);
119
120    struct xomp_task *task = calloc(1, sizeof(struct xomp_task));
121    assert(task);
122    task->arg = data;
123    task->barrier = barrier;
124    task->total_threads = nthreads;
125    task->fn = fn;
126    task->done = 1;  // set the main thread to done
127
128    XMP_DEBUG("distributing work to Xeon Phi\n");
129
130    err = xomp_master_do_work(task);
131
132    if (err_is_fail(err)) {
133        USER_PANIC_ERR(err, "failed to distribute workers\n");
134    }
135}
136
137
138static void xomp_end_processing(void)
139{
140    /* Cleaning of thread_local and work data structures */
141    int i = 0;
142
143    xomp_barrier_enter(g_bomp_state->tld[i]->work->barrier);
144
145    /* Clear the barrier created */
146    bomp_clear_barrier(g_bomp_state->tld[i]->work->barrier);
147
148    free(g_bomp_state->tld);
149
150    g_bomp_state->tld = NULL;
151    g_bomp_state->num_threads = 1;
152}
153
154
155int bomp_xomp_init(void *arg)
156{
157    errval_t err;
158
159    struct xomp_args *args = arg;
160
161    xomp_st = calloc(1, sizeof(struct bomp_state ));
162    if (xomp_st == NULL) {
163        return -1;
164    }
165    xomp_st->backend_type = BOMP_BACKEND_XOMP;
166    xomp_st->backend.get_thread = (backend_get_thread_fn_t)thread_self;
167    xomp_st->backend.get_tls = thread_get_tls;
168    xomp_st->backend.set_tls = thread_set_tls;
169    xomp_st->backend.set_numa = set_numa;
170    xomp_st->backend.thread_exit = thread_exit;
171    xomp_st->backend.synchronize = xomp_synchronize;
172    xomp_st->backend.start_processing = xomp_start_processing;
173    xomp_st->backend.end_processing = xomp_end_processing;
174
175    bomp_common_init(xomp_st);
176    g_bomp_state = xomp_st;
177
178    switch (args->type) {
179        case XOMP_ARG_TYPE_WORKER:
180            g_bomp_state = xomp_st;
181            err = xomp_worker_init(args->args.worker.id);
182            break;
183        case XOMP_ARG_TYPE_UNIFORM:
184        case XOMP_ARG_TYPE_DISTINCT:
185            err = xomp_master_init(args);
186            break;
187        default:
188            return XOMP_ERR_INVALID_ARGUMENTS;
189            break;
190    }
191
192    if (err_is_fail(err)) {
193        USER_PANIC_ERR(err, "Initializing library\n");
194    }
195
196    if (args->type == XOMP_ARG_TYPE_WORKER) {
197        while(1) {
198            messages_wait_and_handle_next();
199        }
200        return -1;
201    }
202
203    if (args->type == XOMP_ARG_TYPE_UNIFORM) {
204        g_thread_limit = args->args.uniform.nthreads;
205    } else {
206        g_thread_limit = args->args.distinct.nthreads;
207    }
208
209    err = xomp_master_spawn_workers(g_thread_limit);
210    if (err_is_fail(err)) {
211        USER_PANIC_ERR(err, "Master spawning workers\n");
212    }
213
214    omp_set_num_threads(g_thread_limit);
215
216    return 0;
217}
218struct bomp_state * bomp_get_backend_state_xomp(void)
219{
220    return xomp_st;
221}
222
223
224
225