1/** 2 * \file 3 * \brief Implementation of the backend functions for domain exclusive address 4 * spaces 5 * 6 * This backend of libbomp deals with worker domains instead of threads. 7 */ 8 9/* 10 * Copyright (c) 2007, 2008, 2009, 2010, ETH Zurich. 11 * All rights reserved. 12 * 13 * This file is distributed under the terms in the attached LICENSE file. 14 * If you do not find this file, copies can be found by writing to: 15 * ETH Zurich D-INFK, Haldeneggsteig 4, CH-8092 Zurich. Attn: Systems Group. 16 */ 17 18#include <barrelfish/barrelfish.h> 19#include <barrelfish/waitset.h> 20#include <stdio.h> 21#include <barrelfish/sys_debug.h> 22 23#include <bomp_internal.h> 24#include <xomp/xomp.h> 25#include <xomp_debug.h> 26 27static struct bomp_state *xomp_st; 28 29/* 30 * ---------------------------------------------------------------------------- 31 * XOMP barriers 32 * ---------------------------------------------------------------------------- 33 */ 34/** 35 * \brief enters the barrier for the master thread and waits until all 36 * workers also entered the barrier 37 * 38 * \param barrier The barrier to enter 39 */ 40static inline void xomp_barrier_enter(struct bomp_barrier *barrier) 41{ 42 errval_t err; 43 int cycle = barrier->cycle; 44 45 if (__sync_fetch_and_add(&barrier->counter, 1) == (barrier->max - 1)) { 46 barrier->counter = 0; 47 barrier->cycle = !barrier->cycle; 48 return; 49 } 50 51 while (cycle == barrier->cycle) { 52 /* dispatch the events such that we recv the done messages */ 53 err = event_dispatch_non_block(get_default_waitset()); 54 switch (err_no(err)) { 55 case SYS_ERR_OK: 56 break; 57 case LIB_ERR_NO_EVENT: 58 thread_yield(); 59 break; 60 default: 61 USER_PANIC_ERR(err, "in event dispatch"); 62 } 63 } 64} 65 66 67static void set_numa(unsigned id) 68{ 69 /* nop */ 70} 71 72static void xomp_synchronize(void) 73{ 74 /* todo */ 75} 76 77static void xomp_start_processing(void (*fn)(void *), 78 void *data, 79 unsigned nthreads) 80{ 81 XMP_DEBUG("start processing fn:%p, data:%p, threads:%u\n", fn, data, nthreads); 82 83 errval_t err; 84 85 /* Create Threads and ask them to process the function specified */ 86 /* Let them die as soon as they are done */ 87 88 struct bomp_work *xdata; 89 90 struct bomp_barrier *barrier; 91 92 g_bomp_state->num_threads = nthreads; 93 94 char *memory = calloc(1, sizeof(void *) + sizeof(*barrier) + sizeof(*xdata)); 95 assert(memory != NULL); 96 97 g_bomp_state->tld = (struct bomp_thread_local_data **) memory; 98 99 memory += sizeof(struct bomp_thread_local_data *); 100 101 /* Create a barier for the work that will be carried out by the threads */ 102 103 barrier = (struct bomp_barrier *) memory; 104 105 memory += sizeof(struct bomp_barrier); 106 bomp_barrier_init(barrier, nthreads); 107 108 /* For main thread */ 109 110 xdata = (struct bomp_work *) memory; 111 112 memory += sizeof(struct bomp_work); 113 114 xdata->fn = fn; 115 xdata->data = data; 116 xdata->thread_id = 0; 117 xdata->barrier = barrier; 118 bomp_set_tls(xdata); 119 120 struct xomp_task *task = calloc(1, sizeof(struct xomp_task)); 121 assert(task); 122 task->arg = data; 123 task->barrier = barrier; 124 task->total_threads = nthreads; 125 task->fn = fn; 126 task->done = 1; // set the main thread to done 127 128 XMP_DEBUG("distributing work to Xeon Phi\n"); 129 130 err = xomp_master_do_work(task); 131 132 if (err_is_fail(err)) { 133 USER_PANIC_ERR(err, "failed to distribute workers\n"); 134 } 135} 136 137 138static void xomp_end_processing(void) 139{ 140 /* Cleaning of thread_local and work data structures */ 141 int i = 0; 142 143 xomp_barrier_enter(g_bomp_state->tld[i]->work->barrier); 144 145 /* Clear the barrier created */ 146 bomp_clear_barrier(g_bomp_state->tld[i]->work->barrier); 147 148 free(g_bomp_state->tld); 149 150 g_bomp_state->tld = NULL; 151 g_bomp_state->num_threads = 1; 152} 153 154 155int bomp_xomp_init(void *arg) 156{ 157 errval_t err; 158 159 struct xomp_args *args = arg; 160 161 xomp_st = calloc(1, sizeof(struct bomp_state )); 162 if (xomp_st == NULL) { 163 return -1; 164 } 165 xomp_st->backend_type = BOMP_BACKEND_XOMP; 166 xomp_st->backend.get_thread = (backend_get_thread_fn_t)thread_self; 167 xomp_st->backend.get_tls = thread_get_tls; 168 xomp_st->backend.set_tls = thread_set_tls; 169 xomp_st->backend.set_numa = set_numa; 170 xomp_st->backend.thread_exit = thread_exit; 171 xomp_st->backend.synchronize = xomp_synchronize; 172 xomp_st->backend.start_processing = xomp_start_processing; 173 xomp_st->backend.end_processing = xomp_end_processing; 174 175 bomp_common_init(xomp_st); 176 g_bomp_state = xomp_st; 177 178 switch (args->type) { 179 case XOMP_ARG_TYPE_WORKER: 180 g_bomp_state = xomp_st; 181 err = xomp_worker_init(args->args.worker.id); 182 break; 183 case XOMP_ARG_TYPE_UNIFORM: 184 case XOMP_ARG_TYPE_DISTINCT: 185 err = xomp_master_init(args); 186 break; 187 default: 188 return XOMP_ERR_INVALID_ARGUMENTS; 189 break; 190 } 191 192 if (err_is_fail(err)) { 193 USER_PANIC_ERR(err, "Initializing library\n"); 194 } 195 196 if (args->type == XOMP_ARG_TYPE_WORKER) { 197 while(1) { 198 messages_wait_and_handle_next(); 199 } 200 return -1; 201 } 202 203 if (args->type == XOMP_ARG_TYPE_UNIFORM) { 204 g_thread_limit = args->args.uniform.nthreads; 205 } else { 206 g_thread_limit = args->args.distinct.nthreads; 207 } 208 209 err = xomp_master_spawn_workers(g_thread_limit); 210 if (err_is_fail(err)) { 211 USER_PANIC_ERR(err, "Master spawning workers\n"); 212 } 213 214 omp_set_num_threads(g_thread_limit); 215 216 return 0; 217} 218struct bomp_state * bomp_get_backend_state_xomp(void) 219{ 220 return xomp_st; 221} 222 223 224 225