1/** 2 * \file 3 * \brief Distops common server/client framework 4 */ 5 6/* 7 * Copyright (c) 2016, ETH Zurich. 8 * All rights reserved. 9 * 10 * This file is distributed under the terms in the attached LICENSE file. 11 * If you do not find this file, copies can be found by writing to: 12 * ETH Zurich D-INFK, Universitaetstr 6, CH-8092 Zurich. Attn: Systems Group. 13 */ 14 15#include <barrelfish/barrelfish.h> 16#include <barrelfish/nameservice_client.h> 17#include <if/bench_distops_defs.h> 18 19#include <trace/trace.h> 20 21#include "benchapi.h" 22 23//{{{1 Shared local state 24 25static const char *service_name = "bench_distops_svc"; 26static coreid_t my_core_id = -1; 27 28//{{{1 Mgmt node 29 30//{{{2 Mgmt node: benchmark-independent state 31 32struct benchmark_state { 33 int clients_seen; 34 int clients_total; 35 struct bench_distops_binding **nodes; 36 void *st; 37 bool tracing; 38}; 39 40struct mgmt_node_state { 41 void *global; 42 uint32_t coreid; 43 void *st; 44}; 45 46// Only for mgmt node 47static struct benchmark_state *bench_state = NULL; 48 49//{{{2 Mgmt node: broadcast helper functions 50 51void broadcast_cmd(uint32_t cmd, uint32_t arg) 52{ 53 errval_t err; 54 if (!bench_state) { 55 printf("Benchmark not initialized, cannot broadcast\n"); 56 return; 57 } 58 if (bench_state->clients_seen < bench_state->clients_total) { 59 printf("Not all clients registered, broadcast not yet possible\n"); 60 return; 61 } 62 for (int i = 0; i < bench_state->clients_total; i++) { 63 assert(bench_state->nodes[i]); 64 err = bench_distops_cmd__tx(bench_state->nodes[i], NOP_CONT, cmd, arg); 65 if (err_is_fail(err)) { 66 DEBUG_ERR(err, "sending cmd msg to binding %p\n", bench_state->nodes[i]); 67 } 68 } 69 return; 70} 71 72void broadcast_caps(uint32_t cmd, uint32_t arg, struct capref cap1) 73{ 74 errval_t err; 75 if (!bench_state) { 76 printf("Benchmark not initialized, cannot broadcast\n"); 77 return; 78 } 79 if (bench_state->clients_seen < bench_state->clients_total) { 80 printf("Not all clients registered, broadcast not yet possible\n"); 81 return; 82 } 83 for (int i = 0; i < bench_state->clients_total; i++) { 84 assert(bench_state->nodes[i]); 85 err = bench_distops_caps__tx(bench_state->nodes[i], NOP_CONT, cmd, arg, cap1); 86 if (err_is_fail(err)) { 87 DEBUG_ERR(err, "sending caps msg to binding %p\n", bench_state->nodes[i]); 88 } 89 } 90 return; 91} 92 93//{{{2 Mgmt node multicast helper functions 94void multicast_caps(uint32_t cmd, uint32_t arg, struct capref cap1, 95 coreid_t *cores, int corecount) 96{ 97 errval_t err; 98 if (!bench_state) { 99 printf("Benchmark not initialized, cannot multicast\n"); 100 return; 101 } 102 if (bench_state->clients_seen < bench_state->clients_total) { 103 printf("Not all clients registered, multicast not yet possible\n"); 104 return; 105 } 106 for (int i = 0; i < bench_state->clients_total; i++) { 107 assert(bench_state->nodes[i]); 108 struct mgmt_node_state *ns = bench_state->nodes[i]->st; 109 for (int c = 0; c < corecount; c++) { 110 if (cores[c] == ns->coreid) { 111 err = bench_distops_caps__tx(bench_state->nodes[i], NOP_CONT, 112 cmd, arg, cap1); 113 if (err_is_fail(err)) { 114 DEBUG_ERR(err, "sending caps msg to binding %p\n", bench_state->nodes[i]); 115 } 116 } 117 } 118 } 119 return; 120} 121 122//{{{2 Mgmt node unicast helper functions 123void unicast_cmd(coreid_t nodeid, uint32_t cmd, uint32_t arg) 124{ 125 errval_t err; 126 if (!bench_state) { 127 printf("Benchmark not initialized, cannot unicast\n"); 128 return; 129 } 130 if (bench_state->clients_seen < bench_state->clients_total) { 131 printf("Not all clients registered, unicast not yet possible\n"); 132 return; 133 } 134 for (int i = 0; i < bench_state->clients_total; i++) { 135 assert(bench_state->nodes[i]); 136 struct mgmt_node_state *ns = bench_state->nodes[i]->st; 137 if (ns->coreid == nodeid) { 138 err = bench_distops_cmd__tx(bench_state->nodes[i], NOP_CONT, cmd, arg); 139 PANIC_IF_ERR(err, "cmd unicast"); 140 break; 141 } 142 } 143 return; 144} 145 146//{{{2 Mgmt node message handlers 147 148static void mgmt_rx_hello(struct bench_distops_binding *b, uint32_t coreid) 149{ 150 DEBUG("mgmt rx_hello from core %"PRIu32"\n", coreid); 151 152 bench_state->clients_seen ++; 153 154 struct mgmt_node_state *ns = b->st; 155 ns->coreid = coreid; 156 157 mgmt_register_node(bench_state->st, coreid); 158 159 if (bench_state->clients_seen == bench_state->clients_total) { 160 mgmt_run_benchmark(bench_state->st); 161 } 162} 163 164static void mgmt_rx_cmd(struct bench_distops_binding *b, uint32_t cmd, uint32_t arg) 165{ 166 DEBUG("server rx_cmd %"PRIu32"\n", arg); 167 168 // Execute command requested by node: implemented in <distop.c> 169 mgmt_cmd(cmd, arg, b); 170} 171 172static void mgmt_rx_caps(struct bench_distops_binding *b, uint32_t cmd, uint32_t arg, 173 struct capref cap1) 174{ 175 // Execute receive caps sent by node: implemented in <distop.c> 176 mgmt_cmd_caps(cmd, arg, cap1, b); 177} 178 179static struct bench_distops_rx_vtbl mgmt_rx_vtbl = { 180 .hello = mgmt_rx_hello, 181 .cmd = mgmt_rx_cmd, 182 .caps = mgmt_rx_caps, 183}; 184 185// {{{2 Mgmt node export & connect 186 187static void export_cb(void *st, errval_t err, iref_t iref) 188{ 189 PANIC_IF_ERR(err, "export failed"); 190 191 printf("service exported at iref %"PRIuIREF"\n", iref); 192 193 // register this iref with the name service 194 err = nameservice_register(service_name, iref); 195 PANIC_IF_ERR(err, "nameservice_register failed"); 196} 197 198static errval_t connect_cb(void *st, struct bench_distops_binding *b) 199{ 200 printf("service got a connection!\n"); 201 202 // copy my message receive handler vtable to the binding 203 b->rx_vtbl = mgmt_rx_vtbl; 204 205 // Create a server state struct for this connection 206 b->st = malloc(sizeof(struct mgmt_node_state)); 207 assert(b->st); 208 if (!b->st) { 209 USER_PANIC("state malloc() in mgmt node"); 210 } 211 struct mgmt_node_state *ns = b->st; 212 ns->global = bench_state; 213 214 errval_t err = mgmt_init_node(&ns->st); 215 PANIC_IF_ERR(err, "init node"); 216 217 // add node binding to list of bindings, for later broadcasts, we don't 218 // care about ordering here, so don't assume that node at index i is 219 // running on core i. 220 static int nidx = 0; 221 bench_state->nodes[nidx++] = b; 222 223 // accept the connection (we could return an error to refuse it) 224 return SYS_ERR_OK; 225} 226 227static void run_benchmark(int nodecount) 228{ 229 errval_t err; 230 231 bench_state = malloc(sizeof(*bench_state)); 232 if (!bench_state) { 233 USER_PANIC("malloc failed"); 234 } 235 236 // Initialize benchmark state 237 bench_state->clients_seen = 0; 238 bench_state->clients_total = nodecount; 239 bench_state->nodes = malloc(nodecount * sizeof(struct bench_distops_binding *)); 240 if (!bench_state->nodes) { 241 USER_PANIC("malloc failed"); 242 } 243 244 err = mgmt_init_tracing(); 245 if (err_is_fail(err)) { 246 USER_PANIC_ERR(err, "initializing tracing"); 247 } 248 249 err = mgmt_init_benchmark(&bench_state->st, nodecount); 250 PANIC_IF_ERR(err, "init benchmark"); 251 252 err = bench_distops_export(NULL, 253 export_cb, connect_cb, get_default_waitset(), 254 IDC_EXPORT_FLAGS_DEFAULT); 255 PANIC_IF_ERR(err, "export failed"); 256} 257 258void *get_global_state(struct bench_distops_binding *b) 259{ 260 struct mgmt_node_state *ns = b->st; 261 struct benchmark_state *bs = ns->global; 262 return bs->st; 263} 264 265//{{{1 Node 266 267//{{{2 Node message handlers 268 269// We use test.basic to signal that server has done it's retypes 270static void node_rx_cmd(struct bench_distops_binding *b, uint32_t cmd, uint32_t arg) 271{ 272 DEBUG("client rx_cmd %"PRIu32": b->st = %p\n", arg, b->st); 273 274 // handle cmd message on node: implemented in <distop.c> 275 node_cmd(cmd, arg, b); 276} 277 278static void node_rx_caps(struct bench_distops_binding *b, uint32_t cmd, uint32_t arg, 279 struct capref cap1) 280{ 281 DEBUG("node %d rx_caps: cmd=%"PRIu32"\n", my_core_id, cmd); 282 283 // handle rx_caps message on node: implemented in <distop.c> 284 node_cmd_caps(cmd, arg, cap1, b); 285} 286 287static struct bench_distops_rx_vtbl client_rx_vtbl = { 288 .cmd = node_rx_cmd, 289 .caps = node_rx_caps, 290}; 291 292//{{{2 Node bind 293 294static void bind_cb(void *st, errval_t err, struct bench_distops_binding *b) 295{ 296 PANIC_IF_ERR(err, "bind failed"); 297 298 printf("node %d bound!\n", my_core_id); 299 300 // copy my message receive handler vtable to the binding 301 b->rx_vtbl = client_rx_vtbl; 302 303 // Initialize node state: implemented in <distop>.c 304 init_node(b); 305 306 // Send hello message 307 printf("%s: node %d sending hello msg\n", __FUNCTION__, my_core_id); 308 err = bench_distops_hello__tx(b, NOP_CONT, my_core_id); 309 PANIC_IF_ERR(err, "in node %d: sending cap to server", my_core_id); 310} 311 312static void run_node(void) 313{ 314 errval_t err; 315 iref_t iref; 316 317 printf("node %d looking up '%s' in name service...\n", my_core_id, service_name); 318 err = nameservice_blocking_lookup(service_name, &iref); 319 PANIC_IF_ERR(err, "nameservice_blocking_lookup failed"); 320 321 printf("node %d binding to %"PRIuIREF"...\n", my_core_id, iref); 322 323 err = bench_distops_bind(iref, bind_cb, NULL, get_default_waitset(), 324 IDC_BIND_FLAGS_DEFAULT); 325 PANIC_IF_ERR(err, "bind failed"); 326} 327 328//{{{1 Main 329int main(int argc, char *argv[]) 330{ 331 char *role; 332 333 my_core_id = disp_get_core_id(); 334 335#ifndef NDEBUG 336 printf("Running with assertions ENABLED!!!\n"); 337#endif 338 339 // TODO: more args 340 if (argc < 2) { 341 printf("distops benchmark needs an argument of \"mgmt\" or \"node\"\n"); 342 return 1; 343 } 344 if (strncmp(argv[1], "mgmt", 4) == 0) { 345 printf("we are the orchestrator\n"); 346 role = "server"; 347 if (argc < 3) { 348 printf("Orchestrator needs number of nodes as 2nd argument\n"); 349 return 2; 350 } 351 run_benchmark(atoi(argv[2])); 352 } 353 if (strncmp(argv[1], "node", 4) == 0) { 354 printf("we are node %d\n", my_core_id); 355 role = "client"; 356 run_node(); 357 } 358 359 errval_t err; 360 struct waitset *ws = get_default_waitset(); 361 while (true) { 362 err = event_dispatch(ws); 363 PANIC_IF_ERR(err, "in %s: event_dispatch", role); 364 } 365 366 return 0; 367} 368