1/**
2 * \file
3 * \brief Distops common server/client framework
4 */
5
6/*
7 * Copyright (c) 2016, ETH Zurich.
8 * All rights reserved.
9 *
10 * This file is distributed under the terms in the attached LICENSE file.
11 * If you do not find this file, copies can be found by writing to:
12 * ETH Zurich D-INFK, Universitaetstr 6, CH-8092 Zurich. Attn: Systems Group.
13 */
14
15#include <barrelfish/barrelfish.h>
16#include <barrelfish/nameservice_client.h>
17#include <if/bench_distops_defs.h>
18
19#include <trace/trace.h>
20
21#include "benchapi.h"
22
23//{{{1 Shared local state
24
25static const char *service_name = "bench_distops_svc";
26static coreid_t my_core_id = -1;
27
28//{{{1 Mgmt node
29
30//{{{2 Mgmt node: benchmark-independent state
31
32struct benchmark_state {
33    int clients_seen;
34    int clients_total;
35    struct bench_distops_binding **nodes;
36    void *st;
37    bool tracing;
38};
39
40struct mgmt_node_state {
41    void *global;
42    uint32_t coreid;
43    void *st;
44};
45
46// Only for mgmt node
47static struct benchmark_state *bench_state = NULL;
48
49//{{{2 Mgmt node: broadcast helper functions
50
51void broadcast_cmd(uint32_t cmd, uint32_t arg)
52{
53    errval_t err;
54    if (!bench_state) {
55        printf("Benchmark not initialized, cannot broadcast\n");
56        return;
57    }
58    if (bench_state->clients_seen < bench_state->clients_total) {
59        printf("Not all clients registered, broadcast not yet possible\n");
60        return;
61    }
62    for (int i = 0; i < bench_state->clients_total; i++) {
63        assert(bench_state->nodes[i]);
64        err = bench_distops_cmd__tx(bench_state->nodes[i], NOP_CONT, cmd, arg);
65        if (err_is_fail(err)) {
66            DEBUG_ERR(err, "sending cmd msg to binding %p\n", bench_state->nodes[i]);
67        }
68    }
69    return;
70}
71
72void broadcast_caps(uint32_t cmd, uint32_t arg, struct capref cap1)
73{
74    errval_t err;
75    if (!bench_state) {
76        printf("Benchmark not initialized, cannot broadcast\n");
77        return;
78    }
79    if (bench_state->clients_seen < bench_state->clients_total) {
80        printf("Not all clients registered, broadcast not yet possible\n");
81        return;
82    }
83    for (int i = 0; i < bench_state->clients_total; i++) {
84        assert(bench_state->nodes[i]);
85        err = bench_distops_caps__tx(bench_state->nodes[i], NOP_CONT, cmd, arg, cap1);
86        if (err_is_fail(err)) {
87            DEBUG_ERR(err, "sending caps msg to binding %p\n", bench_state->nodes[i]);
88        }
89    }
90    return;
91}
92
93//{{{2 Mgmt node multicast helper functions
94void multicast_caps(uint32_t cmd, uint32_t arg, struct capref cap1,
95                    coreid_t *cores, int corecount)
96{
97    errval_t err;
98    if (!bench_state) {
99        printf("Benchmark not initialized, cannot multicast\n");
100        return;
101    }
102    if (bench_state->clients_seen < bench_state->clients_total) {
103        printf("Not all clients registered, multicast not yet possible\n");
104        return;
105    }
106    for (int i = 0; i < bench_state->clients_total; i++) {
107        assert(bench_state->nodes[i]);
108        struct mgmt_node_state *ns = bench_state->nodes[i]->st;
109        for (int c = 0; c < corecount; c++) {
110            if (cores[c] == ns->coreid) {
111                err = bench_distops_caps__tx(bench_state->nodes[i], NOP_CONT,
112                        cmd, arg, cap1);
113                if (err_is_fail(err)) {
114                    DEBUG_ERR(err, "sending caps msg to binding %p\n", bench_state->nodes[i]);
115                }
116            }
117        }
118    }
119    return;
120}
121
122//{{{2 Mgmt node unicast helper functions
123void unicast_cmd(coreid_t nodeid, uint32_t cmd, uint32_t arg)
124{
125    errval_t err;
126    if (!bench_state) {
127        printf("Benchmark not initialized, cannot unicast\n");
128        return;
129    }
130    if (bench_state->clients_seen < bench_state->clients_total) {
131        printf("Not all clients registered, unicast not yet possible\n");
132        return;
133    }
134    for (int i = 0; i < bench_state->clients_total; i++) {
135        assert(bench_state->nodes[i]);
136        struct mgmt_node_state *ns = bench_state->nodes[i]->st;
137        if (ns->coreid == nodeid) {
138            err = bench_distops_cmd__tx(bench_state->nodes[i], NOP_CONT, cmd, arg);
139            PANIC_IF_ERR(err, "cmd unicast");
140            break;
141        }
142    }
143    return;
144}
145
146//{{{2 Mgmt node message handlers
147
148static void mgmt_rx_hello(struct bench_distops_binding *b, uint32_t coreid)
149{
150    DEBUG("mgmt rx_hello from core %"PRIu32"\n", coreid);
151
152    bench_state->clients_seen ++;
153
154    struct mgmt_node_state *ns = b->st;
155    ns->coreid = coreid;
156
157    mgmt_register_node(bench_state->st, coreid);
158
159    if (bench_state->clients_seen == bench_state->clients_total) {
160        mgmt_run_benchmark(bench_state->st);
161    }
162}
163
164static void mgmt_rx_cmd(struct bench_distops_binding *b, uint32_t cmd, uint32_t arg)
165{
166    DEBUG("server rx_cmd %"PRIu32"\n", arg);
167
168    // Execute command requested by node: implemented in <distop.c>
169    mgmt_cmd(cmd, arg, b);
170}
171
172static void mgmt_rx_caps(struct bench_distops_binding *b, uint32_t cmd, uint32_t arg,
173                         struct capref cap1)
174{
175    // Execute receive caps sent by node: implemented in <distop.c>
176    mgmt_cmd_caps(cmd, arg, cap1, b);
177}
178
179static struct bench_distops_rx_vtbl mgmt_rx_vtbl = {
180    .hello = mgmt_rx_hello,
181    .cmd = mgmt_rx_cmd,
182    .caps = mgmt_rx_caps,
183};
184
185// {{{2 Mgmt node export & connect
186
187static void export_cb(void *st, errval_t err, iref_t iref)
188{
189    PANIC_IF_ERR(err, "export failed");
190
191    printf("service exported at iref %"PRIuIREF"\n", iref);
192
193    // register this iref with the name service
194    err = nameservice_register(service_name, iref);
195    PANIC_IF_ERR(err, "nameservice_register failed");
196}
197
198static errval_t connect_cb(void *st, struct bench_distops_binding *b)
199{
200    printf("service got a connection!\n");
201
202    // copy my message receive handler vtable to the binding
203    b->rx_vtbl = mgmt_rx_vtbl;
204
205    // Create a server state struct for this connection
206    b->st = malloc(sizeof(struct mgmt_node_state));
207    assert(b->st);
208    if (!b->st) {
209        USER_PANIC("state malloc() in mgmt node");
210    }
211    struct mgmt_node_state *ns = b->st;
212    ns->global = bench_state;
213
214    errval_t err = mgmt_init_node(&ns->st);
215    PANIC_IF_ERR(err, "init node");
216
217    // add node binding to list of bindings, for later broadcasts, we don't
218    // care about ordering here, so don't assume that node at index i is
219    // running on core i.
220    static int nidx = 0;
221    bench_state->nodes[nidx++] = b;
222
223    // accept the connection (we could return an error to refuse it)
224    return SYS_ERR_OK;
225}
226
227static void run_benchmark(int nodecount)
228{
229    errval_t err;
230
231    bench_state = malloc(sizeof(*bench_state));
232    if (!bench_state) {
233        USER_PANIC("malloc failed");
234    }
235
236    // Initialize benchmark state
237    bench_state->clients_seen = 0;
238    bench_state->clients_total = nodecount;
239    bench_state->nodes = malloc(nodecount * sizeof(struct bench_distops_binding *));
240    if (!bench_state->nodes) {
241        USER_PANIC("malloc failed");
242    }
243
244    err = mgmt_init_tracing();
245    if (err_is_fail(err)) {
246        USER_PANIC_ERR(err, "initializing tracing");
247    }
248
249    err = mgmt_init_benchmark(&bench_state->st, nodecount);
250    PANIC_IF_ERR(err, "init benchmark");
251
252    err = bench_distops_export(NULL,
253            export_cb, connect_cb, get_default_waitset(),
254            IDC_EXPORT_FLAGS_DEFAULT);
255    PANIC_IF_ERR(err, "export failed");
256}
257
258void *get_global_state(struct bench_distops_binding *b)
259{
260    struct mgmt_node_state *ns = b->st;
261    struct benchmark_state *bs = ns->global;
262    return bs->st;
263}
264
265//{{{1 Node
266
267//{{{2 Node message handlers
268
269// We use test.basic to signal that server has done it's retypes
270static void node_rx_cmd(struct bench_distops_binding *b, uint32_t cmd, uint32_t arg)
271{
272    DEBUG("client rx_cmd %"PRIu32": b->st = %p\n", arg, b->st);
273
274    // handle cmd message on node: implemented in <distop.c>
275    node_cmd(cmd, arg, b);
276}
277
278static void node_rx_caps(struct bench_distops_binding *b, uint32_t cmd, uint32_t arg,
279                         struct capref cap1)
280{
281    DEBUG("node %d rx_caps: cmd=%"PRIu32"\n", my_core_id, cmd);
282
283    // handle rx_caps message on node: implemented in <distop.c>
284    node_cmd_caps(cmd, arg, cap1, b);
285}
286
287static struct bench_distops_rx_vtbl client_rx_vtbl = {
288    .cmd = node_rx_cmd,
289    .caps = node_rx_caps,
290};
291
292//{{{2 Node bind
293
294static void bind_cb(void *st, errval_t err, struct bench_distops_binding *b)
295{
296    PANIC_IF_ERR(err, "bind failed");
297
298    printf("node %d bound!\n", my_core_id);
299
300    // copy my message receive handler vtable to the binding
301    b->rx_vtbl = client_rx_vtbl;
302
303    // Initialize node state: implemented in <distop>.c
304    init_node(b);
305
306    // Send hello message
307    printf("%s: node %d sending hello msg\n", __FUNCTION__, my_core_id);
308    err = bench_distops_hello__tx(b, NOP_CONT, my_core_id);
309    PANIC_IF_ERR(err, "in node %d: sending cap to server", my_core_id);
310}
311
312static void run_node(void)
313{
314    errval_t err;
315    iref_t iref;
316
317    printf("node %d looking up '%s' in name service...\n", my_core_id, service_name);
318    err = nameservice_blocking_lookup(service_name, &iref);
319    PANIC_IF_ERR(err, "nameservice_blocking_lookup failed");
320
321    printf("node %d binding to %"PRIuIREF"...\n", my_core_id, iref);
322
323    err = bench_distops_bind(iref, bind_cb, NULL, get_default_waitset(),
324                             IDC_BIND_FLAGS_DEFAULT);
325    PANIC_IF_ERR(err, "bind failed");
326}
327
328//{{{1 Main
329int main(int argc, char *argv[])
330{
331    char *role;
332
333    my_core_id = disp_get_core_id();
334
335#ifndef NDEBUG
336    printf("Running with assertions ENABLED!!!\n");
337#endif
338
339    // TODO: more args
340    if (argc < 2) {
341        printf("distops benchmark needs an argument of \"mgmt\" or \"node\"\n");
342        return 1;
343    }
344    if (strncmp(argv[1], "mgmt", 4) == 0) {
345        printf("we are the orchestrator\n");
346        role = "server";
347        if (argc < 3) {
348            printf("Orchestrator needs number of nodes as 2nd argument\n");
349            return 2;
350        }
351        run_benchmark(atoi(argv[2]));
352    }
353    if (strncmp(argv[1], "node", 4) == 0) {
354        printf("we are node %d\n", my_core_id);
355        role = "client";
356        run_node();
357    }
358
359    errval_t err;
360    struct waitset *ws = get_default_waitset();
361    while (true) {
362        err = event_dispatch(ws);
363        PANIC_IF_ERR(err, "in %s: event_dispatch", role);
364    }
365
366    return 0;
367}
368