1/**
2 * \file
3 * \brief Benchmark empty invocation for baseline
4 */
5
6/*
7 * Copyright (c) 2017, ETH Zurich.
8 * All rights reserved.
9 *
10 * This file is distributed under the terms in the attached LICENSE file.
11 * If you do not find this file, copies can be found by writing to:
12 * ETH Zurich D-INFK, Universitaetstr 6, CH-8092 Zurich. Attn: Systems Group.
13 */
14
15#include <barrelfish/barrelfish.h>
16#include <if/bench_distops_defs.h>
17
18#include <bitmacros.h>
19
20#include <bench/bench.h>
21
22#include "benchapi.h"
23
24//{{{1 debugging helpers
25static void debug_capref(const char *prefix, struct capref cap)
26{
27    char buf[128];
28    debug_print_capref(buf, 128, cap);
29    printf("%s capref = %s\n", prefix, buf);
30}
31
32//{{{1 shared commands
33enum bench_cmd {
34    BENCH_CMD_CREATE_COPIES,
35    BENCH_CMD_COPIES_DONE,
36    BENCH_CMD_CAP_FOR_BENCH,
37    BENCH_CMD_GOT_CAP,
38    BENCH_CMD_DO_DELETE,
39    BENCH_CMD_PRINT_STATS,
40    BENCH_CMD_PRINT_DONE,
41};
42
43//{{{1 Managment node: implement orchestration for benchmark
44
45//{{{2 Management node: state management
46
47struct global_state {
48    struct capref ram;
49    struct capref cap;
50    coreid_t *nodes;
51    int nodes_seen;
52    int nodecount;
53    int copies_done;
54    int printnode;
55    int currcopies;
56};
57
58errval_t mgmt_init_benchmark(void **st, int nodecount)
59{
60     *st = calloc(1, sizeof(struct global_state));
61     if (!*st) {
62         return LIB_ERR_MALLOC_FAIL;
63     }
64     struct global_state *gs = *st;
65     errval_t err;
66     gs->nodes = calloc(nodecount, sizeof(coreid_t));
67     gs->nodecount = nodecount;
68     gs->copies_done = 0;
69     gs->printnode = 0;
70     err = ram_alloc(&gs->ram, BASE_PAGE_BITS);
71     if (err_is_fail(err)) {
72         return err;
73     }
74     err = ram_alloc(&gs->cap, BASE_PAGE_BITS);
75     return err;
76}
77
78static int sort_coreid(const void *a_, const void *b_)
79{
80    // deref pointers as coreids, store as ints
81    int a = *((coreid_t*)a_);
82    int b = *((coreid_t*)b_);
83    // subtract as ints
84    return a-b;
85}
86
87void mgmt_register_node(void *st, coreid_t nodeid)
88{
89    struct global_state *gs = st;
90    gs->nodes[gs->nodes_seen++] = nodeid;
91    // if we've seen all nodes, sort nodes array and configure printnode
92    if (gs->nodes_seen == gs->nodecount) {
93        qsort(gs->nodes, gs->nodecount, sizeof(coreid_t), sort_coreid);
94    }
95}
96
97struct mgmt_node_state {
98};
99
100errval_t mgmt_init_node(void **st)
101{
102     *st = malloc(sizeof(struct mgmt_node_state));
103     if (!*st) {
104         return LIB_ERR_MALLOC_FAIL;
105     }
106    return SYS_ERR_OK;
107}
108
109//{{{2 Management node: benchmark impl
110void mgmt_run_benchmark(void *st)
111{
112    struct global_state *gs = st;
113
114    printf("All clients sent hello! Benchmark starting...\n");
115
116    printf("# Benchmarking NOOP INVOCATION: nodes=%d\n", gs->nodecount);
117
118    printf("# Starting out with %d copies, will increase by factor of two up to %d...\n",
119            NUM_COPIES_START, NUM_COPIES_END);
120
121    gs->currcopies = NUM_COPIES_START;
122    broadcast_caps(BENCH_CMD_CREATE_COPIES, NUM_COPIES_START, gs->ram);
123}
124
125void mgmt_cmd(uint32_t cmd, uint32_t arg, struct bench_distops_binding *b)
126{
127    struct global_state *gs = get_global_state(b);
128    switch(cmd) {
129        case BENCH_CMD_COPIES_DONE:
130            if (arg == 1) {
131                gs->copies_done++;
132                if (gs->copies_done == gs->nodecount) {
133                    printf("# All copies made!\n");
134                    broadcast_cmd(BENCH_CMD_DO_DELETE, ITERS);
135                    unicast_cmd(gs->nodes[gs->printnode++], BENCH_CMD_PRINT_STATS, 0);
136                }
137            } else {
138                bench_distops_caps__tx(b, NOP_CONT, BENCH_CMD_CAP_FOR_BENCH,
139                        0, gs->cap);
140            }
141            break;
142        case BENCH_CMD_GOT_CAP:
143            gs->copies_done++;
144            if (gs->copies_done == gs->nodecount) {
145                printf("# All copies made!\n");
146                broadcast_cmd(BENCH_CMD_DO_DELETE, ITERS);
147                unicast_cmd(gs->nodes[gs->printnode++], BENCH_CMD_PRINT_STATS, 0);
148            }
149            break;
150        case BENCH_CMD_PRINT_DONE:
151            if (gs->printnode == gs->nodecount) {
152                if (gs->currcopies == NUM_COPIES_END) {
153                    printf("# Benchmark done!\n");
154                    return;
155                }
156                printf("# Round done!\n");
157                // Reset counters for next round
158                gs->currcopies *= 2;
159                gs->copies_done = 0;
160                gs->printnode = 0;
161                // Start new round
162                broadcast_cmd(BENCH_CMD_CREATE_COPIES, gs->currcopies);
163                return;
164            }
165            unicast_cmd(gs->nodes[gs->printnode++], BENCH_CMD_PRINT_STATS, 0);
166            break;
167        default:
168            printf("mgmt node got unknown command %d over binding %p\n", cmd, b);
169            break;
170    }
171}
172
173void mgmt_cmd_caps(uint32_t cmd, uint32_t arg, struct capref cap1,
174                   struct bench_distops_binding *b)
175{
176    printf("mgmt node got caps + command %"PRIu32", arg=%d over binding %p:\n",
177            cmd, arg, b);
178    debug_capref("cap1:", cap1);
179}
180
181//{{{1 Node
182
183struct node_state {
184    struct capref cap;
185    struct capref ram;
186    uint32_t numcopies;
187    struct capref *copies;
188    uint64_t *delcycles;
189    uint32_t benchcount;
190};
191
192static coreid_t my_core_id = -1;
193
194void init_node(struct bench_distops_binding *b)
195{
196    printf("%s: binding = %p\n", __FUNCTION__, b);
197
198    my_core_id = disp_get_core_id();
199
200    bench_init();
201
202    // Allocate client state struct
203    b->st = malloc(sizeof(struct node_state));
204    assert(b->st);
205    if (!b->st) {
206        USER_PANIC("state malloc() in client");
207    }
208}
209
210static void node_create_copies(struct node_state *ns)
211{
212    errval_t err;
213    ns->copies = calloc(ns->numcopies, sizeof(struct capref));
214    for (int i = 0; i < ns->numcopies; i++) {
215        err = slot_alloc(&ns->copies[i]);
216        PANIC_IF_ERR(err, "slot_alloc for copy %d\n", i);
217        err = cap_copy(ns->copies[i], ns->ram);
218        PANIC_IF_ERR(err, "cap_copy for copy %d\n", i);
219    }
220}
221
222void node_cmd(uint32_t cmd, uint32_t arg, struct bench_distops_binding *b)
223{
224    struct node_state *ns = b->st;
225    errval_t err;
226    size_t cap_base_count = 0;
227    err = sys_debug_get_mdb_size(&cap_base_count);
228    assert(err_is_ok(err));
229
230    switch(cmd) {
231        case BENCH_CMD_CREATE_COPIES:
232            printf("# node %d: %zu caps before creating copies\n", my_core_id, cap_base_count);
233            printf("# node %d: creating %d cap copies\n", my_core_id, arg);
234            ns->numcopies = arg;
235            node_create_copies(ns);
236            err = bench_distops_cmd__tx(b, NOP_CONT, BENCH_CMD_COPIES_DONE, 1);
237            PANIC_IF_ERR(err, "signaling cap_copy() done\n");
238            break;
239        case BENCH_CMD_DO_DELETE:
240            ns->benchcount = arg;
241            ns->delcycles = calloc(arg, sizeof(uint64_t));
242            assert(ns->delcycles);
243            //printf("node %d: doing delete\n", my_core_id);
244            for (int i = 0; i < ns->benchcount; i++) {
245                uint64_t start, end;
246                start = bench_tsc();
247                err = invoke_ram_noop(ns->cap);
248                end = bench_tsc();
249                ns->delcycles[i] = end - start;
250                assert(err_is_ok(err));
251            }
252            //printf("node %d: deletes done\n", my_core_id);
253            break;
254        case BENCH_CMD_PRINT_STATS:
255            printf("# node %d: tsc_per_us = %ld; numcopies = %d\n", my_core_id, bench_tsc_per_us(), ns->numcopies);
256            printf("# delete latency in cycles\n");
257            for (int i = 0; i < ns->benchcount; i++) {
258                printf("%ld\n", ns->delcycles[i]);
259            }
260            err = bench_distops_cmd__tx(b, NOP_CONT, BENCH_CMD_PRINT_DONE, 0);
261            assert(err_is_ok(err));
262            // Cleanup before next round
263            for (int i = 0; i < ns->numcopies; i++) {
264                err = cap_destroy(ns->copies[i]);
265                assert(err_is_ok(err));
266            }
267            free(ns->copies);
268            free(ns->delcycles);
269            break;
270        default:
271            printf("node %d got command %"PRIu32"\n", my_core_id, cmd);
272            break;
273    }
274}
275
276void node_cmd_caps(uint32_t cmd, uint32_t arg, struct capref cap1,
277                   struct bench_distops_binding *b)
278{
279    errval_t err;
280
281    struct node_state *ns = b->st;
282
283    switch (cmd) {
284        case BENCH_CMD_CREATE_COPIES:
285            printf("# node %d: creating %d cap copies\n", my_core_id, arg);
286            ns->ram = cap1;
287            ns->numcopies = arg;
288            node_create_copies(ns);
289            err = bench_distops_cmd__tx(b, NOP_CONT, BENCH_CMD_COPIES_DONE, 0);
290            PANIC_IF_ERR(err, "signaling cap_copy() done\n");
291            break;
292        case BENCH_CMD_CAP_FOR_BENCH:
293            printf("# node %d: storing cap for benchmark\n", my_core_id);
294            ns->cap = cap1;
295            err = bench_distops_cmd__tx(b, NOP_CONT, BENCH_CMD_GOT_CAP, 0);
296            PANIC_IF_ERR(err, "signaling cap rx done\n");
297            break;
298        default:
299            printf("node %d got caps + command %"PRIu32", arg=%d:\n",
300                my_core_id, cmd, arg);
301            debug_capref("cap1:", cap1);
302            break;
303    }
304}
305