1/**
2 * \file
3 * \brief Benchmark revoke of cap with no remote relations
4 */
5
6/*
7 * Copyright (c) 2017, ETH Zurich.
8 * All rights reserved.
9 *
10 * This file is distributed under the terms in the attached LICENSE file.
11 * If you do not find this file, copies can be found by writing to:
12 * ETH Zurich D-INFK, Universitaetstr 6, CH-8092 Zurich. Attn: Systems Group.
13 */
14
15#include <barrelfish/barrelfish.h>
16#include <if/bench_distops_defs.h>
17
18#include <bitmacros.h>
19
20#include <bench/bench.h>
21#include <trace/trace.h>
22
23#include "benchapi.h"
24
25#define REVOKE_COPIES 10
26
27//{{{1 debugging helpers
28static void debug_capref(const char *prefix, struct capref cap)
29{
30    char buf[128];
31    debug_print_capref(buf, 128, cap);
32    printf("%s capref = %s\n", prefix, buf);
33}
34
35//{{{1 shared commands
36enum bench_cmd {
37    BENCH_CMD_CREATE_COPIES,
38    BENCH_CMD_COPIES_DONE,
39    BENCH_CMD_DO_REVOKE,
40    BENCH_CMD_PRINT_STATS,
41    BENCH_CMD_PRINT_DONE,
42};
43
44//{{{1 Managment node: implement orchestration for benchmark
45
46//{{{2 Management node: state management
47
48struct global_state {
49    struct capref ram;
50    coreid_t *nodes;
51    int nodes_seen;
52    int nodecount;
53    int copies_done;
54    int printnode;
55    int currcopies;
56};
57
58errval_t mgmt_init_benchmark(void **st, int nodecount)
59{
60     *st = calloc(1, sizeof(struct global_state));
61     if (!*st) {
62         return LIB_ERR_MALLOC_FAIL;
63     }
64     struct global_state *gs = *st;
65     gs->nodes = calloc(nodecount, sizeof(coreid_t));
66     gs->nodecount = nodecount;
67     gs->copies_done = 0;
68     gs->printnode = 0;
69     return ram_alloc(&gs->ram, BASE_PAGE_BITS);
70}
71
72static int sort_coreid(const void *a_, const void *b_)
73{
74    // deref pointers as coreids, store as ints
75    int a = *((coreid_t*)a_);
76    int b = *((coreid_t*)b_);
77    // subtract as ints
78    return a-b;
79}
80
81void mgmt_register_node(void *st, coreid_t nodeid)
82{
83    struct global_state *gs = st;
84    gs->nodes[gs->nodes_seen++] = nodeid;
85    // if we've seen all nodes, sort nodes array and configure printnode
86    if (gs->nodes_seen == gs->nodecount) {
87        qsort(gs->nodes, gs->nodecount, sizeof(coreid_t), sort_coreid);
88    }
89}
90
91struct mgmt_node_state {
92};
93
94errval_t mgmt_init_node(void **st)
95{
96     *st = malloc(sizeof(struct mgmt_node_state));
97     if (!*st) {
98         return LIB_ERR_MALLOC_FAIL;
99     }
100    return SYS_ERR_OK;
101}
102
103//{{{2 Management node: benchmark impl
104void mgmt_run_benchmark(void *st)
105{
106    struct global_state *gs = st;
107
108    printf("All clients sent hello! Benchmark starting...\n");
109
110    printf("# Benchmarking REVOKE NO REMOTE: nodes=%d\n", gs->nodecount);
111
112    printf("# Starting out with %d copies, will double up to %d...\n",
113            NUM_COPIES_START, NUM_COPIES_END);
114
115    TRACE(CAPOPS, START, 0);
116
117    gs->currcopies = NUM_COPIES_START;
118    broadcast_caps(BENCH_CMD_CREATE_COPIES, NUM_COPIES_START, gs->ram);
119}
120
121void mgmt_cmd(uint32_t cmd, uint32_t arg, struct bench_distops_binding *b)
122{
123    struct global_state *gs = get_global_state(b);
124    switch(cmd) {
125        case BENCH_CMD_COPIES_DONE:
126            gs->copies_done++;
127            if (gs->copies_done == gs->nodecount) {
128                printf("# All copies made!\n");
129                broadcast_cmd(BENCH_CMD_DO_REVOKE, ITERS);
130                printf("# sending initial print command to %d\n", gs->printnode);
131                unicast_cmd(gs->nodes[gs->printnode++], BENCH_CMD_PRINT_STATS, 0);
132            }
133            break;
134        case BENCH_CMD_PRINT_DONE:
135            if (gs->printnode == gs->nodecount) {
136                if (gs->currcopies == NUM_COPIES_END) {
137                    printf("# Benchmark done!\n");
138                    TRACE(CAPOPS, STOP, 0);
139                    mgmt_trace_flush(NOP_CONT);
140                    return;
141                }
142                printf("# Round done!\n");
143                // Reset counters for next round
144                gs->currcopies *= 2;
145                gs->copies_done = 0;
146                gs->printnode = 0;
147                // Start new round
148                broadcast_cmd(BENCH_CMD_CREATE_COPIES, gs->currcopies);
149                return;
150            }
151            printf("# sending print command to node %d\n", gs->printnode);
152            unicast_cmd(gs->nodes[gs->printnode++], BENCH_CMD_PRINT_STATS, 0);
153            break;
154        default:
155            printf("mgmt node got unknown command %d over binding %p\n", cmd, b);
156            break;
157    }
158}
159
160void mgmt_cmd_caps(uint32_t cmd, uint32_t arg, struct capref cap1,
161                   struct bench_distops_binding *b)
162{
163    printf("mgmt node got caps + command %"PRIu32", arg=%d over binding %p:\n",
164            cmd, arg, b);
165    debug_capref("cap1:", cap1);
166}
167
168//{{{1 Node
169
170struct node_state {
171    struct capref cap;
172    struct capref ram;
173    uint32_t numcopies;
174    struct capref *copies;
175    uint64_t *delcycles;
176    uint32_t benchcount;
177};
178
179static coreid_t my_core_id = -1;
180
181void init_node(struct bench_distops_binding *b)
182{
183    printf("%s: binding = %p\n", __FUNCTION__, b);
184
185    my_core_id = disp_get_core_id();
186
187    bench_init();
188
189    // Allocate client state struct
190    b->st = malloc(sizeof(struct node_state));
191    assert(b->st);
192    if (!b->st) {
193        USER_PANIC("state malloc() in client");
194    }
195}
196
197static void node_create_copies(struct node_state *ns)
198{
199    errval_t err;
200    ns->copies = calloc(ns->numcopies, sizeof(struct capref));
201    for (int i = 0; i < ns->numcopies; i++) {
202        err = slot_alloc(&ns->copies[i]);
203        PANIC_IF_ERR(err, "slot_alloc for copy %d\n", i);
204        err = cap_copy(ns->copies[i], ns->ram);
205        PANIC_IF_ERR(err, "cap_copy for copy %d\n", i);
206    }
207}
208
209static size_t get_mdb_size(void)
210{
211    errval_t err;
212    size_t cap_base_count = 0;
213    err = sys_debug_get_mdb_size(&cap_base_count);
214    assert(err_is_ok(err));
215    return cap_base_count;
216}
217
218void node_cmd(uint32_t cmd, uint32_t arg, struct bench_distops_binding *b)
219{
220    struct node_state *ns = b->st;
221    errval_t err;
222
223    switch(cmd) {
224        case BENCH_CMD_CREATE_COPIES:
225            printf("# node %d: creating %d cap copies\n", my_core_id, arg);
226            ns->numcopies = arg;
227            node_create_copies(ns);
228            printf("# node %d: %zu capabilities on node\n", my_core_id, get_mdb_size());
229            err = bench_distops_cmd__tx(b, NOP_CONT, BENCH_CMD_COPIES_DONE, 1);
230            PANIC_IF_ERR(err, "signaling cap_copy() done\n");
231            break;
232        case BENCH_CMD_DO_REVOKE:
233            // printf("# node %d: doing %d revokes\n", my_core_id, arg);
234            ns->benchcount = arg;
235            ns->delcycles = calloc(arg, sizeof(uint64_t));
236            assert(ns->delcycles);
237            // printf("# node %d: allocated delcycles array\n", my_core_id);
238            struct capref cap;
239            err = ram_alloc(&cap, BASE_PAGE_BITS);
240            assert(err_is_ok(err));
241            // TODO: parametrize
242            ns->benchcount = arg;
243            // printf("# node %d: creating %d slots for copies of cap-to-revoke\n",
244            //        my_core_id, REVOKE_COPIES);
245            struct capref copies[REVOKE_COPIES];
246            for (int c = 0; c < REVOKE_COPIES; c++) {
247                err = slot_alloc(&copies[c]);
248                assert(err_is_ok(err));
249            }
250            // printf("# node %d: starting benchmark iterations\n", my_core_id);
251            for (int i = 0; i < ns->benchcount; i++) {
252                uint64_t start, end;
253                // Make some copies to be deleted during revoke
254                // printf("# node %d: creating copies\n", my_core_id);
255                for (int c = 0; c < REVOKE_COPIES; c++) {
256                    err = cap_copy(copies[c], cap);
257                    PANIC_IF_ERR(err, "creating copy for revoke");
258                    assert(err_is_ok(err));
259                }
260                // printf("# node %d: doing revoke\n", my_core_id);
261                start = bench_tsc();
262                TRACE(CAPOPS, USER_REVOKE_CALL, (ns->numcopies << 16) | i);
263                err = cap_revoke(cap);
264                TRACE(CAPOPS, USER_REVOKE_RESP, (ns->numcopies << 16) | i);
265                end = bench_tsc();
266                ns->delcycles[i] = end - start;
267                assert(err_is_ok(err));
268                PANIC_IF_ERR(err, "# core %d: revoke failed", my_core_id);
269                if (i % (ns->benchcount / 10) == 0) {
270                    // printf("# node %d: %d percent done\n", my_core_id, i / (ns->benchcount/100));
271                }
272            }
273            err = cap_destroy(cap);
274            assert(err_is_ok(err));
275            PANIC_IF_ERR(err, "# core %d: final cap_destroy", my_core_id);
276            for (int c = 0; c < REVOKE_COPIES; c++) {
277                err = slot_free(copies[c]);
278                PANIC_IF_ERR(err, "# core %d: slot_free", my_core_id);
279                assert(err_is_ok(err));
280            }
281            // printf("# node %d: revokes done\n", my_core_id);
282            break;
283        case BENCH_CMD_PRINT_STATS:
284            printf("# node %d: tsc_per_us = %ld; numcopies = %d\n",
285                    my_core_id, bench_tsc_per_us(), ns->numcopies);
286            printf("# delete latency in cycles\n");
287            for (int i = 0; i < ns->benchcount; i++) {
288                printf("%ld\n", ns->delcycles[i]);
289            }
290            err = bench_distops_cmd__tx(b, NOP_CONT, BENCH_CMD_PRINT_DONE, 0);
291            assert(err_is_ok(err));
292            // Cleanup before next round
293            for (int i = 0; i < ns->numcopies; i++) {
294                err = cap_destroy(ns->copies[i]);
295                assert(err_is_ok(err));
296            }
297            free(ns->copies);
298            free(ns->delcycles);
299            break;
300        default:
301            printf("node %d got command %"PRIu32"\n", my_core_id, cmd);
302            break;
303    }
304}
305
306void node_cmd_caps(uint32_t cmd, uint32_t arg, struct capref cap1,
307                   struct bench_distops_binding *b)
308{
309    errval_t err;
310    struct node_state *ns = b->st;
311
312    switch (cmd) {
313        case BENCH_CMD_CREATE_COPIES:
314            printf("# node %d: creating %d cap copies\n", my_core_id, arg);
315            ns->ram = cap1;
316            ns->numcopies = arg;
317            node_create_copies(ns);
318            printf("# node %d: %zu caps on node\n", my_core_id, get_mdb_size());
319            err = bench_distops_cmd__tx(b, NOP_CONT, BENCH_CMD_COPIES_DONE, 0);
320            PANIC_IF_ERR(err, "signaling cap_copy() done\n");
321            break;
322        default:
323            printf("node %d got caps + command %"PRIu32", arg=%d:\n",
324                my_core_id, cmd, arg);
325            debug_capref("cap1:", cap1);
326            break;
327    }
328}
329