1/**
2 * \file
3 * \brief Delete foreign copy of cap
4 */
5
6/*
7 * Copyright (c) 2017, ETH Zurich.
8 * All rights reserved.
9 *
10 * This file is distributed under the terms in the attached LICENSE file.
11 * If you do not find this file, copies can be found by writing to:
12 * ETH Zurich D-INFK, Universitaetstr 6, CH-8092 Zurich. Attn: Systems Group.
13 */
14
15#include <barrelfish/barrelfish.h>
16#include <if/bench_distops_defs.h>
17
18#include <bitmacros.h>
19
20#include <bench/bench.h>
21#include <trace/trace.h>
22
23#include "benchapi.h"
24
25//{{{1 debugging helpers
26static void debug_capref(const char *prefix, struct capref cap)
27{
28    char buf[128];
29    debug_print_capref(buf, 128, cap);
30    printf("%s capref = %s\n", prefix, buf);
31}
32
33//{{{1 shared commands
34enum bench_cmd {
35    BENCH_CMD_CREATE_COPIES,
36    BENCH_CMD_COPIES_DONE,
37    BENCH_CMD_CAP_FOR_BENCH,
38    BENCH_CMD_GOT_CAP,
39    BENCH_CMD_DO_DELETE,
40    BENCH_CMD_PRINT_STATS,
41    BENCH_CMD_PRINT_DONE,
42};
43
44//{{{1 Managment node: implement orchestration for benchmark
45
46//{{{2 Management node: state management
47
48struct global_state {
49    struct capref ram;
50    struct capref cap;
51    coreid_t *nodes;
52    int nodes_seen;
53    int nodecount;
54    int copies_done;
55    int printnode;
56    int currcopies;
57};
58
59errval_t mgmt_init_benchmark(void **st, int nodecount)
60{
61     *st = calloc(1, sizeof(struct global_state));
62     if (!*st) {
63         return LIB_ERR_MALLOC_FAIL;
64     }
65     struct global_state *gs = *st;
66     errval_t err;
67     gs->nodes = calloc(nodecount, sizeof(coreid_t));
68     gs->nodecount = nodecount;
69     gs->copies_done = 0;
70     gs->printnode = 0;
71     err = ram_alloc(&gs->ram, BASE_PAGE_BITS);
72     if (err_is_fail(err)) {
73         return err;
74     }
75     err = ram_alloc(&gs->cap, BASE_PAGE_BITS);
76     return err;
77}
78
79static int sort_coreid(const void *a_, const void *b_)
80{
81    // deref pointers as coreids, store as ints
82    int a = *((coreid_t*)a_);
83    int b = *((coreid_t*)b_);
84    // subtract as ints
85    return a-b;
86}
87
88void mgmt_register_node(void *st, coreid_t nodeid)
89{
90    struct global_state *gs = st;
91    gs->nodes[gs->nodes_seen++] = nodeid;
92    // if we've seen all nodes, sort nodes array and configure printnode
93    if (gs->nodes_seen == gs->nodecount) {
94        qsort(gs->nodes, gs->nodecount, sizeof(coreid_t), sort_coreid);
95    }
96}
97
98
99struct mgmt_node_state {
100};
101
102errval_t mgmt_init_node(void **st)
103{
104     *st = malloc(sizeof(struct mgmt_node_state));
105     if (!*st) {
106         return LIB_ERR_MALLOC_FAIL;
107     }
108    return SYS_ERR_OK;
109}
110
111//{{{2 Management node: benchmark impl
112void mgmt_run_benchmark(void *st)
113{
114    struct global_state *gs = st;
115
116    printf("All clients sent hello! Benchmark starting...\n");
117
118    printf("# Benchmarking DELETE FOREIGN COPY: nodes=%d\n", gs->nodecount);
119
120    printf("# Starting out with %d copies, will increase by factor of two up to %d...\n",
121            NUM_COPIES_START, NUM_COPIES_END);
122
123    TRACE(CAPOPS, START, 0);
124
125    gs->currcopies = NUM_COPIES_START;
126    broadcast_caps(BENCH_CMD_CREATE_COPIES, NUM_COPIES_START, gs->ram);
127}
128
129void mgmt_cmd(uint32_t cmd, uint32_t arg, struct bench_distops_binding *b)
130{
131    struct global_state *gs = get_global_state(b);
132    switch(cmd) {
133        case BENCH_CMD_COPIES_DONE:
134            if (arg == 1) {
135                gs->copies_done++;
136                if (gs->copies_done == gs->nodecount) {
137                    printf("# All copies made!\n");
138                    broadcast_cmd(BENCH_CMD_DO_DELETE, ITERS);
139                    unicast_cmd(gs->nodes[gs->printnode++], BENCH_CMD_PRINT_STATS, 0);
140                }
141            } else {
142                bench_distops_caps__tx(b, NOP_CONT, BENCH_CMD_CAP_FOR_BENCH,
143                        0, gs->cap);
144            }
145            break;
146        case BENCH_CMD_GOT_CAP:
147            gs->copies_done++;
148            if (gs->copies_done == gs->nodecount) {
149                printf("# All copies made!\n");
150                broadcast_cmd(BENCH_CMD_DO_DELETE, ITERS);
151                unicast_cmd(gs->nodes[gs->printnode++], BENCH_CMD_PRINT_STATS, 0);
152            }
153            break;
154        case BENCH_CMD_PRINT_DONE:
155            if (gs->printnode == gs->nodecount) {
156                if (gs->currcopies == NUM_COPIES_END) {
157                    printf("# Benchmark done!\n");
158                    TRACE(CAPOPS, STOP, 0);
159                    mgmt_trace_flush(NOP_CONT);
160                    return;
161                }
162                printf("# Round done!\n");
163                // Reset counters for next round
164                gs->currcopies *= 2;
165                gs->copies_done = 0;
166                gs->printnode = 0;
167                // Start new round
168                broadcast_cmd(BENCH_CMD_CREATE_COPIES, gs->currcopies);
169                return;
170            }
171            unicast_cmd(gs->nodes[gs->printnode++], BENCH_CMD_PRINT_STATS, 0);
172            break;
173        default:
174            printf("mgmt node got unknown command %d over binding %p\n", cmd, b);
175            break;
176    }
177}
178
179void mgmt_cmd_caps(uint32_t cmd, uint32_t arg, struct capref cap1,
180                   struct bench_distops_binding *b)
181{
182    printf("mgmt node got caps + command %"PRIu32", arg=%d over binding %p:\n",
183            cmd, arg, b);
184    debug_capref("cap1:", cap1);
185}
186
187//{{{1 Node
188
189struct node_state {
190    struct capref cap;
191    struct capref ram;
192    uint32_t numcopies;
193    struct capref *copies;
194    uint64_t *delcycles;
195    uint32_t benchcount;
196};
197
198static coreid_t my_core_id = -1;
199
200void init_node(struct bench_distops_binding *b)
201{
202    printf("%s: binding = %p\n", __FUNCTION__, b);
203
204    my_core_id = disp_get_core_id();
205
206    bench_init();
207
208    // Allocate client state struct
209    b->st = malloc(sizeof(struct node_state));
210    assert(b->st);
211    if (!b->st) {
212        USER_PANIC("state malloc() in client");
213    }
214}
215
216static void node_create_copies(struct node_state *ns)
217{
218    errval_t err;
219    ns->copies = calloc(ns->numcopies, sizeof(struct capref));
220    for (int i = 0; i < ns->numcopies; i++) {
221        err = slot_alloc(&ns->copies[i]);
222        PANIC_IF_ERR(err, "slot_alloc for copy %d\n", i);
223        err = cap_copy(ns->copies[i], ns->ram);
224        PANIC_IF_ERR(err, "cap_copy for copy %d\n", i);
225    }
226}
227
228void node_cmd(uint32_t cmd, uint32_t arg, struct bench_distops_binding *b)
229{
230    struct node_state *ns = b->st;
231    errval_t err;
232    size_t cap_base_count = 0;
233    err = sys_debug_get_mdb_size(&cap_base_count);
234    assert(err_is_ok(err));
235
236    switch(cmd) {
237        case BENCH_CMD_CREATE_COPIES:
238            printf("# node %d: %zu caps before creating copies\n", my_core_id, cap_base_count);
239            printf("# node %d: creating %d cap copies\n", my_core_id, arg);
240            ns->numcopies = arg;
241            node_create_copies(ns);
242            err = bench_distops_cmd__tx(b, NOP_CONT, BENCH_CMD_COPIES_DONE, 1);
243            PANIC_IF_ERR(err, "signaling cap_copy() done\n");
244            break;
245        case BENCH_CMD_DO_DELETE:
246            ns->benchcount = arg;
247            ns->delcycles = calloc(arg, sizeof(uint64_t));
248            assert(ns->delcycles);
249            struct capref slot;
250            err = slot_alloc(&slot);
251            assert(err_is_ok(err));
252            err = cap_copy(slot, ns->cap);
253            assert(err_is_ok(err));
254            //printf("node %d: doing delete\n", my_core_id);
255            for (int i = 0; i < ns->benchcount; i++) {
256                uint64_t start, end;
257                start = bench_tsc();
258                TRACE(CAPOPS, USER_DELETE_CALL, (ns->numcopies << 16) | i);
259                err = cap_delete(slot);
260                TRACE(CAPOPS, USER_DELETE_RESP, (ns->numcopies << 16) | i);
261                end = bench_tsc();
262                ns->delcycles[i] = end - start;
263                assert(err_is_ok(err));
264                err = cap_copy(slot, ns->cap);
265                assert(err_is_ok(err));
266            }
267            //printf("node %d: deletes done\n", my_core_id);
268            break;
269        case BENCH_CMD_PRINT_STATS:
270            printf("# node %d: tsc_per_us = %ld; numcopies = %d\n", my_core_id, bench_tsc_per_us(), ns->numcopies);
271            printf("# delete latency in cycles\n");
272            for (int i = 0; i < ns->benchcount; i++) {
273                printf("%ld\n", ns->delcycles[i]);
274            }
275            err = bench_distops_cmd__tx(b, NOP_CONT, BENCH_CMD_PRINT_DONE, 0);
276            assert(err_is_ok(err));
277            // Cleanup before next round
278            for (int i = 0; i < ns->numcopies; i++) {
279                err = cap_destroy(ns->copies[i]);
280                assert(err_is_ok(err));
281            }
282            free(ns->copies);
283            free(ns->delcycles);
284            break;
285        default:
286            printf("node %d got command %"PRIu32"\n", my_core_id, cmd);
287            break;
288    }
289}
290
291void node_cmd_caps(uint32_t cmd, uint32_t arg, struct capref cap1,
292                   struct bench_distops_binding *b)
293{
294    errval_t err;
295
296    struct node_state *ns = b->st;
297
298    switch (cmd) {
299        case BENCH_CMD_CREATE_COPIES:
300            printf("# node %d: creating %d cap copies\n", my_core_id, arg);
301            ns->ram = cap1;
302            ns->numcopies = arg;
303            node_create_copies(ns);
304            err = bench_distops_cmd__tx(b, NOP_CONT, BENCH_CMD_COPIES_DONE, 0);
305            PANIC_IF_ERR(err, "signaling cap_copy() done\n");
306            break;
307        case BENCH_CMD_CAP_FOR_BENCH:
308            printf("# node %d: storing cap for benchmark\n", my_core_id);
309            ns->cap = cap1;
310            err = bench_distops_cmd__tx(b, NOP_CONT, BENCH_CMD_GOT_CAP, 0);
311            PANIC_IF_ERR(err, "signaling cap rx done\n");
312            break;
313        default:
314            printf("node %d got caps + command %"PRIu32", arg=%d:\n",
315                my_core_id, cmd, arg);
316            debug_capref("cap1:", cap1);
317            break;
318    }
319}
320