1/**
2 * \file
3 * \brief Benchmark retype with local descendants present for source cap
4 */
5
6/*
7 * Copyright (c) 2017, ETH Zurich.
8 * All rights reserved.
9 *
10 * This file is distributed under the terms in the attached LICENSE file.
11 * If you do not find this file, copies can be found by writing to:
12 * ETH Zurich D-INFK, Universitaetstr 6, CH-8092 Zurich. Attn: Systems Group.
13 */
14
15#include <barrelfish/barrelfish.h>
16#include <if/bench_distops_defs.h>
17
18#include <bitmacros.h>
19
20#include <bench/bench.h>
21#include <trace/trace.h>
22
23#include "benchapi.h"
24
25//{{{1 debugging helpers
26static void debug_capref(const char *prefix, struct capref cap)
27{
28    char buf[128];
29    debug_print_capref(buf, 128, cap);
30    printf("%s capref = %s\n", prefix, buf);
31}
32
33//{{{1 shared commands
34enum bench_cmd {
35    BENCH_CMD_CREATE_COPIES,
36    BENCH_CMD_COPIES_DONE,
37    BENCH_CMD_DO_RETYPE,
38    BENCH_CMD_PRINT_STATS,
39    BENCH_CMD_PRINT_DONE,
40};
41
42//{{{1 Managment node: implement orchestration for benchmark
43
44//{{{2 Management node: state management
45
46struct global_state {
47    struct capref ram;
48    coreid_t *nodes;
49    int nodes_seen;
50    int nodecount;
51    int copies_done;
52    int printnode;
53    int currcopies;
54};
55
56errval_t mgmt_init_benchmark(void **st, int nodecount)
57{
58     *st = malloc(sizeof(struct global_state));
59     if (!*st) {
60         return LIB_ERR_MALLOC_FAIL;
61     }
62     struct global_state *gs = *st;
63     gs->nodes = calloc(nodecount, sizeof(coreid_t));
64     gs->nodecount = nodecount;
65     gs->copies_done = 0;
66     gs->printnode = 0;
67     return ram_alloc(&gs->ram, BASE_PAGE_BITS);
68}
69
70static int sort_coreid(const void *a_, const void *b_)
71{
72    // deref pointers as coreids, store as ints
73    int a = *((coreid_t*)a_);
74    int b = *((coreid_t*)b_);
75    // subtract as ints
76    return a-b;
77}
78
79void mgmt_register_node(void *st, coreid_t nodeid)
80{
81    struct global_state *gs = st;
82    gs->nodes[gs->nodes_seen++] = nodeid;
83    // if we've seen all nodes, sort nodes array and configure printnode
84    if (gs->nodes_seen == gs->nodecount) {
85        qsort(gs->nodes, gs->nodecount, sizeof(coreid_t), sort_coreid);
86    }
87}
88
89struct mgmt_node_state {
90};
91
92errval_t mgmt_init_node(void **st)
93{
94     *st = malloc(sizeof(struct mgmt_node_state));
95     if (!*st) {
96         return LIB_ERR_MALLOC_FAIL;
97     }
98    return SYS_ERR_OK;
99}
100
101//{{{2 Management node: benchmark impl
102void mgmt_run_benchmark(void *st)
103{
104    struct global_state *gs = st;
105
106    printf("All clients sent hello! Benchmark starting...\n");
107
108    printf("# Benchmarking RETYPE WITH LOCAL DESCS: nodes=%d\n", gs->nodecount);
109
110    printf("# Starting out with %d copies, will increase by factors of two up to %d...\n",
111            NUM_COPIES_START, NUM_COPIES_END);
112
113    TRACE(CAPOPS, START, 0);
114
115    gs->currcopies = NUM_COPIES_START;
116    broadcast_caps(BENCH_CMD_CREATE_COPIES, NUM_COPIES_START, gs->ram);
117}
118
119void mgmt_cmd(uint32_t cmd, uint32_t arg, struct bench_distops_binding *b)
120{
121    struct global_state *gs = get_global_state(b);
122    switch(cmd) {
123        case BENCH_CMD_COPIES_DONE:
124            gs->copies_done++;
125            if (gs->copies_done == gs->nodecount) {
126                printf("# All copies made!\n");
127                broadcast_cmd(BENCH_CMD_DO_RETYPE, ITERS);
128                unicast_cmd(gs->nodes[gs->printnode++], BENCH_CMD_PRINT_STATS, 0);
129            }
130            break;
131        case BENCH_CMD_PRINT_DONE:
132            if (gs->printnode == gs->nodecount) {
133                if (gs->currcopies == NUM_COPIES_END) {
134                    printf("# Benchmark done!\n");
135                    TRACE(CAPOPS, STOP, 0);
136                    mgmt_trace_flush(NOP_CONT);
137                    return;
138                }
139                printf("# Round done!\n");
140                // Reset counters for next round
141                gs->currcopies *= 2;
142                gs->copies_done = 0;
143                gs->printnode = 0;
144                // Start new round
145                broadcast_cmd(BENCH_CMD_CREATE_COPIES, gs->currcopies);
146                return;
147            }
148            unicast_cmd(gs->nodes[gs->printnode++], BENCH_CMD_PRINT_STATS, 0);
149            break;
150        default:
151            printf("mgmt node got unknown command %d over binding %p\n", cmd, b);
152            break;
153    }
154}
155
156void mgmt_cmd_caps(uint32_t cmd, uint32_t arg, struct capref cap1,
157                   struct bench_distops_binding *b)
158{
159    printf("mgmt node got caps + command %"PRIu32", arg=%d over binding %p:\n",
160            cmd, arg, b);
161    debug_capref("cap1:", cap1);
162}
163
164//{{{1 Node
165
166struct node_state {
167    struct capref cap;
168    struct capref desc;
169    struct capref ram;
170    uint32_t numcopies;
171    struct capref *copies;
172    uint64_t *delcycles;
173    uint32_t benchcount;
174};
175
176static coreid_t my_core_id = -1;
177
178void init_node(struct bench_distops_binding *b)
179{
180    printf("%s: binding = %p\n", __FUNCTION__, b);
181
182    my_core_id = disp_get_core_id();
183
184    bench_init();
185
186    // Allocate client state struct
187    b->st = malloc(sizeof(struct node_state));
188    assert(b->st);
189    if (!b->st) {
190        USER_PANIC("state malloc() in client");
191    }
192}
193
194static void node_create_copies(struct node_state *ns)
195{
196    errval_t err;
197    ns->copies = calloc(ns->numcopies, sizeof(struct capref));
198    for (int i = 0; i < ns->numcopies; i++) {
199        err = slot_alloc(&ns->copies[i]);
200        PANIC_IF_ERR(err, "slot_alloc for copy %d\n", i);
201        err = cap_copy(ns->copies[i], ns->ram);
202        PANIC_IF_ERR(err, "cap_copy for copy %d\n", i);
203    }
204}
205
206static size_t get_mdb_size(void)
207{
208    errval_t err;
209    size_t cap_base_count = 0;
210    err = sys_debug_get_mdb_size(&cap_base_count);
211    assert(err_is_ok(err));
212    return cap_base_count;
213}
214
215void node_cmd(uint32_t cmd, uint32_t arg, struct bench_distops_binding *b)
216{
217    struct node_state *ns = b->st;
218    errval_t err;
219
220    switch(cmd) {
221        case BENCH_CMD_CREATE_COPIES:
222            printf("# node %d: creating %d cap copies\n", my_core_id, arg);
223            ns->numcopies = arg;
224            node_create_copies(ns);
225            printf("# node %d: %zu capabilities on node\n", my_core_id, get_mdb_size());
226            err = bench_distops_cmd__tx(b, NOP_CONT, BENCH_CMD_COPIES_DONE, 1);
227            PANIC_IF_ERR(err, "signaling cap_copy() done\n");
228            break;
229        case BENCH_CMD_DO_RETYPE:
230            ns->benchcount = arg;
231            ns->delcycles = calloc(arg, sizeof(uint64_t));
232            assert(ns->delcycles);
233            struct capref slot;
234            err = slot_alloc(&slot);
235            assert(err_is_ok(err));
236            //printf("node %d: doing delete\n", my_core_id);
237            for (int i = 0; i < ns->benchcount; i++) {
238                uint64_t start, end;
239                start = bench_tsc();
240                // Cycle through pages in source cap
241                TRACE(CAPOPS, USER_RETYPE_CALL, (ns->numcopies << 16) | i);
242                err = cap_retype(slot, ns->cap,
243                        (i*BASE_PAGE_SIZE) % LARGE_PAGE_SIZE,
244                        ObjType_Frame, BASE_PAGE_SIZE, 1);
245                TRACE(CAPOPS, USER_RETYPE_RESP, (ns->numcopies << 16) | i);
246                end = bench_tsc();
247                ns->delcycles[i] = end - start;
248                assert(err_is_ok(err));
249                err = cap_delete(slot);
250                assert(err_is_ok(err));
251            }
252            //printf("node %d: deletes done\n", my_core_id);
253            break;
254        case BENCH_CMD_PRINT_STATS:
255            printf("# node %d: tsc_per_us = %ld; numcopies = %d\n",
256                    my_core_id, bench_tsc_per_us(), ns->numcopies);
257            printf("# retype latency in cycles\n");
258            for (int i = 0; i < ns->benchcount; i++) {
259                printf("%ld\n", ns->delcycles[i]);
260            }
261            err = bench_distops_cmd__tx(b, NOP_CONT, BENCH_CMD_PRINT_DONE, 0);
262            assert(err_is_ok(err));
263            // Cleanup before next round
264            for (int i = 0; i < ns->numcopies; i++) {
265                err = cap_destroy(ns->copies[i]);
266                assert(err_is_ok(err));
267            }
268            free(ns->copies);
269            free(ns->delcycles);
270            break;
271        default:
272            printf("node %d got command %"PRIu32"\n", my_core_id, cmd);
273            break;
274    }
275}
276
277void node_cmd_caps(uint32_t cmd, uint32_t arg, struct capref cap1,
278                   struct bench_distops_binding *b)
279{
280    errval_t err;
281    struct node_state *ns = b->st;
282
283    switch (cmd) {
284        case BENCH_CMD_CREATE_COPIES:
285            printf("# node %d: creating %d cap copies\n", my_core_id, arg);
286            ns->ram = cap1;
287            ns->numcopies = arg;
288            node_create_copies(ns);
289            // First round, also get a local RAM cap for the benchmark later
290            err = ram_alloc(&ns->cap, LARGE_PAGE_BITS+1);
291            assert(err_is_ok(err));
292            err = slot_alloc(&ns->desc);
293            assert(err_is_ok(err));
294            err=  cap_retype(ns->desc, ns->cap, LARGE_PAGE_SIZE, ObjType_RAM,
295                    LARGE_PAGE_SIZE, 1);
296            assert(err_is_ok(err));
297            printf("# node %d: %zu caps on node\n", my_core_id, get_mdb_size());
298            err = bench_distops_cmd__tx(b, NOP_CONT, BENCH_CMD_COPIES_DONE, 0);
299            PANIC_IF_ERR(err, "signaling cap_copy() done\n");
300            break;
301        default:
302            printf("node %d got caps + command %"PRIu32", arg=%d:\n",
303                my_core_id, cmd, arg);
304            debug_capref("cap1:", cap1);
305            break;
306    }
307}
308