1/** 2 * \file 3 * \brief delete last copy of moveable cap on owned core with copies existing 4 * on other cores 5 */ 6 7/* 8 * Copyright (c) 2017, ETH Zurich. 9 * All rights reserved. 10 * 11 * This file is distributed under the terms in the attached LICENSE file. 12 * If you do not find this file, copies can be found by writing to: 13 * ETH Zurich D-INFK, Universitaetstr 6, CH-8092 Zurich. Attn: Systems Group. 14 */ 15 16#include <barrelfish/barrelfish.h> 17#include <if/bench_distops_defs.h> 18 19#include <bitmacros.h> 20 21#include <bench/bench.h> 22#include <trace/trace.h> 23 24//#define DEBUG_PROTOCOL 25#include "benchapi.h" 26 27//{{{1 debugging helpers 28static void debug_capref(const char *prefix, struct capref cap) 29{ 30 char buf[128]; 31 debug_print_capref(buf, 128, cap); 32 printf("%s capref = %s\n", prefix, buf); 33} 34 35//{{{1 shared commands 36enum bench_cmd { 37 BENCH_CMD_CREATE_COPIES, 38 BENCH_CMD_COPIES_DONE, 39 BENCH_CMD_DO_ALLOC, 40 BENCH_CMD_DO_DELETE, 41 BENCH_CMD_FORWARD_COPIES, 42 BENCH_CMD_COPIES_RX_DONE, 43 BENCH_CMD_PRINT_DONE, 44}; 45 46//{{{1 shared helper functions 47static size_t get_mdb_size(void) 48{ 49 errval_t err; 50 size_t cap_base_count = 0; 51 err = sys_debug_get_mdb_size(&cap_base_count); 52 assert(err_is_ok(err)); 53 return cap_base_count; 54} 55 56 57//{{{1 Managment node: implement orchestration for benchmark 58 59//{{{2 Management node: state management 60 61struct global_state { 62 struct capref ram; 63 struct capref fwdcap; 64 coreid_t *nodes; 65 int nodes_seen; 66 int nodecount; 67 int copies_done; 68 int copycount; 69 int rx_seen; 70 int masternode; 71 int currcopies; 72}; 73 74errval_t mgmt_init_benchmark(void **st, int nodecount) 75{ 76 *st = calloc(1, sizeof(struct global_state)); 77 if (!*st) { 78 return LIB_ERR_MALLOC_FAIL; 79 } 80 struct global_state *gs = *st; 81 gs->nodes = calloc(nodecount, sizeof(coreid_t)); 82 gs->nodecount = nodecount; 83 gs->copies_done = 0; 84 gs->rx_seen = 0; 85 gs->masternode = -1; 86 return ram_alloc(&gs->ram, BASE_PAGE_BITS); 87} 88 89static int sort_coreid(const void *a_, const void *b_) 90{ 91 // deref pointers as coreids, store as ints 92 int a = *((coreid_t*)a_); 93 int b = *((coreid_t*)b_); 94 // subtract as ints 95 return a-b; 96} 97 98void mgmt_register_node(void *st, coreid_t nodeid) 99{ 100 struct global_state *gs = st; 101 gs->nodes[gs->nodes_seen++] = nodeid; 102 // if we've seen all nodes, sort nodes array and configure printnode 103 if (gs->nodes_seen == gs->nodecount) { 104 qsort(gs->nodes, gs->nodecount, sizeof(coreid_t), sort_coreid); 105 gs->masternode = gs->nodes[0]; 106 } 107} 108 109struct mgmt_node_state { 110}; 111 112errval_t mgmt_init_node(void **st) 113{ 114 *st = malloc(sizeof(struct mgmt_node_state)); 115 if (!*st) { 116 return LIB_ERR_MALLOC_FAIL; 117 } 118 return SYS_ERR_OK; 119} 120 121//{{{2 Management node: benchmark impl 122void mgmt_run_benchmark(void *st) 123{ 124 struct global_state *gs = st; 125 126 printf("All clients sent hello! Benchmark starting...\n"); 127 128 printf("# Benchmarking DELETE LAST: nodes=%d\n", gs->nodecount); 129 130 printf("# Starting out with %d copies, will by powers of 2 up to %d...\n", 131 NUM_COPIES_START, NUM_COPIES_END); 132 133 TRACE(CAPOPS, START, 0); 134 135 gs->currcopies = NUM_COPIES_START; 136 broadcast_caps(BENCH_CMD_CREATE_COPIES, NUM_COPIES_START, gs->ram); 137} 138 139void mgmt_cmd(uint32_t cmd, uint32_t arg, struct bench_distops_binding *b) 140{ 141 errval_t err; 142 struct global_state *gs = get_global_state(b); 143 144 switch(cmd) { 145 case BENCH_CMD_COPIES_DONE: 146 gs->copies_done++; 147 if (gs->copies_done == gs->nodecount) { 148 printf("# All copies made!\n"); 149 unicast_cmd(gs->masternode, BENCH_CMD_DO_ALLOC, ITERS); 150 } 151 break; 152 case BENCH_CMD_COPIES_RX_DONE: 153 gs->rx_seen++; 154 DEBUG("got BENCH_CMD_COPIES_RX_DONE: seen = %d, expected = %d\n", 155 gs->rx_seen, gs->copycount); 156 if (gs->rx_seen == gs->copycount) { 157 DEBUG("# All nodes have copies of cap-to-del\n"); 158 err = cap_destroy(gs->fwdcap); 159 assert(err_is_ok(err)); 160 gs->rx_seen = 0; 161 unicast_cmd(gs->masternode, BENCH_CMD_DO_DELETE, 0); 162 } 163 break; 164 case BENCH_CMD_PRINT_DONE: 165 if (gs->currcopies == NUM_COPIES_END) { 166 printf("# Benchmark done!\n"); 167 // make sure last chunk of traces is flushed 168 TRACE(CAPOPS, STOP, 0); 169 mgmt_trace_flush(NOP_CONT); 170 return; 171 } 172 printf("# Round done!\n"); 173 printf("# mgmt node mdb size: %zu\n", get_mdb_size()); 174 // Reset counters for next round 175 gs->currcopies *= 2; 176 gs->copies_done = 0; 177 gs->rx_seen = 0; 178 // Start new round 179 broadcast_cmd(BENCH_CMD_CREATE_COPIES, gs->currcopies); 180 break; 181 default: 182 printf("mgmt node got unknown command %d over binding %p\n", cmd, b); 183 break; 184 } 185} 186 187void mgmt_cmd_caps(uint32_t cmd, uint32_t arg, struct capref cap1, 188 struct bench_distops_binding *b) 189{ 190 struct global_state *gs = get_global_state(b); 191 switch (cmd) { 192 case BENCH_CMD_FORWARD_COPIES: 193 { 194 coreid_t cores[] = { gs->nodes[1] }; 195 gs->copycount = 1; 196 gs->fwdcap = cap1; 197 multicast_caps(BENCH_CMD_FORWARD_COPIES, arg, cap1, cores, gs->copycount); 198 DEBUG("cmd_fwd_copies: multicast done\n"); 199 break; 200 } 201 default: 202 printf("mgmt node got caps + command %"PRIu32", arg=%d over binding %p:\n", 203 cmd, arg, b); 204 debug_capref("cap1:", cap1); 205 break; 206 } 207} 208 209//{{{1 Node 210 211struct node_state { 212 struct capref cap; 213 struct capref ram; 214 struct capref *copies; 215 int numcopies; 216 struct capref *ramcopies; 217 int numramcopies; 218 uint64_t *delcycles; 219 uint32_t benchcount; 220 uint32_t iter; 221 bool benchnode; 222}; 223 224static coreid_t my_core_id = -1; 225 226void init_node(struct bench_distops_binding *b) 227{ 228 printf("%s: binding = %p\n", __FUNCTION__, b); 229 230 my_core_id = disp_get_core_id(); 231 232 bench_init(); 233 234 // Allocate client state struct 235 b->st = malloc(sizeof(struct node_state)); 236 assert(b->st); 237 if (!b->st) { 238 USER_PANIC("state malloc() in client"); 239 } 240 241 struct node_state *ns = b->st; 242 ns->benchnode = false; 243 ns->ramcopies = NULL; 244} 245 246static void node_create_copies(struct node_state *ns) 247{ 248 errval_t err; 249 ns->copies = calloc(ns->numcopies, sizeof(struct capref)); 250 for (int i = 0; i < ns->numcopies; i++) { 251 err = slot_alloc(&ns->copies[i]); 252 PANIC_IF_ERR(err, "slot_alloc for copy %d\n", i); 253 err = cap_copy(ns->copies[i], ns->ram); 254 PANIC_IF_ERR(err, "cap_copy for copy %d\n", i); 255 } 256} 257 258extern bool info; 259void node_cmd(uint32_t cmd, uint32_t arg, struct bench_distops_binding *b) 260{ 261 struct node_state *ns = b->st; 262 errval_t err; 263 264 switch(cmd) { 265 case BENCH_CMD_CREATE_COPIES: 266 if (ns->copies) { 267 // Cleanup before next round 268 for (int i = 0; i < ns->numcopies; i++) { 269 err = cap_destroy(ns->copies[i]); 270 assert(err_is_ok(err)); 271 } 272 free(ns->copies); 273 } 274 printf("# node %d: creating %d cap copies\n", my_core_id, arg); 275 ns->numcopies = arg; 276 node_create_copies(ns); 277 printf("# node %d: %zu capabilities on node\n", my_core_id, get_mdb_size()); 278 err = bench_distops_cmd__tx(b, NOP_CONT, BENCH_CMD_COPIES_DONE, 1); 279 PANIC_IF_ERR(err, "signaling cap_copy() done\n"); 280 break; 281 case BENCH_CMD_DO_DELETE: 282 DEBUG("# node %d: deleting our copy for benchmark (mdb size %zu)\n", 283 my_core_id, get_mdb_size()); 284 uint64_t start, end; 285 start = bench_tsc(); 286 TRACE(CAPOPS, USER_DELETE_CALL, (ns->numcopies << 16) | ns->iter); 287 err = cap_delete(ns->cap); 288 TRACE(CAPOPS, USER_DELETE_RESP, (ns->numcopies << 16) | ns->iter); 289 end = bench_tsc(); 290 ns->delcycles[ns->iter] = end - start; 291 assert(err_is_ok(err)); 292 err = slot_free(ns->cap); 293 assert(err_is_ok(err)); 294 // increase iteration counter 295 ns->iter ++; 296 // fall-through to next round 297 case BENCH_CMD_DO_ALLOC: 298 if (arg != 0) { 299 DEBUG("Initializing node %d benchmarking meta\n", my_core_id); 300 // First call only 301 ns->benchcount = arg; 302 ns->iter = 0; 303 ns->benchnode = true; 304 ns->delcycles = calloc(ns->benchcount, sizeof(uint64_t)); 305 assert(ns->delcycles); 306 } 307 if (ns->iter == ns->benchcount) { 308 // Exit if we've done enough iterations 309 printf("# node %d: tsc_per_us = %ld; numcopies = %d\n", 310 my_core_id, bench_tsc_per_us(), ns->numcopies); 311 printf("# delete latency in cycles\n"); 312 for (int i = 0; i < ns->benchcount; i++) { 313 printf("%ld\n", ns->delcycles[i]); 314 } 315 free(ns->delcycles); 316 err = bench_distops_cmd__tx(b, NOP_CONT, BENCH_CMD_PRINT_DONE, 0); 317 assert(err_is_ok(err)); 318 break; 319 } 320 DEBUG("# node %d: allocating cap for benchmark (mdb size %zu, iter %d)\n", 321 my_core_id, get_mdb_size(), ns->iter); 322 err = ram_alloc(&ns->cap, BASE_PAGE_BITS); 323 assert(err_is_ok(err)); 324 // TODO: #copies 325 DEBUG("# node %d: forwarding copies for benchmark (mdb size %zu)\n", 326 my_core_id, get_mdb_size()); 327 err = bench_distops_caps__tx(b, NOP_CONT, BENCH_CMD_FORWARD_COPIES, 10, ns->cap); 328 PANIC_IF_ERR(err, "fwd copies"); 329 assert(err_is_ok(err)); 330 break; 331 default: 332 printf("node %d got command %"PRIu32"\n", my_core_id, cmd); 333 break; 334 } 335} 336 337void node_cmd_caps(uint32_t cmd, uint32_t arg, struct capref cap1, 338 struct bench_distops_binding *b) 339{ 340 errval_t err; 341 struct node_state *ns = b->st; 342 343 switch (cmd) { 344 case BENCH_CMD_CREATE_COPIES: 345 printf("# node %d: creating %d cap copies\n", my_core_id, arg); 346 ns->ram = cap1; 347 ns->numcopies = arg; 348 node_create_copies(ns); 349 printf("# node %d: %zu caps on node\n", my_core_id, get_mdb_size()); 350 err = bench_distops_cmd__tx(b, NOP_CONT, BENCH_CMD_COPIES_DONE, 0); 351 PANIC_IF_ERR(err, "signaling cap_copy() done\n"); 352 break; 353 case BENCH_CMD_FORWARD_COPIES: 354 if (ns->benchnode) { 355 printf("# node %d: just deleting forwarded copy\n", my_core_id); 356 printf("# node %d: mdb size: %zu\n", my_core_id, get_mdb_size()); 357 // node on which we benchmark delete 358 // delete forwarded copy 359 err = cap_destroy(cap1); 360 assert(err_is_ok(err)); 361 } else { 362 // other nodes 363 if (ns->ramcopies) { 364 DEBUG("# node %d: deleting %d old copies of cap to delete\n", 365 my_core_id, ns->numramcopies); 366 for (int i = 0; i < ns->numramcopies; i++) { 367 err = cap_destroy(ns->ramcopies[i]); 368 if (err_is_fail(err)) { 369 DEBUG_ERR(err, "cap_destroy on node %d", my_core_id); 370 } 371 assert(err_is_ok(err)); 372 } 373 } 374 ns->numramcopies = arg; 375 ns->ramcopies = malloc(arg * sizeof(struct capref)); 376 assert(ns->ramcopies); 377 DEBUG("# node %d: creating %d copies of cap to delete\n", 378 my_core_id, ns->numramcopies); 379 for (int i = 0; i < ns->numramcopies; i++) { 380 err = slot_alloc(&ns->ramcopies[i]); 381 assert(err_is_ok(err)); 382 err = cap_copy(ns->ramcopies[i], cap1); 383 assert(err_is_ok(err)); 384 } 385 err = cap_destroy(cap1); 386 assert(err_is_ok(err)); 387 } 388 err = bench_distops_cmd__tx(b, NOP_CONT, BENCH_CMD_COPIES_RX_DONE, 0); 389 assert(err_is_ok(err)); 390 break; 391 default: 392 printf("node %d got caps + command %"PRIu32", arg=%d:\n", 393 my_core_id, cmd, arg); 394 debug_capref("cap1:", cap1); 395 break; 396 } 397} 398