1/** 2 * \file 3 * \brief Benchmark revoke of local cap which has remote relations 4 */ 5 6/* 7 * Copyright (c) 2017, ETH Zurich. 8 * All rights reserved. 9 * 10 * This file is distributed under the terms in the attached LICENSE file. 11 * If you do not find this file, copies can be found by writing to: 12 * ETH Zurich D-INFK, Universitaetstr 6, CH-8092 Zurich. Attn: Systems Group. 13 */ 14 15#include <barrelfish/barrelfish.h> 16#include <if/bench_distops_defs.h> 17 18#include <bitmacros.h> 19 20#include <bench/bench.h> 21#include <trace/trace.h> 22 23#include "benchapi.h" 24 25#define REVOKE_COPIES 10 26 27//{{{1 debugging helpers 28static void debug_capref(const char *prefix, struct capref cap) 29{ 30 char buf[128]; 31 debug_print_capref(buf, 128, cap); 32 printf("%s capref = %s\n", prefix, buf); 33} 34 35//{{{1 shared commands 36enum bench_cmd { 37 BENCH_CMD_CREATE_COPIES, 38 BENCH_CMD_COPIES_DONE, 39 BENCH_CMD_DO_ALLOC, 40 BENCH_CMD_DO_REVOKE, 41 BENCH_CMD_FORWARD_COPIES, 42 BENCH_CMD_COPIES_RX_DONE, 43 BENCH_CMD_PRINT_DONE, 44}; 45 46//{{{1 shared helper functions 47static size_t get_mdb_size(void) 48{ 49 errval_t err; 50 size_t cap_base_count = 0; 51 err = sys_debug_get_mdb_size(&cap_base_count); 52 assert(err_is_ok(err)); 53 return cap_base_count; 54} 55 56 57//{{{1 Managment node: implement orchestration for benchmark 58 59//{{{2 Management node: state management 60 61struct global_state { 62 struct capref ram; 63 struct capref fwdcap; 64 coreid_t *nodes; 65 int nodes_seen; 66 int nodecount; 67 int copies_done; 68 int copycount; 69 int rx_seen; 70 int masternode; 71 int currcopies; 72}; 73 74errval_t mgmt_init_benchmark(void **st, int nodecount) 75{ 76 *st = calloc(1, sizeof(struct global_state)); 77 if (!*st) { 78 return LIB_ERR_MALLOC_FAIL; 79 } 80 struct global_state *gs = *st; 81 gs->nodes = calloc(nodecount, sizeof(coreid_t)); 82 gs->nodecount = nodecount; 83 gs->copies_done = 0; 84 gs->rx_seen = 0; 85 gs->masternode = -1; 86 return ram_alloc(&gs->ram, BASE_PAGE_BITS); 87} 88 89static int sort_coreid(const void *a_, const void *b_) 90{ 91 // deref pointers as coreids, store as ints 92 int a = *((coreid_t*)a_); 93 int b = *((coreid_t*)b_); 94 // subtract as ints 95 return a-b; 96} 97 98void mgmt_register_node(void *st, coreid_t nodeid) 99{ 100 struct global_state *gs = st; 101 gs->nodes[gs->nodes_seen++] = nodeid; 102 // if we've seen all nodes, sort nodes array and configure printnode 103 if (gs->nodes_seen == gs->nodecount) { 104 qsort(gs->nodes, gs->nodecount, sizeof(coreid_t), sort_coreid); 105 gs->masternode = gs->nodes[0]; 106 } 107} 108 109struct mgmt_node_state { 110}; 111 112errval_t mgmt_init_node(void **st) 113{ 114 *st = malloc(sizeof(struct mgmt_node_state)); 115 if (!*st) { 116 return LIB_ERR_MALLOC_FAIL; 117 } 118 return SYS_ERR_OK; 119} 120 121//{{{2 Management node: benchmark impl 122void mgmt_run_benchmark(void *st) 123{ 124 struct global_state *gs = st; 125 126 printf("All clients sent hello! Benchmark starting...\n"); 127 128 printf("# Benchmarking REVOKE WITH REMOTE COPIES: nodes=%d\n", gs->nodecount); 129 130 printf("# Starting out with %d copies, will by powers of 2 up to %d...\n", 131 NUM_COPIES_START, NUM_COPIES_END); 132 133 TRACE(CAPOPS, START, 0); 134 135 gs->currcopies = NUM_COPIES_START; 136 broadcast_caps(BENCH_CMD_CREATE_COPIES, NUM_COPIES_START, gs->ram); 137} 138 139void mgmt_cmd(uint32_t cmd, uint32_t arg, struct bench_distops_binding *b) 140{ 141 errval_t err; 142 struct global_state *gs = get_global_state(b); 143 144 switch(cmd) { 145 case BENCH_CMD_COPIES_DONE: 146 gs->copies_done++; 147 if (gs->copies_done == gs->nodecount) { 148 printf("# All copies made!\n"); 149 unicast_cmd(gs->masternode, BENCH_CMD_DO_ALLOC, ITERS); 150 } 151 break; 152 case BENCH_CMD_COPIES_RX_DONE: 153 gs->rx_seen++; 154 DEBUG("got BENCH_CMD_COPIES_RX_DONE: seen = %d, expected = %d\n", 155 gs->rx_seen, gs->copycount); 156 if (gs->rx_seen == gs->copycount) { 157 DEBUG("# All nodes have copies of cap-to-del\n"); 158 err = cap_destroy(gs->fwdcap); 159 assert(err_is_ok(err)); 160 gs->rx_seen = 0; 161 unicast_cmd(gs->masternode, BENCH_CMD_DO_REVOKE, 0); 162 } 163 break; 164 case BENCH_CMD_PRINT_DONE: 165 if (gs->currcopies == NUM_COPIES_END) { 166 printf("# Benchmark done!\n"); 167 TRACE(CAPOPS, STOP, 0); 168 mgmt_trace_flush(NOP_CONT); 169 return; 170 } 171 printf("# Round done!\n"); 172 printf("# mgmt node mdb size: %zu\n", get_mdb_size()); 173 // Reset counters for next round 174 gs->currcopies *= 2; 175 gs->copies_done = 0; 176 gs->rx_seen = 0; 177 // Start new round 178 broadcast_cmd(BENCH_CMD_CREATE_COPIES, gs->currcopies); 179 break; 180 default: 181 printf("mgmt node got unknown command %d over binding %p\n", cmd, b); 182 break; 183 } 184} 185 186void mgmt_cmd_caps(uint32_t cmd, uint32_t arg, struct capref cap1, 187 struct bench_distops_binding *b) 188{ 189 struct global_state *gs = get_global_state(b); 190 switch (cmd) { 191 case BENCH_CMD_FORWARD_COPIES: 192 { 193 coreid_t cores[] = { gs->nodes[1] }; 194 gs->copycount = 1; 195 gs->fwdcap = cap1; 196 multicast_caps(BENCH_CMD_FORWARD_COPIES, arg, cap1, cores, gs->copycount); 197 DEBUG("cmd_fwd_copies: multicast done\n"); 198 break; 199 } 200 default: 201 printf("mgmt node got caps + command %"PRIu32", arg=%d over binding %p:\n", 202 cmd, arg, b); 203 debug_capref("cap1:", cap1); 204 break; 205 } 206} 207 208//{{{1 Node 209 210struct node_state { 211 struct capref cap; 212 struct capref ram; 213 struct capref *copies; 214 int numcopies; 215 struct capref *ramcopies; 216 int numramcopies; 217 uint64_t *delcycles; 218 uint32_t benchcount; 219 uint32_t iter; 220 bool benchnode; 221}; 222 223static coreid_t my_core_id = -1; 224 225void init_node(struct bench_distops_binding *b) 226{ 227 printf("%s: binding = %p\n", __FUNCTION__, b); 228 229 my_core_id = disp_get_core_id(); 230 231 bench_init(); 232 233 // Allocate client state struct 234 b->st = malloc(sizeof(struct node_state)); 235 assert(b->st); 236 if (!b->st) { 237 USER_PANIC("state malloc() in client"); 238 } 239 240 struct node_state *ns = b->st; 241 ns->benchnode = false; 242 ns->ramcopies = NULL; 243 ns->numramcopies = 0; 244} 245 246static void node_create_copies(struct node_state *ns) 247{ 248 errval_t err; 249 ns->copies = calloc(ns->numcopies, sizeof(struct capref)); 250 for (int i = 0; i < ns->numcopies; i++) { 251 err = slot_alloc(&ns->copies[i]); 252 PANIC_IF_ERR(err, "slot_alloc for copy %d\n", i); 253 err = cap_copy(ns->copies[i], ns->ram); 254 PANIC_IF_ERR(err, "cap_copy for copy %d\n", i); 255 } 256} 257 258extern bool info; 259void node_cmd(uint32_t cmd, uint32_t arg, struct bench_distops_binding *b) 260{ 261 struct node_state *ns = b->st; 262 errval_t err; 263 264 switch(cmd) { 265 case BENCH_CMD_CREATE_COPIES: 266 if (ns->copies) { 267 // Cleanup before next round 268 for (int i = 0; i < ns->numcopies; i++) { 269 err = cap_destroy(ns->copies[i]); 270 assert(err_is_ok(err)); 271 } 272 free(ns->copies); 273 } 274 printf("# node %d: creating %d cap copies\n", my_core_id, arg); 275 ns->numcopies = arg; 276 node_create_copies(ns); 277 printf("# node %d: %zu capabilities on node\n", my_core_id, get_mdb_size()); 278 err = bench_distops_cmd__tx(b, NOP_CONT, BENCH_CMD_COPIES_DONE, 1); 279 PANIC_IF_ERR(err, "signaling cap_copy() done\n"); 280 break; 281 case BENCH_CMD_DO_REVOKE: 282#if 0 283 DEBUG("# node %d: making sure the retype checking makes sense\n"); 284 struct capref slot; 285 err = slot_alloc(&slot); 286 assert(err_is_ok(err)); 287 err = cap_retype(slot, ns->cap, 0, ObjType_RAM, BASE_PAGE_SIZE, 1); 288 if (err_is_ok(err)){ 289 printf("node %d: retype after copy succeeded\n", my_core_id); 290 } 291 assert(err_is_ok(err)); 292#endif 293 DEBUG("# node %d: revoking our copy for benchmark (mdb size %zu)\n", 294 my_core_id, get_mdb_size()); 295 uint64_t start, end; 296 start = bench_tsc(); 297 TRACE(CAPOPS, USER_REVOKE_CALL, (ns->numcopies << 16) | ns->iter); 298 err = cap_revoke(ns->cap); 299 TRACE(CAPOPS, USER_REVOKE_RESP, (ns->numcopies << 16) | ns->iter); 300 end = bench_tsc(); 301 ns->delcycles[ns->iter] = end - start; 302 assert(err_is_ok(err)); 303 // Check that revoke went through correctly 304 struct capref slot; 305 err = slot_alloc(&slot); 306 assert(err_is_ok(err)); 307 err = cap_retype(slot, ns->cap, 0, ObjType_RAM, BASE_PAGE_SIZE, 1); 308 PANIC_IF_ERR(err, "retype after revoke!"); 309 err = cap_destroy(slot); 310 assert(err_is_ok(err)); 311 // increase iteration counter 312 ns->iter ++; 313 // fall-through to next round 314 case BENCH_CMD_DO_ALLOC: 315 if (arg != 0) { 316 DEBUG("Initializing node %d benchmarking meta\n", my_core_id); 317 // First call only 318 ns->benchcount = arg; 319 ns->iter = 0; 320 ns->benchnode = true; 321 ns->delcycles = calloc(ns->benchcount, sizeof(uint64_t)); 322 assert(ns->delcycles); 323 err = ram_alloc(&ns->cap, BASE_PAGE_BITS+2); 324 assert(err_is_ok(err)); 325 if (!ns->ramcopies) { 326 ns->ramcopies = calloc(REVOKE_COPIES, sizeof(struct capref)); 327 } 328 assert(ns->ramcopies); 329 for (int c = 0; c < REVOKE_COPIES; c++) { 330 err = slot_alloc(&ns->ramcopies[c]); 331 assert(err_is_ok(err)); 332 } 333 } 334 if (ns->iter == ns->benchcount) { 335 // Exit if we've done enough iterations 336 printf("# node %d: tsc_per_us = %ld; numcopies = %d\n", 337 my_core_id, bench_tsc_per_us(), ns->numcopies); 338 printf("# delete latency in cycles\n"); 339 for (int i = 0; i < ns->benchcount; i++) { 340 printf("%ld\n", ns->delcycles[i]); 341 } 342 free(ns->delcycles); 343 err = cap_destroy(ns->cap); 344 assert(err_is_ok(err)); 345 err = bench_distops_cmd__tx(b, NOP_CONT, BENCH_CMD_PRINT_DONE, 0); 346 assert(err_is_ok(err)); 347 break; 348 } 349 // TODO: #copies / copy/desc mix 350 // XXX: can only create copies on non-owning core, bc. owning core 351 // already has descendants 352 for (int c = 0; c < REVOKE_COPIES; c++) { 353 err = cap_copy(ns->ramcopies[c], ns->cap); 354 assert(err_is_ok(err)); 355 } 356 DEBUG("# node %d: forwarding copies for benchmark (mdb size %zu)\n", 357 my_core_id, get_mdb_size()); 358 err = bench_distops_caps__tx(b, NOP_CONT, BENCH_CMD_FORWARD_COPIES, 10, ns->cap); 359 PANIC_IF_ERR(err, "fwd copies"); 360 assert(err_is_ok(err)); 361 break; 362 default: 363 printf("node %d got command %"PRIu32"\n", my_core_id, cmd); 364 break; 365 } 366} 367 368void node_cmd_caps(uint32_t cmd, uint32_t arg, struct capref cap1, 369 struct bench_distops_binding *b) 370{ 371 errval_t err; 372 struct node_state *ns = b->st; 373 374 switch (cmd) { 375 case BENCH_CMD_CREATE_COPIES: 376 printf("# node %d: creating %d cap copies\n", my_core_id, arg); 377 ns->ram = cap1; 378 ns->numcopies = arg; 379 node_create_copies(ns); 380 printf("# node %d: %zu caps on node\n", my_core_id, get_mdb_size()); 381 err = bench_distops_cmd__tx(b, NOP_CONT, BENCH_CMD_COPIES_DONE, 0); 382 PANIC_IF_ERR(err, "signaling cap_copy() done\n"); 383 break; 384 case BENCH_CMD_FORWARD_COPIES: 385 if (ns->benchnode) { 386 printf("# node %d: just deleting forwarded copy\n", my_core_id); 387 printf("# node %d: mdb size: %zu\n", my_core_id, get_mdb_size()); 388 // node on which we benchmark delete 389 // delete forwarded copy 390 err = cap_destroy(cap1); 391 assert(err_is_ok(err)); 392 } else { 393 // other nodes 394 if (!ns->ramcopies || ns->numramcopies != arg) { 395 if (ns->numramcopies != arg) { 396 free(ns->ramcopies); 397 ns->numramcopies = arg; 398 ns->ramcopies = malloc(arg * sizeof(struct capref)); 399 } 400 for (int i = 0; i < ns->numramcopies; i++) { 401 err = slot_alloc(&ns->ramcopies[i]); 402 assert(err_is_ok(err)); 403 } 404 } 405 assert(ns->ramcopies); 406 DEBUG("# node %d: creating %d copies of cap to delete\n", 407 my_core_id, ns->numramcopies); 408 for (int i = 0; i < ns->numramcopies; i++) { 409 // XXX: magic constant 410 if (i < 4) { 411 // make descendants 412 err = cap_retype(ns->ramcopies[i], cap1, i*BASE_PAGE_SIZE, 413 ObjType_RAM, BASE_PAGE_SIZE, 1); 414 if (err_is_fail(err)) { 415 DEBUG_ERR(err, "[node %d] retyping c=%d", 416 my_core_id, i); 417 } 418 assert(err_is_ok(err)); 419 } else { 420 // copies 421 err = cap_copy(ns->ramcopies[i], cap1); 422 assert(err_is_ok(err)); 423 } 424 } 425 err = cap_destroy(cap1); 426 assert(err_is_ok(err)); 427 } 428 err = bench_distops_cmd__tx(b, NOP_CONT, BENCH_CMD_COPIES_RX_DONE, 0); 429 assert(err_is_ok(err)); 430 break; 431 default: 432 printf("node %d got caps + command %"PRIu32", arg=%d:\n", 433 my_core_id, cmd, arg); 434 debug_capref("cap1:", cap1); 435 break; 436 } 437} 438