1/** 2 * \file 3 * \brief Management of incoming LMP endpoints 4 */ 5 6/* 7 * Copyright (c) 2007, 2008, 2009, 2010, 2011, ETH Zurich. 8 * All rights reserved. 9 * 10 * This file is distributed under the terms in the attached LICENSE file. 11 * If you do not find this file, copies can be found by writing to: 12 * ETH Zurich D-INFK, Universitaetstrasse 6, CH-8092 Zurich. Attn: Systems Group. 13 */ 14 15#include <inttypes.h> 16#include <barrelfish/barrelfish.h> 17#include <barrelfish/dispatch.h> 18#include <barrelfish/dispatcher_arch.h> 19#include <barrelfish/curdispatcher_arch.h> 20#include <barrelfish/lmp_endpoints.h> 21#include <barrelfish/caddr.h> 22#include <barrelfish/waitset_chan.h> 23#include "waitset_chan_priv.h" 24 25static void endpoint_init(struct lmp_endpoint *ep) 26{ 27 ep->k.delivered = ep->k.consumed = 0; 28 ep->k.recv_cspc = 0; 29 ep->k.recv_cptr = 0; 30 ep->seen = 0; 31 waitset_chanstate_init(&ep->waitset_state, CHANTYPE_LMP_IN); 32} 33 34/** 35 * \brief Allocate an LMP endpoint buffer on the current dispatcher 36 * 37 * In order to accomodate for the in-kernel sentinel word, the 38 * allocated size of the buffer will be one larger than buflen. 39 * 40 * \param buflen Length of incoming LMP buffer, in words 41 * \param retep Double pointer to LMP endpoint, filled-in with allocated EP 42 */ 43errval_t lmp_endpoint_alloc(size_t buflen, struct lmp_endpoint **retep) 44{ 45 // sanity-check buflen 46 if (buflen <= LMP_RECV_LENGTH) { 47 return LIB_ERR_LMP_BUFLEN_INVALID; 48 } 49 50 dispatcher_handle_t handle = disp_disable(); 51 struct dispatcher_generic *dg = get_dispatcher_generic(handle); 52 size_t epsize = sizeof(struct lmp_endpoint) + buflen * sizeof(uintptr_t); 53 struct lmp_endpoint *ep = heap_alloc(&dg->lmp_endpoint_heap, epsize); 54 if(ep == NULL) { 55 disp_enable(handle); 56 return LIB_ERR_NO_ENDPOINT_SPACE; 57 } 58 59 endpoint_init(ep); 60 ep->buflen = buflen; 61 62 disp_enable(handle); 63 64 assert(retep != NULL); 65 *retep = ep; 66 return SYS_ERR_OK; 67} 68 69/** 70 * \brief Free an LMP endpoint buffer on the current dispatcher 71 * 72 * Does not delete the endpoint capability nor free any receive slot. 73 * 74 * \param ep LMP endpoint 75 */ 76void lmp_endpoint_free(struct lmp_endpoint *ep) 77{ 78 assert(ep != NULL); 79 waitset_chanstate_destroy(&ep->waitset_state); 80 dispatcher_handle_t dhandle = disp_disable(); 81 struct dispatcher_generic *dg = get_dispatcher_generic(dhandle); 82 heap_free(&dg->lmp_endpoint_heap, ep); 83 disp_enable(dhandle); 84} 85 86/** 87 * \brief Create endpoint to caller on current dispatcher in a specified slot. 88 * 89 * \param buflen Length of incoming LMP buffer, in words 90 * \param dest Location of empty slot in which to create endpoint 91 * \param retep Double pointer to LMP endpoint, filled-in with allocated EP 92 * \param iftype Interface type of this endpoint 93 * 94 * This function mints into the given slot an endpoint capability to the 95 * current dispatcher. 96 */ 97errval_t lmp_endpoint_create_in_slot_with_iftype(size_t buflen, struct capref dest, 98 struct lmp_endpoint **retep, uint16_t iftype) 99{ 100 struct lmp_endpoint *ep = NULL; 101 errval_t err; 102 103 // We increase buflen by 1 here to accomodate for in-kernel sentinel word 104 buflen++; 105 106 err = lmp_endpoint_alloc(buflen, &ep); 107 if (err_is_fail(err)) { 108 return err; 109 } 110 111 assert(ep != NULL); 112 if (retep != NULL) { 113 *retep = ep; 114 } 115 116 uintptr_t epoffset = (uintptr_t)&ep->k - (uintptr_t)curdispatcher(); 117 118 /* stuff buflen and iftype into second param of mint, so we don't need 119 * the ugly invoke_endpoint_set_iftype. buflen should comfortably fit into 120 * 16 bits as it is in *words* rather than bytes */ 121 assert(buflen < UINT16_MAX); 122 buflen = ((uintptr_t)iftype << 16) | buflen; 123 124 //debug_printf("%s: calling mint with epoffset = %"PRIuPTR", buflen = %zu\n", 125 // __FUNCTION__, epoffset, buflen); 126 // mint new badged cap from our existing reply endpoint 127 return cap_mint(dest, cap_selfep, epoffset, buflen); 128} 129 130/** 131 * \brief Create endpoint to caller on current dispatcher in a specified slot. 132 * 133 * \param buflen Length of incoming LMP buffer, in words 134 * \param dest Location of empty slot in which to create endpoint 135 * \param retep Double pointer to LMP endpoint, filled-in with allocated EP 136 * 137 * This function mints into the given slot an endpoint capability to the 138 * current dispatcher. 139 */ 140errval_t lmp_endpoint_create_in_slot(size_t buflen, struct capref dest, 141 struct lmp_endpoint **retep) 142{ 143 return lmp_endpoint_create_in_slot_with_iftype(buflen, dest, retep, 0); 144} 145 146/** 147 * \brief Set the receive capability slot for a given endpoint 148 * 149 * \param ep Endpoint returned from messages_lmp_alloc_endpoint() 150 * \param slot Receive slot 151 */ 152void lmp_endpoint_set_recv_slot(struct lmp_endpoint *ep, struct capref slot) 153{ 154 // debug_printf("%s: recv_cspace = %"PRIxCADDR", recv_cptr = %"PRIxCADDR"\n", 155 // __FUNCTION__, get_croot_addr(slot), get_cap_addr(slot)); 156 ep->k.recv_cspc = get_croot_addr(slot); 157 ep->k.recv_cptr = get_cap_addr(slot); 158 ep->recv_slot = slot; 159} 160 161/** 162 * \brief Returns true iff there are messages in the given endpoint buffer 163 * 164 * May be called enabled or disabled. As a result, when enabled, the result may 165 * be incorrect as soon as the function returns. 166 */ 167inline bool lmp_endpoint_can_recv(struct lmp_endpoint *ep) 168{ 169 return ep->k.delivered != ep->k.consumed; 170} 171 172/// Returns number of words, including headers, previously unseen in endpoint 173static uint32_t lmp_endpoint_words_unseen(struct lmp_endpoint *ep) 174{ 175 uint32_t delivered = ep->k.delivered; 176 uint32_t ret; 177 if (delivered >= ep->seen) { 178 ret = delivered - ep->seen; 179 } else { 180 ret = ep->buflen - ep->seen + delivered; 181 } 182 ep->seen = delivered; 183 return ret; 184} 185 186/** 187 * \brief Check incoming LMP endpoints for messages and notify waitsets 188 * 189 * \param disp_priv Dispatcher's private data 190 * 191 * Must be called while disabled. 192 */ 193void lmp_endpoints_poll_disabled(dispatcher_handle_t handle) 194{ 195 struct dispatcher_shared_generic *disp = 196 get_dispatcher_shared_generic(handle); 197 struct dispatcher_generic *dp = get_dispatcher_generic(handle); 198 struct lmp_endpoint *ep, *nextep, *firstep; 199 errval_t err; 200 201 // get LMP delivered word count at time of entry 202 uint32_t lmp_delivered = disp->lmp_delivered; 203 204 // try the hint EP first if set 205 if (disp->lmp_hint != 0) { 206 assert_disabled(disp->lmp_hint < DISPATCHER_FRAME_SIZE); 207 208 /* compute endpoint location */ 209 ep = (struct lmp_endpoint *) 210 ((char *)handle + disp->lmp_hint - offsetof(struct lmp_endpoint, k)); 211 212 // clear hint now we're about to look at it 213 disp->lmp_hint = 0; 214 215 // if channel has a message, is registered, and isn't already pending 216 if (lmp_endpoint_can_recv(ep) 217 && waitset_chan_is_registered(&ep->waitset_state) 218 && ep->waitset_state.state == CHAN_IDLE) { 219 220 // update seen count 221 disp->lmp_seen += lmp_endpoint_words_unseen(ep); 222 223 // should have been in the poll list if it was registered 224 assert_disabled(ep->next != NULL && ep->prev != NULL); 225 226 err = waitset_chan_trigger_disabled(&ep->waitset_state, handle); 227 assert_disabled(err_is_ok(err)); 228 229 /* remove from poll list */ 230 if (ep->next == ep) { 231 assert(ep->prev == ep); 232 assert(dp->lmp_poll_list == ep); 233 dp->lmp_poll_list = NULL; 234 } else { 235 ep->prev->next = ep->next; 236 ep->next->prev = ep->prev; 237 if (dp->lmp_poll_list == ep) { 238 dp->lmp_poll_list = ep->next; 239 } 240 } 241 } 242 } 243 244 // if there is now nothing outstanding, we can skip the full poll 245 // it's possible that another message arrived while we were polling, but 246 // we'll stay runnable and get that next time 247 if (disp->lmp_seen == lmp_delivered) { 248 return; 249 } 250 251 // there are other polled endpoints with unseen messages: search for them 252 for (ep = dp->lmp_poll_list; ep != NULL; ep = nextep) { 253 nextep = ep->next; 254 firstep = dp->lmp_poll_list; 255 256 if (lmp_endpoint_can_recv(ep)) { 257 err = waitset_chan_trigger_disabled(&ep->waitset_state, handle); 258 assert_disabled(err_is_ok(err)); // can't fail 259 260 // update seen count 261 disp->lmp_seen += lmp_endpoint_words_unseen(ep); 262 263 /* remove from poll list */ 264 if (ep->next == ep) { 265 assert(ep->prev == ep); 266 assert(dp->lmp_poll_list == ep); 267 dp->lmp_poll_list = NULL; 268 break; 269 } else { 270 ep->prev->next = ep->next; 271 ep->next->prev = ep->prev; 272 if (dp->lmp_poll_list == ep) { 273 dp->lmp_poll_list = ep->next; 274 } 275 } 276 } 277 278 if (nextep == firstep) { 279 break; // looped 280 } 281 } 282 283 // if there are now any outstanding unseen messages, they must be in 284 // endpoints which we aren't currently polling / aren't currently 285 // registered, so we can ignore them here, as long as we update 286 // ep->seen = ep->delivered when inserting an endpoint into the poll list. 287 disp->lmp_seen = lmp_delivered; 288} 289 290/** 291 * \brief Register an event handler to be notified when messages can be received 292 * 293 * In the future, call the closure on the given waitset when it is likely that 294 * a message can be received on the endpoint. An endpoint may only be registered 295 * with a single event handler on a single waitset at any one time. 296 * 297 * \param ep LMP endpoint 298 * \param ws Waitset 299 * \param closure Event handler 300 */ 301errval_t lmp_endpoint_register(struct lmp_endpoint *ep, struct waitset *ws, 302 struct event_closure closure) 303{ 304 errval_t err; 305 306 dispatcher_handle_t handle = disp_disable(); 307 struct dispatcher_generic *dp = get_dispatcher_generic(handle); 308 309 // update seen count before checking for any new messages 310 ep->seen = ep->k.delivered; 311 312 if (lmp_endpoint_can_recv(ep)) { // trigger immediately 313 err = waitset_chan_trigger_closure_disabled(ws, &ep->waitset_state, 314 closure, handle); 315 } else { 316 err = waitset_chan_register_disabled(ws, &ep->waitset_state, closure); 317 if (err_is_ok(err)) { 318 /* enqueue on poll list */ 319 if (dp->lmp_poll_list == NULL) { 320 ep->prev = ep->next = ep; 321 } else { 322 ep->next = dp->lmp_poll_list; 323 ep->prev = ep->next->prev; 324 ep->next->prev = ep; 325 ep->prev->next = ep; 326 } 327 dp->lmp_poll_list = ep; 328 } 329 } 330 331 disp_enable(handle); 332 333 return err; 334} 335 336/** 337 * \brief Cancel an event registration made with lmp_endpoint_register() 338 * 339 * \param ep LMP Endpoint 340 */ 341errval_t lmp_endpoint_deregister(struct lmp_endpoint *ep) 342{ 343 assert(ep != NULL); 344 dispatcher_handle_t handle = disp_disable(); 345 struct dispatcher_generic *dp = get_dispatcher_generic(handle); 346 347 errval_t err = waitset_chan_deregister_disabled(&ep->waitset_state, handle); 348 if (err_is_ok(err)) { 349 /* dequeue from poll list */ 350 if (ep->next == ep) { 351 assert(ep->prev == ep); 352 assert(dp->lmp_poll_list == ep); 353 dp->lmp_poll_list = NULL; 354 } else { 355 ep->next->prev = ep->prev; 356 ep->prev->next = ep->next; 357 if (dp->lmp_poll_list == ep) { 358 dp->lmp_poll_list = ep->next; 359 } 360 } 361 } 362 363 disp_enable(handle); 364 365 return err; 366} 367 368/** 369 * \brief Migrate an event registration made with lmp_endpoint_register() to a new waitset. 370 * 371 * \param ep LMP Endpoint 372 * \param ws New waitset 373 */ 374void lmp_endpoint_migrate(struct lmp_endpoint *ep, struct waitset *ws) 375{ 376 waitset_chan_migrate(&ep->waitset_state, ws); 377} 378 379/** 380 * \brief Retrieve an LMP message from an endpoint, if possible 381 * 382 * \param ep Endpoint 383 * \param buf LMP message buffer, to be filled-in 384 * \param cap If non-NULL, filled-in with location of received capability, if any 385 * 386 * \return LIB_ERR_NO_LMP_MSG if no message is available 387 * \return LIB_ERR_LMP_RECV_BUF_OVERFLOW if user-provided receive buffer is too small 388 * to store the entire message 389 */ 390errval_t lmp_endpoint_recv(struct lmp_endpoint *ep, struct lmp_recv_buf *buf, 391 struct capref *cap) 392{ 393 assert(buf != NULL); 394 395 dispatcher_handle_t handle = disp_disable(); 396 397 if (!lmp_endpoint_can_recv(ep)) { 398 disp_enable(handle); 399 return LIB_ERR_NO_LMP_MSG; 400 } 401 402 uint32_t pos = ep->k.consumed; 403 assert(pos < ep->buflen); 404 405 /* look at the header first */ 406 union lmp_recv_header header; 407 header.raw = ep->k.buf[pos]; 408 buf->msglen = header.x.length; 409 410 if (header.x.length >= ep->buflen) { 411 USER_PANIC("lmp_endpoint_recv: insane message (%u words @ %"PRIu32")." 412 " delivered=%"PRIu32" consumed=%"PRIu32" len=%"PRIu32"\n", 413 header.x.length, pos, ep->k.delivered, ep->k.consumed, 414 ep->buflen); 415 } 416 417 /* check for space in the user's buffer */ 418 if (header.x.length > buf->buflen) { 419 disp_enable(handle); 420 debug_printf("lmp_endpoint_recv: recv buf (%zu words @ %p) overflow" 421 " by pending message (%u words @ %"PRIu32")." 422 " delivered=%"PRIu32" consumed=%"PRIu32" len=%"PRIu32"\n", 423 buf->buflen, &buf, header.x.length, pos, 424 ep->k.delivered, ep->k.consumed, ep->buflen); 425 return LIB_ERR_LMP_RECV_BUF_OVERFLOW; 426 } 427 428 /* consume the header */ 429 if (++pos == ep->buflen) { 430 pos = 0; 431 } 432 433 /* copy the rest out one word at a time... */ 434 for (int i = 0; i < header.x.length; i++) { 435 buf->words[i] = ep->k.buf[pos]; 436 if (++pos == ep->buflen) { 437 pos = 0; 438 } 439 } 440 441 /* did we get a cap? */ 442 if (header.x.flags.captransfer) { 443 assert_disabled(ep->k.recv_cptr != 0); 444 if (cap != NULL) { 445 *cap = ep->recv_slot; 446 } 447 ep->k.recv_cptr = ep->k.recv_cspc = 0; 448 } else if (cap != NULL) { 449 *cap = NULL_CAP; 450 } 451 452 /* update dispatcher */ 453 ep->k.consumed = pos; 454 455 disp_enable(handle); 456 457 return SYS_ERR_OK; 458} 459 460/** 461 * \brief Store a newly-received LRPC message into an endpoint buffer 462 * 463 * Must be called while disabled. 464 * 465 * \param ep Endpoint 466 * \param bufpos Reserved position in endpoint message buffer 467 * \param arg1 Message payload 468 * \param arg2 Message payload 469 * \param arg3 Message payload 470 * \param arg4 Message payload 471 */ 472void lmp_endpoint_store_lrpc_disabled(struct lmp_endpoint *ep, uint32_t bufpos, 473 uintptr_t arg1, uintptr_t arg2, 474 uintptr_t arg3, uintptr_t arg4) 475{ 476 /* prefabricated LMP header */ 477 static union lmp_recv_header header = { 478 .x.length = LRPC_MSG_LENGTH, 479 }; 480 481 uint32_t buflen = ep->buflen; 482 assert_disabled(buflen != 0); 483 assert_disabled(bufpos < buflen); 484 485 /* deposit message into buffer, starting from bufpos */ 486 ep->k.buf[bufpos] = header.raw; 487 if (++bufpos == buflen) { 488 bufpos = 0; 489 } 490 ep->k.buf[bufpos] = arg1; 491 if (++bufpos == buflen) { 492 bufpos = 0; 493 } 494 ep->k.buf[bufpos] = arg2; 495 if (++bufpos == buflen) { 496 bufpos = 0; 497 } 498 ep->k.buf[bufpos] = arg3; 499 if (++bufpos == buflen) { 500 bufpos = 0; 501 } 502 ep->k.buf[bufpos] = arg4; 503} 504 505/** 506 * \brief Initialize LMP endpoint subsystem. 507 */ 508void lmp_endpoint_init(void) 509{ 510 dispatcher_handle_t handle = disp_disable(); 511 size_t dispsize = get_dispatcher_size(); 512 void *buf = (char *)get_dispatcher_vaddr(handle) + dispsize; 513 size_t buflen = DISPATCHER_FRAME_SIZE - dispsize; 514 struct dispatcher_generic *d = get_dispatcher_generic(handle); 515 516 heap_init(&d->lmp_endpoint_heap, buf, buflen, NULL); 517 disp_enable(handle); 518} 519