1/** 2 * \file 3 * \brief Dispatcher implementation. 4 */ 5 6/* 7 * Copyright (c) 2007, 2008, 2009, 2010, 2011, 2012, ETH Zurich. 8 * All rights reserved. 9 * 10 * This file is distributed under the terms in the attached LICENSE file. 11 * If you do not find this file, copies can be found by writing to: 12 * ETH Zurich D-INFK, Haldeneggsteig 4, CH-8092 Zurich. Attn: Systems Group. 13 */ 14 15#include <inttypes.h> 16#include <stdlib.h> 17#include <stdio.h> 18#include <string.h> 19#include <barrelfish/barrelfish.h> 20#include <barrelfish/dispatch.h> 21#include <barrelfish/dispatcher_arch.h> 22#include <barrelfish/curdispatcher_arch.h> 23#include <barrelfish/caddr.h> 24#include <barrelfish/debug.h> 25#include <barrelfish/deferred.h> 26#include <barrelfish_kpi/cpu_arch.h> 27#include "threads_priv.h" 28#include <barrelfish/notificator.h> 29#include <barrelfish/systime.h> 30 31#include <trace/trace.h> 32#include <trace_definitions/trace_defs.h> 33 34#ifdef CONFIG_INTERCONNECT_DRIVER_LMP 35# include <barrelfish/lmp_chan.h> 36#endif 37 38 39#if defined(__GNUC__) && !defined(__clang__) && !defined(__ICC) \ 40 && (defined(__x86_64__) || (defined(__i386__))) 41// Disable SSE/MMX -- this code context switches those registers 42# pragma GCC target ("no-mmx,no-sse,no-sse2,no-sse3,no-sse4.1,no-sse4.2,no-sse4,no-sse4a,no-3dnow") 43#endif 44 45void disp_run(dispatcher_handle_t handle); 46void disp_lrpc(struct lmp_endpoint *ep, uint32_t bufpos, 47 uintptr_t arg1, uintptr_t arg2, uintptr_t arg3, uintptr_t arg4, 48 dispatcher_handle_t handle); 49void disp_pagefault(dispatcher_handle_t handle, lvaddr_t fault_address, 50 uintptr_t error, lvaddr_t ip); 51void disp_pagefault_disabled(dispatcher_handle_t handle, lvaddr_t fault_address, 52 uintptr_t error, lvaddr_t ip); 53void disp_trap(dispatcher_handle_t handle, uintptr_t irq, uintptr_t error, 54 lvaddr_t ip); 55 56static inline void assert_print(const char *str) 57{ 58 sys_print(str, strlen(str)); 59} 60 61static uint64_t run_counter = 0; 62uint64_t disp_run_counter(void) 63{ 64 return run_counter; 65} 66/** 67 * \brief Run entry point 68 * 69 * This function is called from assembly code when the kernel enters us to 70 * give us the CPU. 71 * 72 * \param disp Dispatcher 73 */ 74void disp_run(dispatcher_handle_t handle) 75{ 76#ifdef __x86_64__ 77 struct dispatcher_x86_64 *disp_priv = get_dispatcher_x86_64(handle); 78 /* load compatibility dispatcher segment to FS */ 79 __asm volatile("mov %%ax, %%fs" 80 : /* No outputs */ 81 : "a" (disp_priv->disp_seg_selector)); 82#endif 83 // We can't call printf(), so do this silly thing... 84// assert_print("FIXME: infinite while loop\n"); 85// while(1); 86 87 struct dispatcher_generic* disp_gen = get_dispatcher_generic(handle); 88 struct dispatcher_shared_generic* disp = 89 get_dispatcher_shared_generic(handle); 90 91 assert_disabled(disp->disabled); 92 ++run_counter; 93 disp_gen->timeslice++; 94 // Never let 0 be a valid timeslice number 95 if(disp_gen->timeslice == 0) { 96 disp_gen->timeslice++; 97 } 98 99 // Trigger any deferred events 100 trigger_deferred_events_disabled(handle, disp->systime); 101 102#ifdef CONFIG_INTERCONNECT_DRIVER_LMP 103 // Check for incoming LMP messages 104 if (disp->lmp_delivered != disp->lmp_seen) { 105 lmp_endpoints_poll_disabled(handle); 106 } 107 108 // Trigger any send events for LMP channels 109 lmp_channels_retry_send_disabled(handle); 110#endif // CONFIG_INTERCONNECT_DRIVER_LMP 111 // Check polled channels 112 poll_channels_disabled(handle); 113 ump_channels_retry_send_disabled(handle); 114 115 check_notificators_disabled(handle); 116 117 // Run, saving state of previous thread if required 118 thread_run_disabled(handle); 119 120 // NOTREACHED 121 assert_disabled(!"disp_run: thread_run() returned!\n"); 122 for(;;); 123} 124 125/** 126 * \brief LRPC entry point 127 * 128 * This function is called from assembly code when the kernel enters us to 129 * give us the CPU and deliver an LRPC message. The dispatcher is disabled. 130 * 131 * \param ep LMP endpoint structure 132 * \param bufpos Reserved position in endpoint message buffer 133 * \param arg1 Message payload 134 * \param arg2 Message payload 135 * \param arg3 Message payload 136 * \param arg4 Message payload 137 * \param handle Dispatcher pointer 138 * 139 * \note Dispatcher pointer comes last here, because AB was too lazy to shuffle 140 * registers through the whole LRPC path when adding it. 141 */ 142void disp_lrpc(struct lmp_endpoint *ep, uint32_t bufpos, 143 uintptr_t arg1, uintptr_t arg2, uintptr_t arg3, uintptr_t arg4, 144 dispatcher_handle_t handle) 145{ 146 struct dispatcher_shared_generic* disp = 147 get_dispatcher_shared_generic(handle); 148 149 /* deliver LRPC message to buffer */ 150 lmp_endpoint_store_lrpc_disabled(ep, bufpos, arg1, arg2, arg3, arg4); 151 152 // set hint 153 disp->lmp_hint = (lvaddr_t)&ep->k - (lvaddr_t)handle; 154 155 disp_run(handle); 156} 157 158/** 159 * \brief Initialise the dispatcher, while disabled 160 * 161 * This function is called to setup the dispatcher structure while still 162 * disabled. 163 * 164 * \param disp Dispatcher 165 */ 166void disp_init_disabled(dispatcher_handle_t handle) 167{ 168 assert_disabled(handle != 0); 169 struct dispatcher_generic* disp_gen = get_dispatcher_generic(handle); 170 struct dispatcher_shared_generic* disp = 171 get_dispatcher_shared_generic(handle); 172 assert_disabled(disp->disabled); 173 174 // Initialize entry points (and LDT on x86_64) 175 disp_arch_init(handle); 176 177 disp_gen->timeslice = 1; 178 systime_frequency = disp->systime_frequency; 179 // Initialize important capability pointers 180 if (disp_gen->dcb_cap.slot == 0) { 181 disp_gen->dcb_cap.cnode = cnode_task; 182 disp_gen->dcb_cap.slot = TASKCN_SLOT_DISPATCHER; 183 } 184 185 disp_gen->cleanupthread = NULL; 186 thread_mutex_init(&disp_gen->cleanupthread_lock); 187} 188 189/** 190 * \brief Yield the dispatcher's CPU 191 * 192 * This function yields the CPU. It may only be called while disabled. 193 * 194 * \param disp Current dispatcher 195 */ 196void disp_yield_disabled(dispatcher_handle_t handle) 197{ 198 struct dispatcher_shared_generic* disp = 199 get_dispatcher_shared_generic(handle); 200 assert_disabled(disp->disabled); 201 202#ifdef CONFIG_DEBUG_DEADLOCKS 203 disp->yieldcount++; 204#endif 205 206 // FIXME: This perticular trace event is breaking as it is running 207 // into problems due to assumptions about segment register %fs 208// trace_event(TRACE_SUBSYS_THREADS, TRACE_EVENT_THREADS_SYS_YIELD, 209// 2); 210 assert_disabled(disp->disabled); 211 sys_yield(CPTR_NULL); 212 assert_disabled(disp->disabled); 213 assert_print("dispatcher PANIC: sys_yield returned"); 214 for (;;); 215} 216 217/** 218 * \brief Disable the dispatcher 219 * 220 * This function disables the current dispatcher, returning a pointer to it. 221 * The dispatcher must be enabled. 222 * 223 * While the dispatcher is disabled, the current thread cannot be preempted, 224 * and no incoming LMP messages can be received. 225 */ 226dispatcher_handle_t disp_disable(void) 227{ 228 dispatcher_handle_t handle = curdispatcher(); 229 struct dispatcher_shared_generic* disp = 230 get_dispatcher_shared_generic(handle); 231 assert_disabled(disp->disabled == 0); 232 disp->disabled = 1; 233 return handle; 234} 235 236/** 237 * \brief Try to disable the dispatcher 238 * 239 * This function disables the current dispatcher if it's enabled 240 * and returns a pointer to it. 241 * 242 * While the dispatcher is disabled, the current thread cannot be preempted, 243 * and no incoming LMP messages can be received. 244 * 245 * \param was_enabled True, if the dispatcher was enabled 246 */ 247dispatcher_handle_t disp_try_disable(bool *was_enabled) 248{ 249 dispatcher_handle_t handle = curdispatcher(); 250 struct dispatcher_shared_generic* disp = 251 get_dispatcher_shared_generic(handle); 252#ifdef __k1om__ // K1om GCC 4.7.0 does not support __atomic_* functions 253 *was_enabled = __sync_bool_compare_and_swap(&disp->disabled, 0, 1); 254#else 255 *was_enabled = !__atomic_test_and_set(&disp->disabled, __ATOMIC_SEQ_CST); 256#endif 257 return handle; 258} 259 260/** 261 * \brief Re-enable the dispatcher 262 * 263 * This function re-enables the current dispatcher. 264 * The dispatcher must be disabled. 265 */ 266void disp_enable(dispatcher_handle_t handle) 267{ 268 assert_disabled(handle == curdispatcher()); 269 struct dispatcher_shared_generic* disp = 270 get_dispatcher_shared_generic(handle); 271 assert_disabled(disp->disabled == 1); 272 disp->disabled = 0; 273} 274 275/** 276 * \brief Return a pointer to the name field of the current dispatcher 277 * 278 * May be called when the dispatcher is either enabled or disabled. 279 * 280 * \returns a string of at most #DISP_NAME_LEN characters, 281 * which may not be nul-terminated. 282 */ 283const char *disp_name(void) 284{ 285 dispatcher_handle_t handle = curdispatcher(); 286 struct dispatcher_shared_generic* disp = 287 get_dispatcher_shared_generic(handle); 288 return disp->name; 289} 290 291 292#ifdef __k1om__ 293/** 294 * \brief Returns the Xeon Phi ID this dispatcher is running on 295 * 296 * May be called when the dispatcher is either enabled or disabled. 297 * 298 * \returns unsigned integer containing the Xeon Phi Id 299 */ 300uint8_t disp_xeon_phi_id(void) 301{ 302 dispatcher_handle_t handle = curdispatcher(); 303 struct dispatcher_shared_generic* disp = 304 get_dispatcher_shared_generic(handle); 305 return disp->xeon_phi_id; 306} 307#endif 308 309 310/** 311 * \brief Page fault entry point 312 * 313 * This function is called from assembly code when the kernel 314 * enters us to report a page fault while enabled. 315 * 316 * \param handle Dispatcher 317 * \param fault_address Fault address 318 * \param error CPU error code 319 * \param ip Faulting instruction pointer 320 */ 321void disp_pagefault(dispatcher_handle_t handle, lvaddr_t fault_address, 322 uintptr_t error, lvaddr_t ip) 323{ 324 struct dispatcher_shared_generic* disp = 325 get_dispatcher_shared_generic(handle); 326 struct dispatcher_generic *disp_gen = get_dispatcher_generic(handle); 327 arch_registers_state_t *regs = dispatcher_get_enabled_save_area(handle); 328 enum pagefault_exception_type fault_type; 329 330 // FIXME: this logic is x86-specific. it needs to be moved into an arch file 331#if defined(__x86_64__) || defined(__i386__) 332 const uintptr_t ERR_PF_PRESENT = (1 << 0); 333 const uintptr_t ERR_PF_READ_WRITE = (1 << 1); 334 const uintptr_t ERR_PF_USER_SUPERVISOR = (1 << 2); 335 const uintptr_t ERR_PF_RESERVED = (1 << 3); 336 const uintptr_t ERR_PF_INSTRUCTION = (1 << 4); 337 338 if ((error & ERR_PF_INSTRUCTION) != 0) { 339 fault_type = PAGEFLT_EXEC; 340 } else if ((error & ERR_PF_READ_WRITE) != 0) { 341 fault_type = PAGEFLT_WRITE; 342 } else { 343 fault_type = PAGEFLT_READ; 344 } 345#else 346 assert_print("Warning: don't know how to determine fault type on this arch!\n"); 347 fault_type = PAGEFLT_NULL; 348#endif 349 350 // sanity-check IP in save area 351 assert_disabled(ip == registers_get_ip(regs)); 352 353 // sanity-check that we were on a thread 354 assert_disabled(disp_gen->current != NULL); 355 356 // try to deliver exception 357 thread_deliver_exception_disabled(handle, EXCEPT_PAGEFAULT, fault_type, 358 (void *)fault_address, regs); 359 360 // it returned: this means the exception couldn't be delivered and the 361 // thread is dead. better tell them what happened... 362 363 static char str[256]; 364 snprintf(str, sizeof(str), "%.*s: unhandled page fault (error code 0x%" 365 PRIxPTR ") on %" PRIxPTR " at IP %" PRIxPTR "\n", 366 DISP_NAME_LEN, disp->name, error, fault_address, ip); 367 assert_print(str); 368 369 // dump hw page tables 370 debug_dump_hw_ptables(); 371 372#if defined(__x86_64__) || defined(__i386__) 373 snprintf(str, sizeof(str), "%s page fault due to %s%s, while in %s mode%s\n", 374 error & ERR_PF_READ_WRITE ? "write" : "read", 375 error & ERR_PF_PRESENT ? "access violation" : "page not present", 376 error & ERR_PF_RESERVED ? ", reserved bits set in page table" 377 : "", 378 error & ERR_PF_USER_SUPERVISOR ? "user" : "supervisor", 379 error & ERR_PF_INSTRUCTION ? ", by instruction fetch" : ""); 380 assert_print(str); 381#endif 382 383 // print out stuff 384 debug_print_save_area(regs); 385 debug_dump(regs); 386 //debug_call_chain(regs); 387 388 // run something else 389 thread_run_disabled(handle); 390} 391 392 393/** 394 * \brief Disabled page fault entry point 395 * 396 * This function is called from assembly code when the kernel enters us to 397 * report a page fault while disabled. 398 * 399 * \param handle Dispatcher 400 * \param fault_address Fault address 401 * \param error CPU error code 402 * \param ip Faulting instruction pointer 403 */ 404void disp_pagefault_disabled(dispatcher_handle_t handle, lvaddr_t fault_address, 405 uintptr_t error, lvaddr_t ip) 406{ 407 struct dispatcher_shared_generic *disp = 408 get_dispatcher_shared_generic(handle); 409 static char str[256]; 410 snprintf(str, 256, "%.*s: page fault WHILE DISABLED" 411 " (error code 0x%" PRIxPTR ") on %" PRIxPTR " at IP %" PRIxPTR "\n", 412 DISP_NAME_LEN, disp->name, error, fault_address, ip); 413 assert_print(str); 414 if(fault_address == 0) { 415 assert_print("NULL pointer dereferenced!\n"); 416 } 417 418 // NOTE: Based on which code is is causing page fault, only assert_print 419 // is safe bet to print anything here. Anything else would cause 420 // page fault in itself. 421 assert_disabled(disp->disabled); 422 423 424 // FIXME: Make sure that following are using assert_print to avoid 425 // loop of disabled pagefaults 426 // arch_registers_state_t *regs = dispatcher_get_trap_save_area(handle); 427 // debug_print_save_area(regs); 428 429 // disabled by AB, because we can get into a loop of disabled pagefaults 430 // debug_dump(regs); 431 // debug_return_addresses(); 432 for(;;); 433} 434 435#include <barrelfish/barrelfish.h> 436#include <barrelfish_kpi/capabilities.h> 437 438/** 439 * \brief Trap entry point 440 * 441 * This function is called from assembly code when the kernel enters us to 442 * report a trap taken while we were enabled. 443 * 444 * \param handle Dispatcher 445 * \param irq Trap vector 446 * \param error CPU error code 447 * \param ip Faulting instruction pointer 448 */ 449void disp_trap(dispatcher_handle_t handle, uintptr_t irq, uintptr_t error, 450 lvaddr_t ip) 451{ 452 struct dispatcher_shared_generic* disp = 453 get_dispatcher_shared_generic(handle); 454 arch_registers_state_t *regs = dispatcher_get_trap_save_area(handle); 455 struct dispatcher_generic *disp_gen = get_dispatcher_generic(handle); 456 457 // Must've happened on a thread 458 struct thread *t = disp_gen->current; 459 assert_disabled(t != NULL); 460 461 // sanity-check IP in save area 462 // not valid for debug exceptions? 463 // assert_disabled(ip == registers_get_ip(regs)); 464 465 enum exception_type type; 466#if defined(__i386__) || defined(__x86_64__) 467 switch (irq) { 468 case IDT_DB: 469 type = EXCEPT_SINGLESTEP; 470 break; 471 472 case IDT_BP: 473 type = EXCEPT_BREAKPOINT; 474 break; 475 476 default: 477 type = EXCEPT_OTHER; 478 break; 479 } 480#else 481 type = EXCEPT_OTHER; // XXX 482#endif 483 484 // deliver exception (shouldn't return) 485 thread_deliver_exception_disabled(handle, type, irq, (void *)ip, regs); 486 487 // if it failed, say something 488 static char str[256]; 489 snprintf(str, sizeof(str), "%.*s: unhandled trap (IRQ %" PRIuPTR 490 ", error code 0x%" PRIxPTR ") at IP %" PRIxPTR "\n", 491 DISP_NAME_LEN, disp->name, irq, error, ip); 492 assert_print(str); 493 494 // print out stuff 495 debug_print_save_area(regs); 496 //debug_call_chain(regs); 497 498 // run something else 499 thread_run_disabled(handle); 500} 501 502void 503disp_assert_fail(const char *exp, const char *file, const char *func, const char *line) 504{ 505 const char *dispname = disp_name(); 506 507 // We can't call printf(), so do this silly thing... 508 assert_print("Dispatcher assertion failed in "); 509 for(int i = 0; i < DISP_NAME_LEN && dispname[i] != '\0'; i++) { 510 sys_print(&dispname[i], 1); 511 } 512 assert_print(": "); 513 assert_print(exp); 514 assert_print(", function "); 515 assert_print(func); 516 assert_print(", file "); 517 assert_print(file); 518 assert_print(", line "); 519 assert_print(line); 520 assert_print(".\n"); 521 522 for(;;); 523} 524 525void 526disp_warn_fail(const char *exp, const char *file, const char *func, const char *line) 527{ 528 const char *dispname = disp_name(); 529 530 // We can't call printf(), so do this silly thing... 531 assert_print("Dispatcher warning in "); 532 for(int i = 0; i < DISP_NAME_LEN && dispname[i] != '\0'; i++) { 533 sys_print(&dispname[i], 1); 534 } 535 assert_print(": "); 536 assert_print(exp); 537 assert_print(", function "); 538 assert_print(func); 539 assert_print(", file "); 540 assert_print(file); 541 assert_print(", line "); 542 assert_print(line); 543 assert_print(".\n"); 544} 545