1/** 2 * \file 3 * \brief Dispatcher implementation. 4 */ 5 6/* 7 * Copyright (c) 2007, 2008, 2009, 2010, 2011, 2012, ETH Zurich. 8 * All rights reserved. 9 * 10 * This file is distributed under the terms in the attached LICENSE file. 11 * If you do not find this file, copies can be found by writing to: 12 * ETH Zurich D-INFK, Universitaetstrasse 6, CH-8092 Zurich. Attn: Systems Group. 13 */ 14 15#include <inttypes.h> 16#include <stdlib.h> 17#include <stdio.h> 18#include <string.h> 19#include <barrelfish/barrelfish.h> 20#include <barrelfish/dispatch.h> 21#include <barrelfish/dispatcher_arch.h> 22#include <barrelfish/curdispatcher_arch.h> 23#include <barrelfish/caddr.h> 24#include <barrelfish/debug.h> 25#include <barrelfish/deferred.h> 26#include <barrelfish_kpi/cpu_arch.h> 27#include "threads_priv.h" 28#include <barrelfish/notificator.h> 29#include <barrelfish/systime.h> 30 31#include <trace/trace.h> 32#include <trace_definitions/trace_defs.h> 33 34#ifdef CONFIG_INTERCONNECT_DRIVER_LMP 35# include <barrelfish/lmp_chan.h> 36#endif 37 38 39#if defined(__GNUC__) && !defined(__clang__) && !defined(__ICC) \ 40 && (defined(__x86_64__) || (defined(__i386__))) 41// Disable SSE/MMX -- this code context switches those registers 42# pragma GCC target ("no-mmx,no-sse,no-sse2,no-sse3,no-sse4.1,no-sse4.2,no-sse4,no-sse4a,no-3dnow") 43#endif 44 45void disp_run(dispatcher_handle_t handle); 46void disp_lrpc(struct lmp_endpoint *ep, uint32_t bufpos, 47 uintptr_t arg1, uintptr_t arg2, uintptr_t arg3, uintptr_t arg4, 48 dispatcher_handle_t handle); 49void disp_pagefault(dispatcher_handle_t handle, lvaddr_t fault_address, 50 uintptr_t error, lvaddr_t ip); 51void disp_pagefault_disabled(dispatcher_handle_t handle, lvaddr_t fault_address, 52 uintptr_t error, lvaddr_t ip); 53void disp_trap(dispatcher_handle_t handle, uintptr_t irq, uintptr_t error, 54 lvaddr_t ip); 55 56static inline void assert_print(const char *str) 57{ 58 if (!str) { 59 return; 60 } 61 sys_print(str, strlen(str)); 62} 63 64static uint64_t run_counter = 0; 65uint64_t disp_run_counter(void) 66{ 67 return run_counter; 68} 69/** 70 * \brief Run entry point 71 * 72 * This function is called from assembly code when the kernel enters us to 73 * give us the CPU. 74 * 75 * \param disp Dispatcher 76 */ 77void disp_run(dispatcher_handle_t handle) 78{ 79#ifdef __x86_64__ 80 struct dispatcher_x86_64 *disp_priv = get_dispatcher_x86_64(handle); 81 /* load compatibility dispatcher segment to FS */ 82 __asm volatile("mov %%ax, %%fs" 83 : /* No outputs */ 84 : "a" (disp_priv->disp_seg_selector)); 85#endif 86 // We can't call printf(), so do this silly thing... 87// assert_print("FIXME: infinite while loop\n"); 88// while(1); 89 90 struct dispatcher_generic* disp_gen = get_dispatcher_generic(handle); 91 struct dispatcher_shared_generic* disp = 92 get_dispatcher_shared_generic(handle); 93 94 assert_disabled(disp->disabled); 95 ++run_counter; 96 disp_gen->timeslice++; 97 // Never let 0 be a valid timeslice number 98 if(disp_gen->timeslice == 0) { 99 disp_gen->timeslice++; 100 } 101 102 // Trigger any deferred events 103 trigger_deferred_events_disabled(handle, disp->systime); 104 105#ifdef CONFIG_INTERCONNECT_DRIVER_LMP 106 // Check for incoming LMP messages 107 if (disp->lmp_delivered != disp->lmp_seen) { 108 lmp_endpoints_poll_disabled(handle); 109 } 110 111 // Trigger any send events for LMP channels 112 lmp_channels_retry_send_disabled(handle); 113#endif // CONFIG_INTERCONNECT_DRIVER_LMP 114 // Check polled channels 115 poll_channels_disabled(handle); 116 ump_channels_retry_send_disabled(handle); 117 118 check_notificators_disabled(handle); 119 120 // Run, saving state of previous thread if required 121 thread_run_disabled(handle); 122 123 // NOTREACHED 124 assert_disabled(!"disp_run: thread_run() returned!\n"); 125 for(;;); 126} 127 128/** 129 * \brief LRPC entry point 130 * 131 * This function is called from assembly code when the kernel enters us to 132 * give us the CPU and deliver an LRPC message. The dispatcher is disabled. 133 * 134 * \param ep LMP endpoint structure 135 * \param bufpos Reserved position in endpoint message buffer 136 * \param arg1 Message payload 137 * \param arg2 Message payload 138 * \param arg3 Message payload 139 * \param arg4 Message payload 140 * \param handle Dispatcher pointer 141 * 142 * \note Dispatcher pointer comes last here, because AB was too lazy to shuffle 143 * registers through the whole LRPC path when adding it. 144 */ 145void disp_lrpc(struct lmp_endpoint *ep, uint32_t bufpos, 146 uintptr_t arg1, uintptr_t arg2, uintptr_t arg3, uintptr_t arg4, 147 dispatcher_handle_t handle) 148{ 149 struct dispatcher_shared_generic* disp = 150 get_dispatcher_shared_generic(handle); 151 152 /* deliver LRPC message to buffer */ 153 lmp_endpoint_store_lrpc_disabled(ep, bufpos, arg1, arg2, arg3, arg4); 154 155 // set hint 156 disp->lmp_hint = (lvaddr_t)&ep->k - (lvaddr_t)handle; 157 158 disp_run(handle); 159} 160 161/** 162 * \brief Initialise the dispatcher, while disabled 163 * 164 * This function is called to setup the dispatcher structure while still 165 * disabled. 166 * 167 * \param disp Dispatcher 168 */ 169void disp_init_disabled(dispatcher_handle_t handle) 170{ 171 assert_disabled(handle != 0); 172 struct dispatcher_generic* disp_gen = get_dispatcher_generic(handle); 173 struct dispatcher_shared_generic* disp = 174 get_dispatcher_shared_generic(handle); 175 assert_disabled(disp->disabled); 176 177 // Initialize entry points (and LDT on x86_64) 178 disp_arch_init(handle); 179 180 disp_gen->timeslice = 1; 181 systime_frequency = disp->systime_frequency; 182 // Initialize important capability pointers 183 if (disp_gen->dcb_cap.slot == 0) { 184 disp_gen->dcb_cap.cnode = cnode_task; 185 disp_gen->dcb_cap.slot = TASKCN_SLOT_DISPATCHER; 186 } 187 188 disp_gen->cleanupthread = NULL; 189 thread_mutex_init(&disp_gen->cleanupthread_lock); 190} 191 192/** 193 * \brief Yield the dispatcher's CPU 194 * 195 * This function yields the CPU. It may only be called while disabled. 196 * 197 * \param disp Current dispatcher 198 */ 199void disp_yield_disabled(dispatcher_handle_t handle) 200{ 201 struct dispatcher_shared_generic* disp = 202 get_dispatcher_shared_generic(handle); 203 assert_disabled(disp->disabled); 204 205#ifdef CONFIG_DEBUG_DEADLOCKS 206 disp->yieldcount++; 207#endif 208 209 // FIXME: This perticular trace event is breaking as it is running 210 // into problems due to assumptions about segment register %fs 211// trace_event(TRACE_SUBSYS_THREADS, TRACE_EVENT_THREADS_SYS_YIELD, 212// 2); 213 assert_disabled(disp->disabled); 214 sys_yield(CPTR_NULL); 215 assert_disabled(disp->disabled); 216 assert_print("dispatcher PANIC: sys_yield returned"); 217 for (;;); 218} 219 220/** 221 * \brief Disable the dispatcher 222 * 223 * This function disables the current dispatcher, returning a pointer to it. 224 * The dispatcher must be enabled. 225 * 226 * While the dispatcher is disabled, the current thread cannot be preempted, 227 * and no incoming LMP messages can be received. 228 */ 229dispatcher_handle_t disp_disable(void) 230{ 231 dispatcher_handle_t handle = curdispatcher(); 232 struct dispatcher_shared_generic* disp = 233 get_dispatcher_shared_generic(handle); 234 assert_disabled(disp->disabled == 0); 235 disp->disabled = 1; 236 return handle; 237} 238 239/** 240 * \brief Try to disable the dispatcher 241 * 242 * This function disables the current dispatcher if it's enabled 243 * and returns a pointer to it. 244 * 245 * While the dispatcher is disabled, the current thread cannot be preempted, 246 * and no incoming LMP messages can be received. 247 * 248 * \param was_enabled True, if the dispatcher was enabled 249 */ 250dispatcher_handle_t disp_try_disable(bool *was_enabled) 251{ 252 dispatcher_handle_t handle = curdispatcher(); 253 struct dispatcher_shared_generic* disp = 254 get_dispatcher_shared_generic(handle); 255#ifdef __k1om__ // K1om GCC 4.7.0 does not support __atomic_* functions 256 *was_enabled = __sync_bool_compare_and_swap(&disp->disabled, 0, 1); 257#else 258 *was_enabled = !__atomic_test_and_set(&disp->disabled, __ATOMIC_SEQ_CST); 259#endif 260 return handle; 261} 262 263/** 264 * \brief Re-enable the dispatcher 265 * 266 * This function re-enables the current dispatcher. 267 * The dispatcher must be disabled. 268 */ 269void disp_enable(dispatcher_handle_t handle) 270{ 271 assert_disabled(handle == curdispatcher()); 272 struct dispatcher_shared_generic* disp = 273 get_dispatcher_shared_generic(handle); 274 assert_disabled(disp->disabled == 1); 275 disp->disabled = 0; 276} 277 278/** 279 * \brief Return a pointer to the name field of the current dispatcher 280 * 281 * May be called when the dispatcher is either enabled or disabled. 282 * 283 * \returns a string of at most #DISP_NAME_LEN characters, 284 * which may not be nul-terminated. 285 */ 286const char *disp_name(void) 287{ 288 dispatcher_handle_t handle = curdispatcher(); 289 struct dispatcher_shared_generic* disp = 290 get_dispatcher_shared_generic(handle); 291 return disp->name; 292} 293 294 295#ifdef __k1om__ 296/** 297 * \brief Returns the Xeon Phi ID this dispatcher is running on 298 * 299 * May be called when the dispatcher is either enabled or disabled. 300 * 301 * \returns unsigned integer containing the Xeon Phi Id 302 */ 303uint8_t disp_xeon_phi_id(void) 304{ 305 dispatcher_handle_t handle = curdispatcher(); 306 struct dispatcher_shared_generic* disp = 307 get_dispatcher_shared_generic(handle); 308 return disp->xeon_phi_id; 309} 310#endif 311 312 313/** 314 * \brief Page fault entry point 315 * 316 * This function is called from assembly code when the kernel 317 * enters us to report a page fault while enabled. 318 * 319 * \param handle Dispatcher 320 * \param fault_address Fault address 321 * \param error CPU error code 322 * \param ip Faulting instruction pointer 323 */ 324void disp_pagefault(dispatcher_handle_t handle, lvaddr_t fault_address, 325 uintptr_t error, lvaddr_t ip) 326{ 327 struct dispatcher_shared_generic* disp = 328 get_dispatcher_shared_generic(handle); 329 struct dispatcher_generic *disp_gen = get_dispatcher_generic(handle); 330 arch_registers_state_t *regs = dispatcher_get_enabled_save_area(handle); 331 enum pagefault_exception_type fault_type; 332 333 // FIXME: this logic is x86-specific. it needs to be moved into an arch file 334#if defined(__x86_64__) || defined(__i386__) 335 const uintptr_t ERR_PF_PRESENT = (1 << 0); 336 const uintptr_t ERR_PF_READ_WRITE = (1 << 1); 337 const uintptr_t ERR_PF_USER_SUPERVISOR = (1 << 2); 338 const uintptr_t ERR_PF_RESERVED = (1 << 3); 339 const uintptr_t ERR_PF_INSTRUCTION = (1 << 4); 340 341 if ((error & ERR_PF_INSTRUCTION) != 0) { 342 fault_type = PAGEFLT_EXEC; 343 } else if ((error & ERR_PF_READ_WRITE) != 0) { 344 fault_type = PAGEFLT_WRITE; 345 } else { 346 fault_type = PAGEFLT_READ; 347 } 348#elif defined(__ARM_ARCH_8A__) 349 const uintptr_t EC_IABT = 0x20; 350 const uintptr_t EC_DABT = 0x24; 351 const uintptr_t ISS_WRITE_NOT_READ = (1 << 6); 352 353 uintptr_t ec = error >> 26; 354 355 if (ec == EC_IABT) { 356 fault_type = PAGEFLT_EXEC; 357 } 358 else if ((ec == EC_DABT) && (error & ISS_WRITE_NOT_READ)) { 359 fault_type = PAGEFLT_WRITE; 360 } 361 else if (ec == EC_DABT) { 362 fault_type = PAGEFLT_READ; 363 } 364 else { 365 fault_type = PAGEFLT_NULL; 366 } 367#else 368 assert_print("Warning: don't know how to determine fault type on this arch!\n"); 369 fault_type = PAGEFLT_NULL; 370#endif 371 372 // sanity-check IP in save area 373 assert_disabled(ip == registers_get_ip(regs)); 374 375 // sanity-check that we were on a thread 376 assert_disabled(disp_gen->current != NULL); 377 378 // try to deliver exception 379 thread_deliver_exception_disabled(handle, EXCEPT_PAGEFAULT, fault_type, 380 (void *)fault_address, regs); 381 382 // it returned: this means the exception couldn't be delivered and the 383 // thread is dead. better tell them what happened... 384 385 static char str[256]; 386 snprintf(str, sizeof(str), "%.*s.%d: unhandled page fault (error code 0x%" 387 PRIxPTR ") on %" PRIxPTR " at IP %" PRIxPTR "\n", 388 DISP_NAME_LEN, disp->name, disp_get_current_core_id(), error, fault_address, ip); 389 assert_print(str); 390 391 // dump hw page tables 392 //debug_dump_hw_ptables(); 393 394#if defined(__x86_64__) || defined(__i386__) 395 snprintf(str, sizeof(str), "%s page fault due to %s%s, while in %s mode%s\n", 396 error & ERR_PF_READ_WRITE ? "write" : "read", 397 error & ERR_PF_PRESENT ? "access violation" : "page not present", 398 error & ERR_PF_RESERVED ? ", reserved bits set in page table" 399 : "", 400 error & ERR_PF_USER_SUPERVISOR ? "user" : "supervisor", 401 error & ERR_PF_INSTRUCTION ? ", by instruction fetch" : ""); 402 assert_print(str); 403#endif 404 405 // print out stuff 406 debug_print_save_area(regs); 407 debug_dump(regs); 408 //debug_call_chain(regs); 409 410 // run something else 411 thread_run_disabled(handle); 412} 413 414 415/** 416 * \brief Disabled page fault entry point 417 * 418 * This function is called from assembly code when the kernel enters us to 419 * report a page fault while disabled. 420 * 421 * \param handle Dispatcher 422 * \param fault_address Fault address 423 * \param error CPU error code 424 * \param ip Faulting instruction pointer 425 */ 426void disp_pagefault_disabled(dispatcher_handle_t handle, lvaddr_t fault_address, 427 uintptr_t error, lvaddr_t ip) 428{ 429 struct dispatcher_shared_generic *disp = 430 get_dispatcher_shared_generic(handle); 431 static char str[256]; 432 snprintf(str, 256, "%.*s: page fault WHILE DISABLED" 433 " (error code 0x%" PRIxPTR ") on %" PRIxPTR " at IP %" PRIxPTR "\n", 434 DISP_NAME_LEN, disp->name, error, fault_address, ip); 435 assert_print(str); 436 if(fault_address == 0) { 437 assert_print("NULL pointer dereferenced!\n"); 438 } 439 440 // NOTE: Based on which code is is causing page fault, only assert_print 441 // is safe bet to print anything here. Anything else would cause 442 // page fault in itself. 443 assert_disabled(disp->disabled); 444 445 446#ifdef __x86_64__ 447 // give more info about faults that basically cannot be in text section 448 if (fault_address >= (1ULL << 39)) { 449 // FIXME: Make sure that following are using assert_print to avoid 450 // loop of disabled pagefaults 451 arch_registers_state_t *regs = dispatcher_get_trap_save_area(handle); 452 debug_print_save_area(regs); 453 454 // disabled by AB, because we can get into a loop of disabled pagefaults 455 debug_dump(regs); 456 debug_return_addresses(); 457 } 458#endif 459 for(;;); 460} 461 462#include <barrelfish/barrelfish.h> 463#include <barrelfish_kpi/capabilities.h> 464 465/** 466 * \brief Trap entry point 467 * 468 * This function is called from assembly code when the kernel enters us to 469 * report a trap taken while we were enabled. 470 * 471 * \param handle Dispatcher 472 * \param irq Trap vector 473 * \param error CPU error code 474 * \param ip Faulting instruction pointer 475 */ 476void disp_trap(dispatcher_handle_t handle, uintptr_t irq, uintptr_t error, 477 lvaddr_t ip) 478{ 479 struct dispatcher_shared_generic* disp = 480 get_dispatcher_shared_generic(handle); 481 arch_registers_state_t *regs = dispatcher_get_trap_save_area(handle); 482 struct dispatcher_generic *disp_gen = get_dispatcher_generic(handle); 483 484 // Must've happened on a thread 485 struct thread *t = disp_gen->current; 486 assert_disabled(t != NULL); 487 488 // sanity-check IP in save area 489 // not valid for debug exceptions? 490 // assert_disabled(ip == registers_get_ip(regs)); 491 492 enum exception_type type; 493#if defined(__i386__) || defined(__x86_64__) 494 switch (irq) { 495 case IDT_DB: 496 type = EXCEPT_SINGLESTEP; 497 break; 498 499 case IDT_BP: 500 type = EXCEPT_BREAKPOINT; 501 break; 502 503 default: 504 type = EXCEPT_OTHER; 505 break; 506 } 507#else 508 type = EXCEPT_OTHER; // XXX 509#endif 510 511 // deliver exception (shouldn't return) 512 thread_deliver_exception_disabled(handle, type, irq, (void *)ip, regs); 513 514 // if it failed, say something 515 static char str[256]; 516 snprintf(str, sizeof(str), "%.*s: unhandled trap (IRQ %" PRIuPTR 517 ", error code 0x%" PRIxPTR ") at IP %" PRIxPTR "\n", 518 DISP_NAME_LEN, disp->name, irq, error, ip); 519 assert_print(str); 520 521 // print out stuff 522 debug_print_save_area(regs); 523 //debug_call_chain(regs); 524 525 // run something else 526 thread_run_disabled(handle); 527} 528 529void 530disp_assert_fail(const char *exp, const char *file, const char *func, const char *line) 531{ 532 const char *dispname = disp_name(); 533 534 // We can't call printf(), so do this silly thing... 535 assert_print("Dispatcher assertion failed in "); 536 for(int i = 0; i < DISP_NAME_LEN && dispname[i] != '\0'; i++) { 537 sys_print(&dispname[i], 1); 538 } 539 assert_print(": "); 540 assert_print(exp); 541 assert_print(", function "); 542 assert_print(func); 543 assert_print(", file "); 544 assert_print(file); 545 assert_print(", line "); 546 assert_print(line); 547 assert_print(".\n"); 548 549 for(;;); 550} 551 552void 553disp_warn_fail(const char *exp, const char *file, const char *func, const char *line) 554{ 555 const char *dispname = disp_name(); 556 557 // We can't call printf(), so do this silly thing... 558 assert_print("Dispatcher warning in "); 559 for(int i = 0; i < DISP_NAME_LEN && dispname[i] != '\0'; i++) { 560 sys_print(&dispname[i], 1); 561 } 562 assert_print(": "); 563 assert_print(exp); 564 assert_print(", function "); 565 assert_print(func); 566 assert_print(", file "); 567 assert_print(file); 568 assert_print(", line "); 569 assert_print(line); 570 assert_print(".\n"); 571} 572