1/* 2 * Copyright 2013, Pawe�� Dziepak, pdziepak@quarnos.org. 3 * Copyright 2008-2009, Ingo Weinhold, ingo_weinhold@gmx.de. 4 * Copyright 2002-2010, Axel D��rfler, axeld@pinc-software.de. 5 * Distributed under the terms of the MIT License. 6 * 7 * Copyright 2001-2002, Travis Geiselbrecht. All rights reserved. 8 * Distributed under the terms of the NewOS License. 9 */ 10 11 12/*! Functionality for symetrical multi-processors */ 13 14 15#include <smp.h> 16 17#include <stdlib.h> 18#include <string.h> 19 20#include <arch/atomic.h> 21#include <arch/cpu.h> 22#include <arch/debug.h> 23#include <arch/int.h> 24#include <arch/smp.h> 25#include <boot/kernel_args.h> 26#include <cpu.h> 27#include <generic_syscall.h> 28#include <int.h> 29#include <spinlock_contention.h> 30#include <thread.h> 31#include <util/atomic.h> 32 33#include "kernel_debug_config.h" 34 35 36//#define TRACE_SMP 37#ifdef TRACE_SMP 38# define TRACE(...) dprintf_no_syslog(__VA_ARGS__) 39#else 40# define TRACE(...) (void)0 41#endif 42 43 44#undef try_acquire_spinlock 45#undef acquire_spinlock 46#undef release_spinlock 47 48#undef try_acquire_read_spinlock 49#undef acquire_read_spinlock 50#undef release_read_spinlock 51#undef try_acquire_write_spinlock 52#undef acquire_write_spinlock 53#undef release_write_spinlock 54 55#undef try_acquire_write_seqlock 56#undef acquire_write_seqlock 57#undef release_write_seqlock 58#undef acquire_read_seqlock 59#undef release_read_seqlock 60 61 62#define MSG_POOL_SIZE (SMP_MAX_CPUS * 4) 63 64// These macros define the number of unsuccessful iterations in 65// acquire_spinlock() and acquire_spinlock_nocheck() after which the functions 66// panic(), assuming a deadlock. 67#define SPINLOCK_DEADLOCK_COUNT 100000000 68#define SPINLOCK_DEADLOCK_COUNT_NO_CHECK 2000000000 69 70 71struct smp_msg { 72 struct smp_msg *next; 73 int32 message; 74 addr_t data; 75 addr_t data2; 76 addr_t data3; 77 void *data_ptr; 78 uint32 flags; 79 int32 ref_count; 80 int32 done; 81 CPUSet proc_bitmap; 82}; 83 84enum mailbox_source { 85 MAILBOX_LOCAL, 86 MAILBOX_BCAST, 87}; 88 89static int32 sBootCPUSpin = 0; 90 91static int32 sEarlyCPUCallCount; 92static CPUSet sEarlyCPUCallSet; 93static void (*sEarlyCPUCallFunction)(void*, int); 94void* sEarlyCPUCallCookie; 95 96static struct smp_msg* sFreeMessages = NULL; 97static int32 sFreeMessageCount = 0; 98static spinlock sFreeMessageSpinlock = B_SPINLOCK_INITIALIZER; 99 100static struct smp_msg* sCPUMessages[SMP_MAX_CPUS] = { NULL, }; 101 102static struct smp_msg* sBroadcastMessages = NULL; 103static spinlock sBroadcastMessageSpinlock = B_SPINLOCK_INITIALIZER; 104static int32 sBroadcastMessageCounter; 105 106static bool sICIEnabled = false; 107static int32 sNumCPUs = 1; 108 109static int32 process_pending_ici(int32 currentCPU); 110 111 112#if DEBUG_SPINLOCKS 113#define NUM_LAST_CALLERS 32 114 115static struct { 116 void *caller; 117 spinlock *lock; 118} sLastCaller[NUM_LAST_CALLERS]; 119 120static int32 sLastIndex = 0; 121 // Is incremented atomically. Must be % NUM_LAST_CALLERS before being used 122 // as index into sLastCaller. Note, that it has to be casted to uint32 123 // before applying the modulo operation, since otherwise after overflowing 124 // that would yield negative indices. 125 126 127static void 128push_lock_caller(void* caller, spinlock* lock) 129{ 130 int32 index = (uint32)atomic_add(&sLastIndex, 1) % NUM_LAST_CALLERS; 131 132 sLastCaller[index].caller = caller; 133 sLastCaller[index].lock = lock; 134} 135 136 137static void* 138find_lock_caller(spinlock* lock) 139{ 140 int32 lastIndex = (uint32)atomic_get(&sLastIndex) % NUM_LAST_CALLERS; 141 142 for (int32 i = 0; i < NUM_LAST_CALLERS; i++) { 143 int32 index = (NUM_LAST_CALLERS + lastIndex - 1 - i) % NUM_LAST_CALLERS; 144 if (sLastCaller[index].lock == lock) 145 return sLastCaller[index].caller; 146 } 147 148 return NULL; 149} 150 151 152int 153dump_spinlock(int argc, char** argv) 154{ 155 if (argc != 2) { 156 print_debugger_command_usage(argv[0]); 157 return 0; 158 } 159 160 uint64 address; 161 if (!evaluate_debug_expression(argv[1], &address, false)) 162 return 0; 163 164 spinlock* lock = (spinlock*)(addr_t)address; 165 kprintf("spinlock %p:\n", lock); 166 bool locked = B_SPINLOCK_IS_LOCKED(lock); 167 if (locked) { 168 kprintf(" locked from %p\n", find_lock_caller(lock)); 169 } else 170 kprintf(" not locked\n"); 171 172#if B_DEBUG_SPINLOCK_CONTENTION 173 kprintf(" failed try_acquire(): %d\n", lock->failed_try_acquire); 174 kprintf(" total wait time: %" B_PRIdBIGTIME "\n", lock->total_wait); 175 kprintf(" total held time: %" B_PRIdBIGTIME "\n", lock->total_held); 176 kprintf(" last acquired at: %" B_PRIdBIGTIME "\n", lock->last_acquired); 177#endif 178 179 return 0; 180} 181 182 183#endif // DEBUG_SPINLOCKS 184 185 186#if B_DEBUG_SPINLOCK_CONTENTION 187 188 189static inline void 190update_lock_contention(spinlock* lock, bigtime_t start) 191{ 192 const bigtime_t now = system_time(); 193 lock->last_acquired = now; 194 lock->total_wait += (now - start); 195} 196 197 198static inline void 199update_lock_held(spinlock* lock) 200{ 201 const bigtime_t held = (system_time() - lock->last_acquired); 202 lock->total_held += held; 203 204//#define DEBUG_SPINLOCK_LATENCIES 2000 205#if DEBUG_SPINLOCK_LATENCIES 206 if (held > DEBUG_SPINLOCK_LATENCIES) { 207 panic("spinlock %p was held for %" B_PRIdBIGTIME " usecs (%d allowed)\n", 208 lock, held, DEBUG_SPINLOCK_LATENCIES); 209 } 210#endif // DEBUG_SPINLOCK_LATENCIES 211} 212 213 214#endif // B_DEBUG_SPINLOCK_CONTENTION 215 216 217int 218dump_ici_messages(int argc, char** argv) 219{ 220 // count broadcast messages 221 int32 count = 0; 222 int32 doneCount = 0; 223 int32 unreferencedCount = 0; 224 smp_msg* message = sBroadcastMessages; 225 while (message != NULL) { 226 count++; 227 if (message->done == 1) 228 doneCount++; 229 if (message->ref_count <= 0) 230 unreferencedCount++; 231 message = message->next; 232 } 233 234 kprintf("ICI broadcast messages: %" B_PRId32 ", first: %p\n", count, 235 sBroadcastMessages); 236 kprintf(" done: %" B_PRId32 "\n", doneCount); 237 kprintf(" unreferenced: %" B_PRId32 "\n", unreferencedCount); 238 239 // count per-CPU messages 240 for (int32 i = 0; i < sNumCPUs; i++) { 241 count = 0; 242 message = sCPUMessages[i]; 243 while (message != NULL) { 244 count++; 245 message = message->next; 246 } 247 248 kprintf("CPU %" B_PRId32 " messages: %" B_PRId32 ", first: %p\n", i, 249 count, sCPUMessages[i]); 250 } 251 252 return 0; 253} 254 255 256int 257dump_ici_message(int argc, char** argv) 258{ 259 if (argc != 2) { 260 print_debugger_command_usage(argv[0]); 261 return 0; 262 } 263 264 uint64 address; 265 if (!evaluate_debug_expression(argv[1], &address, false)) 266 return 0; 267 268 smp_msg* message = (smp_msg*)(addr_t)address; 269 kprintf("ICI message %p:\n", message); 270 kprintf(" next: %p\n", message->next); 271 kprintf(" message: %" B_PRId32 "\n", message->message); 272 kprintf(" data: 0x%lx\n", message->data); 273 kprintf(" data2: 0x%lx\n", message->data2); 274 kprintf(" data3: 0x%lx\n", message->data3); 275 kprintf(" data_ptr: %p\n", message->data_ptr); 276 kprintf(" flags: %" B_PRIx32 "\n", message->flags); 277 kprintf(" ref_count: %" B_PRIx32 "\n", message->ref_count); 278 kprintf(" done: %s\n", message->done == 1 ? "true" : "false"); 279 280 kprintf(" proc_bitmap: "); 281 for (int32 i = 0; i < sNumCPUs; i++) { 282 if (message->proc_bitmap.GetBit(i)) 283 kprintf("%s%" B_PRId32, i != 0 ? ", " : "", i); 284 } 285 kprintf("\n"); 286 287 return 0; 288} 289 290 291static inline void 292process_all_pending_ici(int32 currentCPU) 293{ 294 while (process_pending_ici(currentCPU) != B_ENTRY_NOT_FOUND) 295 ; 296} 297 298 299bool 300try_acquire_spinlock(spinlock* lock) 301{ 302#if DEBUG_SPINLOCKS 303 if (are_interrupts_enabled()) { 304 panic("try_acquire_spinlock: attempt to acquire lock %p with " 305 "interrupts enabled", lock); 306 } 307#endif 308 309 if (atomic_get_and_set(&lock->lock, 1) != 0) { 310#if B_DEBUG_SPINLOCK_CONTENTION 311 atomic_add(&lock->failed_try_acquire, 1); 312#endif 313 return false; 314 } 315 316#if B_DEBUG_SPINLOCK_CONTENTION 317 update_lock_contention(lock, system_time()); 318#endif 319 320#if DEBUG_SPINLOCKS 321 push_lock_caller(arch_debug_get_caller(), lock); 322#endif 323 324 return true; 325} 326 327 328void 329acquire_spinlock(spinlock* lock) 330{ 331#if DEBUG_SPINLOCKS 332 if (are_interrupts_enabled()) { 333 panic("acquire_spinlock: attempt to acquire lock %p with interrupts " 334 "enabled", lock); 335 } 336#endif 337 338 if (sNumCPUs > 1) { 339#if B_DEBUG_SPINLOCK_CONTENTION 340 const bigtime_t start = system_time(); 341#endif 342 int currentCPU = smp_get_current_cpu(); 343 while (1) { 344 uint32 count = 0; 345 while (lock->lock != 0) { 346 if (++count == SPINLOCK_DEADLOCK_COUNT) { 347#if DEBUG_SPINLOCKS 348 panic("acquire_spinlock(): Failed to acquire spinlock %p " 349 "for a long time (last caller: %p, value: %" B_PRIx32 350 ")", lock, find_lock_caller(lock), lock->lock); 351#else 352 panic("acquire_spinlock(): Failed to acquire spinlock %p " 353 "for a long time (value: %" B_PRIx32 ")", lock, 354 lock->lock); 355#endif 356 count = 0; 357 } 358 359 process_all_pending_ici(currentCPU); 360 cpu_wait(&lock->lock, 0); 361 } 362 if (atomic_get_and_set(&lock->lock, 1) == 0) 363 break; 364 } 365 366#if B_DEBUG_SPINLOCK_CONTENTION 367 update_lock_contention(lock, start); 368#endif 369 370#if DEBUG_SPINLOCKS 371 push_lock_caller(arch_debug_get_caller(), lock); 372#endif 373 } else { 374#if B_DEBUG_SPINLOCK_CONTENTION 375 lock->last_acquired = system_time(); 376#endif 377#if DEBUG_SPINLOCKS 378 int32 oldValue = atomic_get_and_set(&lock->lock, 1); 379 if (oldValue != 0) { 380 panic("acquire_spinlock: attempt to acquire lock %p twice on " 381 "non-SMP system (last caller: %p, value %" B_PRIx32 ")", lock, 382 find_lock_caller(lock), oldValue); 383 } 384 385 push_lock_caller(arch_debug_get_caller(), lock); 386#endif 387 } 388} 389 390 391static void 392acquire_spinlock_nocheck(spinlock *lock) 393{ 394#if DEBUG_SPINLOCKS 395 if (are_interrupts_enabled()) { 396 panic("acquire_spinlock_nocheck: attempt to acquire lock %p with " 397 "interrupts enabled", lock); 398 } 399#endif 400 401 if (sNumCPUs > 1) { 402#if B_DEBUG_SPINLOCK_CONTENTION 403 const bigtime_t start = system_time(); 404#endif 405 while (1) { 406 uint32 count = 0; 407 while (lock->lock != 0) { 408 if (++count == SPINLOCK_DEADLOCK_COUNT_NO_CHECK) { 409#if DEBUG_SPINLOCKS 410 panic("acquire_spinlock_nocheck(): Failed to acquire " 411 "spinlock %p for a long time (last caller: %p, value: %" 412 B_PRIx32 ")", lock, find_lock_caller(lock), lock->lock); 413#else 414 panic("acquire_spinlock_nocheck(): Failed to acquire " 415 "spinlock %p for a long time (value: %" B_PRIx32 ")", 416 lock, lock->lock); 417#endif 418 count = 0; 419 } 420 421 cpu_wait(&lock->lock, 0); 422 } 423 424 if (atomic_get_and_set(&lock->lock, 1) == 0) 425 break; 426 } 427 428#if B_DEBUG_SPINLOCK_CONTENTION 429 update_lock_contention(lock, start); 430#endif 431 432#if DEBUG_SPINLOCKS 433 push_lock_caller(arch_debug_get_caller(), lock); 434#endif 435 } else { 436#if B_DEBUG_SPINLOCK_CONTENTION 437 lock->last_acquired = system_time(); 438#endif 439#if DEBUG_SPINLOCKS 440 int32 oldValue = atomic_get_and_set(&lock->lock, 1); 441 if (oldValue != 0) { 442 panic("acquire_spinlock_nocheck: attempt to acquire lock %p twice " 443 "on non-SMP system (last caller: %p, value %" B_PRIx32 ")", 444 lock, find_lock_caller(lock), oldValue); 445 } 446 447 push_lock_caller(arch_debug_get_caller(), lock); 448#endif 449 } 450} 451 452 453/*! Equivalent to acquire_spinlock(), save for currentCPU parameter. */ 454static void 455acquire_spinlock_cpu(int32 currentCPU, spinlock *lock) 456{ 457#if DEBUG_SPINLOCKS 458 if (are_interrupts_enabled()) { 459 panic("acquire_spinlock_cpu: attempt to acquire lock %p with " 460 "interrupts enabled", lock); 461 } 462#endif 463 464 if (sNumCPUs > 1) { 465#if B_DEBUG_SPINLOCK_CONTENTION 466 const bigtime_t start = system_time(); 467#endif 468 while (1) { 469 uint32 count = 0; 470 while (lock->lock != 0) { 471 if (++count == SPINLOCK_DEADLOCK_COUNT) { 472#if DEBUG_SPINLOCKS 473 panic("acquire_spinlock_cpu(): Failed to acquire spinlock " 474 "%p for a long time (last caller: %p, value: %" B_PRIx32 475 ")", lock, find_lock_caller(lock), lock->lock); 476#else 477 panic("acquire_spinlock_cpu(): Failed to acquire spinlock " 478 "%p for a long time (value: %" B_PRIx32 ")", lock, 479 lock->lock); 480#endif 481 count = 0; 482 } 483 484 process_all_pending_ici(currentCPU); 485 cpu_wait(&lock->lock, 0); 486 } 487 if (atomic_get_and_set(&lock->lock, 1) == 0) 488 break; 489 } 490 491#if B_DEBUG_SPINLOCK_CONTENTION 492 update_lock_contention(lock, start); 493#endif 494 495#if DEBUG_SPINLOCKS 496 push_lock_caller(arch_debug_get_caller(), lock); 497#endif 498 } else { 499#if B_DEBUG_SPINLOCK_CONTENTION 500 lock->last_acquired = system_time(); 501#endif 502#if DEBUG_SPINLOCKS 503 int32 oldValue = atomic_get_and_set(&lock->lock, 1); 504 if (oldValue != 0) { 505 panic("acquire_spinlock_cpu(): attempt to acquire lock %p twice on " 506 "non-SMP system (last caller: %p, value %" B_PRIx32 ")", lock, 507 find_lock_caller(lock), oldValue); 508 } 509 510 push_lock_caller(arch_debug_get_caller(), lock); 511#endif 512 } 513} 514 515 516void 517release_spinlock(spinlock *lock) 518{ 519#if B_DEBUG_SPINLOCK_CONTENTION 520 update_lock_held(lock); 521#endif 522 523 if (sNumCPUs > 1) { 524 if (are_interrupts_enabled()) { 525 panic("release_spinlock: attempt to release lock %p with " 526 "interrupts enabled\n", lock); 527 } 528 529#if DEBUG_SPINLOCKS 530 if (atomic_get_and_set(&lock->lock, 0) != 1) 531 panic("release_spinlock: lock %p was already released\n", lock); 532#else 533 atomic_set(&lock->lock, 0); 534#endif 535 } else { 536#if DEBUG_SPINLOCKS 537 if (are_interrupts_enabled()) { 538 panic("release_spinlock: attempt to release lock %p with " 539 "interrupts enabled\n", lock); 540 } 541 if (atomic_get_and_set(&lock->lock, 0) != 1) 542 panic("release_spinlock: lock %p was already released\n", lock); 543#endif 544 } 545} 546 547 548bool 549try_acquire_write_spinlock(rw_spinlock* lock) 550{ 551#if DEBUG_SPINLOCKS 552 if (are_interrupts_enabled()) { 553 panic("try_acquire_write_spinlock: attempt to acquire lock %p with " 554 "interrupts enabled", lock); 555 } 556 557 if (sNumCPUs < 2 && lock->lock != 0) { 558 panic("try_acquire_write_spinlock(): attempt to acquire lock %p twice " 559 "on non-SMP system", lock); 560 } 561#endif 562 563 return atomic_test_and_set(&lock->lock, 1u << 31, 0) == 0; 564} 565 566 567void 568acquire_write_spinlock(rw_spinlock* lock) 569{ 570#if DEBUG_SPINLOCKS 571 if (are_interrupts_enabled()) { 572 panic("acquire_write_spinlock: attempt to acquire lock %p with " 573 "interrupts enabled", lock); 574 } 575#endif 576 577 uint32 count = 0; 578 int currentCPU = smp_get_current_cpu(); 579 while (true) { 580 if (try_acquire_write_spinlock(lock)) 581 break; 582 583 while (lock->lock != 0) { 584 if (++count == SPINLOCK_DEADLOCK_COUNT) { 585 panic("acquire_write_spinlock(): Failed to acquire spinlock %p " 586 "for a long time!", lock); 587 count = 0; 588 } 589 590 process_all_pending_ici(currentCPU); 591 cpu_wait(&lock->lock, 0); 592 } 593 } 594} 595 596 597void 598release_write_spinlock(rw_spinlock* lock) 599{ 600#if DEBUG_SPINLOCKS 601 uint32 previous = atomic_get_and_set(&lock->lock, 0); 602 if ((previous & 1u << 31) == 0) { 603 panic("release_write_spinlock: lock %p was already released (value: " 604 "%#" B_PRIx32 ")\n", lock, previous); 605 } 606#else 607 atomic_set(&lock->lock, 0); 608#endif 609} 610 611 612bool 613try_acquire_read_spinlock(rw_spinlock* lock) 614{ 615#if DEBUG_SPINLOCKS 616 if (are_interrupts_enabled()) { 617 panic("try_acquire_read_spinlock: attempt to acquire lock %p with " 618 "interrupts enabled", lock); 619 } 620 621 if (sNumCPUs < 2 && lock->lock != 0) { 622 panic("try_acquire_read_spinlock(): attempt to acquire lock %p twice " 623 "on non-SMP system", lock); 624 } 625#endif 626 627 uint32 previous = atomic_add(&lock->lock, 1); 628 return (previous & (1u << 31)) == 0; 629} 630 631 632void 633acquire_read_spinlock(rw_spinlock* lock) 634{ 635#if DEBUG_SPINLOCKS 636 if (are_interrupts_enabled()) { 637 panic("acquire_read_spinlock: attempt to acquire lock %p with " 638 "interrupts enabled", lock); 639 } 640#endif 641 642 uint32 count = 0; 643 int currentCPU = smp_get_current_cpu(); 644 while (1) { 645 if (try_acquire_read_spinlock(lock)) 646 break; 647 648 while ((lock->lock & (1u << 31)) != 0) { 649 if (++count == SPINLOCK_DEADLOCK_COUNT) { 650 panic("acquire_read_spinlock(): Failed to acquire spinlock %p " 651 "for a long time!", lock); 652 count = 0; 653 } 654 655 process_all_pending_ici(currentCPU); 656 cpu_wait(&lock->lock, 0); 657 } 658 } 659} 660 661 662void 663release_read_spinlock(rw_spinlock* lock) 664{ 665#if DEBUG_SPINLOCKS 666 uint32 previous = atomic_add(&lock->lock, -1); 667 if ((previous & 1u << 31) != 0) { 668 panic("release_read_spinlock: lock %p was already released (value:" 669 " %#" B_PRIx32 ")\n", lock, previous); 670 } 671#else 672 atomic_add(&lock->lock, -1); 673#endif 674 675} 676 677 678bool 679try_acquire_write_seqlock(seqlock* lock) 680{ 681 bool succeed = try_acquire_spinlock(&lock->lock); 682 if (succeed) 683 atomic_add((int32*)&lock->count, 1); 684 return succeed; 685} 686 687 688void 689acquire_write_seqlock(seqlock* lock) 690{ 691 acquire_spinlock(&lock->lock); 692 atomic_add((int32*)&lock->count, 1); 693} 694 695 696void 697release_write_seqlock(seqlock* lock) 698{ 699 atomic_add((int32*)&lock->count, 1); 700 release_spinlock(&lock->lock); 701} 702 703 704uint32 705acquire_read_seqlock(seqlock* lock) 706{ 707 return atomic_get((int32*)&lock->count); 708} 709 710 711bool 712release_read_seqlock(seqlock* lock, uint32 count) 713{ 714 memory_read_barrier(); 715 716 uint32 current = *(volatile int32*)&lock->count; 717 718 if (count % 2 == 1 || current != count) { 719 cpu_pause(); 720 return false; 721 } 722 723 return true; 724} 725 726 727/*! Finds a free message and gets it. 728 NOTE: has side effect of disabling interrupts 729 return value is the former interrupt state 730*/ 731static cpu_status 732find_free_message(struct smp_msg** msg) 733{ 734 cpu_status state; 735 736 TRACE("find_free_message: entry\n"); 737 738retry: 739 while (sFreeMessageCount <= 0) 740 cpu_pause(); 741 742 state = disable_interrupts(); 743 acquire_spinlock(&sFreeMessageSpinlock); 744 745 if (sFreeMessageCount <= 0) { 746 // someone grabbed one while we were getting the lock, 747 // go back to waiting for it 748 release_spinlock(&sFreeMessageSpinlock); 749 restore_interrupts(state); 750 goto retry; 751 } 752 753 *msg = sFreeMessages; 754 sFreeMessages = (*msg)->next; 755 sFreeMessageCount--; 756 757 release_spinlock(&sFreeMessageSpinlock); 758 759 TRACE("find_free_message: returning msg %p\n", *msg); 760 761 return state; 762} 763 764 765/*! Similar to find_free_message(), but expects the interrupts to be disabled 766 already. 767*/ 768static void 769find_free_message_interrupts_disabled(int32 currentCPU, 770 struct smp_msg** _message) 771{ 772 TRACE("find_free_message_interrupts_disabled: entry\n"); 773 774 acquire_spinlock_cpu(currentCPU, &sFreeMessageSpinlock); 775 while (sFreeMessageCount <= 0) { 776 release_spinlock(&sFreeMessageSpinlock); 777 process_all_pending_ici(currentCPU); 778 cpu_pause(); 779 acquire_spinlock_cpu(currentCPU, &sFreeMessageSpinlock); 780 } 781 782 *_message = sFreeMessages; 783 sFreeMessages = (*_message)->next; 784 sFreeMessageCount--; 785 786 release_spinlock(&sFreeMessageSpinlock); 787 788 TRACE("find_free_message_interrupts_disabled: returning msg %p\n", 789 *_message); 790} 791 792 793static void 794return_free_message(struct smp_msg* msg) 795{ 796 TRACE("return_free_message: returning msg %p\n", msg); 797 798 acquire_spinlock_nocheck(&sFreeMessageSpinlock); 799 msg->next = sFreeMessages; 800 sFreeMessages = msg; 801 sFreeMessageCount++; 802 release_spinlock(&sFreeMessageSpinlock); 803} 804 805 806static struct smp_msg* 807check_for_message(int currentCPU, mailbox_source& sourceMailbox) 808{ 809 if (!sICIEnabled) 810 return NULL; 811 812 struct smp_msg* msg = atomic_pointer_get(&sCPUMessages[currentCPU]); 813 if (msg != NULL) { 814 do { 815 cpu_pause(); 816 msg = atomic_pointer_get(&sCPUMessages[currentCPU]); 817 ASSERT(msg != NULL); 818 } while (atomic_pointer_test_and_set(&sCPUMessages[currentCPU], 819 msg->next, msg) != msg); 820 821 TRACE(" cpu %d: found msg %p in cpu mailbox\n", currentCPU, msg); 822 sourceMailbox = MAILBOX_LOCAL; 823 } else if (atomic_get(&get_cpu_struct()->ici_counter) 824 != atomic_get(&sBroadcastMessageCounter)) { 825 826 // try getting one from the broadcast mailbox 827 acquire_spinlock_nocheck(&sBroadcastMessageSpinlock); 828 829 msg = sBroadcastMessages; 830 while (msg != NULL) { 831 if (!msg->proc_bitmap.GetBit(currentCPU)) { 832 // we have handled this one already 833 msg = msg->next; 834 continue; 835 } 836 837 // mark it so we wont try to process this one again 838 msg->proc_bitmap.ClearBitAtomic(currentCPU); 839 atomic_add(&gCPU[currentCPU].ici_counter, 1); 840 841 sourceMailbox = MAILBOX_BCAST; 842 break; 843 } 844 release_spinlock(&sBroadcastMessageSpinlock); 845 846 if (msg != NULL) { 847 TRACE(" cpu %d: found msg %p in broadcast mailbox\n", currentCPU, 848 msg); 849 } 850 } 851 return msg; 852} 853 854 855static void 856finish_message_processing(int currentCPU, struct smp_msg* msg, 857 mailbox_source sourceMailbox) 858{ 859 if (atomic_add(&msg->ref_count, -1) != 1) 860 return; 861 862 // we were the last one to decrement the ref_count 863 // it's our job to remove it from the list & possibly clean it up 864 865 // clean up the message 866 if (sourceMailbox == MAILBOX_BCAST) 867 acquire_spinlock_nocheck(&sBroadcastMessageSpinlock); 868 869 TRACE("cleaning up message %p\n", msg); 870 871 if (sourceMailbox != MAILBOX_BCAST) { 872 // local mailbox -- the message has already been removed in 873 // check_for_message() 874 } else if (msg == sBroadcastMessages) { 875 sBroadcastMessages = msg->next; 876 } else { 877 // we need to walk to find the message in the list. 878 // we can't use any data found when previously walking through 879 // the list, since the list may have changed. But, we are guaranteed 880 // to at least have msg in it. 881 struct smp_msg* last = NULL; 882 struct smp_msg* msg1; 883 884 msg1 = sBroadcastMessages; 885 while (msg1 != NULL && msg1 != msg) { 886 last = msg1; 887 msg1 = msg1->next; 888 } 889 890 // by definition, last must be something 891 if (msg1 == msg && last != NULL) 892 last->next = msg->next; 893 else 894 panic("last == NULL or msg != msg1"); 895 } 896 897 if (sourceMailbox == MAILBOX_BCAST) 898 release_spinlock(&sBroadcastMessageSpinlock); 899 900 if ((msg->flags & SMP_MSG_FLAG_FREE_ARG) != 0 && msg->data_ptr != NULL) 901 free(msg->data_ptr); 902 903 if ((msg->flags & SMP_MSG_FLAG_SYNC) != 0) { 904 atomic_set(&msg->done, 1); 905 // the caller cpu should now free the message 906 } else { 907 // in the !SYNC case, we get to free the message 908 return_free_message(msg); 909 } 910} 911 912 913static status_t 914process_pending_ici(int32 currentCPU) 915{ 916 mailbox_source sourceMailbox; 917 struct smp_msg* msg = check_for_message(currentCPU, sourceMailbox); 918 if (msg == NULL) 919 return B_ENTRY_NOT_FOUND; 920 921 TRACE(" cpu %ld message = %ld\n", currentCPU, msg->message); 922 923 bool haltCPU = false; 924 925 switch (msg->message) { 926 case SMP_MSG_INVALIDATE_PAGE_RANGE: 927 arch_cpu_invalidate_TLB_range(msg->data, msg->data2); 928 break; 929 case SMP_MSG_INVALIDATE_PAGE_LIST: 930 arch_cpu_invalidate_TLB_list((addr_t*)msg->data, (int)msg->data2); 931 break; 932 case SMP_MSG_USER_INVALIDATE_PAGES: 933 arch_cpu_user_TLB_invalidate(); 934 break; 935 case SMP_MSG_GLOBAL_INVALIDATE_PAGES: 936 arch_cpu_global_TLB_invalidate(); 937 break; 938 case SMP_MSG_CPU_HALT: 939 haltCPU = true; 940 break; 941 case SMP_MSG_CALL_FUNCTION: 942 { 943 smp_call_func func = (smp_call_func)msg->data_ptr; 944 func(msg->data, currentCPU, msg->data2, msg->data3); 945 break; 946 } 947 case SMP_MSG_RESCHEDULE: 948 scheduler_reschedule_ici(); 949 break; 950 951 default: 952 dprintf("smp_intercpu_int_handler: got unknown message %" B_PRId32 "\n", 953 msg->message); 954 break; 955 } 956 957 // finish dealing with this message, possibly removing it from the list 958 finish_message_processing(currentCPU, msg, sourceMailbox); 959 960 // special case for the halt message 961 if (haltCPU) 962 debug_trap_cpu_in_kdl(currentCPU, false); 963 964 return B_OK; 965} 966 967 968#if B_DEBUG_SPINLOCK_CONTENTION 969 970 971static status_t 972spinlock_contention_syscall(const char* subsystem, uint32 function, 973 void* buffer, size_t bufferSize) 974{ 975 if (function != GET_SPINLOCK_CONTENTION_INFO) 976 return B_BAD_VALUE; 977 978 if (bufferSize < sizeof(spinlock_contention_info)) 979 return B_BAD_VALUE; 980 981 // TODO: This isn't very useful at the moment... 982 983 spinlock_contention_info info; 984 info.thread_creation_spinlock = gThreadCreationLock.total_wait; 985 986 if (!IS_USER_ADDRESS(buffer) 987 || user_memcpy(buffer, &info, sizeof(info)) != B_OK) { 988 return B_BAD_ADDRESS; 989 } 990 991 return B_OK; 992} 993 994 995#endif // B_DEBUG_SPINLOCK_CONTENTION 996 997 998static void 999process_early_cpu_call(int32 cpu) 1000{ 1001 sEarlyCPUCallFunction(sEarlyCPUCallCookie, cpu); 1002 sEarlyCPUCallSet.ClearBitAtomic(cpu); 1003 atomic_add(&sEarlyCPUCallCount, 1); 1004} 1005 1006 1007static void 1008call_all_cpus_early(void (*function)(void*, int), void* cookie) 1009{ 1010 if (sNumCPUs > 1) { 1011 sEarlyCPUCallFunction = function; 1012 sEarlyCPUCallCookie = cookie; 1013 1014 atomic_set(&sEarlyCPUCallCount, 1); 1015 sEarlyCPUCallSet.SetAll(); 1016 sEarlyCPUCallSet.ClearBit(0); 1017 1018 // wait for all CPUs to finish 1019 while (sEarlyCPUCallCount < sNumCPUs) 1020 cpu_wait(&sEarlyCPUCallCount, sNumCPUs); 1021 } 1022 1023 function(cookie, 0); 1024} 1025 1026 1027// #pragma mark - 1028 1029 1030int 1031smp_intercpu_int_handler(int32 cpu) 1032{ 1033 TRACE("smp_intercpu_int_handler: entry on cpu %ld\n", cpu); 1034 1035 process_all_pending_ici(cpu); 1036 1037 TRACE("smp_intercpu_int_handler: done on cpu %ld\n", cpu); 1038 1039 return B_HANDLED_INTERRUPT; 1040} 1041 1042 1043void 1044smp_send_ici(int32 targetCPU, int32 message, addr_t data, addr_t data2, 1045 addr_t data3, void* dataPointer, uint32 flags) 1046{ 1047 struct smp_msg *msg; 1048 1049 TRACE("smp_send_ici: target 0x%lx, mess 0x%lx, data 0x%lx, data2 0x%lx, " 1050 "data3 0x%lx, ptr %p, flags 0x%lx\n", targetCPU, message, data, data2, 1051 data3, dataPointer, flags); 1052 1053 if (sICIEnabled) { 1054 int state; 1055 int currentCPU; 1056 1057 // find_free_message leaves interrupts disabled 1058 state = find_free_message(&msg); 1059 1060 currentCPU = smp_get_current_cpu(); 1061 if (targetCPU == currentCPU) { 1062 return_free_message(msg); 1063 restore_interrupts(state); 1064 return; // nope, cant do that 1065 } 1066 1067 // set up the message 1068 msg->message = message; 1069 msg->data = data; 1070 msg->data2 = data2; 1071 msg->data3 = data3; 1072 msg->data_ptr = dataPointer; 1073 msg->ref_count = 1; 1074 msg->flags = flags; 1075 msg->done = 0; 1076 1077 // stick it in the appropriate cpu's mailbox 1078 struct smp_msg* next; 1079 do { 1080 cpu_pause(); 1081 next = atomic_pointer_get(&sCPUMessages[targetCPU]); 1082 msg->next = next; 1083 } while (atomic_pointer_test_and_set(&sCPUMessages[targetCPU], msg, 1084 next) != next); 1085 1086 arch_smp_send_ici(targetCPU); 1087 1088 if ((flags & SMP_MSG_FLAG_SYNC) != 0) { 1089 // wait for the other cpu to finish processing it 1090 // the interrupt handler will ref count it to <0 1091 // if the message is sync after it has removed it from the mailbox 1092 while (msg->done == 0) { 1093 process_all_pending_ici(currentCPU); 1094 cpu_wait(&msg->done, 1); 1095 } 1096 // for SYNC messages, it's our responsibility to put it 1097 // back into the free list 1098 return_free_message(msg); 1099 } 1100 1101 restore_interrupts(state); 1102 } 1103} 1104 1105 1106void 1107smp_send_multicast_ici(CPUSet& cpuMask, int32 message, addr_t data, 1108 addr_t data2, addr_t data3, void *dataPointer, uint32 flags) 1109{ 1110 if (!sICIEnabled) 1111 return; 1112 1113 int currentCPU = smp_get_current_cpu(); 1114 1115 // find_free_message leaves interrupts disabled 1116 struct smp_msg *msg; 1117 int state = find_free_message(&msg); 1118 1119 msg->proc_bitmap = cpuMask; 1120 msg->proc_bitmap.ClearBit(currentCPU); 1121 1122 int32 targetCPUs = 0; 1123 for (int32 i = 0; i < sNumCPUs; i++) { 1124 if (msg->proc_bitmap.GetBit(i)) 1125 targetCPUs++; 1126 } 1127 1128 if (targetCPUs == 0) { 1129 panic("smp_send_multicast_ici(): 0 CPU mask"); 1130 return; 1131 } 1132 1133 msg->message = message; 1134 msg->data = data; 1135 msg->data2 = data2; 1136 msg->data3 = data3; 1137 msg->data_ptr = dataPointer; 1138 msg->ref_count = targetCPUs; 1139 msg->flags = flags; 1140 msg->done = 0; 1141 1142 bool broadcast = targetCPUs == sNumCPUs - 1; 1143 1144 // stick it in the broadcast mailbox 1145 acquire_spinlock_nocheck(&sBroadcastMessageSpinlock); 1146 msg->next = sBroadcastMessages; 1147 sBroadcastMessages = msg; 1148 release_spinlock(&sBroadcastMessageSpinlock); 1149 1150 atomic_add(&sBroadcastMessageCounter, 1); 1151 for (int32 i = 0; i < sNumCPUs; i++) { 1152 if (!cpuMask.GetBit(i)) 1153 atomic_add(&gCPU[i].ici_counter, 1); 1154 } 1155 1156 if (broadcast) 1157 arch_smp_send_broadcast_ici(); 1158 else 1159 arch_smp_send_multicast_ici(cpuMask); 1160 1161 if ((flags & SMP_MSG_FLAG_SYNC) != 0) { 1162 // wait for the other cpus to finish processing it 1163 // the interrupt handler will ref count it to <0 1164 // if the message is sync after it has removed it from the mailbox 1165 while (msg->done == 0) { 1166 process_all_pending_ici(currentCPU); 1167 cpu_wait(&msg->done, 1); 1168 } 1169 1170 // for SYNC messages, it's our responsibility to put it 1171 // back into the free list 1172 return_free_message(msg); 1173 } 1174 1175 restore_interrupts(state); 1176} 1177 1178 1179void 1180smp_send_broadcast_ici(int32 message, addr_t data, addr_t data2, addr_t data3, 1181 void *dataPointer, uint32 flags) 1182{ 1183 struct smp_msg *msg; 1184 1185 TRACE("smp_send_broadcast_ici: cpu %ld mess 0x%lx, data 0x%lx, data2 " 1186 "0x%lx, data3 0x%lx, ptr %p, flags 0x%lx\n", smp_get_current_cpu(), 1187 message, data, data2, data3, dataPointer, flags); 1188 1189 if (sICIEnabled) { 1190 int state; 1191 int currentCPU; 1192 1193 // find_free_message leaves interrupts disabled 1194 state = find_free_message(&msg); 1195 1196 currentCPU = smp_get_current_cpu(); 1197 1198 msg->message = message; 1199 msg->data = data; 1200 msg->data2 = data2; 1201 msg->data3 = data3; 1202 msg->data_ptr = dataPointer; 1203 msg->ref_count = sNumCPUs - 1; 1204 msg->flags = flags; 1205 msg->proc_bitmap.SetAll(); 1206 msg->proc_bitmap.ClearBit(currentCPU); 1207 msg->done = 0; 1208 1209 TRACE("smp_send_broadcast_ici%d: inserting msg %p into broadcast " 1210 "mbox\n", currentCPU, msg); 1211 1212 // stick it in the appropriate cpu's mailbox 1213 acquire_spinlock_nocheck(&sBroadcastMessageSpinlock); 1214 msg->next = sBroadcastMessages; 1215 sBroadcastMessages = msg; 1216 release_spinlock(&sBroadcastMessageSpinlock); 1217 1218 atomic_add(&sBroadcastMessageCounter, 1); 1219 atomic_add(&gCPU[currentCPU].ici_counter, 1); 1220 1221 arch_smp_send_broadcast_ici(); 1222 1223 TRACE("smp_send_broadcast_ici: sent interrupt\n"); 1224 1225 if ((flags & SMP_MSG_FLAG_SYNC) != 0) { 1226 // wait for the other cpus to finish processing it 1227 // the interrupt handler will ref count it to <0 1228 // if the message is sync after it has removed it from the mailbox 1229 TRACE("smp_send_broadcast_ici: waiting for ack\n"); 1230 1231 while (msg->done == 0) { 1232 process_all_pending_ici(currentCPU); 1233 cpu_wait(&msg->done, 1); 1234 } 1235 1236 TRACE("smp_send_broadcast_ici: returning message to free list\n"); 1237 1238 // for SYNC messages, it's our responsibility to put it 1239 // back into the free list 1240 return_free_message(msg); 1241 } 1242 1243 restore_interrupts(state); 1244 } 1245 1246 TRACE("smp_send_broadcast_ici: done\n"); 1247} 1248 1249 1250void 1251smp_send_broadcast_ici_interrupts_disabled(int32 currentCPU, int32 message, 1252 addr_t data, addr_t data2, addr_t data3, void *dataPointer, uint32 flags) 1253{ 1254 if (!sICIEnabled) 1255 return; 1256 1257 TRACE("smp_send_broadcast_ici_interrupts_disabled: cpu %ld mess 0x%lx, " 1258 "data 0x%lx, data2 0x%lx, data3 0x%lx, ptr %p, flags 0x%lx\n", 1259 currentCPU, message, data, data2, data3, dataPointer, flags); 1260 1261 struct smp_msg *msg; 1262 find_free_message_interrupts_disabled(currentCPU, &msg); 1263 1264 msg->message = message; 1265 msg->data = data; 1266 msg->data2 = data2; 1267 msg->data3 = data3; 1268 msg->data_ptr = dataPointer; 1269 msg->ref_count = sNumCPUs - 1; 1270 msg->flags = flags; 1271 msg->proc_bitmap.SetAll(); 1272 msg->proc_bitmap.ClearBit(currentCPU); 1273 msg->done = 0; 1274 1275 TRACE("smp_send_broadcast_ici_interrupts_disabled %ld: inserting msg %p " 1276 "into broadcast mbox\n", currentCPU, msg); 1277 1278 // stick it in the appropriate cpu's mailbox 1279 acquire_spinlock_nocheck(&sBroadcastMessageSpinlock); 1280 msg->next = sBroadcastMessages; 1281 sBroadcastMessages = msg; 1282 release_spinlock(&sBroadcastMessageSpinlock); 1283 1284 atomic_add(&sBroadcastMessageCounter, 1); 1285 atomic_add(&gCPU[currentCPU].ici_counter, 1); 1286 1287 arch_smp_send_broadcast_ici(); 1288 1289 TRACE("smp_send_broadcast_ici_interrupts_disabled %ld: sent interrupt\n", 1290 currentCPU); 1291 1292 if ((flags & SMP_MSG_FLAG_SYNC) != 0) { 1293 // wait for the other cpus to finish processing it 1294 // the interrupt handler will ref count it to <0 1295 // if the message is sync after it has removed it from the mailbox 1296 TRACE("smp_send_broadcast_ici_interrupts_disabled %ld: waiting for " 1297 "ack\n", currentCPU); 1298 1299 while (msg->done == 0) { 1300 process_all_pending_ici(currentCPU); 1301 cpu_wait(&msg->done, 1); 1302 } 1303 1304 TRACE("smp_send_broadcast_ici_interrupts_disabled %ld: returning " 1305 "message to free list\n", currentCPU); 1306 1307 // for SYNC messages, it's our responsibility to put it 1308 // back into the free list 1309 return_free_message(msg); 1310 } 1311 1312 TRACE("smp_send_broadcast_ici_interrupts_disabled: done\n"); 1313} 1314 1315 1316/*! Spin on non-boot CPUs until smp_wake_up_non_boot_cpus() has been called. 1317 1318 \param cpu The index of the calling CPU. 1319 \param rendezVous A rendez-vous variable to make sure that the boot CPU 1320 does not return before all other CPUs have started waiting. 1321 \return \c true on the boot CPU, \c false otherwise. 1322*/ 1323bool 1324smp_trap_non_boot_cpus(int32 cpu, uint32* rendezVous) 1325{ 1326 if (cpu == 0) { 1327 smp_cpu_rendezvous(rendezVous); 1328 return true; 1329 } 1330 1331 smp_cpu_rendezvous(rendezVous); 1332 1333 while (sBootCPUSpin == 0) { 1334 if (sEarlyCPUCallSet.GetBit(cpu)) 1335 process_early_cpu_call(cpu); 1336 1337 cpu_pause(); 1338 } 1339 1340 return false; 1341} 1342 1343 1344void 1345smp_wake_up_non_boot_cpus() 1346{ 1347 // ICIs were previously being ignored 1348 if (sNumCPUs > 1) 1349 sICIEnabled = true; 1350 1351 // resume non boot CPUs 1352 atomic_set(&sBootCPUSpin, 1); 1353} 1354 1355 1356/*! Spin until all CPUs have reached the rendez-vous point. 1357 1358 The rendez-vous variable \c *var must have been initialized to 0 before the 1359 function is called. The variable will be non-null when the function returns. 1360 1361 Note that when the function returns on one CPU, it only means that all CPU 1362 have already entered the function. It does not mean that the variable can 1363 already be reset. Only when all CPUs have returned (which would have to be 1364 ensured via another rendez-vous) the variable can be reset. 1365*/ 1366void 1367smp_cpu_rendezvous(uint32* var) 1368{ 1369 atomic_add((int32*)var, 1); 1370 1371 while (*var < (uint32)sNumCPUs) 1372 cpu_wait((int32*)var, sNumCPUs); 1373} 1374 1375 1376status_t 1377smp_init(kernel_args* args) 1378{ 1379 TRACE("smp_init: entry\n"); 1380 1381#if DEBUG_SPINLOCKS 1382 add_debugger_command_etc("spinlock", &dump_spinlock, 1383 "Dump info on a spinlock", 1384 "\n" 1385 "Dumps info on a spinlock.\n", 0); 1386#endif 1387 add_debugger_command_etc("ici", &dump_ici_messages, 1388 "Dump info on pending ICI messages", 1389 "\n" 1390 "Dumps info on pending ICI messages.\n", 0); 1391 add_debugger_command_etc("ici_message", &dump_ici_message, 1392 "Dump info on an ICI message", 1393 "\n" 1394 "Dumps info on an ICI message.\n", 0); 1395 1396 if (args->num_cpus > 1) { 1397 sFreeMessages = NULL; 1398 sFreeMessageCount = 0; 1399 for (int i = 0; i < MSG_POOL_SIZE; i++) { 1400 struct smp_msg* msg 1401 = (struct smp_msg*)malloc(sizeof(struct smp_msg)); 1402 if (msg == NULL) { 1403 panic("error creating smp mailboxes\n"); 1404 return B_ERROR; 1405 } 1406 memset(msg, 0, sizeof(struct smp_msg)); 1407 msg->next = sFreeMessages; 1408 sFreeMessages = msg; 1409 sFreeMessageCount++; 1410 } 1411 sNumCPUs = args->num_cpus; 1412 } 1413 TRACE("smp_init: calling arch_smp_init\n"); 1414 1415 return arch_smp_init(args); 1416} 1417 1418 1419status_t 1420smp_per_cpu_init(kernel_args* args, int32 cpu) 1421{ 1422 return arch_smp_per_cpu_init(args, cpu); 1423} 1424 1425 1426status_t 1427smp_init_post_generic_syscalls(void) 1428{ 1429#if B_DEBUG_SPINLOCK_CONTENTION 1430 return register_generic_syscall(SPINLOCK_CONTENTION, 1431 &spinlock_contention_syscall, 0, 0); 1432#else 1433 return B_OK; 1434#endif 1435} 1436 1437 1438void 1439smp_set_num_cpus(int32 numCPUs) 1440{ 1441 sNumCPUs = numCPUs; 1442} 1443 1444 1445int32 1446smp_get_num_cpus() 1447{ 1448 return sNumCPUs; 1449} 1450 1451 1452int32 1453smp_get_current_cpu(void) 1454{ 1455 return thread_get_current_thread()->cpu->cpu_num; 1456} 1457 1458 1459static void 1460_call_single_cpu(uint32 targetCPU, void (*func)(void*, int), void* cookie, bool sync) 1461{ 1462 cpu_status state = disable_interrupts(); 1463 1464 if (targetCPU == (uint32)smp_get_current_cpu()) { 1465 func(cookie, smp_get_current_cpu()); 1466 restore_interrupts(state); 1467 return; 1468 } 1469 1470 if (!sICIEnabled) { 1471 // Early mechanism not available 1472 panic("call_single_cpu is not yet available"); 1473 restore_interrupts(state); 1474 return; 1475 } 1476 1477 smp_send_ici(targetCPU, SMP_MSG_CALL_FUNCTION, (addr_t)cookie, 1478 0, 0, (void*)func, sync ? SMP_MSG_FLAG_SYNC : SMP_MSG_FLAG_ASYNC); 1479 1480 restore_interrupts(state); 1481} 1482 1483 1484void 1485call_single_cpu(uint32 targetCPU, void (*func)(void*, int), void* cookie) 1486{ 1487 return _call_single_cpu(targetCPU, func, cookie, false); 1488} 1489 1490 1491void 1492call_single_cpu_sync(uint32 targetCPU, void (*func)(void*, int), void* cookie) 1493{ 1494 return _call_single_cpu(targetCPU, func, cookie, true); 1495} 1496 1497 1498// #pragma mark - public exported functions 1499 1500 1501void 1502call_all_cpus(void (*func)(void*, int), void* cookie) 1503{ 1504 cpu_status state = disable_interrupts(); 1505 1506 // if inter-CPU communication is not yet enabled, use the early mechanism 1507 if (!sICIEnabled) { 1508 call_all_cpus_early(func, cookie); 1509 restore_interrupts(state); 1510 return; 1511 } 1512 1513 if (smp_get_num_cpus() > 1) { 1514 smp_send_broadcast_ici(SMP_MSG_CALL_FUNCTION, (addr_t)cookie, 1515 0, 0, (void*)func, SMP_MSG_FLAG_ASYNC); 1516 } 1517 1518 // we need to call this function ourselves as well 1519 func(cookie, smp_get_current_cpu()); 1520 1521 restore_interrupts(state); 1522} 1523 1524 1525void 1526call_all_cpus_sync(void (*func)(void*, int), void* cookie) 1527{ 1528 cpu_status state = disable_interrupts(); 1529 1530 // if inter-CPU communication is not yet enabled, use the early mechanism 1531 if (!sICIEnabled) { 1532 call_all_cpus_early(func, cookie); 1533 restore_interrupts(state); 1534 return; 1535 } 1536 1537 if (smp_get_num_cpus() > 1) { 1538 smp_send_broadcast_ici(SMP_MSG_CALL_FUNCTION, (addr_t)cookie, 1539 0, 0, (void*)func, SMP_MSG_FLAG_SYNC); 1540 } 1541 1542 // we need to call this function ourselves as well 1543 func(cookie, smp_get_current_cpu()); 1544 1545 restore_interrupts(state); 1546} 1547 1548 1549// Ensure the symbols for memory_barriers are still included 1550// in the kernel for binary compatibility. Calls are forwarded 1551// to the more efficent per-processor atomic implementations. 1552#undef memory_read_barrier 1553#undef memory_write_barrier 1554 1555 1556void 1557memory_read_barrier() 1558{ 1559 memory_read_barrier_inline(); 1560} 1561 1562 1563void 1564memory_write_barrier() 1565{ 1566 memory_write_barrier_inline(); 1567} 1568 1569