1/* 2 * BK Id: SCCS/s.smp.c 1.40 03/28/02 16:54:23 hozer 3 */ 4/* 5 * Smp support for ppc. 6 * 7 * Written by Cort Dougan (cort@cs.nmt.edu) borrowing a great 8 * deal of code from the sparc and intel versions. 9 * 10 * Copyright (C) 1999 Cort Dougan <cort@cs.nmt.edu> 11 * 12 */ 13 14#include <linux/config.h> 15#include <linux/kernel.h> 16#include <linux/sched.h> 17#include <linux/smp.h> 18#include <linux/smp_lock.h> 19#include <linux/interrupt.h> 20#include <linux/kernel_stat.h> 21#include <linux/delay.h> 22#define __KERNEL_SYSCALLS__ 23#include <linux/unistd.h> 24#include <linux/init.h> 25#include <linux/spinlock.h> 26#include <linux/cache.h> 27 28#include <asm/ptrace.h> 29#include <asm/atomic.h> 30#include <asm/irq.h> 31#include <asm/page.h> 32#include <asm/pgtable.h> 33#include <asm/hardirq.h> 34#include <asm/softirq.h> 35#include <asm/io.h> 36#include <asm/prom.h> 37#include <asm/smp.h> 38#include <asm/residual.h> 39#include <asm/time.h> 40 41int smp_threads_ready; 42volatile int smp_commenced; 43int smp_num_cpus = 1; 44int smp_tb_synchronized; 45struct cpuinfo_PPC cpu_data[NR_CPUS]; 46struct klock_info_struct klock_info = { KLOCK_CLEAR, 0 }; 47atomic_t ipi_recv; 48atomic_t ipi_sent; 49spinlock_t kernel_flag __cacheline_aligned_in_smp = SPIN_LOCK_UNLOCKED; 50unsigned int prof_multiplier[NR_CPUS]; 51unsigned int prof_counter[NR_CPUS]; 52cycles_t cacheflush_time; 53static int max_cpus __initdata = NR_CPUS; 54unsigned long cpu_online_map; 55int smp_hw_index[NR_CPUS]; 56static struct smp_ops_t *smp_ops; 57 58/* all cpu mappings are 1-1 -- Cort */ 59volatile unsigned long cpu_callin_map[NR_CPUS]; 60 61#define TB_SYNC_PASSES 4 62volatile unsigned long __initdata tb_sync_flag = 0; 63volatile unsigned long __initdata tb_offset = 0; 64 65int start_secondary(void *); 66extern int cpu_idle(void *unused); 67void smp_call_function_interrupt(void); 68 69/* Since OpenPIC has only 4 IPIs, we use slightly different message numbers. 70 * 71 * Make sure this matches openpic_request_IPIs in open_pic.c, or what shows up 72 * in /proc/interrupts will be wrong!!! --Troy */ 73#define PPC_MSG_CALL_FUNCTION 0 74#define PPC_MSG_RESCHEDULE 1 75#define PPC_MSG_INVALIDATE_TLB 2 76#define PPC_MSG_XMON_BREAK 3 77 78static inline void 79smp_message_pass(int target, int msg, unsigned long data, int wait) 80{ 81 if (smp_ops){ 82 atomic_inc(&ipi_sent); 83 smp_ops->message_pass(target,msg,data,wait); 84 } 85} 86 87/* 88 * Common functions 89 */ 90void smp_local_timer_interrupt(struct pt_regs * regs) 91{ 92 int cpu = smp_processor_id(); 93 94 if (!--prof_counter[cpu]) { 95 update_process_times(user_mode(regs)); 96 prof_counter[cpu]=prof_multiplier[cpu]; 97 } 98} 99 100void smp_message_recv(int msg, struct pt_regs *regs) 101{ 102 atomic_inc(&ipi_recv); 103 104 switch( msg ) { 105 case PPC_MSG_CALL_FUNCTION: 106 smp_call_function_interrupt(); 107 break; 108 case PPC_MSG_RESCHEDULE: 109 current->need_resched = 1; 110 break; 111 case PPC_MSG_INVALIDATE_TLB: 112 _tlbia(); 113 break; 114#ifdef CONFIG_XMON 115 case PPC_MSG_XMON_BREAK: 116 xmon(regs); 117 break; 118#endif /* CONFIG_XMON */ 119 default: 120 printk("SMP %d: smp_message_recv(): unknown msg %d\n", 121 smp_processor_id(), msg); 122 break; 123 } 124} 125 126#ifdef CONFIG_750_SMP 127/* 128 * 750's don't broadcast tlb invalidates so 129 * we have to emulate that behavior. 130 * -- Cort 131 */ 132void smp_ppc750_send_tlb_invalidate(int cpu) 133{ 134 if ( PVR_VER(mfspr(PVR)) == 8 ) 135 smp_message_pass(MSG_ALL_BUT_SELF, PPC_MSG_INVALIDATE_TLB, 0, 0); 136} 137#endif 138 139void smp_send_reschedule(int cpu) 140{ 141 /* 142 * This is only used if `cpu' is running an idle task, 143 * so it will reschedule itself anyway... 144 * 145 * This isn't the case anymore since the other CPU could be 146 * sleeping and won't reschedule until the next interrupt (such 147 * as the timer). 148 * -- Cort 149 */ 150 /* This is only used if `cpu' is running an idle task, 151 so it will reschedule itself anyway... */ 152 smp_message_pass(cpu, PPC_MSG_RESCHEDULE, 0, 0); 153} 154 155#ifdef CONFIG_XMON 156void smp_send_xmon_break(int cpu) 157{ 158 smp_message_pass(cpu, PPC_MSG_XMON_BREAK, 0, 0); 159} 160#endif /* CONFIG_XMON */ 161 162static void stop_this_cpu(void *dummy) 163{ 164 __cli(); 165 while (1) 166 ; 167} 168 169void smp_send_stop(void) 170{ 171 smp_call_function(stop_this_cpu, NULL, 1, 0); 172 smp_num_cpus = 1; 173} 174 175/* 176 * Structure and data for smp_call_function(). This is designed to minimise 177 * static memory requirements. It also looks cleaner. 178 * Stolen from the i386 version. 179 */ 180static spinlock_t call_lock = SPIN_LOCK_UNLOCKED; 181 182static struct call_data_struct { 183 void (*func) (void *info); 184 void *info; 185 atomic_t started; 186 atomic_t finished; 187 int wait; 188} *call_data; 189 190/* 191 * this function sends a 'generic call function' IPI to all other CPUs 192 * in the system. 193 */ 194 195int smp_call_function (void (*func) (void *info), void *info, int nonatomic, 196 int wait) 197/* 198 * [SUMMARY] Run a function on all other CPUs. 199 * <func> The function to run. This must be fast and non-blocking. 200 * <info> An arbitrary pointer to pass to the function. 201 * <nonatomic> currently unused. 202 * <wait> If true, wait (atomically) until function has completed on other CPUs. 203 * [RETURNS] 0 on success, else a negative status code. Does not return until 204 * remote CPUs are nearly ready to execute <<func>> or are or have executed. 205 * 206 * You must not call this function with disabled interrupts or from a 207 * hardware interrupt handler, you may call it from a bottom half handler. 208 */ 209{ 210 struct call_data_struct data; 211 int ret = -1, cpus = smp_num_cpus-1; 212 int timeout; 213 214 if (!cpus) 215 return 0; 216 217 data.func = func; 218 data.info = info; 219 atomic_set(&data.started, 0); 220 data.wait = wait; 221 if (wait) 222 atomic_set(&data.finished, 0); 223 224 spin_lock_bh(&call_lock); 225 call_data = &data; 226 /* Send a message to all other CPUs and wait for them to respond */ 227 smp_message_pass(MSG_ALL_BUT_SELF, PPC_MSG_CALL_FUNCTION, 0, 0); 228 229 /* Wait for response */ 230 timeout = 1000000; 231 while (atomic_read(&data.started) != cpus) { 232 if (--timeout == 0) { 233 printk("smp_call_function on cpu %d: other cpus not responding (%d)\n", 234 smp_processor_id(), atomic_read(&data.started)); 235 goto out; 236 } 237 barrier(); 238 udelay(1); 239 } 240 241 if (wait) { 242 timeout = 1000000; 243 while (atomic_read(&data.finished) != cpus) { 244 if (--timeout == 0) { 245 printk("smp_call_function on cpu %d: other cpus not finishing (%d/%d)\n", 246 smp_processor_id(), atomic_read(&data.finished), atomic_read(&data.started)); 247 goto out; 248 } 249 barrier(); 250 udelay(1); 251 } 252 } 253 ret = 0; 254 255 out: 256 spin_unlock_bh(&call_lock); 257 return ret; 258} 259 260void smp_call_function_interrupt(void) 261{ 262 void (*func) (void *info) = call_data->func; 263 void *info = call_data->info; 264 int wait = call_data->wait; 265 266 /* 267 * Notify initiating CPU that I've grabbed the data and am 268 * about to execute the function 269 */ 270 atomic_inc(&call_data->started); 271 /* 272 * At this point the info structure may be out of scope unless wait==1 273 */ 274 (*func)(info); 275 if (wait) 276 atomic_inc(&call_data->finished); 277} 278 279void __init smp_boot_cpus(void) 280{ 281 extern struct task_struct *current_set[NR_CPUS]; 282 int i, cpu_nr; 283 struct task_struct *p; 284 285 printk("Entering SMP Mode...\n"); 286 smp_num_cpus = 1; 287 smp_store_cpu_info(0); 288 cpu_online_map = 1UL; 289 290 /* 291 * assume for now that the first cpu booted is 292 * cpu 0, the master -- Cort 293 */ 294 cpu_callin_map[0] = 1; 295 current->processor = 0; 296 297 init_idle(); 298 299 for (i = 0; i < NR_CPUS; i++) { 300 prof_counter[i] = 1; 301 prof_multiplier[i] = 1; 302 } 303 304 cacheflush_time = 5 * 1024; 305 306 smp_ops = ppc_md.smp_ops; 307 if (smp_ops == NULL) { 308 printk("SMP not supported on this machine.\n"); 309 return; 310 } 311 312#ifndef CONFIG_750_SMP 313 /* check for 750's, they just don't work with linux SMP. 314 * If you actually have 750 SMP hardware and want to try to get 315 * it to work, send me a patch to make it work and 316 * I'll make CONFIG_750_SMP a config option. -- Troy (hozer@drgw.net) 317 */ 318 if ( PVR_VER(mfspr(PVR)) == 8 ){ 319 printk("SMP not supported on 750 cpus. %s line %d\n", 320 __FILE__, __LINE__); 321 return; 322 } 323#endif 324 325 326 /* Probe arch for CPUs */ 327 cpu_nr = smp_ops->probe(); 328 329 /* 330 * only check for cpus we know exist. We keep the callin map 331 * with cpus at the bottom -- Cort 332 */ 333 if (cpu_nr > max_cpus) 334 cpu_nr = max_cpus; 335 for (i = 1; i < cpu_nr; i++) { 336 int c; 337 struct pt_regs regs; 338 339 /* create a process for the processor */ 340 /* only regs.msr is actually used, and 0 is OK for it */ 341 memset(®s, 0, sizeof(struct pt_regs)); 342 if (do_fork(CLONE_VM|CLONE_PID, 0, ®s, 0) < 0) 343 panic("failed fork for CPU %d", i); 344 p = init_task.prev_task; 345 if (!p) 346 panic("No idle task for CPU %d", i); 347 del_from_runqueue(p); 348 unhash_process(p); 349 init_tasks[i] = p; 350 351 p->processor = i; 352 p->cpus_runnable = 1 << i; /* we schedule the first task manually */ 353 current_set[i] = p; 354 355 /* 356 * There was a cache flush loop here to flush the cache 357 * to memory for the first 8MB of RAM. The cache flush 358 * has been pushed into the kick_cpu function for those 359 * platforms that need it. 360 */ 361 362 /* wake up cpus */ 363 smp_ops->kick_cpu(i); 364 365 /* 366 * wait to see if the cpu made a callin (is actually up). 367 * use this value that I found through experimentation. 368 * -- Cort 369 */ 370 for ( c = 1000; c && !cpu_callin_map[i] ; c-- ) 371 udelay(100); 372 373 if ( cpu_callin_map[i] ) 374 { 375 char buf[32]; 376 sprintf(buf, "found cpu %d", i); 377 if (ppc_md.progress) ppc_md.progress(buf, 0x350+i); 378 printk("Processor %d found.\n", i); 379 smp_num_cpus++; 380 } else { 381 char buf[32]; 382 sprintf(buf, "didn't find cpu %d", i); 383 if (ppc_md.progress) ppc_md.progress(buf, 0x360+i); 384 printk("Processor %d is stuck.\n", i); 385 } 386 } 387 388 /* Setup CPU 0 last (important) */ 389 smp_ops->setup_cpu(0); 390 391 if (smp_num_cpus < 2) 392 smp_tb_synchronized = 1; 393} 394 395void __init smp_software_tb_sync(int cpu) 396{ 397#define PASSES 4 /* 4 passes.. */ 398 int pass; 399 int i, j; 400 401 /* stop - start will be the number of timebase ticks it takes for cpu0 402 * to send a message to all others and the first reponse to show up. 403 * 404 * ASSUMPTION: this time is similiar for all cpus 405 * ASSUMPTION: the time to send a one-way message is ping/2 406 */ 407 register unsigned long start = 0; 408 register unsigned long stop = 0; 409 register unsigned long temp = 0; 410 411 set_tb(0, 0); 412 413 /* multiple passes to get in l1 cache.. */ 414 for (pass = 2; pass < 2+PASSES; pass++){ 415 if (cpu == 0){ 416 mb(); 417 for (i = j = 1; i < smp_num_cpus; i++, j++){ 418 /* skip stuck cpus */ 419 while (!cpu_callin_map[j]) 420 ++j; 421 while (cpu_callin_map[j] != pass) 422 barrier(); 423 } 424 mb(); 425 tb_sync_flag = pass; 426 start = get_tbl(); /* start timing */ 427 while (tb_sync_flag) 428 mb(); 429 stop = get_tbl(); /* end timing */ 430 /* theoretically, the divisor should be 2, but 431 * I get better results on my dual mtx. someone 432 * please report results on other smp machines.. 433 */ 434 tb_offset = (stop-start)/4; 435 mb(); 436 tb_sync_flag = pass; 437 udelay(10); 438 mb(); 439 tb_sync_flag = 0; 440 mb(); 441 set_tb(0,0); 442 mb(); 443 } else { 444 cpu_callin_map[cpu] = pass; 445 mb(); 446 while (!tb_sync_flag) 447 mb(); /* wait for cpu0 */ 448 mb(); 449 tb_sync_flag = 0; /* send response for timing */ 450 mb(); 451 while (!tb_sync_flag) 452 mb(); 453 temp = tb_offset; /* make sure offset is loaded */ 454 while (tb_sync_flag) 455 mb(); 456 set_tb(0,temp); /* now, set the timebase */ 457 mb(); 458 } 459 } 460 if (cpu == 0) { 461 smp_tb_synchronized = 1; 462 printk("smp_software_tb_sync: %d passes, final offset: %ld\n", 463 PASSES, tb_offset); 464 } 465 /* so time.c doesn't get confused */ 466 set_dec(tb_ticks_per_jiffy); 467 last_jiffy_stamp(cpu) = 0; 468} 469 470void __init smp_commence(void) 471{ 472 /* 473 * Lets the callin's below out of their loop. 474 */ 475 if (ppc_md.progress) ppc_md.progress("smp_commence", 0x370); 476 wmb(); 477 smp_commenced = 1; 478 479 /* if the smp_ops->setup_cpu function has not already synched the 480 * timebases with a nicer hardware-based method, do so now 481 * 482 * I am open to suggestions for improvements to this method 483 * -- Troy <hozer@drgw.net> 484 * 485 * NOTE: if you are debugging, set smp_tb_synchronized for now 486 * since if this code runs pretty early and needs all cpus that 487 * reported in in smp_callin_map to be working 488 * 489 * NOTE2: this code doesn't seem to work on > 2 cpus. -- paulus/BenH 490 */ 491 if (!smp_tb_synchronized && smp_num_cpus == 2) { 492 unsigned long flags; 493 __save_and_cli(flags); 494 smp_software_tb_sync(0); 495 __restore_flags(flags); 496 } 497} 498 499void __init smp_callin(void) 500{ 501 int cpu = current->processor; 502 503 smp_store_cpu_info(cpu); 504 set_dec(tb_ticks_per_jiffy); 505 cpu_callin_map[cpu] = 1; 506 507 smp_ops->setup_cpu(cpu); 508 509 /* 510 * This cpu is now "online". Only set them online 511 * before they enter the loop below since write access 512 * to the below variable is _not_ guaranteed to be 513 * atomic. 514 * -- Cort <cort@fsmlabs.com> 515 */ 516 cpu_online_map |= 1UL << smp_processor_id(); 517 518 while(!smp_commenced) 519 barrier(); 520 521 /* see smp_commence for more info */ 522 if (!smp_tb_synchronized && smp_num_cpus == 2) { 523 smp_software_tb_sync(cpu); 524 } 525 __sti(); 526} 527 528/* intel needs this */ 529void __init initialize_secondary(void) 530{ 531} 532 533/* Activate a secondary processor. */ 534int __init start_secondary(void *unused) 535{ 536 atomic_inc(&init_mm.mm_count); 537 current->active_mm = &init_mm; 538 smp_callin(); 539 return cpu_idle(NULL); 540} 541 542void __init smp_setup(char *str, int *ints) 543{ 544} 545 546int __init setup_profiling_timer(unsigned int multiplier) 547{ 548 return 0; 549} 550 551void __init smp_store_cpu_info(int id) 552{ 553 struct cpuinfo_PPC *c = &cpu_data[id]; 554 555 /* assume bogomips are same for everything */ 556 c->loops_per_jiffy = loops_per_jiffy; 557 c->pvr = mfspr(PVR); 558} 559 560static int __init maxcpus(char *str) 561{ 562 get_option(&str, &max_cpus); 563 return 1; 564} 565 566__setup("maxcpus=", maxcpus); 567