1/* -*- mode: c; c-basic-offset: 8 -*- */ 2 3/* Copyright (C) 1999,2001 4 * 5 * Author: J.E.J.Bottomley@HansenPartnership.com 6 * 7 * linux/arch/i386/kernel/voyager_smp.c 8 * 9 * This file provides all the same external entries as smp.c but uses 10 * the voyager hal to provide the functionality 11 */ 12#include <linux/module.h> 13#include <linux/mm.h> 14#include <linux/kernel_stat.h> 15#include <linux/delay.h> 16#include <linux/mc146818rtc.h> 17#include <linux/cache.h> 18#include <linux/interrupt.h> 19#include <linux/init.h> 20#include <linux/kernel.h> 21#include <linux/bootmem.h> 22#include <linux/completion.h> 23#include <asm/desc.h> 24#include <asm/voyager.h> 25#include <asm/vic.h> 26#include <asm/mtrr.h> 27#include <asm/pgalloc.h> 28#include <asm/tlbflush.h> 29#include <asm/arch_hooks.h> 30 31/* TLB state -- visible externally, indexed physically */ 32DEFINE_PER_CPU(struct tlb_state, cpu_tlbstate) ____cacheline_aligned = { &init_mm, 0 }; 33 34/* CPU IRQ affinity -- set to all ones initially */ 35static unsigned long cpu_irq_affinity[NR_CPUS] __cacheline_aligned = { [0 ... NR_CPUS-1] = ~0UL }; 36 37/* per CPU data structure (for /proc/cpuinfo et al), visible externally 38 * indexed physically */ 39struct cpuinfo_x86 cpu_data[NR_CPUS] __cacheline_aligned; 40EXPORT_SYMBOL(cpu_data); 41 42/* physical ID of the CPU used to boot the system */ 43unsigned char boot_cpu_id; 44 45/* The memory line addresses for the Quad CPIs */ 46struct voyager_qic_cpi *voyager_quad_cpi_addr[NR_CPUS] __cacheline_aligned; 47 48/* The masks for the Extended VIC processors, filled in by cat_init */ 49__u32 voyager_extended_vic_processors = 0; 50 51/* Masks for the extended Quad processors which cannot be VIC booted */ 52__u32 voyager_allowed_boot_processors = 0; 53 54/* The mask for the Quad Processors (both extended and non-extended) */ 55__u32 voyager_quad_processors = 0; 56 57/* Total count of live CPUs, used in process.c to display 58 * the CPU information and in irq.c for the per CPU irq 59 * activity count. Finally exported by i386_ksyms.c */ 60static int voyager_extended_cpus = 1; 61 62/* Have we found an SMP box - used by time.c to do the profiling 63 interrupt for timeslicing; do not set to 1 until the per CPU timer 64 interrupt is active */ 65int smp_found_config = 0; 66 67/* Used for the invalidate map that's also checked in the spinlock */ 68static volatile unsigned long smp_invalidate_needed; 69 70/* Bitmask of currently online CPUs - used by setup.c for 71 /proc/cpuinfo, visible externally but still physical */ 72cpumask_t cpu_online_map = CPU_MASK_NONE; 73EXPORT_SYMBOL(cpu_online_map); 74 75/* Bitmask of CPUs present in the system - exported by i386_syms.c, used 76 * by scheduler but indexed physically */ 77cpumask_t phys_cpu_present_map = CPU_MASK_NONE; 78 79 80/* The internal functions */ 81static void send_CPI(__u32 cpuset, __u8 cpi); 82static void ack_CPI(__u8 cpi); 83static int ack_QIC_CPI(__u8 cpi); 84static void ack_special_QIC_CPI(__u8 cpi); 85static void ack_VIC_CPI(__u8 cpi); 86static void send_CPI_allbutself(__u8 cpi); 87static void mask_vic_irq(unsigned int irq); 88static void unmask_vic_irq(unsigned int irq); 89static unsigned int startup_vic_irq(unsigned int irq); 90static void enable_local_vic_irq(unsigned int irq); 91static void disable_local_vic_irq(unsigned int irq); 92static void before_handle_vic_irq(unsigned int irq); 93static void after_handle_vic_irq(unsigned int irq); 94static void set_vic_irq_affinity(unsigned int irq, cpumask_t mask); 95static void ack_vic_irq(unsigned int irq); 96static void vic_enable_cpi(void); 97static void do_boot_cpu(__u8 cpuid); 98static void do_quad_bootstrap(void); 99 100int hard_smp_processor_id(void); 101int safe_smp_processor_id(void); 102 103/* Inline functions */ 104static inline void 105send_one_QIC_CPI(__u8 cpu, __u8 cpi) 106{ 107 voyager_quad_cpi_addr[cpu]->qic_cpi[cpi].cpi = 108 (smp_processor_id() << 16) + cpi; 109} 110 111static inline void 112send_QIC_CPI(__u32 cpuset, __u8 cpi) 113{ 114 int cpu; 115 116 for_each_online_cpu(cpu) { 117 if(cpuset & (1<<cpu)) { 118#ifdef VOYAGER_DEBUG 119 if(!cpu_isset(cpu, cpu_online_map)) 120 VDEBUG(("CPU%d sending cpi %d to CPU%d not in cpu_online_map\n", hard_smp_processor_id(), cpi, cpu)); 121#endif 122 send_one_QIC_CPI(cpu, cpi - QIC_CPI_OFFSET); 123 } 124 } 125} 126 127static inline void 128wrapper_smp_local_timer_interrupt(void) 129{ 130 irq_enter(); 131 smp_local_timer_interrupt(); 132 irq_exit(); 133} 134 135static inline void 136send_one_CPI(__u8 cpu, __u8 cpi) 137{ 138 if(voyager_quad_processors & (1<<cpu)) 139 send_one_QIC_CPI(cpu, cpi - QIC_CPI_OFFSET); 140 else 141 send_CPI(1<<cpu, cpi); 142} 143 144static inline void 145send_CPI_allbutself(__u8 cpi) 146{ 147 __u8 cpu = smp_processor_id(); 148 __u32 mask = cpus_addr(cpu_online_map)[0] & ~(1 << cpu); 149 send_CPI(mask, cpi); 150} 151 152static inline int 153is_cpu_quad(void) 154{ 155 __u8 cpumask = inb(VIC_PROC_WHO_AM_I); 156 return ((cpumask & QUAD_IDENTIFIER) == QUAD_IDENTIFIER); 157} 158 159static inline int 160is_cpu_extended(void) 161{ 162 __u8 cpu = hard_smp_processor_id(); 163 164 return(voyager_extended_vic_processors & (1<<cpu)); 165} 166 167static inline int 168is_cpu_vic_boot(void) 169{ 170 __u8 cpu = hard_smp_processor_id(); 171 172 return(voyager_extended_vic_processors 173 & voyager_allowed_boot_processors & (1<<cpu)); 174} 175 176 177static inline void 178ack_CPI(__u8 cpi) 179{ 180 switch(cpi) { 181 case VIC_CPU_BOOT_CPI: 182 if(is_cpu_quad() && !is_cpu_vic_boot()) 183 ack_QIC_CPI(cpi); 184 else 185 ack_VIC_CPI(cpi); 186 break; 187 case VIC_SYS_INT: 188 case VIC_CMN_INT: 189 /* These are slightly strange. Even on the Quad card, 190 * They are vectored as VIC CPIs */ 191 if(is_cpu_quad()) 192 ack_special_QIC_CPI(cpi); 193 else 194 ack_VIC_CPI(cpi); 195 break; 196 default: 197 printk("VOYAGER ERROR: CPI%d is in common CPI code\n", cpi); 198 break; 199 } 200} 201 202/* local variables */ 203 204/* The VIC IRQ descriptors -- these look almost identical to the 205 * 8259 IRQs except that masks and things must be kept per processor 206 */ 207static struct irq_chip vic_chip = { 208 .name = "VIC", 209 .startup = startup_vic_irq, 210 .mask = mask_vic_irq, 211 .unmask = unmask_vic_irq, 212 .set_affinity = set_vic_irq_affinity, 213}; 214 215/* used to count up as CPUs are brought on line (starts at 0) */ 216static int cpucount = 0; 217 218/* steal a page from the bottom of memory for the trampoline and 219 * squirrel its address away here. This will be in kernel virtual 220 * space */ 221static __u32 trampoline_base; 222 223/* The per cpu profile stuff - used in smp_local_timer_interrupt */ 224static DEFINE_PER_CPU(int, prof_multiplier) = 1; 225static DEFINE_PER_CPU(int, prof_old_multiplier) = 1; 226static DEFINE_PER_CPU(int, prof_counter) = 1; 227 228/* the map used to check if a CPU has booted */ 229static __u32 cpu_booted_map; 230 231/* the synchronize flag used to hold all secondary CPUs spinning in 232 * a tight loop until the boot sequence is ready for them */ 233static cpumask_t smp_commenced_mask = CPU_MASK_NONE; 234 235/* This is for the new dynamic CPU boot code */ 236cpumask_t cpu_callin_map = CPU_MASK_NONE; 237cpumask_t cpu_callout_map = CPU_MASK_NONE; 238EXPORT_SYMBOL(cpu_callout_map); 239cpumask_t cpu_possible_map = CPU_MASK_NONE; 240EXPORT_SYMBOL(cpu_possible_map); 241 242/* The per processor IRQ masks (these are usually kept in sync) */ 243static __u16 vic_irq_mask[NR_CPUS] __cacheline_aligned; 244 245/* the list of IRQs to be enabled by the VIC_ENABLE_IRQ_CPI */ 246static __u16 vic_irq_enable_mask[NR_CPUS] __cacheline_aligned = { 0 }; 247 248/* Lock for enable/disable of VIC interrupts */ 249static __cacheline_aligned DEFINE_SPINLOCK(vic_irq_lock); 250 251/* The boot processor is correctly set up in PC mode when it 252 * comes up, but the secondaries need their master/slave 8259 253 * pairs initializing correctly */ 254 255/* Interrupt counters (per cpu) and total - used to try to 256 * even up the interrupt handling routines */ 257static long vic_intr_total = 0; 258static long vic_intr_count[NR_CPUS] __cacheline_aligned = { 0 }; 259static unsigned long vic_tick[NR_CPUS] __cacheline_aligned = { 0 }; 260 261/* Since we can only use CPI0, we fake all the other CPIs */ 262static unsigned long vic_cpi_mailbox[NR_CPUS] __cacheline_aligned; 263 264/* debugging routine to read the isr of the cpu's pic */ 265static inline __u16 266vic_read_isr(void) 267{ 268 __u16 isr; 269 270 outb(0x0b, 0xa0); 271 isr = inb(0xa0) << 8; 272 outb(0x0b, 0x20); 273 isr |= inb(0x20); 274 275 return isr; 276} 277 278static __init void 279qic_setup(void) 280{ 281 if(!is_cpu_quad()) { 282 /* not a quad, no setup */ 283 return; 284 } 285 outb(QIC_DEFAULT_MASK0, QIC_MASK_REGISTER0); 286 outb(QIC_CPI_ENABLE, QIC_MASK_REGISTER1); 287 288 if(is_cpu_extended()) { 289 /* the QIC duplicate of the VIC base register */ 290 outb(VIC_DEFAULT_CPI_BASE, QIC_VIC_CPI_BASE_REGISTER); 291 outb(QIC_DEFAULT_CPI_BASE, QIC_CPI_BASE_REGISTER); 292 293 } 294} 295 296static __init void 297vic_setup_pic(void) 298{ 299 outb(1, VIC_REDIRECT_REGISTER_1); 300 /* clear the claim registers for dynamic routing */ 301 outb(0, VIC_CLAIM_REGISTER_0); 302 outb(0, VIC_CLAIM_REGISTER_1); 303 304 outb(0, VIC_PRIORITY_REGISTER); 305 outb(FIRST_EXTERNAL_VECTOR, VIC_PRIMARY_MC_BASE); 306 outb(FIRST_EXTERNAL_VECTOR, VIC_SECONDARY_MC_BASE); 307 /* Now initiallise the master PIC belonging to this CPU by 308 * sending the four ICWs */ 309 310 /* ICW1: level triggered, ICW4 needed */ 311 outb(0x19, 0x20); 312 313 /* ICW2: vector base */ 314 outb(FIRST_EXTERNAL_VECTOR, 0x21); 315 316 /* ICW3: slave at line 2 */ 317 outb(0x04, 0x21); 318 319 /* ICW4: 8086 mode */ 320 outb(0x01, 0x21); 321 322 /* now the same for the slave PIC */ 323 324 /* ICW1: level trigger, ICW4 needed */ 325 outb(0x19, 0xA0); 326 327 /* ICW2: slave vector base */ 328 outb(FIRST_EXTERNAL_VECTOR + 8, 0xA1); 329 330 /* ICW3: slave ID */ 331 outb(0x02, 0xA1); 332 333 /* ICW4: 8086 mode */ 334 outb(0x01, 0xA1); 335} 336 337static void 338do_quad_bootstrap(void) 339{ 340 if(is_cpu_quad() && is_cpu_vic_boot()) { 341 int i; 342 unsigned long flags; 343 __u8 cpuid = hard_smp_processor_id(); 344 345 local_irq_save(flags); 346 347 for(i = 0; i<4; i++) { 348 if(((cpuid >> 2) & 0x03) == i) 349 /* don't lower our own mask! */ 350 continue; 351 352 /* masquerade as local Quad CPU */ 353 outb(QIC_CPUID_ENABLE | i, QIC_PROCESSOR_ID); 354 /* enable the startup CPI */ 355 outb(QIC_BOOT_CPI_MASK, QIC_MASK_REGISTER1); 356 /* restore cpu id */ 357 outb(0, QIC_PROCESSOR_ID); 358 } 359 local_irq_restore(flags); 360 } 361} 362 363 364/* Set up all the basic stuff: read the SMP config and make all the 365 * SMP information reflect only the boot cpu. All others will be 366 * brought on-line later. */ 367void __init 368find_smp_config(void) 369{ 370 int i; 371 372 boot_cpu_id = hard_smp_processor_id(); 373 374 printk("VOYAGER SMP: Boot cpu is %d\n", boot_cpu_id); 375 376 /* initialize the CPU structures (moved from smp_boot_cpus) */ 377 for(i=0; i<NR_CPUS; i++) { 378 cpu_irq_affinity[i] = ~0; 379 } 380 cpu_online_map = cpumask_of_cpu(boot_cpu_id); 381 382 /* The boot CPU must be extended */ 383 voyager_extended_vic_processors = 1<<boot_cpu_id; 384 /* initially, all of the first 8 cpu's can boot */ 385 voyager_allowed_boot_processors = 0xff; 386 /* set up everything for just this CPU, we can alter 387 * this as we start the other CPUs later */ 388 /* now get the CPU disposition from the extended CMOS */ 389 cpus_addr(phys_cpu_present_map)[0] = voyager_extended_cmos_read(VOYAGER_PROCESSOR_PRESENT_MASK); 390 cpus_addr(phys_cpu_present_map)[0] |= voyager_extended_cmos_read(VOYAGER_PROCESSOR_PRESENT_MASK + 1) << 8; 391 cpus_addr(phys_cpu_present_map)[0] |= voyager_extended_cmos_read(VOYAGER_PROCESSOR_PRESENT_MASK + 2) << 16; 392 cpus_addr(phys_cpu_present_map)[0] |= voyager_extended_cmos_read(VOYAGER_PROCESSOR_PRESENT_MASK + 3) << 24; 393 cpu_possible_map = phys_cpu_present_map; 394 printk("VOYAGER SMP: phys_cpu_present_map = 0x%lx\n", cpus_addr(phys_cpu_present_map)[0]); 395 /* Here we set up the VIC to enable SMP */ 396 /* enable the CPIs by writing the base vector to their register */ 397 outb(VIC_DEFAULT_CPI_BASE, VIC_CPI_BASE_REGISTER); 398 outb(1, VIC_REDIRECT_REGISTER_1); 399 /* set the claim registers for static routing --- Boot CPU gets 400 * all interrupts untill all other CPUs started */ 401 outb(0xff, VIC_CLAIM_REGISTER_0); 402 outb(0xff, VIC_CLAIM_REGISTER_1); 403 outb(FIRST_EXTERNAL_VECTOR, VIC_PRIMARY_MC_BASE); 404 outb(FIRST_EXTERNAL_VECTOR, VIC_SECONDARY_MC_BASE); 405 406 /* Finally tell the firmware that we're driving */ 407 outb(inb(VOYAGER_SUS_IN_CONTROL_PORT) | VOYAGER_IN_CONTROL_FLAG, 408 VOYAGER_SUS_IN_CONTROL_PORT); 409 410 current_thread_info()->cpu = boot_cpu_id; 411 x86_write_percpu(cpu_number, boot_cpu_id); 412} 413 414/* 415 * The bootstrap kernel entry code has set these up. Save them 416 * for a given CPU, id is physical */ 417void __init 418smp_store_cpu_info(int id) 419{ 420 struct cpuinfo_x86 *c=&cpu_data[id]; 421 422 *c = boot_cpu_data; 423 424 identify_secondary_cpu(c); 425} 426 427/* set up the trampoline and return the physical address of the code */ 428static __u32 __init 429setup_trampoline(void) 430{ 431 /* these two are global symbols in trampoline.S */ 432 extern __u8 trampoline_end[]; 433 extern __u8 trampoline_data[]; 434 435 memcpy((__u8 *)trampoline_base, trampoline_data, 436 trampoline_end - trampoline_data); 437 return virt_to_phys((__u8 *)trampoline_base); 438} 439 440/* Routine initially called when a non-boot CPU is brought online */ 441static void __init 442start_secondary(void *unused) 443{ 444 __u8 cpuid = hard_smp_processor_id(); 445 /* external functions not defined in the headers */ 446 extern void calibrate_delay(void); 447 448 cpu_init(); 449 450 /* OK, we're in the routine */ 451 ack_CPI(VIC_CPU_BOOT_CPI); 452 453 /* setup the 8259 master slave pair belonging to this CPU --- 454 * we won't actually receive any until the boot CPU 455 * relinquishes it's static routing mask */ 456 vic_setup_pic(); 457 458 qic_setup(); 459 460 if(is_cpu_quad() && !is_cpu_vic_boot()) { 461 /* clear the boot CPI */ 462 __u8 dummy; 463 464 dummy = voyager_quad_cpi_addr[cpuid]->qic_cpi[VIC_CPU_BOOT_CPI].cpi; 465 printk("read dummy %d\n", dummy); 466 } 467 468 /* lower the mask to receive CPIs */ 469 vic_enable_cpi(); 470 471 VDEBUG(("VOYAGER SMP: CPU%d, stack at about %p\n", cpuid, &cpuid)); 472 473 /* enable interrupts */ 474 local_irq_enable(); 475 476 /* get our bogomips */ 477 calibrate_delay(); 478 479 /* save our processor parameters */ 480 smp_store_cpu_info(cpuid); 481 482 /* if we're a quad, we may need to bootstrap other CPUs */ 483 do_quad_bootstrap(); 484 485 local_irq_disable(); 486 cpu_set(cpuid, cpu_callin_map); 487 488 /* signal that we're done */ 489 cpu_booted_map = 1; 490 491 while (!cpu_isset(cpuid, smp_commenced_mask)) 492 rep_nop(); 493 local_irq_enable(); 494 495 local_flush_tlb(); 496 497 cpu_set(cpuid, cpu_online_map); 498 wmb(); 499 cpu_idle(); 500} 501 502 503/* Routine to kick start the given CPU and wait for it to report ready 504 * (or timeout in startup). When this routine returns, the requested 505 * CPU is either fully running and configured or known to be dead. 506 * 507 * We call this routine sequentially 1 CPU at a time, so no need for 508 * locking */ 509 510static void __init 511do_boot_cpu(__u8 cpu) 512{ 513 struct task_struct *idle; 514 int timeout; 515 unsigned long flags; 516 int quad_boot = (1<<cpu) & voyager_quad_processors 517 & ~( voyager_extended_vic_processors 518 & voyager_allowed_boot_processors); 519 520 /* This is an area in head.S which was used to set up the 521 * initial kernel stack. We need to alter this to give the 522 * booting CPU a new stack (taken from its idle process) */ 523 extern struct { 524 __u8 *esp; 525 unsigned short ss; 526 } stack_start; 527 /* This is the format of the CPI IDT gate (in real mode) which 528 * we're hijacking to boot the CPU */ 529 union IDTFormat { 530 struct seg { 531 __u16 Offset; 532 __u16 Segment; 533 } idt; 534 __u32 val; 535 } hijack_source; 536 537 __u32 *hijack_vector; 538 __u32 start_phys_address = setup_trampoline(); 539 540 /* There's a clever trick to this: The linux trampoline is 541 * compiled to begin at absolute location zero, so make the 542 * address zero but have the data segment selector compensate 543 * for the actual address */ 544 hijack_source.idt.Offset = start_phys_address & 0x000F; 545 hijack_source.idt.Segment = (start_phys_address >> 4) & 0xFFFF; 546 547 cpucount++; 548 alternatives_smp_switch(1); 549 550 idle = fork_idle(cpu); 551 if(IS_ERR(idle)) 552 panic("failed fork for CPU%d", cpu); 553 idle->thread.eip = (unsigned long) start_secondary; 554 /* init_tasks (in sched.c) is indexed logically */ 555 stack_start.esp = (void *) idle->thread.esp; 556 557 init_gdt(cpu); 558 per_cpu(current_task, cpu) = idle; 559 early_gdt_descr.address = (unsigned long)get_cpu_gdt_table(cpu); 560 irq_ctx_init(cpu); 561 562 /* Note: Don't modify initial ss override */ 563 VDEBUG(("VOYAGER SMP: Booting CPU%d at 0x%lx[%x:%x], stack %p\n", cpu, 564 (unsigned long)hijack_source.val, hijack_source.idt.Segment, 565 hijack_source.idt.Offset, stack_start.esp)); 566 567 /* init lowmem identity mapping */ 568 clone_pgd_range(swapper_pg_dir, swapper_pg_dir + USER_PGD_PTRS, 569 min_t(unsigned long, KERNEL_PGD_PTRS, USER_PGD_PTRS)); 570 flush_tlb_all(); 571 572 if(quad_boot) { 573 printk("CPU %d: non extended Quad boot\n", cpu); 574 hijack_vector = (__u32 *)phys_to_virt((VIC_CPU_BOOT_CPI + QIC_DEFAULT_CPI_BASE)*4); 575 *hijack_vector = hijack_source.val; 576 } else { 577 printk("CPU%d: extended VIC boot\n", cpu); 578 hijack_vector = (__u32 *)phys_to_virt((VIC_CPU_BOOT_CPI + VIC_DEFAULT_CPI_BASE)*4); 579 *hijack_vector = hijack_source.val; 580 /* VIC errata, may also receive interrupt at this address */ 581 hijack_vector = (__u32 *)phys_to_virt((VIC_CPU_BOOT_ERRATA_CPI + VIC_DEFAULT_CPI_BASE)*4); 582 *hijack_vector = hijack_source.val; 583 } 584 /* All non-boot CPUs start with interrupts fully masked. Need 585 * to lower the mask of the CPI we're about to send. We do 586 * this in the VIC by masquerading as the processor we're 587 * about to boot and lowering its interrupt mask */ 588 local_irq_save(flags); 589 if(quad_boot) { 590 send_one_QIC_CPI(cpu, VIC_CPU_BOOT_CPI); 591 } else { 592 outb(VIC_CPU_MASQUERADE_ENABLE | cpu, VIC_PROCESSOR_ID); 593 /* here we're altering registers belonging to `cpu' */ 594 595 outb(VIC_BOOT_INTERRUPT_MASK, 0x21); 596 /* now go back to our original identity */ 597 outb(boot_cpu_id, VIC_PROCESSOR_ID); 598 599 /* and boot the CPU */ 600 601 send_CPI((1<<cpu), VIC_CPU_BOOT_CPI); 602 } 603 cpu_booted_map = 0; 604 local_irq_restore(flags); 605 606 /* now wait for it to become ready (or timeout) */ 607 for(timeout = 0; timeout < 50000; timeout++) { 608 if(cpu_booted_map) 609 break; 610 udelay(100); 611 } 612 /* reset the page table */ 613 zap_low_mappings(); 614 615 if (cpu_booted_map) { 616 VDEBUG(("CPU%d: Booted successfully, back in CPU %d\n", 617 cpu, smp_processor_id())); 618 619 printk("CPU%d: ", cpu); 620 print_cpu_info(&cpu_data[cpu]); 621 wmb(); 622 cpu_set(cpu, cpu_callout_map); 623 cpu_set(cpu, cpu_present_map); 624 } 625 else { 626 printk("CPU%d FAILED TO BOOT: ", cpu); 627 if (*((volatile unsigned char *)phys_to_virt(start_phys_address))==0xA5) 628 printk("Stuck.\n"); 629 else 630 printk("Not responding.\n"); 631 632 cpucount--; 633 } 634} 635 636void __init 637smp_boot_cpus(void) 638{ 639 int i; 640 641 /* CAT BUS initialisation must be done after the memory */ 642 if(voyager_level == 5) { 643 voyager_cat_init(); 644 645 /* now that the cat has probed the Voyager System Bus, sanity 646 * check the cpu map */ 647 if( ((voyager_quad_processors | voyager_extended_vic_processors) 648 & cpus_addr(phys_cpu_present_map)[0]) != cpus_addr(phys_cpu_present_map)[0]) { 649 /* should panic */ 650 printk("\n\n***WARNING*** Sanity check of CPU present map FAILED\n"); 651 } 652 } else if(voyager_level == 4) 653 voyager_extended_vic_processors = cpus_addr(phys_cpu_present_map)[0]; 654 655 /* this sets up the idle task to run on the current cpu */ 656 voyager_extended_cpus = 1; 657 /* Remove the global_irq_holder setting, it triggers a BUG() on 658 * schedule at the moment */ 659 //global_irq_holder = boot_cpu_id; 660 661 smp_store_cpu_info(boot_cpu_id); 662 printk("CPU%d: ", boot_cpu_id); 663 print_cpu_info(&cpu_data[boot_cpu_id]); 664 665 if(is_cpu_quad()) { 666 /* booting on a Quad CPU */ 667 printk("VOYAGER SMP: Boot CPU is Quad\n"); 668 qic_setup(); 669 do_quad_bootstrap(); 670 } 671 672 /* enable our own CPIs */ 673 vic_enable_cpi(); 674 675 cpu_set(boot_cpu_id, cpu_online_map); 676 cpu_set(boot_cpu_id, cpu_callout_map); 677 678 /* loop over all the extended VIC CPUs and boot them. The 679 * Quad CPUs must be bootstrapped by their extended VIC cpu */ 680 for(i = 0; i < NR_CPUS; i++) { 681 if(i == boot_cpu_id || !cpu_isset(i, phys_cpu_present_map)) 682 continue; 683 do_boot_cpu(i); 684 /* This udelay seems to be needed for the Quad boots 685 * don't remove unless you know what you're doing */ 686 udelay(1000); 687 } 688 /* we could compute the total bogomips here, but why bother?, 689 * Code added from smpboot.c */ 690 { 691 unsigned long bogosum = 0; 692 for (i = 0; i < NR_CPUS; i++) 693 if (cpu_isset(i, cpu_online_map)) 694 bogosum += cpu_data[i].loops_per_jiffy; 695 printk(KERN_INFO "Total of %d processors activated (%lu.%02lu BogoMIPS).\n", 696 cpucount+1, 697 bogosum/(500000/HZ), 698 (bogosum/(5000/HZ))%100); 699 } 700 voyager_extended_cpus = hweight32(voyager_extended_vic_processors); 701 printk("VOYAGER: Extended (interrupt handling CPUs): %d, non-extended: %d\n", voyager_extended_cpus, num_booting_cpus() - voyager_extended_cpus); 702 /* that's it, switch to symmetric mode */ 703 outb(0, VIC_PRIORITY_REGISTER); 704 outb(0, VIC_CLAIM_REGISTER_0); 705 outb(0, VIC_CLAIM_REGISTER_1); 706 707 VDEBUG(("VOYAGER SMP: Booted with %d CPUs\n", num_booting_cpus())); 708} 709 710/* Reload the secondary CPUs task structure (this function does not 711 * return ) */ 712void __init 713initialize_secondary(void) 714{ 715 716 /* 717 * We don't actually need to load the full TSS, 718 * basically just the stack pointer and the eip. 719 */ 720 721 asm volatile( 722 "movl %0,%%esp\n\t" 723 "jmp *%1" 724 : 725 :"r" (current->thread.esp),"r" (current->thread.eip)); 726} 727 728fastcall void 729smp_vic_sys_interrupt(struct pt_regs *regs) 730{ 731 ack_CPI(VIC_SYS_INT); 732 printk("Voyager SYSTEM INTERRUPT\n"); 733} 734 735fastcall void 736smp_vic_cmn_interrupt(struct pt_regs *regs) 737{ 738 static __u8 in_cmn_int = 0; 739 static DEFINE_SPINLOCK(cmn_int_lock); 740 741 /* common ints are broadcast, so make sure we only do this once */ 742 _raw_spin_lock(&cmn_int_lock); 743 if(in_cmn_int) 744 goto unlock_end; 745 746 in_cmn_int++; 747 _raw_spin_unlock(&cmn_int_lock); 748 749 VDEBUG(("Voyager COMMON INTERRUPT\n")); 750 751 if(voyager_level == 5) 752 voyager_cat_do_common_interrupt(); 753 754 _raw_spin_lock(&cmn_int_lock); 755 in_cmn_int = 0; 756 unlock_end: 757 _raw_spin_unlock(&cmn_int_lock); 758 ack_CPI(VIC_CMN_INT); 759} 760 761/* 762 * Reschedule call back. Nothing to do, all the work is done 763 * automatically when we return from the interrupt. */ 764static void 765smp_reschedule_interrupt(void) 766{ 767 /* do nothing */ 768} 769 770static struct mm_struct * flush_mm; 771static unsigned long flush_va; 772static DEFINE_SPINLOCK(tlbstate_lock); 773#define FLUSH_ALL 0xffffffff 774 775/* 776 * We cannot call mmdrop() because we are in interrupt context, 777 * instead update mm->cpu_vm_mask. 778 * 779 * We need to reload %cr3 since the page tables may be going 780 * away from under us.. 781 */ 782static inline void 783leave_mm (unsigned long cpu) 784{ 785 if (per_cpu(cpu_tlbstate, cpu).state == TLBSTATE_OK) 786 BUG(); 787 cpu_clear(cpu, per_cpu(cpu_tlbstate, cpu).active_mm->cpu_vm_mask); 788 load_cr3(swapper_pg_dir); 789} 790 791 792/* 793 * Invalidate call-back 794 */ 795static void 796smp_invalidate_interrupt(void) 797{ 798 __u8 cpu = smp_processor_id(); 799 800 if (!test_bit(cpu, &smp_invalidate_needed)) 801 return; 802 /* This will flood messages. Don't uncomment unless you see 803 * Problems with cross cpu invalidation 804 VDEBUG(("VOYAGER SMP: CPU%d received INVALIDATE_CPI\n", 805 smp_processor_id())); 806 */ 807 808 if (flush_mm == per_cpu(cpu_tlbstate, cpu).active_mm) { 809 if (per_cpu(cpu_tlbstate, cpu).state == TLBSTATE_OK) { 810 if (flush_va == FLUSH_ALL) 811 local_flush_tlb(); 812 else 813 __flush_tlb_one(flush_va); 814 } else 815 leave_mm(cpu); 816 } 817 smp_mb__before_clear_bit(); 818 clear_bit(cpu, &smp_invalidate_needed); 819 smp_mb__after_clear_bit(); 820} 821 822/* All the new flush operations for 2.4 */ 823 824 825/* This routine is called with a physical cpu mask */ 826static void 827voyager_flush_tlb_others (unsigned long cpumask, struct mm_struct *mm, 828 unsigned long va) 829{ 830 int stuck = 50000; 831 832 if (!cpumask) 833 BUG(); 834 if ((cpumask & cpus_addr(cpu_online_map)[0]) != cpumask) 835 BUG(); 836 if (cpumask & (1 << smp_processor_id())) 837 BUG(); 838 if (!mm) 839 BUG(); 840 841 spin_lock(&tlbstate_lock); 842 843 flush_mm = mm; 844 flush_va = va; 845 atomic_set_mask(cpumask, &smp_invalidate_needed); 846 /* 847 * We have to send the CPI only to 848 * CPUs affected. 849 */ 850 send_CPI(cpumask, VIC_INVALIDATE_CPI); 851 852 while (smp_invalidate_needed) { 853 mb(); 854 if(--stuck == 0) { 855 printk("***WARNING*** Stuck doing invalidate CPI (CPU%d)\n", smp_processor_id()); 856 break; 857 } 858 } 859 860 /* Uncomment only to debug invalidation problems 861 VDEBUG(("VOYAGER SMP: Completed invalidate CPI (CPU%d)\n", cpu)); 862 */ 863 864 flush_mm = NULL; 865 flush_va = 0; 866 spin_unlock(&tlbstate_lock); 867} 868 869void 870flush_tlb_current_task(void) 871{ 872 struct mm_struct *mm = current->mm; 873 unsigned long cpu_mask; 874 875 preempt_disable(); 876 877 cpu_mask = cpus_addr(mm->cpu_vm_mask)[0] & ~(1 << smp_processor_id()); 878 local_flush_tlb(); 879 if (cpu_mask) 880 voyager_flush_tlb_others(cpu_mask, mm, FLUSH_ALL); 881 882 preempt_enable(); 883} 884 885 886void 887flush_tlb_mm (struct mm_struct * mm) 888{ 889 unsigned long cpu_mask; 890 891 preempt_disable(); 892 893 cpu_mask = cpus_addr(mm->cpu_vm_mask)[0] & ~(1 << smp_processor_id()); 894 895 if (current->active_mm == mm) { 896 if (current->mm) 897 local_flush_tlb(); 898 else 899 leave_mm(smp_processor_id()); 900 } 901 if (cpu_mask) 902 voyager_flush_tlb_others(cpu_mask, mm, FLUSH_ALL); 903 904 preempt_enable(); 905} 906 907void flush_tlb_page(struct vm_area_struct * vma, unsigned long va) 908{ 909 struct mm_struct *mm = vma->vm_mm; 910 unsigned long cpu_mask; 911 912 preempt_disable(); 913 914 cpu_mask = cpus_addr(mm->cpu_vm_mask)[0] & ~(1 << smp_processor_id()); 915 if (current->active_mm == mm) { 916 if(current->mm) 917 __flush_tlb_one(va); 918 else 919 leave_mm(smp_processor_id()); 920 } 921 922 if (cpu_mask) 923 voyager_flush_tlb_others(cpu_mask, mm, va); 924 925 preempt_enable(); 926} 927EXPORT_SYMBOL(flush_tlb_page); 928 929/* enable the requested IRQs */ 930static void 931smp_enable_irq_interrupt(void) 932{ 933 __u8 irq; 934 __u8 cpu = get_cpu(); 935 936 VDEBUG(("VOYAGER SMP: CPU%d enabling irq mask 0x%x\n", cpu, 937 vic_irq_enable_mask[cpu])); 938 939 spin_lock(&vic_irq_lock); 940 for(irq = 0; irq < 16; irq++) { 941 if(vic_irq_enable_mask[cpu] & (1<<irq)) 942 enable_local_vic_irq(irq); 943 } 944 vic_irq_enable_mask[cpu] = 0; 945 spin_unlock(&vic_irq_lock); 946 947 put_cpu_no_resched(); 948} 949 950/* 951 * CPU halt call-back 952 */ 953static void 954smp_stop_cpu_function(void *dummy) 955{ 956 VDEBUG(("VOYAGER SMP: CPU%d is STOPPING\n", smp_processor_id())); 957 cpu_clear(smp_processor_id(), cpu_online_map); 958 local_irq_disable(); 959 for(;;) 960 halt(); 961} 962 963static DEFINE_SPINLOCK(call_lock); 964 965struct call_data_struct { 966 void (*func) (void *info); 967 void *info; 968 volatile unsigned long started; 969 volatile unsigned long finished; 970 int wait; 971}; 972 973static struct call_data_struct * call_data; 974 975/* execute a thread on a new CPU. The function to be called must be 976 * previously set up. This is used to schedule a function for 977 * execution on all CPU's - set up the function then broadcast a 978 * function_interrupt CPI to come here on each CPU */ 979static void 980smp_call_function_interrupt(void) 981{ 982 void (*func) (void *info) = call_data->func; 983 void *info = call_data->info; 984 /* must take copy of wait because call_data may be replaced 985 * unless the function is waiting for us to finish */ 986 int wait = call_data->wait; 987 __u8 cpu = smp_processor_id(); 988 989 /* 990 * Notify initiating CPU that I've grabbed the data and am 991 * about to execute the function 992 */ 993 mb(); 994 if(!test_and_clear_bit(cpu, &call_data->started)) { 995 /* If the bit wasn't set, this could be a replay */ 996 printk(KERN_WARNING "VOYAGER SMP: CPU %d received call funtion with no call pending\n", cpu); 997 return; 998 } 999 /* 1000 * At this point the info structure may be out of scope unless wait==1 1001 */ 1002 irq_enter(); 1003 (*func)(info); 1004 irq_exit(); 1005 if (wait) { 1006 mb(); 1007 clear_bit(cpu, &call_data->finished); 1008 } 1009} 1010 1011static int 1012voyager_smp_call_function_mask (cpumask_t cpumask, 1013 void (*func) (void *info), void *info, 1014 int wait) 1015{ 1016 struct call_data_struct data; 1017 u32 mask = cpus_addr(cpumask)[0]; 1018 1019 mask &= ~(1<<smp_processor_id()); 1020 1021 if (!mask) 1022 return 0; 1023 1024 /* Can deadlock when called with interrupts disabled */ 1025 WARN_ON(irqs_disabled()); 1026 1027 data.func = func; 1028 data.info = info; 1029 data.started = mask; 1030 data.wait = wait; 1031 if (wait) 1032 data.finished = mask; 1033 1034 spin_lock(&call_lock); 1035 call_data = &data; 1036 wmb(); 1037 /* Send a message to all other CPUs and wait for them to respond */ 1038 send_CPI(mask, VIC_CALL_FUNCTION_CPI); 1039 1040 /* Wait for response */ 1041 while (data.started) 1042 barrier(); 1043 1044 if (wait) 1045 while (data.finished) 1046 barrier(); 1047 1048 spin_unlock(&call_lock); 1049 1050 return 0; 1051} 1052 1053fastcall void 1054smp_apic_timer_interrupt(struct pt_regs *regs) 1055{ 1056 struct pt_regs *old_regs = set_irq_regs(regs); 1057 wrapper_smp_local_timer_interrupt(); 1058 set_irq_regs(old_regs); 1059} 1060 1061/* All of the QUAD interrupt GATES */ 1062fastcall void 1063smp_qic_timer_interrupt(struct pt_regs *regs) 1064{ 1065 struct pt_regs *old_regs = set_irq_regs(regs); 1066 ack_QIC_CPI(QIC_TIMER_CPI); 1067 wrapper_smp_local_timer_interrupt(); 1068 set_irq_regs(old_regs); 1069} 1070 1071fastcall void 1072smp_qic_invalidate_interrupt(struct pt_regs *regs) 1073{ 1074 ack_QIC_CPI(QIC_INVALIDATE_CPI); 1075 smp_invalidate_interrupt(); 1076} 1077 1078fastcall void 1079smp_qic_reschedule_interrupt(struct pt_regs *regs) 1080{ 1081 ack_QIC_CPI(QIC_RESCHEDULE_CPI); 1082 smp_reschedule_interrupt(); 1083} 1084 1085fastcall void 1086smp_qic_enable_irq_interrupt(struct pt_regs *regs) 1087{ 1088 ack_QIC_CPI(QIC_ENABLE_IRQ_CPI); 1089 smp_enable_irq_interrupt(); 1090} 1091 1092fastcall void 1093smp_qic_call_function_interrupt(struct pt_regs *regs) 1094{ 1095 ack_QIC_CPI(QIC_CALL_FUNCTION_CPI); 1096 smp_call_function_interrupt(); 1097} 1098 1099fastcall void 1100smp_vic_cpi_interrupt(struct pt_regs *regs) 1101{ 1102 struct pt_regs *old_regs = set_irq_regs(regs); 1103 __u8 cpu = smp_processor_id(); 1104 1105 if(is_cpu_quad()) 1106 ack_QIC_CPI(VIC_CPI_LEVEL0); 1107 else 1108 ack_VIC_CPI(VIC_CPI_LEVEL0); 1109 1110 if(test_and_clear_bit(VIC_TIMER_CPI, &vic_cpi_mailbox[cpu])) 1111 wrapper_smp_local_timer_interrupt(); 1112 if(test_and_clear_bit(VIC_INVALIDATE_CPI, &vic_cpi_mailbox[cpu])) 1113 smp_invalidate_interrupt(); 1114 if(test_and_clear_bit(VIC_RESCHEDULE_CPI, &vic_cpi_mailbox[cpu])) 1115 smp_reschedule_interrupt(); 1116 if(test_and_clear_bit(VIC_ENABLE_IRQ_CPI, &vic_cpi_mailbox[cpu])) 1117 smp_enable_irq_interrupt(); 1118 if(test_and_clear_bit(VIC_CALL_FUNCTION_CPI, &vic_cpi_mailbox[cpu])) 1119 smp_call_function_interrupt(); 1120 set_irq_regs(old_regs); 1121} 1122 1123static void 1124do_flush_tlb_all(void* info) 1125{ 1126 unsigned long cpu = smp_processor_id(); 1127 1128 __flush_tlb_all(); 1129 if (per_cpu(cpu_tlbstate, cpu).state == TLBSTATE_LAZY) 1130 leave_mm(cpu); 1131} 1132 1133 1134/* flush the TLB of every active CPU in the system */ 1135void 1136flush_tlb_all(void) 1137{ 1138 on_each_cpu(do_flush_tlb_all, 0, 1, 1); 1139} 1140 1141/* used to set up the trampoline for other CPUs when the memory manager 1142 * is sorted out */ 1143void __init 1144smp_alloc_memory(void) 1145{ 1146 trampoline_base = (__u32)alloc_bootmem_low_pages(PAGE_SIZE); 1147 if(__pa(trampoline_base) >= 0x93000) 1148 BUG(); 1149} 1150 1151/* send a reschedule CPI to one CPU by physical CPU number*/ 1152static void 1153voyager_smp_send_reschedule(int cpu) 1154{ 1155 send_one_CPI(cpu, VIC_RESCHEDULE_CPI); 1156} 1157 1158 1159int 1160hard_smp_processor_id(void) 1161{ 1162 __u8 i; 1163 __u8 cpumask = inb(VIC_PROC_WHO_AM_I); 1164 if((cpumask & QUAD_IDENTIFIER) == QUAD_IDENTIFIER) 1165 return cpumask & 0x1F; 1166 1167 for(i = 0; i < 8; i++) { 1168 if(cpumask & (1<<i)) 1169 return i; 1170 } 1171 printk("** WARNING ** Illegal cpuid returned by VIC: %d", cpumask); 1172 return 0; 1173} 1174 1175int 1176safe_smp_processor_id(void) 1177{ 1178 return hard_smp_processor_id(); 1179} 1180 1181/* broadcast a halt to all other CPUs */ 1182static void 1183voyager_smp_send_stop(void) 1184{ 1185 smp_call_function(smp_stop_cpu_function, NULL, 1, 1); 1186} 1187 1188/* this function is triggered in time.c when a clock tick fires 1189 * we need to re-broadcast the tick to all CPUs */ 1190void 1191smp_vic_timer_interrupt(void) 1192{ 1193 send_CPI_allbutself(VIC_TIMER_CPI); 1194 smp_local_timer_interrupt(); 1195} 1196 1197/* local (per CPU) timer interrupt. It does both profiling and 1198 * process statistics/rescheduling. 1199 * 1200 * We do profiling in every local tick, statistics/rescheduling 1201 * happen only every 'profiling multiplier' ticks. The default 1202 * multiplier is 1 and it can be changed by writing the new multiplier 1203 * value into /proc/profile. 1204 */ 1205void 1206smp_local_timer_interrupt(void) 1207{ 1208 int cpu = smp_processor_id(); 1209 long weight; 1210 1211 profile_tick(CPU_PROFILING); 1212 if (--per_cpu(prof_counter, cpu) <= 0) { 1213 /* 1214 * The multiplier may have changed since the last time we got 1215 * to this point as a result of the user writing to 1216 * /proc/profile. In this case we need to adjust the APIC 1217 * timer accordingly. 1218 * 1219 * Interrupts are already masked off at this point. 1220 */ 1221 per_cpu(prof_counter,cpu) = per_cpu(prof_multiplier, cpu); 1222 if (per_cpu(prof_counter, cpu) != 1223 per_cpu(prof_old_multiplier, cpu)) { 1224 per_cpu(prof_old_multiplier, cpu) = 1225 per_cpu(prof_counter, cpu); 1226 } 1227 1228 update_process_times(user_mode_vm(get_irq_regs())); 1229 } 1230 1231 if( ((1<<cpu) & voyager_extended_vic_processors) == 0) 1232 /* only extended VIC processors participate in 1233 * interrupt distribution */ 1234 return; 1235 1236 /* 1237 * We take the 'long' return path, and there every subsystem 1238 * grabs the apropriate locks (kernel lock/ irq lock). 1239 * 1240 * we might want to decouple profiling from the 'long path', 1241 * and do the profiling totally in assembly. 1242 * 1243 * Currently this isn't too much of an issue (performance wise), 1244 * we can take more than 100K local irqs per second on a 100 MHz P5. 1245 */ 1246 1247 if((++vic_tick[cpu] & 0x7) != 0) 1248 return; 1249 /* get here every 16 ticks (about every 1/6 of a second) */ 1250 1251 weight = (vic_intr_count[cpu]*voyager_extended_cpus 1252 - vic_intr_total) >> 4; 1253 weight += 4; 1254 if(weight > 7) 1255 weight = 7; 1256 if(weight < 0) 1257 weight = 0; 1258 1259 outb((__u8)weight, VIC_PRIORITY_REGISTER); 1260 1261#ifdef VOYAGER_DEBUG 1262 if((vic_tick[cpu] & 0xFFF) == 0) { 1263 /* print this message roughly every 25 secs */ 1264 printk("VOYAGER SMP: vic_tick[%d] = %lu, weight = %ld\n", 1265 cpu, vic_tick[cpu], weight); 1266 } 1267#endif 1268} 1269 1270/* setup the profiling timer */ 1271int 1272setup_profiling_timer(unsigned int multiplier) 1273{ 1274 int i; 1275 1276 if ( (!multiplier)) 1277 return -EINVAL; 1278 1279 /* 1280 * Set the new multiplier for each CPU. CPUs don't start using the 1281 * new values until the next timer interrupt in which they do process 1282 * accounting. 1283 */ 1284 for (i = 0; i < NR_CPUS; ++i) 1285 per_cpu(prof_multiplier, i) = multiplier; 1286 1287 return 0; 1288} 1289 1290/* This is a bit of a mess, but forced on us by the genirq changes 1291 * there's no genirq handler that really does what voyager wants 1292 * so hack it up with the simple IRQ handler */ 1293static void fastcall 1294handle_vic_irq(unsigned int irq, struct irq_desc *desc) 1295{ 1296 before_handle_vic_irq(irq); 1297 handle_simple_irq(irq, desc); 1298 after_handle_vic_irq(irq); 1299} 1300 1301 1302/* The CPIs are handled in the per cpu 8259s, so they must be 1303 * enabled to be received: FIX: enabling the CPIs in the early 1304 * boot sequence interferes with bug checking; enable them later 1305 * on in smp_init */ 1306#define VIC_SET_GATE(cpi, vector) \ 1307 set_intr_gate((cpi) + VIC_DEFAULT_CPI_BASE, (vector)) 1308#define QIC_SET_GATE(cpi, vector) \ 1309 set_intr_gate((cpi) + QIC_DEFAULT_CPI_BASE, (vector)) 1310 1311void __init 1312smp_intr_init(void) 1313{ 1314 int i; 1315 1316 /* initialize the per cpu irq mask to all disabled */ 1317 for(i = 0; i < NR_CPUS; i++) 1318 vic_irq_mask[i] = 0xFFFF; 1319 1320 VIC_SET_GATE(VIC_CPI_LEVEL0, vic_cpi_interrupt); 1321 1322 VIC_SET_GATE(VIC_SYS_INT, vic_sys_interrupt); 1323 VIC_SET_GATE(VIC_CMN_INT, vic_cmn_interrupt); 1324 1325 QIC_SET_GATE(QIC_TIMER_CPI, qic_timer_interrupt); 1326 QIC_SET_GATE(QIC_INVALIDATE_CPI, qic_invalidate_interrupt); 1327 QIC_SET_GATE(QIC_RESCHEDULE_CPI, qic_reschedule_interrupt); 1328 QIC_SET_GATE(QIC_ENABLE_IRQ_CPI, qic_enable_irq_interrupt); 1329 QIC_SET_GATE(QIC_CALL_FUNCTION_CPI, qic_call_function_interrupt); 1330 1331 1332 /* now put the VIC descriptor into the first 48 IRQs 1333 * 1334 * This is for later: first 16 correspond to PC IRQs; next 16 1335 * are Primary MC IRQs and final 16 are Secondary MC IRQs */ 1336 for(i = 0; i < 48; i++) 1337 set_irq_chip_and_handler(i, &vic_chip, handle_vic_irq); 1338} 1339 1340/* send a CPI at level cpi to a set of cpus in cpuset (set 1 bit per 1341 * processor to receive CPI */ 1342static void 1343send_CPI(__u32 cpuset, __u8 cpi) 1344{ 1345 int cpu; 1346 __u32 quad_cpuset = (cpuset & voyager_quad_processors); 1347 1348 if(cpi < VIC_START_FAKE_CPI) { 1349 /* fake CPI are only used for booting, so send to the 1350 * extended quads as well---Quads must be VIC booted */ 1351 outb((__u8)(cpuset), VIC_CPI_Registers[cpi]); 1352 return; 1353 } 1354 if(quad_cpuset) 1355 send_QIC_CPI(quad_cpuset, cpi); 1356 cpuset &= ~quad_cpuset; 1357 cpuset &= 0xff; /* only first 8 CPUs vaild for VIC CPI */ 1358 if(cpuset == 0) 1359 return; 1360 for_each_online_cpu(cpu) { 1361 if(cpuset & (1<<cpu)) 1362 set_bit(cpi, &vic_cpi_mailbox[cpu]); 1363 } 1364 if(cpuset) 1365 outb((__u8)cpuset, VIC_CPI_Registers[VIC_CPI_LEVEL0]); 1366} 1367 1368/* Acknowledge receipt of CPI in the QIC, clear in QIC hardware and 1369 * set the cache line to shared by reading it. 1370 * 1371 * DON'T make this inline otherwise the cache line read will be 1372 * optimised away 1373 * */ 1374static int 1375ack_QIC_CPI(__u8 cpi) { 1376 __u8 cpu = hard_smp_processor_id(); 1377 1378 cpi &= 7; 1379 1380 outb(1<<cpi, QIC_INTERRUPT_CLEAR1); 1381 return voyager_quad_cpi_addr[cpu]->qic_cpi[cpi].cpi; 1382} 1383 1384static void 1385ack_special_QIC_CPI(__u8 cpi) 1386{ 1387 switch(cpi) { 1388 case VIC_CMN_INT: 1389 outb(QIC_CMN_INT, QIC_INTERRUPT_CLEAR0); 1390 break; 1391 case VIC_SYS_INT: 1392 outb(QIC_SYS_INT, QIC_INTERRUPT_CLEAR0); 1393 break; 1394 } 1395 /* also clear at the VIC, just in case (nop for non-extended proc) */ 1396 ack_VIC_CPI(cpi); 1397} 1398 1399/* Acknowledge receipt of CPI in the VIC (essentially an EOI) */ 1400static void 1401ack_VIC_CPI(__u8 cpi) 1402{ 1403#ifdef VOYAGER_DEBUG 1404 unsigned long flags; 1405 __u16 isr; 1406 __u8 cpu = smp_processor_id(); 1407 1408 local_irq_save(flags); 1409 isr = vic_read_isr(); 1410 if((isr & (1<<(cpi &7))) == 0) { 1411 printk("VOYAGER SMP: CPU%d lost CPI%d\n", cpu, cpi); 1412 } 1413#endif 1414 /* send specific EOI; the two system interrupts have 1415 * bit 4 set for a separate vector but behave as the 1416 * corresponding 3 bit intr */ 1417 outb_p(0x60|(cpi & 7),0x20); 1418 1419#ifdef VOYAGER_DEBUG 1420 if((vic_read_isr() & (1<<(cpi &7))) != 0) { 1421 printk("VOYAGER SMP: CPU%d still asserting CPI%d\n", cpu, cpi); 1422 } 1423 local_irq_restore(flags); 1424#endif 1425} 1426 1427/* cribbed with thanks from irq.c */ 1428#define __byte(x,y) (((unsigned char *)&(y))[x]) 1429#define cached_21(cpu) (__byte(0,vic_irq_mask[cpu])) 1430#define cached_A1(cpu) (__byte(1,vic_irq_mask[cpu])) 1431 1432static unsigned int 1433startup_vic_irq(unsigned int irq) 1434{ 1435 unmask_vic_irq(irq); 1436 1437 return 0; 1438} 1439 1440/* The enable and disable routines. This is where we run into 1441 * conflicting architectural philosophy. Fundamentally, the voyager 1442 * architecture does not expect to have to disable interrupts globally 1443 * (the IRQ controllers belong to each CPU). The processor masquerade 1444 * which is used to start the system shouldn't be used in a running OS 1445 * since it will cause great confusion if two separate CPUs drive to 1446 * the same IRQ controller (I know, I've tried it). 1447 * 1448 * The solution is a variant on the NCR lazy SPL design: 1449 * 1450 * 1) To disable an interrupt, do nothing (other than set the 1451 * IRQ_DISABLED flag). This dares the interrupt actually to arrive. 1452 * 1453 * 2) If the interrupt dares to come in, raise the local mask against 1454 * it (this will result in all the CPU masks being raised 1455 * eventually). 1456 * 1457 * 3) To enable the interrupt, lower the mask on the local CPU and 1458 * broadcast an Interrupt enable CPI which causes all other CPUs to 1459 * adjust their masks accordingly. */ 1460 1461static void 1462unmask_vic_irq(unsigned int irq) 1463{ 1464 /* linux doesn't to processor-irq affinity, so enable on 1465 * all CPUs we know about */ 1466 int cpu = smp_processor_id(), real_cpu; 1467 __u16 mask = (1<<irq); 1468 __u32 processorList = 0; 1469 unsigned long flags; 1470 1471 VDEBUG(("VOYAGER: unmask_vic_irq(%d) CPU%d affinity 0x%lx\n", 1472 irq, cpu, cpu_irq_affinity[cpu])); 1473 spin_lock_irqsave(&vic_irq_lock, flags); 1474 for_each_online_cpu(real_cpu) { 1475 if(!(voyager_extended_vic_processors & (1<<real_cpu))) 1476 continue; 1477 if(!(cpu_irq_affinity[real_cpu] & mask)) { 1478 /* irq has no affinity for this CPU, ignore */ 1479 continue; 1480 } 1481 if(real_cpu == cpu) { 1482 enable_local_vic_irq(irq); 1483 } 1484 else if(vic_irq_mask[real_cpu] & mask) { 1485 vic_irq_enable_mask[real_cpu] |= mask; 1486 processorList |= (1<<real_cpu); 1487 } 1488 } 1489 spin_unlock_irqrestore(&vic_irq_lock, flags); 1490 if(processorList) 1491 send_CPI(processorList, VIC_ENABLE_IRQ_CPI); 1492} 1493 1494static void 1495mask_vic_irq(unsigned int irq) 1496{ 1497 /* lazy disable, do nothing */ 1498} 1499 1500static void 1501enable_local_vic_irq(unsigned int irq) 1502{ 1503 __u8 cpu = smp_processor_id(); 1504 __u16 mask = ~(1 << irq); 1505 __u16 old_mask = vic_irq_mask[cpu]; 1506 1507 vic_irq_mask[cpu] &= mask; 1508 if(vic_irq_mask[cpu] == old_mask) 1509 return; 1510 1511 VDEBUG(("VOYAGER DEBUG: Enabling irq %d in hardware on CPU %d\n", 1512 irq, cpu)); 1513 1514 if (irq & 8) { 1515 outb_p(cached_A1(cpu),0xA1); 1516 (void)inb_p(0xA1); 1517 } 1518 else { 1519 outb_p(cached_21(cpu),0x21); 1520 (void)inb_p(0x21); 1521 } 1522} 1523 1524static void 1525disable_local_vic_irq(unsigned int irq) 1526{ 1527 __u8 cpu = smp_processor_id(); 1528 __u16 mask = (1 << irq); 1529 __u16 old_mask = vic_irq_mask[cpu]; 1530 1531 if(irq == 7) 1532 return; 1533 1534 vic_irq_mask[cpu] |= mask; 1535 if(old_mask == vic_irq_mask[cpu]) 1536 return; 1537 1538 VDEBUG(("VOYAGER DEBUG: Disabling irq %d in hardware on CPU %d\n", 1539 irq, cpu)); 1540 1541 if (irq & 8) { 1542 outb_p(cached_A1(cpu),0xA1); 1543 (void)inb_p(0xA1); 1544 } 1545 else { 1546 outb_p(cached_21(cpu),0x21); 1547 (void)inb_p(0x21); 1548 } 1549} 1550 1551/* The VIC is level triggered, so the ack can only be issued after the 1552 * interrupt completes. However, we do Voyager lazy interrupt 1553 * handling here: It is an extremely expensive operation to mask an 1554 * interrupt in the vic, so we merely set a flag (IRQ_DISABLED). If 1555 * this interrupt actually comes in, then we mask and ack here to push 1556 * the interrupt off to another CPU */ 1557static void 1558before_handle_vic_irq(unsigned int irq) 1559{ 1560 irq_desc_t *desc = irq_desc + irq; 1561 __u8 cpu = smp_processor_id(); 1562 1563 _raw_spin_lock(&vic_irq_lock); 1564 vic_intr_total++; 1565 vic_intr_count[cpu]++; 1566 1567 if(!(cpu_irq_affinity[cpu] & (1<<irq))) { 1568 /* The irq is not in our affinity mask, push it off 1569 * onto another CPU */ 1570 VDEBUG(("VOYAGER DEBUG: affinity triggered disable of irq %d on cpu %d\n", 1571 irq, cpu)); 1572 disable_local_vic_irq(irq); 1573 /* set IRQ_INPROGRESS to prevent the handler in irq.c from 1574 * actually calling the interrupt routine */ 1575 desc->status |= IRQ_REPLAY | IRQ_INPROGRESS; 1576 } else if(desc->status & IRQ_DISABLED) { 1577 /* Damn, the interrupt actually arrived, do the lazy 1578 * disable thing. The interrupt routine in irq.c will 1579 * not handle a IRQ_DISABLED interrupt, so nothing more 1580 * need be done here */ 1581 VDEBUG(("VOYAGER DEBUG: lazy disable of irq %d on CPU %d\n", 1582 irq, cpu)); 1583 disable_local_vic_irq(irq); 1584 desc->status |= IRQ_REPLAY; 1585 } else { 1586 desc->status &= ~IRQ_REPLAY; 1587 } 1588 1589 _raw_spin_unlock(&vic_irq_lock); 1590} 1591 1592/* Finish the VIC interrupt: basically mask */ 1593static void 1594after_handle_vic_irq(unsigned int irq) 1595{ 1596 irq_desc_t *desc = irq_desc + irq; 1597 1598 _raw_spin_lock(&vic_irq_lock); 1599 { 1600 unsigned int status = desc->status & ~IRQ_INPROGRESS; 1601#ifdef VOYAGER_DEBUG 1602 __u16 isr; 1603#endif 1604 1605 desc->status = status; 1606 if ((status & IRQ_DISABLED)) 1607 disable_local_vic_irq(irq); 1608#ifdef VOYAGER_DEBUG 1609 /* DEBUG: before we ack, check what's in progress */ 1610 isr = vic_read_isr(); 1611 if((isr & (1<<irq) && !(status & IRQ_REPLAY)) == 0) { 1612 int i; 1613 __u8 cpu = smp_processor_id(); 1614 __u8 real_cpu; 1615 int mask; /* Um... initialize me??? --RR */ 1616 1617 printk("VOYAGER SMP: CPU%d lost interrupt %d\n", 1618 cpu, irq); 1619 for_each_possible_cpu(real_cpu, mask) { 1620 1621 outb(VIC_CPU_MASQUERADE_ENABLE | real_cpu, 1622 VIC_PROCESSOR_ID); 1623 isr = vic_read_isr(); 1624 if(isr & (1<<irq)) { 1625 printk("VOYAGER SMP: CPU%d ack irq %d\n", 1626 real_cpu, irq); 1627 ack_vic_irq(irq); 1628 } 1629 outb(cpu, VIC_PROCESSOR_ID); 1630 } 1631 } 1632#endif /* VOYAGER_DEBUG */ 1633 /* as soon as we ack, the interrupt is eligible for 1634 * receipt by another CPU so everything must be in 1635 * order here */ 1636 ack_vic_irq(irq); 1637 if(status & IRQ_REPLAY) { 1638 /* replay is set if we disable the interrupt 1639 * in the before_handle_vic_irq() routine, so 1640 * clear the in progress bit here to allow the 1641 * next CPU to handle this correctly */ 1642 desc->status &= ~(IRQ_REPLAY | IRQ_INPROGRESS); 1643 } 1644#ifdef VOYAGER_DEBUG 1645 isr = vic_read_isr(); 1646 if((isr & (1<<irq)) != 0) 1647 printk("VOYAGER SMP: after_handle_vic_irq() after ack irq=%d, isr=0x%x\n", 1648 irq, isr); 1649#endif /* VOYAGER_DEBUG */ 1650 } 1651 _raw_spin_unlock(&vic_irq_lock); 1652 1653 /* All code after this point is out of the main path - the IRQ 1654 * may be intercepted by another CPU if reasserted */ 1655} 1656 1657 1658/* Linux processor - interrupt affinity manipulations. 1659 * 1660 * For each processor, we maintain a 32 bit irq affinity mask. 1661 * Initially it is set to all 1's so every processor accepts every 1662 * interrupt. In this call, we change the processor's affinity mask: 1663 * 1664 * Change from enable to disable: 1665 * 1666 * If the interrupt ever comes in to the processor, we will disable it 1667 * and ack it to push it off to another CPU, so just accept the mask here. 1668 * 1669 * Change from disable to enable: 1670 * 1671 * change the mask and then do an interrupt enable CPI to re-enable on 1672 * the selected processors */ 1673 1674void 1675set_vic_irq_affinity(unsigned int irq, cpumask_t mask) 1676{ 1677 /* Only extended processors handle interrupts */ 1678 unsigned long real_mask; 1679 unsigned long irq_mask = 1 << irq; 1680 int cpu; 1681 1682 real_mask = cpus_addr(mask)[0] & voyager_extended_vic_processors; 1683 1684 if(cpus_addr(mask)[0] == 0) 1685 /* can't have no cpu's to accept the interrupt -- extremely 1686 * bad things will happen */ 1687 return; 1688 1689 if(irq == 0) 1690 /* can't change the affinity of the timer IRQ. This 1691 * is due to the constraint in the voyager 1692 * architecture that the CPI also comes in on and IRQ 1693 * line and we have chosen IRQ0 for this. If you 1694 * raise the mask on this interrupt, the processor 1695 * will no-longer be able to accept VIC CPIs */ 1696 return; 1697 1698 if(irq >= 32) 1699 /* You can only have 32 interrupts in a voyager system 1700 * (and 32 only if you have a secondary microchannel 1701 * bus) */ 1702 return; 1703 1704 for_each_online_cpu(cpu) { 1705 unsigned long cpu_mask = 1 << cpu; 1706 1707 if(cpu_mask & real_mask) { 1708 /* enable the interrupt for this cpu */ 1709 cpu_irq_affinity[cpu] |= irq_mask; 1710 } else { 1711 /* disable the interrupt for this cpu */ 1712 cpu_irq_affinity[cpu] &= ~irq_mask; 1713 } 1714 } 1715 /* this is magic, we now have the correct affinity maps, so 1716 * enable the interrupt. This will send an enable CPI to 1717 * those cpu's who need to enable it in their local masks, 1718 * causing them to correct for the new affinity . If the 1719 * interrupt is currently globally disabled, it will simply be 1720 * disabled again as it comes in (voyager lazy disable). If 1721 * the affinity map is tightened to disable the interrupt on a 1722 * cpu, it will be pushed off when it comes in */ 1723 unmask_vic_irq(irq); 1724} 1725 1726static void 1727ack_vic_irq(unsigned int irq) 1728{ 1729 if (irq & 8) { 1730 outb(0x62,0x20); /* Specific EOI to cascade */ 1731 outb(0x60|(irq & 7),0xA0); 1732 } else { 1733 outb(0x60 | (irq & 7),0x20); 1734 } 1735} 1736 1737/* enable the CPIs. In the VIC, the CPIs are delivered by the 8259 1738 * but are not vectored by it. This means that the 8259 mask must be 1739 * lowered to receive them */ 1740static __init void 1741vic_enable_cpi(void) 1742{ 1743 __u8 cpu = smp_processor_id(); 1744 1745 /* just take a copy of the current mask (nop for boot cpu) */ 1746 vic_irq_mask[cpu] = vic_irq_mask[boot_cpu_id]; 1747 1748 enable_local_vic_irq(VIC_CPI_LEVEL0); 1749 enable_local_vic_irq(VIC_CPI_LEVEL1); 1750 /* for sys int and cmn int */ 1751 enable_local_vic_irq(7); 1752 1753 if(is_cpu_quad()) { 1754 outb(QIC_DEFAULT_MASK0, QIC_MASK_REGISTER0); 1755 outb(QIC_CPI_ENABLE, QIC_MASK_REGISTER1); 1756 VDEBUG(("VOYAGER SMP: QIC ENABLE CPI: CPU%d: MASK 0x%x\n", 1757 cpu, QIC_CPI_ENABLE)); 1758 } 1759 1760 VDEBUG(("VOYAGER SMP: ENABLE CPI: CPU%d: MASK 0x%x\n", 1761 cpu, vic_irq_mask[cpu])); 1762} 1763 1764void 1765voyager_smp_dump() 1766{ 1767 int old_cpu = smp_processor_id(), cpu; 1768 1769 /* dump the interrupt masks of each processor */ 1770 for_each_online_cpu(cpu) { 1771 __u16 imr, isr, irr; 1772 unsigned long flags; 1773 1774 local_irq_save(flags); 1775 outb(VIC_CPU_MASQUERADE_ENABLE | cpu, VIC_PROCESSOR_ID); 1776 imr = (inb(0xa1) << 8) | inb(0x21); 1777 outb(0x0a, 0xa0); 1778 irr = inb(0xa0) << 8; 1779 outb(0x0a, 0x20); 1780 irr |= inb(0x20); 1781 outb(0x0b, 0xa0); 1782 isr = inb(0xa0) << 8; 1783 outb(0x0b, 0x20); 1784 isr |= inb(0x20); 1785 outb(old_cpu, VIC_PROCESSOR_ID); 1786 local_irq_restore(flags); 1787 printk("\tCPU%d: mask=0x%x, IMR=0x%x, IRR=0x%x, ISR=0x%x\n", 1788 cpu, vic_irq_mask[cpu], imr, irr, isr); 1789 } 1790} 1791 1792void 1793smp_voyager_power_off(void *dummy) 1794{ 1795 if(smp_processor_id() == boot_cpu_id) 1796 voyager_power_off(); 1797 else 1798 smp_stop_cpu_function(NULL); 1799} 1800 1801static void __init 1802voyager_smp_prepare_cpus(unsigned int max_cpus) 1803{ 1804 smp_boot_cpus(); 1805} 1806 1807static void __devinit voyager_smp_prepare_boot_cpu(void) 1808{ 1809 init_gdt(smp_processor_id()); 1810 switch_to_new_gdt(); 1811 1812 cpu_set(smp_processor_id(), cpu_online_map); 1813 cpu_set(smp_processor_id(), cpu_callout_map); 1814 cpu_set(smp_processor_id(), cpu_possible_map); 1815 cpu_set(smp_processor_id(), cpu_present_map); 1816} 1817 1818static int __devinit 1819voyager_cpu_up(unsigned int cpu) 1820{ 1821 /* This only works at boot for x86. See "rewrite" above. */ 1822 if (cpu_isset(cpu, smp_commenced_mask)) 1823 return -ENOSYS; 1824 1825 /* In case one didn't come up */ 1826 if (!cpu_isset(cpu, cpu_callin_map)) 1827 return -EIO; 1828 /* Unleash the CPU! */ 1829 cpu_set(cpu, smp_commenced_mask); 1830 while (!cpu_isset(cpu, cpu_online_map)) 1831 mb(); 1832 return 0; 1833} 1834 1835static void __init 1836voyager_smp_cpus_done(unsigned int max_cpus) 1837{ 1838 zap_low_mappings(); 1839} 1840 1841void __init 1842smp_setup_processor_id(void) 1843{ 1844 current_thread_info()->cpu = hard_smp_processor_id(); 1845 x86_write_percpu(cpu_number, hard_smp_processor_id()); 1846} 1847 1848struct smp_ops smp_ops = { 1849 .smp_prepare_boot_cpu = voyager_smp_prepare_boot_cpu, 1850 .smp_prepare_cpus = voyager_smp_prepare_cpus, 1851 .cpu_up = voyager_cpu_up, 1852 .smp_cpus_done = voyager_smp_cpus_done, 1853 1854 .smp_send_stop = voyager_smp_send_stop, 1855 .smp_send_reschedule = voyager_smp_send_reschedule, 1856 .smp_call_function_mask = voyager_smp_call_function_mask, 1857}; 1858