1/* 2 * SMP Support 3 * 4 * Copyright (C) 1999 Walt Drummond <drummond@valinux.com> 5 * Copyright (C) 1999, 2001, 2003 David Mosberger-Tang <davidm@hpl.hp.com> 6 * 7 * Lots of stuff stolen from arch/alpha/kernel/smp.c 8 * 9 * 01/05/16 Rohit Seth <rohit.seth@intel.com> IA64-SMP functions. Reorganized 10 * the existing code (on the lines of x86 port). 11 * 00/09/11 David Mosberger <davidm@hpl.hp.com> Do loops_per_jiffy 12 * calibration on each CPU. 13 * 00/08/23 Asit Mallick <asit.k.mallick@intel.com> fixed logical processor id 14 * 00/03/31 Rohit Seth <rohit.seth@intel.com> Fixes for Bootstrap Processor 15 * & cpu_online_map now gets done here (instead of setup.c) 16 * 99/10/05 davidm Update to bring it in sync with new command-line processing 17 * scheme. 18 * 10/13/00 Goutham Rao <goutham.rao@intel.com> Updated smp_call_function and 19 * smp_call_function_single to resend IPI on timeouts 20 */ 21#include <linux/module.h> 22#include <linux/kernel.h> 23#include <linux/sched.h> 24#include <linux/init.h> 25#include <linux/interrupt.h> 26#include <linux/smp.h> 27#include <linux/kernel_stat.h> 28#include <linux/mm.h> 29#include <linux/cache.h> 30#include <linux/delay.h> 31#include <linux/efi.h> 32#include <linux/bitops.h> 33#include <linux/kexec.h> 34 35#include <asm/atomic.h> 36#include <asm/current.h> 37#include <asm/delay.h> 38#include <asm/machvec.h> 39#include <asm/io.h> 40#include <asm/irq.h> 41#include <asm/page.h> 42#include <asm/pgalloc.h> 43#include <asm/pgtable.h> 44#include <asm/processor.h> 45#include <asm/ptrace.h> 46#include <asm/sal.h> 47#include <asm/system.h> 48#include <asm/tlbflush.h> 49#include <asm/unistd.h> 50#include <asm/mca.h> 51 52/* 53 * Note: alignment of 4 entries/cacheline was empirically determined 54 * to be a good tradeoff between hot cachelines & spreading the array 55 * across too many cacheline. 56 */ 57static struct local_tlb_flush_counts { 58 unsigned int count; 59} __attribute__((__aligned__(32))) local_tlb_flush_counts[NR_CPUS]; 60 61static DEFINE_PER_CPU_SHARED_ALIGNED(unsigned short [NR_CPUS], 62 shadow_flush_counts); 63 64#define IPI_CALL_FUNC 0 65#define IPI_CPU_STOP 1 66#define IPI_CALL_FUNC_SINGLE 2 67#define IPI_KDUMP_CPU_STOP 3 68 69/* This needs to be cacheline aligned because it is written to by *other* CPUs. */ 70static DEFINE_PER_CPU_SHARED_ALIGNED(unsigned long, ipi_operation); 71 72extern void cpu_halt (void); 73 74static void 75stop_this_cpu(void) 76{ 77 /* 78 * Remove this CPU: 79 */ 80 cpu_clear(smp_processor_id(), cpu_online_map); 81 max_xtp(); 82 local_irq_disable(); 83 cpu_halt(); 84} 85 86void 87cpu_die(void) 88{ 89 max_xtp(); 90 local_irq_disable(); 91 cpu_halt(); 92 /* Should never be here */ 93 BUG(); 94 for (;;); 95} 96 97irqreturn_t 98handle_IPI (int irq, void *dev_id) 99{ 100 int this_cpu = get_cpu(); 101 unsigned long *pending_ipis = &__ia64_per_cpu_var(ipi_operation); 102 unsigned long ops; 103 104 mb(); /* Order interrupt and bit testing. */ 105 while ((ops = xchg(pending_ipis, 0)) != 0) { 106 mb(); /* Order bit clearing and data access. */ 107 do { 108 unsigned long which; 109 110 which = ffz(~ops); 111 ops &= ~(1 << which); 112 113 switch (which) { 114 case IPI_CPU_STOP: 115 stop_this_cpu(); 116 break; 117 case IPI_CALL_FUNC: 118 generic_smp_call_function_interrupt(); 119 break; 120 case IPI_CALL_FUNC_SINGLE: 121 generic_smp_call_function_single_interrupt(); 122 break; 123#ifdef CONFIG_KEXEC 124 case IPI_KDUMP_CPU_STOP: 125 unw_init_running(kdump_cpu_freeze, NULL); 126 break; 127#endif 128 default: 129 printk(KERN_CRIT "Unknown IPI on CPU %d: %lu\n", 130 this_cpu, which); 131 break; 132 } 133 } while (ops); 134 mb(); /* Order data access and bit testing. */ 135 } 136 put_cpu(); 137 return IRQ_HANDLED; 138} 139 140 141 142/* 143 * Called with preemption disabled. 144 */ 145static inline void 146send_IPI_single (int dest_cpu, int op) 147{ 148 set_bit(op, &per_cpu(ipi_operation, dest_cpu)); 149 platform_send_ipi(dest_cpu, IA64_IPI_VECTOR, IA64_IPI_DM_INT, 0); 150} 151 152/* 153 * Called with preemption disabled. 154 */ 155static inline void 156send_IPI_allbutself (int op) 157{ 158 unsigned int i; 159 160 for_each_online_cpu(i) { 161 if (i != smp_processor_id()) 162 send_IPI_single(i, op); 163 } 164} 165 166/* 167 * Called with preemption disabled. 168 */ 169static inline void 170send_IPI_mask(const struct cpumask *mask, int op) 171{ 172 unsigned int cpu; 173 174 for_each_cpu(cpu, mask) { 175 send_IPI_single(cpu, op); 176 } 177} 178 179/* 180 * Called with preemption disabled. 181 */ 182static inline void 183send_IPI_all (int op) 184{ 185 int i; 186 187 for_each_online_cpu(i) { 188 send_IPI_single(i, op); 189 } 190} 191 192/* 193 * Called with preemption disabled. 194 */ 195static inline void 196send_IPI_self (int op) 197{ 198 send_IPI_single(smp_processor_id(), op); 199} 200 201#ifdef CONFIG_KEXEC 202void 203kdump_smp_send_stop(void) 204{ 205 send_IPI_allbutself(IPI_KDUMP_CPU_STOP); 206} 207 208void 209kdump_smp_send_init(void) 210{ 211 unsigned int cpu, self_cpu; 212 self_cpu = smp_processor_id(); 213 for_each_online_cpu(cpu) { 214 if (cpu != self_cpu) { 215 if(kdump_status[cpu] == 0) 216 platform_send_ipi(cpu, 0, IA64_IPI_DM_INIT, 0); 217 } 218 } 219} 220#endif 221/* 222 * Called with preemption disabled. 223 */ 224void 225smp_send_reschedule (int cpu) 226{ 227 platform_send_ipi(cpu, IA64_IPI_RESCHEDULE, IA64_IPI_DM_INT, 0); 228} 229EXPORT_SYMBOL_GPL(smp_send_reschedule); 230 231/* 232 * Called with preemption disabled. 233 */ 234static void 235smp_send_local_flush_tlb (int cpu) 236{ 237 platform_send_ipi(cpu, IA64_IPI_LOCAL_TLB_FLUSH, IA64_IPI_DM_INT, 0); 238} 239 240void 241smp_local_flush_tlb(void) 242{ 243 /* 244 * Use atomic ops. Otherwise, the load/increment/store sequence from 245 * a "++" operation can have the line stolen between the load & store. 246 * The overhead of the atomic op in negligible in this case & offers 247 * significant benefit for the brief periods where lots of cpus 248 * are simultaneously flushing TLBs. 249 */ 250 ia64_fetchadd(1, &local_tlb_flush_counts[smp_processor_id()].count, acq); 251 local_flush_tlb_all(); 252} 253 254#define FLUSH_DELAY 5 /* Usec backoff to eliminate excessive cacheline bouncing */ 255 256void 257smp_flush_tlb_cpumask(cpumask_t xcpumask) 258{ 259 unsigned short *counts = __ia64_per_cpu_var(shadow_flush_counts); 260 cpumask_t cpumask = xcpumask; 261 int mycpu, cpu, flush_mycpu = 0; 262 263 preempt_disable(); 264 mycpu = smp_processor_id(); 265 266 for_each_cpu_mask(cpu, cpumask) 267 counts[cpu] = local_tlb_flush_counts[cpu].count & 0xffff; 268 269 mb(); 270 for_each_cpu_mask(cpu, cpumask) { 271 if (cpu == mycpu) 272 flush_mycpu = 1; 273 else 274 smp_send_local_flush_tlb(cpu); 275 } 276 277 if (flush_mycpu) 278 smp_local_flush_tlb(); 279 280 for_each_cpu_mask(cpu, cpumask) 281 while(counts[cpu] == (local_tlb_flush_counts[cpu].count & 0xffff)) 282 udelay(FLUSH_DELAY); 283 284 preempt_enable(); 285} 286 287void 288smp_flush_tlb_all (void) 289{ 290 on_each_cpu((void (*)(void *))local_flush_tlb_all, NULL, 1); 291} 292 293void 294smp_flush_tlb_mm (struct mm_struct *mm) 295{ 296 preempt_disable(); 297 /* this happens for the common case of a single-threaded fork(): */ 298 if (likely(mm == current->active_mm && atomic_read(&mm->mm_users) == 1)) 299 { 300 local_finish_flush_tlb_mm(mm); 301 preempt_enable(); 302 return; 303 } 304 305 smp_call_function_many(mm_cpumask(mm), 306 (void (*)(void *))local_finish_flush_tlb_mm, mm, 1); 307 local_irq_disable(); 308 local_finish_flush_tlb_mm(mm); 309 local_irq_enable(); 310 preempt_enable(); 311} 312 313void arch_send_call_function_single_ipi(int cpu) 314{ 315 send_IPI_single(cpu, IPI_CALL_FUNC_SINGLE); 316} 317 318void arch_send_call_function_ipi_mask(const struct cpumask *mask) 319{ 320 send_IPI_mask(mask, IPI_CALL_FUNC); 321} 322 323/* 324 * this function calls the 'stop' function on all other CPUs in the system. 325 */ 326void 327smp_send_stop (void) 328{ 329 send_IPI_allbutself(IPI_CPU_STOP); 330} 331 332int 333setup_profiling_timer (unsigned int multiplier) 334{ 335 return -EINVAL; 336} 337