1/* -*- linux-c -*- 2 * linux/arch/blackfin/kernel/ipipe.c 3 * 4 * Copyright (C) 2005-2007 Philippe Gerum. 5 * 6 * This program is free software; you can redistribute it and/or modify 7 * it under the terms of the GNU General Public License as published by 8 * the Free Software Foundation, Inc., 675 Mass Ave, Cambridge MA 02139, 9 * USA; either version 2 of the License, or (at your option) any later 10 * version. 11 * 12 * This program is distributed in the hope that it will be useful, 13 * but WITHOUT ANY WARRANTY; without even the implied warranty of 14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 * GNU General Public License for more details. 16 * 17 * You should have received a copy of the GNU General Public License 18 * along with this program; if not, write to the Free Software 19 * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. 20 * 21 * Architecture-dependent I-pipe support for the Blackfin. 22 */ 23 24#include <linux/kernel.h> 25#include <linux/sched.h> 26#include <linux/module.h> 27#include <linux/interrupt.h> 28#include <linux/percpu.h> 29#include <linux/bitops.h> 30#include <linux/errno.h> 31#include <linux/kthread.h> 32#include <linux/unistd.h> 33#include <linux/io.h> 34#include <asm/system.h> 35#include <asm/atomic.h> 36 37DEFINE_PER_CPU(struct pt_regs, __ipipe_tick_regs); 38 39asmlinkage void asm_do_IRQ(unsigned int irq, struct pt_regs *regs); 40 41static void __ipipe_no_irqtail(void); 42 43unsigned long __ipipe_irq_tail_hook = (unsigned long)&__ipipe_no_irqtail; 44EXPORT_SYMBOL(__ipipe_irq_tail_hook); 45 46unsigned long __ipipe_core_clock; 47EXPORT_SYMBOL(__ipipe_core_clock); 48 49unsigned long __ipipe_freq_scale; 50EXPORT_SYMBOL(__ipipe_freq_scale); 51 52atomic_t __ipipe_irq_lvdepth[IVG15 + 1]; 53 54unsigned long __ipipe_irq_lvmask = bfin_no_irqs; 55EXPORT_SYMBOL(__ipipe_irq_lvmask); 56 57static void __ipipe_ack_irq(unsigned irq, struct irq_desc *desc) 58{ 59 desc->ipipe_ack(irq, desc); 60} 61 62/* 63 * __ipipe_enable_pipeline() -- We are running on the boot CPU, hw 64 * interrupts are off, and secondary CPUs are still lost in space. 65 */ 66void __ipipe_enable_pipeline(void) 67{ 68 unsigned irq; 69 70 __ipipe_core_clock = get_cclk(); /* Fetch this once. */ 71 __ipipe_freq_scale = 1000000000UL / __ipipe_core_clock; 72 73 for (irq = 0; irq < NR_IRQS; ++irq) 74 ipipe_virtualize_irq(ipipe_root_domain, 75 irq, 76 (ipipe_irq_handler_t)&asm_do_IRQ, 77 NULL, 78 &__ipipe_ack_irq, 79 IPIPE_HANDLE_MASK | IPIPE_PASS_MASK); 80} 81 82/* 83 * __ipipe_handle_irq() -- IPIPE's generic IRQ handler. An optimistic 84 * interrupt protection log is maintained here for each domain. Hw 85 * interrupts are masked on entry. 86 */ 87void __ipipe_handle_irq(unsigned irq, struct pt_regs *regs) 88{ 89 struct ipipe_percpu_domain_data *p = ipipe_root_cpudom_ptr(); 90 struct ipipe_domain *this_domain, *next_domain; 91 struct list_head *head, *pos; 92 struct ipipe_irqdesc *idesc; 93 int m_ack, s = -1; 94 95 /* 96 * Software-triggered IRQs do not need any ack. The contents 97 * of the register frame should only be used when processing 98 * the timer interrupt, but not for handling any other 99 * interrupt. 100 */ 101 m_ack = (regs == NULL || irq == IRQ_SYSTMR || irq == IRQ_CORETMR); 102 this_domain = __ipipe_current_domain; 103 idesc = &this_domain->irqs[irq]; 104 105 if (unlikely(test_bit(IPIPE_STICKY_FLAG, &idesc->control))) 106 head = &this_domain->p_link; 107 else { 108 head = __ipipe_pipeline.next; 109 next_domain = list_entry(head, struct ipipe_domain, p_link); 110 idesc = &next_domain->irqs[irq]; 111 if (likely(test_bit(IPIPE_WIRED_FLAG, &idesc->control))) { 112 if (!m_ack && idesc->acknowledge != NULL) 113 idesc->acknowledge(irq, irq_to_desc(irq)); 114 if (test_bit(IPIPE_SYNCDEFER_FLAG, &p->status)) 115 s = __test_and_set_bit(IPIPE_STALL_FLAG, 116 &p->status); 117 __ipipe_dispatch_wired(next_domain, irq); 118 goto out; 119 } 120 } 121 122 /* Ack the interrupt. */ 123 124 pos = head; 125 while (pos != &__ipipe_pipeline) { 126 next_domain = list_entry(pos, struct ipipe_domain, p_link); 127 idesc = &next_domain->irqs[irq]; 128 if (test_bit(IPIPE_HANDLE_FLAG, &idesc->control)) { 129 __ipipe_set_irq_pending(next_domain, irq); 130 if (!m_ack && idesc->acknowledge != NULL) { 131 idesc->acknowledge(irq, irq_to_desc(irq)); 132 m_ack = 1; 133 } 134 } 135 if (!test_bit(IPIPE_PASS_FLAG, &idesc->control)) 136 break; 137 pos = next_domain->p_link.next; 138 } 139 140 /* 141 * Now walk the pipeline, yielding control to the highest 142 * priority domain that has pending interrupt(s) or 143 * immediately to the current domain if the interrupt has been 144 * marked as 'sticky'. This search does not go beyond the 145 * current domain in the pipeline. We also enforce the 146 * additional root stage lock (blackfin-specific). 147 */ 148 if (test_bit(IPIPE_SYNCDEFER_FLAG, &p->status)) 149 s = __test_and_set_bit(IPIPE_STALL_FLAG, &p->status); 150 151 /* 152 * If the interrupt preempted the head domain, then do not 153 * even try to walk the pipeline, unless an interrupt is 154 * pending for it. 155 */ 156 if (test_bit(IPIPE_AHEAD_FLAG, &this_domain->flags) && 157 ipipe_head_cpudom_var(irqpend_himask) == 0) 158 goto out; 159 160 __ipipe_walk_pipeline(head); 161out: 162 if (!s) 163 __clear_bit(IPIPE_STALL_FLAG, &p->status); 164} 165 166void __ipipe_enable_irqdesc(struct ipipe_domain *ipd, unsigned irq) 167{ 168 struct irq_desc *desc = irq_to_desc(irq); 169 int prio = __ipipe_get_irq_priority(irq); 170 171 desc->depth = 0; 172 if (ipd != &ipipe_root && 173 atomic_inc_return(&__ipipe_irq_lvdepth[prio]) == 1) 174 __set_bit(prio, &__ipipe_irq_lvmask); 175} 176EXPORT_SYMBOL(__ipipe_enable_irqdesc); 177 178void __ipipe_disable_irqdesc(struct ipipe_domain *ipd, unsigned irq) 179{ 180 int prio = __ipipe_get_irq_priority(irq); 181 182 if (ipd != &ipipe_root && 183 atomic_dec_and_test(&__ipipe_irq_lvdepth[prio])) 184 __clear_bit(prio, &__ipipe_irq_lvmask); 185} 186EXPORT_SYMBOL(__ipipe_disable_irqdesc); 187 188int __ipipe_syscall_root(struct pt_regs *regs) 189{ 190 struct ipipe_percpu_domain_data *p; 191 unsigned long flags; 192 int ret; 193 194 /* 195 * We need to run the IRQ tail hook whenever we don't 196 * propagate a syscall to higher domains, because we know that 197 * important operations might be pending there (e.g. Xenomai 198 * deferred rescheduling). 199 */ 200 201 if (regs->orig_p0 < NR_syscalls) { 202 void (*hook)(void) = (void (*)(void))__ipipe_irq_tail_hook; 203 hook(); 204 if ((current->flags & PF_EVNOTIFY) == 0) 205 return 0; 206 } 207 208 /* 209 * This routine either returns: 210 * 0 -- if the syscall is to be passed to Linux; 211 * >0 -- if the syscall should not be passed to Linux, and no 212 * tail work should be performed; 213 * <0 -- if the syscall should not be passed to Linux but the 214 * tail work has to be performed (for handling signals etc). 215 */ 216 217 if (!__ipipe_event_monitored_p(IPIPE_EVENT_SYSCALL)) 218 return 0; 219 220 ret = __ipipe_dispatch_event(IPIPE_EVENT_SYSCALL, regs); 221 222 local_irq_save_hw(flags); 223 224 if (!__ipipe_root_domain_p) { 225 local_irq_restore_hw(flags); 226 return 1; 227 } 228 229 p = ipipe_root_cpudom_ptr(); 230 if ((p->irqpend_himask & IPIPE_IRQMASK_VIRT) != 0) 231 __ipipe_sync_pipeline(IPIPE_IRQMASK_VIRT); 232 233 local_irq_restore_hw(flags); 234 235 return -ret; 236} 237 238unsigned long ipipe_critical_enter(void (*syncfn) (void)) 239{ 240 unsigned long flags; 241 242 local_irq_save_hw(flags); 243 244 return flags; 245} 246 247void ipipe_critical_exit(unsigned long flags) 248{ 249 local_irq_restore_hw(flags); 250} 251 252static void __ipipe_no_irqtail(void) 253{ 254} 255 256int ipipe_get_sysinfo(struct ipipe_sysinfo *info) 257{ 258 info->ncpus = num_online_cpus(); 259 info->cpufreq = ipipe_cpu_freq(); 260 info->archdep.tmirq = IPIPE_TIMER_IRQ; 261 info->archdep.tmfreq = info->cpufreq; 262 263 return 0; 264} 265 266/* 267 * ipipe_trigger_irq() -- Push the interrupt at front of the pipeline 268 * just like if it has been actually received from a hw source. Also 269 * works for virtual interrupts. 270 */ 271int ipipe_trigger_irq(unsigned irq) 272{ 273 unsigned long flags; 274 275#ifdef CONFIG_IPIPE_DEBUG 276 if (irq >= IPIPE_NR_IRQS || 277 (ipipe_virtual_irq_p(irq) 278 && !test_bit(irq - IPIPE_VIRQ_BASE, &__ipipe_virtual_irq_map))) 279 return -EINVAL; 280#endif 281 282 local_irq_save_hw(flags); 283 __ipipe_handle_irq(irq, NULL); 284 local_irq_restore_hw(flags); 285 286 return 1; 287} 288 289asmlinkage void __ipipe_sync_root(void) 290{ 291 void (*irq_tail_hook)(void) = (void (*)(void))__ipipe_irq_tail_hook; 292 unsigned long flags; 293 294 BUG_ON(irqs_disabled()); 295 296 local_irq_save_hw(flags); 297 298 if (irq_tail_hook) 299 irq_tail_hook(); 300 301 clear_thread_flag(TIF_IRQ_SYNC); 302 303 if (ipipe_root_cpudom_var(irqpend_himask) != 0) 304 __ipipe_sync_pipeline(IPIPE_IRQMASK_ANY); 305 306 local_irq_restore_hw(flags); 307} 308 309void ___ipipe_sync_pipeline(unsigned long syncmask) 310{ 311 if (__ipipe_root_domain_p && 312 test_bit(IPIPE_SYNCDEFER_FLAG, &ipipe_root_cpudom_var(status))) 313 return; 314 315 __ipipe_sync_stage(syncmask); 316} 317 318void __ipipe_disable_root_irqs_hw(void) 319{ 320 /* 321 * This code is called by the ins{bwl} routines (see 322 * arch/blackfin/lib/ins.S), which are heavily used by the 323 * network stack. It masks all interrupts but those handled by 324 * non-root domains, so that we keep decent network transfer 325 * rates for Linux without inducing pathological jitter for 326 * the real-time domain. 327 */ 328 bfin_sti(__ipipe_irq_lvmask); 329 __set_bit(IPIPE_STALL_FLAG, &ipipe_root_cpudom_var(status)); 330} 331 332void __ipipe_enable_root_irqs_hw(void) 333{ 334 __clear_bit(IPIPE_STALL_FLAG, &ipipe_root_cpudom_var(status)); 335 bfin_sti(bfin_irq_flags); 336} 337 338/* 339 * We could use standard atomic bitops in the following root status 340 * manipulation routines, but let's prepare for SMP support in the 341 * same move, preventing CPU migration as required. 342 */ 343void __ipipe_stall_root(void) 344{ 345 unsigned long *p, flags; 346 347 local_irq_save_hw(flags); 348 p = &__ipipe_root_status; 349 __set_bit(IPIPE_STALL_FLAG, p); 350 local_irq_restore_hw(flags); 351} 352EXPORT_SYMBOL(__ipipe_stall_root); 353 354unsigned long __ipipe_test_and_stall_root(void) 355{ 356 unsigned long *p, flags; 357 int x; 358 359 local_irq_save_hw(flags); 360 p = &__ipipe_root_status; 361 x = __test_and_set_bit(IPIPE_STALL_FLAG, p); 362 local_irq_restore_hw(flags); 363 364 return x; 365} 366EXPORT_SYMBOL(__ipipe_test_and_stall_root); 367 368unsigned long __ipipe_test_root(void) 369{ 370 const unsigned long *p; 371 unsigned long flags; 372 int x; 373 374 local_irq_save_hw_smp(flags); 375 p = &__ipipe_root_status; 376 x = test_bit(IPIPE_STALL_FLAG, p); 377 local_irq_restore_hw_smp(flags); 378 379 return x; 380} 381EXPORT_SYMBOL(__ipipe_test_root); 382 383void __ipipe_lock_root(void) 384{ 385 unsigned long *p, flags; 386 387 local_irq_save_hw(flags); 388 p = &__ipipe_root_status; 389 __set_bit(IPIPE_SYNCDEFER_FLAG, p); 390 local_irq_restore_hw(flags); 391} 392EXPORT_SYMBOL(__ipipe_lock_root); 393 394void __ipipe_unlock_root(void) 395{ 396 unsigned long *p, flags; 397 398 local_irq_save_hw(flags); 399 p = &__ipipe_root_status; 400 __clear_bit(IPIPE_SYNCDEFER_FLAG, p); 401 local_irq_restore_hw(flags); 402} 403EXPORT_SYMBOL(__ipipe_unlock_root); 404