1/* Modified by Broadcom Corp. Portions Copyright (c) Broadcom Corp, 2012. */ 2/* 3 * linux/kernel/softirq.c 4 * 5 * Copyright (C) 1992 Linus Torvalds 6 * 7 * Distribute under GPLv2. 8 * 9 * Rewritten. Old one was good in 2.2, but in 2.3 it was immoral. --ANK (990903) 10 * 11 * Remote softirq infrastructure is by Jens Axboe. 12 */ 13 14#include <linux/module.h> 15#include <linux/kernel_stat.h> 16#include <linux/interrupt.h> 17#include <linux/init.h> 18#include <linux/mm.h> 19#include <linux/notifier.h> 20#include <linux/percpu.h> 21#include <linux/cpu.h> 22#include <linux/freezer.h> 23#include <linux/kthread.h> 24#include <linux/rcupdate.h> 25#include <linux/ftrace.h> 26#include <linux/smp.h> 27#include <linux/tick.h> 28 29#define CREATE_TRACE_POINTS 30#include <trace/events/irq.h> 31 32#include <asm/irq.h> 33 34#if defined(CONFIG_BUZZZ) 35#include <asm/buzzz.h> 36#endif /* CONFIG_BUZZZ */ 37 38#include <typedefs.h> 39#include <bcmdefs.h> 40 41/* 42 - No shared variables, all the data are CPU local. 43 - If a softirq needs serialization, let it serialize itself 44 by its own spinlocks. 45 - Even if softirq is serialized, only local cpu is marked for 46 execution. Hence, we get something sort of weak cpu binding. 47 Though it is still not clear, will it result in better locality 48 or will not. 49 50 Examples: 51 - NET RX softirq. It is multithreaded and does not require 52 any global serialization. 53 - NET TX softirq. It kicks software netdevice queues, hence 54 it is logically serialized per device, but this serialization 55 is invisible to common code. 56 - Tasklets: serialized wrt itself. 57 */ 58 59#ifndef __ARCH_IRQ_STAT 60irq_cpustat_t irq_stat[NR_CPUS] ____cacheline_aligned; 61EXPORT_SYMBOL(irq_stat); 62#endif 63 64static struct softirq_action softirq_vec[NR_SOFTIRQS] __cacheline_aligned_in_smp; 65 66static DEFINE_PER_CPU(struct task_struct *, ksoftirqd); 67 68char *softirq_to_name[NR_SOFTIRQS] = { 69 "HI", "TIMER", "NET_TX", "NET_RX", "BLOCK", "BLOCK_IOPOLL", 70 "TASKLET", "SCHED", "HRTIMER", "RCU" 71}; 72 73/* 74 * we cannot loop indefinitely here to avoid userspace starvation, 75 * but we also don't want to introduce a worst case 1/HZ latency 76 * to the pending events, so lets the scheduler to balance 77 * the softirq load for us. 78 */ 79void wakeup_softirqd(void) 80{ 81 /* Interrupts are disabled: no need to stop preemption */ 82 struct task_struct *tsk = __get_cpu_var(ksoftirqd); 83 84 if (tsk && tsk->state != TASK_RUNNING) 85 wake_up_process(tsk); 86} 87 88/* 89 * This one is for softirq.c-internal use, 90 * where hardirqs are disabled legitimately: 91 */ 92#ifdef CONFIG_TRACE_IRQFLAGS 93static void __local_bh_disable(unsigned long ip) 94{ 95 unsigned long flags; 96 97 WARN_ON_ONCE(in_irq()); 98 99 raw_local_irq_save(flags); 100 /* 101 * The preempt tracer hooks into add_preempt_count and will break 102 * lockdep because it calls back into lockdep after SOFTIRQ_OFFSET 103 * is set and before current->softirq_enabled is cleared. 104 * We must manually increment preempt_count here and manually 105 * call the trace_preempt_off later. 106 */ 107 preempt_count() += SOFTIRQ_OFFSET; 108 /* 109 * Were softirqs turned off above: 110 */ 111 if (softirq_count() == SOFTIRQ_OFFSET) 112 trace_softirqs_off(ip); 113 raw_local_irq_restore(flags); 114 115 if (preempt_count() == SOFTIRQ_OFFSET) 116 trace_preempt_off(CALLER_ADDR0, get_parent_ip(CALLER_ADDR1)); 117} 118#else /* !CONFIG_TRACE_IRQFLAGS */ 119static inline void __local_bh_disable(unsigned long ip) 120{ 121 add_preempt_count(SOFTIRQ_OFFSET); 122 barrier(); 123} 124#endif /* CONFIG_TRACE_IRQFLAGS */ 125 126void BCMFASTPATH local_bh_disable(void) 127{ 128 __local_bh_disable((unsigned long)__builtin_return_address(0)); 129} 130 131EXPORT_SYMBOL(local_bh_disable); 132 133/* 134 * Special-case - softirqs can safely be enabled in 135 * cond_resched_softirq(), or by __do_softirq(), 136 * without processing still-pending softirqs: 137 */ 138void _local_bh_enable(void) 139{ 140 141 if (softirq_count() == SOFTIRQ_OFFSET) 142 trace_softirqs_on((unsigned long)__builtin_return_address(0)); 143 sub_preempt_count(SOFTIRQ_OFFSET); 144} 145 146EXPORT_SYMBOL(_local_bh_enable); 147 148static inline void _local_bh_enable_ip(unsigned long ip) 149{ 150#ifdef CONFIG_TRACE_IRQFLAGS 151 local_irq_disable(); 152#endif 153 /* 154 * Are softirqs going to be turned on now: 155 */ 156 if (softirq_count() == SOFTIRQ_OFFSET) 157 trace_softirqs_on(ip); 158 /* 159 * Keep preemption disabled until we are done with 160 * softirq processing: 161 */ 162 sub_preempt_count(SOFTIRQ_OFFSET - 1); 163 164 if (unlikely(!in_interrupt() && local_softirq_pending())) 165 do_softirq(); 166 167 dec_preempt_count(); 168#ifdef CONFIG_TRACE_IRQFLAGS 169 local_irq_enable(); 170#endif 171 preempt_check_resched(); 172} 173 174void BCMFASTPATH local_bh_enable(void) 175{ 176 _local_bh_enable_ip((unsigned long)__builtin_return_address(0)); 177} 178EXPORT_SYMBOL(local_bh_enable); 179 180void local_bh_enable_ip(unsigned long ip) 181{ 182 _local_bh_enable_ip(ip); 183} 184EXPORT_SYMBOL(local_bh_enable_ip); 185 186/* 187 * We restart softirq processing MAX_SOFTIRQ_RESTART times, 188 * and we fall back to softirqd after that. 189 * 190 * This number has been established via experimentation. 191 * The two things to balance is latency against fairness - 192 * we want to handle softirqs as soon as possible, but they 193 * should not be able to lock up the box. 194 */ 195#define MAX_SOFTIRQ_RESTART 10 196 197asmlinkage void BCMFASTPATH __do_softirq(void) 198{ 199 struct softirq_action *h; 200 __u32 pending; 201 int max_restart = MAX_SOFTIRQ_RESTART; 202 int cpu; 203 204 pending = local_softirq_pending(); 205 account_system_vtime(current); 206 207 __local_bh_disable((unsigned long)__builtin_return_address(0)); 208 lockdep_softirq_enter(); 209 210 cpu = smp_processor_id(); 211restart: 212 /* Reset the pending bitmask before enabling irqs */ 213 set_softirq_pending(0); 214 215 local_irq_enable(); 216 217 h = softirq_vec; 218 219 do { 220 if (pending & 1) { 221 int prev_count = preempt_count(); 222 kstat_incr_softirqs_this_cpu(h - softirq_vec); 223 224 trace_softirq_entry(h, softirq_vec); 225 226#if defined(BUZZZ_KEVT_LVL) && (BUZZZ_KEVT_LVL >= 1) 227 buzzz_kevt_log1(BUZZZ_KEVT_ID_SIRQ_ENTRY, (int)h->action); 228#endif /* BUZZZ_KEVT_LVL */ 229 230 h->action(h); 231 232#if defined(BUZZZ_KEVT_LVL) && (BUZZZ_KEVT_LVL >= 1) 233 buzzz_kevt_log1(BUZZZ_KEVT_ID_SIRQ_EXIT, (int)h->action); 234#endif /* BUZZZ_KEVT_LVL */ 235 236 trace_softirq_exit(h, softirq_vec); 237 if (unlikely(prev_count != preempt_count())) { 238 printk(KERN_ERR "huh, entered softirq %td %s %p" 239 "with preempt_count %08x," 240 " exited with %08x?\n", h - softirq_vec, 241 softirq_to_name[h - softirq_vec], 242 h->action, prev_count, preempt_count()); 243 preempt_count() = prev_count; 244 } 245 246 rcu_bh_qs(cpu); 247 } 248 h++; 249 pending >>= 1; 250 } while (pending); 251 252 local_irq_disable(); 253 254 pending = local_softirq_pending(); 255 if (pending && --max_restart) 256 goto restart; 257 258 if (pending) 259 wakeup_softirqd(); 260 261 lockdep_softirq_exit(); 262 263 account_system_vtime(current); 264 _local_bh_enable(); 265} 266 267#ifndef __ARCH_HAS_DO_SOFTIRQ 268 269asmlinkage void do_softirq(void) 270{ 271 __u32 pending; 272 unsigned long flags; 273 274 if (in_interrupt()) 275 return; 276 277 local_irq_save(flags); 278 279 pending = local_softirq_pending(); 280 281 if (pending) 282 __do_softirq(); 283 284 local_irq_restore(flags); 285} 286 287#endif 288 289/* 290 * Enter an interrupt context. 291 */ 292void irq_enter(void) 293{ 294 int cpu = smp_processor_id(); 295 296 rcu_irq_enter(); 297 if (idle_cpu(cpu) && !in_interrupt()) { 298 __irq_enter(); 299 tick_check_idle(cpu); 300 } else 301 __irq_enter(); 302} 303 304#ifdef __ARCH_IRQ_EXIT_IRQS_DISABLED 305# define invoke_softirq() __do_softirq() 306#else 307# define invoke_softirq() do_softirq() 308#endif 309 310/* 311 * Exit an interrupt context. Process softirqs if needed and possible: 312 */ 313void irq_exit(void) 314{ 315 account_system_vtime(current); 316 trace_hardirq_exit(); 317 sub_preempt_count(IRQ_EXIT_OFFSET); 318 if (!in_interrupt() && local_softirq_pending()) 319 invoke_softirq(); 320 321 rcu_irq_exit(); 322#ifdef CONFIG_NO_HZ 323 /* Make sure that timer wheel updates are propagated */ 324 if (idle_cpu(smp_processor_id()) && !in_interrupt() && !need_resched()) 325 tick_nohz_stop_sched_tick(0); 326#endif 327 preempt_enable_no_resched(); 328} 329 330/* 331 * This function must run with irqs disabled! 332 */ 333inline void raise_softirq_irqoff(unsigned int nr) 334{ 335 __raise_softirq_irqoff(nr); 336 337 /* 338 * If we're in an interrupt or softirq, we're done 339 * (this also catches softirq-disabled code). We will 340 * actually run the softirq once we return from 341 * the irq or softirq. 342 * 343 * Otherwise we wake up ksoftirqd to make sure we 344 * schedule the softirq soon. 345 */ 346 if (!in_interrupt()) 347 wakeup_softirqd(); 348} 349 350void raise_softirq(unsigned int nr) 351{ 352 unsigned long flags; 353 354 local_irq_save(flags); 355 raise_softirq_irqoff(nr); 356 local_irq_restore(flags); 357} 358 359void open_softirq(int nr, void (*action)(struct softirq_action *)) 360{ 361 softirq_vec[nr].action = action; 362} 363 364/* 365 * Tasklets 366 */ 367struct tasklet_head 368{ 369 struct tasklet_struct *head; 370 struct tasklet_struct **tail; 371}; 372 373static DEFINE_PER_CPU(struct tasklet_head, tasklet_vec); 374static DEFINE_PER_CPU(struct tasklet_head, tasklet_hi_vec); 375 376void __tasklet_schedule(struct tasklet_struct *t) 377{ 378 unsigned long flags; 379 380 local_irq_save(flags); 381 t->next = NULL; 382 *__get_cpu_var(tasklet_vec).tail = t; 383 __get_cpu_var(tasklet_vec).tail = &(t->next); 384 raise_softirq_irqoff(TASKLET_SOFTIRQ); 385 local_irq_restore(flags); 386} 387 388EXPORT_SYMBOL(__tasklet_schedule); 389 390void __tasklet_hi_schedule(struct tasklet_struct *t) 391{ 392 unsigned long flags; 393 394 local_irq_save(flags); 395 t->next = NULL; 396 *__get_cpu_var(tasklet_hi_vec).tail = t; 397 __get_cpu_var(tasklet_hi_vec).tail = &(t->next); 398 raise_softirq_irqoff(HI_SOFTIRQ); 399 local_irq_restore(flags); 400} 401 402EXPORT_SYMBOL(__tasklet_hi_schedule); 403 404void __tasklet_hi_schedule_first(struct tasklet_struct *t) 405{ 406 BUG_ON(!irqs_disabled()); 407 408 t->next = __get_cpu_var(tasklet_hi_vec).head; 409 __get_cpu_var(tasklet_hi_vec).head = t; 410 __raise_softirq_irqoff(HI_SOFTIRQ); 411} 412 413EXPORT_SYMBOL(__tasklet_hi_schedule_first); 414 415static void tasklet_action(struct softirq_action *a) 416{ 417 struct tasklet_struct *list; 418 419 local_irq_disable(); 420 list = __get_cpu_var(tasklet_vec).head; 421 __get_cpu_var(tasklet_vec).head = NULL; 422 __get_cpu_var(tasklet_vec).tail = &__get_cpu_var(tasklet_vec).head; 423 local_irq_enable(); 424 425 while (list) { 426 struct tasklet_struct *t = list; 427 428 list = list->next; 429 430 if (tasklet_trylock(t)) { 431 if (!atomic_read(&t->count)) { 432 if (!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state)) 433 BUG(); 434 t->func(t->data); 435 tasklet_unlock(t); 436 continue; 437 } 438 tasklet_unlock(t); 439 } 440 441 local_irq_disable(); 442 t->next = NULL; 443 *__get_cpu_var(tasklet_vec).tail = t; 444 __get_cpu_var(tasklet_vec).tail = &(t->next); 445 __raise_softirq_irqoff(TASKLET_SOFTIRQ); 446 local_irq_enable(); 447 } 448} 449 450static void tasklet_hi_action(struct softirq_action *a) 451{ 452 struct tasklet_struct *list; 453 454 local_irq_disable(); 455 list = __get_cpu_var(tasklet_hi_vec).head; 456 __get_cpu_var(tasklet_hi_vec).head = NULL; 457 __get_cpu_var(tasklet_hi_vec).tail = &__get_cpu_var(tasklet_hi_vec).head; 458 local_irq_enable(); 459 460 while (list) { 461 struct tasklet_struct *t = list; 462 463 list = list->next; 464 465 if (tasklet_trylock(t)) { 466 if (!atomic_read(&t->count)) { 467 if (!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state)) 468 BUG(); 469 t->func(t->data); 470 tasklet_unlock(t); 471 continue; 472 } 473 tasklet_unlock(t); 474 } 475 476 local_irq_disable(); 477 t->next = NULL; 478 *__get_cpu_var(tasklet_hi_vec).tail = t; 479 __get_cpu_var(tasklet_hi_vec).tail = &(t->next); 480 __raise_softirq_irqoff(HI_SOFTIRQ); 481 local_irq_enable(); 482 } 483} 484 485 486void tasklet_init(struct tasklet_struct *t, 487 void (*func)(unsigned long), unsigned long data) 488{ 489 t->next = NULL; 490 t->state = 0; 491 atomic_set(&t->count, 0); 492 t->func = func; 493 t->data = data; 494} 495 496EXPORT_SYMBOL(tasklet_init); 497 498void tasklet_kill(struct tasklet_struct *t) 499{ 500 if (in_interrupt()) 501 printk("Attempt to kill tasklet from interrupt\n"); 502 503 while (test_and_set_bit(TASKLET_STATE_SCHED, &t->state)) { 504 do { 505 yield(); 506 } while (test_bit(TASKLET_STATE_SCHED, &t->state)); 507 } 508 tasklet_unlock_wait(t); 509 clear_bit(TASKLET_STATE_SCHED, &t->state); 510} 511 512EXPORT_SYMBOL(tasklet_kill); 513 514/* 515 * tasklet_hrtimer 516 */ 517 518/* 519 * The trampoline is called when the hrtimer expires. It schedules a tasklet 520 * to run __tasklet_hrtimer_trampoline() which in turn will call the intended 521 * hrtimer callback, but from softirq context. 522 */ 523static enum hrtimer_restart __hrtimer_tasklet_trampoline(struct hrtimer *timer) 524{ 525 struct tasklet_hrtimer *ttimer = 526 container_of(timer, struct tasklet_hrtimer, timer); 527 528 tasklet_hi_schedule(&ttimer->tasklet); 529 return HRTIMER_NORESTART; 530} 531 532/* 533 * Helper function which calls the hrtimer callback from 534 * tasklet/softirq context 535 */ 536static void __tasklet_hrtimer_trampoline(unsigned long data) 537{ 538 struct tasklet_hrtimer *ttimer = (void *)data; 539 enum hrtimer_restart restart; 540 541 restart = ttimer->function(&ttimer->timer); 542 if (restart != HRTIMER_NORESTART) 543 hrtimer_restart(&ttimer->timer); 544} 545 546/** 547 * tasklet_hrtimer_init - Init a tasklet/hrtimer combo for softirq callbacks 548 * @ttimer: tasklet_hrtimer which is initialized 549 * @function: hrtimer callback funtion which gets called from softirq context 550 * @which_clock: clock id (CLOCK_MONOTONIC/CLOCK_REALTIME) 551 * @mode: hrtimer mode (HRTIMER_MODE_ABS/HRTIMER_MODE_REL) 552 */ 553void tasklet_hrtimer_init(struct tasklet_hrtimer *ttimer, 554 enum hrtimer_restart (*function)(struct hrtimer *), 555 clockid_t which_clock, enum hrtimer_mode mode) 556{ 557 hrtimer_init(&ttimer->timer, which_clock, mode); 558 ttimer->timer.function = __hrtimer_tasklet_trampoline; 559 tasklet_init(&ttimer->tasklet, __tasklet_hrtimer_trampoline, 560 (unsigned long)ttimer); 561 ttimer->function = function; 562} 563EXPORT_SYMBOL_GPL(tasklet_hrtimer_init); 564 565/* 566 * Remote softirq bits 567 */ 568 569DEFINE_PER_CPU(struct list_head [NR_SOFTIRQS], softirq_work_list); 570EXPORT_PER_CPU_SYMBOL(softirq_work_list); 571 572static void __local_trigger(struct call_single_data *cp, int softirq) 573{ 574 struct list_head *head = &__get_cpu_var(softirq_work_list[softirq]); 575 576 list_add_tail(&cp->list, head); 577 578 /* Trigger the softirq only if the list was previously empty. */ 579 if (head->next == &cp->list) 580 raise_softirq_irqoff(softirq); 581} 582 583#ifdef CONFIG_USE_GENERIC_SMP_HELPERS 584static void remote_softirq_receive(void *data) 585{ 586 struct call_single_data *cp = data; 587 unsigned long flags; 588 int softirq; 589 590 softirq = cp->priv; 591 592 local_irq_save(flags); 593 __local_trigger(cp, softirq); 594 local_irq_restore(flags); 595} 596 597static int __try_remote_softirq(struct call_single_data *cp, int cpu, int softirq) 598{ 599 if (cpu_online(cpu)) { 600 cp->func = remote_softirq_receive; 601 cp->info = cp; 602 cp->flags = 0; 603 cp->priv = softirq; 604 605 __smp_call_function_single(cpu, cp, 0); 606 return 0; 607 } 608 return 1; 609} 610#else /* CONFIG_USE_GENERIC_SMP_HELPERS */ 611static int __try_remote_softirq(struct call_single_data *cp, int cpu, int softirq) 612{ 613 return 1; 614} 615#endif 616 617/** 618 * __send_remote_softirq - try to schedule softirq work on a remote cpu 619 * @cp: private SMP call function data area 620 * @cpu: the remote cpu 621 * @this_cpu: the currently executing cpu 622 * @softirq: the softirq for the work 623 * 624 * Attempt to schedule softirq work on a remote cpu. If this cannot be 625 * done, the work is instead queued up on the local cpu. 626 * 627 * Interrupts must be disabled. 628 */ 629void __send_remote_softirq(struct call_single_data *cp, int cpu, int this_cpu, int softirq) 630{ 631 if (cpu == this_cpu || __try_remote_softirq(cp, cpu, softirq)) 632 __local_trigger(cp, softirq); 633} 634EXPORT_SYMBOL(__send_remote_softirq); 635 636/** 637 * send_remote_softirq - try to schedule softirq work on a remote cpu 638 * @cp: private SMP call function data area 639 * @cpu: the remote cpu 640 * @softirq: the softirq for the work 641 * 642 * Like __send_remote_softirq except that disabling interrupts and 643 * computing the current cpu is done for the caller. 644 */ 645void send_remote_softirq(struct call_single_data *cp, int cpu, int softirq) 646{ 647 unsigned long flags; 648 int this_cpu; 649 650 local_irq_save(flags); 651 this_cpu = smp_processor_id(); 652 __send_remote_softirq(cp, cpu, this_cpu, softirq); 653 local_irq_restore(flags); 654} 655EXPORT_SYMBOL(send_remote_softirq); 656 657static int __cpuinit remote_softirq_cpu_notify(struct notifier_block *self, 658 unsigned long action, void *hcpu) 659{ 660 /* 661 * If a CPU goes away, splice its entries to the current CPU 662 * and trigger a run of the softirq 663 */ 664 if (action == CPU_DEAD || action == CPU_DEAD_FROZEN) { 665 int cpu = (unsigned long) hcpu; 666 int i; 667 668 local_irq_disable(); 669 for (i = 0; i < NR_SOFTIRQS; i++) { 670 struct list_head *head = &per_cpu(softirq_work_list[i], cpu); 671 struct list_head *local_head; 672 673 if (list_empty(head)) 674 continue; 675 676 local_head = &__get_cpu_var(softirq_work_list[i]); 677 list_splice_init(head, local_head); 678 raise_softirq_irqoff(i); 679 } 680 local_irq_enable(); 681 } 682 683 return NOTIFY_OK; 684} 685 686static struct notifier_block __cpuinitdata remote_softirq_cpu_notifier = { 687 .notifier_call = remote_softirq_cpu_notify, 688}; 689 690void __init softirq_init(void) 691{ 692 int cpu; 693 694 for_each_possible_cpu(cpu) { 695 int i; 696 697 per_cpu(tasklet_vec, cpu).tail = 698 &per_cpu(tasklet_vec, cpu).head; 699 per_cpu(tasklet_hi_vec, cpu).tail = 700 &per_cpu(tasklet_hi_vec, cpu).head; 701 for (i = 0; i < NR_SOFTIRQS; i++) 702 INIT_LIST_HEAD(&per_cpu(softirq_work_list[i], cpu)); 703 } 704 705 register_hotcpu_notifier(&remote_softirq_cpu_notifier); 706 707 open_softirq(TASKLET_SOFTIRQ, tasklet_action); 708 open_softirq(HI_SOFTIRQ, tasklet_hi_action); 709} 710 711static int run_ksoftirqd(void * __bind_cpu) 712{ 713 set_current_state(TASK_INTERRUPTIBLE); 714 715 while (!kthread_should_stop()) { 716 preempt_disable(); 717 if (!local_softirq_pending()) { 718 preempt_enable_no_resched(); 719 schedule(); 720 preempt_disable(); 721 } 722 723 __set_current_state(TASK_RUNNING); 724 725 while (local_softirq_pending()) { 726 /* Preempt disable stops cpu going offline. 727 If already offline, we'll be on wrong CPU: 728 don't process */ 729 if (cpu_is_offline((long)__bind_cpu)) 730 goto wait_to_die; 731 do_softirq(); 732 preempt_enable_no_resched(); 733 cond_resched(); 734 preempt_disable(); 735 rcu_note_context_switch((long)__bind_cpu); 736 } 737 preempt_enable(); 738 set_current_state(TASK_INTERRUPTIBLE); 739 } 740 __set_current_state(TASK_RUNNING); 741 return 0; 742 743wait_to_die: 744 preempt_enable(); 745 /* Wait for kthread_stop */ 746 set_current_state(TASK_INTERRUPTIBLE); 747 while (!kthread_should_stop()) { 748 schedule(); 749 set_current_state(TASK_INTERRUPTIBLE); 750 } 751 __set_current_state(TASK_RUNNING); 752 return 0; 753} 754 755#ifdef CONFIG_HOTPLUG_CPU 756/* 757 * tasklet_kill_immediate is called to remove a tasklet which can already be 758 * scheduled for execution on @cpu. 759 * 760 * Unlike tasklet_kill, this function removes the tasklet 761 * _immediately_, even if the tasklet is in TASKLET_STATE_SCHED state. 762 * 763 * When this function is called, @cpu must be in the CPU_DEAD state. 764 */ 765void tasklet_kill_immediate(struct tasklet_struct *t, unsigned int cpu) 766{ 767 struct tasklet_struct **i; 768 769 BUG_ON(cpu_online(cpu)); 770 BUG_ON(test_bit(TASKLET_STATE_RUN, &t->state)); 771 772 if (!test_bit(TASKLET_STATE_SCHED, &t->state)) 773 return; 774 775 /* CPU is dead, so no lock needed. */ 776 for (i = &per_cpu(tasklet_vec, cpu).head; *i; i = &(*i)->next) { 777 if (*i == t) { 778 *i = t->next; 779 /* If this was the tail element, move the tail ptr */ 780 if (*i == NULL) 781 per_cpu(tasklet_vec, cpu).tail = i; 782 return; 783 } 784 } 785 BUG(); 786} 787 788static void takeover_tasklets(unsigned int cpu) 789{ 790 /* CPU is dead, so no lock needed. */ 791 local_irq_disable(); 792 793 /* Find end, append list for that CPU. */ 794 if (&per_cpu(tasklet_vec, cpu).head != per_cpu(tasklet_vec, cpu).tail) { 795 *(__get_cpu_var(tasklet_vec).tail) = per_cpu(tasklet_vec, cpu).head; 796 __get_cpu_var(tasklet_vec).tail = per_cpu(tasklet_vec, cpu).tail; 797 per_cpu(tasklet_vec, cpu).head = NULL; 798 per_cpu(tasklet_vec, cpu).tail = &per_cpu(tasklet_vec, cpu).head; 799 } 800 raise_softirq_irqoff(TASKLET_SOFTIRQ); 801 802 if (&per_cpu(tasklet_hi_vec, cpu).head != per_cpu(tasklet_hi_vec, cpu).tail) { 803 *__get_cpu_var(tasklet_hi_vec).tail = per_cpu(tasklet_hi_vec, cpu).head; 804 __get_cpu_var(tasklet_hi_vec).tail = per_cpu(tasklet_hi_vec, cpu).tail; 805 per_cpu(tasklet_hi_vec, cpu).head = NULL; 806 per_cpu(tasklet_hi_vec, cpu).tail = &per_cpu(tasklet_hi_vec, cpu).head; 807 } 808 raise_softirq_irqoff(HI_SOFTIRQ); 809 810 local_irq_enable(); 811} 812#endif /* CONFIG_HOTPLUG_CPU */ 813 814static int __cpuinit cpu_callback(struct notifier_block *nfb, 815 unsigned long action, 816 void *hcpu) 817{ 818 int hotcpu = (unsigned long)hcpu; 819 struct task_struct *p; 820 821 switch (action) { 822 case CPU_UP_PREPARE: 823 case CPU_UP_PREPARE_FROZEN: 824 p = kthread_create(run_ksoftirqd, hcpu, "ksoftirqd/%d", hotcpu); 825 if (IS_ERR(p)) { 826 printk("ksoftirqd for %i failed\n", hotcpu); 827 return notifier_from_errno(PTR_ERR(p)); 828 } 829 kthread_bind(p, hotcpu); 830 per_cpu(ksoftirqd, hotcpu) = p; 831 break; 832 case CPU_ONLINE: 833 case CPU_ONLINE_FROZEN: 834 wake_up_process(per_cpu(ksoftirqd, hotcpu)); 835 break; 836#ifdef CONFIG_HOTPLUG_CPU 837 case CPU_UP_CANCELED: 838 case CPU_UP_CANCELED_FROZEN: 839 if (!per_cpu(ksoftirqd, hotcpu)) 840 break; 841 /* Unbind so it can run. Fall thru. */ 842 kthread_bind(per_cpu(ksoftirqd, hotcpu), 843 cpumask_any(cpu_online_mask)); 844 case CPU_DEAD: 845 case CPU_DEAD_FROZEN: { 846 struct sched_param param = { .sched_priority = MAX_RT_PRIO-1 }; 847 848 p = per_cpu(ksoftirqd, hotcpu); 849 per_cpu(ksoftirqd, hotcpu) = NULL; 850 sched_setscheduler_nocheck(p, SCHED_FIFO, ¶m); 851 kthread_stop(p); 852 takeover_tasklets(hotcpu); 853 break; 854 } 855#endif /* CONFIG_HOTPLUG_CPU */ 856 } 857 return NOTIFY_OK; 858} 859 860static struct notifier_block __cpuinitdata cpu_nfb = { 861 .notifier_call = cpu_callback 862}; 863 864static __init int spawn_ksoftirqd(void) 865{ 866 void *cpu = (void *)(long)smp_processor_id(); 867 int err = cpu_callback(&cpu_nfb, CPU_UP_PREPARE, cpu); 868 869 BUG_ON(err != NOTIFY_OK); 870 cpu_callback(&cpu_nfb, CPU_ONLINE, cpu); 871 register_cpu_notifier(&cpu_nfb); 872 return 0; 873} 874early_initcall(spawn_ksoftirqd); 875 876#ifdef CONFIG_SMP 877/* 878 * Call a function on all processors 879 */ 880int on_each_cpu(void (*func) (void *info), void *info, int wait) 881{ 882 int ret = 0; 883 884 preempt_disable(); 885 ret = smp_call_function(func, info, wait); 886 local_irq_disable(); 887 func(info); 888 local_irq_enable(); 889 preempt_enable(); 890 return ret; 891} 892EXPORT_SYMBOL(on_each_cpu); 893#endif 894 895/* 896 * [ These __weak aliases are kept in a separate compilation unit, so that 897 * GCC does not inline them incorrectly. ] 898 */ 899 900int __init __weak early_irq_init(void) 901{ 902 return 0; 903} 904 905int __init __weak arch_probe_nr_irqs(void) 906{ 907 return 0; 908} 909 910int __init __weak arch_early_irq_init(void) 911{ 912 return 0; 913} 914 915int __weak arch_init_chip_data(struct irq_desc *desc, int node) 916{ 917 return 0; 918} 919