1/* 2 * I/O SAPIC support. 3 * 4 * Copyright (C) 1999 Intel Corp. 5 * Copyright (C) 1999 Asit Mallick <asit.k.mallick@intel.com> 6 * Copyright (C) 2000-2002 J.I. Lee <jung-ik.lee@intel.com> 7 * Copyright (C) 1999-2000, 2002-2003 Hewlett-Packard Co. 8 * David Mosberger-Tang <davidm@hpl.hp.com> 9 * Copyright (C) 1999 VA Linux Systems 10 * Copyright (C) 1999,2000 Walt Drummond <drummond@valinux.com> 11 * 12 * 00/04/19 D. Mosberger Rewritten to mirror more closely the x86 I/O 13 * APIC code. In particular, we now have separate 14 * handlers for edge and level triggered 15 * interrupts. 16 * 00/10/27 Asit Mallick, Goutham Rao <goutham.rao@intel.com> IRQ vector 17 * allocation PCI to vector mapping, shared PCI 18 * interrupts. 19 * 00/10/27 D. Mosberger Document things a bit more to make them more 20 * understandable. Clean up much of the old 21 * IOSAPIC cruft. 22 * 01/07/27 J.I. Lee PCI irq routing, Platform/Legacy interrupts 23 * and fixes for ACPI S5(SoftOff) support. 24 * 02/01/23 J.I. Lee iosapic pgm fixes for PCI irq routing from _PRT 25 * 02/01/07 E. Focht <efocht@ess.nec.de> Redirectable interrupt 26 * vectors in iosapic_set_affinity(), 27 * initializations for /proc/irq/#/smp_affinity 28 * 02/04/02 P. Diefenbaugh Cleaned up ACPI PCI IRQ routing. 29 * 02/04/18 J.I. Lee bug fix in iosapic_init_pci_irq 30 * 02/04/30 J.I. Lee bug fix in find_iosapic to fix ACPI PCI IRQ to 31 * IOSAPIC mapping error 32 * 02/07/29 T. Kochi Allocate interrupt vectors dynamically 33 * 02/08/04 T. Kochi Cleaned up terminology (irq, global system 34 * interrupt, vector, etc.) 35 * 02/09/20 D. Mosberger Simplified by taking advantage of ACPI's 36 * pci_irq code. 37 * 03/02/19 B. Helgaas Make pcat_compat system-wide, not per-IOSAPIC. 38 * Remove iosapic_address & gsi_base from 39 * external interfaces. Rationalize 40 * __init/__devinit attributes. 41 * 04/12/04 Ashok Raj <ashok.raj@intel.com> Intel Corporation 2004 42 * Updated to work with irq migration necessary 43 * for CPU Hotplug 44 */ 45/* 46 * Here is what the interrupt logic between a PCI device and the kernel looks 47 * like: 48 * 49 * (1) A PCI device raises one of the four interrupt pins (INTA, INTB, INTC, 50 * INTD). The device is uniquely identified by its bus-, and slot-number 51 * (the function number does not matter here because all functions share 52 * the same interrupt lines). 53 * 54 * (2) The motherboard routes the interrupt line to a pin on a IOSAPIC 55 * controller. Multiple interrupt lines may have to share the same 56 * IOSAPIC pin (if they're level triggered and use the same polarity). 57 * Each interrupt line has a unique Global System Interrupt (GSI) number 58 * which can be calculated as the sum of the controller's base GSI number 59 * and the IOSAPIC pin number to which the line connects. 60 * 61 * (3) The IOSAPIC uses an internal routing table entries (RTEs) to map the 62 * IOSAPIC pin into the IA-64 interrupt vector. This interrupt vector is then 63 * sent to the CPU. 64 * 65 * (4) The kernel recognizes an interrupt as an IRQ. The IRQ interface is 66 * used as architecture-independent interrupt handling mechanism in Linux. 67 * As an IRQ is a number, we have to have 68 * IA-64 interrupt vector number <-> IRQ number mapping. On smaller 69 * systems, we use one-to-one mapping between IA-64 vector and IRQ. A 70 * platform can implement platform_irq_to_vector(irq) and 71 * platform_local_vector_to_irq(vector) APIs to differentiate the mapping. 72 * Please see also arch/ia64/include/asm/hw_irq.h for those APIs. 73 * 74 * To sum up, there are three levels of mappings involved: 75 * 76 * PCI pin -> global system interrupt (GSI) -> IA-64 vector <-> IRQ 77 * 78 * Note: The term "IRQ" is loosely used everywhere in Linux kernel to 79 * describeinterrupts. Now we use "IRQ" only for Linux IRQ's. ISA IRQ 80 * (isa_irq) is the only exception in this source code. 81 */ 82 83#include <linux/acpi.h> 84#include <linux/init.h> 85#include <linux/irq.h> 86#include <linux/kernel.h> 87#include <linux/list.h> 88#include <linux/pci.h> 89#include <linux/slab.h> 90#include <linux/smp.h> 91#include <linux/string.h> 92#include <linux/bootmem.h> 93 94#include <asm/delay.h> 95#include <asm/hw_irq.h> 96#include <asm/io.h> 97#include <asm/iosapic.h> 98#include <asm/machvec.h> 99#include <asm/processor.h> 100#include <asm/ptrace.h> 101#include <asm/system.h> 102 103#undef DEBUG_INTERRUPT_ROUTING 104 105#ifdef DEBUG_INTERRUPT_ROUTING 106#define DBG(fmt...) printk(fmt) 107#else 108#define DBG(fmt...) 109#endif 110 111#define NR_PREALLOCATE_RTE_ENTRIES \ 112 (PAGE_SIZE / sizeof(struct iosapic_rte_info)) 113#define RTE_PREALLOCATED (1) 114 115static DEFINE_SPINLOCK(iosapic_lock); 116 117/* 118 * These tables map IA-64 vectors to the IOSAPIC pin that generates this 119 * vector. 120 */ 121 122#define NO_REF_RTE 0 123 124static struct iosapic { 125 char __iomem *addr; /* base address of IOSAPIC */ 126 unsigned int gsi_base; /* GSI base */ 127 unsigned short num_rte; /* # of RTEs on this IOSAPIC */ 128 int rtes_inuse; /* # of RTEs in use on this IOSAPIC */ 129#ifdef CONFIG_NUMA 130 unsigned short node; /* numa node association via pxm */ 131#endif 132 spinlock_t lock; /* lock for indirect reg access */ 133} iosapic_lists[NR_IOSAPICS]; 134 135struct iosapic_rte_info { 136 struct list_head rte_list; /* RTEs sharing the same vector */ 137 char rte_index; /* IOSAPIC RTE index */ 138 int refcnt; /* reference counter */ 139 unsigned int flags; /* flags */ 140 struct iosapic *iosapic; 141} ____cacheline_aligned; 142 143static struct iosapic_intr_info { 144 struct list_head rtes; /* RTEs using this vector (empty => 145 * not an IOSAPIC interrupt) */ 146 int count; /* # of registered RTEs */ 147 u32 low32; /* current value of low word of 148 * Redirection table entry */ 149 unsigned int dest; /* destination CPU physical ID */ 150 unsigned char dmode : 3; /* delivery mode (see iosapic.h) */ 151 unsigned char polarity: 1; /* interrupt polarity 152 * (see iosapic.h) */ 153 unsigned char trigger : 1; /* trigger mode (see iosapic.h) */ 154} iosapic_intr_info[NR_IRQS]; 155 156static unsigned char pcat_compat __devinitdata; /* 8259 compatibility flag */ 157 158static int iosapic_kmalloc_ok; 159static LIST_HEAD(free_rte_list); 160 161static inline void 162iosapic_write(struct iosapic *iosapic, unsigned int reg, u32 val) 163{ 164 unsigned long flags; 165 166 spin_lock_irqsave(&iosapic->lock, flags); 167 __iosapic_write(iosapic->addr, reg, val); 168 spin_unlock_irqrestore(&iosapic->lock, flags); 169} 170 171/* 172 * Find an IOSAPIC associated with a GSI 173 */ 174static inline int 175find_iosapic (unsigned int gsi) 176{ 177 int i; 178 179 for (i = 0; i < NR_IOSAPICS; i++) { 180 if ((unsigned) (gsi - iosapic_lists[i].gsi_base) < 181 iosapic_lists[i].num_rte) 182 return i; 183 } 184 185 return -1; 186} 187 188static inline int __gsi_to_irq(unsigned int gsi) 189{ 190 int irq; 191 struct iosapic_intr_info *info; 192 struct iosapic_rte_info *rte; 193 194 for (irq = 0; irq < NR_IRQS; irq++) { 195 info = &iosapic_intr_info[irq]; 196 list_for_each_entry(rte, &info->rtes, rte_list) 197 if (rte->iosapic->gsi_base + rte->rte_index == gsi) 198 return irq; 199 } 200 return -1; 201} 202 203int 204gsi_to_irq (unsigned int gsi) 205{ 206 unsigned long flags; 207 int irq; 208 209 spin_lock_irqsave(&iosapic_lock, flags); 210 irq = __gsi_to_irq(gsi); 211 spin_unlock_irqrestore(&iosapic_lock, flags); 212 return irq; 213} 214 215static struct iosapic_rte_info *find_rte(unsigned int irq, unsigned int gsi) 216{ 217 struct iosapic_rte_info *rte; 218 219 list_for_each_entry(rte, &iosapic_intr_info[irq].rtes, rte_list) 220 if (rte->iosapic->gsi_base + rte->rte_index == gsi) 221 return rte; 222 return NULL; 223} 224 225static void 226set_rte (unsigned int gsi, unsigned int irq, unsigned int dest, int mask) 227{ 228 unsigned long pol, trigger, dmode; 229 u32 low32, high32; 230 int rte_index; 231 char redir; 232 struct iosapic_rte_info *rte; 233 ia64_vector vector = irq_to_vector(irq); 234 235 DBG(KERN_DEBUG"IOSAPIC: routing vector %d to 0x%x\n", vector, dest); 236 237 rte = find_rte(irq, gsi); 238 if (!rte) 239 return; /* not an IOSAPIC interrupt */ 240 241 rte_index = rte->rte_index; 242 pol = iosapic_intr_info[irq].polarity; 243 trigger = iosapic_intr_info[irq].trigger; 244 dmode = iosapic_intr_info[irq].dmode; 245 246 redir = (dmode == IOSAPIC_LOWEST_PRIORITY) ? 1 : 0; 247 248#ifdef CONFIG_SMP 249 set_irq_affinity_info(irq, (int)(dest & 0xffff), redir); 250#endif 251 252 low32 = ((pol << IOSAPIC_POLARITY_SHIFT) | 253 (trigger << IOSAPIC_TRIGGER_SHIFT) | 254 (dmode << IOSAPIC_DELIVERY_SHIFT) | 255 ((mask ? 1 : 0) << IOSAPIC_MASK_SHIFT) | 256 vector); 257 258 /* dest contains both id and eid */ 259 high32 = (dest << IOSAPIC_DEST_SHIFT); 260 261 iosapic_write(rte->iosapic, IOSAPIC_RTE_HIGH(rte_index), high32); 262 iosapic_write(rte->iosapic, IOSAPIC_RTE_LOW(rte_index), low32); 263 iosapic_intr_info[irq].low32 = low32; 264 iosapic_intr_info[irq].dest = dest; 265} 266 267static void 268nop (unsigned int irq) 269{ 270 /* do nothing... */ 271} 272 273 274#ifdef CONFIG_KEXEC 275void 276kexec_disable_iosapic(void) 277{ 278 struct iosapic_intr_info *info; 279 struct iosapic_rte_info *rte; 280 ia64_vector vec; 281 int irq; 282 283 for (irq = 0; irq < NR_IRQS; irq++) { 284 info = &iosapic_intr_info[irq]; 285 vec = irq_to_vector(irq); 286 list_for_each_entry(rte, &info->rtes, 287 rte_list) { 288 iosapic_write(rte->iosapic, 289 IOSAPIC_RTE_LOW(rte->rte_index), 290 IOSAPIC_MASK|vec); 291 iosapic_eoi(rte->iosapic->addr, vec); 292 } 293 } 294} 295#endif 296 297static void 298mask_irq (unsigned int irq) 299{ 300 u32 low32; 301 int rte_index; 302 struct iosapic_rte_info *rte; 303 304 if (!iosapic_intr_info[irq].count) 305 return; /* not an IOSAPIC interrupt! */ 306 307 /* set only the mask bit */ 308 low32 = iosapic_intr_info[irq].low32 |= IOSAPIC_MASK; 309 list_for_each_entry(rte, &iosapic_intr_info[irq].rtes, rte_list) { 310 rte_index = rte->rte_index; 311 iosapic_write(rte->iosapic, IOSAPIC_RTE_LOW(rte_index), low32); 312 } 313} 314 315static void 316unmask_irq (unsigned int irq) 317{ 318 u32 low32; 319 int rte_index; 320 struct iosapic_rte_info *rte; 321 322 if (!iosapic_intr_info[irq].count) 323 return; /* not an IOSAPIC interrupt! */ 324 325 low32 = iosapic_intr_info[irq].low32 &= ~IOSAPIC_MASK; 326 list_for_each_entry(rte, &iosapic_intr_info[irq].rtes, rte_list) { 327 rte_index = rte->rte_index; 328 iosapic_write(rte->iosapic, IOSAPIC_RTE_LOW(rte_index), low32); 329 } 330} 331 332 333static int 334iosapic_set_affinity(unsigned int irq, const struct cpumask *mask) 335{ 336#ifdef CONFIG_SMP 337 u32 high32, low32; 338 int cpu, dest, rte_index; 339 int redir = (irq & IA64_IRQ_REDIRECTED) ? 1 : 0; 340 struct iosapic_rte_info *rte; 341 struct iosapic *iosapic; 342 343 irq &= (~IA64_IRQ_REDIRECTED); 344 345 cpu = cpumask_first_and(cpu_online_mask, mask); 346 if (cpu >= nr_cpu_ids) 347 return -1; 348 349 if (irq_prepare_move(irq, cpu)) 350 return -1; 351 352 dest = cpu_physical_id(cpu); 353 354 if (!iosapic_intr_info[irq].count) 355 return -1; /* not an IOSAPIC interrupt */ 356 357 set_irq_affinity_info(irq, dest, redir); 358 359 /* dest contains both id and eid */ 360 high32 = dest << IOSAPIC_DEST_SHIFT; 361 362 low32 = iosapic_intr_info[irq].low32 & ~(7 << IOSAPIC_DELIVERY_SHIFT); 363 if (redir) 364 /* change delivery mode to lowest priority */ 365 low32 |= (IOSAPIC_LOWEST_PRIORITY << IOSAPIC_DELIVERY_SHIFT); 366 else 367 /* change delivery mode to fixed */ 368 low32 |= (IOSAPIC_FIXED << IOSAPIC_DELIVERY_SHIFT); 369 low32 &= IOSAPIC_VECTOR_MASK; 370 low32 |= irq_to_vector(irq); 371 372 iosapic_intr_info[irq].low32 = low32; 373 iosapic_intr_info[irq].dest = dest; 374 list_for_each_entry(rte, &iosapic_intr_info[irq].rtes, rte_list) { 375 iosapic = rte->iosapic; 376 rte_index = rte->rte_index; 377 iosapic_write(iosapic, IOSAPIC_RTE_HIGH(rte_index), high32); 378 iosapic_write(iosapic, IOSAPIC_RTE_LOW(rte_index), low32); 379 } 380 381#endif 382 return 0; 383} 384 385/* 386 * Handlers for level-triggered interrupts. 387 */ 388 389static unsigned int 390iosapic_startup_level_irq (unsigned int irq) 391{ 392 unmask_irq(irq); 393 return 0; 394} 395 396static void 397iosapic_end_level_irq (unsigned int irq) 398{ 399 ia64_vector vec = irq_to_vector(irq); 400 struct iosapic_rte_info *rte; 401 int do_unmask_irq = 0; 402 403 irq_complete_move(irq); 404 if (unlikely(irq_desc[irq].status & IRQ_MOVE_PENDING)) { 405 do_unmask_irq = 1; 406 mask_irq(irq); 407 } 408 409 list_for_each_entry(rte, &iosapic_intr_info[irq].rtes, rte_list) 410 iosapic_eoi(rte->iosapic->addr, vec); 411 412 if (unlikely(do_unmask_irq)) { 413 move_masked_irq(irq); 414 unmask_irq(irq); 415 } 416} 417 418#define iosapic_shutdown_level_irq mask_irq 419#define iosapic_enable_level_irq unmask_irq 420#define iosapic_disable_level_irq mask_irq 421#define iosapic_ack_level_irq nop 422 423static struct irq_chip irq_type_iosapic_level = { 424 .name = "IO-SAPIC-level", 425 .startup = iosapic_startup_level_irq, 426 .shutdown = iosapic_shutdown_level_irq, 427 .enable = iosapic_enable_level_irq, 428 .disable = iosapic_disable_level_irq, 429 .ack = iosapic_ack_level_irq, 430 .end = iosapic_end_level_irq, 431 .mask = mask_irq, 432 .unmask = unmask_irq, 433 .set_affinity = iosapic_set_affinity 434}; 435 436/* 437 * Handlers for edge-triggered interrupts. 438 */ 439 440static unsigned int 441iosapic_startup_edge_irq (unsigned int irq) 442{ 443 unmask_irq(irq); 444 /* 445 * IOSAPIC simply drops interrupts pended while the 446 * corresponding pin was masked, so we can't know if an 447 * interrupt is pending already. Let's hope not... 448 */ 449 return 0; 450} 451 452static void 453iosapic_ack_edge_irq (unsigned int irq) 454{ 455 struct irq_desc *idesc = irq_desc + irq; 456 457 irq_complete_move(irq); 458 move_native_irq(irq); 459 /* 460 * Once we have recorded IRQ_PENDING already, we can mask the 461 * interrupt for real. This prevents IRQ storms from unhandled 462 * devices. 463 */ 464 if ((idesc->status & (IRQ_PENDING|IRQ_DISABLED)) == 465 (IRQ_PENDING|IRQ_DISABLED)) 466 mask_irq(irq); 467} 468 469#define iosapic_enable_edge_irq unmask_irq 470#define iosapic_disable_edge_irq nop 471#define iosapic_end_edge_irq nop 472 473static struct irq_chip irq_type_iosapic_edge = { 474 .name = "IO-SAPIC-edge", 475 .startup = iosapic_startup_edge_irq, 476 .shutdown = iosapic_disable_edge_irq, 477 .enable = iosapic_enable_edge_irq, 478 .disable = iosapic_disable_edge_irq, 479 .ack = iosapic_ack_edge_irq, 480 .end = iosapic_end_edge_irq, 481 .mask = mask_irq, 482 .unmask = unmask_irq, 483 .set_affinity = iosapic_set_affinity 484}; 485 486static unsigned int 487iosapic_version (char __iomem *addr) 488{ 489 /* 490 * IOSAPIC Version Register return 32 bit structure like: 491 * { 492 * unsigned int version : 8; 493 * unsigned int reserved1 : 8; 494 * unsigned int max_redir : 8; 495 * unsigned int reserved2 : 8; 496 * } 497 */ 498 return __iosapic_read(addr, IOSAPIC_VERSION); 499} 500 501static int iosapic_find_sharable_irq(unsigned long trigger, unsigned long pol) 502{ 503 int i, irq = -ENOSPC, min_count = -1; 504 struct iosapic_intr_info *info; 505 506 /* 507 * shared vectors for edge-triggered interrupts are not 508 * supported yet 509 */ 510 if (trigger == IOSAPIC_EDGE) 511 return -EINVAL; 512 513 for (i = 0; i < NR_IRQS; i++) { 514 info = &iosapic_intr_info[i]; 515 if (info->trigger == trigger && info->polarity == pol && 516 (info->dmode == IOSAPIC_FIXED || 517 info->dmode == IOSAPIC_LOWEST_PRIORITY) && 518 can_request_irq(i, IRQF_SHARED)) { 519 if (min_count == -1 || info->count < min_count) { 520 irq = i; 521 min_count = info->count; 522 } 523 } 524 } 525 return irq; 526} 527 528/* 529 * if the given vector is already owned by other, 530 * assign a new vector for the other and make the vector available 531 */ 532static void __init 533iosapic_reassign_vector (int irq) 534{ 535 int new_irq; 536 537 if (iosapic_intr_info[irq].count) { 538 new_irq = create_irq(); 539 if (new_irq < 0) 540 panic("%s: out of interrupt vectors!\n", __func__); 541 printk(KERN_INFO "Reassigning vector %d to %d\n", 542 irq_to_vector(irq), irq_to_vector(new_irq)); 543 memcpy(&iosapic_intr_info[new_irq], &iosapic_intr_info[irq], 544 sizeof(struct iosapic_intr_info)); 545 INIT_LIST_HEAD(&iosapic_intr_info[new_irq].rtes); 546 list_move(iosapic_intr_info[irq].rtes.next, 547 &iosapic_intr_info[new_irq].rtes); 548 memset(&iosapic_intr_info[irq], 0, 549 sizeof(struct iosapic_intr_info)); 550 iosapic_intr_info[irq].low32 = IOSAPIC_MASK; 551 INIT_LIST_HEAD(&iosapic_intr_info[irq].rtes); 552 } 553} 554 555static struct iosapic_rte_info * __init_refok iosapic_alloc_rte (void) 556{ 557 int i; 558 struct iosapic_rte_info *rte; 559 int preallocated = 0; 560 561 if (!iosapic_kmalloc_ok && list_empty(&free_rte_list)) { 562 rte = alloc_bootmem(sizeof(struct iosapic_rte_info) * 563 NR_PREALLOCATE_RTE_ENTRIES); 564 for (i = 0; i < NR_PREALLOCATE_RTE_ENTRIES; i++, rte++) 565 list_add(&rte->rte_list, &free_rte_list); 566 } 567 568 if (!list_empty(&free_rte_list)) { 569 rte = list_entry(free_rte_list.next, struct iosapic_rte_info, 570 rte_list); 571 list_del(&rte->rte_list); 572 preallocated++; 573 } else { 574 rte = kmalloc(sizeof(struct iosapic_rte_info), GFP_ATOMIC); 575 if (!rte) 576 return NULL; 577 } 578 579 memset(rte, 0, sizeof(struct iosapic_rte_info)); 580 if (preallocated) 581 rte->flags |= RTE_PREALLOCATED; 582 583 return rte; 584} 585 586static inline int irq_is_shared (int irq) 587{ 588 return (iosapic_intr_info[irq].count > 1); 589} 590 591struct irq_chip* 592ia64_native_iosapic_get_irq_chip(unsigned long trigger) 593{ 594 if (trigger == IOSAPIC_EDGE) 595 return &irq_type_iosapic_edge; 596 else 597 return &irq_type_iosapic_level; 598} 599 600static int 601register_intr (unsigned int gsi, int irq, unsigned char delivery, 602 unsigned long polarity, unsigned long trigger) 603{ 604 struct irq_desc *idesc; 605 struct irq_chip *irq_type; 606 int index; 607 struct iosapic_rte_info *rte; 608 609 index = find_iosapic(gsi); 610 if (index < 0) { 611 printk(KERN_WARNING "%s: No IOSAPIC for GSI %u\n", 612 __func__, gsi); 613 return -ENODEV; 614 } 615 616 rte = find_rte(irq, gsi); 617 if (!rte) { 618 rte = iosapic_alloc_rte(); 619 if (!rte) { 620 printk(KERN_WARNING "%s: cannot allocate memory\n", 621 __func__); 622 return -ENOMEM; 623 } 624 625 rte->iosapic = &iosapic_lists[index]; 626 rte->rte_index = gsi - rte->iosapic->gsi_base; 627 rte->refcnt++; 628 list_add_tail(&rte->rte_list, &iosapic_intr_info[irq].rtes); 629 iosapic_intr_info[irq].count++; 630 iosapic_lists[index].rtes_inuse++; 631 } 632 else if (rte->refcnt == NO_REF_RTE) { 633 struct iosapic_intr_info *info = &iosapic_intr_info[irq]; 634 if (info->count > 0 && 635 (info->trigger != trigger || info->polarity != polarity)){ 636 printk (KERN_WARNING 637 "%s: cannot override the interrupt\n", 638 __func__); 639 return -EINVAL; 640 } 641 rte->refcnt++; 642 iosapic_intr_info[irq].count++; 643 iosapic_lists[index].rtes_inuse++; 644 } 645 646 iosapic_intr_info[irq].polarity = polarity; 647 iosapic_intr_info[irq].dmode = delivery; 648 iosapic_intr_info[irq].trigger = trigger; 649 650 irq_type = iosapic_get_irq_chip(trigger); 651 652 idesc = irq_desc + irq; 653 if (irq_type != NULL && idesc->chip != irq_type) { 654 if (idesc->chip != &no_irq_chip) 655 printk(KERN_WARNING 656 "%s: changing vector %d from %s to %s\n", 657 __func__, irq_to_vector(irq), 658 idesc->chip->name, irq_type->name); 659 idesc->chip = irq_type; 660 } 661 return 0; 662} 663 664static unsigned int 665get_target_cpu (unsigned int gsi, int irq) 666{ 667#ifdef CONFIG_SMP 668 static int cpu = -1; 669 extern int cpe_vector; 670 cpumask_t domain = irq_to_domain(irq); 671 672 /* 673 * In case of vector shared by multiple RTEs, all RTEs that 674 * share the vector need to use the same destination CPU. 675 */ 676 if (iosapic_intr_info[irq].count) 677 return iosapic_intr_info[irq].dest; 678 679 /* 680 * If the platform supports redirection via XTP, let it 681 * distribute interrupts. 682 */ 683 if (smp_int_redirect & SMP_IRQ_REDIRECTION) 684 return cpu_physical_id(smp_processor_id()); 685 686 /* 687 * Some interrupts (ACPI SCI, for instance) are registered 688 * before the BSP is marked as online. 689 */ 690 if (!cpu_online(smp_processor_id())) 691 return cpu_physical_id(smp_processor_id()); 692 693#ifdef CONFIG_ACPI 694 if (cpe_vector > 0 && irq_to_vector(irq) == IA64_CPEP_VECTOR) 695 return get_cpei_target_cpu(); 696#endif 697 698#ifdef CONFIG_NUMA 699 { 700 int num_cpus, cpu_index, iosapic_index, numa_cpu, i = 0; 701 const struct cpumask *cpu_mask; 702 703 iosapic_index = find_iosapic(gsi); 704 if (iosapic_index < 0 || 705 iosapic_lists[iosapic_index].node == MAX_NUMNODES) 706 goto skip_numa_setup; 707 708 cpu_mask = cpumask_of_node(iosapic_lists[iosapic_index].node); 709 num_cpus = 0; 710 for_each_cpu_and(numa_cpu, cpu_mask, &domain) { 711 if (cpu_online(numa_cpu)) 712 num_cpus++; 713 } 714 715 if (!num_cpus) 716 goto skip_numa_setup; 717 718 /* Use irq assignment to distribute across cpus in node */ 719 cpu_index = irq % num_cpus; 720 721 for_each_cpu_and(numa_cpu, cpu_mask, &domain) 722 if (cpu_online(numa_cpu) && i++ >= cpu_index) 723 break; 724 725 if (numa_cpu < nr_cpu_ids) 726 return cpu_physical_id(numa_cpu); 727 } 728skip_numa_setup: 729#endif 730 /* 731 * Otherwise, round-robin interrupt vectors across all the 732 * processors. (It'd be nice if we could be smarter in the 733 * case of NUMA.) 734 */ 735 do { 736 if (++cpu >= nr_cpu_ids) 737 cpu = 0; 738 } while (!cpu_online(cpu) || !cpu_isset(cpu, domain)); 739 740 return cpu_physical_id(cpu); 741#else /* CONFIG_SMP */ 742 return cpu_physical_id(smp_processor_id()); 743#endif 744} 745 746static inline unsigned char choose_dmode(void) 747{ 748#ifdef CONFIG_SMP 749 if (smp_int_redirect & SMP_IRQ_REDIRECTION) 750 return IOSAPIC_LOWEST_PRIORITY; 751#endif 752 return IOSAPIC_FIXED; 753} 754 755/* 756 * ACPI can describe IOSAPIC interrupts via static tables and namespace 757 * methods. This provides an interface to register those interrupts and 758 * program the IOSAPIC RTE. 759 */ 760int 761iosapic_register_intr (unsigned int gsi, 762 unsigned long polarity, unsigned long trigger) 763{ 764 int irq, mask = 1, err; 765 unsigned int dest; 766 unsigned long flags; 767 struct iosapic_rte_info *rte; 768 u32 low32; 769 unsigned char dmode; 770 771 /* 772 * If this GSI has already been registered (i.e., it's a 773 * shared interrupt, or we lost a race to register it), 774 * don't touch the RTE. 775 */ 776 spin_lock_irqsave(&iosapic_lock, flags); 777 irq = __gsi_to_irq(gsi); 778 if (irq > 0) { 779 rte = find_rte(irq, gsi); 780 if(iosapic_intr_info[irq].count == 0) { 781 assign_irq_vector(irq); 782 dynamic_irq_init(irq); 783 } else if (rte->refcnt != NO_REF_RTE) { 784 rte->refcnt++; 785 goto unlock_iosapic_lock; 786 } 787 } else 788 irq = create_irq(); 789 790 /* If vector is running out, we try to find a sharable vector */ 791 if (irq < 0) { 792 irq = iosapic_find_sharable_irq(trigger, polarity); 793 if (irq < 0) 794 goto unlock_iosapic_lock; 795 } 796 797 raw_spin_lock(&irq_desc[irq].lock); 798 dest = get_target_cpu(gsi, irq); 799 dmode = choose_dmode(); 800 err = register_intr(gsi, irq, dmode, polarity, trigger); 801 if (err < 0) { 802 raw_spin_unlock(&irq_desc[irq].lock); 803 irq = err; 804 goto unlock_iosapic_lock; 805 } 806 807 /* 808 * If the vector is shared and already unmasked for other 809 * interrupt sources, don't mask it. 810 */ 811 low32 = iosapic_intr_info[irq].low32; 812 if (irq_is_shared(irq) && !(low32 & IOSAPIC_MASK)) 813 mask = 0; 814 set_rte(gsi, irq, dest, mask); 815 816 printk(KERN_INFO "GSI %u (%s, %s) -> CPU %d (0x%04x) vector %d\n", 817 gsi, (trigger == IOSAPIC_EDGE ? "edge" : "level"), 818 (polarity == IOSAPIC_POL_HIGH ? "high" : "low"), 819 cpu_logical_id(dest), dest, irq_to_vector(irq)); 820 821 raw_spin_unlock(&irq_desc[irq].lock); 822 unlock_iosapic_lock: 823 spin_unlock_irqrestore(&iosapic_lock, flags); 824 return irq; 825} 826 827void 828iosapic_unregister_intr (unsigned int gsi) 829{ 830 unsigned long flags; 831 int irq, index; 832 struct irq_desc *idesc; 833 u32 low32; 834 unsigned long trigger, polarity; 835 unsigned int dest; 836 struct iosapic_rte_info *rte; 837 838 /* 839 * If the irq associated with the gsi is not found, 840 * iosapic_unregister_intr() is unbalanced. We need to check 841 * this again after getting locks. 842 */ 843 irq = gsi_to_irq(gsi); 844 if (irq < 0) { 845 printk(KERN_ERR "iosapic_unregister_intr(%u) unbalanced\n", 846 gsi); 847 WARN_ON(1); 848 return; 849 } 850 851 spin_lock_irqsave(&iosapic_lock, flags); 852 if ((rte = find_rte(irq, gsi)) == NULL) { 853 printk(KERN_ERR "iosapic_unregister_intr(%u) unbalanced\n", 854 gsi); 855 WARN_ON(1); 856 goto out; 857 } 858 859 if (--rte->refcnt > 0) 860 goto out; 861 862 idesc = irq_desc + irq; 863 rte->refcnt = NO_REF_RTE; 864 865 /* Mask the interrupt */ 866 low32 = iosapic_intr_info[irq].low32 | IOSAPIC_MASK; 867 iosapic_write(rte->iosapic, IOSAPIC_RTE_LOW(rte->rte_index), low32); 868 869 iosapic_intr_info[irq].count--; 870 index = find_iosapic(gsi); 871 iosapic_lists[index].rtes_inuse--; 872 WARN_ON(iosapic_lists[index].rtes_inuse < 0); 873 874 trigger = iosapic_intr_info[irq].trigger; 875 polarity = iosapic_intr_info[irq].polarity; 876 dest = iosapic_intr_info[irq].dest; 877 printk(KERN_INFO 878 "GSI %u (%s, %s) -> CPU %d (0x%04x) vector %d unregistered\n", 879 gsi, (trigger == IOSAPIC_EDGE ? "edge" : "level"), 880 (polarity == IOSAPIC_POL_HIGH ? "high" : "low"), 881 cpu_logical_id(dest), dest, irq_to_vector(irq)); 882 883 if (iosapic_intr_info[irq].count == 0) { 884#ifdef CONFIG_SMP 885 /* Clear affinity */ 886 cpumask_setall(idesc->affinity); 887#endif 888 /* Clear the interrupt information */ 889 iosapic_intr_info[irq].dest = 0; 890 iosapic_intr_info[irq].dmode = 0; 891 iosapic_intr_info[irq].polarity = 0; 892 iosapic_intr_info[irq].trigger = 0; 893 iosapic_intr_info[irq].low32 |= IOSAPIC_MASK; 894 895 /* Destroy and reserve IRQ */ 896 destroy_and_reserve_irq(irq); 897 } 898 out: 899 spin_unlock_irqrestore(&iosapic_lock, flags); 900} 901 902/* 903 * ACPI calls this when it finds an entry for a platform interrupt. 904 */ 905int __init 906iosapic_register_platform_intr (u32 int_type, unsigned int gsi, 907 int iosapic_vector, u16 eid, u16 id, 908 unsigned long polarity, unsigned long trigger) 909{ 910 static const char * const name[] = {"unknown", "PMI", "INIT", "CPEI"}; 911 unsigned char delivery; 912 int irq, vector, mask = 0; 913 unsigned int dest = ((id << 8) | eid) & 0xffff; 914 915 switch (int_type) { 916 case ACPI_INTERRUPT_PMI: 917 irq = vector = iosapic_vector; 918 bind_irq_vector(irq, vector, CPU_MASK_ALL); 919 /* 920 * since PMI vector is alloc'd by FW(ACPI) not by kernel, 921 * we need to make sure the vector is available 922 */ 923 iosapic_reassign_vector(irq); 924 delivery = IOSAPIC_PMI; 925 break; 926 case ACPI_INTERRUPT_INIT: 927 irq = create_irq(); 928 if (irq < 0) 929 panic("%s: out of interrupt vectors!\n", __func__); 930 vector = irq_to_vector(irq); 931 delivery = IOSAPIC_INIT; 932 break; 933 case ACPI_INTERRUPT_CPEI: 934 irq = vector = IA64_CPE_VECTOR; 935 BUG_ON(bind_irq_vector(irq, vector, CPU_MASK_ALL)); 936 delivery = IOSAPIC_FIXED; 937 mask = 1; 938 break; 939 default: 940 printk(KERN_ERR "%s: invalid int type 0x%x\n", __func__, 941 int_type); 942 return -1; 943 } 944 945 register_intr(gsi, irq, delivery, polarity, trigger); 946 947 printk(KERN_INFO 948 "PLATFORM int %s (0x%x): GSI %u (%s, %s) -> CPU %d (0x%04x)" 949 " vector %d\n", 950 int_type < ARRAY_SIZE(name) ? name[int_type] : "unknown", 951 int_type, gsi, (trigger == IOSAPIC_EDGE ? "edge" : "level"), 952 (polarity == IOSAPIC_POL_HIGH ? "high" : "low"), 953 cpu_logical_id(dest), dest, vector); 954 955 set_rte(gsi, irq, dest, mask); 956 return vector; 957} 958 959/* 960 * ACPI calls this when it finds an entry for a legacy ISA IRQ override. 961 */ 962void __devinit 963iosapic_override_isa_irq (unsigned int isa_irq, unsigned int gsi, 964 unsigned long polarity, 965 unsigned long trigger) 966{ 967 int vector, irq; 968 unsigned int dest = cpu_physical_id(smp_processor_id()); 969 unsigned char dmode; 970 971 irq = vector = isa_irq_to_vector(isa_irq); 972 BUG_ON(bind_irq_vector(irq, vector, CPU_MASK_ALL)); 973 dmode = choose_dmode(); 974 register_intr(gsi, irq, dmode, polarity, trigger); 975 976 DBG("ISA: IRQ %u -> GSI %u (%s,%s) -> CPU %d (0x%04x) vector %d\n", 977 isa_irq, gsi, trigger == IOSAPIC_EDGE ? "edge" : "level", 978 polarity == IOSAPIC_POL_HIGH ? "high" : "low", 979 cpu_logical_id(dest), dest, vector); 980 981 set_rte(gsi, irq, dest, 1); 982} 983 984void __init 985ia64_native_iosapic_pcat_compat_init(void) 986{ 987 if (pcat_compat) { 988 /* 989 * Disable the compatibility mode interrupts (8259 style), 990 * needs IN/OUT support enabled. 991 */ 992 printk(KERN_INFO 993 "%s: Disabling PC-AT compatible 8259 interrupts\n", 994 __func__); 995 outb(0xff, 0xA1); 996 outb(0xff, 0x21); 997 } 998} 999 1000void __init 1001iosapic_system_init (int system_pcat_compat) 1002{ 1003 int irq; 1004 1005 for (irq = 0; irq < NR_IRQS; ++irq) { 1006 iosapic_intr_info[irq].low32 = IOSAPIC_MASK; 1007 /* mark as unused */ 1008 INIT_LIST_HEAD(&iosapic_intr_info[irq].rtes); 1009 1010 iosapic_intr_info[irq].count = 0; 1011 } 1012 1013 pcat_compat = system_pcat_compat; 1014 if (pcat_compat) 1015 iosapic_pcat_compat_init(); 1016} 1017 1018static inline int 1019iosapic_alloc (void) 1020{ 1021 int index; 1022 1023 for (index = 0; index < NR_IOSAPICS; index++) 1024 if (!iosapic_lists[index].addr) 1025 return index; 1026 1027 printk(KERN_WARNING "%s: failed to allocate iosapic\n", __func__); 1028 return -1; 1029} 1030 1031static inline void 1032iosapic_free (int index) 1033{ 1034 memset(&iosapic_lists[index], 0, sizeof(iosapic_lists[0])); 1035} 1036 1037static inline int 1038iosapic_check_gsi_range (unsigned int gsi_base, unsigned int ver) 1039{ 1040 int index; 1041 unsigned int gsi_end, base, end; 1042 1043 /* check gsi range */ 1044 gsi_end = gsi_base + ((ver >> 16) & 0xff); 1045 for (index = 0; index < NR_IOSAPICS; index++) { 1046 if (!iosapic_lists[index].addr) 1047 continue; 1048 1049 base = iosapic_lists[index].gsi_base; 1050 end = base + iosapic_lists[index].num_rte - 1; 1051 1052 if (gsi_end < base || end < gsi_base) 1053 continue; /* OK */ 1054 1055 return -EBUSY; 1056 } 1057 return 0; 1058} 1059 1060int __devinit 1061iosapic_init (unsigned long phys_addr, unsigned int gsi_base) 1062{ 1063 int num_rte, err, index; 1064 unsigned int isa_irq, ver; 1065 char __iomem *addr; 1066 unsigned long flags; 1067 1068 spin_lock_irqsave(&iosapic_lock, flags); 1069 index = find_iosapic(gsi_base); 1070 if (index >= 0) { 1071 spin_unlock_irqrestore(&iosapic_lock, flags); 1072 return -EBUSY; 1073 } 1074 1075 addr = ioremap(phys_addr, 0); 1076 if (addr == NULL) { 1077 spin_unlock_irqrestore(&iosapic_lock, flags); 1078 return -ENOMEM; 1079 } 1080 ver = iosapic_version(addr); 1081 if ((err = iosapic_check_gsi_range(gsi_base, ver))) { 1082 iounmap(addr); 1083 spin_unlock_irqrestore(&iosapic_lock, flags); 1084 return err; 1085 } 1086 1087 /* 1088 * The MAX_REDIR register holds the highest input pin number 1089 * (starting from 0). We add 1 so that we can use it for 1090 * number of pins (= RTEs) 1091 */ 1092 num_rte = ((ver >> 16) & 0xff) + 1; 1093 1094 index = iosapic_alloc(); 1095 iosapic_lists[index].addr = addr; 1096 iosapic_lists[index].gsi_base = gsi_base; 1097 iosapic_lists[index].num_rte = num_rte; 1098#ifdef CONFIG_NUMA 1099 iosapic_lists[index].node = MAX_NUMNODES; 1100#endif 1101 spin_lock_init(&iosapic_lists[index].lock); 1102 spin_unlock_irqrestore(&iosapic_lock, flags); 1103 1104 if ((gsi_base == 0) && pcat_compat) { 1105 /* 1106 * Map the legacy ISA devices into the IOSAPIC data. Some of 1107 * these may get reprogrammed later on with data from the ACPI 1108 * Interrupt Source Override table. 1109 */ 1110 for (isa_irq = 0; isa_irq < 16; ++isa_irq) 1111 iosapic_override_isa_irq(isa_irq, isa_irq, 1112 IOSAPIC_POL_HIGH, 1113 IOSAPIC_EDGE); 1114 } 1115 return 0; 1116} 1117 1118#ifdef CONFIG_HOTPLUG 1119int 1120iosapic_remove (unsigned int gsi_base) 1121{ 1122 int index, err = 0; 1123 unsigned long flags; 1124 1125 spin_lock_irqsave(&iosapic_lock, flags); 1126 index = find_iosapic(gsi_base); 1127 if (index < 0) { 1128 printk(KERN_WARNING "%s: No IOSAPIC for GSI base %u\n", 1129 __func__, gsi_base); 1130 goto out; 1131 } 1132 1133 if (iosapic_lists[index].rtes_inuse) { 1134 err = -EBUSY; 1135 printk(KERN_WARNING "%s: IOSAPIC for GSI base %u is busy\n", 1136 __func__, gsi_base); 1137 goto out; 1138 } 1139 1140 iounmap(iosapic_lists[index].addr); 1141 iosapic_free(index); 1142 out: 1143 spin_unlock_irqrestore(&iosapic_lock, flags); 1144 return err; 1145} 1146#endif /* CONFIG_HOTPLUG */ 1147 1148#ifdef CONFIG_NUMA 1149void __devinit 1150map_iosapic_to_node(unsigned int gsi_base, int node) 1151{ 1152 int index; 1153 1154 index = find_iosapic(gsi_base); 1155 if (index < 0) { 1156 printk(KERN_WARNING "%s: No IOSAPIC for GSI %u\n", 1157 __func__, gsi_base); 1158 return; 1159 } 1160 iosapic_lists[index].node = node; 1161 return; 1162} 1163#endif 1164 1165static int __init iosapic_enable_kmalloc (void) 1166{ 1167 iosapic_kmalloc_ok = 1; 1168 return 0; 1169} 1170core_initcall (iosapic_enable_kmalloc); 1171