1/* irq.c: UltraSparc IRQ handling/init/registry. 2 * 3 * Copyright (C) 1997, 2007 David S. Miller (davem@davemloft.net) 4 * Copyright (C) 1998 Eddie C. Dost (ecd@skynet.be) 5 * Copyright (C) 1998 Jakub Jelinek (jj@ultra.linux.cz) 6 */ 7 8#include <linux/module.h> 9#include <linux/sched.h> 10#include <linux/ptrace.h> 11#include <linux/errno.h> 12#include <linux/kernel_stat.h> 13#include <linux/signal.h> 14#include <linux/mm.h> 15#include <linux/interrupt.h> 16#include <linux/slab.h> 17#include <linux/random.h> 18#include <linux/init.h> 19#include <linux/delay.h> 20#include <linux/proc_fs.h> 21#include <linux/seq_file.h> 22#include <linux/bootmem.h> 23#include <linux/irq.h> 24#include <linux/msi.h> 25 26#include <asm/ptrace.h> 27#include <asm/processor.h> 28#include <asm/atomic.h> 29#include <asm/system.h> 30#include <asm/irq.h> 31#include <asm/io.h> 32#include <asm/sbus.h> 33#include <asm/iommu.h> 34#include <asm/upa.h> 35#include <asm/oplib.h> 36#include <asm/prom.h> 37#include <asm/timer.h> 38#include <asm/smp.h> 39#include <asm/starfire.h> 40#include <asm/uaccess.h> 41#include <asm/cache.h> 42#include <asm/cpudata.h> 43#include <asm/auxio.h> 44#include <asm/head.h> 45#include <asm/hypervisor.h> 46 47/* UPA nodes send interrupt packet to UltraSparc with first data reg 48 * value low 5 (7 on Starfire) bits holding the IRQ identifier being 49 * delivered. We must translate this into a non-vector IRQ so we can 50 * set the softint on this cpu. 51 * 52 * To make processing these packets efficient and race free we use 53 * an array of irq buckets below. The interrupt vector handler in 54 * entry.S feeds incoming packets into per-cpu pil-indexed lists. 55 * The IVEC handler does not need to act atomically, the PIL dispatch 56 * code uses CAS to get an atomic snapshot of the list and clear it 57 * at the same time. 58 * 59 * If you make changes to ino_bucket, please update hand coded assembler 60 * of the vectored interrupt trap handler(s) in entry.S and sun4v_ivec.S 61 */ 62struct ino_bucket { 63 /* Next handler in per-CPU IRQ worklist. We know that 64 * bucket pointers have the high 32-bits clear, so to 65 * save space we only store the bits we need. 66 */ 67/*0x00*/unsigned int irq_chain; 68 69 /* Virtual interrupt number assigned to this INO. */ 70/*0x04*/unsigned int virt_irq; 71}; 72 73#define NUM_IVECS (IMAP_INR + 1) 74struct ino_bucket ivector_table[NUM_IVECS] __attribute__ ((aligned (SMP_CACHE_BYTES))); 75 76#define __irq_ino(irq) \ 77 (((struct ino_bucket *)(unsigned long)(irq)) - &ivector_table[0]) 78#define __bucket(irq) ((struct ino_bucket *)(unsigned long)(irq)) 79#define __irq(bucket) ((unsigned int)(unsigned long)(bucket)) 80 81/* This has to be in the main kernel image, it cannot be 82 * turned into per-cpu data. The reason is that the main 83 * kernel image is locked into the TLB and this structure 84 * is accessed from the vectored interrupt trap handler. If 85 * access to this structure takes a TLB miss it could cause 86 * the 5-level sparc v9 trap stack to overflow. 87 */ 88#define irq_work(__cpu) &(trap_block[(__cpu)].irq_worklist) 89 90static unsigned int virt_to_real_irq_table[NR_IRQS]; 91 92static unsigned char virt_irq_alloc(unsigned int real_irq) 93{ 94 unsigned char ent; 95 96 BUILD_BUG_ON(NR_IRQS >= 256); 97 98 for (ent = 1; ent < NR_IRQS; ent++) { 99 if (!virt_to_real_irq_table[ent]) 100 break; 101 } 102 if (ent >= NR_IRQS) { 103 printk(KERN_ERR "IRQ: Out of virtual IRQs.\n"); 104 return 0; 105 } 106 107 virt_to_real_irq_table[ent] = real_irq; 108 109 return ent; 110} 111 112#ifdef CONFIG_PCI_MSI 113static void virt_irq_free(unsigned int virt_irq) 114{ 115 unsigned int real_irq; 116 117 if (virt_irq >= NR_IRQS) 118 return; 119 120 real_irq = virt_to_real_irq_table[virt_irq]; 121 virt_to_real_irq_table[virt_irq] = 0; 122 123 __bucket(real_irq)->virt_irq = 0; 124} 125#endif 126 127static unsigned int virt_to_real_irq(unsigned char virt_irq) 128{ 129 return virt_to_real_irq_table[virt_irq]; 130} 131 132/* 133 * /proc/interrupts printing: 134 */ 135 136int show_interrupts(struct seq_file *p, void *v) 137{ 138 int i = *(loff_t *) v, j; 139 struct irqaction * action; 140 unsigned long flags; 141 142 if (i == 0) { 143 seq_printf(p, " "); 144 for_each_online_cpu(j) 145 seq_printf(p, "CPU%d ",j); 146 seq_putc(p, '\n'); 147 } 148 149 if (i < NR_IRQS) { 150 spin_lock_irqsave(&irq_desc[i].lock, flags); 151 action = irq_desc[i].action; 152 if (!action) 153 goto skip; 154 seq_printf(p, "%3d: ",i); 155#ifndef CONFIG_SMP 156 seq_printf(p, "%10u ", kstat_irqs(i)); 157#else 158 for_each_online_cpu(j) 159 seq_printf(p, "%10u ", kstat_cpu(j).irqs[i]); 160#endif 161 seq_printf(p, " %9s", irq_desc[i].chip->typename); 162 seq_printf(p, " %s", action->name); 163 164 for (action=action->next; action; action = action->next) 165 seq_printf(p, ", %s", action->name); 166 167 seq_putc(p, '\n'); 168skip: 169 spin_unlock_irqrestore(&irq_desc[i].lock, flags); 170 } 171 return 0; 172} 173 174static unsigned int sun4u_compute_tid(unsigned long imap, unsigned long cpuid) 175{ 176 unsigned int tid; 177 178 if (this_is_starfire) { 179 tid = starfire_translate(imap, cpuid); 180 tid <<= IMAP_TID_SHIFT; 181 tid &= IMAP_TID_UPA; 182 } else { 183 if (tlb_type == cheetah || tlb_type == cheetah_plus) { 184 unsigned long ver; 185 186 __asm__ ("rdpr %%ver, %0" : "=r" (ver)); 187 if ((ver >> 32UL) == __JALAPENO_ID || 188 (ver >> 32UL) == __SERRANO_ID) { 189 tid = cpuid << IMAP_TID_SHIFT; 190 tid &= IMAP_TID_JBUS; 191 } else { 192 unsigned int a = cpuid & 0x1f; 193 unsigned int n = (cpuid >> 5) & 0x1f; 194 195 tid = ((a << IMAP_AID_SHIFT) | 196 (n << IMAP_NID_SHIFT)); 197 tid &= (IMAP_AID_SAFARI | 198 IMAP_NID_SAFARI);; 199 } 200 } else { 201 tid = cpuid << IMAP_TID_SHIFT; 202 tid &= IMAP_TID_UPA; 203 } 204 } 205 206 return tid; 207} 208 209struct irq_handler_data { 210 unsigned long iclr; 211 unsigned long imap; 212 213 void (*pre_handler)(unsigned int, void *, void *); 214 void *pre_handler_arg1; 215 void *pre_handler_arg2; 216}; 217 218static inline struct ino_bucket *virt_irq_to_bucket(unsigned int virt_irq) 219{ 220 unsigned int real_irq = virt_to_real_irq(virt_irq); 221 struct ino_bucket *bucket = NULL; 222 223 if (likely(real_irq)) 224 bucket = __bucket(real_irq); 225 226 return bucket; 227} 228 229#ifdef CONFIG_SMP 230static int irq_choose_cpu(unsigned int virt_irq) 231{ 232 cpumask_t mask = irq_desc[virt_irq].affinity; 233 int cpuid; 234 235 if (cpus_equal(mask, CPU_MASK_ALL)) { 236 static int irq_rover; 237 static DEFINE_SPINLOCK(irq_rover_lock); 238 unsigned long flags; 239 240 /* Round-robin distribution... */ 241 do_round_robin: 242 spin_lock_irqsave(&irq_rover_lock, flags); 243 244 while (!cpu_online(irq_rover)) { 245 if (++irq_rover >= NR_CPUS) 246 irq_rover = 0; 247 } 248 cpuid = irq_rover; 249 do { 250 if (++irq_rover >= NR_CPUS) 251 irq_rover = 0; 252 } while (!cpu_online(irq_rover)); 253 254 spin_unlock_irqrestore(&irq_rover_lock, flags); 255 } else { 256 cpumask_t tmp; 257 258 cpus_and(tmp, cpu_online_map, mask); 259 260 if (cpus_empty(tmp)) 261 goto do_round_robin; 262 263 cpuid = first_cpu(tmp); 264 } 265 266 return cpuid; 267} 268#else 269static int irq_choose_cpu(unsigned int virt_irq) 270{ 271 return real_hard_smp_processor_id(); 272} 273#endif 274 275static void sun4u_irq_enable(unsigned int virt_irq) 276{ 277 struct irq_handler_data *data = get_irq_chip_data(virt_irq); 278 279 if (likely(data)) { 280 unsigned long cpuid, imap, val; 281 unsigned int tid; 282 283 cpuid = irq_choose_cpu(virt_irq); 284 imap = data->imap; 285 286 tid = sun4u_compute_tid(imap, cpuid); 287 288 val = upa_readq(imap); 289 val &= ~(IMAP_TID_UPA | IMAP_TID_JBUS | 290 IMAP_AID_SAFARI | IMAP_NID_SAFARI); 291 val |= tid | IMAP_VALID; 292 upa_writeq(val, imap); 293 } 294} 295 296static void sun4u_irq_disable(unsigned int virt_irq) 297{ 298 struct irq_handler_data *data = get_irq_chip_data(virt_irq); 299 300 if (likely(data)) { 301 unsigned long imap = data->imap; 302 u32 tmp = upa_readq(imap); 303 304 tmp &= ~IMAP_VALID; 305 upa_writeq(tmp, imap); 306 } 307} 308 309static void sun4u_irq_end(unsigned int virt_irq) 310{ 311 struct irq_handler_data *data = get_irq_chip_data(virt_irq); 312 313 if (likely(data)) 314 upa_writeq(ICLR_IDLE, data->iclr); 315} 316 317static void sun4v_irq_enable(unsigned int virt_irq) 318{ 319 struct ino_bucket *bucket = virt_irq_to_bucket(virt_irq); 320 unsigned int ino = bucket - &ivector_table[0]; 321 322 if (likely(bucket)) { 323 unsigned long cpuid; 324 int err; 325 326 cpuid = irq_choose_cpu(virt_irq); 327 328 err = sun4v_intr_settarget(ino, cpuid); 329 if (err != HV_EOK) 330 printk("sun4v_intr_settarget(%x,%lu): err(%d)\n", 331 ino, cpuid, err); 332 err = sun4v_intr_setstate(ino, HV_INTR_STATE_IDLE); 333 if (err != HV_EOK) 334 printk("sun4v_intr_setstate(%x): " 335 "err(%d)\n", ino, err); 336 err = sun4v_intr_setenabled(ino, HV_INTR_ENABLED); 337 if (err != HV_EOK) 338 printk("sun4v_intr_setenabled(%x): err(%d)\n", 339 ino, err); 340 } 341} 342 343static void sun4v_irq_disable(unsigned int virt_irq) 344{ 345 struct ino_bucket *bucket = virt_irq_to_bucket(virt_irq); 346 unsigned int ino = bucket - &ivector_table[0]; 347 348 if (likely(bucket)) { 349 int err; 350 351 err = sun4v_intr_setenabled(ino, HV_INTR_DISABLED); 352 if (err != HV_EOK) 353 printk("sun4v_intr_setenabled(%x): " 354 "err(%d)\n", ino, err); 355 } 356} 357 358#ifdef CONFIG_PCI_MSI 359static void sun4v_msi_enable(unsigned int virt_irq) 360{ 361 sun4v_irq_enable(virt_irq); 362 unmask_msi_irq(virt_irq); 363} 364 365static void sun4v_msi_disable(unsigned int virt_irq) 366{ 367 mask_msi_irq(virt_irq); 368 sun4v_irq_disable(virt_irq); 369} 370#endif 371 372static void sun4v_irq_end(unsigned int virt_irq) 373{ 374 struct ino_bucket *bucket = virt_irq_to_bucket(virt_irq); 375 unsigned int ino = bucket - &ivector_table[0]; 376 377 if (likely(bucket)) { 378 int err; 379 380 err = sun4v_intr_setstate(ino, HV_INTR_STATE_IDLE); 381 if (err != HV_EOK) 382 printk("sun4v_intr_setstate(%x): " 383 "err(%d)\n", ino, err); 384 } 385} 386 387static void sun4v_virq_enable(unsigned int virt_irq) 388{ 389 struct ino_bucket *bucket = virt_irq_to_bucket(virt_irq); 390 unsigned int ino = bucket - &ivector_table[0]; 391 392 if (likely(bucket)) { 393 unsigned long cpuid, dev_handle, dev_ino; 394 int err; 395 396 cpuid = irq_choose_cpu(virt_irq); 397 398 dev_handle = ino & IMAP_IGN; 399 dev_ino = ino & IMAP_INO; 400 401 err = sun4v_vintr_set_target(dev_handle, dev_ino, cpuid); 402 if (err != HV_EOK) 403 printk("sun4v_vintr_set_target(%lx,%lx,%lu): " 404 "err(%d)\n", 405 dev_handle, dev_ino, cpuid, err); 406 err = sun4v_vintr_set_state(dev_handle, dev_ino, 407 HV_INTR_STATE_IDLE); 408 if (err != HV_EOK) 409 printk("sun4v_vintr_set_state(%lx,%lx," 410 "HV_INTR_STATE_IDLE): err(%d)\n", 411 dev_handle, dev_ino, err); 412 err = sun4v_vintr_set_valid(dev_handle, dev_ino, 413 HV_INTR_ENABLED); 414 if (err != HV_EOK) 415 printk("sun4v_vintr_set_state(%lx,%lx," 416 "HV_INTR_ENABLED): err(%d)\n", 417 dev_handle, dev_ino, err); 418 } 419} 420 421static void sun4v_virq_disable(unsigned int virt_irq) 422{ 423 struct ino_bucket *bucket = virt_irq_to_bucket(virt_irq); 424 unsigned int ino = bucket - &ivector_table[0]; 425 426 if (likely(bucket)) { 427 unsigned long dev_handle, dev_ino; 428 int err; 429 430 dev_handle = ino & IMAP_IGN; 431 dev_ino = ino & IMAP_INO; 432 433 err = sun4v_vintr_set_valid(dev_handle, dev_ino, 434 HV_INTR_DISABLED); 435 if (err != HV_EOK) 436 printk("sun4v_vintr_set_state(%lx,%lx," 437 "HV_INTR_DISABLED): err(%d)\n", 438 dev_handle, dev_ino, err); 439 } 440} 441 442static void sun4v_virq_end(unsigned int virt_irq) 443{ 444 struct ino_bucket *bucket = virt_irq_to_bucket(virt_irq); 445 unsigned int ino = bucket - &ivector_table[0]; 446 447 if (likely(bucket)) { 448 unsigned long dev_handle, dev_ino; 449 int err; 450 451 dev_handle = ino & IMAP_IGN; 452 dev_ino = ino & IMAP_INO; 453 454 err = sun4v_vintr_set_state(dev_handle, dev_ino, 455 HV_INTR_STATE_IDLE); 456 if (err != HV_EOK) 457 printk("sun4v_vintr_set_state(%lx,%lx," 458 "HV_INTR_STATE_IDLE): err(%d)\n", 459 dev_handle, dev_ino, err); 460 } 461} 462 463static void run_pre_handler(unsigned int virt_irq) 464{ 465 struct ino_bucket *bucket = virt_irq_to_bucket(virt_irq); 466 struct irq_handler_data *data = get_irq_chip_data(virt_irq); 467 468 if (likely(data->pre_handler)) { 469 data->pre_handler(__irq_ino(__irq(bucket)), 470 data->pre_handler_arg1, 471 data->pre_handler_arg2); 472 } 473} 474 475static struct irq_chip sun4u_irq = { 476 .typename = "sun4u", 477 .enable = sun4u_irq_enable, 478 .disable = sun4u_irq_disable, 479 .end = sun4u_irq_end, 480}; 481 482static struct irq_chip sun4u_irq_ack = { 483 .typename = "sun4u+ack", 484 .enable = sun4u_irq_enable, 485 .disable = sun4u_irq_disable, 486 .ack = run_pre_handler, 487 .end = sun4u_irq_end, 488}; 489 490static struct irq_chip sun4v_irq = { 491 .typename = "sun4v", 492 .enable = sun4v_irq_enable, 493 .disable = sun4v_irq_disable, 494 .end = sun4v_irq_end, 495}; 496 497static struct irq_chip sun4v_irq_ack = { 498 .typename = "sun4v+ack", 499 .enable = sun4v_irq_enable, 500 .disable = sun4v_irq_disable, 501 .ack = run_pre_handler, 502 .end = sun4v_irq_end, 503}; 504 505#ifdef CONFIG_PCI_MSI 506static struct irq_chip sun4v_msi = { 507 .typename = "sun4v+msi", 508 .mask = mask_msi_irq, 509 .unmask = unmask_msi_irq, 510 .enable = sun4v_msi_enable, 511 .disable = sun4v_msi_disable, 512 .ack = run_pre_handler, 513 .end = sun4v_irq_end, 514}; 515#endif 516 517static struct irq_chip sun4v_virq = { 518 .typename = "vsun4v", 519 .enable = sun4v_virq_enable, 520 .disable = sun4v_virq_disable, 521 .end = sun4v_virq_end, 522}; 523 524static struct irq_chip sun4v_virq_ack = { 525 .typename = "vsun4v+ack", 526 .enable = sun4v_virq_enable, 527 .disable = sun4v_virq_disable, 528 .ack = run_pre_handler, 529 .end = sun4v_virq_end, 530}; 531 532void irq_install_pre_handler(int virt_irq, 533 void (*func)(unsigned int, void *, void *), 534 void *arg1, void *arg2) 535{ 536 struct irq_handler_data *data = get_irq_chip_data(virt_irq); 537 struct irq_chip *chip; 538 539 data->pre_handler = func; 540 data->pre_handler_arg1 = arg1; 541 data->pre_handler_arg2 = arg2; 542 543 chip = get_irq_chip(virt_irq); 544 if (chip == &sun4u_irq_ack || 545 chip == &sun4v_irq_ack || 546 chip == &sun4v_virq_ack 547#ifdef CONFIG_PCI_MSI 548 || chip == &sun4v_msi 549#endif 550 ) 551 return; 552 553 chip = (chip == &sun4u_irq ? 554 &sun4u_irq_ack : 555 (chip == &sun4v_irq ? 556 &sun4v_irq_ack : &sun4v_virq_ack)); 557 set_irq_chip(virt_irq, chip); 558} 559 560unsigned int build_irq(int inofixup, unsigned long iclr, unsigned long imap) 561{ 562 struct ino_bucket *bucket; 563 struct irq_handler_data *data; 564 int ino; 565 566 BUG_ON(tlb_type == hypervisor); 567 568 ino = (upa_readq(imap) & (IMAP_IGN | IMAP_INO)) + inofixup; 569 bucket = &ivector_table[ino]; 570 if (!bucket->virt_irq) { 571 bucket->virt_irq = virt_irq_alloc(__irq(bucket)); 572 set_irq_chip(bucket->virt_irq, &sun4u_irq); 573 } 574 575 data = get_irq_chip_data(bucket->virt_irq); 576 if (unlikely(data)) 577 goto out; 578 579 data = kzalloc(sizeof(struct irq_handler_data), GFP_ATOMIC); 580 if (unlikely(!data)) { 581 prom_printf("IRQ: kzalloc(irq_handler_data) failed.\n"); 582 prom_halt(); 583 } 584 set_irq_chip_data(bucket->virt_irq, data); 585 586 data->imap = imap; 587 data->iclr = iclr; 588 589out: 590 return bucket->virt_irq; 591} 592 593static unsigned int sun4v_build_common(unsigned long sysino, 594 struct irq_chip *chip) 595{ 596 struct ino_bucket *bucket; 597 struct irq_handler_data *data; 598 599 BUG_ON(tlb_type != hypervisor); 600 601 bucket = &ivector_table[sysino]; 602 if (!bucket->virt_irq) { 603 bucket->virt_irq = virt_irq_alloc(__irq(bucket)); 604 set_irq_chip(bucket->virt_irq, chip); 605 } 606 607 data = get_irq_chip_data(bucket->virt_irq); 608 if (unlikely(data)) 609 goto out; 610 611 data = kzalloc(sizeof(struct irq_handler_data), GFP_ATOMIC); 612 if (unlikely(!data)) { 613 prom_printf("IRQ: kzalloc(irq_handler_data) failed.\n"); 614 prom_halt(); 615 } 616 set_irq_chip_data(bucket->virt_irq, data); 617 618 /* Catch accidental accesses to these things. IMAP/ICLR handling 619 * is done by hypervisor calls on sun4v platforms, not by direct 620 * register accesses. 621 */ 622 data->imap = ~0UL; 623 data->iclr = ~0UL; 624 625out: 626 return bucket->virt_irq; 627} 628 629unsigned int sun4v_build_irq(u32 devhandle, unsigned int devino) 630{ 631 unsigned long sysino = sun4v_devino_to_sysino(devhandle, devino); 632 633 return sun4v_build_common(sysino, &sun4v_irq); 634} 635 636unsigned int sun4v_build_virq(u32 devhandle, unsigned int devino) 637{ 638 unsigned long sysino, hv_err; 639 640 BUG_ON(devhandle & ~IMAP_IGN); 641 BUG_ON(devino & ~IMAP_INO); 642 643 sysino = devhandle | devino; 644 645 hv_err = sun4v_vintr_set_cookie(devhandle, devino, sysino); 646 if (hv_err) { 647 prom_printf("IRQ: Fatal, cannot set cookie for [%x:%x] " 648 "err=%lu\n", devhandle, devino, hv_err); 649 prom_halt(); 650 } 651 652 return sun4v_build_common(sysino, &sun4v_virq); 653} 654 655#ifdef CONFIG_PCI_MSI 656unsigned int sun4v_build_msi(u32 devhandle, unsigned int *virt_irq_p, 657 unsigned int msi_start, unsigned int msi_end) 658{ 659 struct ino_bucket *bucket; 660 struct irq_handler_data *data; 661 unsigned long sysino; 662 unsigned int devino; 663 664 BUG_ON(tlb_type != hypervisor); 665 666 /* Find a free devino in the given range. */ 667 for (devino = msi_start; devino < msi_end; devino++) { 668 sysino = sun4v_devino_to_sysino(devhandle, devino); 669 bucket = &ivector_table[sysino]; 670 if (!bucket->virt_irq) 671 break; 672 } 673 if (devino >= msi_end) 674 return 0; 675 676 sysino = sun4v_devino_to_sysino(devhandle, devino); 677 bucket = &ivector_table[sysino]; 678 bucket->virt_irq = virt_irq_alloc(__irq(bucket)); 679 *virt_irq_p = bucket->virt_irq; 680 set_irq_chip(bucket->virt_irq, &sun4v_msi); 681 682 data = get_irq_chip_data(bucket->virt_irq); 683 if (unlikely(data)) 684 return devino; 685 686 data = kzalloc(sizeof(struct irq_handler_data), GFP_ATOMIC); 687 if (unlikely(!data)) { 688 prom_printf("IRQ: kzalloc(irq_handler_data) failed.\n"); 689 prom_halt(); 690 } 691 set_irq_chip_data(bucket->virt_irq, data); 692 693 data->imap = ~0UL; 694 data->iclr = ~0UL; 695 696 return devino; 697} 698 699void sun4v_destroy_msi(unsigned int virt_irq) 700{ 701 virt_irq_free(virt_irq); 702} 703#endif 704 705void ack_bad_irq(unsigned int virt_irq) 706{ 707 struct ino_bucket *bucket = virt_irq_to_bucket(virt_irq); 708 unsigned int ino = 0xdeadbeef; 709 710 if (bucket) 711 ino = bucket - &ivector_table[0]; 712 713 printk(KERN_CRIT "Unexpected IRQ from ino[%x] virt_irq[%u]\n", 714 ino, virt_irq); 715} 716 717void handler_irq(int irq, struct pt_regs *regs) 718{ 719 struct ino_bucket *bucket; 720 struct pt_regs *old_regs; 721 722 clear_softint(1 << irq); 723 724 old_regs = set_irq_regs(regs); 725 irq_enter(); 726 727 /* Sliiiick... */ 728 bucket = __bucket(xchg32(irq_work(smp_processor_id()), 0)); 729 while (bucket) { 730 struct ino_bucket *next = __bucket(bucket->irq_chain); 731 732 bucket->irq_chain = 0; 733 __do_IRQ(bucket->virt_irq); 734 735 bucket = next; 736 } 737 738 irq_exit(); 739 set_irq_regs(old_regs); 740} 741 742struct sun5_timer { 743 u64 count0; 744 u64 limit0; 745 u64 count1; 746 u64 limit1; 747}; 748 749static struct sun5_timer *prom_timers; 750static u64 prom_limit0, prom_limit1; 751 752static void map_prom_timers(void) 753{ 754 struct device_node *dp; 755 const unsigned int *addr; 756 757 /* PROM timer node hangs out in the top level of device siblings... */ 758 dp = of_find_node_by_path("/"); 759 dp = dp->child; 760 while (dp) { 761 if (!strcmp(dp->name, "counter-timer")) 762 break; 763 dp = dp->sibling; 764 } 765 766 /* Assume if node is not present, PROM uses different tick mechanism 767 * which we should not care about. 768 */ 769 if (!dp) { 770 prom_timers = (struct sun5_timer *) 0; 771 return; 772 } 773 774 /* If PROM is really using this, it must be mapped by him. */ 775 addr = of_get_property(dp, "address", NULL); 776 if (!addr) { 777 prom_printf("PROM does not have timer mapped, trying to continue.\n"); 778 prom_timers = (struct sun5_timer *) 0; 779 return; 780 } 781 prom_timers = (struct sun5_timer *) ((unsigned long)addr[0]); 782} 783 784static void kill_prom_timer(void) 785{ 786 if (!prom_timers) 787 return; 788 789 /* Save them away for later. */ 790 prom_limit0 = prom_timers->limit0; 791 prom_limit1 = prom_timers->limit1; 792 793 /* Just as in sun4c/sun4m PROM uses timer which ticks at IRQ 14. 794 * We turn both off here just to be paranoid. 795 */ 796 prom_timers->limit0 = 0; 797 prom_timers->limit1 = 0; 798 799 /* Wheee, eat the interrupt packet too... */ 800 __asm__ __volatile__( 801" mov 0x40, %%g2\n" 802" ldxa [%%g0] %0, %%g1\n" 803" ldxa [%%g2] %1, %%g1\n" 804" stxa %%g0, [%%g0] %0\n" 805" membar #Sync\n" 806 : /* no outputs */ 807 : "i" (ASI_INTR_RECEIVE), "i" (ASI_INTR_R) 808 : "g1", "g2"); 809} 810 811void init_irqwork_curcpu(void) 812{ 813 int cpu = hard_smp_processor_id(); 814 815 trap_block[cpu].irq_worklist = 0; 816} 817 818/* Please be very careful with register_one_mondo() and 819 * sun4v_register_mondo_queues(). 820 * 821 * On SMP this gets invoked from the CPU trampoline before 822 * the cpu has fully taken over the trap table from OBP, 823 * and it's kernel stack + %g6 thread register state is 824 * not fully cooked yet. 825 * 826 * Therefore you cannot make any OBP calls, not even prom_printf, 827 * from these two routines. 828 */ 829static void __cpuinit register_one_mondo(unsigned long paddr, unsigned long type, unsigned long qmask) 830{ 831 unsigned long num_entries = (qmask + 1) / 64; 832 unsigned long status; 833 834 status = sun4v_cpu_qconf(type, paddr, num_entries); 835 if (status != HV_EOK) { 836 prom_printf("SUN4V: sun4v_cpu_qconf(%lu:%lx:%lu) failed, " 837 "err %lu\n", type, paddr, num_entries, status); 838 prom_halt(); 839 } 840} 841 842static void __cpuinit sun4v_register_mondo_queues(int this_cpu) 843{ 844 struct trap_per_cpu *tb = &trap_block[this_cpu]; 845 846 register_one_mondo(tb->cpu_mondo_pa, HV_CPU_QUEUE_CPU_MONDO, 847 tb->cpu_mondo_qmask); 848 register_one_mondo(tb->dev_mondo_pa, HV_CPU_QUEUE_DEVICE_MONDO, 849 tb->dev_mondo_qmask); 850 register_one_mondo(tb->resum_mondo_pa, HV_CPU_QUEUE_RES_ERROR, 851 tb->resum_qmask); 852 register_one_mondo(tb->nonresum_mondo_pa, HV_CPU_QUEUE_NONRES_ERROR, 853 tb->nonresum_qmask); 854} 855 856static void __cpuinit alloc_one_mondo(unsigned long *pa_ptr, unsigned long qmask, int use_bootmem) 857{ 858 unsigned long size = PAGE_ALIGN(qmask + 1); 859 unsigned long order = get_order(size); 860 void *p = NULL; 861 862 if (use_bootmem) { 863 p = __alloc_bootmem_low(size, size, 0); 864 } else { 865 struct page *page = alloc_pages(GFP_ATOMIC | __GFP_ZERO, order); 866 if (page) 867 p = page_address(page); 868 } 869 870 if (!p) { 871 prom_printf("SUN4V: Error, cannot allocate mondo queue.\n"); 872 prom_halt(); 873 } 874 875 *pa_ptr = __pa(p); 876} 877 878static void __cpuinit alloc_one_kbuf(unsigned long *pa_ptr, unsigned long qmask, int use_bootmem) 879{ 880 unsigned long size = PAGE_ALIGN(qmask + 1); 881 unsigned long order = get_order(size); 882 void *p = NULL; 883 884 if (use_bootmem) { 885 p = __alloc_bootmem_low(size, size, 0); 886 } else { 887 struct page *page = alloc_pages(GFP_ATOMIC | __GFP_ZERO, order); 888 if (page) 889 p = page_address(page); 890 } 891 892 if (!p) { 893 prom_printf("SUN4V: Error, cannot allocate kbuf page.\n"); 894 prom_halt(); 895 } 896 897 *pa_ptr = __pa(p); 898} 899 900static void __cpuinit init_cpu_send_mondo_info(struct trap_per_cpu *tb, int use_bootmem) 901{ 902#ifdef CONFIG_SMP 903 void *page; 904 905 BUILD_BUG_ON((NR_CPUS * sizeof(u16)) > (PAGE_SIZE - 64)); 906 907 if (use_bootmem) 908 page = alloc_bootmem_low_pages(PAGE_SIZE); 909 else 910 page = (void *) get_zeroed_page(GFP_ATOMIC); 911 912 if (!page) { 913 prom_printf("SUN4V: Error, cannot allocate cpu mondo page.\n"); 914 prom_halt(); 915 } 916 917 tb->cpu_mondo_block_pa = __pa(page); 918 tb->cpu_list_pa = __pa(page + 64); 919#endif 920} 921 922/* Allocate and register the mondo and error queues for this cpu. */ 923void __cpuinit sun4v_init_mondo_queues(int use_bootmem, int cpu, int alloc, int load) 924{ 925 struct trap_per_cpu *tb = &trap_block[cpu]; 926 927 if (alloc) { 928 alloc_one_mondo(&tb->cpu_mondo_pa, tb->cpu_mondo_qmask, use_bootmem); 929 alloc_one_mondo(&tb->dev_mondo_pa, tb->dev_mondo_qmask, use_bootmem); 930 alloc_one_mondo(&tb->resum_mondo_pa, tb->resum_qmask, use_bootmem); 931 alloc_one_kbuf(&tb->resum_kernel_buf_pa, tb->resum_qmask, use_bootmem); 932 alloc_one_mondo(&tb->nonresum_mondo_pa, tb->nonresum_qmask, use_bootmem); 933 alloc_one_kbuf(&tb->nonresum_kernel_buf_pa, tb->nonresum_qmask, use_bootmem); 934 935 init_cpu_send_mondo_info(tb, use_bootmem); 936 } 937 938 if (load) { 939 if (cpu != hard_smp_processor_id()) { 940 prom_printf("SUN4V: init mondo on cpu %d not %d\n", 941 cpu, hard_smp_processor_id()); 942 prom_halt(); 943 } 944 sun4v_register_mondo_queues(cpu); 945 } 946} 947 948static struct irqaction timer_irq_action = { 949 .name = "timer", 950}; 951 952/* Only invoked on boot processor. */ 953void __init init_IRQ(void) 954{ 955 map_prom_timers(); 956 kill_prom_timer(); 957 memset(&ivector_table[0], 0, sizeof(ivector_table)); 958 959 if (tlb_type == hypervisor) 960 sun4v_init_mondo_queues(1, hard_smp_processor_id(), 1, 1); 961 962 /* We need to clear any IRQ's pending in the soft interrupt 963 * registers, a spurious one could be left around from the 964 * PROM timer which we just disabled. 965 */ 966 clear_softint(get_softint()); 967 968 /* Now that ivector table is initialized, it is safe 969 * to receive IRQ vector traps. We will normally take 970 * one or two right now, in case some device PROM used 971 * to boot us wants to speak to us. We just ignore them. 972 */ 973 __asm__ __volatile__("rdpr %%pstate, %%g1\n\t" 974 "or %%g1, %0, %%g1\n\t" 975 "wrpr %%g1, 0x0, %%pstate" 976 : /* No outputs */ 977 : "i" (PSTATE_IE) 978 : "g1"); 979 980 irq_desc[0].action = &timer_irq_action; 981} 982