1/* 2 * This file contains miscellaneous low-level functions. 3 * Copyright (C) 1995-1996 Gary Thomas (gdt@linuxppc.org) 4 * 5 * Largely rewritten by Cort Dougan (cort@cs.nmt.edu) 6 * and Paul Mackerras. 7 * Adapted for iSeries by Mike Corrigan (mikejc@us.ibm.com) 8 * PPC64 updates by Dave Engebretsen (engebret@us.ibm.com) 9 * 10 * This program is free software; you can redistribute it and/or 11 * modify it under the terms of the GNU General Public License 12 * as published by the Free Software Foundation; either version 13 * 2 of the License, or (at your option) any later version. 14 * 15 */ 16 17#include <linux/sys.h> 18#include <asm/unistd.h> 19#include <asm/errno.h> 20#include <asm/processor.h> 21#include <asm/page.h> 22#include <asm/cache.h> 23#include <asm/ppc_asm.h> 24#include <asm/asm-offsets.h> 25#include <asm/cputable.h> 26#include <asm/thread_info.h> 27 28 .text 29 30_GLOBAL(get_msr) 31 mfmsr r3 32 blr 33 34_GLOBAL(get_srr0) 35 mfsrr0 r3 36 blr 37 38_GLOBAL(get_srr1) 39 mfsrr1 r3 40 blr 41 42#ifdef CONFIG_IRQSTACKS 43_GLOBAL(call_do_softirq) 44 mflr r0 45 std r0,16(r1) 46 stdu r1,THREAD_SIZE-112(r3) 47 mr r1,r3 48 bl .__do_softirq 49 ld r1,0(r1) 50 ld r0,16(r1) 51 mtlr r0 52 blr 53 54_GLOBAL(call_handle_irq) 55 ld r8,0(r6) 56 mflr r0 57 std r0,16(r1) 58 mtctr r8 59 stdu r1,THREAD_SIZE-112(r5) 60 mr r1,r5 61 bctrl 62 ld r1,0(r1) 63 ld r0,16(r1) 64 mtlr r0 65 blr 66#endif /* CONFIG_IRQSTACKS */ 67 68 .section ".toc","aw" 69PPC64_CACHES: 70 .tc ppc64_caches[TC],ppc64_caches 71 .section ".text" 72 73/* 74 * Write any modified data cache blocks out to memory 75 * and invalidate the corresponding instruction cache blocks. 76 * 77 * flush_icache_range(unsigned long start, unsigned long stop) 78 * 79 * flush all bytes from start through stop-1 inclusive 80 */ 81 82_KPROBE(__flush_icache_range) 83 84/* 85 * Flush the data cache to memory 86 * 87 * Different systems have different cache line sizes 88 * and in some cases i-cache and d-cache line sizes differ from 89 * each other. 90 */ 91 ld r10,PPC64_CACHES@toc(r2) 92 lwz r7,DCACHEL1LINESIZE(r10)/* Get cache line size */ 93 addi r5,r7,-1 94 andc r6,r3,r5 /* round low to line bdy */ 95 subf r8,r6,r4 /* compute length */ 96 add r8,r8,r5 /* ensure we get enough */ 97 lwz r9,DCACHEL1LOGLINESIZE(r10) /* Get log-2 of cache line size */ 98 srw. r8,r8,r9 /* compute line count */ 99 beqlr /* nothing to do? */ 100 mtctr r8 1011: dcbst 0,r6 102 add r6,r6,r7 103 bdnz 1b 104 sync 105 106/* Now invalidate the instruction cache */ 107 108 lwz r7,ICACHEL1LINESIZE(r10) /* Get Icache line size */ 109 addi r5,r7,-1 110 andc r6,r3,r5 /* round low to line bdy */ 111 subf r8,r6,r4 /* compute length */ 112 add r8,r8,r5 113 lwz r9,ICACHEL1LOGLINESIZE(r10) /* Get log-2 of Icache line size */ 114 srw. r8,r8,r9 /* compute line count */ 115 beqlr /* nothing to do? */ 116 mtctr r8 1172: icbi 0,r6 118 add r6,r6,r7 119 bdnz 2b 120 isync 121 blr 122 .previous .text 123/* 124 * Like above, but only do the D-cache. 125 * 126 * flush_dcache_range(unsigned long start, unsigned long stop) 127 * 128 * flush all bytes from start to stop-1 inclusive 129 */ 130_GLOBAL(flush_dcache_range) 131 132/* 133 * Flush the data cache to memory 134 * 135 * Different systems have different cache line sizes 136 */ 137 ld r10,PPC64_CACHES@toc(r2) 138 lwz r7,DCACHEL1LINESIZE(r10) /* Get dcache line size */ 139 addi r5,r7,-1 140 andc r6,r3,r5 /* round low to line bdy */ 141 subf r8,r6,r4 /* compute length */ 142 add r8,r8,r5 /* ensure we get enough */ 143 lwz r9,DCACHEL1LOGLINESIZE(r10) /* Get log-2 of dcache line size */ 144 srw. r8,r8,r9 /* compute line count */ 145 beqlr /* nothing to do? */ 146 mtctr r8 1470: dcbst 0,r6 148 add r6,r6,r7 149 bdnz 0b 150 sync 151 blr 152 153/* 154 * Like above, but works on non-mapped physical addresses. 155 * Use only for non-LPAR setups ! It also assumes real mode 156 * is cacheable. Used for flushing out the DART before using 157 * it as uncacheable memory 158 * 159 * flush_dcache_phys_range(unsigned long start, unsigned long stop) 160 * 161 * flush all bytes from start to stop-1 inclusive 162 */ 163_GLOBAL(flush_dcache_phys_range) 164 ld r10,PPC64_CACHES@toc(r2) 165 lwz r7,DCACHEL1LINESIZE(r10) /* Get dcache line size */ 166 addi r5,r7,-1 167 andc r6,r3,r5 /* round low to line bdy */ 168 subf r8,r6,r4 /* compute length */ 169 add r8,r8,r5 /* ensure we get enough */ 170 lwz r9,DCACHEL1LOGLINESIZE(r10) /* Get log-2 of dcache line size */ 171 srw. r8,r8,r9 /* compute line count */ 172 beqlr /* nothing to do? */ 173 mfmsr r5 /* Disable MMU Data Relocation */ 174 ori r0,r5,MSR_DR 175 xori r0,r0,MSR_DR 176 sync 177 mtmsr r0 178 sync 179 isync 180 mtctr r8 1810: dcbst 0,r6 182 add r6,r6,r7 183 bdnz 0b 184 sync 185 isync 186 mtmsr r5 /* Re-enable MMU Data Relocation */ 187 sync 188 isync 189 blr 190 191_GLOBAL(flush_inval_dcache_range) 192 ld r10,PPC64_CACHES@toc(r2) 193 lwz r7,DCACHEL1LINESIZE(r10) /* Get dcache line size */ 194 addi r5,r7,-1 195 andc r6,r3,r5 /* round low to line bdy */ 196 subf r8,r6,r4 /* compute length */ 197 add r8,r8,r5 /* ensure we get enough */ 198 lwz r9,DCACHEL1LOGLINESIZE(r10)/* Get log-2 of dcache line size */ 199 srw. r8,r8,r9 /* compute line count */ 200 beqlr /* nothing to do? */ 201 sync 202 isync 203 mtctr r8 2040: dcbf 0,r6 205 add r6,r6,r7 206 bdnz 0b 207 sync 208 isync 209 blr 210 211 212/* 213 * Flush a particular page from the data cache to RAM. 214 * Note: this is necessary because the instruction cache does *not* 215 * snoop from the data cache. 216 * 217 * void __flush_dcache_icache(void *page) 218 */ 219_GLOBAL(__flush_dcache_icache) 220/* 221 * Flush the data cache to memory 222 * 223 * Different systems have different cache line sizes 224 */ 225 226/* Flush the dcache */ 227 ld r7,PPC64_CACHES@toc(r2) 228 clrrdi r3,r3,PAGE_SHIFT /* Page align */ 229 lwz r4,DCACHEL1LINESPERPAGE(r7) /* Get # dcache lines per page */ 230 lwz r5,DCACHEL1LINESIZE(r7) /* Get dcache line size */ 231 mr r6,r3 232 mtctr r4 2330: dcbst 0,r6 234 add r6,r6,r5 235 bdnz 0b 236 sync 237 238/* Now invalidate the icache */ 239 240 lwz r4,ICACHEL1LINESPERPAGE(r7) /* Get # icache lines per page */ 241 lwz r5,ICACHEL1LINESIZE(r7) /* Get icache line size */ 242 mtctr r4 2431: icbi 0,r3 244 add r3,r3,r5 245 bdnz 1b 246 isync 247 blr 248 249 250#if defined(CONFIG_PPC_PMAC) || defined(CONFIG_PPC_MAPLE) 251/* 252 * Do an IO access in real mode 253 */ 254_GLOBAL(real_readb) 255 mfmsr r7 256 ori r0,r7,MSR_DR 257 xori r0,r0,MSR_DR 258 sync 259 mtmsrd r0 260 sync 261 isync 262 mfspr r6,SPRN_HID4 263 rldicl r5,r6,32,0 264 ori r5,r5,0x100 265 rldicl r5,r5,32,0 266 sync 267 mtspr SPRN_HID4,r5 268 isync 269 slbia 270 isync 271 lbz r3,0(r3) 272 sync 273 mtspr SPRN_HID4,r6 274 isync 275 slbia 276 isync 277 mtmsrd r7 278 sync 279 isync 280 blr 281 282 /* 283 * Do an IO access in real mode 284 */ 285_GLOBAL(real_writeb) 286 mfmsr r7 287 ori r0,r7,MSR_DR 288 xori r0,r0,MSR_DR 289 sync 290 mtmsrd r0 291 sync 292 isync 293 mfspr r6,SPRN_HID4 294 rldicl r5,r6,32,0 295 ori r5,r5,0x100 296 rldicl r5,r5,32,0 297 sync 298 mtspr SPRN_HID4,r5 299 isync 300 slbia 301 isync 302 stb r3,0(r4) 303 sync 304 mtspr SPRN_HID4,r6 305 isync 306 slbia 307 isync 308 mtmsrd r7 309 sync 310 isync 311 blr 312#endif /* defined(CONFIG_PPC_PMAC) || defined(CONFIG_PPC_MAPLE) */ 313 314#ifdef CONFIG_PPC_PASEMI 315 316/* No support in all binutils for these yet, so use defines */ 317#define LBZCIX(RT,RA,RB) .long (0x7c0006aa|(RT<<21)|(RA<<16)|(RB << 11)) 318#define STBCIX(RS,RA,RB) .long (0x7c0007aa|(RS<<21)|(RA<<16)|(RB << 11)) 319 320 321_GLOBAL(real_205_readb) 322 mfmsr r7 323 ori r0,r7,MSR_DR 324 xori r0,r0,MSR_DR 325 sync 326 mtmsrd r0 327 sync 328 isync 329 LBZCIX(r3,0,r3) 330 isync 331 mtmsrd r7 332 sync 333 isync 334 blr 335 336_GLOBAL(real_205_writeb) 337 mfmsr r7 338 ori r0,r7,MSR_DR 339 xori r0,r0,MSR_DR 340 sync 341 mtmsrd r0 342 sync 343 isync 344 STBCIX(r3,0,r4) 345 isync 346 mtmsrd r7 347 sync 348 isync 349 blr 350 351#endif /* CONFIG_PPC_PASEMI */ 352 353 354#ifdef CONFIG_CPU_FREQ_PMAC64 355/* 356 * SCOM access functions for 970 (FX only for now) 357 * 358 * unsigned long scom970_read(unsigned int address); 359 * void scom970_write(unsigned int address, unsigned long value); 360 * 361 * The address passed in is the 24 bits register address. This code 362 * is 970 specific and will not check the status bits, so you should 363 * know what you are doing. 364 */ 365_GLOBAL(scom970_read) 366 /* interrupts off */ 367 mfmsr r4 368 ori r0,r4,MSR_EE 369 xori r0,r0,MSR_EE 370 mtmsrd r0,1 371 372 /* rotate 24 bits SCOM address 8 bits left and mask out it's low 8 bits 373 * (including parity). On current CPUs they must be 0'd, 374 * and finally or in RW bit 375 */ 376 rlwinm r3,r3,8,0,15 377 ori r3,r3,0x8000 378 379 /* do the actual scom read */ 380 sync 381 mtspr SPRN_SCOMC,r3 382 isync 383 mfspr r3,SPRN_SCOMD 384 isync 385 mfspr r0,SPRN_SCOMC 386 isync 387 388 389 /* restore interrupts */ 390 mtmsrd r4,1 391 blr 392 393 394_GLOBAL(scom970_write) 395 /* interrupts off */ 396 mfmsr r5 397 ori r0,r5,MSR_EE 398 xori r0,r0,MSR_EE 399 mtmsrd r0,1 400 401 /* rotate 24 bits SCOM address 8 bits left and mask out it's low 8 bits 402 * (including parity). On current CPUs they must be 0'd. 403 */ 404 405 rlwinm r3,r3,8,0,15 406 407 sync 408 mtspr SPRN_SCOMD,r4 /* write data */ 409 isync 410 mtspr SPRN_SCOMC,r3 /* write command */ 411 isync 412 mfspr 3,SPRN_SCOMC 413 isync 414 415 /* restore interrupts */ 416 mtmsrd r5,1 417 blr 418#endif /* CONFIG_CPU_FREQ_PMAC64 */ 419 420 421/* 422 * Create a kernel thread 423 * kernel_thread(fn, arg, flags) 424 */ 425_GLOBAL(kernel_thread) 426 std r29,-24(r1) 427 std r30,-16(r1) 428 stdu r1,-STACK_FRAME_OVERHEAD(r1) 429 mr r29,r3 430 mr r30,r4 431 ori r3,r5,CLONE_VM /* flags */ 432 oris r3,r3,(CLONE_UNTRACED>>16) 433 li r4,0 /* new sp (unused) */ 434 li r0,__NR_clone 435 sc 436 cmpdi 0,r3,0 /* parent or child? */ 437 bne 1f /* return if parent */ 438 li r0,0 439 stdu r0,-STACK_FRAME_OVERHEAD(r1) 440 ld r2,8(r29) 441 ld r29,0(r29) 442 mtlr r29 /* fn addr in lr */ 443 mr r3,r30 /* load arg and call fn */ 444 blrl 445 li r0,__NR_exit /* exit after child exits */ 446 li r3,0 447 sc 4481: addi r1,r1,STACK_FRAME_OVERHEAD 449 ld r29,-24(r1) 450 ld r30,-16(r1) 451 blr 452 453/* 454 * disable_kernel_fp() 455 * Disable the FPU. 456 */ 457_GLOBAL(disable_kernel_fp) 458 mfmsr r3 459 rldicl r0,r3,(63-MSR_FP_LG),1 460 rldicl r3,r0,(MSR_FP_LG+1),0 461 mtmsrd r3 /* disable use of fpu now */ 462 isync 463 blr 464 465#ifdef CONFIG_ALTIVEC 466 467 468/* 469 * giveup_altivec(tsk) 470 * Disable VMX for the task given as the argument, 471 * and save the vector registers in its thread_struct. 472 * Enables the VMX for use in the kernel on return. 473 */ 474_GLOBAL(giveup_altivec) 475 mfmsr r5 476 oris r5,r5,MSR_VEC@h 477 mtmsrd r5 /* enable use of VMX now */ 478 isync 479 cmpdi 0,r3,0 480 beqlr- /* if no previous owner, done */ 481 addi r3,r3,THREAD /* want THREAD of task */ 482 ld r5,PT_REGS(r3) 483 cmpdi 0,r5,0 484 SAVE_32VRS(0,r4,r3) 485 mfvscr vr0 486 li r4,THREAD_VSCR 487 stvx vr0,r4,r3 488 beq 1f 489 ld r4,_MSR-STACK_FRAME_OVERHEAD(r5) 490 lis r3,MSR_VEC@h 491 andc r4,r4,r3 /* disable FP for previous task */ 492 std r4,_MSR-STACK_FRAME_OVERHEAD(r5) 4931: 494#ifndef CONFIG_SMP 495 li r5,0 496 ld r4,last_task_used_altivec@got(r2) 497 std r5,0(r4) 498#endif /* CONFIG_SMP */ 499 blr 500 501#endif /* CONFIG_ALTIVEC */ 502 503_GLOBAL(kernel_execve) 504 li r0,__NR_execve 505 sc 506 bnslr 507 neg r3,r3 508 blr 509 510/* kexec_wait(phys_cpu) 511 * 512 * wait for the flag to change, indicating this kernel is going away but 513 * the slave code for the next one is at addresses 0 to 100. 514 * 515 * This is used by all slaves. 516 * 517 * Physical (hardware) cpu id should be in r3. 518 */ 519_GLOBAL(kexec_wait) 520 bl 1f 5211: mflr r5 522 addi r5,r5,kexec_flag-1b 523 52499: HMT_LOW 525#ifdef CONFIG_KEXEC /* use no memory without kexec */ 526 lwz r4,0(r5) 527 cmpwi 0,r4,0 528 bnea 0x60 529#endif 530 b 99b 531 532/* this can be in text because we won't change it until we are 533 * running in real anyways 534 */ 535kexec_flag: 536 .long 0 537 538 539#ifdef CONFIG_KEXEC 540 541/* kexec_smp_wait(void) 542 * 543 * call with interrupts off 544 * note: this is a terminal routine, it does not save lr 545 * 546 * get phys id from paca 547 * set paca id to -1 to say we got here 548 * switch to real mode 549 * join other cpus in kexec_wait(phys_id) 550 */ 551_GLOBAL(kexec_smp_wait) 552 lhz r3,PACAHWCPUID(r13) 553 li r4,-1 554 sth r4,PACAHWCPUID(r13) /* let others know we left */ 555 bl real_mode 556 b .kexec_wait 557 558/* 559 * switch to real mode (turn mmu off) 560 * we use the early kernel trick that the hardware ignores bits 561 * 0 and 1 (big endian) of the effective address in real mode 562 * 563 * don't overwrite r3 here, it is live for kexec_wait above. 564 */ 565real_mode: /* assume normal blr return */ 5661: li r9,MSR_RI 567 li r10,MSR_DR|MSR_IR 568 mflr r11 /* return address to SRR0 */ 569 mfmsr r12 570 andc r9,r12,r9 571 andc r10,r12,r10 572 573 mtmsrd r9,1 574 mtspr SPRN_SRR1,r10 575 mtspr SPRN_SRR0,r11 576 rfid 577 578 579/* 580 * kexec_sequence(newstack, start, image, control, clear_all()) 581 * 582 * does the grungy work with stack switching and real mode switches 583 * also does simple calls to other code 584 */ 585 586_GLOBAL(kexec_sequence) 587 mflr r0 588 std r0,16(r1) 589 590 /* switch stacks to newstack -- &kexec_stack.stack */ 591 stdu r1,THREAD_SIZE-112(r3) 592 mr r1,r3 593 594 li r0,0 595 std r0,16(r1) 596 597 /* save regs for local vars on new stack. 598 * yes, we won't go back, but ... 599 */ 600 std r31,-8(r1) 601 std r30,-16(r1) 602 std r29,-24(r1) 603 std r28,-32(r1) 604 std r27,-40(r1) 605 std r26,-48(r1) 606 std r25,-56(r1) 607 608 stdu r1,-112-64(r1) 609 610 /* save args into preserved regs */ 611 mr r31,r3 /* newstack (both) */ 612 mr r30,r4 /* start (real) */ 613 mr r29,r5 /* image (virt) */ 614 mr r28,r6 /* control, unused */ 615 mr r27,r7 /* clear_all() fn desc */ 616 mr r26,r8 /* spare */ 617 lhz r25,PACAHWCPUID(r13) /* get our phys cpu from paca */ 618 619 /* disable interrupts, we are overwriting kernel data next */ 620 mfmsr r3 621 rlwinm r3,r3,0,17,15 622 mtmsrd r3,1 623 624 /* copy dest pages, flush whole dest image */ 625 mr r3,r29 626 bl .kexec_copy_flush /* (image) */ 627 628 /* turn off mmu */ 629 bl real_mode 630 631 /* clear out hardware hash page table and tlb */ 632 ld r5,0(r27) /* deref function descriptor */ 633 mtctr r5 634 bctrl /* ppc_md.hpte_clear_all(void); */ 635 636/* 637 * kexec image calling is: 638 * the first 0x100 bytes of the entry point are copied to 0 639 * 640 * all slaves branch to slave = 0x60 (absolute) 641 * slave(phys_cpu_id); 642 * 643 * master goes to start = entry point 644 * start(phys_cpu_id, start, 0); 645 * 646 * 647 * a wrapper is needed to call existing kernels, here is an approximate 648 * description of one method: 649 * 650 * v2: (2.6.10) 651 * start will be near the boot_block (maybe 0x100 bytes before it?) 652 * it will have a 0x60, which will b to boot_block, where it will wait 653 * and 0 will store phys into struct boot-block and load r3 from there, 654 * copy kernel 0-0x100 and tell slaves to back down to 0x60 again 655 * 656 * v1: (2.6.9) 657 * boot block will have all cpus scanning device tree to see if they 658 * are the boot cpu ????? 659 * other device tree differences (prop sizes, va vs pa, etc)... 660 */ 661 662 /* copy 0x100 bytes starting at start to 0 */ 663 li r3,0 664 mr r4,r30 665 li r5,0x100 666 li r6,0 667 bl .copy_and_flush /* (dest, src, copy limit, start offset) */ 6681: /* assume normal blr return */ 669 670 /* release other cpus to the new kernel secondary start at 0x60 */ 671 mflr r5 672 li r6,1 673 stw r6,kexec_flag-1b(5) 674 mr r3,r25 # my phys cpu 675 mr r4,r30 # start, aka phys mem offset 676 mtlr 4 677 li r5,0 678 blr /* image->start(physid, image->start, 0); */ 679#endif /* CONFIG_KEXEC */ 680