1/* 2 * ppc64 MMU hashtable management routines 3 * 4 * (c) Copyright IBM Corp. 2003, 2005 5 * 6 * Maintained by: Benjamin Herrenschmidt 7 * <benh@kernel.crashing.org> 8 * 9 * This file is covered by the GNU Public Licence v2 as 10 * described in the kernel's COPYING file. 11 */ 12 13#include <asm/reg.h> 14#include <asm/pgtable.h> 15#include <asm/mmu.h> 16#include <asm/page.h> 17#include <asm/types.h> 18#include <asm/ppc_asm.h> 19#include <asm/asm-offsets.h> 20#include <asm/cputable.h> 21 22 .text 23 24/* 25 * Stackframe: 26 * 27 * +-> Back chain (SP + 256) 28 * | General register save area (SP + 112) 29 * | Parameter save area (SP + 48) 30 * | TOC save area (SP + 40) 31 * | link editor doubleword (SP + 32) 32 * | compiler doubleword (SP + 24) 33 * | LR save area (SP + 16) 34 * | CR save area (SP + 8) 35 * SP ---> +-- Back chain (SP + 0) 36 */ 37#define STACKFRAMESIZE 256 38 39/* Save parameters offsets */ 40#define STK_PARM(i) (STACKFRAMESIZE + 48 + ((i)-3)*8) 41 42/* Save non-volatile offsets */ 43#define STK_REG(i) (112 + ((i)-14)*8) 44 45 46#ifndef CONFIG_PPC_64K_PAGES 47 48/***************************************************************************** 49 * * 50 * 4K SW & 4K HW pages implementation * 51 * * 52 *****************************************************************************/ 53 54 55/* 56 * _hash_page_4K(unsigned long ea, unsigned long access, unsigned long vsid, 57 * pte_t *ptep, unsigned long trap, int local) 58 * 59 * Adds a 4K page to the hash table in a segment of 4K pages only 60 */ 61 62_GLOBAL(__hash_page_4K) 63 mflr r0 64 std r0,16(r1) 65 stdu r1,-STACKFRAMESIZE(r1) 66 /* Save all params that we need after a function call */ 67 std r6,STK_PARM(r6)(r1) 68 std r8,STK_PARM(r8)(r1) 69 70 /* Add _PAGE_PRESENT to access */ 71 ori r4,r4,_PAGE_PRESENT 72 73 /* Save non-volatile registers. 74 * r31 will hold "old PTE" 75 * r30 is "new PTE" 76 * r29 is "va" 77 * r28 is a hash value 78 * r27 is hashtab mask (maybe dynamic patched instead ?) 79 */ 80 std r27,STK_REG(r27)(r1) 81 std r28,STK_REG(r28)(r1) 82 std r29,STK_REG(r29)(r1) 83 std r30,STK_REG(r30)(r1) 84 std r31,STK_REG(r31)(r1) 85 86 /* Step 1: 87 * 88 * Check permissions, atomically mark the linux PTE busy 89 * and hashed. 90 */ 911: 92 ldarx r31,0,r6 93 /* Check access rights (access & ~(pte_val(*ptep))) */ 94 andc. r0,r4,r31 95 bne- htab_wrong_access 96 /* Check if PTE is busy */ 97 andi. r0,r31,_PAGE_BUSY 98 /* If so, just bail out and refault if needed. Someone else 99 * is changing this PTE anyway and might hash it. 100 */ 101 bne- htab_bail_ok 102 103 /* Prepare new PTE value (turn access RW into DIRTY, then 104 * add BUSY,HASHPTE and ACCESSED) 105 */ 106 rlwinm r30,r4,32-9+7,31-7,31-7 /* _PAGE_RW -> _PAGE_DIRTY */ 107 or r30,r30,r31 108 ori r30,r30,_PAGE_BUSY | _PAGE_ACCESSED | _PAGE_HASHPTE 109 /* Write the linux PTE atomically (setting busy) */ 110 stdcx. r30,0,r6 111 bne- 1b 112 isync 113 114 /* Step 2: 115 * 116 * Insert/Update the HPTE in the hash table. At this point, 117 * r4 (access) is re-useable, we use it for the new HPTE flags 118 */ 119 120 /* Calc va and put it in r29 */ 121 rldicr r29,r5,28,63-28 122 rldicl r3,r3,0,36 123 or r29,r3,r29 124 125 /* Calculate hash value for primary slot and store it in r28 */ 126 rldicl r5,r5,0,25 /* vsid & 0x0000007fffffffff */ 127 rldicl r0,r3,64-12,48 /* (ea >> 12) & 0xffff */ 128 xor r28,r5,r0 129 130 /* Convert linux PTE bits into HW equivalents */ 131 andi. r3,r30,0x1fe /* Get basic set of flags */ 132 xori r3,r3,HPTE_R_N /* _PAGE_EXEC -> NOEXEC */ 133 rlwinm r0,r30,32-9+1,30,30 /* _PAGE_RW -> _PAGE_USER (r0) */ 134 rlwinm r4,r30,32-7+1,30,30 /* _PAGE_DIRTY -> _PAGE_USER (r4) */ 135 and r0,r0,r4 /* _PAGE_RW & _PAGE_DIRTY ->r0 bit 30*/ 136 andc r0,r30,r0 /* r0 = pte & ~r0 */ 137 rlwimi r3,r0,32-1,31,31 /* Insert result into PP lsb */ 138 ori r3,r3,HPTE_R_C /* Always add "C" bit for perf. */ 139 140 /* We eventually do the icache sync here (maybe inline that 141 * code rather than call a C function...) 142 */ 143BEGIN_FTR_SECTION 144 mr r4,r30 145 mr r5,r7 146 bl .hash_page_do_lazy_icache 147END_FTR_SECTION(CPU_FTR_NOEXECUTE|CPU_FTR_COHERENT_ICACHE, CPU_FTR_NOEXECUTE) 148 149 /* At this point, r3 contains new PP bits, save them in 150 * place of "access" in the param area (sic) 151 */ 152 std r3,STK_PARM(r4)(r1) 153 154 /* Get htab_hash_mask */ 155 ld r4,htab_hash_mask@got(2) 156 ld r27,0(r4) /* htab_hash_mask -> r27 */ 157 158 /* Check if we may already be in the hashtable, in this case, we 159 * go to out-of-line code to try to modify the HPTE 160 */ 161 andi. r0,r31,_PAGE_HASHPTE 162 bne htab_modify_pte 163 164htab_insert_pte: 165 /* Clear hpte bits in new pte (we also clear BUSY btw) and 166 * add _PAGE_HASHPTE 167 */ 168 lis r0,_PAGE_HPTEFLAGS@h 169 ori r0,r0,_PAGE_HPTEFLAGS@l 170 andc r30,r30,r0 171 ori r30,r30,_PAGE_HASHPTE 172 173 /* physical address r5 */ 174 rldicl r5,r31,64-PTE_RPN_SHIFT,PTE_RPN_SHIFT 175 sldi r5,r5,PAGE_SHIFT 176 177 /* Calculate primary group hash */ 178 and r0,r28,r27 179 rldicr r3,r0,3,63-3 /* r3 = (hash & mask) << 3 */ 180 181 /* Call ppc_md.hpte_insert */ 182 ld r6,STK_PARM(r4)(r1) /* Retreive new pp bits */ 183 mr r4,r29 /* Retreive va */ 184 li r7,0 /* !bolted, !secondary */ 185 li r8,MMU_PAGE_4K /* page size */ 186_GLOBAL(htab_call_hpte_insert1) 187 bl . /* Patched by htab_finish_init() */ 188 cmpdi 0,r3,0 189 bge htab_pte_insert_ok /* Insertion successful */ 190 cmpdi 0,r3,-2 /* Critical failure */ 191 beq- htab_pte_insert_failure 192 193 /* Now try secondary slot */ 194 195 /* physical address r5 */ 196 rldicl r5,r31,64-PTE_RPN_SHIFT,PTE_RPN_SHIFT 197 sldi r5,r5,PAGE_SHIFT 198 199 /* Calculate secondary group hash */ 200 andc r0,r27,r28 201 rldicr r3,r0,3,63-3 /* r0 = (~hash & mask) << 3 */ 202 203 /* Call ppc_md.hpte_insert */ 204 ld r6,STK_PARM(r4)(r1) /* Retreive new pp bits */ 205 mr r4,r29 /* Retreive va */ 206 li r7,HPTE_V_SECONDARY /* !bolted, secondary */ 207 li r8,MMU_PAGE_4K /* page size */ 208_GLOBAL(htab_call_hpte_insert2) 209 bl . /* Patched by htab_finish_init() */ 210 cmpdi 0,r3,0 211 bge+ htab_pte_insert_ok /* Insertion successful */ 212 cmpdi 0,r3,-2 /* Critical failure */ 213 beq- htab_pte_insert_failure 214 215 /* Both are full, we need to evict something */ 216 mftb r0 217 /* Pick a random group based on TB */ 218 andi. r0,r0,1 219 mr r5,r28 220 bne 2f 221 not r5,r5 2222: and r0,r5,r27 223 rldicr r3,r0,3,63-3 /* r0 = (hash & mask) << 3 */ 224 /* Call ppc_md.hpte_remove */ 225_GLOBAL(htab_call_hpte_remove) 226 bl . /* Patched by htab_finish_init() */ 227 228 /* Try all again */ 229 b htab_insert_pte 230 231htab_bail_ok: 232 li r3,0 233 b htab_bail 234 235htab_pte_insert_ok: 236 /* Insert slot number & secondary bit in PTE */ 237 rldimi r30,r3,12,63-15 238 239 /* Write out the PTE with a normal write 240 * (maybe add eieio may be good still ?) 241 */ 242htab_write_out_pte: 243 ld r6,STK_PARM(r6)(r1) 244 std r30,0(r6) 245 li r3, 0 246htab_bail: 247 ld r27,STK_REG(r27)(r1) 248 ld r28,STK_REG(r28)(r1) 249 ld r29,STK_REG(r29)(r1) 250 ld r30,STK_REG(r30)(r1) 251 ld r31,STK_REG(r31)(r1) 252 addi r1,r1,STACKFRAMESIZE 253 ld r0,16(r1) 254 mtlr r0 255 blr 256 257htab_modify_pte: 258 /* Keep PP bits in r4 and slot idx from the PTE around in r3 */ 259 mr r4,r3 260 rlwinm r3,r31,32-12,29,31 261 262 /* Secondary group ? if yes, get a inverted hash value */ 263 mr r5,r28 264 andi. r0,r31,_PAGE_SECONDARY 265 beq 1f 266 not r5,r5 2671: 268 /* Calculate proper slot value for ppc_md.hpte_updatepp */ 269 and r0,r5,r27 270 rldicr r0,r0,3,63-3 /* r0 = (hash & mask) << 3 */ 271 add r3,r0,r3 /* add slot idx */ 272 273 /* Call ppc_md.hpte_updatepp */ 274 mr r5,r29 /* va */ 275 li r6,MMU_PAGE_4K /* page size */ 276 ld r7,STK_PARM(r8)(r1) /* get "local" param */ 277_GLOBAL(htab_call_hpte_updatepp) 278 bl . /* Patched by htab_finish_init() */ 279 280 /* if we failed because typically the HPTE wasn't really here 281 * we try an insertion. 282 */ 283 cmpdi 0,r3,-1 284 beq- htab_insert_pte 285 286 /* Clear the BUSY bit and Write out the PTE */ 287 li r0,_PAGE_BUSY 288 andc r30,r30,r0 289 b htab_write_out_pte 290 291htab_wrong_access: 292 /* Bail out clearing reservation */ 293 stdcx. r31,0,r6 294 li r3,1 295 b htab_bail 296 297htab_pte_insert_failure: 298 /* Bail out restoring old PTE */ 299 ld r6,STK_PARM(r6)(r1) 300 std r31,0(r6) 301 li r3,-1 302 b htab_bail 303 304 305#else /* CONFIG_PPC_64K_PAGES */ 306 307 308/***************************************************************************** 309 * * 310 * 64K SW & 4K or 64K HW in a 4K segment pages implementation * 311 * * 312 *****************************************************************************/ 313 314/* _hash_page_4K(unsigned long ea, unsigned long access, unsigned long vsid, 315 * pte_t *ptep, unsigned long trap, int local) 316 */ 317 318/* 319 * For now, we do NOT implement Admixed pages 320 */ 321_GLOBAL(__hash_page_4K) 322 mflr r0 323 std r0,16(r1) 324 stdu r1,-STACKFRAMESIZE(r1) 325 /* Save all params that we need after a function call */ 326 std r6,STK_PARM(r6)(r1) 327 std r8,STK_PARM(r8)(r1) 328 329 /* Add _PAGE_PRESENT to access */ 330 ori r4,r4,_PAGE_PRESENT 331 332 /* Save non-volatile registers. 333 * r31 will hold "old PTE" 334 * r30 is "new PTE" 335 * r29 is "va" 336 * r28 is a hash value 337 * r27 is hashtab mask (maybe dynamic patched instead ?) 338 * r26 is the hidx mask 339 * r25 is the index in combo page 340 */ 341 std r25,STK_REG(r25)(r1) 342 std r26,STK_REG(r26)(r1) 343 std r27,STK_REG(r27)(r1) 344 std r28,STK_REG(r28)(r1) 345 std r29,STK_REG(r29)(r1) 346 std r30,STK_REG(r30)(r1) 347 std r31,STK_REG(r31)(r1) 348 349 /* Step 1: 350 * 351 * Check permissions, atomically mark the linux PTE busy 352 * and hashed. 353 */ 3541: 355 ldarx r31,0,r6 356 /* Check access rights (access & ~(pte_val(*ptep))) */ 357 andc. r0,r4,r31 358 bne- htab_wrong_access 359 /* Check if PTE is busy */ 360 andi. r0,r31,_PAGE_BUSY 361 /* If so, just bail out and refault if needed. Someone else 362 * is changing this PTE anyway and might hash it. 363 */ 364 bne- htab_bail_ok 365 /* Prepare new PTE value (turn access RW into DIRTY, then 366 * add BUSY and ACCESSED) 367 */ 368 rlwinm r30,r4,32-9+7,31-7,31-7 /* _PAGE_RW -> _PAGE_DIRTY */ 369 or r30,r30,r31 370 ori r30,r30,_PAGE_BUSY | _PAGE_ACCESSED | _PAGE_HASHPTE 371 oris r30,r30,_PAGE_COMBO@h 372 /* Write the linux PTE atomically (setting busy) */ 373 stdcx. r30,0,r6 374 bne- 1b 375 isync 376 377 /* Step 2: 378 * 379 * Insert/Update the HPTE in the hash table. At this point, 380 * r4 (access) is re-useable, we use it for the new HPTE flags 381 */ 382 383 /* Load the hidx index */ 384 rldicl r25,r3,64-12,60 385 386 /* Calc va and put it in r29 */ 387 rldicr r29,r5,28,63-28 /* r29 = (vsid << 28) */ 388 rldicl r3,r3,0,36 /* r3 = (ea & 0x0fffffff) */ 389 or r29,r3,r29 /* r29 = va 390 391 /* Calculate hash value for primary slot and store it in r28 */ 392 rldicl r5,r5,0,25 /* vsid & 0x0000007fffffffff */ 393 rldicl r0,r3,64-12,48 /* (ea >> 12) & 0xffff */ 394 xor r28,r5,r0 395 396 /* Convert linux PTE bits into HW equivalents */ 397 andi. r3,r30,0x1fe /* Get basic set of flags */ 398 xori r3,r3,HPTE_R_N /* _PAGE_EXEC -> NOEXEC */ 399 rlwinm r0,r30,32-9+1,30,30 /* _PAGE_RW -> _PAGE_USER (r0) */ 400 rlwinm r4,r30,32-7+1,30,30 /* _PAGE_DIRTY -> _PAGE_USER (r4) */ 401 and r0,r0,r4 /* _PAGE_RW & _PAGE_DIRTY ->r0 bit 30*/ 402 andc r0,r30,r0 /* r0 = pte & ~r0 */ 403 rlwimi r3,r0,32-1,31,31 /* Insert result into PP lsb */ 404 ori r3,r3,HPTE_R_C /* Always add "C" bit for perf. */ 405 406 /* We eventually do the icache sync here (maybe inline that 407 * code rather than call a C function...) 408 */ 409BEGIN_FTR_SECTION 410 mr r4,r30 411 mr r5,r7 412 bl .hash_page_do_lazy_icache 413END_FTR_SECTION(CPU_FTR_NOEXECUTE|CPU_FTR_COHERENT_ICACHE, CPU_FTR_NOEXECUTE) 414 415 /* At this point, r3 contains new PP bits, save them in 416 * place of "access" in the param area (sic) 417 */ 418 std r3,STK_PARM(r4)(r1) 419 420 /* Get htab_hash_mask */ 421 ld r4,htab_hash_mask@got(2) 422 ld r27,0(r4) /* htab_hash_mask -> r27 */ 423 424 /* Check if we may already be in the hashtable, in this case, we 425 * go to out-of-line code to try to modify the HPTE. We look for 426 * the bit at (1 >> (index + 32)) 427 */ 428 andi. r0,r31,_PAGE_HASHPTE 429 li r26,0 /* Default hidx */ 430 beq htab_insert_pte 431 432 /* 433 * Check if the pte was already inserted into the hash table 434 * as a 64k HW page, and invalidate the 64k HPTE if so. 435 */ 436 andis. r0,r31,_PAGE_COMBO@h 437 beq htab_inval_old_hpte 438 439 ld r6,STK_PARM(r6)(r1) 440 ori r26,r6,0x8000 /* Load the hidx mask */ 441 ld r26,0(r26) 442 addi r5,r25,36 /* Check actual HPTE_SUB bit, this */ 443 rldcr. r0,r31,r5,0 /* must match pgtable.h definition */ 444 bne htab_modify_pte 445 446htab_insert_pte: 447 /* real page number in r5, PTE RPN value + index */ 448 andis. r0,r31,_PAGE_4K_PFN@h 449 srdi r5,r31,PTE_RPN_SHIFT 450 bne- htab_special_pfn 451 sldi r5,r5,PAGE_SHIFT-HW_PAGE_SHIFT 452 add r5,r5,r25 453htab_special_pfn: 454 sldi r5,r5,HW_PAGE_SHIFT 455 456 /* Calculate primary group hash */ 457 and r0,r28,r27 458 rldicr r3,r0,3,63-3 /* r0 = (hash & mask) << 3 */ 459 460 /* Call ppc_md.hpte_insert */ 461 ld r6,STK_PARM(r4)(r1) /* Retreive new pp bits */ 462 mr r4,r29 /* Retreive va */ 463 li r7,0 /* !bolted, !secondary */ 464 li r8,MMU_PAGE_4K /* page size */ 465_GLOBAL(htab_call_hpte_insert1) 466 bl . /* patched by htab_finish_init() */ 467 cmpdi 0,r3,0 468 bge htab_pte_insert_ok /* Insertion successful */ 469 cmpdi 0,r3,-2 /* Critical failure */ 470 beq- htab_pte_insert_failure 471 472 /* Now try secondary slot */ 473 474 /* real page number in r5, PTE RPN value + index */ 475 rldicl r5,r31,64-PTE_RPN_SHIFT,PTE_RPN_SHIFT 476 sldi r5,r5,PAGE_SHIFT-HW_PAGE_SHIFT 477 add r5,r5,r25 478 sldi r5,r5,HW_PAGE_SHIFT 479 480 /* Calculate secondary group hash */ 481 andc r0,r27,r28 482 rldicr r3,r0,3,63-3 /* r0 = (~hash & mask) << 3 */ 483 484 /* Call ppc_md.hpte_insert */ 485 ld r6,STK_PARM(r4)(r1) /* Retreive new pp bits */ 486 mr r4,r29 /* Retreive va */ 487 li r7,HPTE_V_SECONDARY /* !bolted, secondary */ 488 li r8,MMU_PAGE_4K /* page size */ 489_GLOBAL(htab_call_hpte_insert2) 490 bl . /* patched by htab_finish_init() */ 491 cmpdi 0,r3,0 492 bge+ htab_pte_insert_ok /* Insertion successful */ 493 cmpdi 0,r3,-2 /* Critical failure */ 494 beq- htab_pte_insert_failure 495 496 /* Both are full, we need to evict something */ 497 mftb r0 498 /* Pick a random group based on TB */ 499 andi. r0,r0,1 500 mr r5,r28 501 bne 2f 502 not r5,r5 5032: and r0,r5,r27 504 rldicr r3,r0,3,63-3 /* r0 = (hash & mask) << 3 */ 505 /* Call ppc_md.hpte_remove */ 506_GLOBAL(htab_call_hpte_remove) 507 bl . /* patched by htab_finish_init() */ 508 509 /* Try all again */ 510 b htab_insert_pte 511 512 /* 513 * Call out to C code to invalidate an 64k HW HPTE that is 514 * useless now that the segment has been switched to 4k pages. 515 */ 516htab_inval_old_hpte: 517 mr r3,r29 /* virtual addr */ 518 mr r4,r31 /* PTE.pte */ 519 li r5,0 /* PTE.hidx */ 520 li r6,MMU_PAGE_64K /* psize */ 521 ld r7,STK_PARM(r8)(r1) /* local */ 522 bl .flush_hash_page 523 b htab_insert_pte 524 525htab_bail_ok: 526 li r3,0 527 b htab_bail 528 529htab_pte_insert_ok: 530 /* Insert slot number & secondary bit in PTE second half, 531 * clear _PAGE_BUSY and set approriate HPTE slot bit 532 */ 533 ld r6,STK_PARM(r6)(r1) 534 li r0,_PAGE_BUSY 535 andc r30,r30,r0 536 /* HPTE SUB bit */ 537 li r0,1 538 subfic r5,r25,27 /* Must match bit position in */ 539 sld r0,r0,r5 /* pgtable.h */ 540 or r30,r30,r0 541 /* hindx */ 542 sldi r5,r25,2 543 sld r3,r3,r5 544 li r4,0xf 545 sld r4,r4,r5 546 andc r26,r26,r4 547 or r26,r26,r3 548 ori r5,r6,0x8000 549 std r26,0(r5) 550 lwsync 551 std r30,0(r6) 552 li r3, 0 553htab_bail: 554 ld r25,STK_REG(r25)(r1) 555 ld r26,STK_REG(r26)(r1) 556 ld r27,STK_REG(r27)(r1) 557 ld r28,STK_REG(r28)(r1) 558 ld r29,STK_REG(r29)(r1) 559 ld r30,STK_REG(r30)(r1) 560 ld r31,STK_REG(r31)(r1) 561 addi r1,r1,STACKFRAMESIZE 562 ld r0,16(r1) 563 mtlr r0 564 blr 565 566htab_modify_pte: 567 /* Keep PP bits in r4 and slot idx from the PTE around in r3 */ 568 mr r4,r3 569 sldi r5,r25,2 570 srd r3,r26,r5 571 572 /* Secondary group ? if yes, get a inverted hash value */ 573 mr r5,r28 574 andi. r0,r3,0x8 /* page secondary ? */ 575 beq 1f 576 not r5,r5 5771: andi. r3,r3,0x7 /* extract idx alone */ 578 579 /* Calculate proper slot value for ppc_md.hpte_updatepp */ 580 and r0,r5,r27 581 rldicr r0,r0,3,63-3 /* r0 = (hash & mask) << 3 */ 582 add r3,r0,r3 /* add slot idx */ 583 584 /* Call ppc_md.hpte_updatepp */ 585 mr r5,r29 /* va */ 586 li r6,MMU_PAGE_4K /* page size */ 587 ld r7,STK_PARM(r8)(r1) /* get "local" param */ 588_GLOBAL(htab_call_hpte_updatepp) 589 bl . /* patched by htab_finish_init() */ 590 591 /* if we failed because typically the HPTE wasn't really here 592 * we try an insertion. 593 */ 594 cmpdi 0,r3,-1 595 beq- htab_insert_pte 596 597 /* Clear the BUSY bit and Write out the PTE */ 598 li r0,_PAGE_BUSY 599 andc r30,r30,r0 600 ld r6,STK_PARM(r6)(r1) 601 std r30,0(r6) 602 li r3,0 603 b htab_bail 604 605htab_wrong_access: 606 /* Bail out clearing reservation */ 607 stdcx. r31,0,r6 608 li r3,1 609 b htab_bail 610 611htab_pte_insert_failure: 612 /* Bail out restoring old PTE */ 613 ld r6,STK_PARM(r6)(r1) 614 std r31,0(r6) 615 li r3,-1 616 b htab_bail 617 618#endif /* CONFIG_PPC_64K_PAGES */ 619 620#ifdef CONFIG_PPC_HAS_HASH_64K 621 622/***************************************************************************** 623 * * 624 * 64K SW & 64K HW in a 64K segment pages implementation * 625 * * 626 *****************************************************************************/ 627 628_GLOBAL(__hash_page_64K) 629 mflr r0 630 std r0,16(r1) 631 stdu r1,-STACKFRAMESIZE(r1) 632 /* Save all params that we need after a function call */ 633 std r6,STK_PARM(r6)(r1) 634 std r8,STK_PARM(r8)(r1) 635 636 /* Add _PAGE_PRESENT to access */ 637 ori r4,r4,_PAGE_PRESENT 638 639 /* Save non-volatile registers. 640 * r31 will hold "old PTE" 641 * r30 is "new PTE" 642 * r29 is "va" 643 * r28 is a hash value 644 * r27 is hashtab mask (maybe dynamic patched instead ?) 645 */ 646 std r27,STK_REG(r27)(r1) 647 std r28,STK_REG(r28)(r1) 648 std r29,STK_REG(r29)(r1) 649 std r30,STK_REG(r30)(r1) 650 std r31,STK_REG(r31)(r1) 651 652 /* Step 1: 653 * 654 * Check permissions, atomically mark the linux PTE busy 655 * and hashed. 656 */ 6571: 658 ldarx r31,0,r6 659 /* Check access rights (access & ~(pte_val(*ptep))) */ 660 andc. r0,r4,r31 661 bne- ht64_wrong_access 662 /* Check if PTE is busy */ 663 andi. r0,r31,_PAGE_BUSY 664 /* If so, just bail out and refault if needed. Someone else 665 * is changing this PTE anyway and might hash it. 666 */ 667 bne- ht64_bail_ok 668BEGIN_FTR_SECTION 669 /* Check if PTE has the cache-inhibit bit set */ 670 andi. r0,r31,_PAGE_NO_CACHE 671 /* If so, bail out and refault as a 4k page */ 672 bne- ht64_bail_ok 673END_FTR_SECTION_IFCLR(CPU_FTR_CI_LARGE_PAGE) 674 /* Prepare new PTE value (turn access RW into DIRTY, then 675 * add BUSY,HASHPTE and ACCESSED) 676 */ 677 rlwinm r30,r4,32-9+7,31-7,31-7 /* _PAGE_RW -> _PAGE_DIRTY */ 678 or r30,r30,r31 679 ori r30,r30,_PAGE_BUSY | _PAGE_ACCESSED | _PAGE_HASHPTE 680 /* Write the linux PTE atomically (setting busy) */ 681 stdcx. r30,0,r6 682 bne- 1b 683 isync 684 685 /* Step 2: 686 * 687 * Insert/Update the HPTE in the hash table. At this point, 688 * r4 (access) is re-useable, we use it for the new HPTE flags 689 */ 690 691 /* Calc va and put it in r29 */ 692 rldicr r29,r5,28,63-28 693 rldicl r3,r3,0,36 694 or r29,r3,r29 695 696 /* Calculate hash value for primary slot and store it in r28 */ 697 rldicl r5,r5,0,25 /* vsid & 0x0000007fffffffff */ 698 rldicl r0,r3,64-16,52 /* (ea >> 16) & 0xfff */ 699 xor r28,r5,r0 700 701 /* Convert linux PTE bits into HW equivalents */ 702 andi. r3,r30,0x1fe /* Get basic set of flags */ 703 xori r3,r3,HPTE_R_N /* _PAGE_EXEC -> NOEXEC */ 704 rlwinm r0,r30,32-9+1,30,30 /* _PAGE_RW -> _PAGE_USER (r0) */ 705 rlwinm r4,r30,32-7+1,30,30 /* _PAGE_DIRTY -> _PAGE_USER (r4) */ 706 and r0,r0,r4 /* _PAGE_RW & _PAGE_DIRTY ->r0 bit 30*/ 707 andc r0,r30,r0 /* r0 = pte & ~r0 */ 708 rlwimi r3,r0,32-1,31,31 /* Insert result into PP lsb */ 709 ori r3,r3,HPTE_R_C /* Always add "C" bit for perf. */ 710 711 /* We eventually do the icache sync here (maybe inline that 712 * code rather than call a C function...) 713 */ 714BEGIN_FTR_SECTION 715 mr r4,r30 716 mr r5,r7 717 bl .hash_page_do_lazy_icache 718END_FTR_SECTION(CPU_FTR_NOEXECUTE|CPU_FTR_COHERENT_ICACHE, CPU_FTR_NOEXECUTE) 719 720 /* At this point, r3 contains new PP bits, save them in 721 * place of "access" in the param area (sic) 722 */ 723 std r3,STK_PARM(r4)(r1) 724 725 /* Get htab_hash_mask */ 726 ld r4,htab_hash_mask@got(2) 727 ld r27,0(r4) /* htab_hash_mask -> r27 */ 728 729 /* Check if we may already be in the hashtable, in this case, we 730 * go to out-of-line code to try to modify the HPTE 731 */ 732 andi. r0,r31,_PAGE_HASHPTE 733 bne ht64_modify_pte 734 735ht64_insert_pte: 736 /* Clear hpte bits in new pte (we also clear BUSY btw) and 737 * add _PAGE_HASHPTE 738 */ 739 lis r0,_PAGE_HPTEFLAGS@h 740 ori r0,r0,_PAGE_HPTEFLAGS@l 741 andc r30,r30,r0 742 ori r30,r30,_PAGE_HASHPTE 743 744 /* Phyical address in r5 */ 745 rldicl r5,r31,64-PTE_RPN_SHIFT,PTE_RPN_SHIFT 746 sldi r5,r5,PAGE_SHIFT 747 748 /* Calculate primary group hash */ 749 and r0,r28,r27 750 rldicr r3,r0,3,63-3 /* r0 = (hash & mask) << 3 */ 751 752 /* Call ppc_md.hpte_insert */ 753 ld r6,STK_PARM(r4)(r1) /* Retreive new pp bits */ 754 mr r4,r29 /* Retreive va */ 755 li r7,0 /* !bolted, !secondary */ 756 li r8,MMU_PAGE_64K 757_GLOBAL(ht64_call_hpte_insert1) 758 bl . /* patched by htab_finish_init() */ 759 cmpdi 0,r3,0 760 bge ht64_pte_insert_ok /* Insertion successful */ 761 cmpdi 0,r3,-2 /* Critical failure */ 762 beq- ht64_pte_insert_failure 763 764 /* Now try secondary slot */ 765 766 /* Phyical address in r5 */ 767 rldicl r5,r31,64-PTE_RPN_SHIFT,PTE_RPN_SHIFT 768 sldi r5,r5,PAGE_SHIFT 769 770 /* Calculate secondary group hash */ 771 andc r0,r27,r28 772 rldicr r3,r0,3,63-3 /* r0 = (~hash & mask) << 3 */ 773 774 /* Call ppc_md.hpte_insert */ 775 ld r6,STK_PARM(r4)(r1) /* Retreive new pp bits */ 776 mr r4,r29 /* Retreive va */ 777 li r7,HPTE_V_SECONDARY /* !bolted, secondary */ 778 li r8,MMU_PAGE_64K 779_GLOBAL(ht64_call_hpte_insert2) 780 bl . /* patched by htab_finish_init() */ 781 cmpdi 0,r3,0 782 bge+ ht64_pte_insert_ok /* Insertion successful */ 783 cmpdi 0,r3,-2 /* Critical failure */ 784 beq- ht64_pte_insert_failure 785 786 /* Both are full, we need to evict something */ 787 mftb r0 788 /* Pick a random group based on TB */ 789 andi. r0,r0,1 790 mr r5,r28 791 bne 2f 792 not r5,r5 7932: and r0,r5,r27 794 rldicr r3,r0,3,63-3 /* r0 = (hash & mask) << 3 */ 795 /* Call ppc_md.hpte_remove */ 796_GLOBAL(ht64_call_hpte_remove) 797 bl . /* patched by htab_finish_init() */ 798 799 /* Try all again */ 800 b ht64_insert_pte 801 802ht64_bail_ok: 803 li r3,0 804 b ht64_bail 805 806ht64_pte_insert_ok: 807 /* Insert slot number & secondary bit in PTE */ 808 rldimi r30,r3,12,63-15 809 810 /* Write out the PTE with a normal write 811 * (maybe add eieio may be good still ?) 812 */ 813ht64_write_out_pte: 814 ld r6,STK_PARM(r6)(r1) 815 std r30,0(r6) 816 li r3, 0 817ht64_bail: 818 ld r27,STK_REG(r27)(r1) 819 ld r28,STK_REG(r28)(r1) 820 ld r29,STK_REG(r29)(r1) 821 ld r30,STK_REG(r30)(r1) 822 ld r31,STK_REG(r31)(r1) 823 addi r1,r1,STACKFRAMESIZE 824 ld r0,16(r1) 825 mtlr r0 826 blr 827 828ht64_modify_pte: 829 /* Keep PP bits in r4 and slot idx from the PTE around in r3 */ 830 mr r4,r3 831 rlwinm r3,r31,32-12,29,31 832 833 /* Secondary group ? if yes, get a inverted hash value */ 834 mr r5,r28 835 andi. r0,r31,_PAGE_F_SECOND 836 beq 1f 837 not r5,r5 8381: 839 /* Calculate proper slot value for ppc_md.hpte_updatepp */ 840 and r0,r5,r27 841 rldicr r0,r0,3,63-3 /* r0 = (hash & mask) << 3 */ 842 add r3,r0,r3 /* add slot idx */ 843 844 /* Call ppc_md.hpte_updatepp */ 845 mr r5,r29 /* va */ 846 li r6,MMU_PAGE_64K 847 ld r7,STK_PARM(r8)(r1) /* get "local" param */ 848_GLOBAL(ht64_call_hpte_updatepp) 849 bl . /* patched by htab_finish_init() */ 850 851 /* if we failed because typically the HPTE wasn't really here 852 * we try an insertion. 853 */ 854 cmpdi 0,r3,-1 855 beq- ht64_insert_pte 856 857 /* Clear the BUSY bit and Write out the PTE */ 858 li r0,_PAGE_BUSY 859 andc r30,r30,r0 860 b ht64_write_out_pte 861 862ht64_wrong_access: 863 /* Bail out clearing reservation */ 864 stdcx. r31,0,r6 865 li r3,1 866 b ht64_bail 867 868ht64_pte_insert_failure: 869 /* Bail out restoring old PTE */ 870 ld r6,STK_PARM(r6)(r1) 871 std r31,0(r6) 872 li r3,-1 873 b ht64_bail 874 875 876#endif /* CONFIG_PPC_HAS_HASH_64K */ 877 878 879/***************************************************************************** 880 * * 881 * Huge pages implementation is in hugetlbpage.c * 882 * * 883 *****************************************************************************/ 884