1/* 2 * PPC64 Huge TLB Page Support for hash based MMUs (POWER4 and later) 3 * 4 * Copyright (C) 2003 David Gibson, IBM Corporation. 5 * 6 * Based on the IA-32 version: 7 * Copyright (C) 2002, Rohit Seth <rohit.seth@intel.com> 8 */ 9 10#include <linux/mm.h> 11#include <linux/hugetlb.h> 12#include <asm/pgtable.h> 13#include <asm/pgalloc.h> 14#include <asm/cacheflush.h> 15#include <asm/machdep.h> 16 17int __hash_page_huge(unsigned long ea, unsigned long access, unsigned long vsid, 18 pte_t *ptep, unsigned long trap, int local, int ssize, 19 unsigned int shift, unsigned int mmu_psize) 20{ 21 unsigned long old_pte, new_pte; 22 unsigned long va, rflags, pa, sz; 23 long slot; 24 25 BUG_ON(shift != mmu_psize_defs[mmu_psize].shift); 26 27 /* Search the Linux page table for a match with va */ 28 va = hpt_va(ea, vsid, ssize); 29 30 /* At this point, we have a pte (old_pte) which can be used to build 31 * or update an HPTE. There are 2 cases: 32 * 33 * 1. There is a valid (present) pte with no associated HPTE (this is 34 * the most common case) 35 * 2. There is a valid (present) pte with an associated HPTE. The 36 * current values of the pp bits in the HPTE prevent access 37 * because we are doing software DIRTY bit management and the 38 * page is currently not DIRTY. 39 */ 40 41 42 do { 43 old_pte = pte_val(*ptep); 44 /* If PTE busy, retry the access */ 45 if (unlikely(old_pte & _PAGE_BUSY)) 46 return 0; 47 /* If PTE permissions don't match, take page fault */ 48 if (unlikely(access & ~old_pte)) 49 return 1; 50 /* Try to lock the PTE, add ACCESSED and DIRTY if it was 51 * a write access */ 52 new_pte = old_pte | _PAGE_BUSY | _PAGE_ACCESSED; 53 if (access & _PAGE_RW) 54 new_pte |= _PAGE_DIRTY; 55 } while(old_pte != __cmpxchg_u64((unsigned long *)ptep, 56 old_pte, new_pte)); 57 58 rflags = 0x2 | (!(new_pte & _PAGE_RW)); 59 /* _PAGE_EXEC -> HW_NO_EXEC since it's inverted */ 60 rflags |= ((new_pte & _PAGE_EXEC) ? 0 : HPTE_R_N); 61 sz = ((1UL) << shift); 62 if (!cpu_has_feature(CPU_FTR_COHERENT_ICACHE)) 63 /* No CPU has hugepages but lacks no execute, so we 64 * don't need to worry about that case */ 65 rflags = hash_page_do_lazy_icache(rflags, __pte(old_pte), trap); 66 67 /* Check if pte already has an hpte (case 2) */ 68 if (unlikely(old_pte & _PAGE_HASHPTE)) { 69 /* There MIGHT be an HPTE for this pte */ 70 unsigned long hash, slot; 71 72 hash = hpt_hash(va, shift, ssize); 73 if (old_pte & _PAGE_F_SECOND) 74 hash = ~hash; 75 slot = (hash & htab_hash_mask) * HPTES_PER_GROUP; 76 slot += (old_pte & _PAGE_F_GIX) >> 12; 77 78 if (ppc_md.hpte_updatepp(slot, rflags, va, mmu_psize, 79 ssize, local) == -1) 80 old_pte &= ~_PAGE_HPTEFLAGS; 81 } 82 83 if (likely(!(old_pte & _PAGE_HASHPTE))) { 84 unsigned long hash = hpt_hash(va, shift, ssize); 85 unsigned long hpte_group; 86 87 pa = pte_pfn(__pte(old_pte)) << PAGE_SHIFT; 88 89repeat: 90 hpte_group = ((hash & htab_hash_mask) * 91 HPTES_PER_GROUP) & ~0x7UL; 92 93 /* clear HPTE slot informations in new PTE */ 94#ifdef CONFIG_PPC_64K_PAGES 95 new_pte = (new_pte & ~_PAGE_HPTEFLAGS) | _PAGE_HPTE_SUB0; 96#else 97 new_pte = (new_pte & ~_PAGE_HPTEFLAGS) | _PAGE_HASHPTE; 98#endif 99 /* Add in WIMG bits */ 100 rflags |= (new_pte & (_PAGE_WRITETHRU | _PAGE_NO_CACHE | 101 _PAGE_COHERENT | _PAGE_GUARDED)); 102 103 /* Insert into the hash table, primary slot */ 104 slot = ppc_md.hpte_insert(hpte_group, va, pa, rflags, 0, 105 mmu_psize, ssize); 106 107 /* Primary is full, try the secondary */ 108 if (unlikely(slot == -1)) { 109 hpte_group = ((~hash & htab_hash_mask) * 110 HPTES_PER_GROUP) & ~0x7UL; 111 slot = ppc_md.hpte_insert(hpte_group, va, pa, rflags, 112 HPTE_V_SECONDARY, 113 mmu_psize, ssize); 114 if (slot == -1) { 115 if (mftb() & 0x1) 116 hpte_group = ((hash & htab_hash_mask) * 117 HPTES_PER_GROUP)&~0x7UL; 118 119 ppc_md.hpte_remove(hpte_group); 120 goto repeat; 121 } 122 } 123 124 /* 125 * Hypervisor failure. Restore old pte and return -1 126 * similar to __hash_page_* 127 */ 128 if (unlikely(slot == -2)) { 129 *ptep = __pte(old_pte); 130 hash_failure_debug(ea, access, vsid, trap, ssize, 131 mmu_psize, old_pte); 132 return -1; 133 } 134 135 new_pte |= (slot << 12) & (_PAGE_F_SECOND | _PAGE_F_GIX); 136 } 137 138 /* 139 * No need to use ldarx/stdcx here 140 */ 141 *ptep = __pte(new_pte & ~_PAGE_BUSY); 142 return 0; 143} 144