1/* 2 * This file is subject to the terms and conditions of the GNU General Public 3 * License. See the file "COPYING" in the main directory of this archive 4 * for more details. 5 * 6 * arch/sh64/mm/fault.c 7 * 8 * Copyright (C) 2000, 2001 Paolo Alberelli 9 * Copyright (C) 2003 Richard Curnow (/proc/tlb, bug fixes) 10 * Copyright (C) 2003 Paul Mundt 11 * 12 */ 13 14#include <linux/signal.h> 15#include <linux/rwsem.h> 16#include <linux/sched.h> 17#include <linux/kernel.h> 18#include <linux/errno.h> 19#include <linux/string.h> 20#include <linux/types.h> 21#include <linux/ptrace.h> 22#include <linux/mman.h> 23#include <linux/mm.h> 24#include <linux/smp.h> 25#include <linux/interrupt.h> 26 27#include <asm/system.h> 28#include <asm/io.h> 29#include <asm/tlb.h> 30#include <asm/uaccess.h> 31#include <asm/pgalloc.h> 32#include <asm/mmu_context.h> 33#include <asm/registers.h> /* required by inline asm statements */ 34 35#if defined(CONFIG_SH64_PROC_TLB) 36#include <linux/init.h> 37#include <linux/proc_fs.h> 38/* Count numbers of tlb refills in each region */ 39static unsigned long long calls_to_update_mmu_cache = 0ULL; 40static unsigned long long calls_to_flush_tlb_page = 0ULL; 41static unsigned long long calls_to_flush_tlb_range = 0ULL; 42static unsigned long long calls_to_flush_tlb_mm = 0ULL; 43static unsigned long long calls_to_flush_tlb_all = 0ULL; 44unsigned long long calls_to_do_slow_page_fault = 0ULL; 45unsigned long long calls_to_do_fast_page_fault = 0ULL; 46 47/* Count size of ranges for flush_tlb_range */ 48static unsigned long long flush_tlb_range_1 = 0ULL; 49static unsigned long long flush_tlb_range_2 = 0ULL; 50static unsigned long long flush_tlb_range_3_4 = 0ULL; 51static unsigned long long flush_tlb_range_5_7 = 0ULL; 52static unsigned long long flush_tlb_range_8_11 = 0ULL; 53static unsigned long long flush_tlb_range_12_15 = 0ULL; 54static unsigned long long flush_tlb_range_16_up = 0ULL; 55 56static unsigned long long page_not_present = 0ULL; 57 58#endif 59 60extern void die(const char *,struct pt_regs *,long); 61 62#define PFLAG(val,flag) (( (val) & (flag) ) ? #flag : "" ) 63#define PPROT(flag) PFLAG(pgprot_val(prot),flag) 64 65static inline void print_prots(pgprot_t prot) 66{ 67 printk("prot is 0x%08lx\n",pgprot_val(prot)); 68 69 printk("%s %s %s %s %s\n",PPROT(_PAGE_SHARED),PPROT(_PAGE_READ), 70 PPROT(_PAGE_EXECUTE),PPROT(_PAGE_WRITE),PPROT(_PAGE_USER)); 71} 72 73static inline void print_vma(struct vm_area_struct *vma) 74{ 75 printk("vma start 0x%08lx\n", vma->vm_start); 76 printk("vma end 0x%08lx\n", vma->vm_end); 77 78 print_prots(vma->vm_page_prot); 79 printk("vm_flags 0x%08lx\n", vma->vm_flags); 80} 81 82static inline void print_task(struct task_struct *tsk) 83{ 84 printk("Task pid %d\n", tsk->pid); 85} 86 87static pte_t *lookup_pte(struct mm_struct *mm, unsigned long address) 88{ 89 pgd_t *dir; 90 pmd_t *pmd; 91 pte_t *pte; 92 pte_t entry; 93 94 dir = pgd_offset(mm, address); 95 if (pgd_none(*dir)) { 96 return NULL; 97 } 98 99 pmd = pmd_offset(dir, address); 100 if (pmd_none(*pmd)) { 101 return NULL; 102 } 103 104 pte = pte_offset_kernel(pmd, address); 105 entry = *pte; 106 107 if (pte_none(entry)) { 108 return NULL; 109 } 110 if (!pte_present(entry)) { 111 return NULL; 112 } 113 114 return pte; 115} 116 117/* 118 * This routine handles page faults. It determines the address, 119 * and the problem, and then passes it off to one of the appropriate 120 * routines. 121 */ 122asmlinkage void do_page_fault(struct pt_regs *regs, unsigned long writeaccess, 123 unsigned long textaccess, unsigned long address) 124{ 125 struct task_struct *tsk; 126 struct mm_struct *mm; 127 struct vm_area_struct * vma; 128 const struct exception_table_entry *fixup; 129 pte_t *pte; 130 131#if defined(CONFIG_SH64_PROC_TLB) 132 ++calls_to_do_slow_page_fault; 133#endif 134 135 /* SIM 136 * Note this is now called with interrupts still disabled 137 * This is to cope with being called for a missing IO port 138 * address with interrupts disabled. This should be fixed as 139 * soon as we have a better 'fast path' miss handler. 140 * 141 * Plus take care how you try and debug this stuff. 142 * For example, writing debug data to a port which you 143 * have just faulted on is not going to work. 144 */ 145 146 tsk = current; 147 mm = tsk->mm; 148 149 /* Not an IO address, so reenable interrupts */ 150 local_irq_enable(); 151 152 /* 153 * If we're in an interrupt or have no user 154 * context, we must not take the fault.. 155 */ 156 if (in_atomic() || !mm) 157 goto no_context; 158 159 /* TLB misses upon some cache flushes get done under cli() */ 160 down_read(&mm->mmap_sem); 161 162 vma = find_vma(mm, address); 163 164 if (!vma) { 165#ifdef DEBUG_FAULT 166 print_task(tsk); 167 printk("%s:%d fault, address is 0x%08x PC %016Lx textaccess %d writeaccess %d\n", 168 __FUNCTION__,__LINE__, 169 address,regs->pc,textaccess,writeaccess); 170 show_regs(regs); 171#endif 172 goto bad_area; 173 } 174 if (vma->vm_start <= address) { 175 goto good_area; 176 } 177 178 if (!(vma->vm_flags & VM_GROWSDOWN)) { 179#ifdef DEBUG_FAULT 180 print_task(tsk); 181 printk("%s:%d fault, address is 0x%08x PC %016Lx textaccess %d writeaccess %d\n", 182 __FUNCTION__,__LINE__, 183 address,regs->pc,textaccess,writeaccess); 184 show_regs(regs); 185 186 print_vma(vma); 187#endif 188 goto bad_area; 189 } 190 if (expand_stack(vma, address)) { 191#ifdef DEBUG_FAULT 192 print_task(tsk); 193 printk("%s:%d fault, address is 0x%08x PC %016Lx textaccess %d writeaccess %d\n", 194 __FUNCTION__,__LINE__, 195 address,regs->pc,textaccess,writeaccess); 196 show_regs(regs); 197#endif 198 goto bad_area; 199 } 200/* 201 * Ok, we have a good vm_area for this memory access, so 202 * we can handle it.. 203 */ 204good_area: 205 if (textaccess) { 206 if (!(vma->vm_flags & VM_EXEC)) 207 goto bad_area; 208 } else { 209 if (writeaccess) { 210 if (!(vma->vm_flags & VM_WRITE)) 211 goto bad_area; 212 } else { 213 if (!(vma->vm_flags & VM_READ)) 214 goto bad_area; 215 } 216 } 217 218 /* 219 * If for any reason at all we couldn't handle the fault, 220 * make sure we exit gracefully rather than endlessly redo 221 * the fault. 222 */ 223survive: 224 switch (handle_mm_fault(mm, vma, address, writeaccess)) { 225 case VM_FAULT_MINOR: 226 tsk->min_flt++; 227 break; 228 case VM_FAULT_MAJOR: 229 tsk->maj_flt++; 230 break; 231 case VM_FAULT_SIGBUS: 232 goto do_sigbus; 233 default: 234 goto out_of_memory; 235 } 236 /* If we get here, the page fault has been handled. Do the TLB refill 237 now from the newly-setup PTE, to avoid having to fault again right 238 away on the same instruction. */ 239 pte = lookup_pte (mm, address); 240 if (!pte) { 241 /* From empirical evidence, we can get here, due to 242 !pte_present(pte). (e.g. if a swap-in occurs, and the page 243 is swapped back out again before the process that wanted it 244 gets rescheduled?) */ 245 goto no_pte; 246 } 247 248 __do_tlb_refill(address, textaccess, pte); 249 250no_pte: 251 252 up_read(&mm->mmap_sem); 253 return; 254 255/* 256 * Something tried to access memory that isn't in our memory map.. 257 * Fix it, but check if it's kernel or user first.. 258 */ 259bad_area: 260#ifdef DEBUG_FAULT 261 printk("fault:bad area\n"); 262#endif 263 up_read(&mm->mmap_sem); 264 265 if (user_mode(regs)) { 266 static int count=0; 267 siginfo_t info; 268 if (count < 4) { 269 /* This is really to help debug faults when starting 270 * usermode, so only need a few */ 271 count++; 272 printk("user mode bad_area address=%08lx pid=%d (%s) pc=%08lx\n", 273 address, current->pid, current->comm, 274 (unsigned long) regs->pc); 275 } 276 if (is_init(tsk)) { 277 panic("INIT had user mode bad_area\n"); 278 } 279 tsk->thread.address = address; 280 tsk->thread.error_code = writeaccess; 281 info.si_signo = SIGSEGV; 282 info.si_errno = 0; 283 info.si_addr = (void *) address; 284 force_sig_info(SIGSEGV, &info, tsk); 285 return; 286 } 287 288no_context: 289#ifdef DEBUG_FAULT 290 printk("fault:No context\n"); 291#endif 292 /* Are we prepared to handle this kernel fault? */ 293 fixup = search_exception_tables(regs->pc); 294 if (fixup) { 295 regs->pc = fixup->fixup; 296 return; 297 } 298 299/* 300 * Oops. The kernel tried to access some bad page. We'll have to 301 * terminate things with extreme prejudice. 302 * 303 */ 304 if (address < PAGE_SIZE) 305 printk(KERN_ALERT "Unable to handle kernel NULL pointer dereference"); 306 else 307 printk(KERN_ALERT "Unable to handle kernel paging request"); 308 printk(" at virtual address %08lx\n", address); 309 printk(KERN_ALERT "pc = %08Lx%08Lx\n", regs->pc >> 32, regs->pc & 0xffffffff); 310 die("Oops", regs, writeaccess); 311 do_exit(SIGKILL); 312 313/* 314 * We ran out of memory, or some other thing happened to us that made 315 * us unable to handle the page fault gracefully. 316 */ 317out_of_memory: 318 if (is_init(current)) { 319 panic("INIT out of memory\n"); 320 yield(); 321 goto survive; 322 } 323 printk("fault:Out of memory\n"); 324 up_read(&mm->mmap_sem); 325 if (is_init(current)) { 326 yield(); 327 down_read(&mm->mmap_sem); 328 goto survive; 329 } 330 printk("VM: killing process %s\n", tsk->comm); 331 if (user_mode(regs)) 332 do_exit(SIGKILL); 333 goto no_context; 334 335do_sigbus: 336 printk("fault:Do sigbus\n"); 337 up_read(&mm->mmap_sem); 338 339 /* 340 * Send a sigbus, regardless of whether we were in kernel 341 * or user mode. 342 */ 343 tsk->thread.address = address; 344 tsk->thread.error_code = writeaccess; 345 tsk->thread.trap_no = 14; 346 force_sig(SIGBUS, tsk); 347 348 /* Kernel mode? Handle exceptions or die */ 349 if (!user_mode(regs)) 350 goto no_context; 351} 352 353 354void flush_tlb_all(void); 355 356void update_mmu_cache(struct vm_area_struct * vma, 357 unsigned long address, pte_t pte) 358{ 359#if defined(CONFIG_SH64_PROC_TLB) 360 ++calls_to_update_mmu_cache; 361#endif 362 363 /* 364 * This appears to get called once for every pte entry that gets 365 * established => I don't think it's efficient to try refilling the 366 * TLBs with the pages - some may not get accessed even. Also, for 367 * executable pages, it is impossible to determine reliably here which 368 * TLB they should be mapped into (or both even). 369 * 370 * So, just do nothing here and handle faults on demand. In the 371 * TLBMISS handling case, the refill is now done anyway after the pte 372 * has been fixed up, so that deals with most useful cases. 373 */ 374} 375 376static void __flush_tlb_page(struct vm_area_struct *vma, unsigned long page) 377{ 378 unsigned long long match, pteh=0, lpage; 379 unsigned long tlb; 380 struct mm_struct *mm; 381 382 mm = vma->vm_mm; 383 384 if (mm->context == NO_CONTEXT) 385 return; 386 387 /* 388 * Sign-extend based on neff. 389 */ 390 lpage = (page & NEFF_SIGN) ? (page | NEFF_MASK) : page; 391 match = ((mm->context & MMU_CONTEXT_ASID_MASK) << PTEH_ASID_SHIFT) | PTEH_VALID; 392 match |= lpage; 393 394 /* Do ITLB : don't bother for pages in non-exectutable VMAs */ 395 if (vma->vm_flags & VM_EXEC) { 396 for_each_itlb_entry(tlb) { 397 asm volatile ("getcfg %1, 0, %0" 398 : "=r" (pteh) 399 : "r" (tlb) ); 400 401 if (pteh == match) { 402 __flush_tlb_slot(tlb); 403 break; 404 } 405 406 } 407 } 408 409 /* Do DTLB : any page could potentially be in here. */ 410 for_each_dtlb_entry(tlb) { 411 asm volatile ("getcfg %1, 0, %0" 412 : "=r" (pteh) 413 : "r" (tlb) ); 414 415 if (pteh == match) { 416 __flush_tlb_slot(tlb); 417 break; 418 } 419 420 } 421} 422 423void flush_tlb_page(struct vm_area_struct *vma, unsigned long page) 424{ 425 unsigned long flags; 426 427#if defined(CONFIG_SH64_PROC_TLB) 428 ++calls_to_flush_tlb_page; 429#endif 430 431 if (vma->vm_mm) { 432 page &= PAGE_MASK; 433 local_irq_save(flags); 434 __flush_tlb_page(vma, page); 435 local_irq_restore(flags); 436 } 437} 438 439void flush_tlb_range(struct vm_area_struct *vma, unsigned long start, 440 unsigned long end) 441{ 442 unsigned long flags; 443 unsigned long long match, pteh=0, pteh_epn, pteh_low; 444 unsigned long tlb; 445 struct mm_struct *mm; 446 447 mm = vma->vm_mm; 448 449#if defined(CONFIG_SH64_PROC_TLB) 450 ++calls_to_flush_tlb_range; 451 452 { 453 unsigned long size = (end - 1) - start; 454 size >>= 12; /* divide by PAGE_SIZE */ 455 size++; /* end=start+4096 => 1 page */ 456 switch (size) { 457 case 1 : flush_tlb_range_1++; break; 458 case 2 : flush_tlb_range_2++; break; 459 case 3 ... 4 : flush_tlb_range_3_4++; break; 460 case 5 ... 7 : flush_tlb_range_5_7++; break; 461 case 8 ... 11 : flush_tlb_range_8_11++; break; 462 case 12 ... 15 : flush_tlb_range_12_15++; break; 463 default : flush_tlb_range_16_up++; break; 464 } 465 } 466#endif 467 468 if (mm->context == NO_CONTEXT) 469 return; 470 471 local_irq_save(flags); 472 473 start &= PAGE_MASK; 474 end &= PAGE_MASK; 475 476 match = ((mm->context & MMU_CONTEXT_ASID_MASK) << PTEH_ASID_SHIFT) | PTEH_VALID; 477 478 /* Flush ITLB */ 479 for_each_itlb_entry(tlb) { 480 asm volatile ("getcfg %1, 0, %0" 481 : "=r" (pteh) 482 : "r" (tlb) ); 483 484 pteh_epn = pteh & PAGE_MASK; 485 pteh_low = pteh & ~PAGE_MASK; 486 487 if (pteh_low == match && pteh_epn >= start && pteh_epn <= end) 488 __flush_tlb_slot(tlb); 489 } 490 491 /* Flush DTLB */ 492 for_each_dtlb_entry(tlb) { 493 asm volatile ("getcfg %1, 0, %0" 494 : "=r" (pteh) 495 : "r" (tlb) ); 496 497 pteh_epn = pteh & PAGE_MASK; 498 pteh_low = pteh & ~PAGE_MASK; 499 500 if (pteh_low == match && pteh_epn >= start && pteh_epn <= end) 501 __flush_tlb_slot(tlb); 502 } 503 504 local_irq_restore(flags); 505} 506 507void flush_tlb_mm(struct mm_struct *mm) 508{ 509 unsigned long flags; 510 511#if defined(CONFIG_SH64_PROC_TLB) 512 ++calls_to_flush_tlb_mm; 513#endif 514 515 if (mm->context == NO_CONTEXT) 516 return; 517 518 local_irq_save(flags); 519 520 mm->context=NO_CONTEXT; 521 if(mm==current->mm) 522 activate_context(mm); 523 524 local_irq_restore(flags); 525 526} 527 528void flush_tlb_all(void) 529{ 530 /* Invalidate all, including shared pages, excluding fixed TLBs */ 531 532 unsigned long flags, tlb; 533 534#if defined(CONFIG_SH64_PROC_TLB) 535 ++calls_to_flush_tlb_all; 536#endif 537 538 local_irq_save(flags); 539 540 /* Flush each ITLB entry */ 541 for_each_itlb_entry(tlb) { 542 __flush_tlb_slot(tlb); 543 } 544 545 /* Flush each DTLB entry */ 546 for_each_dtlb_entry(tlb) { 547 __flush_tlb_slot(tlb); 548 } 549 550 local_irq_restore(flags); 551} 552 553void flush_tlb_kernel_range(unsigned long start, unsigned long end) 554{ 555 flush_tlb_all(); 556} 557 558#if defined(CONFIG_SH64_PROC_TLB) 559/* Procfs interface to read the performance information */ 560 561static int 562tlb_proc_info(char *buf, char **start, off_t fpos, int length, int *eof, void *data) 563{ 564 int len=0; 565 len += sprintf(buf+len, "do_fast_page_fault called %12lld times\n", calls_to_do_fast_page_fault); 566 len += sprintf(buf+len, "do_slow_page_fault called %12lld times\n", calls_to_do_slow_page_fault); 567 len += sprintf(buf+len, "update_mmu_cache called %12lld times\n", calls_to_update_mmu_cache); 568 len += sprintf(buf+len, "flush_tlb_page called %12lld times\n", calls_to_flush_tlb_page); 569 len += sprintf(buf+len, "flush_tlb_range called %12lld times\n", calls_to_flush_tlb_range); 570 len += sprintf(buf+len, "flush_tlb_mm called %12lld times\n", calls_to_flush_tlb_mm); 571 len += sprintf(buf+len, "flush_tlb_all called %12lld times\n", calls_to_flush_tlb_all); 572 len += sprintf(buf+len, "flush_tlb_range_sizes\n" 573 " 1 : %12lld\n" 574 " 2 : %12lld\n" 575 " 3 - 4 : %12lld\n" 576 " 5 - 7 : %12lld\n" 577 " 8 - 11 : %12lld\n" 578 "12 - 15 : %12lld\n" 579 "16+ : %12lld\n", 580 flush_tlb_range_1, flush_tlb_range_2, flush_tlb_range_3_4, 581 flush_tlb_range_5_7, flush_tlb_range_8_11, flush_tlb_range_12_15, 582 flush_tlb_range_16_up); 583 len += sprintf(buf+len, "page not present %12lld times\n", page_not_present); 584 *eof = 1; 585 return len; 586} 587 588static int __init register_proc_tlb(void) 589{ 590 create_proc_read_entry("tlb", 0, NULL, tlb_proc_info, NULL); 591 return 0; 592} 593 594__initcall(register_proc_tlb); 595 596#endif 597