1/* sun4c.c: Doing in software what should be done in hardware. 2 * 3 * Copyright (C) 1996 David S. Miller (davem@davemloft.net) 4 * Copyright (C) 1996 Eddie C. Dost (ecd@skynet.be) 5 * Copyright (C) 1996 Andrew Tridgell (Andrew.Tridgell@anu.edu.au) 6 * Copyright (C) 1997-2000 Anton Blanchard (anton@samba.org) 7 * Copyright (C) 1998 Jakub Jelinek (jj@sunsite.mff.cuni.cz) 8 */ 9 10#define NR_TASK_BUCKETS 512 11 12#include <linux/kernel.h> 13#include <linux/mm.h> 14#include <linux/init.h> 15#include <linux/slab.h> 16#include <linux/bootmem.h> 17#include <linux/highmem.h> 18#include <linux/fs.h> 19#include <linux/seq_file.h> 20#include <linux/scatterlist.h> 21#include <linux/bitmap.h> 22 23#include <asm/sections.h> 24#include <asm/page.h> 25#include <asm/pgalloc.h> 26#include <asm/pgtable.h> 27#include <asm/vaddrs.h> 28#include <asm/idprom.h> 29#include <asm/machines.h> 30#include <asm/memreg.h> 31#include <asm/processor.h> 32#include <asm/auxio.h> 33#include <asm/io.h> 34#include <asm/oplib.h> 35#include <asm/openprom.h> 36#include <asm/mmu_context.h> 37#include <asm/highmem.h> 38#include <asm/btfixup.h> 39#include <asm/cacheflush.h> 40#include <asm/tlbflush.h> 41 42/* Because of our dynamic kernel TLB miss strategy, and how 43 * our DVMA mapping allocation works, you _MUST_: 44 * 45 * 1) Disable interrupts _and_ not touch any dynamic kernel 46 * memory while messing with kernel MMU state. By 47 * dynamic memory I mean any object which is not in 48 * the kernel image itself or a thread_union (both of 49 * which are locked into the MMU). 50 * 2) Disable interrupts while messing with user MMU state. 51 */ 52 53extern int num_segmaps, num_contexts; 54 55extern unsigned long page_kernel; 56 57/* That's it, we prom_halt() on sun4c if the cache size is something other than 65536. 58 * So let's save some cycles and just use that everywhere except for that bootup 59 * sanity check. 60 */ 61#define SUN4C_VAC_SIZE 65536 62 63#define SUN4C_KERNEL_BUCKETS 32 64 65/* Flushing the cache. */ 66struct sun4c_vac_props sun4c_vacinfo; 67unsigned long sun4c_kernel_faults; 68 69/* Invalidate every sun4c cache line tag. */ 70static void __init sun4c_flush_all(void) 71{ 72 unsigned long begin, end; 73 74 if (sun4c_vacinfo.on) 75 panic("SUN4C: AIEEE, trying to invalidate vac while it is on."); 76 77 /* Clear 'valid' bit in all cache line tags */ 78 begin = AC_CACHETAGS; 79 end = (AC_CACHETAGS + SUN4C_VAC_SIZE); 80 while (begin < end) { 81 __asm__ __volatile__("sta %%g0, [%0] %1\n\t" : : 82 "r" (begin), "i" (ASI_CONTROL)); 83 begin += sun4c_vacinfo.linesize; 84 } 85} 86 87static void sun4c_flush_context_hw(void) 88{ 89 unsigned long end = SUN4C_VAC_SIZE; 90 91 __asm__ __volatile__( 92 "1: addcc %0, -4096, %0\n\t" 93 " bne 1b\n\t" 94 " sta %%g0, [%0] %2" 95 : "=&r" (end) 96 : "0" (end), "i" (ASI_HWFLUSHCONTEXT) 97 : "cc"); 98} 99 100/* Must be called minimally with IRQs disabled. */ 101static void sun4c_flush_segment_hw(unsigned long addr) 102{ 103 if (sun4c_get_segmap(addr) != invalid_segment) { 104 unsigned long vac_size = SUN4C_VAC_SIZE; 105 106 __asm__ __volatile__( 107 "1: addcc %0, -4096, %0\n\t" 108 " bne 1b\n\t" 109 " sta %%g0, [%2 + %0] %3" 110 : "=&r" (vac_size) 111 : "0" (vac_size), "r" (addr), "i" (ASI_HWFLUSHSEG) 112 : "cc"); 113 } 114} 115 116/* File local boot time fixups. */ 117BTFIXUPDEF_CALL(void, sun4c_flush_page, unsigned long) 118BTFIXUPDEF_CALL(void, sun4c_flush_segment, unsigned long) 119BTFIXUPDEF_CALL(void, sun4c_flush_context, void) 120 121#define sun4c_flush_page(addr) BTFIXUP_CALL(sun4c_flush_page)(addr) 122#define sun4c_flush_segment(addr) BTFIXUP_CALL(sun4c_flush_segment)(addr) 123#define sun4c_flush_context() BTFIXUP_CALL(sun4c_flush_context)() 124 125/* Must be called minimally with interrupts disabled. */ 126static void sun4c_flush_page_hw(unsigned long addr) 127{ 128 addr &= PAGE_MASK; 129 if ((int)sun4c_get_pte(addr) < 0) 130 __asm__ __volatile__("sta %%g0, [%0] %1" 131 : : "r" (addr), "i" (ASI_HWFLUSHPAGE)); 132} 133 134/* Don't inline the software version as it eats too many cache lines if expanded. */ 135static void sun4c_flush_context_sw(void) 136{ 137 unsigned long nbytes = SUN4C_VAC_SIZE; 138 unsigned long lsize = sun4c_vacinfo.linesize; 139 140 __asm__ __volatile__( 141 "add %2, %2, %%g1\n\t" 142 "add %2, %%g1, %%g2\n\t" 143 "add %2, %%g2, %%g3\n\t" 144 "add %2, %%g3, %%g4\n\t" 145 "add %2, %%g4, %%g5\n\t" 146 "add %2, %%g5, %%o4\n\t" 147 "add %2, %%o4, %%o5\n" 148 "1:\n\t" 149 "subcc %0, %%o5, %0\n\t" 150 "sta %%g0, [%0] %3\n\t" 151 "sta %%g0, [%0 + %2] %3\n\t" 152 "sta %%g0, [%0 + %%g1] %3\n\t" 153 "sta %%g0, [%0 + %%g2] %3\n\t" 154 "sta %%g0, [%0 + %%g3] %3\n\t" 155 "sta %%g0, [%0 + %%g4] %3\n\t" 156 "sta %%g0, [%0 + %%g5] %3\n\t" 157 "bg 1b\n\t" 158 " sta %%g0, [%1 + %%o4] %3\n" 159 : "=&r" (nbytes) 160 : "0" (nbytes), "r" (lsize), "i" (ASI_FLUSHCTX) 161 : "g1", "g2", "g3", "g4", "g5", "o4", "o5", "cc"); 162} 163 164/* Don't inline the software version as it eats too many cache lines if expanded. */ 165static void sun4c_flush_segment_sw(unsigned long addr) 166{ 167 if (sun4c_get_segmap(addr) != invalid_segment) { 168 unsigned long nbytes = SUN4C_VAC_SIZE; 169 unsigned long lsize = sun4c_vacinfo.linesize; 170 171 __asm__ __volatile__( 172 "add %2, %2, %%g1\n\t" 173 "add %2, %%g1, %%g2\n\t" 174 "add %2, %%g2, %%g3\n\t" 175 "add %2, %%g3, %%g4\n\t" 176 "add %2, %%g4, %%g5\n\t" 177 "add %2, %%g5, %%o4\n\t" 178 "add %2, %%o4, %%o5\n" 179 "1:\n\t" 180 "subcc %1, %%o5, %1\n\t" 181 "sta %%g0, [%0] %6\n\t" 182 "sta %%g0, [%0 + %2] %6\n\t" 183 "sta %%g0, [%0 + %%g1] %6\n\t" 184 "sta %%g0, [%0 + %%g2] %6\n\t" 185 "sta %%g0, [%0 + %%g3] %6\n\t" 186 "sta %%g0, [%0 + %%g4] %6\n\t" 187 "sta %%g0, [%0 + %%g5] %6\n\t" 188 "sta %%g0, [%0 + %%o4] %6\n\t" 189 "bg 1b\n\t" 190 " add %0, %%o5, %0\n" 191 : "=&r" (addr), "=&r" (nbytes), "=&r" (lsize) 192 : "0" (addr), "1" (nbytes), "2" (lsize), 193 "i" (ASI_FLUSHSEG) 194 : "g1", "g2", "g3", "g4", "g5", "o4", "o5", "cc"); 195 } 196} 197 198/* Don't inline the software version as it eats too many cache lines if expanded. */ 199static void sun4c_flush_page_sw(unsigned long addr) 200{ 201 addr &= PAGE_MASK; 202 if ((sun4c_get_pte(addr) & (_SUN4C_PAGE_NOCACHE | _SUN4C_PAGE_VALID)) == 203 _SUN4C_PAGE_VALID) { 204 unsigned long left = PAGE_SIZE; 205 unsigned long lsize = sun4c_vacinfo.linesize; 206 207 __asm__ __volatile__( 208 "add %2, %2, %%g1\n\t" 209 "add %2, %%g1, %%g2\n\t" 210 "add %2, %%g2, %%g3\n\t" 211 "add %2, %%g3, %%g4\n\t" 212 "add %2, %%g4, %%g5\n\t" 213 "add %2, %%g5, %%o4\n\t" 214 "add %2, %%o4, %%o5\n" 215 "1:\n\t" 216 "subcc %1, %%o5, %1\n\t" 217 "sta %%g0, [%0] %6\n\t" 218 "sta %%g0, [%0 + %2] %6\n\t" 219 "sta %%g0, [%0 + %%g1] %6\n\t" 220 "sta %%g0, [%0 + %%g2] %6\n\t" 221 "sta %%g0, [%0 + %%g3] %6\n\t" 222 "sta %%g0, [%0 + %%g4] %6\n\t" 223 "sta %%g0, [%0 + %%g5] %6\n\t" 224 "sta %%g0, [%0 + %%o4] %6\n\t" 225 "bg 1b\n\t" 226 " add %0, %%o5, %0\n" 227 : "=&r" (addr), "=&r" (left), "=&r" (lsize) 228 : "0" (addr), "1" (left), "2" (lsize), 229 "i" (ASI_FLUSHPG) 230 : "g1", "g2", "g3", "g4", "g5", "o4", "o5", "cc"); 231 } 232} 233 234/* The sun4c's do have an on chip store buffer. And the way you 235 * clear them out isn't so obvious. The only way I can think of 236 * to accomplish this is to read the current context register, 237 * store the same value there, then read an external hardware 238 * register. 239 */ 240void sun4c_complete_all_stores(void) 241{ 242 volatile int _unused; 243 244 _unused = sun4c_get_context(); 245 sun4c_set_context(_unused); 246 _unused = get_auxio(); 247} 248 249/* Bootup utility functions. */ 250static inline void sun4c_init_clean_segmap(unsigned char pseg) 251{ 252 unsigned long vaddr; 253 254 sun4c_put_segmap(0, pseg); 255 for (vaddr = 0; vaddr < SUN4C_REAL_PGDIR_SIZE; vaddr += PAGE_SIZE) 256 sun4c_put_pte(vaddr, 0); 257 sun4c_put_segmap(0, invalid_segment); 258} 259 260static inline void sun4c_init_clean_mmu(unsigned long kernel_end) 261{ 262 unsigned long vaddr; 263 unsigned char savectx, ctx; 264 265 savectx = sun4c_get_context(); 266 for (ctx = 0; ctx < num_contexts; ctx++) { 267 sun4c_set_context(ctx); 268 for (vaddr = 0; vaddr < 0x20000000; vaddr += SUN4C_REAL_PGDIR_SIZE) 269 sun4c_put_segmap(vaddr, invalid_segment); 270 for (vaddr = 0xe0000000; vaddr < KERNBASE; vaddr += SUN4C_REAL_PGDIR_SIZE) 271 sun4c_put_segmap(vaddr, invalid_segment); 272 for (vaddr = kernel_end; vaddr < KADB_DEBUGGER_BEGVM; vaddr += SUN4C_REAL_PGDIR_SIZE) 273 sun4c_put_segmap(vaddr, invalid_segment); 274 for (vaddr = LINUX_OPPROM_ENDVM; vaddr; vaddr += SUN4C_REAL_PGDIR_SIZE) 275 sun4c_put_segmap(vaddr, invalid_segment); 276 } 277 sun4c_set_context(savectx); 278} 279 280void __init sun4c_probe_vac(void) 281{ 282 sun4c_disable_vac(); 283 284 if ((idprom->id_machtype == (SM_SUN4C | SM_4C_SS1)) || 285 (idprom->id_machtype == (SM_SUN4C | SM_4C_SS1PLUS))) { 286 /* PROM on SS1 lacks this info, to be super safe we 287 * hard code it here since this arch is cast in stone. 288 */ 289 sun4c_vacinfo.num_bytes = 65536; 290 sun4c_vacinfo.linesize = 16; 291 } else { 292 sun4c_vacinfo.num_bytes = 293 prom_getintdefault(prom_root_node, "vac-size", 65536); 294 sun4c_vacinfo.linesize = 295 prom_getintdefault(prom_root_node, "vac-linesize", 16); 296 } 297 sun4c_vacinfo.do_hwflushes = 298 prom_getintdefault(prom_root_node, "vac-hwflush", 0); 299 300 if (sun4c_vacinfo.do_hwflushes == 0) 301 sun4c_vacinfo.do_hwflushes = 302 prom_getintdefault(prom_root_node, "vac_hwflush", 0); 303 304 if (sun4c_vacinfo.num_bytes != 65536) { 305 prom_printf("WEIRD Sun4C VAC cache size, " 306 "tell sparclinux@vger.kernel.org"); 307 prom_halt(); 308 } 309 310 switch (sun4c_vacinfo.linesize) { 311 case 16: 312 sun4c_vacinfo.log2lsize = 4; 313 break; 314 case 32: 315 sun4c_vacinfo.log2lsize = 5; 316 break; 317 default: 318 prom_printf("probe_vac: Didn't expect vac-linesize of %d, halting\n", 319 sun4c_vacinfo.linesize); 320 prom_halt(); 321 }; 322 323 sun4c_flush_all(); 324 sun4c_enable_vac(); 325} 326 327/* Patch instructions for the low level kernel fault handler. */ 328extern unsigned long invalid_segment_patch1, invalid_segment_patch1_ff; 329extern unsigned long invalid_segment_patch2, invalid_segment_patch2_ff; 330extern unsigned long invalid_segment_patch1_1ff, invalid_segment_patch2_1ff; 331extern unsigned long num_context_patch1, num_context_patch1_16; 332extern unsigned long num_context_patch2_16; 333extern unsigned long vac_linesize_patch, vac_linesize_patch_32; 334extern unsigned long vac_hwflush_patch1, vac_hwflush_patch1_on; 335extern unsigned long vac_hwflush_patch2, vac_hwflush_patch2_on; 336 337#define PATCH_INSN(src, dst) do { \ 338 daddr = &(dst); \ 339 iaddr = &(src); \ 340 *daddr = *iaddr; \ 341 } while (0) 342 343static void __init patch_kernel_fault_handler(void) 344{ 345 unsigned long *iaddr, *daddr; 346 347 switch (num_segmaps) { 348 case 128: 349 /* Default, nothing to do. */ 350 break; 351 case 256: 352 PATCH_INSN(invalid_segment_patch1_ff, 353 invalid_segment_patch1); 354 PATCH_INSN(invalid_segment_patch2_ff, 355 invalid_segment_patch2); 356 break; 357 case 512: 358 PATCH_INSN(invalid_segment_patch1_1ff, 359 invalid_segment_patch1); 360 PATCH_INSN(invalid_segment_patch2_1ff, 361 invalid_segment_patch2); 362 break; 363 default: 364 prom_printf("Unhandled number of segmaps: %d\n", 365 num_segmaps); 366 prom_halt(); 367 }; 368 switch (num_contexts) { 369 case 8: 370 /* Default, nothing to do. */ 371 break; 372 case 16: 373 PATCH_INSN(num_context_patch1_16, 374 num_context_patch1); 375 break; 376 default: 377 prom_printf("Unhandled number of contexts: %d\n", 378 num_contexts); 379 prom_halt(); 380 }; 381 382 if (sun4c_vacinfo.do_hwflushes != 0) { 383 PATCH_INSN(vac_hwflush_patch1_on, vac_hwflush_patch1); 384 PATCH_INSN(vac_hwflush_patch2_on, vac_hwflush_patch2); 385 } else { 386 switch (sun4c_vacinfo.linesize) { 387 case 16: 388 /* Default, nothing to do. */ 389 break; 390 case 32: 391 PATCH_INSN(vac_linesize_patch_32, vac_linesize_patch); 392 break; 393 default: 394 prom_printf("Impossible VAC linesize %d, halting...\n", 395 sun4c_vacinfo.linesize); 396 prom_halt(); 397 }; 398 } 399} 400 401static void __init sun4c_probe_mmu(void) 402{ 403 if ((idprom->id_machtype == (SM_SUN4C | SM_4C_SS1)) || 404 (idprom->id_machtype == (SM_SUN4C | SM_4C_SS1PLUS))) { 405 /* Hardcode these just to be safe, PROM on SS1 does 406 * not have this info available in the root node. 407 */ 408 num_segmaps = 128; 409 num_contexts = 8; 410 } else { 411 num_segmaps = 412 prom_getintdefault(prom_root_node, "mmu-npmg", 128); 413 num_contexts = 414 prom_getintdefault(prom_root_node, "mmu-nctx", 0x8); 415 } 416 patch_kernel_fault_handler(); 417} 418 419volatile unsigned long __iomem *sun4c_memerr_reg = NULL; 420 421void __init sun4c_probe_memerr_reg(void) 422{ 423 int node; 424 struct linux_prom_registers regs[1]; 425 426 node = prom_getchild(prom_root_node); 427 node = prom_searchsiblings(prom_root_node, "memory-error"); 428 if (!node) 429 return; 430 if (prom_getproperty(node, "reg", (char *)regs, sizeof(regs)) <= 0) 431 return; 432 /* hmm I think regs[0].which_io is zero here anyways */ 433 sun4c_memerr_reg = ioremap(regs[0].phys_addr, regs[0].reg_size); 434} 435 436static inline void sun4c_init_ss2_cache_bug(void) 437{ 438 extern unsigned long start; 439 440 if ((idprom->id_machtype == (SM_SUN4C | SM_4C_SS2)) || 441 (idprom->id_machtype == (SM_SUN4C | SM_4C_IPX)) || 442 (idprom->id_machtype == (SM_SUN4C | SM_4C_ELC))) { 443 /* Whee.. */ 444 printk("SS2 cache bug detected, uncaching trap table page\n"); 445 sun4c_flush_page((unsigned int) &start); 446 sun4c_put_pte(((unsigned long) &start), 447 (sun4c_get_pte((unsigned long) &start) | _SUN4C_PAGE_NOCACHE)); 448 } 449} 450 451/* Addr is always aligned on a page boundary for us already. */ 452static int sun4c_map_dma_area(struct device *dev, dma_addr_t *pba, unsigned long va, 453 unsigned long addr, int len) 454{ 455 unsigned long page, end; 456 457 *pba = addr; 458 459 end = PAGE_ALIGN((addr + len)); 460 while (addr < end) { 461 page = va; 462 sun4c_flush_page(page); 463 page -= PAGE_OFFSET; 464 page >>= PAGE_SHIFT; 465 page |= (_SUN4C_PAGE_VALID | _SUN4C_PAGE_DIRTY | 466 _SUN4C_PAGE_NOCACHE | _SUN4C_PAGE_PRIV); 467 sun4c_put_pte(addr, page); 468 addr += PAGE_SIZE; 469 va += PAGE_SIZE; 470 } 471 472 return 0; 473} 474 475static void sun4c_unmap_dma_area(struct device *dev, unsigned long busa, int len) 476{ 477 /* Fortunately for us, bus_addr == uncached_virt in sun4c. */ 478} 479 480/* TLB management. */ 481 482/* Don't change this struct without changing entry.S. This is used 483 * in the in-window kernel fault handler, and you don't want to mess 484 * with that. (See sun4c_fault in entry.S). 485 */ 486struct sun4c_mmu_entry { 487 struct sun4c_mmu_entry *next; 488 struct sun4c_mmu_entry *prev; 489 unsigned long vaddr; 490 unsigned char pseg; 491 unsigned char locked; 492 493 /* For user mappings only, and completely hidden from kernel 494 * TLB miss code. 495 */ 496 unsigned char ctx; 497 struct sun4c_mmu_entry *lru_next; 498 struct sun4c_mmu_entry *lru_prev; 499}; 500 501static struct sun4c_mmu_entry mmu_entry_pool[SUN4C_MAX_SEGMAPS]; 502 503static void __init sun4c_init_mmu_entry_pool(void) 504{ 505 int i; 506 507 for (i=0; i < SUN4C_MAX_SEGMAPS; i++) { 508 mmu_entry_pool[i].pseg = i; 509 mmu_entry_pool[i].next = NULL; 510 mmu_entry_pool[i].prev = NULL; 511 mmu_entry_pool[i].vaddr = 0; 512 mmu_entry_pool[i].locked = 0; 513 mmu_entry_pool[i].ctx = 0; 514 mmu_entry_pool[i].lru_next = NULL; 515 mmu_entry_pool[i].lru_prev = NULL; 516 } 517 mmu_entry_pool[invalid_segment].locked = 1; 518} 519 520static inline void fix_permissions(unsigned long vaddr, unsigned long bits_on, 521 unsigned long bits_off) 522{ 523 unsigned long start, end; 524 525 end = vaddr + SUN4C_REAL_PGDIR_SIZE; 526 for (start = vaddr; start < end; start += PAGE_SIZE) 527 if (sun4c_get_pte(start) & _SUN4C_PAGE_VALID) 528 sun4c_put_pte(start, (sun4c_get_pte(start) | bits_on) & 529 ~bits_off); 530} 531 532static inline void sun4c_init_map_kernelprom(unsigned long kernel_end) 533{ 534 unsigned long vaddr; 535 unsigned char pseg, ctx; 536 537 for (vaddr = KADB_DEBUGGER_BEGVM; 538 vaddr < LINUX_OPPROM_ENDVM; 539 vaddr += SUN4C_REAL_PGDIR_SIZE) { 540 pseg = sun4c_get_segmap(vaddr); 541 if (pseg != invalid_segment) { 542 mmu_entry_pool[pseg].locked = 1; 543 for (ctx = 0; ctx < num_contexts; ctx++) 544 prom_putsegment(ctx, vaddr, pseg); 545 fix_permissions(vaddr, _SUN4C_PAGE_PRIV, 0); 546 } 547 } 548 549 for (vaddr = KERNBASE; vaddr < kernel_end; vaddr += SUN4C_REAL_PGDIR_SIZE) { 550 pseg = sun4c_get_segmap(vaddr); 551 mmu_entry_pool[pseg].locked = 1; 552 for (ctx = 0; ctx < num_contexts; ctx++) 553 prom_putsegment(ctx, vaddr, pseg); 554 fix_permissions(vaddr, _SUN4C_PAGE_PRIV, _SUN4C_PAGE_NOCACHE); 555 } 556} 557 558static void __init sun4c_init_lock_area(unsigned long start, unsigned long end) 559{ 560 int i, ctx; 561 562 while (start < end) { 563 for (i = 0; i < invalid_segment; i++) 564 if (!mmu_entry_pool[i].locked) 565 break; 566 mmu_entry_pool[i].locked = 1; 567 sun4c_init_clean_segmap(i); 568 for (ctx = 0; ctx < num_contexts; ctx++) 569 prom_putsegment(ctx, start, mmu_entry_pool[i].pseg); 570 start += SUN4C_REAL_PGDIR_SIZE; 571 } 572} 573 574/* Don't change this struct without changing entry.S. This is used 575 * in the in-window kernel fault handler, and you don't want to mess 576 * with that. (See sun4c_fault in entry.S). 577 */ 578struct sun4c_mmu_ring { 579 struct sun4c_mmu_entry ringhd; 580 int num_entries; 581}; 582 583static struct sun4c_mmu_ring sun4c_context_ring[SUN4C_MAX_CONTEXTS]; /* used user entries */ 584static struct sun4c_mmu_ring sun4c_ufree_ring; /* free user entries */ 585static struct sun4c_mmu_ring sun4c_ulru_ring; /* LRU user entries */ 586struct sun4c_mmu_ring sun4c_kernel_ring; /* used kernel entries */ 587struct sun4c_mmu_ring sun4c_kfree_ring; /* free kernel entries */ 588 589static inline void sun4c_init_rings(void) 590{ 591 int i; 592 593 for (i = 0; i < SUN4C_MAX_CONTEXTS; i++) { 594 sun4c_context_ring[i].ringhd.next = 595 sun4c_context_ring[i].ringhd.prev = 596 &sun4c_context_ring[i].ringhd; 597 sun4c_context_ring[i].num_entries = 0; 598 } 599 sun4c_ufree_ring.ringhd.next = sun4c_ufree_ring.ringhd.prev = 600 &sun4c_ufree_ring.ringhd; 601 sun4c_ufree_ring.num_entries = 0; 602 sun4c_ulru_ring.ringhd.lru_next = sun4c_ulru_ring.ringhd.lru_prev = 603 &sun4c_ulru_ring.ringhd; 604 sun4c_ulru_ring.num_entries = 0; 605 sun4c_kernel_ring.ringhd.next = sun4c_kernel_ring.ringhd.prev = 606 &sun4c_kernel_ring.ringhd; 607 sun4c_kernel_ring.num_entries = 0; 608 sun4c_kfree_ring.ringhd.next = sun4c_kfree_ring.ringhd.prev = 609 &sun4c_kfree_ring.ringhd; 610 sun4c_kfree_ring.num_entries = 0; 611} 612 613static void add_ring(struct sun4c_mmu_ring *ring, 614 struct sun4c_mmu_entry *entry) 615{ 616 struct sun4c_mmu_entry *head = &ring->ringhd; 617 618 entry->prev = head; 619 (entry->next = head->next)->prev = entry; 620 head->next = entry; 621 ring->num_entries++; 622} 623 624static inline void add_lru(struct sun4c_mmu_entry *entry) 625{ 626 struct sun4c_mmu_ring *ring = &sun4c_ulru_ring; 627 struct sun4c_mmu_entry *head = &ring->ringhd; 628 629 entry->lru_next = head; 630 (entry->lru_prev = head->lru_prev)->lru_next = entry; 631 head->lru_prev = entry; 632} 633 634static void add_ring_ordered(struct sun4c_mmu_ring *ring, 635 struct sun4c_mmu_entry *entry) 636{ 637 struct sun4c_mmu_entry *head = &ring->ringhd; 638 unsigned long addr = entry->vaddr; 639 640 while ((head->next != &ring->ringhd) && (head->next->vaddr < addr)) 641 head = head->next; 642 643 entry->prev = head; 644 (entry->next = head->next)->prev = entry; 645 head->next = entry; 646 ring->num_entries++; 647 648 add_lru(entry); 649} 650 651static inline void remove_ring(struct sun4c_mmu_ring *ring, 652 struct sun4c_mmu_entry *entry) 653{ 654 struct sun4c_mmu_entry *next = entry->next; 655 656 (next->prev = entry->prev)->next = next; 657 ring->num_entries--; 658} 659 660static void remove_lru(struct sun4c_mmu_entry *entry) 661{ 662 struct sun4c_mmu_entry *next = entry->lru_next; 663 664 (next->lru_prev = entry->lru_prev)->lru_next = next; 665} 666 667static void free_user_entry(int ctx, struct sun4c_mmu_entry *entry) 668{ 669 remove_ring(sun4c_context_ring+ctx, entry); 670 remove_lru(entry); 671 add_ring(&sun4c_ufree_ring, entry); 672} 673 674static void free_kernel_entry(struct sun4c_mmu_entry *entry, 675 struct sun4c_mmu_ring *ring) 676{ 677 remove_ring(ring, entry); 678 add_ring(&sun4c_kfree_ring, entry); 679} 680 681static void __init sun4c_init_fill_kernel_ring(int howmany) 682{ 683 int i; 684 685 while (howmany) { 686 for (i = 0; i < invalid_segment; i++) 687 if (!mmu_entry_pool[i].locked) 688 break; 689 mmu_entry_pool[i].locked = 1; 690 sun4c_init_clean_segmap(i); 691 add_ring(&sun4c_kfree_ring, &mmu_entry_pool[i]); 692 howmany--; 693 } 694} 695 696static void __init sun4c_init_fill_user_ring(void) 697{ 698 int i; 699 700 for (i = 0; i < invalid_segment; i++) { 701 if (mmu_entry_pool[i].locked) 702 continue; 703 sun4c_init_clean_segmap(i); 704 add_ring(&sun4c_ufree_ring, &mmu_entry_pool[i]); 705 } 706} 707 708static void sun4c_kernel_unmap(struct sun4c_mmu_entry *kentry) 709{ 710 int savectx, ctx; 711 712 savectx = sun4c_get_context(); 713 for (ctx = 0; ctx < num_contexts; ctx++) { 714 sun4c_set_context(ctx); 715 sun4c_put_segmap(kentry->vaddr, invalid_segment); 716 } 717 sun4c_set_context(savectx); 718} 719 720static void sun4c_kernel_map(struct sun4c_mmu_entry *kentry) 721{ 722 int savectx, ctx; 723 724 savectx = sun4c_get_context(); 725 for (ctx = 0; ctx < num_contexts; ctx++) { 726 sun4c_set_context(ctx); 727 sun4c_put_segmap(kentry->vaddr, kentry->pseg); 728 } 729 sun4c_set_context(savectx); 730} 731 732#define sun4c_user_unmap(__entry) \ 733 sun4c_put_segmap((__entry)->vaddr, invalid_segment) 734 735static void sun4c_demap_context(struct sun4c_mmu_ring *crp, unsigned char ctx) 736{ 737 struct sun4c_mmu_entry *head = &crp->ringhd; 738 unsigned long flags; 739 740 local_irq_save(flags); 741 if (head->next != head) { 742 struct sun4c_mmu_entry *entry = head->next; 743 int savectx = sun4c_get_context(); 744 745 flush_user_windows(); 746 sun4c_set_context(ctx); 747 sun4c_flush_context(); 748 do { 749 struct sun4c_mmu_entry *next = entry->next; 750 751 sun4c_user_unmap(entry); 752 free_user_entry(ctx, entry); 753 754 entry = next; 755 } while (entry != head); 756 sun4c_set_context(savectx); 757 } 758 local_irq_restore(flags); 759} 760 761static int sun4c_user_taken_entries; /* This is how much we have. */ 762static int max_user_taken_entries; /* This limits us and prevents deadlock. */ 763 764static struct sun4c_mmu_entry *sun4c_kernel_strategy(void) 765{ 766 struct sun4c_mmu_entry *this_entry; 767 768 /* If some are free, return first one. */ 769 if (sun4c_kfree_ring.num_entries) { 770 this_entry = sun4c_kfree_ring.ringhd.next; 771 return this_entry; 772 } 773 774 /* Else free one up. */ 775 this_entry = sun4c_kernel_ring.ringhd.prev; 776 sun4c_flush_segment(this_entry->vaddr); 777 sun4c_kernel_unmap(this_entry); 778 free_kernel_entry(this_entry, &sun4c_kernel_ring); 779 this_entry = sun4c_kfree_ring.ringhd.next; 780 781 return this_entry; 782} 783 784/* Using this method to free up mmu entries eliminates a lot of 785 * potential races since we have a kernel that incurs tlb 786 * replacement faults. There may be performance penalties. 787 * 788 * NOTE: Must be called with interrupts disabled. 789 */ 790static struct sun4c_mmu_entry *sun4c_user_strategy(void) 791{ 792 struct sun4c_mmu_entry *entry; 793 unsigned char ctx; 794 int savectx; 795 796 /* If some are free, return first one. */ 797 if (sun4c_ufree_ring.num_entries) { 798 entry = sun4c_ufree_ring.ringhd.next; 799 goto unlink_out; 800 } 801 802 if (sun4c_user_taken_entries) { 803 entry = sun4c_kernel_strategy(); 804 sun4c_user_taken_entries--; 805 goto kunlink_out; 806 } 807 808 /* Grab from the beginning of the LRU list. */ 809 entry = sun4c_ulru_ring.ringhd.lru_next; 810 ctx = entry->ctx; 811 812 savectx = sun4c_get_context(); 813 flush_user_windows(); 814 sun4c_set_context(ctx); 815 sun4c_flush_segment(entry->vaddr); 816 sun4c_user_unmap(entry); 817 remove_ring(sun4c_context_ring + ctx, entry); 818 remove_lru(entry); 819 sun4c_set_context(savectx); 820 821 return entry; 822 823unlink_out: 824 remove_ring(&sun4c_ufree_ring, entry); 825 return entry; 826kunlink_out: 827 remove_ring(&sun4c_kfree_ring, entry); 828 return entry; 829} 830 831/* NOTE: Must be called with interrupts disabled. */ 832void sun4c_grow_kernel_ring(void) 833{ 834 struct sun4c_mmu_entry *entry; 835 836 /* Prevent deadlock condition. */ 837 if (sun4c_user_taken_entries >= max_user_taken_entries) 838 return; 839 840 if (sun4c_ufree_ring.num_entries) { 841 entry = sun4c_ufree_ring.ringhd.next; 842 remove_ring(&sun4c_ufree_ring, entry); 843 add_ring(&sun4c_kfree_ring, entry); 844 sun4c_user_taken_entries++; 845 } 846} 847 848/* 2 page buckets for task struct and kernel stack allocation. 849 * 850 * TASK_STACK_BEGIN 851 * bucket[0] 852 * bucket[1] 853 * [ ... ] 854 * bucket[NR_TASK_BUCKETS-1] 855 * TASK_STACK_BEGIN + (sizeof(struct task_bucket) * NR_TASK_BUCKETS) 856 * 857 * Each slot looks like: 858 * 859 * page 1 -- task struct + beginning of kernel stack 860 * page 2 -- rest of kernel stack 861 */ 862 863union task_union *sun4c_bucket[NR_TASK_BUCKETS]; 864 865static int sun4c_lowbucket_avail; 866 867#define BUCKET_EMPTY ((union task_union *) 0) 868#define BUCKET_SHIFT (PAGE_SHIFT + 1) /* log2(sizeof(struct task_bucket)) */ 869#define BUCKET_SIZE (1 << BUCKET_SHIFT) 870#define BUCKET_NUM(addr) ((((addr) - SUN4C_LOCK_VADDR) >> BUCKET_SHIFT)) 871#define BUCKET_ADDR(num) (((num) << BUCKET_SHIFT) + SUN4C_LOCK_VADDR) 872#define BUCKET_PTE(page) \ 873 ((((page) - PAGE_OFFSET) >> PAGE_SHIFT) | pgprot_val(SUN4C_PAGE_KERNEL)) 874#define BUCKET_PTE_PAGE(pte) \ 875 (PAGE_OFFSET + (((pte) & SUN4C_PFN_MASK) << PAGE_SHIFT)) 876 877static void get_locked_segment(unsigned long addr) 878{ 879 struct sun4c_mmu_entry *stolen; 880 unsigned long flags; 881 882 local_irq_save(flags); 883 addr &= SUN4C_REAL_PGDIR_MASK; 884 stolen = sun4c_user_strategy(); 885 max_user_taken_entries--; 886 stolen->vaddr = addr; 887 flush_user_windows(); 888 sun4c_kernel_map(stolen); 889 local_irq_restore(flags); 890} 891 892static void free_locked_segment(unsigned long addr) 893{ 894 struct sun4c_mmu_entry *entry; 895 unsigned long flags; 896 unsigned char pseg; 897 898 local_irq_save(flags); 899 addr &= SUN4C_REAL_PGDIR_MASK; 900 pseg = sun4c_get_segmap(addr); 901 entry = &mmu_entry_pool[pseg]; 902 903 flush_user_windows(); 904 sun4c_flush_segment(addr); 905 sun4c_kernel_unmap(entry); 906 add_ring(&sun4c_ufree_ring, entry); 907 max_user_taken_entries++; 908 local_irq_restore(flags); 909} 910 911static inline void garbage_collect(int entry) 912{ 913 int start, end; 914 915 /* 32 buckets per segment... */ 916 entry &= ~31; 917 start = entry; 918 for (end = (start + 32); start < end; start++) 919 if (sun4c_bucket[start] != BUCKET_EMPTY) 920 return; 921 922 /* Entire segment empty, release it. */ 923 free_locked_segment(BUCKET_ADDR(entry)); 924} 925 926static struct thread_info *sun4c_alloc_thread_info(void) 927{ 928 unsigned long addr, pages; 929 int entry; 930 931 pages = __get_free_pages(GFP_KERNEL, THREAD_INFO_ORDER); 932 if (!pages) 933 return NULL; 934 935 for (entry = sun4c_lowbucket_avail; entry < NR_TASK_BUCKETS; entry++) 936 if (sun4c_bucket[entry] == BUCKET_EMPTY) 937 break; 938 if (entry == NR_TASK_BUCKETS) { 939 free_pages(pages, THREAD_INFO_ORDER); 940 return NULL; 941 } 942 if (entry >= sun4c_lowbucket_avail) 943 sun4c_lowbucket_avail = entry + 1; 944 945 addr = BUCKET_ADDR(entry); 946 sun4c_bucket[entry] = (union task_union *) addr; 947 if(sun4c_get_segmap(addr) == invalid_segment) 948 get_locked_segment(addr); 949 950 /* We are changing the virtual color of the page(s) 951 * so we must flush the cache to guarantee consistency. 952 */ 953 sun4c_flush_page(pages); 954 sun4c_flush_page(pages + PAGE_SIZE); 955 956 sun4c_put_pte(addr, BUCKET_PTE(pages)); 957 sun4c_put_pte(addr + PAGE_SIZE, BUCKET_PTE(pages + PAGE_SIZE)); 958 959#ifdef CONFIG_DEBUG_STACK_USAGE 960 memset((void *)addr, 0, PAGE_SIZE << THREAD_INFO_ORDER); 961#endif /* DEBUG_STACK_USAGE */ 962 963 return (struct thread_info *) addr; 964} 965 966static void sun4c_free_thread_info(struct thread_info *ti) 967{ 968 unsigned long tiaddr = (unsigned long) ti; 969 unsigned long pages = BUCKET_PTE_PAGE(sun4c_get_pte(tiaddr)); 970 int entry = BUCKET_NUM(tiaddr); 971 972 /* We are deleting a mapping, so the flush here is mandatory. */ 973 sun4c_flush_page(tiaddr); 974 sun4c_flush_page(tiaddr + PAGE_SIZE); 975 976 sun4c_put_pte(tiaddr, 0); 977 sun4c_put_pte(tiaddr + PAGE_SIZE, 0); 978 979 sun4c_bucket[entry] = BUCKET_EMPTY; 980 if (entry < sun4c_lowbucket_avail) 981 sun4c_lowbucket_avail = entry; 982 983 free_pages(pages, THREAD_INFO_ORDER); 984 garbage_collect(entry); 985} 986 987static void __init sun4c_init_buckets(void) 988{ 989 int entry; 990 991 if (sizeof(union thread_union) != (PAGE_SIZE << THREAD_INFO_ORDER)) { 992 extern void thread_info_size_is_bolixed_pete(void); 993 thread_info_size_is_bolixed_pete(); 994 } 995 996 for (entry = 0; entry < NR_TASK_BUCKETS; entry++) 997 sun4c_bucket[entry] = BUCKET_EMPTY; 998 sun4c_lowbucket_avail = 0; 999} 1000 1001static unsigned long sun4c_iobuffer_start; 1002static unsigned long sun4c_iobuffer_end; 1003static unsigned long sun4c_iobuffer_high; 1004static unsigned long *sun4c_iobuffer_map; 1005static int iobuffer_map_size; 1006 1007/* 1008 * Alias our pages so they do not cause a trap. 1009 * Also one page may be aliased into several I/O areas and we may 1010 * finish these I/O separately. 1011 */ 1012static char *sun4c_lockarea(char *vaddr, unsigned long size) 1013{ 1014 unsigned long base, scan; 1015 unsigned long npages; 1016 unsigned long vpage; 1017 unsigned long pte; 1018 unsigned long apage; 1019 unsigned long high; 1020 unsigned long flags; 1021 1022 npages = (((unsigned long)vaddr & ~PAGE_MASK) + 1023 size + (PAGE_SIZE-1)) >> PAGE_SHIFT; 1024 1025 local_irq_save(flags); 1026 base = bitmap_find_next_zero_area(sun4c_iobuffer_map, iobuffer_map_size, 1027 0, npages, 0); 1028 if (base >= iobuffer_map_size) 1029 goto abend; 1030 1031 high = ((base + npages) << PAGE_SHIFT) + sun4c_iobuffer_start; 1032 high = SUN4C_REAL_PGDIR_ALIGN(high); 1033 while (high > sun4c_iobuffer_high) { 1034 get_locked_segment(sun4c_iobuffer_high); 1035 sun4c_iobuffer_high += SUN4C_REAL_PGDIR_SIZE; 1036 } 1037 1038 vpage = ((unsigned long) vaddr) & PAGE_MASK; 1039 for (scan = base; scan < base+npages; scan++) { 1040 pte = ((vpage-PAGE_OFFSET) >> PAGE_SHIFT); 1041 pte |= pgprot_val(SUN4C_PAGE_KERNEL); 1042 pte |= _SUN4C_PAGE_NOCACHE; 1043 set_bit(scan, sun4c_iobuffer_map); 1044 apage = (scan << PAGE_SHIFT) + sun4c_iobuffer_start; 1045 1046 /* Flush original mapping so we see the right things later. */ 1047 sun4c_flush_page(vpage); 1048 1049 sun4c_put_pte(apage, pte); 1050 vpage += PAGE_SIZE; 1051 } 1052 local_irq_restore(flags); 1053 return (char *) ((base << PAGE_SHIFT) + sun4c_iobuffer_start + 1054 (((unsigned long) vaddr) & ~PAGE_MASK)); 1055 1056abend: 1057 local_irq_restore(flags); 1058 printk("DMA vaddr=0x%p size=%08lx\n", vaddr, size); 1059 panic("Out of iobuffer table"); 1060 return NULL; 1061} 1062 1063static void sun4c_unlockarea(char *vaddr, unsigned long size) 1064{ 1065 unsigned long vpage, npages; 1066 unsigned long flags; 1067 int scan, high; 1068 1069 vpage = (unsigned long)vaddr & PAGE_MASK; 1070 npages = (((unsigned long)vaddr & ~PAGE_MASK) + 1071 size + (PAGE_SIZE-1)) >> PAGE_SHIFT; 1072 1073 local_irq_save(flags); 1074 while (npages != 0) { 1075 --npages; 1076 1077 /* This mapping is marked non-cachable, no flush necessary. */ 1078 sun4c_put_pte(vpage, 0); 1079 clear_bit((vpage - sun4c_iobuffer_start) >> PAGE_SHIFT, 1080 sun4c_iobuffer_map); 1081 vpage += PAGE_SIZE; 1082 } 1083 1084 /* garbage collect */ 1085 scan = (sun4c_iobuffer_high - sun4c_iobuffer_start) >> PAGE_SHIFT; 1086 while (scan >= 0 && !sun4c_iobuffer_map[scan >> 5]) 1087 scan -= 32; 1088 scan += 32; 1089 high = sun4c_iobuffer_start + (scan << PAGE_SHIFT); 1090 high = SUN4C_REAL_PGDIR_ALIGN(high) + SUN4C_REAL_PGDIR_SIZE; 1091 while (high < sun4c_iobuffer_high) { 1092 sun4c_iobuffer_high -= SUN4C_REAL_PGDIR_SIZE; 1093 free_locked_segment(sun4c_iobuffer_high); 1094 } 1095 local_irq_restore(flags); 1096} 1097 1098/* Note the scsi code at init time passes to here buffers 1099 * which sit on the kernel stack, those are already locked 1100 * by implication and fool the page locking code above 1101 * if passed to by mistake. 1102 */ 1103static __u32 sun4c_get_scsi_one(struct device *dev, char *bufptr, unsigned long len) 1104{ 1105 unsigned long page; 1106 1107 page = ((unsigned long)bufptr) & PAGE_MASK; 1108 if (!virt_addr_valid(page)) { 1109 sun4c_flush_page(page); 1110 return (__u32)bufptr; /* already locked */ 1111 } 1112 return (__u32)sun4c_lockarea(bufptr, len); 1113} 1114 1115static void sun4c_get_scsi_sgl(struct device *dev, struct scatterlist *sg, int sz) 1116{ 1117 while (sz != 0) { 1118 --sz; 1119 sg->dma_address = (__u32)sun4c_lockarea(sg_virt(sg), sg->length); 1120 sg->dma_length = sg->length; 1121 sg = sg_next(sg); 1122 } 1123} 1124 1125static void sun4c_release_scsi_one(struct device *dev, __u32 bufptr, unsigned long len) 1126{ 1127 if (bufptr < sun4c_iobuffer_start) 1128 return; /* On kernel stack or similar, see above */ 1129 sun4c_unlockarea((char *)bufptr, len); 1130} 1131 1132static void sun4c_release_scsi_sgl(struct device *dev, struct scatterlist *sg, int sz) 1133{ 1134 while (sz != 0) { 1135 --sz; 1136 sun4c_unlockarea((char *)sg->dma_address, sg->length); 1137 sg = sg_next(sg); 1138 } 1139} 1140 1141#define TASK_ENTRY_SIZE BUCKET_SIZE /* see above */ 1142#define LONG_ALIGN(x) (((x)+(sizeof(long))-1)&~((sizeof(long))-1)) 1143 1144struct vm_area_struct sun4c_kstack_vma; 1145 1146static void __init sun4c_init_lock_areas(void) 1147{ 1148 unsigned long sun4c_taskstack_start; 1149 unsigned long sun4c_taskstack_end; 1150 int bitmap_size; 1151 1152 sun4c_init_buckets(); 1153 sun4c_taskstack_start = SUN4C_LOCK_VADDR; 1154 sun4c_taskstack_end = (sun4c_taskstack_start + 1155 (TASK_ENTRY_SIZE * NR_TASK_BUCKETS)); 1156 if (sun4c_taskstack_end >= SUN4C_LOCK_END) { 1157 prom_printf("Too many tasks, decrease NR_TASK_BUCKETS please.\n"); 1158 prom_halt(); 1159 } 1160 1161 sun4c_iobuffer_start = sun4c_iobuffer_high = 1162 SUN4C_REAL_PGDIR_ALIGN(sun4c_taskstack_end); 1163 sun4c_iobuffer_end = SUN4C_LOCK_END; 1164 bitmap_size = (sun4c_iobuffer_end - sun4c_iobuffer_start) >> PAGE_SHIFT; 1165 bitmap_size = (bitmap_size + 7) >> 3; 1166 bitmap_size = LONG_ALIGN(bitmap_size); 1167 iobuffer_map_size = bitmap_size << 3; 1168 sun4c_iobuffer_map = __alloc_bootmem(bitmap_size, SMP_CACHE_BYTES, 0UL); 1169 memset((void *) sun4c_iobuffer_map, 0, bitmap_size); 1170 1171 sun4c_kstack_vma.vm_mm = &init_mm; 1172 sun4c_kstack_vma.vm_start = sun4c_taskstack_start; 1173 sun4c_kstack_vma.vm_end = sun4c_taskstack_end; 1174 sun4c_kstack_vma.vm_page_prot = PAGE_SHARED; 1175 sun4c_kstack_vma.vm_flags = VM_READ | VM_WRITE | VM_EXEC; 1176 insert_vm_struct(&init_mm, &sun4c_kstack_vma); 1177} 1178 1179/* Cache flushing on the sun4c. */ 1180static void sun4c_flush_cache_all(void) 1181{ 1182 unsigned long begin, end; 1183 1184 flush_user_windows(); 1185 begin = (KERNBASE + SUN4C_REAL_PGDIR_SIZE); 1186 end = (begin + SUN4C_VAC_SIZE); 1187 1188 if (sun4c_vacinfo.linesize == 32) { 1189 while (begin < end) { 1190 __asm__ __volatile__( 1191 "ld [%0 + 0x00], %%g0\n\t" 1192 "ld [%0 + 0x20], %%g0\n\t" 1193 "ld [%0 + 0x40], %%g0\n\t" 1194 "ld [%0 + 0x60], %%g0\n\t" 1195 "ld [%0 + 0x80], %%g0\n\t" 1196 "ld [%0 + 0xa0], %%g0\n\t" 1197 "ld [%0 + 0xc0], %%g0\n\t" 1198 "ld [%0 + 0xe0], %%g0\n\t" 1199 "ld [%0 + 0x100], %%g0\n\t" 1200 "ld [%0 + 0x120], %%g0\n\t" 1201 "ld [%0 + 0x140], %%g0\n\t" 1202 "ld [%0 + 0x160], %%g0\n\t" 1203 "ld [%0 + 0x180], %%g0\n\t" 1204 "ld [%0 + 0x1a0], %%g0\n\t" 1205 "ld [%0 + 0x1c0], %%g0\n\t" 1206 "ld [%0 + 0x1e0], %%g0\n" 1207 : : "r" (begin)); 1208 begin += 512; 1209 } 1210 } else { 1211 while (begin < end) { 1212 __asm__ __volatile__( 1213 "ld [%0 + 0x00], %%g0\n\t" 1214 "ld [%0 + 0x10], %%g0\n\t" 1215 "ld [%0 + 0x20], %%g0\n\t" 1216 "ld [%0 + 0x30], %%g0\n\t" 1217 "ld [%0 + 0x40], %%g0\n\t" 1218 "ld [%0 + 0x50], %%g0\n\t" 1219 "ld [%0 + 0x60], %%g0\n\t" 1220 "ld [%0 + 0x70], %%g0\n\t" 1221 "ld [%0 + 0x80], %%g0\n\t" 1222 "ld [%0 + 0x90], %%g0\n\t" 1223 "ld [%0 + 0xa0], %%g0\n\t" 1224 "ld [%0 + 0xb0], %%g0\n\t" 1225 "ld [%0 + 0xc0], %%g0\n\t" 1226 "ld [%0 + 0xd0], %%g0\n\t" 1227 "ld [%0 + 0xe0], %%g0\n\t" 1228 "ld [%0 + 0xf0], %%g0\n" 1229 : : "r" (begin)); 1230 begin += 256; 1231 } 1232 } 1233} 1234 1235static void sun4c_flush_cache_mm(struct mm_struct *mm) 1236{ 1237 int new_ctx = mm->context; 1238 1239 if (new_ctx != NO_CONTEXT) { 1240 flush_user_windows(); 1241 1242 if (sun4c_context_ring[new_ctx].num_entries) { 1243 struct sun4c_mmu_entry *head = &sun4c_context_ring[new_ctx].ringhd; 1244 unsigned long flags; 1245 1246 local_irq_save(flags); 1247 if (head->next != head) { 1248 struct sun4c_mmu_entry *entry = head->next; 1249 int savectx = sun4c_get_context(); 1250 1251 sun4c_set_context(new_ctx); 1252 sun4c_flush_context(); 1253 do { 1254 struct sun4c_mmu_entry *next = entry->next; 1255 1256 sun4c_user_unmap(entry); 1257 free_user_entry(new_ctx, entry); 1258 1259 entry = next; 1260 } while (entry != head); 1261 sun4c_set_context(savectx); 1262 } 1263 local_irq_restore(flags); 1264 } 1265 } 1266} 1267 1268static void sun4c_flush_cache_range(struct vm_area_struct *vma, unsigned long start, unsigned long end) 1269{ 1270 struct mm_struct *mm = vma->vm_mm; 1271 int new_ctx = mm->context; 1272 1273 if (new_ctx != NO_CONTEXT) { 1274 struct sun4c_mmu_entry *head = &sun4c_context_ring[new_ctx].ringhd; 1275 struct sun4c_mmu_entry *entry; 1276 unsigned long flags; 1277 1278 flush_user_windows(); 1279 1280 local_irq_save(flags); 1281 /* All user segmap chains are ordered on entry->vaddr. */ 1282 for (entry = head->next; 1283 (entry != head) && ((entry->vaddr+SUN4C_REAL_PGDIR_SIZE) < start); 1284 entry = entry->next) 1285 ; 1286 1287 /* Tracing various job mixtures showed that this conditional 1288 * only passes ~35% of the time for most worse case situations, 1289 * therefore we avoid all of this gross overhead ~65% of the time. 1290 */ 1291 if ((entry != head) && (entry->vaddr < end)) { 1292 int octx = sun4c_get_context(); 1293 sun4c_set_context(new_ctx); 1294 1295 /* At this point, always, (start >= entry->vaddr) and 1296 * (entry->vaddr < end), once the latter condition 1297 * ceases to hold, or we hit the end of the list, we 1298 * exit the loop. The ordering of all user allocated 1299 * segmaps makes this all work out so beautifully. 1300 */ 1301 do { 1302 struct sun4c_mmu_entry *next = entry->next; 1303 unsigned long realend; 1304 1305 /* "realstart" is always >= entry->vaddr */ 1306 realend = entry->vaddr + SUN4C_REAL_PGDIR_SIZE; 1307 if (end < realend) 1308 realend = end; 1309 if ((realend - entry->vaddr) <= (PAGE_SIZE << 3)) { 1310 unsigned long page = entry->vaddr; 1311 while (page < realend) { 1312 sun4c_flush_page(page); 1313 page += PAGE_SIZE; 1314 } 1315 } else { 1316 sun4c_flush_segment(entry->vaddr); 1317 sun4c_user_unmap(entry); 1318 free_user_entry(new_ctx, entry); 1319 } 1320 entry = next; 1321 } while ((entry != head) && (entry->vaddr < end)); 1322 sun4c_set_context(octx); 1323 } 1324 local_irq_restore(flags); 1325 } 1326} 1327 1328static void sun4c_flush_cache_page(struct vm_area_struct *vma, unsigned long page) 1329{ 1330 struct mm_struct *mm = vma->vm_mm; 1331 int new_ctx = mm->context; 1332 1333 /* Sun4c has no separate I/D caches so cannot optimize for non 1334 * text page flushes. 1335 */ 1336 if (new_ctx != NO_CONTEXT) { 1337 int octx = sun4c_get_context(); 1338 unsigned long flags; 1339 1340 flush_user_windows(); 1341 local_irq_save(flags); 1342 sun4c_set_context(new_ctx); 1343 sun4c_flush_page(page); 1344 sun4c_set_context(octx); 1345 local_irq_restore(flags); 1346 } 1347} 1348 1349static void sun4c_flush_page_to_ram(unsigned long page) 1350{ 1351 unsigned long flags; 1352 1353 local_irq_save(flags); 1354 sun4c_flush_page(page); 1355 local_irq_restore(flags); 1356} 1357 1358/* Sun4c cache is unified, both instructions and data live there, so 1359 * no need to flush the on-stack instructions for new signal handlers. 1360 */ 1361static void sun4c_flush_sig_insns(struct mm_struct *mm, unsigned long insn_addr) 1362{ 1363} 1364 1365/* TLB flushing on the sun4c. These routines count on the cache 1366 * flushing code to flush the user register windows so that we need 1367 * not do so when we get here. 1368 */ 1369 1370static void sun4c_flush_tlb_all(void) 1371{ 1372 struct sun4c_mmu_entry *this_entry, *next_entry; 1373 unsigned long flags; 1374 int savectx, ctx; 1375 1376 local_irq_save(flags); 1377 this_entry = sun4c_kernel_ring.ringhd.next; 1378 savectx = sun4c_get_context(); 1379 flush_user_windows(); 1380 while (sun4c_kernel_ring.num_entries) { 1381 next_entry = this_entry->next; 1382 sun4c_flush_segment(this_entry->vaddr); 1383 for (ctx = 0; ctx < num_contexts; ctx++) { 1384 sun4c_set_context(ctx); 1385 sun4c_put_segmap(this_entry->vaddr, invalid_segment); 1386 } 1387 free_kernel_entry(this_entry, &sun4c_kernel_ring); 1388 this_entry = next_entry; 1389 } 1390 sun4c_set_context(savectx); 1391 local_irq_restore(flags); 1392} 1393 1394static void sun4c_flush_tlb_mm(struct mm_struct *mm) 1395{ 1396 int new_ctx = mm->context; 1397 1398 if (new_ctx != NO_CONTEXT) { 1399 struct sun4c_mmu_entry *head = &sun4c_context_ring[new_ctx].ringhd; 1400 unsigned long flags; 1401 1402 local_irq_save(flags); 1403 if (head->next != head) { 1404 struct sun4c_mmu_entry *entry = head->next; 1405 int savectx = sun4c_get_context(); 1406 1407 sun4c_set_context(new_ctx); 1408 sun4c_flush_context(); 1409 do { 1410 struct sun4c_mmu_entry *next = entry->next; 1411 1412 sun4c_user_unmap(entry); 1413 free_user_entry(new_ctx, entry); 1414 1415 entry = next; 1416 } while (entry != head); 1417 sun4c_set_context(savectx); 1418 } 1419 local_irq_restore(flags); 1420 } 1421} 1422 1423static void sun4c_flush_tlb_range(struct vm_area_struct *vma, unsigned long start, unsigned long end) 1424{ 1425 struct mm_struct *mm = vma->vm_mm; 1426 int new_ctx = mm->context; 1427 1428 if (new_ctx != NO_CONTEXT) { 1429 struct sun4c_mmu_entry *head = &sun4c_context_ring[new_ctx].ringhd; 1430 struct sun4c_mmu_entry *entry; 1431 unsigned long flags; 1432 1433 local_irq_save(flags); 1434 /* See commentary in sun4c_flush_cache_range(). */ 1435 for (entry = head->next; 1436 (entry != head) && ((entry->vaddr+SUN4C_REAL_PGDIR_SIZE) < start); 1437 entry = entry->next) 1438 ; 1439 1440 if ((entry != head) && (entry->vaddr < end)) { 1441 int octx = sun4c_get_context(); 1442 1443 sun4c_set_context(new_ctx); 1444 do { 1445 struct sun4c_mmu_entry *next = entry->next; 1446 1447 sun4c_flush_segment(entry->vaddr); 1448 sun4c_user_unmap(entry); 1449 free_user_entry(new_ctx, entry); 1450 1451 entry = next; 1452 } while ((entry != head) && (entry->vaddr < end)); 1453 sun4c_set_context(octx); 1454 } 1455 local_irq_restore(flags); 1456 } 1457} 1458 1459static void sun4c_flush_tlb_page(struct vm_area_struct *vma, unsigned long page) 1460{ 1461 struct mm_struct *mm = vma->vm_mm; 1462 int new_ctx = mm->context; 1463 1464 if (new_ctx != NO_CONTEXT) { 1465 int savectx = sun4c_get_context(); 1466 unsigned long flags; 1467 1468 local_irq_save(flags); 1469 sun4c_set_context(new_ctx); 1470 page &= PAGE_MASK; 1471 sun4c_flush_page(page); 1472 sun4c_put_pte(page, 0); 1473 sun4c_set_context(savectx); 1474 local_irq_restore(flags); 1475 } 1476} 1477 1478static inline void sun4c_mapioaddr(unsigned long physaddr, unsigned long virt_addr) 1479{ 1480 unsigned long page_entry, pg_iobits; 1481 1482 pg_iobits = _SUN4C_PAGE_PRESENT | _SUN4C_READABLE | _SUN4C_WRITEABLE | 1483 _SUN4C_PAGE_IO | _SUN4C_PAGE_NOCACHE; 1484 1485 page_entry = ((physaddr >> PAGE_SHIFT) & SUN4C_PFN_MASK); 1486 page_entry |= ((pg_iobits | _SUN4C_PAGE_PRIV) & ~(_SUN4C_PAGE_PRESENT)); 1487 sun4c_put_pte(virt_addr, page_entry); 1488} 1489 1490static void sun4c_mapiorange(unsigned int bus, unsigned long xpa, 1491 unsigned long xva, unsigned int len) 1492{ 1493 while (len != 0) { 1494 len -= PAGE_SIZE; 1495 sun4c_mapioaddr(xpa, xva); 1496 xva += PAGE_SIZE; 1497 xpa += PAGE_SIZE; 1498 } 1499} 1500 1501static void sun4c_unmapiorange(unsigned long virt_addr, unsigned int len) 1502{ 1503 while (len != 0) { 1504 len -= PAGE_SIZE; 1505 sun4c_put_pte(virt_addr, 0); 1506 virt_addr += PAGE_SIZE; 1507 } 1508} 1509 1510static void sun4c_alloc_context(struct mm_struct *old_mm, struct mm_struct *mm) 1511{ 1512 struct ctx_list *ctxp; 1513 1514 ctxp = ctx_free.next; 1515 if (ctxp != &ctx_free) { 1516 remove_from_ctx_list(ctxp); 1517 add_to_used_ctxlist(ctxp); 1518 mm->context = ctxp->ctx_number; 1519 ctxp->ctx_mm = mm; 1520 return; 1521 } 1522 ctxp = ctx_used.next; 1523 if (ctxp->ctx_mm == old_mm) 1524 ctxp = ctxp->next; 1525 remove_from_ctx_list(ctxp); 1526 add_to_used_ctxlist(ctxp); 1527 ctxp->ctx_mm->context = NO_CONTEXT; 1528 ctxp->ctx_mm = mm; 1529 mm->context = ctxp->ctx_number; 1530 sun4c_demap_context(&sun4c_context_ring[ctxp->ctx_number], 1531 ctxp->ctx_number); 1532} 1533 1534/* Switch the current MM context. */ 1535static void sun4c_switch_mm(struct mm_struct *old_mm, struct mm_struct *mm, struct task_struct *tsk, int cpu) 1536{ 1537 struct ctx_list *ctx; 1538 int dirty = 0; 1539 1540 if (mm->context == NO_CONTEXT) { 1541 dirty = 1; 1542 sun4c_alloc_context(old_mm, mm); 1543 } else { 1544 /* Update the LRU ring of contexts. */ 1545 ctx = ctx_list_pool + mm->context; 1546 remove_from_ctx_list(ctx); 1547 add_to_used_ctxlist(ctx); 1548 } 1549 if (dirty || old_mm != mm) 1550 sun4c_set_context(mm->context); 1551} 1552 1553static void sun4c_destroy_context(struct mm_struct *mm) 1554{ 1555 struct ctx_list *ctx_old; 1556 1557 if (mm->context != NO_CONTEXT) { 1558 sun4c_demap_context(&sun4c_context_ring[mm->context], mm->context); 1559 ctx_old = ctx_list_pool + mm->context; 1560 remove_from_ctx_list(ctx_old); 1561 add_to_free_ctxlist(ctx_old); 1562 mm->context = NO_CONTEXT; 1563 } 1564} 1565 1566static void sun4c_mmu_info(struct seq_file *m) 1567{ 1568 int used_user_entries, i; 1569 1570 used_user_entries = 0; 1571 for (i = 0; i < num_contexts; i++) 1572 used_user_entries += sun4c_context_ring[i].num_entries; 1573 1574 seq_printf(m, 1575 "vacsize\t\t: %d bytes\n" 1576 "vachwflush\t: %s\n" 1577 "vaclinesize\t: %d bytes\n" 1578 "mmuctxs\t\t: %d\n" 1579 "mmupsegs\t: %d\n" 1580 "kernelpsegs\t: %d\n" 1581 "kfreepsegs\t: %d\n" 1582 "usedpsegs\t: %d\n" 1583 "ufreepsegs\t: %d\n" 1584 "user_taken\t: %d\n" 1585 "max_taken\t: %d\n", 1586 sun4c_vacinfo.num_bytes, 1587 (sun4c_vacinfo.do_hwflushes ? "yes" : "no"), 1588 sun4c_vacinfo.linesize, 1589 num_contexts, 1590 (invalid_segment + 1), 1591 sun4c_kernel_ring.num_entries, 1592 sun4c_kfree_ring.num_entries, 1593 used_user_entries, 1594 sun4c_ufree_ring.num_entries, 1595 sun4c_user_taken_entries, 1596 max_user_taken_entries); 1597} 1598 1599/* Nothing below here should touch the mmu hardware nor the mmu_entry 1600 * data structures. 1601 */ 1602 1603/* First the functions which the mid-level code uses to directly 1604 * manipulate the software page tables. Some defines since we are 1605 * emulating the i386 page directory layout. 1606 */ 1607#define PGD_PRESENT 0x001 1608#define PGD_RW 0x002 1609#define PGD_USER 0x004 1610#define PGD_ACCESSED 0x020 1611#define PGD_DIRTY 0x040 1612#define PGD_TABLE (PGD_PRESENT | PGD_RW | PGD_USER | PGD_ACCESSED | PGD_DIRTY) 1613 1614static void sun4c_set_pte(pte_t *ptep, pte_t pte) 1615{ 1616 *ptep = pte; 1617} 1618 1619static void sun4c_pgd_set(pgd_t * pgdp, pmd_t * pmdp) 1620{ 1621} 1622 1623static void sun4c_pmd_set(pmd_t * pmdp, pte_t * ptep) 1624{ 1625 pmdp->pmdv[0] = PGD_TABLE | (unsigned long) ptep; 1626} 1627 1628static void sun4c_pmd_populate(pmd_t * pmdp, struct page * ptep) 1629{ 1630 if (page_address(ptep) == NULL) BUG(); /* No highmem on sun4c */ 1631 pmdp->pmdv[0] = PGD_TABLE | (unsigned long) page_address(ptep); 1632} 1633 1634static int sun4c_pte_present(pte_t pte) 1635{ 1636 return ((pte_val(pte) & (_SUN4C_PAGE_PRESENT | _SUN4C_PAGE_PRIV)) != 0); 1637} 1638static void sun4c_pte_clear(pte_t *ptep) { *ptep = __pte(0); } 1639 1640static int sun4c_pmd_bad(pmd_t pmd) 1641{ 1642 return (((pmd_val(pmd) & ~PAGE_MASK) != PGD_TABLE) || 1643 (!virt_addr_valid(pmd_val(pmd)))); 1644} 1645 1646static int sun4c_pmd_present(pmd_t pmd) 1647{ 1648 return ((pmd_val(pmd) & PGD_PRESENT) != 0); 1649} 1650 1651static void sun4c_pmd_clear(pmd_t *pmdp) { 1652 memset((void *)pmdp, 0, sizeof(pmd_t)); 1653} 1654 1655static int sun4c_pgd_none(pgd_t pgd) { return 0; } 1656static int sun4c_pgd_bad(pgd_t pgd) { return 0; } 1657static int sun4c_pgd_present(pgd_t pgd) { return 1; } 1658static void sun4c_pgd_clear(pgd_t * pgdp) { } 1659 1660/* 1661 * The following only work if pte_present() is true. 1662 * Undefined behaviour if not.. 1663 */ 1664static pte_t sun4c_pte_mkwrite(pte_t pte) 1665{ 1666 pte = __pte(pte_val(pte) | _SUN4C_PAGE_WRITE); 1667 if (pte_val(pte) & _SUN4C_PAGE_MODIFIED) 1668 pte = __pte(pte_val(pte) | _SUN4C_PAGE_SILENT_WRITE); 1669 return pte; 1670} 1671 1672static pte_t sun4c_pte_mkdirty(pte_t pte) 1673{ 1674 pte = __pte(pte_val(pte) | _SUN4C_PAGE_MODIFIED); 1675 if (pte_val(pte) & _SUN4C_PAGE_WRITE) 1676 pte = __pte(pte_val(pte) | _SUN4C_PAGE_SILENT_WRITE); 1677 return pte; 1678} 1679 1680static pte_t sun4c_pte_mkyoung(pte_t pte) 1681{ 1682 pte = __pte(pte_val(pte) | _SUN4C_PAGE_ACCESSED); 1683 if (pte_val(pte) & _SUN4C_PAGE_READ) 1684 pte = __pte(pte_val(pte) | _SUN4C_PAGE_SILENT_READ); 1685 return pte; 1686} 1687 1688/* 1689 * Conversion functions: convert a page and protection to a page entry, 1690 * and a page entry and page directory to the page they refer to. 1691 */ 1692static pte_t sun4c_mk_pte(struct page *page, pgprot_t pgprot) 1693{ 1694 return __pte(page_to_pfn(page) | pgprot_val(pgprot)); 1695} 1696 1697static pte_t sun4c_mk_pte_phys(unsigned long phys_page, pgprot_t pgprot) 1698{ 1699 return __pte((phys_page >> PAGE_SHIFT) | pgprot_val(pgprot)); 1700} 1701 1702static pte_t sun4c_mk_pte_io(unsigned long page, pgprot_t pgprot, int space) 1703{ 1704 return __pte(((page - PAGE_OFFSET) >> PAGE_SHIFT) | pgprot_val(pgprot)); 1705} 1706 1707static unsigned long sun4c_pte_pfn(pte_t pte) 1708{ 1709 return pte_val(pte) & SUN4C_PFN_MASK; 1710} 1711 1712static pte_t sun4c_pgoff_to_pte(unsigned long pgoff) 1713{ 1714 return __pte(pgoff | _SUN4C_PAGE_FILE); 1715} 1716 1717static unsigned long sun4c_pte_to_pgoff(pte_t pte) 1718{ 1719 return pte_val(pte) & ((1UL << PTE_FILE_MAX_BITS) - 1); 1720} 1721 1722 1723static inline unsigned long sun4c_pmd_page_v(pmd_t pmd) 1724{ 1725 return (pmd_val(pmd) & PAGE_MASK); 1726} 1727 1728static struct page *sun4c_pmd_page(pmd_t pmd) 1729{ 1730 return virt_to_page(sun4c_pmd_page_v(pmd)); 1731} 1732 1733static unsigned long sun4c_pgd_page(pgd_t pgd) { return 0; } 1734 1735/* to find an entry in a page-table-directory */ 1736static inline pgd_t *sun4c_pgd_offset(struct mm_struct * mm, unsigned long address) 1737{ 1738 return mm->pgd + (address >> SUN4C_PGDIR_SHIFT); 1739} 1740 1741/* Find an entry in the second-level page table.. */ 1742static pmd_t *sun4c_pmd_offset(pgd_t * dir, unsigned long address) 1743{ 1744 return (pmd_t *) dir; 1745} 1746 1747/* Find an entry in the third-level page table.. */ 1748pte_t *sun4c_pte_offset_kernel(pmd_t * dir, unsigned long address) 1749{ 1750 return (pte_t *) sun4c_pmd_page_v(*dir) + 1751 ((address >> PAGE_SHIFT) & (SUN4C_PTRS_PER_PTE - 1)); 1752} 1753 1754static unsigned long sun4c_swp_type(swp_entry_t entry) 1755{ 1756 return (entry.val & SUN4C_SWP_TYPE_MASK); 1757} 1758 1759static unsigned long sun4c_swp_offset(swp_entry_t entry) 1760{ 1761 return (entry.val >> SUN4C_SWP_OFF_SHIFT) & SUN4C_SWP_OFF_MASK; 1762} 1763 1764static swp_entry_t sun4c_swp_entry(unsigned long type, unsigned long offset) 1765{ 1766 return (swp_entry_t) { 1767 (offset & SUN4C_SWP_OFF_MASK) << SUN4C_SWP_OFF_SHIFT 1768 | (type & SUN4C_SWP_TYPE_MASK) }; 1769} 1770 1771static void sun4c_free_pte_slow(pte_t *pte) 1772{ 1773 free_page((unsigned long)pte); 1774} 1775 1776static void sun4c_free_pgd_slow(pgd_t *pgd) 1777{ 1778 free_page((unsigned long)pgd); 1779} 1780 1781static pgd_t *sun4c_get_pgd_fast(void) 1782{ 1783 unsigned long *ret; 1784 1785 if ((ret = pgd_quicklist) != NULL) { 1786 pgd_quicklist = (unsigned long *)(*ret); 1787 ret[0] = ret[1]; 1788 pgtable_cache_size--; 1789 } else { 1790 pgd_t *init; 1791 1792 ret = (unsigned long *)__get_free_page(GFP_KERNEL); 1793 memset (ret, 0, (KERNBASE / SUN4C_PGDIR_SIZE) * sizeof(pgd_t)); 1794 init = sun4c_pgd_offset(&init_mm, 0); 1795 memcpy (((pgd_t *)ret) + USER_PTRS_PER_PGD, init + USER_PTRS_PER_PGD, 1796 (PTRS_PER_PGD - USER_PTRS_PER_PGD) * sizeof(pgd_t)); 1797 } 1798 return (pgd_t *)ret; 1799} 1800 1801static void sun4c_free_pgd_fast(pgd_t *pgd) 1802{ 1803 *(unsigned long *)pgd = (unsigned long) pgd_quicklist; 1804 pgd_quicklist = (unsigned long *) pgd; 1805 pgtable_cache_size++; 1806} 1807 1808 1809static inline pte_t * 1810sun4c_pte_alloc_one_fast(struct mm_struct *mm, unsigned long address) 1811{ 1812 unsigned long *ret; 1813 1814 if ((ret = (unsigned long *)pte_quicklist) != NULL) { 1815 pte_quicklist = (unsigned long *)(*ret); 1816 ret[0] = ret[1]; 1817 pgtable_cache_size--; 1818 } 1819 return (pte_t *)ret; 1820} 1821 1822static pte_t *sun4c_pte_alloc_one_kernel(struct mm_struct *mm, unsigned long address) 1823{ 1824 pte_t *pte; 1825 1826 if ((pte = sun4c_pte_alloc_one_fast(mm, address)) != NULL) 1827 return pte; 1828 1829 pte = (pte_t *)get_zeroed_page(GFP_KERNEL|__GFP_REPEAT); 1830 return pte; 1831} 1832 1833static pgtable_t sun4c_pte_alloc_one(struct mm_struct *mm, unsigned long address) 1834{ 1835 pte_t *pte; 1836 struct page *page; 1837 1838 pte = sun4c_pte_alloc_one_kernel(mm, address); 1839 if (pte == NULL) 1840 return NULL; 1841 page = virt_to_page(pte); 1842 pgtable_page_ctor(page); 1843 return page; 1844} 1845 1846static inline void sun4c_free_pte_fast(pte_t *pte) 1847{ 1848 *(unsigned long *)pte = (unsigned long) pte_quicklist; 1849 pte_quicklist = (unsigned long *) pte; 1850 pgtable_cache_size++; 1851} 1852 1853static void sun4c_pte_free(pgtable_t pte) 1854{ 1855 pgtable_page_dtor(pte); 1856 sun4c_free_pte_fast(page_address(pte)); 1857} 1858 1859/* 1860 * allocating and freeing a pmd is trivial: the 1-entry pmd is 1861 * inside the pgd, so has no extra memory associated with it. 1862 */ 1863static pmd_t *sun4c_pmd_alloc_one(struct mm_struct *mm, unsigned long address) 1864{ 1865 BUG(); 1866 return NULL; 1867} 1868 1869static void sun4c_free_pmd_fast(pmd_t * pmd) { } 1870 1871static void sun4c_check_pgt_cache(int low, int high) 1872{ 1873 if (pgtable_cache_size > high) { 1874 do { 1875 if (pgd_quicklist) 1876 sun4c_free_pgd_slow(sun4c_get_pgd_fast()); 1877 if (pte_quicklist) 1878 sun4c_free_pte_slow(sun4c_pte_alloc_one_fast(NULL, 0)); 1879 } while (pgtable_cache_size > low); 1880 } 1881} 1882 1883/* An experiment, turn off by default for now... -DaveM */ 1884#define SUN4C_PRELOAD_PSEG 1885 1886void sun4c_update_mmu_cache(struct vm_area_struct *vma, unsigned long address, pte_t *ptep) 1887{ 1888 unsigned long flags; 1889 int pseg; 1890 1891 if (vma->vm_mm->context == NO_CONTEXT) 1892 return; 1893 1894 local_irq_save(flags); 1895 address &= PAGE_MASK; 1896 if ((pseg = sun4c_get_segmap(address)) == invalid_segment) { 1897 struct sun4c_mmu_entry *entry = sun4c_user_strategy(); 1898 struct mm_struct *mm = vma->vm_mm; 1899 unsigned long start, end; 1900 1901 entry->vaddr = start = (address & SUN4C_REAL_PGDIR_MASK); 1902 entry->ctx = mm->context; 1903 add_ring_ordered(sun4c_context_ring + mm->context, entry); 1904 sun4c_put_segmap(entry->vaddr, entry->pseg); 1905 end = start + SUN4C_REAL_PGDIR_SIZE; 1906 while (start < end) { 1907#ifdef SUN4C_PRELOAD_PSEG 1908 pgd_t *pgdp = sun4c_pgd_offset(mm, start); 1909 pte_t *ptep; 1910 1911 if (!pgdp) 1912 goto no_mapping; 1913 ptep = sun4c_pte_offset_kernel((pmd_t *) pgdp, start); 1914 if (!ptep || !(pte_val(*ptep) & _SUN4C_PAGE_PRESENT)) 1915 goto no_mapping; 1916 sun4c_put_pte(start, pte_val(*ptep)); 1917 goto next; 1918 1919 no_mapping: 1920#endif 1921 sun4c_put_pte(start, 0); 1922#ifdef SUN4C_PRELOAD_PSEG 1923 next: 1924#endif 1925 start += PAGE_SIZE; 1926 } 1927#ifndef SUN4C_PRELOAD_PSEG 1928 sun4c_put_pte(address, pte_val(*ptep)); 1929#endif 1930 local_irq_restore(flags); 1931 return; 1932 } else { 1933 struct sun4c_mmu_entry *entry = &mmu_entry_pool[pseg]; 1934 1935 remove_lru(entry); 1936 add_lru(entry); 1937 } 1938 1939 sun4c_put_pte(address, pte_val(*ptep)); 1940 local_irq_restore(flags); 1941} 1942 1943extern void sparc_context_init(int); 1944extern unsigned long bootmem_init(unsigned long *pages_avail); 1945extern unsigned long last_valid_pfn; 1946 1947void __init sun4c_paging_init(void) 1948{ 1949 int i, cnt; 1950 unsigned long kernel_end, vaddr; 1951 extern struct resource sparc_iomap; 1952 unsigned long end_pfn, pages_avail; 1953 1954 kernel_end = (unsigned long) &_end; 1955 kernel_end = SUN4C_REAL_PGDIR_ALIGN(kernel_end); 1956 1957 pages_avail = 0; 1958 last_valid_pfn = bootmem_init(&pages_avail); 1959 end_pfn = last_valid_pfn; 1960 1961 sun4c_probe_mmu(); 1962 invalid_segment = (num_segmaps - 1); 1963 sun4c_init_mmu_entry_pool(); 1964 sun4c_init_rings(); 1965 sun4c_init_map_kernelprom(kernel_end); 1966 sun4c_init_clean_mmu(kernel_end); 1967 sun4c_init_fill_kernel_ring(SUN4C_KERNEL_BUCKETS); 1968 sun4c_init_lock_area(sparc_iomap.start, IOBASE_END); 1969 sun4c_init_lock_area(DVMA_VADDR, DVMA_END); 1970 sun4c_init_lock_areas(); 1971 sun4c_init_fill_user_ring(); 1972 1973 sun4c_set_context(0); 1974 memset(swapper_pg_dir, 0, PAGE_SIZE); 1975 memset(pg0, 0, PAGE_SIZE); 1976 memset(pg1, 0, PAGE_SIZE); 1977 memset(pg2, 0, PAGE_SIZE); 1978 memset(pg3, 0, PAGE_SIZE); 1979 1980 /* Save work later. */ 1981 vaddr = VMALLOC_START; 1982 swapper_pg_dir[vaddr>>SUN4C_PGDIR_SHIFT] = __pgd(PGD_TABLE | (unsigned long) pg0); 1983 vaddr += SUN4C_PGDIR_SIZE; 1984 swapper_pg_dir[vaddr>>SUN4C_PGDIR_SHIFT] = __pgd(PGD_TABLE | (unsigned long) pg1); 1985 vaddr += SUN4C_PGDIR_SIZE; 1986 swapper_pg_dir[vaddr>>SUN4C_PGDIR_SHIFT] = __pgd(PGD_TABLE | (unsigned long) pg2); 1987 vaddr += SUN4C_PGDIR_SIZE; 1988 swapper_pg_dir[vaddr>>SUN4C_PGDIR_SHIFT] = __pgd(PGD_TABLE | (unsigned long) pg3); 1989 sun4c_init_ss2_cache_bug(); 1990 sparc_context_init(num_contexts); 1991 1992 { 1993 unsigned long zones_size[MAX_NR_ZONES]; 1994 unsigned long zholes_size[MAX_NR_ZONES]; 1995 unsigned long npages; 1996 int znum; 1997 1998 for (znum = 0; znum < MAX_NR_ZONES; znum++) 1999 zones_size[znum] = zholes_size[znum] = 0; 2000 2001 npages = max_low_pfn - pfn_base; 2002 2003 zones_size[ZONE_DMA] = npages; 2004 zholes_size[ZONE_DMA] = npages - pages_avail; 2005 2006 npages = highend_pfn - max_low_pfn; 2007 zones_size[ZONE_HIGHMEM] = npages; 2008 zholes_size[ZONE_HIGHMEM] = npages - calc_highpages(); 2009 2010 free_area_init_node(0, zones_size, pfn_base, zholes_size); 2011 } 2012 2013 cnt = 0; 2014 for (i = 0; i < num_segmaps; i++) 2015 if (mmu_entry_pool[i].locked) 2016 cnt++; 2017 2018 max_user_taken_entries = num_segmaps - cnt - 40 - 1; 2019 2020 printk("SUN4C: %d mmu entries for the kernel\n", cnt); 2021} 2022 2023static pgprot_t sun4c_pgprot_noncached(pgprot_t prot) 2024{ 2025 prot |= __pgprot(_SUN4C_PAGE_IO | _SUN4C_PAGE_NOCACHE); 2026 2027 return prot; 2028} 2029 2030/* Load up routines and constants for sun4c mmu */ 2031void __init ld_mmu_sun4c(void) 2032{ 2033 extern void ___xchg32_sun4c(void); 2034 2035 printk("Loading sun4c MMU routines\n"); 2036 2037 /* First the constants */ 2038 BTFIXUPSET_SIMM13(pgdir_shift, SUN4C_PGDIR_SHIFT); 2039 BTFIXUPSET_SETHI(pgdir_size, SUN4C_PGDIR_SIZE); 2040 BTFIXUPSET_SETHI(pgdir_mask, SUN4C_PGDIR_MASK); 2041 2042 BTFIXUPSET_SIMM13(ptrs_per_pmd, SUN4C_PTRS_PER_PMD); 2043 BTFIXUPSET_SIMM13(ptrs_per_pgd, SUN4C_PTRS_PER_PGD); 2044 BTFIXUPSET_SIMM13(user_ptrs_per_pgd, KERNBASE / SUN4C_PGDIR_SIZE); 2045 2046 BTFIXUPSET_INT(page_none, pgprot_val(SUN4C_PAGE_NONE)); 2047 PAGE_SHARED = pgprot_val(SUN4C_PAGE_SHARED); 2048 BTFIXUPSET_INT(page_copy, pgprot_val(SUN4C_PAGE_COPY)); 2049 BTFIXUPSET_INT(page_readonly, pgprot_val(SUN4C_PAGE_READONLY)); 2050 BTFIXUPSET_INT(page_kernel, pgprot_val(SUN4C_PAGE_KERNEL)); 2051 page_kernel = pgprot_val(SUN4C_PAGE_KERNEL); 2052 2053 /* Functions */ 2054 BTFIXUPSET_CALL(pgprot_noncached, sun4c_pgprot_noncached, BTFIXUPCALL_NORM); 2055 BTFIXUPSET_CALL(___xchg32, ___xchg32_sun4c, BTFIXUPCALL_NORM); 2056 BTFIXUPSET_CALL(do_check_pgt_cache, sun4c_check_pgt_cache, BTFIXUPCALL_NORM); 2057 2058 BTFIXUPSET_CALL(flush_cache_all, sun4c_flush_cache_all, BTFIXUPCALL_NORM); 2059 2060 if (sun4c_vacinfo.do_hwflushes) { 2061 BTFIXUPSET_CALL(sun4c_flush_page, sun4c_flush_page_hw, BTFIXUPCALL_NORM); 2062 BTFIXUPSET_CALL(sun4c_flush_segment, sun4c_flush_segment_hw, BTFIXUPCALL_NORM); 2063 BTFIXUPSET_CALL(sun4c_flush_context, sun4c_flush_context_hw, BTFIXUPCALL_NORM); 2064 } else { 2065 BTFIXUPSET_CALL(sun4c_flush_page, sun4c_flush_page_sw, BTFIXUPCALL_NORM); 2066 BTFIXUPSET_CALL(sun4c_flush_segment, sun4c_flush_segment_sw, BTFIXUPCALL_NORM); 2067 BTFIXUPSET_CALL(sun4c_flush_context, sun4c_flush_context_sw, BTFIXUPCALL_NORM); 2068 } 2069 2070 BTFIXUPSET_CALL(flush_tlb_mm, sun4c_flush_tlb_mm, BTFIXUPCALL_NORM); 2071 BTFIXUPSET_CALL(flush_cache_mm, sun4c_flush_cache_mm, BTFIXUPCALL_NORM); 2072 BTFIXUPSET_CALL(destroy_context, sun4c_destroy_context, BTFIXUPCALL_NORM); 2073 BTFIXUPSET_CALL(switch_mm, sun4c_switch_mm, BTFIXUPCALL_NORM); 2074 BTFIXUPSET_CALL(flush_cache_page, sun4c_flush_cache_page, BTFIXUPCALL_NORM); 2075 BTFIXUPSET_CALL(flush_tlb_page, sun4c_flush_tlb_page, BTFIXUPCALL_NORM); 2076 BTFIXUPSET_CALL(flush_tlb_range, sun4c_flush_tlb_range, BTFIXUPCALL_NORM); 2077 BTFIXUPSET_CALL(flush_cache_range, sun4c_flush_cache_range, BTFIXUPCALL_NORM); 2078 BTFIXUPSET_CALL(__flush_page_to_ram, sun4c_flush_page_to_ram, BTFIXUPCALL_NORM); 2079 BTFIXUPSET_CALL(flush_tlb_all, sun4c_flush_tlb_all, BTFIXUPCALL_NORM); 2080 2081 BTFIXUPSET_CALL(flush_sig_insns, sun4c_flush_sig_insns, BTFIXUPCALL_NOP); 2082 2083 BTFIXUPSET_CALL(set_pte, sun4c_set_pte, BTFIXUPCALL_STO1O0); 2084 2085 BTFIXUPSET_CALL(pte_pfn, sun4c_pte_pfn, BTFIXUPCALL_NORM); 2086 BTFIXUPSET_CALL(pmd_page, sun4c_pmd_page, BTFIXUPCALL_NORM); 2087 BTFIXUPSET_CALL(pmd_set, sun4c_pmd_set, BTFIXUPCALL_NORM); 2088 BTFIXUPSET_CALL(pmd_populate, sun4c_pmd_populate, BTFIXUPCALL_NORM); 2089 2090 BTFIXUPSET_CALL(pte_present, sun4c_pte_present, BTFIXUPCALL_NORM); 2091 BTFIXUPSET_CALL(pte_clear, sun4c_pte_clear, BTFIXUPCALL_STG0O0); 2092 2093 BTFIXUPSET_CALL(pmd_bad, sun4c_pmd_bad, BTFIXUPCALL_NORM); 2094 BTFIXUPSET_CALL(pmd_present, sun4c_pmd_present, BTFIXUPCALL_NORM); 2095 BTFIXUPSET_CALL(pmd_clear, sun4c_pmd_clear, BTFIXUPCALL_STG0O0); 2096 2097 BTFIXUPSET_CALL(pgd_none, sun4c_pgd_none, BTFIXUPCALL_RETINT(0)); 2098 BTFIXUPSET_CALL(pgd_bad, sun4c_pgd_bad, BTFIXUPCALL_RETINT(0)); 2099 BTFIXUPSET_CALL(pgd_present, sun4c_pgd_present, BTFIXUPCALL_RETINT(1)); 2100 BTFIXUPSET_CALL(pgd_clear, sun4c_pgd_clear, BTFIXUPCALL_NOP); 2101 2102 BTFIXUPSET_CALL(mk_pte, sun4c_mk_pte, BTFIXUPCALL_NORM); 2103 BTFIXUPSET_CALL(mk_pte_phys, sun4c_mk_pte_phys, BTFIXUPCALL_NORM); 2104 BTFIXUPSET_CALL(mk_pte_io, sun4c_mk_pte_io, BTFIXUPCALL_NORM); 2105 2106 BTFIXUPSET_INT(pte_modify_mask, _SUN4C_PAGE_CHG_MASK); 2107 BTFIXUPSET_CALL(pmd_offset, sun4c_pmd_offset, BTFIXUPCALL_NORM); 2108 BTFIXUPSET_CALL(pte_offset_kernel, sun4c_pte_offset_kernel, BTFIXUPCALL_NORM); 2109 BTFIXUPSET_CALL(free_pte_fast, sun4c_free_pte_fast, BTFIXUPCALL_NORM); 2110 BTFIXUPSET_CALL(pte_free, sun4c_pte_free, BTFIXUPCALL_NORM); 2111 BTFIXUPSET_CALL(pte_alloc_one_kernel, sun4c_pte_alloc_one_kernel, BTFIXUPCALL_NORM); 2112 BTFIXUPSET_CALL(pte_alloc_one, sun4c_pte_alloc_one, BTFIXUPCALL_NORM); 2113 BTFIXUPSET_CALL(free_pmd_fast, sun4c_free_pmd_fast, BTFIXUPCALL_NOP); 2114 BTFIXUPSET_CALL(pmd_alloc_one, sun4c_pmd_alloc_one, BTFIXUPCALL_RETO0); 2115 BTFIXUPSET_CALL(free_pgd_fast, sun4c_free_pgd_fast, BTFIXUPCALL_NORM); 2116 BTFIXUPSET_CALL(get_pgd_fast, sun4c_get_pgd_fast, BTFIXUPCALL_NORM); 2117 2118 BTFIXUPSET_HALF(pte_writei, _SUN4C_PAGE_WRITE); 2119 BTFIXUPSET_HALF(pte_dirtyi, _SUN4C_PAGE_MODIFIED); 2120 BTFIXUPSET_HALF(pte_youngi, _SUN4C_PAGE_ACCESSED); 2121 BTFIXUPSET_HALF(pte_filei, _SUN4C_PAGE_FILE); 2122 BTFIXUPSET_HALF(pte_wrprotecti, _SUN4C_PAGE_WRITE|_SUN4C_PAGE_SILENT_WRITE); 2123 BTFIXUPSET_HALF(pte_mkcleani, _SUN4C_PAGE_MODIFIED|_SUN4C_PAGE_SILENT_WRITE); 2124 BTFIXUPSET_HALF(pte_mkoldi, _SUN4C_PAGE_ACCESSED|_SUN4C_PAGE_SILENT_READ); 2125 BTFIXUPSET_CALL(pte_mkwrite, sun4c_pte_mkwrite, BTFIXUPCALL_NORM); 2126 BTFIXUPSET_CALL(pte_mkdirty, sun4c_pte_mkdirty, BTFIXUPCALL_NORM); 2127 BTFIXUPSET_CALL(pte_mkyoung, sun4c_pte_mkyoung, BTFIXUPCALL_NORM); 2128 BTFIXUPSET_CALL(update_mmu_cache, sun4c_update_mmu_cache, BTFIXUPCALL_NORM); 2129 2130 BTFIXUPSET_CALL(pte_to_pgoff, sun4c_pte_to_pgoff, BTFIXUPCALL_NORM); 2131 BTFIXUPSET_CALL(pgoff_to_pte, sun4c_pgoff_to_pte, BTFIXUPCALL_NORM); 2132 2133 BTFIXUPSET_CALL(mmu_lockarea, sun4c_lockarea, BTFIXUPCALL_NORM); 2134 BTFIXUPSET_CALL(mmu_unlockarea, sun4c_unlockarea, BTFIXUPCALL_NORM); 2135 2136 BTFIXUPSET_CALL(mmu_get_scsi_one, sun4c_get_scsi_one, BTFIXUPCALL_NORM); 2137 BTFIXUPSET_CALL(mmu_get_scsi_sgl, sun4c_get_scsi_sgl, BTFIXUPCALL_NORM); 2138 BTFIXUPSET_CALL(mmu_release_scsi_one, sun4c_release_scsi_one, BTFIXUPCALL_NORM); 2139 BTFIXUPSET_CALL(mmu_release_scsi_sgl, sun4c_release_scsi_sgl, BTFIXUPCALL_NORM); 2140 2141 BTFIXUPSET_CALL(mmu_map_dma_area, sun4c_map_dma_area, BTFIXUPCALL_NORM); 2142 BTFIXUPSET_CALL(mmu_unmap_dma_area, sun4c_unmap_dma_area, BTFIXUPCALL_NORM); 2143 2144 BTFIXUPSET_CALL(sparc_mapiorange, sun4c_mapiorange, BTFIXUPCALL_NORM); 2145 BTFIXUPSET_CALL(sparc_unmapiorange, sun4c_unmapiorange, BTFIXUPCALL_NORM); 2146 2147 BTFIXUPSET_CALL(__swp_type, sun4c_swp_type, BTFIXUPCALL_NORM); 2148 BTFIXUPSET_CALL(__swp_offset, sun4c_swp_offset, BTFIXUPCALL_NORM); 2149 BTFIXUPSET_CALL(__swp_entry, sun4c_swp_entry, BTFIXUPCALL_NORM); 2150 2151 BTFIXUPSET_CALL(alloc_thread_info, sun4c_alloc_thread_info, BTFIXUPCALL_NORM); 2152 BTFIXUPSET_CALL(free_thread_info, sun4c_free_thread_info, BTFIXUPCALL_NORM); 2153 2154 BTFIXUPSET_CALL(mmu_info, sun4c_mmu_info, BTFIXUPCALL_NORM); 2155 2156 /* These should _never_ get called with two level tables. */ 2157 BTFIXUPSET_CALL(pgd_set, sun4c_pgd_set, BTFIXUPCALL_NOP); 2158 BTFIXUPSET_CALL(pgd_page_vaddr, sun4c_pgd_page, BTFIXUPCALL_RETO0); 2159} 2160