1/* 2 * This file is subject to the terms and conditions of the GNU General Public 3 * License. See the file "COPYING" in the main directory of this archive 4 * for more details. 5 * 6 * r4xx0.c: R4000 processor variant specific MMU/Cache routines. 7 * 8 * Copyright (C) 1996 David S. Miller (dm@engr.sgi.com) 9 * Copyright (C) 1997, 1998, 1999, 2000 Ralf Baechle ralf@gnu.org 10 */ 11#include <linux/config.h> 12#include <asm/addrspace.h> 13#include <asm/asm.h> 14#include <asm/regdef.h> 15#include <asm/cacheops.h> 16#include <asm/mipsregs.h> 17#include <asm/offset.h> 18 19#ifdef CONFIG_64BIT_PHYS_ADDR 20#define PGD_SIZE 0x2000 21#else 22#define PGD_SIZE 0x1000 23#endif 24 25 .text 26 .set mips3 27 .set noat 28 29/* 30 * Zero an entire page. Basically a simple unrolled loop should do the 31 * job but we want more performance by saving memory bus bandwidth. We 32 * have five flavours of the routine available for: 33 * 34 * - 16byte cachelines and no second level cache 35 * - 32byte cachelines second level cache 36 * - a version which handles the buggy R4600 v1.x 37 * - a version which handles the buggy R4600 v2.0 38 * - Finally a last version without fancy cache games for the SC and MC 39 * versions of R4000 and R4400. 40 */ 41 42LEAF(r4k_clear_page_d16) 43 addiu AT, a0, _PAGE_SIZE 441: cache Create_Dirty_Excl_D, (a0) 45 sd zero, (a0) 46 sd zero, 8(a0) 47 cache Create_Dirty_Excl_D, 16(a0) 48 sd zero, 16(a0) 49 sd zero, 24(a0) 50 addiu a0, 64 51 cache Create_Dirty_Excl_D, -32(a0) 52 sd zero, -32(a0) 53 sd zero, -24(a0) 54 cache Create_Dirty_Excl_D, -16(a0) 55 sd zero, -16(a0) 56 sd zero, -8(a0) 57 bne AT, a0, 1b 58 jr ra 59 END(r4k_clear_page_d16) 60 61LEAF(r4k_clear_page_d32) 62 addiu AT, a0, _PAGE_SIZE 631: cache Create_Dirty_Excl_D, (a0) 64 sd zero, (a0) 65 sd zero, 8(a0) 66 sd zero, 16(a0) 67 sd zero, 24(a0) 68 addiu a0, 64 69 cache Create_Dirty_Excl_D, -32(a0) 70 sd zero, -32(a0) 71 sd zero, -24(a0) 72 sd zero, -16(a0) 73 sd zero, -8(a0) 74 bne AT, a0, 1b 75 jr ra 76 END(r4k_clear_page_d32) 77 78/* 79 * This flavour of r4k_clear_page is for the R4600 V1.x. Cite from the 80 * IDT R4600 V1.7 errata: 81 * 82 * 18. The CACHE instructions Hit_Writeback_Invalidate_D, Hit_Writeback_D, 83 * Hit_Invalidate_D and Create_Dirty_Excl_D should only be 84 * executed if there is no other dcache activity. If the dcache is 85 * accessed for another instruction immeidately preceding when these 86 * cache instructions are executing, it is possible that the dcache 87 * tag match outputs used by these cache instructions will be 88 * incorrect. These cache instructions should be preceded by at least 89 * four instructions that are not any kind of load or store 90 * instruction. 91 * 92 * This is not allowed: lw 93 * nop 94 * nop 95 * nop 96 * cache Hit_Writeback_Invalidate_D 97 * 98 * This is allowed: lw 99 * nop 100 * nop 101 * nop 102 * nop 103 * cache Hit_Writeback_Invalidate_D 104 */ 105 106LEAF(r4k_clear_page_r4600_v1) 107 addiu AT, a0, _PAGE_SIZE 1081: nop 109 nop 110 nop 111 nop 112 cache Create_Dirty_Excl_D, (a0) 113 sd zero, (a0) 114 sd zero, 8(a0) 115 sd zero, 16(a0) 116 sd zero, 24(a0) 117 addiu a0, 64 118 nop 119 nop 120 nop 121 cache Create_Dirty_Excl_D, -32(a0) 122 sd zero, -32(a0) 123 sd zero, -24(a0) 124 sd zero, -16(a0) 125 sd zero, -8(a0) 126 bne AT, a0, 1b 127 jr ra 128 END(r4k_clear_page_r4600_v1) 129 130LEAF(r4k_clear_page_r4600_v2) 131 mfc0 a1, CP0_STATUS 132 ori AT, a1, 1 133 xori AT, 1 134 mtc0 AT, CP0_STATUS 135 nop 136 nop 137 nop 138 139 .set volatile 140 la AT, KSEG1 141 lw zero, (AT) 142 .set novolatile 143 144 addiu AT, a0, _PAGE_SIZE 1451: cache Create_Dirty_Excl_D, (a0) 146 sd zero, (a0) 147 sd zero, 8(a0) 148 sd zero, 16(a0) 149 sd zero, 24(a0) 150 addiu a0, 64 151 cache Create_Dirty_Excl_D, -32(a0) 152 sd zero, -32(a0) 153 sd zero, -24(a0) 154 sd zero, -16(a0) 155 sd zero, -8(a0) 156 bne AT, a0, 1b 157 158 mfc0 AT, CP0_STATUS # __restore_flags 159 andi a1, 1 160 ori AT, 1 161 xori AT, 1 162 or a1, AT 163 mtc0 a1, CP0_STATUS 164 nop 165 nop 166 nop 167 168 jr ra 169 END(r4k_clear_page_r4600_v2) 170 171/* 172 * The next 4 versions are optimized for all possible scache configurations 173 * of the SC / MC versions of R4000 and R4400 ... 174 * 175 * Todo: For even better performance we should have a routine optimized for 176 * every legal combination of dcache / scache linesize. When I (Ralf) tried 177 * this the kernel crashed shortly after mounting the root filesystem. CPU 178 * bug? Weirdo cache instruction semantics? 179 */ 180 181LEAF(r4k_clear_page_s16) 182 addiu AT, a0, _PAGE_SIZE 1831: cache Create_Dirty_Excl_SD, (a0) 184 sd zero, (a0) 185 sd zero, 8(a0) 186 cache Create_Dirty_Excl_SD, 16(a0) 187 sd zero, 16(a0) 188 sd zero, 24(a0) 189 addiu a0, 64 190 cache Create_Dirty_Excl_SD, -32(a0) 191 sd zero, -32(a0) 192 sd zero, -24(a0) 193 cache Create_Dirty_Excl_SD, -16(a0) 194 sd zero, -16(a0) 195 sd zero, -8(a0) 196 bne AT, a0, 1b 197 jr ra 198 END(r4k_clear_page_s16) 199 200LEAF(r4k_clear_page_s32) 201 addiu AT, a0, _PAGE_SIZE 2021: cache Create_Dirty_Excl_SD, (a0) 203 sd zero, (a0) 204 sd zero, 8(a0) 205 sd zero, 16(a0) 206 sd zero, 24(a0) 207 addiu a0, 64 208 cache Create_Dirty_Excl_SD, -32(a0) 209 sd zero, -32(a0) 210 sd zero, -24(a0) 211 sd zero, -16(a0) 212 sd zero, -8(a0) 213 bne AT, a0, 1b 214 jr ra 215 END(r4k_clear_page_s32) 216 217LEAF(r4k_clear_page_s64) 218 addiu AT, a0, _PAGE_SIZE 2191: cache Create_Dirty_Excl_SD, (a0) 220 sd zero, (a0) 221 sd zero, 8(a0) 222 sd zero, 16(a0) 223 sd zero, 24(a0) 224 addiu a0, 64 225 sd zero, -32(a0) 226 sd zero, -24(a0) 227 sd zero, -16(a0) 228 sd zero, -8(a0) 229 bne AT, a0, 1b 230 jr ra 231 END(r4k_clear_page_s64) 232 233LEAF(r4k_clear_page_s128) 234 addiu AT, a0, _PAGE_SIZE 2351: cache Create_Dirty_Excl_SD, (a0) 236 sd zero, (a0) 237 sd zero, 8(a0) 238 sd zero, 16(a0) 239 sd zero, 24(a0) 240 sd zero, 32(a0) 241 sd zero, 40(a0) 242 sd zero, 48(a0) 243 sd zero, 56(a0) 244 addiu a0, 128 245 sd zero, -64(a0) 246 sd zero, -56(a0) 247 sd zero, -48(a0) 248 sd zero, -40(a0) 249 sd zero, -32(a0) 250 sd zero, -24(a0) 251 sd zero, -16(a0) 252 sd zero, -8(a0) 253 bne AT, a0, 1b 254 jr ra 255 END(r4k_clear_page_s128) 256 257/* 258 * This is still inefficient. We only can do better if we know the 259 * virtual address where the copy will be accessed. 260 */ 261 262LEAF(r4k_copy_page_d16) 263 addiu AT, a0, _PAGE_SIZE 2641: cache Create_Dirty_Excl_D, (a0) 265 lw a3, (a1) 266 lw a2, 4(a1) 267 lw v1, 8(a1) 268 lw v0, 12(a1) 269 sw a3, (a0) 270 sw a2, 4(a0) 271 sw v1, 8(a0) 272 sw v0, 12(a0) 273 cache Create_Dirty_Excl_D, 16(a0) 274 lw a3, 16(a1) 275 lw a2, 20(a1) 276 lw v1, 24(a1) 277 lw v0, 28(a1) 278 sw a3, 16(a0) 279 sw a2, 20(a0) 280 sw v1, 24(a0) 281 sw v0, 28(a0) 282 cache Create_Dirty_Excl_D, 32(a0) 283 addiu a0, 64 284 addiu a1, 64 285 lw a3, -32(a1) 286 lw a2, -28(a1) 287 lw v1, -24(a1) 288 lw v0, -20(a1) 289 sw a3, -32(a0) 290 sw a2, -28(a0) 291 sw v1, -24(a0) 292 sw v0, -20(a0) 293 cache Create_Dirty_Excl_D, -16(a0) 294 lw a3, -16(a1) 295 lw a2, -12(a1) 296 lw v1, -8(a1) 297 lw v0, -4(a1) 298 sw a3, -16(a0) 299 sw a2, -12(a0) 300 sw v1, -8(a0) 301 sw v0, -4(a0) 302 bne AT, a0, 1b 303 jr ra 304 END(r4k_copy_page_d16) 305 306LEAF(r4k_copy_page_d32) 307 addiu AT, a0, _PAGE_SIZE 3081: cache Create_Dirty_Excl_D, (a0) 309 lw a3, (a1) 310 lw a2, 4(a1) 311 lw v1, 8(a1) 312 lw v0, 12(a1) 313 sw a3, (a0) 314 sw a2, 4(a0) 315 sw v1, 8(a0) 316 sw v0, 12(a0) 317 lw a3, 16(a1) 318 lw a2, 20(a1) 319 lw v1, 24(a1) 320 lw v0, 28(a1) 321 sw a3, 16(a0) 322 sw a2, 20(a0) 323 sw v1, 24(a0) 324 sw v0, 28(a0) 325 cache Create_Dirty_Excl_D, 32(a0) 326 addiu a0, 64 327 addiu a1, 64 328 lw a3, -32(a1) 329 lw a2, -28(a1) 330 lw v1, -24(a1) 331 lw v0, -20(a1) 332 sw a3, -32(a0) 333 sw a2, -28(a0) 334 sw v1, -24(a0) 335 sw v0, -20(a0) 336 lw a3, -16(a1) 337 lw a2, -12(a1) 338 lw v1, -8(a1) 339 lw v0, -4(a1) 340 sw a3, -16(a0) 341 sw a2, -12(a0) 342 sw v1, -8(a0) 343 sw v0, -4(a0) 344 bne AT, a0, 1b 345 jr ra 346 END(r4k_copy_page_d32) 347 348/* 349 * Again a special version for the R4600 V1.x 350 */ 351 352LEAF(r4k_copy_page_r4600_v1) 353 addiu AT, a0, _PAGE_SIZE 3541: nop 355 nop 356 nop 357 nop 358 cache Create_Dirty_Excl_D, (a0) 359 lw a3, (a1) 360 lw a2, 4(a1) 361 lw v1, 8(a1) 362 lw v0, 12(a1) 363 sw a3, (a0) 364 sw a2, 4(a0) 365 sw v1, 8(a0) 366 sw v0, 12(a0) 367 lw a3, 16(a1) 368 lw a2, 20(a1) 369 lw v1, 24(a1) 370 lw v0, 28(a1) 371 sw a3, 16(a0) 372 sw a2, 20(a0) 373 sw v1, 24(a0) 374 sw v0, 28(a0) 375 nop 376 nop 377 nop 378 nop 379 cache Create_Dirty_Excl_D, 32(a0) 380 addiu a0, 64 381 addiu a1, 64 382 lw a3, -32(a1) 383 lw a2, -28(a1) 384 lw v1, -24(a1) 385 lw v0, -20(a1) 386 sw a3, -32(a0) 387 sw a2, -28(a0) 388 sw v1, -24(a0) 389 sw v0, -20(a0) 390 lw a3, -16(a1) 391 lw a2, -12(a1) 392 lw v1, -8(a1) 393 lw v0, -4(a1) 394 sw a3, -16(a0) 395 sw a2, -12(a0) 396 sw v1, -8(a0) 397 sw v0, -4(a0) 398 bne AT, a0, 1b 399 jr ra 400 END(r4k_copy_page_r4600_v1) 401 402LEAF(r4k_copy_page_r4600_v2) 403 mfc0 v1, CP0_STATUS 404 ori AT, v1, 1 405 xori AT, 1 406 407 mtc0 AT, CP0_STATUS 408 nop 409 nop 410 nop 411 412 addiu AT, a0, _PAGE_SIZE 4131: nop 414 nop 415 nop 416 nop 417 cache Create_Dirty_Excl_D, (a0) 418 lw t1, (a1) 419 lw t0, 4(a1) 420 lw a3, 8(a1) 421 lw a2, 12(a1) 422 sw t1, (a0) 423 sw t0, 4(a0) 424 sw a3, 8(a0) 425 sw a2, 12(a0) 426 lw t1, 16(a1) 427 lw t0, 20(a1) 428 lw a3, 24(a1) 429 lw a2, 28(a1) 430 sw t1, 16(a0) 431 sw t0, 20(a0) 432 sw a3, 24(a0) 433 sw a2, 28(a0) 434 nop 435 nop 436 nop 437 nop 438 cache Create_Dirty_Excl_D, 32(a0) 439 addiu a0, 64 440 addiu a1, 64 441 lw t1, -32(a1) 442 lw t0, -28(a1) 443 lw a3, -24(a1) 444 lw a2, -20(a1) 445 sw t1, -32(a0) 446 sw t0, -28(a0) 447 sw a3, -24(a0) 448 sw a2, -20(a0) 449 lw t1, -16(a1) 450 lw t0, -12(a1) 451 lw a3, -8(a1) 452 lw a2, -4(a1) 453 sw t1, -16(a0) 454 sw t0, -12(a0) 455 sw a3, -8(a0) 456 sw a2, -4(a0) 457 bne AT, a0, 1b 458 459 mfc0 AT, CP0_STATUS # __restore_flags 460 andi v1, 1 461 ori AT, 1 462 xori AT, 1 463 or v1, AT 464 mtc0 v1, CP0_STATUS 465 nop 466 nop 467 nop 468 jr ra 469 END(r4k_copy_page_r4600_v2) 470 471/* 472 * These are for R4000SC / R4400MC 473 */ 474 475LEAF(r4k_copy_page_s16) 476 addiu AT, a0, _PAGE_SIZE 4771: cache Create_Dirty_Excl_SD, (a0) 478 lw a3, (a1) 479 lw a2, 4(a1) 480 lw v1, 8(a1) 481 lw v0, 12(a1) 482 sw a3, (a0) 483 sw a2, 4(a0) 484 sw v1, 8(a0) 485 sw v0, 12(a0) 486 cache Create_Dirty_Excl_SD, 16(a0) 487 lw a3, 16(a1) 488 lw a2, 20(a1) 489 lw v1, 24(a1) 490 lw v0, 28(a1) 491 sw a3, 16(a0) 492 sw a2, 20(a0) 493 sw v1, 24(a0) 494 sw v0, 28(a0) 495 cache Create_Dirty_Excl_SD, 32(a0) 496 addiu a0, 64 497 addiu a1, 64 498 lw a3, -32(a1) 499 lw a2, -28(a1) 500 lw v1, -24(a1) 501 lw v0, -20(a1) 502 sw a3, -32(a0) 503 sw a2, -28(a0) 504 sw v1, -24(a0) 505 sw v0, -20(a0) 506 cache Create_Dirty_Excl_SD, -16(a0) 507 lw a3, -16(a1) 508 lw a2, -12(a1) 509 lw v1, -8(a1) 510 lw v0, -4(a1) 511 sw a3, -16(a0) 512 sw a2, -12(a0) 513 sw v1, -8(a0) 514 sw v0, -4(a0) 515 bne AT, a0, 1b 516 jr ra 517 END(r4k_copy_page_s16) 518 519LEAF(r4k_copy_page_s32) 520 addiu AT, a0, _PAGE_SIZE 5211: cache Create_Dirty_Excl_SD, (a0) 522 lw a3, (a1) 523 lw a2, 4(a1) 524 lw v1, 8(a1) 525 lw v0, 12(a1) 526 sw a3, (a0) 527 sw a2, 4(a0) 528 sw v1, 8(a0) 529 sw v0, 12(a0) 530 lw a3, 16(a1) 531 lw a2, 20(a1) 532 lw v1, 24(a1) 533 lw v0, 28(a1) 534 sw a3, 16(a0) 535 sw a2, 20(a0) 536 sw v1, 24(a0) 537 sw v0, 28(a0) 538 cache Create_Dirty_Excl_SD, 32(a0) 539 addiu a0, 64 540 addiu a1, 64 541 lw a3, -32(a1) 542 lw a2, -28(a1) 543 lw v1, -24(a1) 544 lw v0, -20(a1) 545 sw a3, -32(a0) 546 sw a2, -28(a0) 547 sw v1, -24(a0) 548 sw v0, -20(a0) 549 lw a3, -16(a1) 550 lw a2, -12(a1) 551 lw v1, -8(a1) 552 lw v0, -4(a1) 553 sw a3, -16(a0) 554 sw a2, -12(a0) 555 sw v1, -8(a0) 556 sw v0, -4(a0) 557 bne AT, a0, 1b 558 jr ra 559 END(r4k_copy_page_s32) 560 561LEAF(r4k_copy_page_s64) 562 addiu AT, a0, _PAGE_SIZE 5631: cache Create_Dirty_Excl_SD, (a0) 564 lw a3, (a1) 565 lw a2, 4(a1) 566 lw v1, 8(a1) 567 lw v0, 12(a1) 568 sw a3, (a0) 569 sw a2, 4(a0) 570 sw v1, 8(a0) 571 sw v0, 12(a0) 572 lw a3, 16(a1) 573 lw a2, 20(a1) 574 lw v1, 24(a1) 575 lw v0, 28(a1) 576 sw a3, 16(a0) 577 sw a2, 20(a0) 578 sw v1, 24(a0) 579 sw v0, 28(a0) 580 addiu a0, 64 581 addiu a1, 64 582 lw a3, -32(a1) 583 lw a2, -28(a1) 584 lw v1, -24(a1) 585 lw v0, -20(a1) 586 sw a3, -32(a0) 587 sw a2, -28(a0) 588 sw v1, -24(a0) 589 sw v0, -20(a0) 590 lw a3, -16(a1) 591 lw a2, -12(a1) 592 lw v1, -8(a1) 593 lw v0, -4(a1) 594 sw a3, -16(a0) 595 sw a2, -12(a0) 596 sw v1, -8(a0) 597 sw v0, -4(a0) 598 bne AT, a0, 1b 599 jr ra 600 END(r4k_copy_page_s64) 601 602LEAF(r4k_copy_page_s128) 603 addiu AT, a0, _PAGE_SIZE 6041: cache Create_Dirty_Excl_SD, (a0) 605 lw a3, (a1) 606 lw a2, 4(a1) 607 lw v1, 8(a1) 608 lw v0, 12(a1) 609 sw a3, (a0) 610 sw a2, 4(a0) 611 sw v1, 8(a0) 612 sw v0, 12(a0) 613 lw a3, 16(a1) 614 lw a2, 20(a1) 615 lw v1, 24(a1) 616 lw v0, 28(a1) 617 sw a3, 16(a0) 618 sw a2, 20(a0) 619 sw v1, 24(a0) 620 sw v0, 28(a0) 621 lw a3, 32(a1) 622 lw a2, 36(a1) 623 lw v1, 40(a1) 624 lw v0, 44(a1) 625 sw a3, 32(a0) 626 sw a2, 36(a0) 627 sw v1, 40(a0) 628 sw v0, 44(a0) 629 lw a3, 48(a1) 630 lw a2, 52(a1) 631 lw v1, 56(a1) 632 lw v0, 60(a1) 633 sw a3, 48(a0) 634 sw a2, 52(a0) 635 sw v1, 56(a0) 636 sw v0, 60(a0) 637 addiu a0, 128 638 addiu a1, 128 639 lw a3, -64(a1) 640 lw a2, -60(a1) 641 lw v1, -56(a1) 642 lw v0, -52(a1) 643 sw a3, -64(a0) 644 sw a2, -60(a0) 645 sw v1, -56(a0) 646 sw v0, -52(a0) 647 lw a3, -48(a1) 648 lw a2, -44(a1) 649 lw v1, -40(a1) 650 lw v0, -36(a1) 651 sw a3, -48(a0) 652 sw a2, -44(a0) 653 sw v1, -40(a0) 654 sw v0, -36(a0) 655 lw a3, -32(a1) 656 lw a2, -28(a1) 657 lw v1, -24(a1) 658 lw v0, -20(a1) 659 sw a3, -32(a0) 660 sw a2, -28(a0) 661 sw v1, -24(a0) 662 sw v0, -20(a0) 663 lw a3, -16(a1) 664 lw a2, -12(a1) 665 lw v1, -8(a1) 666 lw v0, -4(a1) 667 sw a3, -16(a0) 668 sw a2, -12(a0) 669 sw v1, -8(a0) 670 sw v0, -4(a0) 671 bne AT, a0, 1b 672 jr ra 673 END(r4k_copy_page_s128) 674 675/* This one still needs to receive cache optimizations */ 676LEAF(pgd_init) 677 addiu AT, a0, PGD_SIZE / 2 678 la v0, invalid_pte_table 6791: sw v0, (a0) 680 sw v0, 4(a0) 681 sw v0, 8(a0) 682 sw v0, 12(a0) 683 addiu a0, 32 684 sw v0, -16(a0) 685 sw v0, -12(a0) 686 sw v0, -8(a0) 687 sw v0, -4(a0) 688 bne AT, a0, 1b 689 jr ra 690 END(pgd_init) 691