1/* Written 2003 by Andi Kleen, based on a kernel by Evandro Menezes */ 2 3#include <linux/linkage.h> 4#include <asm/dwarf2.h> 5 6 ALIGN 7copy_page_c: 8 CFI_STARTPROC 9 movl $4096/8,%ecx 10 rep movsq 11 ret 12 CFI_ENDPROC 13ENDPROC(copy_page_c) 14 15/* Don't use streaming store because it's better when the target 16 ends up in cache. */ 17 18/* Could vary the prefetch distance based on SMP/UP */ 19 20ENTRY(copy_page) 21 CFI_STARTPROC 22 subq $3*8,%rsp 23 CFI_ADJUST_CFA_OFFSET 3*8 24 movq %rbx,(%rsp) 25 CFI_REL_OFFSET rbx, 0 26 movq %r12,1*8(%rsp) 27 CFI_REL_OFFSET r12, 1*8 28 movq %r13,2*8(%rsp) 29 CFI_REL_OFFSET r13, 2*8 30 31 movl $(4096/64)-5,%ecx 32 .p2align 4 33.Loop64: 34 dec %rcx 35 36 movq (%rsi), %rax 37 movq 8 (%rsi), %rbx 38 movq 16 (%rsi), %rdx 39 movq 24 (%rsi), %r8 40 movq 32 (%rsi), %r9 41 movq 40 (%rsi), %r10 42 movq 48 (%rsi), %r11 43 movq 56 (%rsi), %r12 44 45 prefetcht0 5*64(%rsi) 46 47 movq %rax, (%rdi) 48 movq %rbx, 8 (%rdi) 49 movq %rdx, 16 (%rdi) 50 movq %r8, 24 (%rdi) 51 movq %r9, 32 (%rdi) 52 movq %r10, 40 (%rdi) 53 movq %r11, 48 (%rdi) 54 movq %r12, 56 (%rdi) 55 56 leaq 64 (%rsi), %rsi 57 leaq 64 (%rdi), %rdi 58 59 jnz .Loop64 60 61 movl $5,%ecx 62 .p2align 4 63.Loop2: 64 decl %ecx 65 66 movq (%rsi), %rax 67 movq 8 (%rsi), %rbx 68 movq 16 (%rsi), %rdx 69 movq 24 (%rsi), %r8 70 movq 32 (%rsi), %r9 71 movq 40 (%rsi), %r10 72 movq 48 (%rsi), %r11 73 movq 56 (%rsi), %r12 74 75 movq %rax, (%rdi) 76 movq %rbx, 8 (%rdi) 77 movq %rdx, 16 (%rdi) 78 movq %r8, 24 (%rdi) 79 movq %r9, 32 (%rdi) 80 movq %r10, 40 (%rdi) 81 movq %r11, 48 (%rdi) 82 movq %r12, 56 (%rdi) 83 84 leaq 64(%rdi),%rdi 85 leaq 64(%rsi),%rsi 86 87 jnz .Loop2 88 89 movq (%rsp),%rbx 90 CFI_RESTORE rbx 91 movq 1*8(%rsp),%r12 92 CFI_RESTORE r12 93 movq 2*8(%rsp),%r13 94 CFI_RESTORE r13 95 addq $3*8,%rsp 96 CFI_ADJUST_CFA_OFFSET -3*8 97 ret 98.Lcopy_page_end: 99 CFI_ENDPROC 100ENDPROC(copy_page) 101 102 /* Some CPUs run faster using the string copy instructions. 103 It is also a lot simpler. Use this when possible */ 104 105#include <asm/cpufeature.h> 106 107 .section .altinstr_replacement,"ax" 1081: .byte 0xeb /* jmp <disp8> */ 109 .byte (copy_page_c - copy_page) - (2f - 1b) /* offset */ 1102: 111 .previous 112 .section .altinstructions,"a" 113 .align 8 114 .quad copy_page 115 .quad 1b 116 .word X86_FEATURE_REP_GOOD 117 .byte .Lcopy_page_end - copy_page 118 .byte 2b - 1b 119 .previous 120