1/*
2 * Copyright (c) 2006 Apple Computer, Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29#include <machine/cpu_capabilities.h>
30#include <machine/commpage.h>
31
32
33/*
34 * The bcopy/memcpy loops for very long operands, tuned for Pentium-M
35 * class processors with Supplemental SSE3 and 64-byte cache lines.
36 *
37 * The following #defines are tightly coupled to the u-architecture:
38 */
39
40#define kBigChunk   (256*1024)          // outer loop chunk size for kVeryLong sized operands
41
42
43// Very long forward moves.  These are at least several pages, so we loop over big
44// chunks of memory (kBigChunk in size.)  We first prefetch the chunk, and then copy
45// it using non-temporal stores.  Hopefully all the reads occur in the prefetch loop,
46// so the copy loop reads from L2 and writes directly to memory (with write combining.)
47// This minimizes bus turnaround and maintains good DRAM page locality.
48// Note that for this scheme to work, kVeryLong must be a large fraction of L2 cache
49// size.  Otherwise, it is counter-productive to bypass L2 on the stores.
50//
51// We are called from the commpage bcopy loops when they encounter very long
52// operands, with the standard ABI.
53//
54//	void longcopy(const void *dest, void *sou, size_t len)
55
56        .text
57        .align	5, 0x90
58Llongcopy_sse3x:			// void longcopy(const void *dest, void *sou, size_t len)
59	pushl	%ebp			// set up a frame for backtraces
60	movl	%esp,%ebp
61        pushl   %esi
62        pushl   %edi
63	pushl	%ebx			// we'll need to use this too
64        movl    8(%ebp),%edi		// get dest ptr
65        movl    12(%ebp),%esi           // get source ptr
66        movl    16(%ebp),%ecx           // get length
67	movl	%edi,%ebx		// copy dest ptr
68	negl	%ebx
69	andl	$63,%ebx		// get #bytes to cache line align destination
70	jz	LBigChunkLoop		// already aligned
71
72// Cache line align destination, so temporal stores in copy loops work right.
73
74	pushl	%ebx			// arg3 - #bytes to align destination (1..63)
75	pushl	%esi			// arg2 - source
76	pushl	%edi			// arg1 - dest
77	movl	$(_COMM_PAGE_MEMCPY),%eax
78	call	*%eax			// align the destination
79	addl	$12,%esp
80        movl    8(%ebp),%edi		// recover dest ptr
81        movl    12(%ebp),%esi           // recover source ptr
82        movl    16(%ebp),%ecx           // recover length
83	addl	%ebx,%esi		// adjust ptrs and lengths past copy
84	addl	%ebx,%edi
85	subl	%ebx,%ecx
86
87// Loop over big chunks.
88//      ecx = length remaining (>= 4096)
89//      edi = dest (64-byte aligned)
90//      esi = source (may be unaligned)
91
92LBigChunkLoop:
93        movl    $(kBigChunk),%edx       // assume we can do a full chunk
94	cmpl	%edx,%ecx		// do we have a full chunk left to do?
95	cmovbl	%ecx,%edx		// if not, only move what we have left
96	andl	$-4096,%edx		// we work in page multiples
97	xor	%eax,%eax		// initialize chunk offset
98	jmp	LTouchLoop
99
100// Touch in the next chunk.  We try to keep the prefetch unit in "kick-start" mode,
101// by touching two adjacent cache lines every 8 lines of each page, in four slices.
102// Because the source may be unaligned, we use byte loads to touch.
103//      ecx = length remaining (including this chunk)
104//      edi = ptr to start of dest chunk
105//      esi = ptr to start of source chunk
106//	edx = chunk length (multiples of pages)
107//	ebx = scratch reg used to read a byte of each cache line
108//	eax = chunk offset
109
110	.align	4,0x90			// 16-byte align inner loops
111LTouchLoop:
112	movzb	(%esi,%eax),%ebx	// touch line 0, 2, 4, or 6 of page
113	movzb	1*64(%esi,%eax),%ebx	// touch line 1, 3, 5, or 7
114	movzb	8*64(%esi,%eax),%ebx	// touch line 8, 10, 12, or 14
115	movzb	9*64(%esi,%eax),%ebx	// etc
116
117	movzb	16*64(%esi,%eax),%ebx
118	movzb	17*64(%esi,%eax),%ebx
119	movzb	24*64(%esi,%eax),%ebx
120	movzb	25*64(%esi,%eax),%ebx
121
122	movzb	32*64(%esi,%eax),%ebx
123	movzb	33*64(%esi,%eax),%ebx
124	movzb	40*64(%esi,%eax),%ebx
125	movzb	41*64(%esi,%eax),%ebx
126
127	movzb	48*64(%esi,%eax),%ebx
128	movzb	49*64(%esi,%eax),%ebx
129	movzb	56*64(%esi,%eax),%ebx
130	movzb	57*64(%esi,%eax),%ebx
131
132	subl	$-128,%eax		// next slice of page (adding 128 w 8-bit immediate)
133	testl	$512,%eax		// done with this page?
134	jz	LTouchLoop		// no, next of four slices
135	addl	$(4096-512),%eax	// move on to next page
136	cmpl	%eax,%edx		// done with this chunk?
137	jnz	LTouchLoop		// no, do next page
138
139// The chunk has been pre-fetched, now copy it using non-temporal stores.
140// There are two copy loops, depending on whether the source is 16-byte aligned
141// or not.
142
143	addl	%edx,%esi		// increment ptrs by chunk length
144	addl	%edx,%edi
145	subl	%edx,%ecx		// adjust remaining length
146	negl	%edx			// prepare loop index (counts up to 0)
147	testl	$15,%esi		// is source 16-byte aligned?
148	jnz	LVeryLongUnaligned	// source is not aligned
149	jmp	LVeryLongAligned
150
151	.align	4,0x90			// 16-byte align inner loops
152LVeryLongAligned:			// aligned loop over 128-bytes
153        movdqa  (%esi,%edx),%xmm0
154        movdqa  16(%esi,%edx),%xmm1
155        movdqa  32(%esi,%edx),%xmm2
156        movdqa  48(%esi,%edx),%xmm3
157        movdqa  64(%esi,%edx),%xmm4
158        movdqa  80(%esi,%edx),%xmm5
159        movdqa  96(%esi,%edx),%xmm6
160        movdqa  112(%esi,%edx),%xmm7
161
162        movntdq %xmm0,(%edi,%edx)
163        movntdq %xmm1,16(%edi,%edx)
164        movntdq %xmm2,32(%edi,%edx)
165        movntdq %xmm3,48(%edi,%edx)
166        movntdq %xmm4,64(%edi,%edx)
167        movntdq %xmm5,80(%edi,%edx)
168        movntdq %xmm6,96(%edi,%edx)
169        movntdq %xmm7,112(%edi,%edx)
170
171        subl    $-128,%edx		// add 128 with an 8-bit immediate
172        jnz	LVeryLongAligned
173	jmp	LVeryLongChunkEnd
174
175	.align	4,0x90			// 16-byte align inner loops
176LVeryLongUnaligned:			// unaligned loop over 128-bytes
177        movdqu  (%esi,%edx),%xmm0
178        movdqu  16(%esi,%edx),%xmm1
179        movdqu  32(%esi,%edx),%xmm2
180        movdqu  48(%esi,%edx),%xmm3
181        movdqu  64(%esi,%edx),%xmm4
182        movdqu  80(%esi,%edx),%xmm5
183        movdqu  96(%esi,%edx),%xmm6
184        movdqu  112(%esi,%edx),%xmm7
185
186        movntdq %xmm0,(%edi,%edx)
187        movntdq %xmm1,16(%edi,%edx)
188        movntdq %xmm2,32(%edi,%edx)
189        movntdq %xmm3,48(%edi,%edx)
190        movntdq %xmm4,64(%edi,%edx)
191        movntdq %xmm5,80(%edi,%edx)
192        movntdq %xmm6,96(%edi,%edx)
193        movntdq %xmm7,112(%edi,%edx)
194
195        subl    $-128,%edx		// add 128 with an 8-bit immediate
196        jnz     LVeryLongUnaligned
197
198LVeryLongChunkEnd:
199	cmpl	$4096,%ecx		// at least another page to go?
200	jae	LBigChunkLoop		// yes
201
202// Done.  Call memcpy() again to handle the 0-4095 bytes at the end.
203
204	sfence				// required by non-temporal stores
205	testl	%ecx,%ecx		// anything left to copy?
206	jz	1f
207	pushl	%ecx			// arg3 - #bytes to align destination (1..63)
208	pushl	%esi			// arg2 - source
209	pushl	%edi			// arg1 - dest
210	movl	$(_COMM_PAGE_MEMCPY),%eax
211	call	*%eax			// align the destination
212	addl	$12,%esp		// pop off arguments
2131:
214	popl	%ebx
215	popl	%edi
216	popl	%esi
217	popl	%ebp
218	ret
219
220	/* always match for now, as commpage_stuff_routine() will panic if no match */
221	COMMPAGE_DESCRIPTOR(longcopy_sse3x, _COMM_PAGE_LONGCOPY, 0 ,0)
222