1// SPDX-License-Identifier: GPL-2.0+
2/*
3 * Copyright (C) 1991,1992,1993,1997,1998,2003, 2005 Free Software Foundation, Inc.
4 * This file is part of the GNU C Library.
5 * Copyright (c) 2011 The Chromium OS Authors.
6 */
7
8/* From glibc-2.14, sysdeps/i386/memset.c */
9
10#include <linux/types.h>
11#include <linux/compiler.h>
12#include <asm/string.h>
13
14typedef uint32_t op_t;
15
16void *memset(void *dstpp, int c, size_t len)
17{
18	int d0;
19	unsigned long int dstp = (unsigned long int) dstpp;
20
21	/* This explicit register allocation improves code very much indeed. */
22	register op_t x asm("ax");
23
24	x = (unsigned char) c;
25
26	/* Clear the direction flag, so filling will move forward.  */
27	asm volatile("cld");
28
29	/* This threshold value is optimal.  */
30	if (len >= 12) {
31		/* Fill X with four copies of the char we want to fill with. */
32		x |= (x << 8);
33		x |= (x << 16);
34
35		/* Adjust LEN for the bytes handled in the first loop.  */
36		len -= (-dstp) % sizeof(op_t);
37
38		/*
39		 * There are at least some bytes to set. No need to test for
40		 * LEN == 0 in this alignment loop.
41		 */
42
43		/* Fill bytes until DSTP is aligned on a longword boundary. */
44		asm volatile(
45			"rep\n"
46			"stosb" /* %0, %2, %3 */ :
47			"=D" (dstp), "=c" (d0) :
48			"0" (dstp), "1" ((-dstp) % sizeof(op_t)), "a" (x) :
49			"memory");
50
51		/* Fill longwords.  */
52		asm volatile(
53			"rep\n"
54			"stosl" /* %0, %2, %3 */ :
55			"=D" (dstp), "=c" (d0) :
56			"0" (dstp), "1" (len / sizeof(op_t)), "a" (x) :
57			"memory");
58		len %= sizeof(op_t);
59	}
60
61	/* Write the last few bytes. */
62	asm volatile(
63		"rep\n"
64		"stosb" /* %0, %2, %3 */ :
65		"=D" (dstp), "=c" (d0) :
66		"0" (dstp), "1" (len), "a" (x) :
67		"memory");
68
69	return dstpp;
70}
71
72#define	OP_T_THRES	8
73#define OPSIZ	(sizeof(op_t))
74
75#define BYTE_COPY_FWD(dst_bp, src_bp, nbytes)				  \
76do {									  \
77	int __d0;							  \
78	asm volatile(							  \
79		/* Clear the direction flag, so copying goes forward.  */ \
80		"cld\n"							  \
81		/* Copy bytes.  */					  \
82		"rep\n"							  \
83		"movsb" :						  \
84		"=D" (dst_bp), "=S" (src_bp), "=c" (__d0) :		  \
85		"0" (dst_bp), "1" (src_bp), "2" (nbytes) :		  \
86		"memory");						  \
87} while (0)
88
89#define WORD_COPY_FWD(dst_bp, src_bp, nbytes_left, nbytes)		  \
90do {									  \
91	int __d0;							  \
92	asm volatile(							  \
93		/* Clear the direction flag, so copying goes forward.  */ \
94		"cld\n"							  \
95		/* Copy longwords.  */					  \
96		"rep\n"							  \
97		"movsl" :						  \
98		"=D" (dst_bp), "=S" (src_bp), "=c" (__d0) :		  \
99		"0" (dst_bp), "1" (src_bp), "2" ((nbytes) / 4) :	  \
100		"memory");						  \
101	(nbytes_left) = (nbytes) % 4;					  \
102} while (0)
103
104void *memcpy(void *dstpp, const void *srcpp, size_t len)
105{
106	unsigned long int dstp = (long int)dstpp;
107	unsigned long int srcp = (long int)srcpp;
108
109	/* Copy from the beginning to the end.  */
110
111	/* If there not too few bytes to copy, use word copy.  */
112	if (len >= OP_T_THRES) {
113		/* Copy just a few bytes to make DSTP aligned.  */
114		len -= (-dstp) % OPSIZ;
115		BYTE_COPY_FWD(dstp, srcp, (-dstp) % OPSIZ);
116
117		/* Copy from SRCP to DSTP taking advantage of the known
118		 * alignment of DSTP.  Number of bytes remaining is put
119		 * in the third argument, i.e. in LEN.  This number may
120		 * vary from machine to machine.
121		 */
122		WORD_COPY_FWD(dstp, srcp, len, len);
123
124		/* Fall out and copy the tail.  */
125	}
126
127	/* There are just a few bytes to copy. Use byte memory operations. */
128	BYTE_COPY_FWD(dstp, srcp, len);
129
130	return dstpp;
131}
132
133void *memmove(void *dest, const void *src, size_t n)
134{
135	int d0, d1, d2, d3, d4, d5;
136	char *ret = dest;
137
138	__asm__ __volatile__(
139		/* Handle more 16 bytes in loop */
140		"cmp $0x10, %0\n\t"
141		"jb	1f\n\t"
142
143		/* Decide forward/backward copy mode */
144		"cmp %2, %1\n\t"
145		"jb	2f\n\t"
146
147		/*
148		 * movs instruction have many startup latency
149		 * so we handle small size by general register.
150		 */
151		"cmp  $680, %0\n\t"
152		"jb 3f\n\t"
153		/* movs instruction is only good for aligned case */
154		"mov %1, %3\n\t"
155		"xor %2, %3\n\t"
156		"and $0xff, %3\n\t"
157		"jz 4f\n\t"
158		"3:\n\t"
159		"sub $0x10, %0\n\t"
160
161		/* We gobble 16 bytes forward in each loop */
162		"3:\n\t"
163		"sub $0x10, %0\n\t"
164		"mov 0*4(%1), %3\n\t"
165		"mov 1*4(%1), %4\n\t"
166		"mov  %3, 0*4(%2)\n\t"
167		"mov  %4, 1*4(%2)\n\t"
168		"mov 2*4(%1), %3\n\t"
169		"mov 3*4(%1), %4\n\t"
170		"mov  %3, 2*4(%2)\n\t"
171		"mov  %4, 3*4(%2)\n\t"
172		"lea  0x10(%1), %1\n\t"
173		"lea  0x10(%2), %2\n\t"
174		"jae 3b\n\t"
175		"add $0x10, %0\n\t"
176		"jmp 1f\n\t"
177
178		/* Handle data forward by movs */
179		".p2align 4\n\t"
180		"4:\n\t"
181		"mov -4(%1, %0), %3\n\t"
182		"lea -4(%2, %0), %4\n\t"
183		"shr $2, %0\n\t"
184		"rep movsl\n\t"
185		"mov %3, (%4)\n\t"
186		"jmp 11f\n\t"
187		/* Handle data backward by movs */
188		".p2align 4\n\t"
189		"6:\n\t"
190		"mov (%1), %3\n\t"
191		"mov %2, %4\n\t"
192		"lea -4(%1, %0), %1\n\t"
193		"lea -4(%2, %0), %2\n\t"
194		"shr $2, %0\n\t"
195		"std\n\t"
196		"rep movsl\n\t"
197		"mov %3,(%4)\n\t"
198		"cld\n\t"
199		"jmp 11f\n\t"
200
201		/* Start to prepare for backward copy */
202		".p2align 4\n\t"
203		"2:\n\t"
204		"cmp  $680, %0\n\t"
205		"jb 5f\n\t"
206		"mov %1, %3\n\t"
207		"xor %2, %3\n\t"
208		"and $0xff, %3\n\t"
209		"jz 6b\n\t"
210
211		/* Calculate copy position to tail */
212		"5:\n\t"
213		"add %0, %1\n\t"
214		"add %0, %2\n\t"
215		"sub $0x10, %0\n\t"
216
217		/* We gobble 16 bytes backward in each loop */
218		"7:\n\t"
219		"sub $0x10, %0\n\t"
220
221		"mov -1*4(%1), %3\n\t"
222		"mov -2*4(%1), %4\n\t"
223		"mov  %3, -1*4(%2)\n\t"
224		"mov  %4, -2*4(%2)\n\t"
225		"mov -3*4(%1), %3\n\t"
226		"mov -4*4(%1), %4\n\t"
227		"mov  %3, -3*4(%2)\n\t"
228		"mov  %4, -4*4(%2)\n\t"
229		"lea  -0x10(%1), %1\n\t"
230		"lea  -0x10(%2), %2\n\t"
231		"jae 7b\n\t"
232		/* Calculate copy position to head */
233		"add $0x10, %0\n\t"
234		"sub %0, %1\n\t"
235		"sub %0, %2\n\t"
236
237		/* Move data from 8 bytes to 15 bytes */
238		".p2align 4\n\t"
239		"1:\n\t"
240		"cmp $8, %0\n\t"
241		"jb 8f\n\t"
242		"mov 0*4(%1), %3\n\t"
243		"mov 1*4(%1), %4\n\t"
244		"mov -2*4(%1, %0), %5\n\t"
245		"mov -1*4(%1, %0), %1\n\t"
246
247		"mov  %3, 0*4(%2)\n\t"
248		"mov  %4, 1*4(%2)\n\t"
249		"mov  %5, -2*4(%2, %0)\n\t"
250		"mov  %1, -1*4(%2, %0)\n\t"
251		"jmp 11f\n\t"
252
253		/* Move data from 4 bytes to 7 bytes */
254		".p2align 4\n\t"
255		"8:\n\t"
256		"cmp $4, %0\n\t"
257		"jb 9f\n\t"
258		"mov 0*4(%1), %3\n\t"
259		"mov -1*4(%1, %0), %4\n\t"
260		"mov  %3, 0*4(%2)\n\t"
261		"mov  %4, -1*4(%2, %0)\n\t"
262		"jmp 11f\n\t"
263
264		/* Move data from 2 bytes to 3 bytes */
265		".p2align 4\n\t"
266		"9:\n\t"
267		"cmp $2, %0\n\t"
268		"jb 10f\n\t"
269		"movw 0*2(%1), %%dx\n\t"
270		"movw -1*2(%1, %0), %%bx\n\t"
271		"movw %%dx, 0*2(%2)\n\t"
272		"movw %%bx, -1*2(%2, %0)\n\t"
273		"jmp 11f\n\t"
274
275		/* Move data for 1 byte */
276		".p2align 4\n\t"
277		"10:\n\t"
278		"cmp $1, %0\n\t"
279		"jb 11f\n\t"
280		"movb (%1), %%cl\n\t"
281		"movb %%cl, (%2)\n\t"
282		".p2align 4\n\t"
283		"11:"
284		: "=&c" (d0), "=&S" (d1), "=&D" (d2),
285		  "=r" (d3), "=r" (d4), "=r"(d5)
286		: "0" (n),
287		 "1" (src),
288		 "2" (dest)
289		: "memory");
290
291	return ret;
292}
293