1 // SPDX-License-Identifier: GPL-2.0
2 #include <linux/string.h>
3 #include <linux/export.h>
4
5 #undef memcpy
6 #undef memset
7
8 __visible void *memcpy(void *to, const void *from, size_t n)
9 {
10 #if defined(CONFIG_X86_USE_3DNOW) && !defined(CONFIG_FORTIFY_SOURCE)
11 return __memcpy3d(to, from, n);
12 #else
13 return __memcpy(to, from, n);
14 #endif
15 }
16 EXPORT_SYMBOL(memcpy);
17
18 __visible void *memset(void *s, int c, size_t count)
19 {
20 return __memset(s, c, count);
21 }
22 EXPORT_SYMBOL(memset);
23
24 __visible void *memmove(void *dest, const void *src, size_t n)
25 {
26 int d0,d1,d2,d3,d4,d5;
27 char *ret = dest;
28
29 __asm__ __volatile__(
30 /* Handle more 16 bytes in loop */
31 "cmp $0x10, %0\n\t"
32 "jb 1f\n\t"
33
34 /* Decide forward/backward copy mode */
35 "cmp %2, %1\n\t"
36 "jb 2f\n\t"
37
38 /*
39 * movs instruction have many startup latency
40 * so we handle small size by general register.
41 */
42 "cmp $680, %0\n\t"
43 "jb 3f\n\t"
44 /*
45 * movs instruction is only good for aligned case.
46 */
47 "mov %1, %3\n\t"
48 "xor %2, %3\n\t"
49 "and $0xff, %3\n\t"
50 "jz 4f\n\t"
51 "3:\n\t"
52 "sub $0x10, %0\n\t"
53
54 /*
55 * We gobble 16 bytes forward in each loop.
56 */
57 "3:\n\t"
58 "sub $0x10, %0\n\t"
59 "mov 0*4(%1), %3\n\t"
60 "mov 1*4(%1), %4\n\t"
61 "mov %3, 0*4(%2)\n\t"
62 "mov %4, 1*4(%2)\n\t"
63 "mov 2*4(%1), %3\n\t"
64 "mov 3*4(%1), %4\n\t"
65 "mov %3, 2*4(%2)\n\t"
66 "mov %4, 3*4(%2)\n\t"
67 "lea 0x10(%1), %1\n\t"
68 "lea 0x10(%2), %2\n\t"
69 "jae 3b\n\t"
70 "add $0x10, %0\n\t"
71 "jmp 1f\n\t"
72
73 /*
74 * Handle data forward by movs.
75 */
76 ".p2align 4\n\t"
77 "4:\n\t"
78 "mov -4(%1, %0), %3\n\t"
79 "lea -4(%2, %0), %4\n\t"
80 "shr $2, %0\n\t"
81 "rep movsl\n\t"
82 "mov %3, (%4)\n\t"
83 "jmp 11f\n\t"
84 /*
85 * Handle data backward by movs.
86 */
87 ".p2align 4\n\t"
88 "6:\n\t"
89 "mov (%1), %3\n\t"
90 "mov %2, %4\n\t"
91 "lea -4(%1, %0), %1\n\t"
92 "lea -4(%2, %0), %2\n\t"
93 "shr $2, %0\n\t"
94 "std\n\t"
95 "rep movsl\n\t"
96 "mov %3,(%4)\n\t"
97 "cld\n\t"
98 "jmp 11f\n\t"
99
100 /*
101 * Start to prepare for backward copy.
102 */
103 ".p2align 4\n\t"
104 "2:\n\t"
105 "cmp $680, %0\n\t"
106 "jb 5f\n\t"
107 "mov %1, %3\n\t"
108 "xor %2, %3\n\t"
109 "and $0xff, %3\n\t"
110 "jz 6b\n\t"
111
112 /*
113 * Calculate copy position to tail.
114 */
115 "5:\n\t"
116 "add %0, %1\n\t"
117 "add %0, %2\n\t"
118 "sub $0x10, %0\n\t"
119
120 /*
121 * We gobble 16 bytes backward in each loop.
122 */
123 "7:\n\t"
124 "sub $0x10, %0\n\t"
125
126 "mov -1*4(%1), %3\n\t"
127 "mov -2*4(%1), %4\n\t"
128 "mov %3, -1*4(%2)\n\t"
129 "mov %4, -2*4(%2)\n\t"
130 "mov -3*4(%1), %3\n\t"
131 "mov -4*4(%1), %4\n\t"
132 "mov %3, -3*4(%2)\n\t"
133 "mov %4, -4*4(%2)\n\t"
134 "lea -0x10(%1), %1\n\t"
135 "lea -0x10(%2), %2\n\t"
136 "jae 7b\n\t"
137 /*
138 * Calculate copy position to head.
139 */
140 "add $0x10, %0\n\t"
141 "sub %0, %1\n\t"
142 "sub %0, %2\n\t"
143
144 /*
145 * Move data from 8 bytes to 15 bytes.
146 */
147 ".p2align 4\n\t"
148 "1:\n\t"
149 "cmp $8, %0\n\t"
150 "jb 8f\n\t"
151 "mov 0*4(%1), %3\n\t"
152 "mov 1*4(%1), %4\n\t"
153 "mov -2*4(%1, %0), %5\n\t"
154 "mov -1*4(%1, %0), %1\n\t"
155
156 "mov %3, 0*4(%2)\n\t"
157 "mov %4, 1*4(%2)\n\t"
158 "mov %5, -2*4(%2, %0)\n\t"
159 "mov %1, -1*4(%2, %0)\n\t"
160 "jmp 11f\n\t"
161
162 /*
163 * Move data from 4 bytes to 7 bytes.
164 */
165 ".p2align 4\n\t"
166 "8:\n\t"
167 "cmp $4, %0\n\t"
168 "jb 9f\n\t"
169 "mov 0*4(%1), %3\n\t"
170 "mov -1*4(%1, %0), %4\n\t"
171 "mov %3, 0*4(%2)\n\t"
172 "mov %4, -1*4(%2, %0)\n\t"
173 "jmp 11f\n\t"
174
175 /*
176 * Move data from 2 bytes to 3 bytes.
177 */
178 ".p2align 4\n\t"
179 "9:\n\t"
180 "cmp $2, %0\n\t"
181 "jb 10f\n\t"
182 "movw 0*2(%1), %%dx\n\t"
183 "movw -1*2(%1, %0), %%bx\n\t"
184 "movw %%dx, 0*2(%2)\n\t"
185 "movw %%bx, -1*2(%2, %0)\n\t"
186 "jmp 11f\n\t"
187
188 /*
189 * Move data for 1 byte.
190 */
191 ".p2align 4\n\t"
192 "10:\n\t"
193 "cmp $1, %0\n\t"
194 "jb 11f\n\t"
195 "movb (%1), %%cl\n\t"
196 "movb %%cl, (%2)\n\t"
197 ".p2align 4\n\t"
198 "11:"
199 : "=&c" (d0), "=&S" (d1), "=&D" (d2),
200 "=r" (d3),"=r" (d4), "=r"(d5)
201 :"0" (n),
202 "1" (src),
203 "2" (dest)
204 :"memory");
205
206 return ret;
207
208 }
209 EXPORT_SYMBOL(memmove);