1/*
2 * User address space access functions.
3 * The non-inlined parts of asm-cris/uaccess.h are here.
4 *
5 * Copyright (C) 2000, 2003 Axis Communications AB.
6 *
7 * Written by Hans-Peter Nilsson.
8 * Pieces used from memcpy, originally by Kenny Ranerup long time ago.
9 */
10
11#include <asm/uaccess.h>
12
13/* Asm:s have been tweaked (within the domain of correctness) to give
14   satisfactory results for "gcc version 3.2.1 Axis release R53/1.53-v32".
15
16   Check regularly...
17
18   Note that for CRISv32, the PC saved at a bus-fault is the address
19   *at* the faulting instruction, with a special case for instructions
20   in delay slots: then it's the address of the branch.  Note also that
21   in contrast to v10, a postincrement in the instruction is *not*
22   performed at a bus-fault; the register is seen having the original
23   value in fault handlers.  */
24
25
26/* Copy to userspace.  This is based on the memcpy used for
27   kernel-to-kernel copying; see "string.c".  */
28
29unsigned long __copy_user(void __user *pdst, const void *psrc, unsigned long pn)
30{
31  /* We want the parameters put in special registers.
32     Make sure the compiler is able to make something useful of this.
33     As it is now: r10 -> r13; r11 -> r11 (nop); r12 -> r12 (nop).
34
35     FIXME: Comment for old gcc version.  Check.
36     If gcc was alright, it really would need no temporaries, and no
37     stack space to save stuff on. */
38
39  register char *dst __asm__ ("r13") = pdst;
40  register const char *src __asm__ ("r11") = psrc;
41  register int n __asm__ ("r12") = pn;
42  register int retn __asm__ ("r10") = 0;
43
44
45  /* When src is aligned but not dst, this makes a few extra needless
46     cycles.  I believe it would take as many to check that the
47     re-alignment was unnecessary.  */
48  if (((unsigned long) dst & 3) != 0
49      /* Don't align if we wouldn't copy more than a few bytes; so we
50	 don't have to check further for overflows.  */
51      && n >= 3)
52  {
53    if ((unsigned long) dst & 1)
54    {
55      __asm_copy_to_user_1 (dst, src, retn);
56      n--;
57    }
58
59    if ((unsigned long) dst & 2)
60    {
61      __asm_copy_to_user_2 (dst, src, retn);
62      n -= 2;
63    }
64  }
65
66  /* Movem is dirt cheap.  The overheap is low enough to always use the
67     minimum possible block size as the threshold.  */
68  if (n >= 44)
69  {
70    /* For large copies we use 'movem'.  */
71
72    /* It is not optimal to tell the compiler about clobbering any
73       registers; that will move the saving/restoring of those registers
74       to the function prologue/epilogue, and make non-movem sizes
75       suboptimal.  */
76    __asm__ volatile ("\
77        ;; Check that the register asm declaration got right.		\n\
78        ;; The GCC manual explicitly says TRT will happen.		\n\
79	.ifnc %0%1%2%3,$r13$r11$r12$r10					\n\
80	.err								\n\
81	.endif								\n\
82									\n\
83	;; Save the registers we'll use in the movem process		\n\
84	;; on the stack.						\n\
85	subq	11*4,$sp						\n\
86	movem	$r10,[$sp]						\n\
87									\n\
88	;; Now we've got this:						\n\
89	;; r11 - src							\n\
90	;; r13 - dst							\n\
91	;; r12 - n							\n\
92									\n\
93	;; Update n for the first loop					\n\
94	subq	44,$r12							\n\
950:									\n\
96	movem	[$r11+],$r10						\n\
97	subq   44,$r12							\n\
981:	bge	0b							\n\
99	movem	$r10,[$r13+]						\n\
1003:									\n\
101	addq   44,$r12  ;; compensate for last loop underflowing n	\n\
102									\n\
103	;; Restore registers from stack					\n\
104	movem [$sp+],$r10						\n\
1052:									\n\
106	.section .fixup,\"ax\"						\n\
1074:									\n\
108; When failing on any of the 1..44 bytes in a chunk, we adjust back the	\n\
109; source pointer and just drop through	to the by-16 and by-4 loops to	\n\
110; get the correct number of failing bytes.  This necessarily means a	\n\
111; few extra exceptions, but invalid user pointers shouldn't happen in	\n\
112; time-critical code anyway.						\n\
113	jump 3b								\n\
114	subq 44,$r11							\n\
115									\n\
116	.previous							\n\
117	.section __ex_table,\"a\"					\n\
118	.dword 1b,4b							\n\
119	.previous"
120
121     /* Outputs */ : "=r" (dst), "=r" (src), "=r" (n), "=r" (retn)
122     /* Inputs */ : "0" (dst), "1" (src), "2" (n), "3" (retn));
123
124  }
125
126  while (n >= 16)
127  {
128    __asm_copy_to_user_16 (dst, src, retn);
129    n -= 16;
130  }
131
132  /* Having a separate by-four loops cuts down on cache footprint.
133     FIXME:  Test with and without; increasing switch to be 0..15.  */
134  while (n >= 4)
135  {
136    __asm_copy_to_user_4 (dst, src, retn);
137    n -= 4;
138  }
139
140  switch (n)
141  {
142    case 0:
143      break;
144    case 1:
145      __asm_copy_to_user_1 (dst, src, retn);
146      break;
147    case 2:
148      __asm_copy_to_user_2 (dst, src, retn);
149      break;
150    case 3:
151      __asm_copy_to_user_3 (dst, src, retn);
152      break;
153  }
154
155  return retn;
156}
157EXPORT_SYMBOL(__copy_user);
158
159/* Copy from user to kernel, zeroing the bytes that were inaccessible in
160   userland.  The return-value is the number of bytes that were
161   inaccessible.  */
162unsigned long __copy_user_zeroing(void *pdst, const void __user *psrc,
163				  unsigned long pn)
164{
165  /* We want the parameters put in special registers.
166     Make sure the compiler is able to make something useful of this.
167     As it is now: r10 -> r13; r11 -> r11 (nop); r12 -> r12 (nop).
168
169     FIXME: Comment for old gcc version.  Check.
170     If gcc was alright, it really would need no temporaries, and no
171     stack space to save stuff on.  */
172
173  register char *dst __asm__ ("r13") = pdst;
174  register const char *src __asm__ ("r11") = psrc;
175  register int n __asm__ ("r12") = pn;
176  register int retn __asm__ ("r10") = 0;
177
178  /* The best reason to align src is that we then know that a read-fault
179     was for aligned bytes; there's no 1..3 remaining good bytes to
180     pickle.  */
181  if (((unsigned long) src & 3) != 0)
182  {
183    if (((unsigned long) src & 1) && n != 0)
184    {
185      __asm_copy_from_user_1 (dst, src, retn);
186      n--;
187    }
188
189    if (((unsigned long) src & 2) && n >= 2)
190    {
191      __asm_copy_from_user_2 (dst, src, retn);
192      n -= 2;
193    }
194
195    /* We only need one check after the unalignment-adjustments, because
196       if both adjustments were done, either both or neither reference
197       had an exception.  */
198    if (retn != 0)
199      goto copy_exception_bytes;
200  }
201
202  /* Movem is dirt cheap.  The overheap is low enough to always use the
203     minimum possible block size as the threshold.  */
204  if (n >= 44)
205  {
206    /* It is not optimal to tell the compiler about clobbering any
207       registers; that will move the saving/restoring of those registers
208       to the function prologue/epilogue, and make non-movem sizes
209       suboptimal.  */
210    __asm__ volatile ("\
211	.ifnc %0%1%2%3,$r13$r11$r12$r10					\n\
212	.err								\n\
213	.endif								\n\
214									\n\
215	;; Save the registers we'll use in the movem process		\n\
216	;; on the stack.						\n\
217	subq	11*4,$sp						\n\
218	movem	$r10,[$sp]						\n\
219									\n\
220	;; Now we've got this:						\n\
221	;; r11 - src							\n\
222	;; r13 - dst							\n\
223	;; r12 - n							\n\
224									\n\
225	;; Update n for the first loop					\n\
226	subq	44,$r12							\n\
2270:									\n\
228	movem	[$r11+],$r10						\n\
229									\n\
230	subq   44,$r12							\n\
231	bge	0b							\n\
232	movem	$r10,[$r13+]						\n\
233									\n\
2344:									\n\
235	addq   44,$r12  ;; compensate for last loop underflowing n	\n\
236									\n\
237	;; Restore registers from stack					\n\
238	movem [$sp+],$r10						\n\
239	.section .fixup,\"ax\"						\n\
240									\n\
241;; Do not jump back into the loop if we fail.  For some uses, we get a	\n\
242;; page fault somewhere on the line.  Without checking for page limits,	\n\
243;; we don't know where, but we need to copy accurately and keep an	\n\
244;; accurate count; not just clear the whole line.  To do that, we fall	\n\
245;; down in the code below, proceeding with smaller amounts.  It should	\n\
246;; be kept in mind that we have to cater to code like what at one time	\n\
247;; was in fs/super.c:							\n\
248;;  i = size - copy_from_user((void *)page, data, size);		\n\
249;; which would cause repeated faults while clearing the remainder of	\n\
250;; the SIZE bytes at PAGE after the first fault.			\n\
251;; A caveat here is that we must not fall through from a failing page	\n\
252;; to a valid page.							\n\
253									\n\
2543:									\n\
255	jump	4b ;; Fall through, pretending the fault didn't happen.	\n\
256	nop								\n\
257									\n\
258	.previous							\n\
259	.section __ex_table,\"a\"					\n\
260	.dword 0b,3b							\n\
261	.previous"
262
263     /* Outputs */ : "=r" (dst), "=r" (src), "=r" (n), "=r" (retn)
264     /* Inputs */ : "0" (dst), "1" (src), "2" (n), "3" (retn));
265  }
266
267  /* Either we directly start copying here, using dword copying in a loop,
268     or we copy as much as possible with 'movem' and then the last block
269     (<44 bytes) is copied here.  This will work since 'movem' will have
270     updated src, dst and n.  (Except with failing src.)
271
272     Since we want to keep src accurate, we can't use
273     __asm_copy_from_user_N with N != (1, 2, 4); it updates dst and
274     retn, but not src (by design; it's value is ignored elsewhere).  */
275
276  while (n >= 4)
277  {
278    __asm_copy_from_user_4 (dst, src, retn);
279    n -= 4;
280
281    if (retn)
282      goto copy_exception_bytes;
283  }
284
285  /* If we get here, there were no memory read faults.  */
286  switch (n)
287  {
288    /* These copies are at least "naturally aligned" (so we don't have
289       to check each byte), due to the src alignment code before the
290       movem loop.  The *_3 case *will* get the correct count for retn.  */
291    case 0:
292      /* This case deliberately left in (if you have doubts check the
293	 generated assembly code).  */
294      break;
295    case 1:
296      __asm_copy_from_user_1 (dst, src, retn);
297      break;
298    case 2:
299      __asm_copy_from_user_2 (dst, src, retn);
300      break;
301    case 3:
302      __asm_copy_from_user_3 (dst, src, retn);
303      break;
304  }
305
306  /* If we get here, retn correctly reflects the number of failing
307     bytes.  */
308  return retn;
309
310copy_exception_bytes:
311  /* We already have "retn" bytes cleared, and need to clear the
312     remaining "n" bytes.  A non-optimized simple byte-for-byte in-line
313     memset is preferred here, since this isn't speed-critical code and
314     we'd rather have this a leaf-function than calling memset.  */
315  {
316    char *endp;
317    for (endp = dst + n; dst < endp; dst++)
318      *dst = 0;
319  }
320
321  return retn + n;
322}
323EXPORT_SYMBOL(__copy_user_zeroing);
324
325/* Zero userspace.  */
326unsigned long __do_clear_user(void __user *pto, unsigned long pn)
327{
328  /* We want the parameters put in special registers.
329     Make sure the compiler is able to make something useful of this.
330      As it is now: r10 -> r13; r11 -> r11 (nop); r12 -> r12 (nop).
331
332     FIXME: Comment for old gcc version.  Check.
333     If gcc was alright, it really would need no temporaries, and no
334     stack space to save stuff on. */
335
336  register char *dst __asm__ ("r13") = pto;
337  register int n __asm__ ("r12") = pn;
338  register int retn __asm__ ("r10") = 0;
339
340
341  if (((unsigned long) dst & 3) != 0
342     /* Don't align if we wouldn't copy more than a few bytes.  */
343      && n >= 3)
344  {
345    if ((unsigned long) dst & 1)
346    {
347      __asm_clear_1 (dst, retn);
348      n--;
349    }
350
351    if ((unsigned long) dst & 2)
352    {
353      __asm_clear_2 (dst, retn);
354      n -= 2;
355    }
356  }
357
358  /* Decide which copying method to use.
359     FIXME: This number is from the "ordinary" kernel memset.  */
360  if (n >= 48)
361  {
362    /* For large clears we use 'movem' */
363
364    /* It is not optimal to tell the compiler about clobbering any
365       call-saved registers; that will move the saving/restoring of
366       those registers to the function prologue/epilogue, and make
367       non-movem sizes suboptimal.
368
369       This method is not foolproof; it assumes that the "asm reg"
370       declarations at the beginning of the function really are used
371       here (beware: they may be moved to temporary registers).
372       This way, we do not have to save/move the registers around into
373       temporaries; we can safely use them straight away.
374
375      If you want to check that the allocation was right; then
376      check the equalities in the first comment.  It should say
377      something like "r13=r13, r11=r11, r12=r12". */
378    __asm__ volatile ("\
379	.ifnc %0%1%2,$r13$r12$r10					\n\
380	.err								\n\
381	.endif								\n\
382									\n\
383	;; Save the registers we'll clobber in the movem process	\n\
384	;; on the stack.  Don't mention them to gcc, it will only be	\n\
385	;; upset.							\n\
386	subq	11*4,$sp						\n\
387	movem	$r10,[$sp]						\n\
388									\n\
389	clear.d $r0							\n\
390	clear.d $r1							\n\
391	clear.d $r2							\n\
392	clear.d $r3							\n\
393	clear.d $r4							\n\
394	clear.d $r5							\n\
395	clear.d $r6							\n\
396	clear.d $r7							\n\
397	clear.d $r8							\n\
398	clear.d $r9							\n\
399	clear.d $r10							\n\
400	clear.d $r11							\n\
401									\n\
402	;; Now we've got this:						\n\
403	;; r13 - dst							\n\
404	;; r12 - n							\n\
405									\n\
406	;; Update n for the first loop					\n\
407	subq	12*4,$r12						\n\
4080:									\n\
409	subq   12*4,$r12						\n\
4101:									\n\
411	bge	0b							\n\
412	movem	$r11,[$r13+]						\n\
413									\n\
414	addq   12*4,$r12 ;; compensate for last loop underflowing n	\n\
415									\n\
416	;; Restore registers from stack					\n\
417	movem [$sp+],$r10						\n\
4182:									\n\
419	.section .fixup,\"ax\"						\n\
4203:									\n\
421	movem [$sp],$r10						\n\
422	addq 12*4,$r10							\n\
423	addq 12*4,$r13							\n\
424	movem $r10,[$sp]						\n\
425	jump 0b								\n\
426	clear.d $r10							\n\
427									\n\
428	.previous							\n\
429	.section __ex_table,\"a\"					\n\
430	.dword 1b,3b							\n\
431	.previous"
432
433     /* Outputs */ : "=r" (dst), "=r" (n), "=r" (retn)
434     /* Inputs */ : "0" (dst), "1" (n), "2" (retn)
435     /* Clobber */ : "r11");
436  }
437
438  while (n >= 16)
439  {
440    __asm_clear_16 (dst, retn);
441    n -= 16;
442  }
443
444  /* Having a separate by-four loops cuts down on cache footprint.
445     FIXME:  Test with and without; increasing switch to be 0..15.  */
446  while (n >= 4)
447  {
448    __asm_clear_4 (dst, retn);
449    n -= 4;
450  }
451
452  switch (n)
453  {
454    case 0:
455      break;
456    case 1:
457      __asm_clear_1 (dst, retn);
458      break;
459    case 2:
460      __asm_clear_2 (dst, retn);
461      break;
462    case 3:
463      __asm_clear_3 (dst, retn);
464      break;
465  }
466
467  return retn;
468}
469EXPORT_SYMBOL(__do_clear_user);
470