1/*
2 * INET		An implementation of the TCP/IP protocol suite for the LINUX
3 *		operating system.  INET is implemented using the  BSD Socket
4 *		interface as the means of communication with the user level.
5 *
6 *		IP/TCP/UDP checksumming routines
7 *
8 * Authors:	Jorge Cwik, <jorge@laser.satlink.net>
9 *		Arnt Gulbrandsen, <agulbra@nvg.unit.no>
10 *		Tom May, <ftom@netcom.com>
11 *              Pentium Pro/II routines:
12 *              Alexander Kjeldaas <astor@guardian.no>
13 *              Finn Arne Gangstad <finnag@guardian.no>
14 *		Lots of code moved from tcp.c and ip.c; see those files
15 *		for more names.
16 *
17 * Changes:     Ingo Molnar, converted csum_partial_copy() to 2.1 exception
18 *			     handling.
19 *		Andi Kleen,  add zeroing on error
20 *                   converted to pure assembler
21 *
22 *		This program is free software; you can redistribute it and/or
23 *		modify it under the terms of the GNU General Public License
24 *		as published by the Free Software Foundation; either version
25 *		2 of the License, or (at your option) any later version.
26 */
27
28#include <linux/linkage.h>
29#include <asm/dwarf2.h>
30#include <asm/errno.h>
31#include <asm/asm.h>
32
33/*
34 * computes a partial checksum, e.g. for TCP/UDP fragments
35 */
36
37/*
38unsigned int csum_partial(const unsigned char * buff, int len, unsigned int sum)
39 */
40
41.text
42
43#ifndef CONFIG_X86_USE_PPRO_CHECKSUM
44
45	  /*
46	   * Experiments with Ethernet and SLIP connections show that buff
47	   * is aligned on either a 2-byte or 4-byte boundary.  We get at
48	   * least a twofold speedup on 486 and Pentium if it is 4-byte aligned.
49	   * Fortunately, it is easy to convert 2-byte alignment to 4-byte
50	   * alignment for the unrolled loop.
51	   */
52ENTRY(csum_partial)
53	CFI_STARTPROC
54	pushl_cfi_reg esi
55	pushl_cfi_reg ebx
56	movl 20(%esp),%eax	# Function arg: unsigned int sum
57	movl 16(%esp),%ecx	# Function arg: int len
58	movl 12(%esp),%esi	# Function arg: unsigned char *buff
59	testl $3, %esi		# Check alignment.
60	jz 2f			# Jump if alignment is ok.
61	testl $1, %esi		# Check alignment.
62	jz 10f			# Jump if alignment is boundary of 2 bytes.
63
64	# buf is odd
65	dec %ecx
66	jl 8f
67	movzbl (%esi), %ebx
68	adcl %ebx, %eax
69	roll $8, %eax
70	inc %esi
71	testl $2, %esi
72	jz 2f
7310:
74	subl $2, %ecx		# Alignment uses up two bytes.
75	jae 1f			# Jump if we had at least two bytes.
76	addl $2, %ecx		# ecx was < 2.  Deal with it.
77	jmp 4f
781:	movw (%esi), %bx
79	addl $2, %esi
80	addw %bx, %ax
81	adcl $0, %eax
822:
83	movl %ecx, %edx
84	shrl $5, %ecx
85	jz 2f
86	testl %esi, %esi
871:	movl (%esi), %ebx
88	adcl %ebx, %eax
89	movl 4(%esi), %ebx
90	adcl %ebx, %eax
91	movl 8(%esi), %ebx
92	adcl %ebx, %eax
93	movl 12(%esi), %ebx
94	adcl %ebx, %eax
95	movl 16(%esi), %ebx
96	adcl %ebx, %eax
97	movl 20(%esi), %ebx
98	adcl %ebx, %eax
99	movl 24(%esi), %ebx
100	adcl %ebx, %eax
101	movl 28(%esi), %ebx
102	adcl %ebx, %eax
103	lea 32(%esi), %esi
104	dec %ecx
105	jne 1b
106	adcl $0, %eax
1072:	movl %edx, %ecx
108	andl $0x1c, %edx
109	je 4f
110	shrl $2, %edx		# This clears CF
1113:	adcl (%esi), %eax
112	lea 4(%esi), %esi
113	dec %edx
114	jne 3b
115	adcl $0, %eax
1164:	andl $3, %ecx
117	jz 7f
118	cmpl $2, %ecx
119	jb 5f
120	movw (%esi),%cx
121	leal 2(%esi),%esi
122	je 6f
123	shll $16,%ecx
1245:	movb (%esi),%cl
1256:	addl %ecx,%eax
126	adcl $0, %eax
1277:
128	testb $1, 12(%esp)
129	jz 8f
130	roll $8, %eax
1318:
132	popl_cfi_reg ebx
133	popl_cfi_reg esi
134	ret
135	CFI_ENDPROC
136ENDPROC(csum_partial)
137
138#else
139
140/* Version for PentiumII/PPro */
141
142ENTRY(csum_partial)
143	CFI_STARTPROC
144	pushl_cfi_reg esi
145	pushl_cfi_reg ebx
146	movl 20(%esp),%eax	# Function arg: unsigned int sum
147	movl 16(%esp),%ecx	# Function arg: int len
148	movl 12(%esp),%esi	# Function arg:	const unsigned char *buf
149
150	testl $3, %esi
151	jnz 25f
15210:
153	movl %ecx, %edx
154	movl %ecx, %ebx
155	andl $0x7c, %ebx
156	shrl $7, %ecx
157	addl %ebx,%esi
158	shrl $2, %ebx
159	negl %ebx
160	lea 45f(%ebx,%ebx,2), %ebx
161	testl %esi, %esi
162	jmp *%ebx
163
164	# Handle 2-byte-aligned regions
16520:	addw (%esi), %ax
166	lea 2(%esi), %esi
167	adcl $0, %eax
168	jmp 10b
16925:
170	testl $1, %esi
171	jz 30f
172	# buf is odd
173	dec %ecx
174	jl 90f
175	movzbl (%esi), %ebx
176	addl %ebx, %eax
177	adcl $0, %eax
178	roll $8, %eax
179	inc %esi
180	testl $2, %esi
181	jz 10b
182
18330:	subl $2, %ecx
184	ja 20b
185	je 32f
186	addl $2, %ecx
187	jz 80f
188	movzbl (%esi),%ebx	# csumming 1 byte, 2-aligned
189	addl %ebx, %eax
190	adcl $0, %eax
191	jmp 80f
19232:
193	addw (%esi), %ax	# csumming 2 bytes, 2-aligned
194	adcl $0, %eax
195	jmp 80f
196
19740:
198	addl -128(%esi), %eax
199	adcl -124(%esi), %eax
200	adcl -120(%esi), %eax
201	adcl -116(%esi), %eax
202	adcl -112(%esi), %eax
203	adcl -108(%esi), %eax
204	adcl -104(%esi), %eax
205	adcl -100(%esi), %eax
206	adcl -96(%esi), %eax
207	adcl -92(%esi), %eax
208	adcl -88(%esi), %eax
209	adcl -84(%esi), %eax
210	adcl -80(%esi), %eax
211	adcl -76(%esi), %eax
212	adcl -72(%esi), %eax
213	adcl -68(%esi), %eax
214	adcl -64(%esi), %eax
215	adcl -60(%esi), %eax
216	adcl -56(%esi), %eax
217	adcl -52(%esi), %eax
218	adcl -48(%esi), %eax
219	adcl -44(%esi), %eax
220	adcl -40(%esi), %eax
221	adcl -36(%esi), %eax
222	adcl -32(%esi), %eax
223	adcl -28(%esi), %eax
224	adcl -24(%esi), %eax
225	adcl -20(%esi), %eax
226	adcl -16(%esi), %eax
227	adcl -12(%esi), %eax
228	adcl -8(%esi), %eax
229	adcl -4(%esi), %eax
23045:
231	lea 128(%esi), %esi
232	adcl $0, %eax
233	dec %ecx
234	jge 40b
235	movl %edx, %ecx
23650:	andl $3, %ecx
237	jz 80f
238
239	# Handle the last 1-3 bytes without jumping
240	notl %ecx		# 1->2, 2->1, 3->0, higher bits are masked
241	movl $0xffffff,%ebx	# by the shll and shrl instructions
242	shll $3,%ecx
243	shrl %cl,%ebx
244	andl -128(%esi),%ebx	# esi is 4-aligned so should be ok
245	addl %ebx,%eax
246	adcl $0,%eax
24780:
248	testb $1, 12(%esp)
249	jz 90f
250	roll $8, %eax
25190:
252	popl_cfi_reg ebx
253	popl_cfi_reg esi
254	ret
255	CFI_ENDPROC
256ENDPROC(csum_partial)
257
258#endif
259
260/*
261unsigned int csum_partial_copy_generic (const char *src, char *dst,
262				  int len, int sum, int *src_err_ptr, int *dst_err_ptr)
263 */
264
265/*
266 * Copy from ds while checksumming, otherwise like csum_partial
267 *
268 * The macros SRC and DST specify the type of access for the instruction.
269 * thus we can call a custom exception handler for all access types.
270 *
271 * FIXME: could someone double-check whether I haven't mixed up some SRC and
272 *	  DST definitions? It's damn hard to trigger all cases.  I hope I got
273 *	  them all but there's no guarantee.
274 */
275
276#define SRC(y...)			\
277	9999: y;			\
278	_ASM_EXTABLE(9999b, 6001f)
279
280#define DST(y...)			\
281	9999: y;			\
282	_ASM_EXTABLE(9999b, 6002f)
283
284#ifndef CONFIG_X86_USE_PPRO_CHECKSUM
285
286#define ARGBASE 16
287#define FP		12
288
289ENTRY(csum_partial_copy_generic)
290	CFI_STARTPROC
291	subl  $4,%esp
292	CFI_ADJUST_CFA_OFFSET 4
293	pushl_cfi_reg edi
294	pushl_cfi_reg esi
295	pushl_cfi_reg ebx
296	movl ARGBASE+16(%esp),%eax	# sum
297	movl ARGBASE+12(%esp),%ecx	# len
298	movl ARGBASE+4(%esp),%esi	# src
299	movl ARGBASE+8(%esp),%edi	# dst
300
301	testl $2, %edi			# Check alignment.
302	jz 2f				# Jump if alignment is ok.
303	subl $2, %ecx			# Alignment uses up two bytes.
304	jae 1f				# Jump if we had at least two bytes.
305	addl $2, %ecx			# ecx was < 2.  Deal with it.
306	jmp 4f
307SRC(1:	movw (%esi), %bx	)
308	addl $2, %esi
309DST(	movw %bx, (%edi)	)
310	addl $2, %edi
311	addw %bx, %ax
312	adcl $0, %eax
3132:
314	movl %ecx, FP(%esp)
315	shrl $5, %ecx
316	jz 2f
317	testl %esi, %esi
318SRC(1:	movl (%esi), %ebx	)
319SRC(	movl 4(%esi), %edx	)
320	adcl %ebx, %eax
321DST(	movl %ebx, (%edi)	)
322	adcl %edx, %eax
323DST(	movl %edx, 4(%edi)	)
324
325SRC(	movl 8(%esi), %ebx	)
326SRC(	movl 12(%esi), %edx	)
327	adcl %ebx, %eax
328DST(	movl %ebx, 8(%edi)	)
329	adcl %edx, %eax
330DST(	movl %edx, 12(%edi)	)
331
332SRC(	movl 16(%esi), %ebx 	)
333SRC(	movl 20(%esi), %edx	)
334	adcl %ebx, %eax
335DST(	movl %ebx, 16(%edi)	)
336	adcl %edx, %eax
337DST(	movl %edx, 20(%edi)	)
338
339SRC(	movl 24(%esi), %ebx	)
340SRC(	movl 28(%esi), %edx	)
341	adcl %ebx, %eax
342DST(	movl %ebx, 24(%edi)	)
343	adcl %edx, %eax
344DST(	movl %edx, 28(%edi)	)
345
346	lea 32(%esi), %esi
347	lea 32(%edi), %edi
348	dec %ecx
349	jne 1b
350	adcl $0, %eax
3512:	movl FP(%esp), %edx
352	movl %edx, %ecx
353	andl $0x1c, %edx
354	je 4f
355	shrl $2, %edx			# This clears CF
356SRC(3:	movl (%esi), %ebx	)
357	adcl %ebx, %eax
358DST(	movl %ebx, (%edi)	)
359	lea 4(%esi), %esi
360	lea 4(%edi), %edi
361	dec %edx
362	jne 3b
363	adcl $0, %eax
3644:	andl $3, %ecx
365	jz 7f
366	cmpl $2, %ecx
367	jb 5f
368SRC(	movw (%esi), %cx	)
369	leal 2(%esi), %esi
370DST(	movw %cx, (%edi)	)
371	leal 2(%edi), %edi
372	je 6f
373	shll $16,%ecx
374SRC(5:	movb (%esi), %cl	)
375DST(	movb %cl, (%edi)	)
3766:	addl %ecx, %eax
377	adcl $0, %eax
3787:
3795000:
380
381# Exception handler:
382.section .fixup, "ax"
383
3846001:
385	movl ARGBASE+20(%esp), %ebx	# src_err_ptr
386	movl $-EFAULT, (%ebx)
387
388	# zero the complete destination - computing the rest
389	# is too much work
390	movl ARGBASE+8(%esp), %edi	# dst
391	movl ARGBASE+12(%esp), %ecx	# len
392	xorl %eax,%eax
393	rep ; stosb
394
395	jmp 5000b
396
3976002:
398	movl ARGBASE+24(%esp), %ebx	# dst_err_ptr
399	movl $-EFAULT,(%ebx)
400	jmp 5000b
401
402.previous
403
404	popl_cfi_reg ebx
405	popl_cfi_reg esi
406	popl_cfi_reg edi
407	popl_cfi %ecx			# equivalent to addl $4,%esp
408	ret
409	CFI_ENDPROC
410ENDPROC(csum_partial_copy_generic)
411
412#else
413
414/* Version for PentiumII/PPro */
415
416#define ROUND1(x) \
417	SRC(movl x(%esi), %ebx	)	;	\
418	addl %ebx, %eax			;	\
419	DST(movl %ebx, x(%edi)	)	;
420
421#define ROUND(x) \
422	SRC(movl x(%esi), %ebx	)	;	\
423	adcl %ebx, %eax			;	\
424	DST(movl %ebx, x(%edi)	)	;
425
426#define ARGBASE 12
427
428ENTRY(csum_partial_copy_generic)
429	CFI_STARTPROC
430	pushl_cfi_reg ebx
431	pushl_cfi_reg edi
432	pushl_cfi_reg esi
433	movl ARGBASE+4(%esp),%esi	#src
434	movl ARGBASE+8(%esp),%edi	#dst
435	movl ARGBASE+12(%esp),%ecx	#len
436	movl ARGBASE+16(%esp),%eax	#sum
437#	movl %ecx, %edx
438	movl %ecx, %ebx
439	movl %esi, %edx
440	shrl $6, %ecx
441	andl $0x3c, %ebx
442	negl %ebx
443	subl %ebx, %esi
444	subl %ebx, %edi
445	lea  -1(%esi),%edx
446	andl $-32,%edx
447	lea 3f(%ebx,%ebx), %ebx
448	testl %esi, %esi
449	jmp *%ebx
4501:	addl $64,%esi
451	addl $64,%edi
452	SRC(movb -32(%edx),%bl)	; SRC(movb (%edx),%bl)
453	ROUND1(-64) ROUND(-60) ROUND(-56) ROUND(-52)
454	ROUND (-48) ROUND(-44) ROUND(-40) ROUND(-36)
455	ROUND (-32) ROUND(-28) ROUND(-24) ROUND(-20)
456	ROUND (-16) ROUND(-12) ROUND(-8)  ROUND(-4)
4573:	adcl $0,%eax
458	addl $64, %edx
459	dec %ecx
460	jge 1b
4614:	movl ARGBASE+12(%esp),%edx	#len
462	andl $3, %edx
463	jz 7f
464	cmpl $2, %edx
465	jb 5f
466SRC(	movw (%esi), %dx         )
467	leal 2(%esi), %esi
468DST(	movw %dx, (%edi)         )
469	leal 2(%edi), %edi
470	je 6f
471	shll $16,%edx
4725:
473SRC(	movb (%esi), %dl         )
474DST(	movb %dl, (%edi)         )
4756:	addl %edx, %eax
476	adcl $0, %eax
4777:
478.section .fixup, "ax"
4796001:	movl	ARGBASE+20(%esp), %ebx	# src_err_ptr
480	movl $-EFAULT, (%ebx)
481	# zero the complete destination (computing the rest is too much work)
482	movl ARGBASE+8(%esp),%edi	# dst
483	movl ARGBASE+12(%esp),%ecx	# len
484	xorl %eax,%eax
485	rep; stosb
486	jmp 7b
4876002:	movl ARGBASE+24(%esp), %ebx	# dst_err_ptr
488	movl $-EFAULT, (%ebx)
489	jmp  7b
490.previous
491
492	popl_cfi_reg esi
493	popl_cfi_reg edi
494	popl_cfi_reg ebx
495	ret
496	CFI_ENDPROC
497ENDPROC(csum_partial_copy_generic)
498
499#undef ROUND
500#undef ROUND1
501
502#endif
503