1/* 2 * INET An implementation of the TCP/IP protocol suite for the LINUX 3 * operating system. INET is implemented using the BSD Socket 4 * interface as the means of communication with the user level. 5 * 6 * IP/TCP/UDP checksumming routines 7 * 8 * Authors: Jorge Cwik, <jorge@laser.satlink.net> 9 * Arnt Gulbrandsen, <agulbra@nvg.unit.no> 10 * Tom May, <ftom@netcom.com> 11 * Pentium Pro/II routines: 12 * Alexander Kjeldaas <astor@guardian.no> 13 * Finn Arne Gangstad <finnag@guardian.no> 14 * Lots of code moved from tcp.c and ip.c; see those files 15 * for more names. 16 * 17 * Changes: Ingo Molnar, converted csum_partial_copy() to 2.1 exception 18 * handling. 19 * Andi Kleen, add zeroing on error 20 * converted to pure assembler 21 * 22 * This program is free software; you can redistribute it and/or 23 * modify it under the terms of the GNU General Public License 24 * as published by the Free Software Foundation; either version 25 * 2 of the License, or (at your option) any later version. 26 */ 27 28#include <linux/linkage.h> 29#include <asm/dwarf2.h> 30#include <asm/errno.h> 31#include <asm/asm.h> 32 33/* 34 * computes a partial checksum, e.g. for TCP/UDP fragments 35 */ 36 37/* 38unsigned int csum_partial(const unsigned char * buff, int len, unsigned int sum) 39 */ 40 41.text 42 43#ifndef CONFIG_X86_USE_PPRO_CHECKSUM 44 45 /* 46 * Experiments with Ethernet and SLIP connections show that buff 47 * is aligned on either a 2-byte or 4-byte boundary. We get at 48 * least a twofold speedup on 486 and Pentium if it is 4-byte aligned. 49 * Fortunately, it is easy to convert 2-byte alignment to 4-byte 50 * alignment for the unrolled loop. 51 */ 52ENTRY(csum_partial) 53 CFI_STARTPROC 54 pushl_cfi_reg esi 55 pushl_cfi_reg ebx 56 movl 20(%esp),%eax # Function arg: unsigned int sum 57 movl 16(%esp),%ecx # Function arg: int len 58 movl 12(%esp),%esi # Function arg: unsigned char *buff 59 testl $3, %esi # Check alignment. 60 jz 2f # Jump if alignment is ok. 61 testl $1, %esi # Check alignment. 62 jz 10f # Jump if alignment is boundary of 2 bytes. 63 64 # buf is odd 65 dec %ecx 66 jl 8f 67 movzbl (%esi), %ebx 68 adcl %ebx, %eax 69 roll $8, %eax 70 inc %esi 71 testl $2, %esi 72 jz 2f 7310: 74 subl $2, %ecx # Alignment uses up two bytes. 75 jae 1f # Jump if we had at least two bytes. 76 addl $2, %ecx # ecx was < 2. Deal with it. 77 jmp 4f 781: movw (%esi), %bx 79 addl $2, %esi 80 addw %bx, %ax 81 adcl $0, %eax 822: 83 movl %ecx, %edx 84 shrl $5, %ecx 85 jz 2f 86 testl %esi, %esi 871: movl (%esi), %ebx 88 adcl %ebx, %eax 89 movl 4(%esi), %ebx 90 adcl %ebx, %eax 91 movl 8(%esi), %ebx 92 adcl %ebx, %eax 93 movl 12(%esi), %ebx 94 adcl %ebx, %eax 95 movl 16(%esi), %ebx 96 adcl %ebx, %eax 97 movl 20(%esi), %ebx 98 adcl %ebx, %eax 99 movl 24(%esi), %ebx 100 adcl %ebx, %eax 101 movl 28(%esi), %ebx 102 adcl %ebx, %eax 103 lea 32(%esi), %esi 104 dec %ecx 105 jne 1b 106 adcl $0, %eax 1072: movl %edx, %ecx 108 andl $0x1c, %edx 109 je 4f 110 shrl $2, %edx # This clears CF 1113: adcl (%esi), %eax 112 lea 4(%esi), %esi 113 dec %edx 114 jne 3b 115 adcl $0, %eax 1164: andl $3, %ecx 117 jz 7f 118 cmpl $2, %ecx 119 jb 5f 120 movw (%esi),%cx 121 leal 2(%esi),%esi 122 je 6f 123 shll $16,%ecx 1245: movb (%esi),%cl 1256: addl %ecx,%eax 126 adcl $0, %eax 1277: 128 testb $1, 12(%esp) 129 jz 8f 130 roll $8, %eax 1318: 132 popl_cfi_reg ebx 133 popl_cfi_reg esi 134 ret 135 CFI_ENDPROC 136ENDPROC(csum_partial) 137 138#else 139 140/* Version for PentiumII/PPro */ 141 142ENTRY(csum_partial) 143 CFI_STARTPROC 144 pushl_cfi_reg esi 145 pushl_cfi_reg ebx 146 movl 20(%esp),%eax # Function arg: unsigned int sum 147 movl 16(%esp),%ecx # Function arg: int len 148 movl 12(%esp),%esi # Function arg: const unsigned char *buf 149 150 testl $3, %esi 151 jnz 25f 15210: 153 movl %ecx, %edx 154 movl %ecx, %ebx 155 andl $0x7c, %ebx 156 shrl $7, %ecx 157 addl %ebx,%esi 158 shrl $2, %ebx 159 negl %ebx 160 lea 45f(%ebx,%ebx,2), %ebx 161 testl %esi, %esi 162 jmp *%ebx 163 164 # Handle 2-byte-aligned regions 16520: addw (%esi), %ax 166 lea 2(%esi), %esi 167 adcl $0, %eax 168 jmp 10b 16925: 170 testl $1, %esi 171 jz 30f 172 # buf is odd 173 dec %ecx 174 jl 90f 175 movzbl (%esi), %ebx 176 addl %ebx, %eax 177 adcl $0, %eax 178 roll $8, %eax 179 inc %esi 180 testl $2, %esi 181 jz 10b 182 18330: subl $2, %ecx 184 ja 20b 185 je 32f 186 addl $2, %ecx 187 jz 80f 188 movzbl (%esi),%ebx # csumming 1 byte, 2-aligned 189 addl %ebx, %eax 190 adcl $0, %eax 191 jmp 80f 19232: 193 addw (%esi), %ax # csumming 2 bytes, 2-aligned 194 adcl $0, %eax 195 jmp 80f 196 19740: 198 addl -128(%esi), %eax 199 adcl -124(%esi), %eax 200 adcl -120(%esi), %eax 201 adcl -116(%esi), %eax 202 adcl -112(%esi), %eax 203 adcl -108(%esi), %eax 204 adcl -104(%esi), %eax 205 adcl -100(%esi), %eax 206 adcl -96(%esi), %eax 207 adcl -92(%esi), %eax 208 adcl -88(%esi), %eax 209 adcl -84(%esi), %eax 210 adcl -80(%esi), %eax 211 adcl -76(%esi), %eax 212 adcl -72(%esi), %eax 213 adcl -68(%esi), %eax 214 adcl -64(%esi), %eax 215 adcl -60(%esi), %eax 216 adcl -56(%esi), %eax 217 adcl -52(%esi), %eax 218 adcl -48(%esi), %eax 219 adcl -44(%esi), %eax 220 adcl -40(%esi), %eax 221 adcl -36(%esi), %eax 222 adcl -32(%esi), %eax 223 adcl -28(%esi), %eax 224 adcl -24(%esi), %eax 225 adcl -20(%esi), %eax 226 adcl -16(%esi), %eax 227 adcl -12(%esi), %eax 228 adcl -8(%esi), %eax 229 adcl -4(%esi), %eax 23045: 231 lea 128(%esi), %esi 232 adcl $0, %eax 233 dec %ecx 234 jge 40b 235 movl %edx, %ecx 23650: andl $3, %ecx 237 jz 80f 238 239 # Handle the last 1-3 bytes without jumping 240 notl %ecx # 1->2, 2->1, 3->0, higher bits are masked 241 movl $0xffffff,%ebx # by the shll and shrl instructions 242 shll $3,%ecx 243 shrl %cl,%ebx 244 andl -128(%esi),%ebx # esi is 4-aligned so should be ok 245 addl %ebx,%eax 246 adcl $0,%eax 24780: 248 testb $1, 12(%esp) 249 jz 90f 250 roll $8, %eax 25190: 252 popl_cfi_reg ebx 253 popl_cfi_reg esi 254 ret 255 CFI_ENDPROC 256ENDPROC(csum_partial) 257 258#endif 259 260/* 261unsigned int csum_partial_copy_generic (const char *src, char *dst, 262 int len, int sum, int *src_err_ptr, int *dst_err_ptr) 263 */ 264 265/* 266 * Copy from ds while checksumming, otherwise like csum_partial 267 * 268 * The macros SRC and DST specify the type of access for the instruction. 269 * thus we can call a custom exception handler for all access types. 270 * 271 * FIXME: could someone double-check whether I haven't mixed up some SRC and 272 * DST definitions? It's damn hard to trigger all cases. I hope I got 273 * them all but there's no guarantee. 274 */ 275 276#define SRC(y...) \ 277 9999: y; \ 278 _ASM_EXTABLE(9999b, 6001f) 279 280#define DST(y...) \ 281 9999: y; \ 282 _ASM_EXTABLE(9999b, 6002f) 283 284#ifndef CONFIG_X86_USE_PPRO_CHECKSUM 285 286#define ARGBASE 16 287#define FP 12 288 289ENTRY(csum_partial_copy_generic) 290 CFI_STARTPROC 291 subl $4,%esp 292 CFI_ADJUST_CFA_OFFSET 4 293 pushl_cfi_reg edi 294 pushl_cfi_reg esi 295 pushl_cfi_reg ebx 296 movl ARGBASE+16(%esp),%eax # sum 297 movl ARGBASE+12(%esp),%ecx # len 298 movl ARGBASE+4(%esp),%esi # src 299 movl ARGBASE+8(%esp),%edi # dst 300 301 testl $2, %edi # Check alignment. 302 jz 2f # Jump if alignment is ok. 303 subl $2, %ecx # Alignment uses up two bytes. 304 jae 1f # Jump if we had at least two bytes. 305 addl $2, %ecx # ecx was < 2. Deal with it. 306 jmp 4f 307SRC(1: movw (%esi), %bx ) 308 addl $2, %esi 309DST( movw %bx, (%edi) ) 310 addl $2, %edi 311 addw %bx, %ax 312 adcl $0, %eax 3132: 314 movl %ecx, FP(%esp) 315 shrl $5, %ecx 316 jz 2f 317 testl %esi, %esi 318SRC(1: movl (%esi), %ebx ) 319SRC( movl 4(%esi), %edx ) 320 adcl %ebx, %eax 321DST( movl %ebx, (%edi) ) 322 adcl %edx, %eax 323DST( movl %edx, 4(%edi) ) 324 325SRC( movl 8(%esi), %ebx ) 326SRC( movl 12(%esi), %edx ) 327 adcl %ebx, %eax 328DST( movl %ebx, 8(%edi) ) 329 adcl %edx, %eax 330DST( movl %edx, 12(%edi) ) 331 332SRC( movl 16(%esi), %ebx ) 333SRC( movl 20(%esi), %edx ) 334 adcl %ebx, %eax 335DST( movl %ebx, 16(%edi) ) 336 adcl %edx, %eax 337DST( movl %edx, 20(%edi) ) 338 339SRC( movl 24(%esi), %ebx ) 340SRC( movl 28(%esi), %edx ) 341 adcl %ebx, %eax 342DST( movl %ebx, 24(%edi) ) 343 adcl %edx, %eax 344DST( movl %edx, 28(%edi) ) 345 346 lea 32(%esi), %esi 347 lea 32(%edi), %edi 348 dec %ecx 349 jne 1b 350 adcl $0, %eax 3512: movl FP(%esp), %edx 352 movl %edx, %ecx 353 andl $0x1c, %edx 354 je 4f 355 shrl $2, %edx # This clears CF 356SRC(3: movl (%esi), %ebx ) 357 adcl %ebx, %eax 358DST( movl %ebx, (%edi) ) 359 lea 4(%esi), %esi 360 lea 4(%edi), %edi 361 dec %edx 362 jne 3b 363 adcl $0, %eax 3644: andl $3, %ecx 365 jz 7f 366 cmpl $2, %ecx 367 jb 5f 368SRC( movw (%esi), %cx ) 369 leal 2(%esi), %esi 370DST( movw %cx, (%edi) ) 371 leal 2(%edi), %edi 372 je 6f 373 shll $16,%ecx 374SRC(5: movb (%esi), %cl ) 375DST( movb %cl, (%edi) ) 3766: addl %ecx, %eax 377 adcl $0, %eax 3787: 3795000: 380 381# Exception handler: 382.section .fixup, "ax" 383 3846001: 385 movl ARGBASE+20(%esp), %ebx # src_err_ptr 386 movl $-EFAULT, (%ebx) 387 388 # zero the complete destination - computing the rest 389 # is too much work 390 movl ARGBASE+8(%esp), %edi # dst 391 movl ARGBASE+12(%esp), %ecx # len 392 xorl %eax,%eax 393 rep ; stosb 394 395 jmp 5000b 396 3976002: 398 movl ARGBASE+24(%esp), %ebx # dst_err_ptr 399 movl $-EFAULT,(%ebx) 400 jmp 5000b 401 402.previous 403 404 popl_cfi_reg ebx 405 popl_cfi_reg esi 406 popl_cfi_reg edi 407 popl_cfi %ecx # equivalent to addl $4,%esp 408 ret 409 CFI_ENDPROC 410ENDPROC(csum_partial_copy_generic) 411 412#else 413 414/* Version for PentiumII/PPro */ 415 416#define ROUND1(x) \ 417 SRC(movl x(%esi), %ebx ) ; \ 418 addl %ebx, %eax ; \ 419 DST(movl %ebx, x(%edi) ) ; 420 421#define ROUND(x) \ 422 SRC(movl x(%esi), %ebx ) ; \ 423 adcl %ebx, %eax ; \ 424 DST(movl %ebx, x(%edi) ) ; 425 426#define ARGBASE 12 427 428ENTRY(csum_partial_copy_generic) 429 CFI_STARTPROC 430 pushl_cfi_reg ebx 431 pushl_cfi_reg edi 432 pushl_cfi_reg esi 433 movl ARGBASE+4(%esp),%esi #src 434 movl ARGBASE+8(%esp),%edi #dst 435 movl ARGBASE+12(%esp),%ecx #len 436 movl ARGBASE+16(%esp),%eax #sum 437# movl %ecx, %edx 438 movl %ecx, %ebx 439 movl %esi, %edx 440 shrl $6, %ecx 441 andl $0x3c, %ebx 442 negl %ebx 443 subl %ebx, %esi 444 subl %ebx, %edi 445 lea -1(%esi),%edx 446 andl $-32,%edx 447 lea 3f(%ebx,%ebx), %ebx 448 testl %esi, %esi 449 jmp *%ebx 4501: addl $64,%esi 451 addl $64,%edi 452 SRC(movb -32(%edx),%bl) ; SRC(movb (%edx),%bl) 453 ROUND1(-64) ROUND(-60) ROUND(-56) ROUND(-52) 454 ROUND (-48) ROUND(-44) ROUND(-40) ROUND(-36) 455 ROUND (-32) ROUND(-28) ROUND(-24) ROUND(-20) 456 ROUND (-16) ROUND(-12) ROUND(-8) ROUND(-4) 4573: adcl $0,%eax 458 addl $64, %edx 459 dec %ecx 460 jge 1b 4614: movl ARGBASE+12(%esp),%edx #len 462 andl $3, %edx 463 jz 7f 464 cmpl $2, %edx 465 jb 5f 466SRC( movw (%esi), %dx ) 467 leal 2(%esi), %esi 468DST( movw %dx, (%edi) ) 469 leal 2(%edi), %edi 470 je 6f 471 shll $16,%edx 4725: 473SRC( movb (%esi), %dl ) 474DST( movb %dl, (%edi) ) 4756: addl %edx, %eax 476 adcl $0, %eax 4777: 478.section .fixup, "ax" 4796001: movl ARGBASE+20(%esp), %ebx # src_err_ptr 480 movl $-EFAULT, (%ebx) 481 # zero the complete destination (computing the rest is too much work) 482 movl ARGBASE+8(%esp),%edi # dst 483 movl ARGBASE+12(%esp),%ecx # len 484 xorl %eax,%eax 485 rep; stosb 486 jmp 7b 4876002: movl ARGBASE+24(%esp), %ebx # dst_err_ptr 488 movl $-EFAULT, (%ebx) 489 jmp 7b 490.previous 491 492 popl_cfi_reg esi 493 popl_cfi_reg edi 494 popl_cfi_reg ebx 495 ret 496 CFI_ENDPROC 497ENDPROC(csum_partial_copy_generic) 498 499#undef ROUND 500#undef ROUND1 501 502#endif 503