1/* GPL HEADER START
2 *
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License version 2 only,
7 * as published by the Free Software Foundation.
8 *
9 * This program is distributed in the hope that it will be useful, but
10 * WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
12 * General Public License version 2 for more details (a copy is included
13 * in the LICENSE file that accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License
16 * version 2 along with this program; If not, see http://www.gnu.org/licenses
17 *
18 * Please  visit http://www.xyratex.com/contact if you need additional
19 * information or have any questions.
20 *
21 * GPL HEADER END
22 */
23
24/*
25 * Copyright 2012 Xyratex Technology Limited
26 *
27 * Using hardware provided PCLMULQDQ instruction to accelerate the CRC32
28 * calculation.
29 * CRC32 polynomial:0x04c11db7(BE)/0xEDB88320(LE)
30 * PCLMULQDQ is a new instruction in Intel SSE4.2, the reference can be found
31 * at:
32 * http://www.intel.com/products/processor/manuals/
33 * Intel(R) 64 and IA-32 Architectures Software Developer's Manual
34 * Volume 2B: Instruction Set Reference, N-Z
35 *
36 * Authors:   Gregory Prestas <Gregory_Prestas@us.xyratex.com>
37 *	      Alexander Boyko <Alexander_Boyko@xyratex.com>
38 */
39
40#include <linux/linkage.h>
41#include <asm/inst.h>
42
43
44.align 16
45/*
46 * [x4*128+32 mod P(x) << 32)]'  << 1   = 0x154442bd4
47 * #define CONSTANT_R1  0x154442bd4LL
48 *
49 * [(x4*128-32 mod P(x) << 32)]' << 1   = 0x1c6e41596
50 * #define CONSTANT_R2  0x1c6e41596LL
51 */
52.Lconstant_R2R1:
53	.octa 0x00000001c6e415960000000154442bd4
54/*
55 * [(x128+32 mod P(x) << 32)]'   << 1   = 0x1751997d0
56 * #define CONSTANT_R3  0x1751997d0LL
57 *
58 * [(x128-32 mod P(x) << 32)]'   << 1   = 0x0ccaa009e
59 * #define CONSTANT_R4  0x0ccaa009eLL
60 */
61.Lconstant_R4R3:
62	.octa 0x00000000ccaa009e00000001751997d0
63/*
64 * [(x64 mod P(x) << 32)]'       << 1   = 0x163cd6124
65 * #define CONSTANT_R5  0x163cd6124LL
66 */
67.Lconstant_R5:
68	.octa 0x00000000000000000000000163cd6124
69.Lconstant_mask32:
70	.octa 0x000000000000000000000000FFFFFFFF
71/*
72 * #define CRCPOLY_TRUE_LE_FULL 0x1DB710641LL
73 *
74 * Barrett Reduction constant (u64`) = u` = (x**64 / P(x))` = 0x1F7011641LL
75 * #define CONSTANT_RU  0x1F7011641LL
76 */
77.Lconstant_RUpoly:
78	.octa 0x00000001F701164100000001DB710641
79
80#define CONSTANT %xmm0
81
82#ifdef __x86_64__
83#define BUF     %rdi
84#define LEN     %rsi
85#define CRC     %edx
86#else
87#define BUF     %eax
88#define LEN     %edx
89#define CRC     %ecx
90#endif
91
92
93
94.text
95/**
96 *      Calculate crc32
97 *      BUF - buffer (16 bytes aligned)
98 *      LEN - sizeof buffer (16 bytes aligned), LEN should be grater than 63
99 *      CRC - initial crc32
100 *      return %eax crc32
101 *      uint crc32_pclmul_le_16(unsigned char const *buffer,
102 *	                     size_t len, uint crc32)
103 */
104
105ENTRY(crc32_pclmul_le_16) /* buffer and buffer size are 16 bytes aligned */
106	movdqa  (BUF), %xmm1
107	movdqa  0x10(BUF), %xmm2
108	movdqa  0x20(BUF), %xmm3
109	movdqa  0x30(BUF), %xmm4
110	movd    CRC, CONSTANT
111	pxor    CONSTANT, %xmm1
112	sub     $0x40, LEN
113	add     $0x40, BUF
114#ifndef __x86_64__
115	/* This is for position independent code(-fPIC) support for 32bit */
116	call    delta
117delta:
118	pop     %ecx
119#endif
120	cmp     $0x40, LEN
121	jb      less_64
122
123#ifdef __x86_64__
124	movdqa .Lconstant_R2R1(%rip), CONSTANT
125#else
126	movdqa .Lconstant_R2R1 - delta(%ecx), CONSTANT
127#endif
128
129loop_64:/*  64 bytes Full cache line folding */
130	prefetchnta    0x40(BUF)
131	movdqa  %xmm1, %xmm5
132	movdqa  %xmm2, %xmm6
133	movdqa  %xmm3, %xmm7
134#ifdef __x86_64__
135	movdqa  %xmm4, %xmm8
136#endif
137	PCLMULQDQ 00, CONSTANT, %xmm1
138	PCLMULQDQ 00, CONSTANT, %xmm2
139	PCLMULQDQ 00, CONSTANT, %xmm3
140#ifdef __x86_64__
141	PCLMULQDQ 00, CONSTANT, %xmm4
142#endif
143	PCLMULQDQ 0x11, CONSTANT, %xmm5
144	PCLMULQDQ 0x11, CONSTANT, %xmm6
145	PCLMULQDQ 0x11, CONSTANT, %xmm7
146#ifdef __x86_64__
147	PCLMULQDQ 0x11, CONSTANT, %xmm8
148#endif
149	pxor    %xmm5, %xmm1
150	pxor    %xmm6, %xmm2
151	pxor    %xmm7, %xmm3
152#ifdef __x86_64__
153	pxor    %xmm8, %xmm4
154#else
155	/* xmm8 unsupported for x32 */
156	movdqa  %xmm4, %xmm5
157	PCLMULQDQ 00, CONSTANT, %xmm4
158	PCLMULQDQ 0x11, CONSTANT, %xmm5
159	pxor    %xmm5, %xmm4
160#endif
161
162	pxor    (BUF), %xmm1
163	pxor    0x10(BUF), %xmm2
164	pxor    0x20(BUF), %xmm3
165	pxor    0x30(BUF), %xmm4
166
167	sub     $0x40, LEN
168	add     $0x40, BUF
169	cmp     $0x40, LEN
170	jge     loop_64
171less_64:/*  Folding cache line into 128bit */
172#ifdef __x86_64__
173	movdqa  .Lconstant_R4R3(%rip), CONSTANT
174#else
175	movdqa  .Lconstant_R4R3 - delta(%ecx), CONSTANT
176#endif
177	prefetchnta     (BUF)
178
179	movdqa  %xmm1, %xmm5
180	PCLMULQDQ 0x00, CONSTANT, %xmm1
181	PCLMULQDQ 0x11, CONSTANT, %xmm5
182	pxor    %xmm5, %xmm1
183	pxor    %xmm2, %xmm1
184
185	movdqa  %xmm1, %xmm5
186	PCLMULQDQ 0x00, CONSTANT, %xmm1
187	PCLMULQDQ 0x11, CONSTANT, %xmm5
188	pxor    %xmm5, %xmm1
189	pxor    %xmm3, %xmm1
190
191	movdqa  %xmm1, %xmm5
192	PCLMULQDQ 0x00, CONSTANT, %xmm1
193	PCLMULQDQ 0x11, CONSTANT, %xmm5
194	pxor    %xmm5, %xmm1
195	pxor    %xmm4, %xmm1
196
197	cmp     $0x10, LEN
198	jb      fold_64
199loop_16:/* Folding rest buffer into 128bit */
200	movdqa  %xmm1, %xmm5
201	PCLMULQDQ 0x00, CONSTANT, %xmm1
202	PCLMULQDQ 0x11, CONSTANT, %xmm5
203	pxor    %xmm5, %xmm1
204	pxor    (BUF), %xmm1
205	sub     $0x10, LEN
206	add     $0x10, BUF
207	cmp     $0x10, LEN
208	jge     loop_16
209
210fold_64:
211	/* perform the last 64 bit fold, also adds 32 zeroes
212	 * to the input stream */
213	PCLMULQDQ 0x01, %xmm1, CONSTANT /* R4 * xmm1.low */
214	psrldq  $0x08, %xmm1
215	pxor    CONSTANT, %xmm1
216
217	/* final 32-bit fold */
218	movdqa  %xmm1, %xmm2
219#ifdef __x86_64__
220	movdqa  .Lconstant_R5(%rip), CONSTANT
221	movdqa  .Lconstant_mask32(%rip), %xmm3
222#else
223	movdqa  .Lconstant_R5 - delta(%ecx), CONSTANT
224	movdqa  .Lconstant_mask32 - delta(%ecx), %xmm3
225#endif
226	psrldq  $0x04, %xmm2
227	pand    %xmm3, %xmm1
228	PCLMULQDQ 0x00, CONSTANT, %xmm1
229	pxor    %xmm2, %xmm1
230
231	/* Finish up with the bit-reversed barrett reduction 64 ==> 32 bits */
232#ifdef __x86_64__
233	movdqa  .Lconstant_RUpoly(%rip), CONSTANT
234#else
235	movdqa  .Lconstant_RUpoly - delta(%ecx), CONSTANT
236#endif
237	movdqa  %xmm1, %xmm2
238	pand    %xmm3, %xmm1
239	PCLMULQDQ 0x10, CONSTANT, %xmm1
240	pand    %xmm3, %xmm1
241	PCLMULQDQ 0x00, CONSTANT, %xmm1
242	pxor    %xmm2, %xmm1
243	PEXTRD  0x01, %xmm1, %eax
244
245	ret
246ENDPROC(crc32_pclmul_le_16)
247