1 /* SPDX-License-Identifier: GPL-2.0-only */
2 /*
3 * Accelerated GHASH implementation with NEON/ARMv8 vmull.p8/64 instructions.
4 *
5 * Copyright (C) 2015 - 2017 Linaro Ltd. <ard.biesheuvel@linaro.org>
6 */
7
8 #include <linux/linkage.h>
9 #include <asm/assembler.h>
10
11 SHASH .req q0
12 T1 .req q1
13 XL .req q2
14 XM .req q3
15 XH .req q4
16 IN1 .req q4
17
18 SHASH_L .req d0
19 SHASH_H .req d1
20 T1_L .req d2
21 T1_H .req d3
22 XL_L .req d4
23 XL_H .req d5
24 XM_L .req d6
25 XM_H .req d7
26 XH_L .req d8
27
28 t0l .req d10
29 t0h .req d11
30 t1l .req d12
31 t1h .req d13
32 t2l .req d14
33 t2h .req d15
34 t3l .req d16
35 t3h .req d17
36 t4l .req d18
37 t4h .req d19
38
39 t0q .req q5
40 t1q .req q6
41 t2q .req q7
42 t3q .req q8
43 t4q .req q9
44 T2 .req q9
45
46 s1l .req d20
47 s1h .req d21
48 s2l .req d22
49 s2h .req d23
50 s3l .req d24
51 s3h .req d25
52 s4l .req d26
53 s4h .req d27
54
55 MASK .req d28
56 SHASH2_p8 .req d28
57
58 k16 .req d29
59 k32 .req d30
60 k48 .req d31
61 SHASH2_p64 .req d31
62
63 HH .req q10
64 HH3 .req q11
65 HH4 .req q12
66 HH34 .req q13
67
68 HH_L .req d20
69 HH_H .req d21
70 HH3_L .req d22
71 HH3_H .req d23
72 HH4_L .req d24
73 HH4_H .req d25
74 HH34_L .req d26
75 HH34_H .req d27
76 SHASH2_H .req d29
77
78 XL2 .req q5
79 XM2 .req q6
80 XH2 .req q7
81 T3 .req q8
82
83 XL2_L .req d10
84 XL2_H .req d11
85 XM2_L .req d12
86 XM2_H .req d13
87 T3_L .req d16
88 T3_H .req d17
89
90 .text
91 .fpu crypto-neon-fp-armv8
92
93 .macro __pmull_p64, rd, rn, rm, b1, b2, b3, b4
94 vmull.p64 \rd, \rn, \rm
95 .endm
96
97 /*
98 * This implementation of 64x64 -> 128 bit polynomial multiplication
99 * using vmull.p8 instructions (8x8 -> 16) is taken from the paper
100 * "Fast Software Polynomial Multiplication on ARM Processors Using
101 * the NEON Engine" by Danilo Camara, Conrado Gouvea, Julio Lopez and
102 * Ricardo Dahab (https://hal.inria.fr/hal-01506572)
103 *
104 * It has been slightly tweaked for in-order performance, and to allow
105 * 'rq' to overlap with 'ad' or 'bd'.
106 */
107 .macro __pmull_p8, rq, ad, bd, b1=t4l, b2=t3l, b3=t4l, b4=t3l
108 vext.8 t0l, \ad, \ad, #1 @ A1
109 .ifc \b1, t4l
110 vext.8 t4l, \bd, \bd, #1 @ B1
111 .endif
112 vmull.p8 t0q, t0l, \bd @ F = A1*B
113 vext.8 t1l, \ad, \ad, #2 @ A2
114 vmull.p8 t4q, \ad, \b1 @ E = A*B1
115 .ifc \b2, t3l
116 vext.8 t3l, \bd, \bd, #2 @ B2
117 .endif
118 vmull.p8 t1q, t1l, \bd @ H = A2*B
119 vext.8 t2l, \ad, \ad, #3 @ A3
120 vmull.p8 t3q, \ad, \b2 @ G = A*B2
121 veor t0q, t0q, t4q @ L = E + F
122 .ifc \b3, t4l
123 vext.8 t4l, \bd, \bd, #3 @ B3
124 .endif
125 vmull.p8 t2q, t2l, \bd @ J = A3*B
126 veor t0l, t0l, t0h @ t0 = (L) (P0 + P1) << 8
127 veor t1q, t1q, t3q @ M = G + H
128 .ifc \b4, t3l
129 vext.8 t3l, \bd, \bd, #4 @ B4
130 .endif
131 vmull.p8 t4q, \ad, \b3 @ I = A*B3
132 veor t1l, t1l, t1h @ t1 = (M) (P2 + P3) << 16
133 vmull.p8 t3q, \ad, \b4 @ K = A*B4
134 vand t0h, t0h, k48
135 vand t1h, t1h, k32
136 veor t2q, t2q, t4q @ N = I + J
137 veor t0l, t0l, t0h
138 veor t1l, t1l, t1h
139 veor t2l, t2l, t2h @ t2 = (N) (P4 + P5) << 24
140 vand t2h, t2h, k16
141 veor t3l, t3l, t3h @ t3 = (K) (P6 + P7) << 32
142 vmov.i64 t3h, #0
143 vext.8 t0q, t0q, t0q, #15
144 veor t2l, t2l, t2h
145 vext.8 t1q, t1q, t1q, #14
146 vmull.p8 \rq, \ad, \bd @ D = A*B
147 vext.8 t2q, t2q, t2q, #13
148 vext.8 t3q, t3q, t3q, #12
149 veor t0q, t0q, t1q
150 veor t2q, t2q, t3q
151 veor \rq, \rq, t0q
152 veor \rq, \rq, t2q
153 .endm
154
155 //
156 // PMULL (64x64->128) based reduction for CPUs that can do
157 // it in a single instruction.
158 //
159 .macro __pmull_reduce_p64
160 vmull.p64 T1, XL_L, MASK
161
162 veor XH_L, XH_L, XM_H
163 vext.8 T1, T1, T1, #8
164 veor XL_H, XL_H, XM_L
165 veor T1, T1, XL
166
167 vmull.p64 XL, T1_H, MASK
168 .endm
169
170 //
171 // Alternative reduction for CPUs that lack support for the
172 // 64x64->128 PMULL instruction
173 //
174 .macro __pmull_reduce_p8
175 veor XL_H, XL_H, XM_L
176 veor XH_L, XH_L, XM_H
177
178 vshl.i64 T1, XL, #57
179 vshl.i64 T2, XL, #62
180 veor T1, T1, T2
181 vshl.i64 T2, XL, #63
182 veor T1, T1, T2
183 veor XL_H, XL_H, T1_L
184 veor XH_L, XH_L, T1_H
185
186 vshr.u64 T1, XL, #1
187 veor XH, XH, XL
188 veor XL, XL, T1
189 vshr.u64 T1, T1, #6
190 vshr.u64 XL, XL, #1
191 .endm
192
193 .macro ghash_update, pn
194 vld1.64 {XL}, [r1]
195
196 /* do the head block first, if supplied */
197 ldr ip, [sp]
198 teq ip, #0
199 beq 0f
200 vld1.64 {T1}, [ip]
201 teq r0, #0
202 b 3f
203
204 0: .ifc \pn, p64
205 tst r0, #3 // skip until #blocks is a
206 bne 2f // round multiple of 4
207
208 vld1.8 {XL2-XM2}, [r2]!
209 1: vld1.8 {T3-T2}, [r2]!
210 vrev64.8 XL2, XL2
211 vrev64.8 XM2, XM2
212
213 subs r0, r0, #4
214
215 vext.8 T1, XL2, XL2, #8
216 veor XL2_H, XL2_H, XL_L
217 veor XL, XL, T1
218
219 vrev64.8 T3, T3
220 vrev64.8 T1, T2
221
222 vmull.p64 XH, HH4_H, XL_H // a1 * b1
223 veor XL2_H, XL2_H, XL_H
224 vmull.p64 XL, HH4_L, XL_L // a0 * b0
225 vmull.p64 XM, HH34_H, XL2_H // (a1 + a0)(b1 + b0)
226
227 vmull.p64 XH2, HH3_H, XM2_L // a1 * b1
228 veor XM2_L, XM2_L, XM2_H
229 vmull.p64 XL2, HH3_L, XM2_H // a0 * b0
230 vmull.p64 XM2, HH34_L, XM2_L // (a1 + a0)(b1 + b0)
231
232 veor XH, XH, XH2
233 veor XL, XL, XL2
234 veor XM, XM, XM2
235
236 vmull.p64 XH2, HH_H, T3_L // a1 * b1
237 veor T3_L, T3_L, T3_H
238 vmull.p64 XL2, HH_L, T3_H // a0 * b0
239 vmull.p64 XM2, SHASH2_H, T3_L // (a1 + a0)(b1 + b0)
240
241 veor XH, XH, XH2
242 veor XL, XL, XL2
243 veor XM, XM, XM2
244
245 vmull.p64 XH2, SHASH_H, T1_L // a1 * b1
246 veor T1_L, T1_L, T1_H
247 vmull.p64 XL2, SHASH_L, T1_H // a0 * b0
248 vmull.p64 XM2, SHASH2_p64, T1_L // (a1 + a0)(b1 + b0)
249
250 veor XH, XH, XH2
251 veor XL, XL, XL2
252 veor XM, XM, XM2
253
254 beq 4f
255
256 vld1.8 {XL2-XM2}, [r2]!
257
258 veor T1, XL, XH
259 veor XM, XM, T1
260
261 __pmull_reduce_p64
262
263 veor T1, T1, XH
264 veor XL, XL, T1
265
266 b 1b
267 .endif
268
269 2: vld1.64 {T1}, [r2]!
270 subs r0, r0, #1
271
272 3: /* multiply XL by SHASH in GF(2^128) */
273 #ifndef CONFIG_CPU_BIG_ENDIAN
274 vrev64.8 T1, T1
275 #endif
276 vext.8 IN1, T1, T1, #8
277 veor T1_L, T1_L, XL_H
278 veor XL, XL, IN1
279
280 __pmull_\pn XH, XL_H, SHASH_H, s1h, s2h, s3h, s4h @ a1 * b1
281 veor T1, T1, XL
282 __pmull_\pn XL, XL_L, SHASH_L, s1l, s2l, s3l, s4l @ a0 * b0
283 __pmull_\pn XM, T1_L, SHASH2_\pn @ (a1+a0)(b1+b0)
284
285 4: veor T1, XL, XH
286 veor XM, XM, T1
287
288 __pmull_reduce_\pn
289
290 veor T1, T1, XH
291 veor XL, XL, T1
292
293 bne 0b
294
295 vst1.64 {XL}, [r1]
296 bx lr
297 .endm
298
299 /*
300 * void pmull_ghash_update(int blocks, u64 dg[], const char *src,
301 * struct ghash_key const *k, const char *head)
302 */
303 ENTRY(pmull_ghash_update_p64)
304 vld1.64 {SHASH}, [r3]!
305 vld1.64 {HH}, [r3]!
306 vld1.64 {HH3-HH4}, [r3]
307
308 veor SHASH2_p64, SHASH_L, SHASH_H
309 veor SHASH2_H, HH_L, HH_H
310 veor HH34_L, HH3_L, HH3_H
311 veor HH34_H, HH4_L, HH4_H
312
313 vmov.i8 MASK, #0xe1
314 vshl.u64 MASK, MASK, #57
315
316 ghash_update p64
317 ENDPROC(pmull_ghash_update_p64)
318
319 ENTRY(pmull_ghash_update_p8)
320 vld1.64 {SHASH}, [r3]
321 veor SHASH2_p8, SHASH_L, SHASH_H
322
323 vext.8 s1l, SHASH_L, SHASH_L, #1
324 vext.8 s2l, SHASH_L, SHASH_L, #2
325 vext.8 s3l, SHASH_L, SHASH_L, #3
326 vext.8 s4l, SHASH_L, SHASH_L, #4
327 vext.8 s1h, SHASH_H, SHASH_H, #1
328 vext.8 s2h, SHASH_H, SHASH_H, #2
329 vext.8 s3h, SHASH_H, SHASH_H, #3
330 vext.8 s4h, SHASH_H, SHASH_H, #4
331
332 vmov.i64 k16, #0xffff
333 vmov.i64 k32, #0xffffffff
334 vmov.i64 k48, #0xffffffffffff
335
336 ghash_update p8
337 ENDPROC(pmull_ghash_update_p8)