1 /* SPDX-License-Identifier: GPL-2.0 */
2 /*
3 * NH - ε-almost-universal hash function, NEON accelerated version
4 *
5 * Copyright 2018 Google LLC
6 *
7 * Author: Eric Biggers <ebiggers@google.com>
8 */
9
10 #include <linux/linkage.h>
11
12 .text
13 .fpu neon
14
15 KEY .req r0
16 MESSAGE .req r1
17 MESSAGE_LEN .req r2
18 HASH .req r3
19
20 PASS0_SUMS .req q0
21 PASS0_SUM_A .req d0
22 PASS0_SUM_B .req d1
23 PASS1_SUMS .req q1
24 PASS1_SUM_A .req d2
25 PASS1_SUM_B .req d3
26 PASS2_SUMS .req q2
27 PASS2_SUM_A .req d4
28 PASS2_SUM_B .req d5
29 PASS3_SUMS .req q3
30 PASS3_SUM_A .req d6
31 PASS3_SUM_B .req d7
32 K0 .req q4
33 K1 .req q5
34 K2 .req q6
35 K3 .req q7
36 T0 .req q8
37 T0_L .req d16
38 T0_H .req d17
39 T1 .req q9
40 T1_L .req d18
41 T1_H .req d19
42 T2 .req q10
43 T2_L .req d20
44 T2_H .req d21
45 T3 .req q11
46 T3_L .req d22
47 T3_H .req d23
48
49 .macro _nh_stride k0, k1, k2, k3
50
51 // Load next message stride
52 vld1.8 {T3}, [MESSAGE]!
53
54 // Load next key stride
55 vld1.32 {\k3}, [KEY]!
56
57 // Add message words to key words
58 vadd.u32 T0, T3, \k0
59 vadd.u32 T1, T3, \k1
60 vadd.u32 T2, T3, \k2
61 vadd.u32 T3, T3, \k3
62
63 // Multiply 32x32 => 64 and accumulate
64 vmlal.u32 PASS0_SUMS, T0_L, T0_H
65 vmlal.u32 PASS1_SUMS, T1_L, T1_H
66 vmlal.u32 PASS2_SUMS, T2_L, T2_H
67 vmlal.u32 PASS3_SUMS, T3_L, T3_H
68 .endm
69
70 /*
71 * void nh_neon(const u32 *key, const u8 *message, size_t message_len,
72 * u8 hash[NH_HASH_BYTES])
73 *
74 * It's guaranteed that message_len % 16 == 0.
75 */
76 ENTRY(nh_neon)
77
78 vld1.32 {K0,K1}, [KEY]!
79 vmov.u64 PASS0_SUMS, #0
80 vmov.u64 PASS1_SUMS, #0
81 vld1.32 {K2}, [KEY]!
82 vmov.u64 PASS2_SUMS, #0
83 vmov.u64 PASS3_SUMS, #0
84
85 subs MESSAGE_LEN, MESSAGE_LEN, #64
86 blt .Lloop4_done
87 .Lloop4:
88 _nh_stride K0, K1, K2, K3
89 _nh_stride K1, K2, K3, K0
90 _nh_stride K2, K3, K0, K1
91 _nh_stride K3, K0, K1, K2
92 subs MESSAGE_LEN, MESSAGE_LEN, #64
93 bge .Lloop4
94
95 .Lloop4_done:
96 ands MESSAGE_LEN, MESSAGE_LEN, #63
97 beq .Ldone
98 _nh_stride K0, K1, K2, K3
99
100 subs MESSAGE_LEN, MESSAGE_LEN, #16
101 beq .Ldone
102 _nh_stride K1, K2, K3, K0
103
104 subs MESSAGE_LEN, MESSAGE_LEN, #16
105 beq .Ldone
106 _nh_stride K2, K3, K0, K1
107
108 .Ldone:
109 // Sum the accumulators for each pass, then store the sums to 'hash'
110 vadd.u64 T0_L, PASS0_SUM_A, PASS0_SUM_B
111 vadd.u64 T0_H, PASS1_SUM_A, PASS1_SUM_B
112 vadd.u64 T1_L, PASS2_SUM_A, PASS2_SUM_B
113 vadd.u64 T1_H, PASS3_SUM_A, PASS3_SUM_B
114 vst1.8 {T0-T1}, [HASH]
115 bx lr
116 ENDPROC(nh_neon)