1 /* SPDX-License-Identifier: GPL-2.0 */
2 /*
3 * NH - ε-almost-universal hash function, ARM64 NEON accelerated version
4 *
5 * Copyright 2018 Google LLC
6 *
7 * Author: Eric Biggers <ebiggers@google.com>
8 */
9
10 #include <linux/linkage.h>
11
12 KEY .req x0
13 MESSAGE .req x1
14 MESSAGE_LEN .req x2
15 HASH .req x3
16
17 PASS0_SUMS .req v0
18 PASS1_SUMS .req v1
19 PASS2_SUMS .req v2
20 PASS3_SUMS .req v3
21 K0 .req v4
22 K1 .req v5
23 K2 .req v6
24 K3 .req v7
25 T0 .req v8
26 T1 .req v9
27 T2 .req v10
28 T3 .req v11
29 T4 .req v12
30 T5 .req v13
31 T6 .req v14
32 T7 .req v15
33
34 .macro _nh_stride k0, k1, k2, k3
35
36 // Load next message stride
37 ld1 {T3.16b}, [MESSAGE], #16
38
39 // Load next key stride
40 ld1 {\k3\().4s}, [KEY], #16
41
42 // Add message words to key words
43 add T0.4s, T3.4s, \k0\().4s
44 add T1.4s, T3.4s, \k1\().4s
45 add T2.4s, T3.4s, \k2\().4s
46 add T3.4s, T3.4s, \k3\().4s
47
48 // Multiply 32x32 => 64 and accumulate
49 mov T4.d[0], T0.d[1]
50 mov T5.d[0], T1.d[1]
51 mov T6.d[0], T2.d[1]
52 mov T7.d[0], T3.d[1]
53 umlal PASS0_SUMS.2d, T0.2s, T4.2s
54 umlal PASS1_SUMS.2d, T1.2s, T5.2s
55 umlal PASS2_SUMS.2d, T2.2s, T6.2s
56 umlal PASS3_SUMS.2d, T3.2s, T7.2s
57 .endm
58
59 /*
60 * void nh_neon(const u32 *key, const u8 *message, size_t message_len,
61 * u8 hash[NH_HASH_BYTES])
62 *
63 * It's guaranteed that message_len % 16 == 0.
64 */
65 ENTRY(nh_neon)
66
67 ld1 {K0.4s,K1.4s}, [KEY], #32
68 movi PASS0_SUMS.2d, #0
69 movi PASS1_SUMS.2d, #0
70 ld1 {K2.4s}, [KEY], #16
71 movi PASS2_SUMS.2d, #0
72 movi PASS3_SUMS.2d, #0
73
74 subs MESSAGE_LEN, MESSAGE_LEN, #64
75 blt .Lloop4_done
76 .Lloop4:
77 _nh_stride K0, K1, K2, K3
78 _nh_stride K1, K2, K3, K0
79 _nh_stride K2, K3, K0, K1
80 _nh_stride K3, K0, K1, K2
81 subs MESSAGE_LEN, MESSAGE_LEN, #64
82 bge .Lloop4
83
84 .Lloop4_done:
85 ands MESSAGE_LEN, MESSAGE_LEN, #63
86 beq .Ldone
87 _nh_stride K0, K1, K2, K3
88
89 subs MESSAGE_LEN, MESSAGE_LEN, #16
90 beq .Ldone
91 _nh_stride K1, K2, K3, K0
92
93 subs MESSAGE_LEN, MESSAGE_LEN, #16
94 beq .Ldone
95 _nh_stride K2, K3, K0, K1
96
97 .Ldone:
98 // Sum the accumulators for each pass, then store the sums to 'hash'
99 addp T0.2d, PASS0_SUMS.2d, PASS1_SUMS.2d
100 addp T1.2d, PASS2_SUMS.2d, PASS3_SUMS.2d
101 st1 {T0.16b,T1.16b}, [HASH]
102 ret
103 ENDPROC(nh_neon)