This source file includes following definitions.
- crypto_sha256_arm64_update
- crypto_sha256_arm64_finup
- crypto_sha256_arm64_final
- sha256_update_neon
- sha256_finup_neon
- sha256_final_neon
- sha256_mod_init
- sha256_mod_fini
1
2
3
4
5
6
7
8 #include <asm/hwcap.h>
9 #include <asm/neon.h>
10 #include <asm/simd.h>
11 #include <crypto/internal/hash.h>
12 #include <crypto/internal/simd.h>
13 #include <crypto/sha.h>
14 #include <crypto/sha256_base.h>
15 #include <linux/cryptohash.h>
16 #include <linux/types.h>
17 #include <linux/string.h>
18
19 MODULE_DESCRIPTION("SHA-224/SHA-256 secure hash for arm64");
20 MODULE_AUTHOR("Andy Polyakov <appro@openssl.org>");
21 MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");
22 MODULE_LICENSE("GPL v2");
23 MODULE_ALIAS_CRYPTO("sha224");
24 MODULE_ALIAS_CRYPTO("sha256");
25
26 asmlinkage void sha256_block_data_order(u32 *digest, const void *data,
27 unsigned int num_blks);
28 EXPORT_SYMBOL(sha256_block_data_order);
29
30 asmlinkage void sha256_block_neon(u32 *digest, const void *data,
31 unsigned int num_blks);
32
33 static int crypto_sha256_arm64_update(struct shash_desc *desc, const u8 *data,
34 unsigned int len)
35 {
36 return sha256_base_do_update(desc, data, len,
37 (sha256_block_fn *)sha256_block_data_order);
38 }
39
40 static int crypto_sha256_arm64_finup(struct shash_desc *desc, const u8 *data,
41 unsigned int len, u8 *out)
42 {
43 if (len)
44 sha256_base_do_update(desc, data, len,
45 (sha256_block_fn *)sha256_block_data_order);
46 sha256_base_do_finalize(desc,
47 (sha256_block_fn *)sha256_block_data_order);
48
49 return sha256_base_finish(desc, out);
50 }
51
52 static int crypto_sha256_arm64_final(struct shash_desc *desc, u8 *out)
53 {
54 return crypto_sha256_arm64_finup(desc, NULL, 0, out);
55 }
56
57 static struct shash_alg algs[] = { {
58 .digestsize = SHA256_DIGEST_SIZE,
59 .init = sha256_base_init,
60 .update = crypto_sha256_arm64_update,
61 .final = crypto_sha256_arm64_final,
62 .finup = crypto_sha256_arm64_finup,
63 .descsize = sizeof(struct sha256_state),
64 .base.cra_name = "sha256",
65 .base.cra_driver_name = "sha256-arm64",
66 .base.cra_priority = 125,
67 .base.cra_blocksize = SHA256_BLOCK_SIZE,
68 .base.cra_module = THIS_MODULE,
69 }, {
70 .digestsize = SHA224_DIGEST_SIZE,
71 .init = sha224_base_init,
72 .update = crypto_sha256_arm64_update,
73 .final = crypto_sha256_arm64_final,
74 .finup = crypto_sha256_arm64_finup,
75 .descsize = sizeof(struct sha256_state),
76 .base.cra_name = "sha224",
77 .base.cra_driver_name = "sha224-arm64",
78 .base.cra_priority = 125,
79 .base.cra_blocksize = SHA224_BLOCK_SIZE,
80 .base.cra_module = THIS_MODULE,
81 } };
82
83 static int sha256_update_neon(struct shash_desc *desc, const u8 *data,
84 unsigned int len)
85 {
86 struct sha256_state *sctx = shash_desc_ctx(desc);
87
88 if (!crypto_simd_usable())
89 return sha256_base_do_update(desc, data, len,
90 (sha256_block_fn *)sha256_block_data_order);
91
92 while (len > 0) {
93 unsigned int chunk = len;
94
95
96
97
98
99
100 if (IS_ENABLED(CONFIG_PREEMPT) &&
101 chunk + sctx->count % SHA256_BLOCK_SIZE > SHA256_BLOCK_SIZE)
102 chunk = SHA256_BLOCK_SIZE -
103 sctx->count % SHA256_BLOCK_SIZE;
104
105 kernel_neon_begin();
106 sha256_base_do_update(desc, data, chunk,
107 (sha256_block_fn *)sha256_block_neon);
108 kernel_neon_end();
109 data += chunk;
110 len -= chunk;
111 }
112 return 0;
113 }
114
115 static int sha256_finup_neon(struct shash_desc *desc, const u8 *data,
116 unsigned int len, u8 *out)
117 {
118 if (!crypto_simd_usable()) {
119 if (len)
120 sha256_base_do_update(desc, data, len,
121 (sha256_block_fn *)sha256_block_data_order);
122 sha256_base_do_finalize(desc,
123 (sha256_block_fn *)sha256_block_data_order);
124 } else {
125 if (len)
126 sha256_update_neon(desc, data, len);
127 kernel_neon_begin();
128 sha256_base_do_finalize(desc,
129 (sha256_block_fn *)sha256_block_neon);
130 kernel_neon_end();
131 }
132 return sha256_base_finish(desc, out);
133 }
134
135 static int sha256_final_neon(struct shash_desc *desc, u8 *out)
136 {
137 return sha256_finup_neon(desc, NULL, 0, out);
138 }
139
140 static struct shash_alg neon_algs[] = { {
141 .digestsize = SHA256_DIGEST_SIZE,
142 .init = sha256_base_init,
143 .update = sha256_update_neon,
144 .final = sha256_final_neon,
145 .finup = sha256_finup_neon,
146 .descsize = sizeof(struct sha256_state),
147 .base.cra_name = "sha256",
148 .base.cra_driver_name = "sha256-arm64-neon",
149 .base.cra_priority = 150,
150 .base.cra_blocksize = SHA256_BLOCK_SIZE,
151 .base.cra_module = THIS_MODULE,
152 }, {
153 .digestsize = SHA224_DIGEST_SIZE,
154 .init = sha224_base_init,
155 .update = sha256_update_neon,
156 .final = sha256_final_neon,
157 .finup = sha256_finup_neon,
158 .descsize = sizeof(struct sha256_state),
159 .base.cra_name = "sha224",
160 .base.cra_driver_name = "sha224-arm64-neon",
161 .base.cra_priority = 150,
162 .base.cra_blocksize = SHA224_BLOCK_SIZE,
163 .base.cra_module = THIS_MODULE,
164 } };
165
166 static int __init sha256_mod_init(void)
167 {
168 int ret = crypto_register_shashes(algs, ARRAY_SIZE(algs));
169 if (ret)
170 return ret;
171
172 if (cpu_have_named_feature(ASIMD)) {
173 ret = crypto_register_shashes(neon_algs, ARRAY_SIZE(neon_algs));
174 if (ret)
175 crypto_unregister_shashes(algs, ARRAY_SIZE(algs));
176 }
177 return ret;
178 }
179
180 static void __exit sha256_mod_fini(void)
181 {
182 if (cpu_have_named_feature(ASIMD))
183 crypto_unregister_shashes(neon_algs, ARRAY_SIZE(neon_algs));
184 crypto_unregister_shashes(algs, ARRAY_SIZE(algs));
185 }
186
187 module_init(sha256_mod_init);
188 module_exit(sha256_mod_fini);