1/*
2 * Accelerated GHASH implementation with ARMv8 vmull.p64 instructions.
3 *
4 * Copyright (C) 2015 Linaro Ltd. <ard.biesheuvel@linaro.org>
5 *
6 * This program is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU General Public License version 2 as published
8 * by the Free Software Foundation.
9 */
10
11#include <asm/hwcap.h>
12#include <asm/neon.h>
13#include <asm/simd.h>
14#include <asm/unaligned.h>
15#include <crypto/cryptd.h>
16#include <crypto/internal/hash.h>
17#include <crypto/gf128mul.h>
18#include <linux/crypto.h>
19#include <linux/module.h>
20
21MODULE_DESCRIPTION("GHASH secure hash using ARMv8 Crypto Extensions");
22MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");
23MODULE_LICENSE("GPL v2");
24
25#define GHASH_BLOCK_SIZE	16
26#define GHASH_DIGEST_SIZE	16
27
28struct ghash_key {
29	u64	a;
30	u64	b;
31};
32
33struct ghash_desc_ctx {
34	u64 digest[GHASH_DIGEST_SIZE/sizeof(u64)];
35	u8 buf[GHASH_BLOCK_SIZE];
36	u32 count;
37};
38
39struct ghash_async_ctx {
40	struct cryptd_ahash *cryptd_tfm;
41};
42
43asmlinkage void pmull_ghash_update(int blocks, u64 dg[], const char *src,
44				   struct ghash_key const *k, const char *head);
45
46static int ghash_init(struct shash_desc *desc)
47{
48	struct ghash_desc_ctx *ctx = shash_desc_ctx(desc);
49
50	*ctx = (struct ghash_desc_ctx){};
51	return 0;
52}
53
54static int ghash_update(struct shash_desc *desc, const u8 *src,
55			unsigned int len)
56{
57	struct ghash_desc_ctx *ctx = shash_desc_ctx(desc);
58	unsigned int partial = ctx->count % GHASH_BLOCK_SIZE;
59
60	ctx->count += len;
61
62	if ((partial + len) >= GHASH_BLOCK_SIZE) {
63		struct ghash_key *key = crypto_shash_ctx(desc->tfm);
64		int blocks;
65
66		if (partial) {
67			int p = GHASH_BLOCK_SIZE - partial;
68
69			memcpy(ctx->buf + partial, src, p);
70			src += p;
71			len -= p;
72		}
73
74		blocks = len / GHASH_BLOCK_SIZE;
75		len %= GHASH_BLOCK_SIZE;
76
77		kernel_neon_begin();
78		pmull_ghash_update(blocks, ctx->digest, src, key,
79				   partial ? ctx->buf : NULL);
80		kernel_neon_end();
81		src += blocks * GHASH_BLOCK_SIZE;
82		partial = 0;
83	}
84	if (len)
85		memcpy(ctx->buf + partial, src, len);
86	return 0;
87}
88
89static int ghash_final(struct shash_desc *desc, u8 *dst)
90{
91	struct ghash_desc_ctx *ctx = shash_desc_ctx(desc);
92	unsigned int partial = ctx->count % GHASH_BLOCK_SIZE;
93
94	if (partial) {
95		struct ghash_key *key = crypto_shash_ctx(desc->tfm);
96
97		memset(ctx->buf + partial, 0, GHASH_BLOCK_SIZE - partial);
98		kernel_neon_begin();
99		pmull_ghash_update(1, ctx->digest, ctx->buf, key, NULL);
100		kernel_neon_end();
101	}
102	put_unaligned_be64(ctx->digest[1], dst);
103	put_unaligned_be64(ctx->digest[0], dst + 8);
104
105	*ctx = (struct ghash_desc_ctx){};
106	return 0;
107}
108
109static int ghash_setkey(struct crypto_shash *tfm,
110			const u8 *inkey, unsigned int keylen)
111{
112	struct ghash_key *key = crypto_shash_ctx(tfm);
113	u64 a, b;
114
115	if (keylen != GHASH_BLOCK_SIZE) {
116		crypto_shash_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
117		return -EINVAL;
118	}
119
120	/* perform multiplication by 'x' in GF(2^128) */
121	b = get_unaligned_be64(inkey);
122	a = get_unaligned_be64(inkey + 8);
123
124	key->a = (a << 1) | (b >> 63);
125	key->b = (b << 1) | (a >> 63);
126
127	if (b >> 63)
128		key->b ^= 0xc200000000000000UL;
129
130	return 0;
131}
132
133static struct shash_alg ghash_alg = {
134	.digestsize		= GHASH_DIGEST_SIZE,
135	.init			= ghash_init,
136	.update			= ghash_update,
137	.final			= ghash_final,
138	.setkey			= ghash_setkey,
139	.descsize		= sizeof(struct ghash_desc_ctx),
140	.base			= {
141		.cra_name	= "ghash",
142		.cra_driver_name = "__driver-ghash-ce",
143		.cra_priority	= 0,
144		.cra_flags	= CRYPTO_ALG_TYPE_SHASH | CRYPTO_ALG_INTERNAL,
145		.cra_blocksize	= GHASH_BLOCK_SIZE,
146		.cra_ctxsize	= sizeof(struct ghash_key),
147		.cra_module	= THIS_MODULE,
148	},
149};
150
151static int ghash_async_init(struct ahash_request *req)
152{
153	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
154	struct ghash_async_ctx *ctx = crypto_ahash_ctx(tfm);
155	struct ahash_request *cryptd_req = ahash_request_ctx(req);
156	struct cryptd_ahash *cryptd_tfm = ctx->cryptd_tfm;
157
158	if (!may_use_simd()) {
159		memcpy(cryptd_req, req, sizeof(*req));
160		ahash_request_set_tfm(cryptd_req, &cryptd_tfm->base);
161		return crypto_ahash_init(cryptd_req);
162	} else {
163		struct shash_desc *desc = cryptd_shash_desc(cryptd_req);
164		struct crypto_shash *child = cryptd_ahash_child(cryptd_tfm);
165
166		desc->tfm = child;
167		desc->flags = req->base.flags;
168		return crypto_shash_init(desc);
169	}
170}
171
172static int ghash_async_update(struct ahash_request *req)
173{
174	struct ahash_request *cryptd_req = ahash_request_ctx(req);
175
176	if (!may_use_simd()) {
177		struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
178		struct ghash_async_ctx *ctx = crypto_ahash_ctx(tfm);
179		struct cryptd_ahash *cryptd_tfm = ctx->cryptd_tfm;
180
181		memcpy(cryptd_req, req, sizeof(*req));
182		ahash_request_set_tfm(cryptd_req, &cryptd_tfm->base);
183		return crypto_ahash_update(cryptd_req);
184	} else {
185		struct shash_desc *desc = cryptd_shash_desc(cryptd_req);
186		return shash_ahash_update(req, desc);
187	}
188}
189
190static int ghash_async_final(struct ahash_request *req)
191{
192	struct ahash_request *cryptd_req = ahash_request_ctx(req);
193
194	if (!may_use_simd()) {
195		struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
196		struct ghash_async_ctx *ctx = crypto_ahash_ctx(tfm);
197		struct cryptd_ahash *cryptd_tfm = ctx->cryptd_tfm;
198
199		memcpy(cryptd_req, req, sizeof(*req));
200		ahash_request_set_tfm(cryptd_req, &cryptd_tfm->base);
201		return crypto_ahash_final(cryptd_req);
202	} else {
203		struct shash_desc *desc = cryptd_shash_desc(cryptd_req);
204		return crypto_shash_final(desc, req->result);
205	}
206}
207
208static int ghash_async_digest(struct ahash_request *req)
209{
210	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
211	struct ghash_async_ctx *ctx = crypto_ahash_ctx(tfm);
212	struct ahash_request *cryptd_req = ahash_request_ctx(req);
213	struct cryptd_ahash *cryptd_tfm = ctx->cryptd_tfm;
214
215	if (!may_use_simd()) {
216		memcpy(cryptd_req, req, sizeof(*req));
217		ahash_request_set_tfm(cryptd_req, &cryptd_tfm->base);
218		return crypto_ahash_digest(cryptd_req);
219	} else {
220		struct shash_desc *desc = cryptd_shash_desc(cryptd_req);
221		struct crypto_shash *child = cryptd_ahash_child(cryptd_tfm);
222
223		desc->tfm = child;
224		desc->flags = req->base.flags;
225		return shash_ahash_digest(req, desc);
226	}
227}
228
229static int ghash_async_setkey(struct crypto_ahash *tfm, const u8 *key,
230			      unsigned int keylen)
231{
232	struct ghash_async_ctx *ctx = crypto_ahash_ctx(tfm);
233	struct crypto_ahash *child = &ctx->cryptd_tfm->base;
234	int err;
235
236	crypto_ahash_clear_flags(child, CRYPTO_TFM_REQ_MASK);
237	crypto_ahash_set_flags(child, crypto_ahash_get_flags(tfm)
238			       & CRYPTO_TFM_REQ_MASK);
239	err = crypto_ahash_setkey(child, key, keylen);
240	crypto_ahash_set_flags(tfm, crypto_ahash_get_flags(child)
241			       & CRYPTO_TFM_RES_MASK);
242
243	return err;
244}
245
246static int ghash_async_init_tfm(struct crypto_tfm *tfm)
247{
248	struct cryptd_ahash *cryptd_tfm;
249	struct ghash_async_ctx *ctx = crypto_tfm_ctx(tfm);
250
251	cryptd_tfm = cryptd_alloc_ahash("__driver-ghash-ce",
252					CRYPTO_ALG_INTERNAL,
253					CRYPTO_ALG_INTERNAL);
254	if (IS_ERR(cryptd_tfm))
255		return PTR_ERR(cryptd_tfm);
256	ctx->cryptd_tfm = cryptd_tfm;
257	crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
258				 sizeof(struct ahash_request) +
259				 crypto_ahash_reqsize(&cryptd_tfm->base));
260
261	return 0;
262}
263
264static void ghash_async_exit_tfm(struct crypto_tfm *tfm)
265{
266	struct ghash_async_ctx *ctx = crypto_tfm_ctx(tfm);
267
268	cryptd_free_ahash(ctx->cryptd_tfm);
269}
270
271static struct ahash_alg ghash_async_alg = {
272	.init			= ghash_async_init,
273	.update			= ghash_async_update,
274	.final			= ghash_async_final,
275	.setkey			= ghash_async_setkey,
276	.digest			= ghash_async_digest,
277	.halg.digestsize	= GHASH_DIGEST_SIZE,
278	.halg.base		= {
279		.cra_name	= "ghash",
280		.cra_driver_name = "ghash-ce",
281		.cra_priority	= 300,
282		.cra_flags	= CRYPTO_ALG_TYPE_AHASH | CRYPTO_ALG_ASYNC,
283		.cra_blocksize	= GHASH_BLOCK_SIZE,
284		.cra_type	= &crypto_ahash_type,
285		.cra_ctxsize	= sizeof(struct ghash_async_ctx),
286		.cra_module	= THIS_MODULE,
287		.cra_init	= ghash_async_init_tfm,
288		.cra_exit	= ghash_async_exit_tfm,
289	},
290};
291
292static int __init ghash_ce_mod_init(void)
293{
294	int err;
295
296	if (!(elf_hwcap2 & HWCAP2_PMULL))
297		return -ENODEV;
298
299	err = crypto_register_shash(&ghash_alg);
300	if (err)
301		return err;
302	err = crypto_register_ahash(&ghash_async_alg);
303	if (err)
304		goto err_shash;
305
306	return 0;
307
308err_shash:
309	crypto_unregister_shash(&ghash_alg);
310	return err;
311}
312
313static void __exit ghash_ce_mod_exit(void)
314{
315	crypto_unregister_ahash(&ghash_async_alg);
316	crypto_unregister_shash(&ghash_alg);
317}
318
319module_init(ghash_ce_mod_init);
320module_exit(ghash_ce_mod_exit);
321