1/*
2 * Accelerated GHASH implementation with ARMv8 PMULL instructions.
3 *
4 * Copyright (C) 2014 Linaro Ltd. <ard.biesheuvel@linaro.org>
5 *
6 * This program is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU General Public License version 2 as published
8 * by the Free Software Foundation.
9 */
10
11#include <asm/neon.h>
12#include <asm/unaligned.h>
13#include <crypto/internal/hash.h>
14#include <linux/cpufeature.h>
15#include <linux/crypto.h>
16#include <linux/module.h>
17
18MODULE_DESCRIPTION("GHASH secure hash using ARMv8 Crypto Extensions");
19MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");
20MODULE_LICENSE("GPL v2");
21
22#define GHASH_BLOCK_SIZE	16
23#define GHASH_DIGEST_SIZE	16
24
25struct ghash_key {
26	u64 a;
27	u64 b;
28};
29
30struct ghash_desc_ctx {
31	u64 digest[GHASH_DIGEST_SIZE/sizeof(u64)];
32	u8 buf[GHASH_BLOCK_SIZE];
33	u32 count;
34};
35
36asmlinkage void pmull_ghash_update(int blocks, u64 dg[], const char *src,
37				   struct ghash_key const *k, const char *head);
38
39static int ghash_init(struct shash_desc *desc)
40{
41	struct ghash_desc_ctx *ctx = shash_desc_ctx(desc);
42
43	*ctx = (struct ghash_desc_ctx){};
44	return 0;
45}
46
47static int ghash_update(struct shash_desc *desc, const u8 *src,
48			unsigned int len)
49{
50	struct ghash_desc_ctx *ctx = shash_desc_ctx(desc);
51	unsigned int partial = ctx->count % GHASH_BLOCK_SIZE;
52
53	ctx->count += len;
54
55	if ((partial + len) >= GHASH_BLOCK_SIZE) {
56		struct ghash_key *key = crypto_shash_ctx(desc->tfm);
57		int blocks;
58
59		if (partial) {
60			int p = GHASH_BLOCK_SIZE - partial;
61
62			memcpy(ctx->buf + partial, src, p);
63			src += p;
64			len -= p;
65		}
66
67		blocks = len / GHASH_BLOCK_SIZE;
68		len %= GHASH_BLOCK_SIZE;
69
70		kernel_neon_begin_partial(8);
71		pmull_ghash_update(blocks, ctx->digest, src, key,
72				   partial ? ctx->buf : NULL);
73		kernel_neon_end();
74		src += blocks * GHASH_BLOCK_SIZE;
75		partial = 0;
76	}
77	if (len)
78		memcpy(ctx->buf + partial, src, len);
79	return 0;
80}
81
82static int ghash_final(struct shash_desc *desc, u8 *dst)
83{
84	struct ghash_desc_ctx *ctx = shash_desc_ctx(desc);
85	unsigned int partial = ctx->count % GHASH_BLOCK_SIZE;
86
87	if (partial) {
88		struct ghash_key *key = crypto_shash_ctx(desc->tfm);
89
90		memset(ctx->buf + partial, 0, GHASH_BLOCK_SIZE - partial);
91
92		kernel_neon_begin_partial(8);
93		pmull_ghash_update(1, ctx->digest, ctx->buf, key, NULL);
94		kernel_neon_end();
95	}
96	put_unaligned_be64(ctx->digest[1], dst);
97	put_unaligned_be64(ctx->digest[0], dst + 8);
98
99	*ctx = (struct ghash_desc_ctx){};
100	return 0;
101}
102
103static int ghash_setkey(struct crypto_shash *tfm,
104			const u8 *inkey, unsigned int keylen)
105{
106	struct ghash_key *key = crypto_shash_ctx(tfm);
107	u64 a, b;
108
109	if (keylen != GHASH_BLOCK_SIZE) {
110		crypto_shash_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
111		return -EINVAL;
112	}
113
114	/* perform multiplication by 'x' in GF(2^128) */
115	b = get_unaligned_be64(inkey);
116	a = get_unaligned_be64(inkey + 8);
117
118	key->a = (a << 1) | (b >> 63);
119	key->b = (b << 1) | (a >> 63);
120
121	if (b >> 63)
122		key->b ^= 0xc200000000000000UL;
123
124	return 0;
125}
126
127static struct shash_alg ghash_alg = {
128	.digestsize	= GHASH_DIGEST_SIZE,
129	.init		= ghash_init,
130	.update		= ghash_update,
131	.final		= ghash_final,
132	.setkey		= ghash_setkey,
133	.descsize	= sizeof(struct ghash_desc_ctx),
134	.base		= {
135		.cra_name		= "ghash",
136		.cra_driver_name	= "ghash-ce",
137		.cra_priority		= 200,
138		.cra_flags		= CRYPTO_ALG_TYPE_SHASH,
139		.cra_blocksize		= GHASH_BLOCK_SIZE,
140		.cra_ctxsize		= sizeof(struct ghash_key),
141		.cra_module		= THIS_MODULE,
142	},
143};
144
145static int __init ghash_ce_mod_init(void)
146{
147	return crypto_register_shash(&ghash_alg);
148}
149
150static void __exit ghash_ce_mod_exit(void)
151{
152	crypto_unregister_shash(&ghash_alg);
153}
154
155module_cpu_feature_match(PMULL, ghash_ce_mod_init);
156module_exit(ghash_ce_mod_exit);
157