1/*
2 * crc32-arm64.c - CRC32 and CRC32C using optional ARMv8 instructions
3 *
4 * Module based on crypto/crc32c_generic.c
5 *
6 * CRC32 loop taken from Ed Nevill's Hadoop CRC patch
7 * http://mail-archives.apache.org/mod_mbox/hadoop-common-dev/201406.mbox/%3C1403687030.3355.19.camel%40localhost.localdomain%3E
8 *
9 * Using inline assembly instead of intrinsics in order to be backwards
10 * compatible with older compilers.
11 *
12 * Copyright (C) 2014 Linaro Ltd <yazen.ghannam@linaro.org>
13 *
14 * This program is free software; you can redistribute it and/or modify
15 * it under the terms of the GNU General Public License version 2 as
16 * published by the Free Software Foundation.
17 */
18
19#include <linux/unaligned/access_ok.h>
20#include <linux/cpufeature.h>
21#include <linux/init.h>
22#include <linux/kernel.h>
23#include <linux/module.h>
24#include <linux/string.h>
25
26#include <crypto/internal/hash.h>
27
28MODULE_AUTHOR("Yazen Ghannam <yazen.ghannam@linaro.org>");
29MODULE_DESCRIPTION("CRC32 and CRC32C using optional ARMv8 instructions");
30MODULE_LICENSE("GPL v2");
31
32#define CRC32X(crc, value) __asm__("crc32x %w[c], %w[c], %x[v]":[c]"+r"(crc):[v]"r"(value))
33#define CRC32W(crc, value) __asm__("crc32w %w[c], %w[c], %w[v]":[c]"+r"(crc):[v]"r"(value))
34#define CRC32H(crc, value) __asm__("crc32h %w[c], %w[c], %w[v]":[c]"+r"(crc):[v]"r"(value))
35#define CRC32B(crc, value) __asm__("crc32b %w[c], %w[c], %w[v]":[c]"+r"(crc):[v]"r"(value))
36#define CRC32CX(crc, value) __asm__("crc32cx %w[c], %w[c], %x[v]":[c]"+r"(crc):[v]"r"(value))
37#define CRC32CW(crc, value) __asm__("crc32cw %w[c], %w[c], %w[v]":[c]"+r"(crc):[v]"r"(value))
38#define CRC32CH(crc, value) __asm__("crc32ch %w[c], %w[c], %w[v]":[c]"+r"(crc):[v]"r"(value))
39#define CRC32CB(crc, value) __asm__("crc32cb %w[c], %w[c], %w[v]":[c]"+r"(crc):[v]"r"(value))
40
41static u32 crc32_arm64_le_hw(u32 crc, const u8 *p, unsigned int len)
42{
43	s64 length = len;
44
45	while ((length -= sizeof(u64)) >= 0) {
46		CRC32X(crc, get_unaligned_le64(p));
47		p += sizeof(u64);
48	}
49
50	/* The following is more efficient than the straight loop */
51	if (length & sizeof(u32)) {
52		CRC32W(crc, get_unaligned_le32(p));
53		p += sizeof(u32);
54	}
55	if (length & sizeof(u16)) {
56		CRC32H(crc, get_unaligned_le16(p));
57		p += sizeof(u16);
58	}
59	if (length & sizeof(u8))
60		CRC32B(crc, *p);
61
62	return crc;
63}
64
65static u32 crc32c_arm64_le_hw(u32 crc, const u8 *p, unsigned int len)
66{
67	s64 length = len;
68
69	while ((length -= sizeof(u64)) >= 0) {
70		CRC32CX(crc, get_unaligned_le64(p));
71		p += sizeof(u64);
72	}
73
74	/* The following is more efficient than the straight loop */
75	if (length & sizeof(u32)) {
76		CRC32CW(crc, get_unaligned_le32(p));
77		p += sizeof(u32);
78	}
79	if (length & sizeof(u16)) {
80		CRC32CH(crc, get_unaligned_le16(p));
81		p += sizeof(u16);
82	}
83	if (length & sizeof(u8))
84		CRC32CB(crc, *p);
85
86	return crc;
87}
88
89#define CHKSUM_BLOCK_SIZE	1
90#define CHKSUM_DIGEST_SIZE	4
91
92struct chksum_ctx {
93	u32 key;
94};
95
96struct chksum_desc_ctx {
97	u32 crc;
98};
99
100static int chksum_init(struct shash_desc *desc)
101{
102	struct chksum_ctx *mctx = crypto_shash_ctx(desc->tfm);
103	struct chksum_desc_ctx *ctx = shash_desc_ctx(desc);
104
105	ctx->crc = mctx->key;
106
107	return 0;
108}
109
110/*
111 * Setting the seed allows arbitrary accumulators and flexible XOR policy
112 * If your algorithm starts with ~0, then XOR with ~0 before you set
113 * the seed.
114 */
115static int chksum_setkey(struct crypto_shash *tfm, const u8 *key,
116			 unsigned int keylen)
117{
118	struct chksum_ctx *mctx = crypto_shash_ctx(tfm);
119
120	if (keylen != sizeof(mctx->key)) {
121		crypto_shash_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
122		return -EINVAL;
123	}
124	mctx->key = get_unaligned_le32(key);
125	return 0;
126}
127
128static int chksum_update(struct shash_desc *desc, const u8 *data,
129			 unsigned int length)
130{
131	struct chksum_desc_ctx *ctx = shash_desc_ctx(desc);
132
133	ctx->crc = crc32_arm64_le_hw(ctx->crc, data, length);
134	return 0;
135}
136
137static int chksumc_update(struct shash_desc *desc, const u8 *data,
138			 unsigned int length)
139{
140	struct chksum_desc_ctx *ctx = shash_desc_ctx(desc);
141
142	ctx->crc = crc32c_arm64_le_hw(ctx->crc, data, length);
143	return 0;
144}
145
146static int chksum_final(struct shash_desc *desc, u8 *out)
147{
148	struct chksum_desc_ctx *ctx = shash_desc_ctx(desc);
149
150	put_unaligned_le32(ctx->crc, out);
151	return 0;
152}
153
154static int chksumc_final(struct shash_desc *desc, u8 *out)
155{
156	struct chksum_desc_ctx *ctx = shash_desc_ctx(desc);
157
158	put_unaligned_le32(~ctx->crc, out);
159	return 0;
160}
161
162static int __chksum_finup(u32 crc, const u8 *data, unsigned int len, u8 *out)
163{
164	put_unaligned_le32(crc32_arm64_le_hw(crc, data, len), out);
165	return 0;
166}
167
168static int __chksumc_finup(u32 crc, const u8 *data, unsigned int len, u8 *out)
169{
170	put_unaligned_le32(~crc32c_arm64_le_hw(crc, data, len), out);
171	return 0;
172}
173
174static int chksum_finup(struct shash_desc *desc, const u8 *data,
175			unsigned int len, u8 *out)
176{
177	struct chksum_desc_ctx *ctx = shash_desc_ctx(desc);
178
179	return __chksum_finup(ctx->crc, data, len, out);
180}
181
182static int chksumc_finup(struct shash_desc *desc, const u8 *data,
183			unsigned int len, u8 *out)
184{
185	struct chksum_desc_ctx *ctx = shash_desc_ctx(desc);
186
187	return __chksumc_finup(ctx->crc, data, len, out);
188}
189
190static int chksum_digest(struct shash_desc *desc, const u8 *data,
191			 unsigned int length, u8 *out)
192{
193	struct chksum_ctx *mctx = crypto_shash_ctx(desc->tfm);
194
195	return __chksum_finup(mctx->key, data, length, out);
196}
197
198static int chksumc_digest(struct shash_desc *desc, const u8 *data,
199			 unsigned int length, u8 *out)
200{
201	struct chksum_ctx *mctx = crypto_shash_ctx(desc->tfm);
202
203	return __chksumc_finup(mctx->key, data, length, out);
204}
205
206static int crc32_cra_init(struct crypto_tfm *tfm)
207{
208	struct chksum_ctx *mctx = crypto_tfm_ctx(tfm);
209
210	mctx->key = 0;
211	return 0;
212}
213
214static int crc32c_cra_init(struct crypto_tfm *tfm)
215{
216	struct chksum_ctx *mctx = crypto_tfm_ctx(tfm);
217
218	mctx->key = ~0;
219	return 0;
220}
221
222static struct shash_alg crc32_alg = {
223	.digestsize		=	CHKSUM_DIGEST_SIZE,
224	.setkey			=	chksum_setkey,
225	.init			=	chksum_init,
226	.update			=	chksum_update,
227	.final			=	chksum_final,
228	.finup			=	chksum_finup,
229	.digest			=	chksum_digest,
230	.descsize		=	sizeof(struct chksum_desc_ctx),
231	.base			=	{
232		.cra_name		=	"crc32",
233		.cra_driver_name	=	"crc32-arm64-hw",
234		.cra_priority		=	300,
235		.cra_blocksize		=	CHKSUM_BLOCK_SIZE,
236		.cra_alignmask		=	0,
237		.cra_ctxsize		=	sizeof(struct chksum_ctx),
238		.cra_module		=	THIS_MODULE,
239		.cra_init		=	crc32_cra_init,
240	}
241};
242
243static struct shash_alg crc32c_alg = {
244	.digestsize		=	CHKSUM_DIGEST_SIZE,
245	.setkey			=	chksum_setkey,
246	.init			=	chksum_init,
247	.update			=	chksumc_update,
248	.final			=	chksumc_final,
249	.finup			=	chksumc_finup,
250	.digest			=	chksumc_digest,
251	.descsize		=	sizeof(struct chksum_desc_ctx),
252	.base			=	{
253		.cra_name		=	"crc32c",
254		.cra_driver_name	=	"crc32c-arm64-hw",
255		.cra_priority		=	300,
256		.cra_blocksize		=	CHKSUM_BLOCK_SIZE,
257		.cra_alignmask		=	0,
258		.cra_ctxsize		=	sizeof(struct chksum_ctx),
259		.cra_module		=	THIS_MODULE,
260		.cra_init		=	crc32c_cra_init,
261	}
262};
263
264static int __init crc32_mod_init(void)
265{
266	int err;
267
268	err = crypto_register_shash(&crc32_alg);
269
270	if (err)
271		return err;
272
273	err = crypto_register_shash(&crc32c_alg);
274
275	if (err) {
276		crypto_unregister_shash(&crc32_alg);
277		return err;
278	}
279
280	return 0;
281}
282
283static void __exit crc32_mod_exit(void)
284{
285	crypto_unregister_shash(&crc32_alg);
286	crypto_unregister_shash(&crc32c_alg);
287}
288
289module_cpu_feature_match(CRC32, crc32_mod_init);
290module_exit(crc32_mod_exit);
291