50 	struct ahash_request req; /* must be last member */  member
56 	struct ablkcipher_request req; /* must be last member */  member
76 static inline void async_done_continue(struct aead_request *req, int err,  in async_done_continue()  argument
80 		err = cont(req);  in async_done_continue()
83 		aead_request_complete(req, err);  in async_done_continue()
86 static void chacha_iv(u8 *iv, struct aead_request *req, u32 icb)  in chacha_iv()  argument
88 	struct chachapoly_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req));  in chacha_iv()
93 	memcpy(iv + sizeof(leicb) + ctx->saltlen, req->iv,  in chacha_iv()
97 static int poly_verify_tag(struct aead_request *req)  in poly_verify_tag()  argument
99 	struct chachapoly_req_ctx *rctx = aead_request_ctx(req);  in poly_verify_tag()
102 	scatterwalk_map_and_copy(tag, req->src,  in poly_verify_tag()
103 				 req->assoclen + rctx->cryptlen,  in poly_verify_tag()
110 static int poly_copy_tag(struct aead_request *req)  in poly_copy_tag()  argument
112 	struct chachapoly_req_ctx *rctx = aead_request_ctx(req);  in poly_copy_tag()
114 	scatterwalk_map_and_copy(rctx->tag, req->dst,  in poly_copy_tag()
115 				 req->assoclen + rctx->cryptlen,  in poly_copy_tag()
125 static int chacha_decrypt(struct aead_request *req)  in chacha_decrypt()  argument
127 	struct chachapoly_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req));  in chacha_decrypt()
128 	struct chachapoly_req_ctx *rctx = aead_request_ctx(req);  in chacha_decrypt()
133 	chacha_iv(creq->iv, req, 1);  in chacha_decrypt()
136 	src = scatterwalk_ffwd(rctx->src, req->src, req->assoclen);  in chacha_decrypt()
139 	if (req->src != req->dst) {  in chacha_decrypt()
141 		dst = scatterwalk_ffwd(rctx->dst, req->dst, req->assoclen);  in chacha_decrypt()
144 	ablkcipher_request_set_callback(&creq->req, aead_request_flags(req),  in chacha_decrypt()
145 					chacha_decrypt_done, req);  in chacha_decrypt()
146 	ablkcipher_request_set_tfm(&creq->req, ctx->chacha);  in chacha_decrypt()
147 	ablkcipher_request_set_crypt(&creq->req, src, dst,  in chacha_decrypt()
149 	err = crypto_ablkcipher_decrypt(&creq->req);  in chacha_decrypt()
153 	return poly_verify_tag(req);  in chacha_decrypt()
156 static int poly_tail_continue(struct aead_request *req)  in poly_tail_continue()  argument
158 	struct chachapoly_req_ctx *rctx = aead_request_ctx(req);  in poly_tail_continue()
160 	if (rctx->cryptlen == req->cryptlen) /* encrypting */  in poly_tail_continue()
161 		return poly_copy_tag(req);  in poly_tail_continue()
163 	return chacha_decrypt(req);  in poly_tail_continue()
171 static int poly_tail(struct aead_request *req)  in poly_tail()  argument
173 	struct crypto_aead *tfm = crypto_aead_reqtfm(req);  in poly_tail()
175 	struct chachapoly_req_ctx *rctx = aead_request_ctx(req);  in poly_tail()
187 	ahash_request_set_callback(&preq->req, aead_request_flags(req),  in poly_tail()
188 				   poly_tail_done, req);  in poly_tail()
189 	ahash_request_set_tfm(&preq->req, ctx->poly);  in poly_tail()
190 	ahash_request_set_crypt(&preq->req, preq->src,  in poly_tail()
193 	err = crypto_ahash_finup(&preq->req);  in poly_tail()
197 	return poly_tail_continue(req);  in poly_tail()
205 static int poly_cipherpad(struct aead_request *req)  in poly_cipherpad()  argument
207 	struct chachapoly_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req));  in poly_cipherpad()
208 	struct chachapoly_req_ctx *rctx = aead_request_ctx(req);  in poly_cipherpad()
218 	ahash_request_set_callback(&preq->req, aead_request_flags(req),  in poly_cipherpad()
219 				   poly_cipherpad_done, req);  in poly_cipherpad()
220 	ahash_request_set_tfm(&preq->req, ctx->poly);  in poly_cipherpad()
221 	ahash_request_set_crypt(&preq->req, preq->src, NULL, padlen);  in poly_cipherpad()
223 	err = crypto_ahash_update(&preq->req);  in poly_cipherpad()
227 	return poly_tail(req);  in poly_cipherpad()
235 static int poly_cipher(struct aead_request *req)  in poly_cipher()  argument
237 	struct chachapoly_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req));  in poly_cipher()
238 	struct chachapoly_req_ctx *rctx = aead_request_ctx(req);  in poly_cipher()
240 	struct scatterlist *crypt = req->src;  in poly_cipher()
243 	if (rctx->cryptlen == req->cryptlen) /* encrypting */  in poly_cipher()
244 		crypt = req->dst;  in poly_cipher()
247 	crypt = scatterwalk_ffwd(rctx->src, crypt, req->assoclen);  in poly_cipher()
249 	ahash_request_set_callback(&preq->req, aead_request_flags(req),  in poly_cipher()
250 				   poly_cipher_done, req);  in poly_cipher()
251 	ahash_request_set_tfm(&preq->req, ctx->poly);  in poly_cipher()
252 	ahash_request_set_crypt(&preq->req, crypt, NULL, rctx->cryptlen);  in poly_cipher()
254 	err = crypto_ahash_update(&preq->req);  in poly_cipher()
258 	return poly_cipherpad(req);  in poly_cipher()
266 static int poly_adpad(struct aead_request *req)  in poly_adpad()  argument
268 	struct chachapoly_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req));  in poly_adpad()
269 	struct chachapoly_req_ctx *rctx = aead_request_ctx(req);  in poly_adpad()
279 	ahash_request_set_callback(&preq->req, aead_request_flags(req),  in poly_adpad()
280 				   poly_adpad_done, req);  in poly_adpad()
281 	ahash_request_set_tfm(&preq->req, ctx->poly);  in poly_adpad()
282 	ahash_request_set_crypt(&preq->req, preq->src, NULL, padlen);  in poly_adpad()
284 	err = crypto_ahash_update(&preq->req);  in poly_adpad()
288 	return poly_cipher(req);  in poly_adpad()
296 static int poly_ad(struct aead_request *req)  in poly_ad()  argument
298 	struct chachapoly_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req));  in poly_ad()
299 	struct chachapoly_req_ctx *rctx = aead_request_ctx(req);  in poly_ad()
303 	ahash_request_set_callback(&preq->req, aead_request_flags(req),  in poly_ad()
304 				   poly_ad_done, req);  in poly_ad()
305 	ahash_request_set_tfm(&preq->req, ctx->poly);  in poly_ad()
306 	ahash_request_set_crypt(&preq->req, req->src, NULL, rctx->assoclen);  in poly_ad()
308 	err = crypto_ahash_update(&preq->req);  in poly_ad()
312 	return poly_adpad(req);  in poly_ad()
320 static int poly_setkey(struct aead_request *req)  in poly_setkey()  argument
322 	struct chachapoly_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req));  in poly_setkey()
323 	struct chachapoly_req_ctx *rctx = aead_request_ctx(req);  in poly_setkey()
330 	ahash_request_set_callback(&preq->req, aead_request_flags(req),  in poly_setkey()
331 				   poly_setkey_done, req);  in poly_setkey()
332 	ahash_request_set_tfm(&preq->req, ctx->poly);  in poly_setkey()
333 	ahash_request_set_crypt(&preq->req, preq->src, NULL, sizeof(rctx->key));  in poly_setkey()
335 	err = crypto_ahash_update(&preq->req);  in poly_setkey()
339 	return poly_ad(req);  in poly_setkey()
347 static int poly_init(struct aead_request *req)  in poly_init()  argument
349 	struct chachapoly_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req));  in poly_init()
350 	struct chachapoly_req_ctx *rctx = aead_request_ctx(req);  in poly_init()
354 	ahash_request_set_callback(&preq->req, aead_request_flags(req),  in poly_init()
355 				   poly_init_done, req);  in poly_init()
356 	ahash_request_set_tfm(&preq->req, ctx->poly);  in poly_init()
358 	err = crypto_ahash_init(&preq->req);  in poly_init()
362 	return poly_setkey(req);  in poly_init()
370 static int poly_genkey(struct aead_request *req)  in poly_genkey()  argument
372 	struct crypto_aead *tfm = crypto_aead_reqtfm(req);  in poly_genkey()
374 	struct chachapoly_req_ctx *rctx = aead_request_ctx(req);  in poly_genkey()
378 	rctx->assoclen = req->assoclen;  in poly_genkey()
390 	chacha_iv(creq->iv, req, 0);  in poly_genkey()
392 	ablkcipher_request_set_callback(&creq->req, aead_request_flags(req),  in poly_genkey()
393 					poly_genkey_done, req);  in poly_genkey()
394 	ablkcipher_request_set_tfm(&creq->req, ctx->chacha);  in poly_genkey()
395 	ablkcipher_request_set_crypt(&creq->req, creq->src, creq->src,  in poly_genkey()
398 	err = crypto_ablkcipher_decrypt(&creq->req);  in poly_genkey()
402 	return poly_init(req);  in poly_genkey()
410 static int chacha_encrypt(struct aead_request *req)  in chacha_encrypt()  argument
412 	struct chachapoly_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req));  in chacha_encrypt()
413 	struct chachapoly_req_ctx *rctx = aead_request_ctx(req);  in chacha_encrypt()
418 	chacha_iv(creq->iv, req, 1);  in chacha_encrypt()
421 	src = scatterwalk_ffwd(rctx->src, req->src, req->assoclen);  in chacha_encrypt()
424 	if (req->src != req->dst) {  in chacha_encrypt()
426 		dst = scatterwalk_ffwd(rctx->dst, req->dst, req->assoclen);  in chacha_encrypt()
429 	ablkcipher_request_set_callback(&creq->req, aead_request_flags(req),  in chacha_encrypt()
430 					chacha_encrypt_done, req);  in chacha_encrypt()
431 	ablkcipher_request_set_tfm(&creq->req, ctx->chacha);  in chacha_encrypt()
432 	ablkcipher_request_set_crypt(&creq->req, src, dst,  in chacha_encrypt()
433 				     req->cryptlen, creq->iv);  in chacha_encrypt()
434 	err = crypto_ablkcipher_encrypt(&creq->req);  in chacha_encrypt()
438 	return poly_genkey(req);  in chacha_encrypt()
441 static int chachapoly_encrypt(struct aead_request *req)  in chachapoly_encrypt()  argument
443 	struct chachapoly_req_ctx *rctx = aead_request_ctx(req);  in chachapoly_encrypt()
445 	rctx->cryptlen = req->cryptlen;  in chachapoly_encrypt()
459 	return chacha_encrypt(req);  in chachapoly_encrypt()
462 static int chachapoly_decrypt(struct aead_request *req)  in chachapoly_decrypt()  argument
464 	struct chachapoly_req_ctx *rctx = aead_request_ctx(req);  in chachapoly_decrypt()
466 	rctx->cryptlen = req->cryptlen - POLY1305_DIGEST_SIZE;  in chachapoly_decrypt()
480 	return poly_genkey(req);  in chachapoly_decrypt()
542 		max(offsetof(struct chacha_req, req) +  in chachapoly_init()
545 		    offsetof(struct poly_req, req) +  in chachapoly_init()