1/* 2 * Software multibuffer async crypto daemon. 3 * 4 * Copyright (c) 2014 Tim Chen <tim.c.chen@linux.intel.com> 5 * 6 * Adapted from crypto daemon. 7 * 8 * This program is free software; you can redistribute it and/or modify it 9 * under the terms of the GNU General Public License as published by the Free 10 * Software Foundation; either version 2 of the License, or (at your option) 11 * any later version. 12 * 13 */ 14 15#include <crypto/algapi.h> 16#include <crypto/internal/hash.h> 17#include <crypto/internal/aead.h> 18#include <crypto/mcryptd.h> 19#include <crypto/crypto_wq.h> 20#include <linux/err.h> 21#include <linux/init.h> 22#include <linux/kernel.h> 23#include <linux/list.h> 24#include <linux/module.h> 25#include <linux/scatterlist.h> 26#include <linux/sched.h> 27#include <linux/slab.h> 28#include <linux/hardirq.h> 29 30#define MCRYPTD_MAX_CPU_QLEN 100 31#define MCRYPTD_BATCH 9 32 33static void *mcryptd_alloc_instance(struct crypto_alg *alg, unsigned int head, 34 unsigned int tail); 35 36struct mcryptd_flush_list { 37 struct list_head list; 38 struct mutex lock; 39}; 40 41static struct mcryptd_flush_list __percpu *mcryptd_flist; 42 43struct hashd_instance_ctx { 44 struct crypto_shash_spawn spawn; 45 struct mcryptd_queue *queue; 46}; 47 48static void mcryptd_queue_worker(struct work_struct *work); 49 50void mcryptd_arm_flusher(struct mcryptd_alg_cstate *cstate, unsigned long delay) 51{ 52 struct mcryptd_flush_list *flist; 53 54 if (!cstate->flusher_engaged) { 55 /* put the flusher on the flush list */ 56 flist = per_cpu_ptr(mcryptd_flist, smp_processor_id()); 57 mutex_lock(&flist->lock); 58 list_add_tail(&cstate->flush_list, &flist->list); 59 cstate->flusher_engaged = true; 60 cstate->next_flush = jiffies + delay; 61 queue_delayed_work_on(smp_processor_id(), kcrypto_wq, 62 &cstate->flush, delay); 63 mutex_unlock(&flist->lock); 64 } 65} 66EXPORT_SYMBOL(mcryptd_arm_flusher); 67 68static int mcryptd_init_queue(struct mcryptd_queue *queue, 69 unsigned int max_cpu_qlen) 70{ 71 int cpu; 72 struct mcryptd_cpu_queue *cpu_queue; 73 74 queue->cpu_queue = alloc_percpu(struct mcryptd_cpu_queue); 75 pr_debug("mqueue:%p mcryptd_cpu_queue %p\n", queue, queue->cpu_queue); 76 if (!queue->cpu_queue) 77 return -ENOMEM; 78 for_each_possible_cpu(cpu) { 79 cpu_queue = per_cpu_ptr(queue->cpu_queue, cpu); 80 pr_debug("cpu_queue #%d %p\n", cpu, queue->cpu_queue); 81 crypto_init_queue(&cpu_queue->queue, max_cpu_qlen); 82 INIT_WORK(&cpu_queue->work, mcryptd_queue_worker); 83 } 84 return 0; 85} 86 87static void mcryptd_fini_queue(struct mcryptd_queue *queue) 88{ 89 int cpu; 90 struct mcryptd_cpu_queue *cpu_queue; 91 92 for_each_possible_cpu(cpu) { 93 cpu_queue = per_cpu_ptr(queue->cpu_queue, cpu); 94 BUG_ON(cpu_queue->queue.qlen); 95 } 96 free_percpu(queue->cpu_queue); 97} 98 99static int mcryptd_enqueue_request(struct mcryptd_queue *queue, 100 struct crypto_async_request *request, 101 struct mcryptd_hash_request_ctx *rctx) 102{ 103 int cpu, err; 104 struct mcryptd_cpu_queue *cpu_queue; 105 106 cpu = get_cpu(); 107 cpu_queue = this_cpu_ptr(queue->cpu_queue); 108 rctx->tag.cpu = cpu; 109 110 err = crypto_enqueue_request(&cpu_queue->queue, request); 111 pr_debug("enqueue request: cpu %d cpu_queue %p request %p\n", 112 cpu, cpu_queue, request); 113 queue_work_on(cpu, kcrypto_wq, &cpu_queue->work); 114 put_cpu(); 115 116 return err; 117} 118 119/* 120 * Try to opportunisticlly flush the partially completed jobs if 121 * crypto daemon is the only task running. 122 */ 123static void mcryptd_opportunistic_flush(void) 124{ 125 struct mcryptd_flush_list *flist; 126 struct mcryptd_alg_cstate *cstate; 127 128 flist = per_cpu_ptr(mcryptd_flist, smp_processor_id()); 129 while (single_task_running()) { 130 mutex_lock(&flist->lock); 131 if (list_empty(&flist->list)) { 132 mutex_unlock(&flist->lock); 133 return; 134 } 135 cstate = list_entry(flist->list.next, 136 struct mcryptd_alg_cstate, flush_list); 137 if (!cstate->flusher_engaged) { 138 mutex_unlock(&flist->lock); 139 return; 140 } 141 list_del(&cstate->flush_list); 142 cstate->flusher_engaged = false; 143 mutex_unlock(&flist->lock); 144 cstate->alg_state->flusher(cstate); 145 } 146} 147 148/* 149 * Called in workqueue context, do one real cryption work (via 150 * req->complete) and reschedule itself if there are more work to 151 * do. 152 */ 153static void mcryptd_queue_worker(struct work_struct *work) 154{ 155 struct mcryptd_cpu_queue *cpu_queue; 156 struct crypto_async_request *req, *backlog; 157 int i; 158 159 /* 160 * Need to loop through more than once for multi-buffer to 161 * be effective. 162 */ 163 164 cpu_queue = container_of(work, struct mcryptd_cpu_queue, work); 165 i = 0; 166 while (i < MCRYPTD_BATCH || single_task_running()) { 167 /* 168 * preempt_disable/enable is used to prevent 169 * being preempted by mcryptd_enqueue_request() 170 */ 171 local_bh_disable(); 172 preempt_disable(); 173 backlog = crypto_get_backlog(&cpu_queue->queue); 174 req = crypto_dequeue_request(&cpu_queue->queue); 175 preempt_enable(); 176 local_bh_enable(); 177 178 if (!req) { 179 mcryptd_opportunistic_flush(); 180 return; 181 } 182 183 if (backlog) 184 backlog->complete(backlog, -EINPROGRESS); 185 req->complete(req, 0); 186 if (!cpu_queue->queue.qlen) 187 return; 188 ++i; 189 } 190 if (cpu_queue->queue.qlen) 191 queue_work(kcrypto_wq, &cpu_queue->work); 192} 193 194void mcryptd_flusher(struct work_struct *__work) 195{ 196 struct mcryptd_alg_cstate *alg_cpu_state; 197 struct mcryptd_alg_state *alg_state; 198 struct mcryptd_flush_list *flist; 199 int cpu; 200 201 cpu = smp_processor_id(); 202 alg_cpu_state = container_of(to_delayed_work(__work), 203 struct mcryptd_alg_cstate, flush); 204 alg_state = alg_cpu_state->alg_state; 205 if (alg_cpu_state->cpu != cpu) 206 pr_debug("mcryptd error: work on cpu %d, should be cpu %d\n", 207 cpu, alg_cpu_state->cpu); 208 209 if (alg_cpu_state->flusher_engaged) { 210 flist = per_cpu_ptr(mcryptd_flist, cpu); 211 mutex_lock(&flist->lock); 212 list_del(&alg_cpu_state->flush_list); 213 alg_cpu_state->flusher_engaged = false; 214 mutex_unlock(&flist->lock); 215 alg_state->flusher(alg_cpu_state); 216 } 217} 218EXPORT_SYMBOL_GPL(mcryptd_flusher); 219 220static inline struct mcryptd_queue *mcryptd_get_queue(struct crypto_tfm *tfm) 221{ 222 struct crypto_instance *inst = crypto_tfm_alg_instance(tfm); 223 struct mcryptd_instance_ctx *ictx = crypto_instance_ctx(inst); 224 225 return ictx->queue; 226} 227 228static void *mcryptd_alloc_instance(struct crypto_alg *alg, unsigned int head, 229 unsigned int tail) 230{ 231 char *p; 232 struct crypto_instance *inst; 233 int err; 234 235 p = kzalloc(head + sizeof(*inst) + tail, GFP_KERNEL); 236 if (!p) 237 return ERR_PTR(-ENOMEM); 238 239 inst = (void *)(p + head); 240 241 err = -ENAMETOOLONG; 242 if (snprintf(inst->alg.cra_driver_name, CRYPTO_MAX_ALG_NAME, 243 "mcryptd(%s)", alg->cra_driver_name) >= CRYPTO_MAX_ALG_NAME) 244 goto out_free_inst; 245 246 memcpy(inst->alg.cra_name, alg->cra_name, CRYPTO_MAX_ALG_NAME); 247 248 inst->alg.cra_priority = alg->cra_priority + 50; 249 inst->alg.cra_blocksize = alg->cra_blocksize; 250 inst->alg.cra_alignmask = alg->cra_alignmask; 251 252out: 253 return p; 254 255out_free_inst: 256 kfree(p); 257 p = ERR_PTR(err); 258 goto out; 259} 260 261static inline void mcryptd_check_internal(struct rtattr **tb, u32 *type, 262 u32 *mask) 263{ 264 struct crypto_attr_type *algt; 265 266 algt = crypto_get_attr_type(tb); 267 if (IS_ERR(algt)) 268 return; 269 if ((algt->type & CRYPTO_ALG_INTERNAL)) 270 *type |= CRYPTO_ALG_INTERNAL; 271 if ((algt->mask & CRYPTO_ALG_INTERNAL)) 272 *mask |= CRYPTO_ALG_INTERNAL; 273} 274 275static int mcryptd_hash_init_tfm(struct crypto_tfm *tfm) 276{ 277 struct crypto_instance *inst = crypto_tfm_alg_instance(tfm); 278 struct hashd_instance_ctx *ictx = crypto_instance_ctx(inst); 279 struct crypto_shash_spawn *spawn = &ictx->spawn; 280 struct mcryptd_hash_ctx *ctx = crypto_tfm_ctx(tfm); 281 struct crypto_shash *hash; 282 283 hash = crypto_spawn_shash(spawn); 284 if (IS_ERR(hash)) 285 return PTR_ERR(hash); 286 287 ctx->child = hash; 288 crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm), 289 sizeof(struct mcryptd_hash_request_ctx) + 290 crypto_shash_descsize(hash)); 291 return 0; 292} 293 294static void mcryptd_hash_exit_tfm(struct crypto_tfm *tfm) 295{ 296 struct mcryptd_hash_ctx *ctx = crypto_tfm_ctx(tfm); 297 298 crypto_free_shash(ctx->child); 299} 300 301static int mcryptd_hash_setkey(struct crypto_ahash *parent, 302 const u8 *key, unsigned int keylen) 303{ 304 struct mcryptd_hash_ctx *ctx = crypto_ahash_ctx(parent); 305 struct crypto_shash *child = ctx->child; 306 int err; 307 308 crypto_shash_clear_flags(child, CRYPTO_TFM_REQ_MASK); 309 crypto_shash_set_flags(child, crypto_ahash_get_flags(parent) & 310 CRYPTO_TFM_REQ_MASK); 311 err = crypto_shash_setkey(child, key, keylen); 312 crypto_ahash_set_flags(parent, crypto_shash_get_flags(child) & 313 CRYPTO_TFM_RES_MASK); 314 return err; 315} 316 317static int mcryptd_hash_enqueue(struct ahash_request *req, 318 crypto_completion_t complete) 319{ 320 int ret; 321 322 struct mcryptd_hash_request_ctx *rctx = ahash_request_ctx(req); 323 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 324 struct mcryptd_queue *queue = 325 mcryptd_get_queue(crypto_ahash_tfm(tfm)); 326 327 rctx->complete = req->base.complete; 328 req->base.complete = complete; 329 330 ret = mcryptd_enqueue_request(queue, &req->base, rctx); 331 332 return ret; 333} 334 335static void mcryptd_hash_init(struct crypto_async_request *req_async, int err) 336{ 337 struct mcryptd_hash_ctx *ctx = crypto_tfm_ctx(req_async->tfm); 338 struct crypto_shash *child = ctx->child; 339 struct ahash_request *req = ahash_request_cast(req_async); 340 struct mcryptd_hash_request_ctx *rctx = ahash_request_ctx(req); 341 struct shash_desc *desc = &rctx->desc; 342 343 if (unlikely(err == -EINPROGRESS)) 344 goto out; 345 346 desc->tfm = child; 347 desc->flags = CRYPTO_TFM_REQ_MAY_SLEEP; 348 349 err = crypto_shash_init(desc); 350 351 req->base.complete = rctx->complete; 352 353out: 354 local_bh_disable(); 355 rctx->complete(&req->base, err); 356 local_bh_enable(); 357} 358 359static int mcryptd_hash_init_enqueue(struct ahash_request *req) 360{ 361 return mcryptd_hash_enqueue(req, mcryptd_hash_init); 362} 363 364static void mcryptd_hash_update(struct crypto_async_request *req_async, int err) 365{ 366 struct ahash_request *req = ahash_request_cast(req_async); 367 struct mcryptd_hash_request_ctx *rctx = ahash_request_ctx(req); 368 369 if (unlikely(err == -EINPROGRESS)) 370 goto out; 371 372 err = shash_ahash_mcryptd_update(req, &rctx->desc); 373 if (err) { 374 req->base.complete = rctx->complete; 375 goto out; 376 } 377 378 return; 379out: 380 local_bh_disable(); 381 rctx->complete(&req->base, err); 382 local_bh_enable(); 383} 384 385static int mcryptd_hash_update_enqueue(struct ahash_request *req) 386{ 387 return mcryptd_hash_enqueue(req, mcryptd_hash_update); 388} 389 390static void mcryptd_hash_final(struct crypto_async_request *req_async, int err) 391{ 392 struct ahash_request *req = ahash_request_cast(req_async); 393 struct mcryptd_hash_request_ctx *rctx = ahash_request_ctx(req); 394 395 if (unlikely(err == -EINPROGRESS)) 396 goto out; 397 398 err = shash_ahash_mcryptd_final(req, &rctx->desc); 399 if (err) { 400 req->base.complete = rctx->complete; 401 goto out; 402 } 403 404 return; 405out: 406 local_bh_disable(); 407 rctx->complete(&req->base, err); 408 local_bh_enable(); 409} 410 411static int mcryptd_hash_final_enqueue(struct ahash_request *req) 412{ 413 return mcryptd_hash_enqueue(req, mcryptd_hash_final); 414} 415 416static void mcryptd_hash_finup(struct crypto_async_request *req_async, int err) 417{ 418 struct ahash_request *req = ahash_request_cast(req_async); 419 struct mcryptd_hash_request_ctx *rctx = ahash_request_ctx(req); 420 421 if (unlikely(err == -EINPROGRESS)) 422 goto out; 423 424 err = shash_ahash_mcryptd_finup(req, &rctx->desc); 425 426 if (err) { 427 req->base.complete = rctx->complete; 428 goto out; 429 } 430 431 return; 432out: 433 local_bh_disable(); 434 rctx->complete(&req->base, err); 435 local_bh_enable(); 436} 437 438static int mcryptd_hash_finup_enqueue(struct ahash_request *req) 439{ 440 return mcryptd_hash_enqueue(req, mcryptd_hash_finup); 441} 442 443static void mcryptd_hash_digest(struct crypto_async_request *req_async, int err) 444{ 445 struct mcryptd_hash_ctx *ctx = crypto_tfm_ctx(req_async->tfm); 446 struct crypto_shash *child = ctx->child; 447 struct ahash_request *req = ahash_request_cast(req_async); 448 struct mcryptd_hash_request_ctx *rctx = ahash_request_ctx(req); 449 struct shash_desc *desc = &rctx->desc; 450 451 if (unlikely(err == -EINPROGRESS)) 452 goto out; 453 454 desc->tfm = child; 455 desc->flags = CRYPTO_TFM_REQ_MAY_SLEEP; /* check this again */ 456 457 err = shash_ahash_mcryptd_digest(req, desc); 458 459 if (err) { 460 req->base.complete = rctx->complete; 461 goto out; 462 } 463 464 return; 465out: 466 local_bh_disable(); 467 rctx->complete(&req->base, err); 468 local_bh_enable(); 469} 470 471static int mcryptd_hash_digest_enqueue(struct ahash_request *req) 472{ 473 return mcryptd_hash_enqueue(req, mcryptd_hash_digest); 474} 475 476static int mcryptd_hash_export(struct ahash_request *req, void *out) 477{ 478 struct mcryptd_hash_request_ctx *rctx = ahash_request_ctx(req); 479 480 return crypto_shash_export(&rctx->desc, out); 481} 482 483static int mcryptd_hash_import(struct ahash_request *req, const void *in) 484{ 485 struct mcryptd_hash_request_ctx *rctx = ahash_request_ctx(req); 486 487 return crypto_shash_import(&rctx->desc, in); 488} 489 490static int mcryptd_create_hash(struct crypto_template *tmpl, struct rtattr **tb, 491 struct mcryptd_queue *queue) 492{ 493 struct hashd_instance_ctx *ctx; 494 struct ahash_instance *inst; 495 struct shash_alg *salg; 496 struct crypto_alg *alg; 497 u32 type = 0; 498 u32 mask = 0; 499 int err; 500 501 mcryptd_check_internal(tb, &type, &mask); 502 503 salg = shash_attr_alg(tb[1], type, mask); 504 if (IS_ERR(salg)) 505 return PTR_ERR(salg); 506 507 alg = &salg->base; 508 pr_debug("crypto: mcryptd hash alg: %s\n", alg->cra_name); 509 inst = mcryptd_alloc_instance(alg, ahash_instance_headroom(), 510 sizeof(*ctx)); 511 err = PTR_ERR(inst); 512 if (IS_ERR(inst)) 513 goto out_put_alg; 514 515 ctx = ahash_instance_ctx(inst); 516 ctx->queue = queue; 517 518 err = crypto_init_shash_spawn(&ctx->spawn, salg, 519 ahash_crypto_instance(inst)); 520 if (err) 521 goto out_free_inst; 522 523 type = CRYPTO_ALG_ASYNC; 524 if (alg->cra_flags & CRYPTO_ALG_INTERNAL) 525 type |= CRYPTO_ALG_INTERNAL; 526 inst->alg.halg.base.cra_flags = type; 527 528 inst->alg.halg.digestsize = salg->digestsize; 529 inst->alg.halg.base.cra_ctxsize = sizeof(struct mcryptd_hash_ctx); 530 531 inst->alg.halg.base.cra_init = mcryptd_hash_init_tfm; 532 inst->alg.halg.base.cra_exit = mcryptd_hash_exit_tfm; 533 534 inst->alg.init = mcryptd_hash_init_enqueue; 535 inst->alg.update = mcryptd_hash_update_enqueue; 536 inst->alg.final = mcryptd_hash_final_enqueue; 537 inst->alg.finup = mcryptd_hash_finup_enqueue; 538 inst->alg.export = mcryptd_hash_export; 539 inst->alg.import = mcryptd_hash_import; 540 inst->alg.setkey = mcryptd_hash_setkey; 541 inst->alg.digest = mcryptd_hash_digest_enqueue; 542 543 err = ahash_register_instance(tmpl, inst); 544 if (err) { 545 crypto_drop_shash(&ctx->spawn); 546out_free_inst: 547 kfree(inst); 548 } 549 550out_put_alg: 551 crypto_mod_put(alg); 552 return err; 553} 554 555static struct mcryptd_queue mqueue; 556 557static int mcryptd_create(struct crypto_template *tmpl, struct rtattr **tb) 558{ 559 struct crypto_attr_type *algt; 560 561 algt = crypto_get_attr_type(tb); 562 if (IS_ERR(algt)) 563 return PTR_ERR(algt); 564 565 switch (algt->type & algt->mask & CRYPTO_ALG_TYPE_MASK) { 566 case CRYPTO_ALG_TYPE_DIGEST: 567 return mcryptd_create_hash(tmpl, tb, &mqueue); 568 break; 569 } 570 571 return -EINVAL; 572} 573 574static void mcryptd_free(struct crypto_instance *inst) 575{ 576 struct mcryptd_instance_ctx *ctx = crypto_instance_ctx(inst); 577 struct hashd_instance_ctx *hctx = crypto_instance_ctx(inst); 578 579 switch (inst->alg.cra_flags & CRYPTO_ALG_TYPE_MASK) { 580 case CRYPTO_ALG_TYPE_AHASH: 581 crypto_drop_shash(&hctx->spawn); 582 kfree(ahash_instance(inst)); 583 return; 584 default: 585 crypto_drop_spawn(&ctx->spawn); 586 kfree(inst); 587 } 588} 589 590static struct crypto_template mcryptd_tmpl = { 591 .name = "mcryptd", 592 .create = mcryptd_create, 593 .free = mcryptd_free, 594 .module = THIS_MODULE, 595}; 596 597struct mcryptd_ahash *mcryptd_alloc_ahash(const char *alg_name, 598 u32 type, u32 mask) 599{ 600 char mcryptd_alg_name[CRYPTO_MAX_ALG_NAME]; 601 struct crypto_ahash *tfm; 602 603 if (snprintf(mcryptd_alg_name, CRYPTO_MAX_ALG_NAME, 604 "mcryptd(%s)", alg_name) >= CRYPTO_MAX_ALG_NAME) 605 return ERR_PTR(-EINVAL); 606 tfm = crypto_alloc_ahash(mcryptd_alg_name, type, mask); 607 if (IS_ERR(tfm)) 608 return ERR_CAST(tfm); 609 if (tfm->base.__crt_alg->cra_module != THIS_MODULE) { 610 crypto_free_ahash(tfm); 611 return ERR_PTR(-EINVAL); 612 } 613 614 return __mcryptd_ahash_cast(tfm); 615} 616EXPORT_SYMBOL_GPL(mcryptd_alloc_ahash); 617 618int shash_ahash_mcryptd_digest(struct ahash_request *req, 619 struct shash_desc *desc) 620{ 621 int err; 622 623 err = crypto_shash_init(desc) ?: 624 shash_ahash_mcryptd_finup(req, desc); 625 626 return err; 627} 628EXPORT_SYMBOL_GPL(shash_ahash_mcryptd_digest); 629 630int shash_ahash_mcryptd_update(struct ahash_request *req, 631 struct shash_desc *desc) 632{ 633 struct crypto_shash *tfm = desc->tfm; 634 struct shash_alg *shash = crypto_shash_alg(tfm); 635 636 /* alignment is to be done by multi-buffer crypto algorithm if needed */ 637 638 return shash->update(desc, NULL, 0); 639} 640EXPORT_SYMBOL_GPL(shash_ahash_mcryptd_update); 641 642int shash_ahash_mcryptd_finup(struct ahash_request *req, 643 struct shash_desc *desc) 644{ 645 struct crypto_shash *tfm = desc->tfm; 646 struct shash_alg *shash = crypto_shash_alg(tfm); 647 648 /* alignment is to be done by multi-buffer crypto algorithm if needed */ 649 650 return shash->finup(desc, NULL, 0, req->result); 651} 652EXPORT_SYMBOL_GPL(shash_ahash_mcryptd_finup); 653 654int shash_ahash_mcryptd_final(struct ahash_request *req, 655 struct shash_desc *desc) 656{ 657 struct crypto_shash *tfm = desc->tfm; 658 struct shash_alg *shash = crypto_shash_alg(tfm); 659 660 /* alignment is to be done by multi-buffer crypto algorithm if needed */ 661 662 return shash->final(desc, req->result); 663} 664EXPORT_SYMBOL_GPL(shash_ahash_mcryptd_final); 665 666struct crypto_shash *mcryptd_ahash_child(struct mcryptd_ahash *tfm) 667{ 668 struct mcryptd_hash_ctx *ctx = crypto_ahash_ctx(&tfm->base); 669 670 return ctx->child; 671} 672EXPORT_SYMBOL_GPL(mcryptd_ahash_child); 673 674struct shash_desc *mcryptd_shash_desc(struct ahash_request *req) 675{ 676 struct mcryptd_hash_request_ctx *rctx = ahash_request_ctx(req); 677 return &rctx->desc; 678} 679EXPORT_SYMBOL_GPL(mcryptd_shash_desc); 680 681void mcryptd_free_ahash(struct mcryptd_ahash *tfm) 682{ 683 crypto_free_ahash(&tfm->base); 684} 685EXPORT_SYMBOL_GPL(mcryptd_free_ahash); 686 687 688static int __init mcryptd_init(void) 689{ 690 int err, cpu; 691 struct mcryptd_flush_list *flist; 692 693 mcryptd_flist = alloc_percpu(struct mcryptd_flush_list); 694 for_each_possible_cpu(cpu) { 695 flist = per_cpu_ptr(mcryptd_flist, cpu); 696 INIT_LIST_HEAD(&flist->list); 697 mutex_init(&flist->lock); 698 } 699 700 err = mcryptd_init_queue(&mqueue, MCRYPTD_MAX_CPU_QLEN); 701 if (err) { 702 free_percpu(mcryptd_flist); 703 return err; 704 } 705 706 err = crypto_register_template(&mcryptd_tmpl); 707 if (err) { 708 mcryptd_fini_queue(&mqueue); 709 free_percpu(mcryptd_flist); 710 } 711 712 return err; 713} 714 715static void __exit mcryptd_exit(void) 716{ 717 mcryptd_fini_queue(&mqueue); 718 crypto_unregister_template(&mcryptd_tmpl); 719 free_percpu(mcryptd_flist); 720} 721 722subsys_initcall(mcryptd_init); 723module_exit(mcryptd_exit); 724 725MODULE_LICENSE("GPL"); 726MODULE_DESCRIPTION("Software async multibuffer crypto daemon"); 727MODULE_ALIAS_CRYPTO("mcryptd"); 728