94 	struct spacc_engine		*engine;  member
108 	struct spacc_engine		*engine;  member
151 	struct spacc_engine		*engine;  member
159 	struct spacc_engine		*engine;  member
199 static inline int spacc_fifo_cmd_full(struct spacc_engine *engine)  in spacc_fifo_cmd_full()  argument
201 	u32 fifo_stat = readl(engine->regs + SPA_FIFO_STAT_REG_OFFSET);  in spacc_fifo_cmd_full()
217 	return is_cipher_ctx ? ctx->engine->cipher_ctx_base +  in spacc_ctx_page_addr()
218 			(indx * ctx->engine->cipher_pg_sz) :  in spacc_ctx_page_addr()
219 		ctx->engine->hash_key_base + (indx * ctx->engine->hash_pg_sz);  in spacc_ctx_page_addr()
253 	unsigned indx = ctx->engine->next_ctx++;  in spacc_load_ctx()
259 	ctx->engine->next_ctx &= ctx->engine->fifo_sz - 1;  in spacc_load_ctx()
264 	       ctx->engine->regs + SPA_KEY_SZ_REG_OFFSET);  in spacc_load_ctx()
269 		       ctx->engine->regs + SPA_KEY_SZ_REG_OFFSET);  in spacc_load_ctx()
292 static struct spacc_ddt *spacc_sg_to_ddt(struct spacc_engine *engine,  in spacc_sg_to_ddt()  argument
304 	mapped_ents = dma_map_sg(engine->dev, payload, nents, dir);  in spacc_sg_to_ddt()
309 	ddt = dma_pool_alloc(engine->req_pool, GFP_ATOMIC, ddt_phys);  in spacc_sg_to_ddt()
320 	dma_unmap_sg(engine->dev, payload, nents, dir);  in spacc_sg_to_ddt()
328 	struct spacc_engine *engine = req->engine;  in spacc_aead_make_ddts()  local
350 	src_ddt = dma_pool_alloc(engine->req_pool, GFP_ATOMIC, &req->src_addr);  in spacc_aead_make_ddts()
354 	dst_ddt = dma_pool_alloc(engine->req_pool, GFP_ATOMIC, &req->dst_addr);  in spacc_aead_make_ddts()
362 		src_ents = dma_map_sg(engine->dev, areq->src, src_nents,  in spacc_aead_make_ddts()
367 		dst_ents = dma_map_sg(engine->dev, areq->dst, dst_nents,  in spacc_aead_make_ddts()
371 			dma_unmap_sg(engine->dev, areq->src, src_nents,  in spacc_aead_make_ddts()
376 		src_ents = dma_map_sg(engine->dev, areq->src, src_nents,  in spacc_aead_make_ddts()
409 	dma_pool_free(engine->req_pool, dst_ddt, req->dst_addr);  in spacc_aead_make_ddts()
411 	dma_pool_free(engine->req_pool, src_ddt, req->src_addr);  in spacc_aead_make_ddts()
424 	struct spacc_engine *engine = aead_ctx->generic.engine;  in spacc_aead_free_ddts()  local
428 		dma_unmap_sg(engine->dev, areq->src, nents, DMA_TO_DEVICE);  in spacc_aead_free_ddts()
429 		dma_unmap_sg(engine->dev, areq->dst,  in spacc_aead_free_ddts()
433 		dma_unmap_sg(engine->dev, areq->src, nents, DMA_BIDIRECTIONAL);  in spacc_aead_free_ddts()
435 	dma_pool_free(engine->req_pool, req->src_ddt, req->src_addr);  in spacc_aead_free_ddts()
436 	dma_pool_free(engine->req_pool, req->dst_ddt, req->dst_addr);  in spacc_aead_free_ddts()
445 	dma_unmap_sg(req->engine->dev, payload, nents, dir);  in spacc_free_ddt()
446 	dma_pool_free(req->engine->req_pool, ddt, ddt_addr);  in spacc_free_ddt()
554 	struct spacc_engine *engine = ctx->generic.engine;  in spacc_aead_submit()  local
563 	writel(req->src_addr, engine->regs + SPA_SRC_PTR_REG_OFFSET);  in spacc_aead_submit()
564 	writel(req->dst_addr, engine->regs + SPA_DST_PTR_REG_OFFSET);  in spacc_aead_submit()
565 	writel(0, engine->regs + SPA_OFFSET_REG_OFFSET);  in spacc_aead_submit()
577 	writel(proc_len, engine->regs + SPA_PROC_LEN_REG_OFFSET);  in spacc_aead_submit()
578 	writel(assoc_len, engine->regs + SPA_AAD_LEN_REG_OFFSET);  in spacc_aead_submit()
579 	writel(authsize, engine->regs + SPA_ICV_LEN_REG_OFFSET);  in spacc_aead_submit()
580 	writel(0, engine->regs + SPA_ICV_OFFSET_REG_OFFSET);  in spacc_aead_submit()
581 	writel(0, engine->regs + SPA_AUX_INFO_REG_OFFSET);  in spacc_aead_submit()
590 	mod_timer(&engine->packet_timeout, jiffies + PACKET_TIMEOUT);  in spacc_aead_submit()
592 	writel(ctrl, engine->regs + SPA_CTRL_REG_OFFSET);  in spacc_aead_submit()
599 static void spacc_push(struct spacc_engine *engine)  in spacc_push()  argument
603 	while (!list_empty(&engine->pending) &&  in spacc_push()
604 	       engine->in_flight + 1 <= engine->fifo_sz) {  in spacc_push()
606 		++engine->in_flight;  in spacc_push()
607 		req = list_first_entry(&engine->pending, struct spacc_req,  in spacc_push()
609 		list_move_tail(&req->list, &engine->in_progress);  in spacc_push()
624 	struct spacc_engine *engine = to_spacc_aead(alg)->engine;  in spacc_aead_setup()  local
632 	dev_req->engine		= engine;  in spacc_aead_setup()
643 	spin_lock_irqsave(&engine->hw_lock, flags);  in spacc_aead_setup()
644 	if (unlikely(spacc_fifo_cmd_full(engine)) ||  in spacc_aead_setup()
645 	    engine->in_flight + 1 > engine->fifo_sz) {  in spacc_aead_setup()
648 			spin_unlock_irqrestore(&engine->hw_lock, flags);  in spacc_aead_setup()
651 		list_add_tail(&dev_req->list, &engine->pending);  in spacc_aead_setup()
653 		list_add_tail(&dev_req->list, &engine->pending);  in spacc_aead_setup()
654 		spacc_push(engine);  in spacc_aead_setup()
656 	spin_unlock_irqrestore(&engine->hw_lock, flags);  in spacc_aead_setup()
691 	struct spacc_engine *engine = spacc_alg->engine;  in spacc_aead_cra_init()  local
694 	ctx->generic.engine = engine;  in spacc_aead_cra_init()
860 	struct spacc_engine *engine = ctx->generic.engine;  in spacc_ablk_submit()  local
867 	writel(req->src_addr, engine->regs + SPA_SRC_PTR_REG_OFFSET);  in spacc_ablk_submit()
868 	writel(req->dst_addr, engine->regs + SPA_DST_PTR_REG_OFFSET);  in spacc_ablk_submit()
869 	writel(0, engine->regs + SPA_OFFSET_REG_OFFSET);  in spacc_ablk_submit()
871 	writel(ablk_req->nbytes, engine->regs + SPA_PROC_LEN_REG_OFFSET);  in spacc_ablk_submit()
872 	writel(0, engine->regs + SPA_ICV_OFFSET_REG_OFFSET);  in spacc_ablk_submit()
873 	writel(0, engine->regs + SPA_AUX_INFO_REG_OFFSET);  in spacc_ablk_submit()
874 	writel(0, engine->regs + SPA_AAD_LEN_REG_OFFSET);  in spacc_ablk_submit()
880 	mod_timer(&engine->packet_timeout, jiffies + PACKET_TIMEOUT);  in spacc_ablk_submit()
882 	writel(ctrl, engine->regs + SPA_CTRL_REG_OFFSET);  in spacc_ablk_submit()
915 	struct spacc_engine *engine = to_spacc_alg(alg)->engine;  in spacc_ablk_setup()  local
922 	dev_req->engine		= engine;  in spacc_ablk_setup()
934 		dev_req->src_ddt = spacc_sg_to_ddt(engine, req->src,  in spacc_ablk_setup()
939 		dev_req->dst_ddt = spacc_sg_to_ddt(engine, req->dst,  in spacc_ablk_setup()
944 		dev_req->dst_ddt = spacc_sg_to_ddt(engine, req->dst,  in spacc_ablk_setup()
954 	spin_lock_irqsave(&engine->hw_lock, flags);  in spacc_ablk_setup()
960 	if (unlikely(spacc_fifo_cmd_full(engine)) ||  in spacc_ablk_setup()
961 	    engine->in_flight + 1 > engine->fifo_sz) {  in spacc_ablk_setup()
964 			spin_unlock_irqrestore(&engine->hw_lock, flags);  in spacc_ablk_setup()
967 		list_add_tail(&dev_req->list, &engine->pending);  in spacc_ablk_setup()
969 		list_add_tail(&dev_req->list, &engine->pending);  in spacc_ablk_setup()
970 		spacc_push(engine);  in spacc_ablk_setup()
972 	spin_unlock_irqrestore(&engine->hw_lock, flags);  in spacc_ablk_setup()
993 	struct spacc_engine *engine = spacc_alg->engine;  in spacc_ablk_cra_init()  local
996 	ctx->generic.engine = engine;  in spacc_ablk_cra_init()
1001 			dev_warn(engine->dev, "failed to allocate fallback for %s\n",  in spacc_ablk_cra_init()
1041 static inline int spacc_fifo_stat_empty(struct spacc_engine *engine)  in spacc_fifo_stat_empty()  argument
1043 	return readl(engine->regs + SPA_FIFO_STAT_REG_OFFSET) &  in spacc_fifo_stat_empty()
1047 static void spacc_process_done(struct spacc_engine *engine)  in spacc_process_done()  argument
1052 	spin_lock_irqsave(&engine->hw_lock, flags);  in spacc_process_done()
1054 	while (!spacc_fifo_stat_empty(engine)) {  in spacc_process_done()
1055 		req = list_first_entry(&engine->in_progress, struct spacc_req,  in spacc_process_done()
1057 		list_move_tail(&req->list, &engine->completed);  in spacc_process_done()
1058 		--engine->in_flight;  in spacc_process_done()
1061 		writel(~0, engine->regs + SPA_STAT_POP_REG_OFFSET);  in spacc_process_done()
1062 		req->result = (readl(engine->regs + SPA_STATUS_REG_OFFSET) &  in spacc_process_done()
1076 				dev_warn(engine->dev,  in spacc_process_done()
1082 				dev_warn(engine->dev,  in spacc_process_done()
1090 	tasklet_schedule(&engine->complete);  in spacc_process_done()
1092 	spin_unlock_irqrestore(&engine->hw_lock, flags);  in spacc_process_done()
1097 	struct spacc_engine *engine = (struct spacc_engine *)dev;  in spacc_spacc_irq()  local
1098 	u32 spacc_irq_stat = readl(engine->regs + SPA_IRQ_STAT_REG_OFFSET);  in spacc_spacc_irq()
1100 	writel(spacc_irq_stat, engine->regs + SPA_IRQ_STAT_REG_OFFSET);  in spacc_spacc_irq()
1101 	spacc_process_done(engine);  in spacc_spacc_irq()
1108 	struct spacc_engine *engine = (struct spacc_engine *)data;  in spacc_packet_timeout()  local
1110 	spacc_process_done(engine);  in spacc_packet_timeout()
1125 	struct spacc_engine *engine = (struct spacc_engine *)data;  in spacc_spacc_complete()  local
1130 	spin_lock_irqsave(&engine->hw_lock, flags);  in spacc_spacc_complete()
1132 	list_splice_init(&engine->completed, &completed);  in spacc_spacc_complete()
1133 	spacc_push(engine);  in spacc_spacc_complete()
1134 	if (engine->in_flight)  in spacc_spacc_complete()
1135 		mod_timer(&engine->packet_timeout, jiffies + PACKET_TIMEOUT);  in spacc_spacc_complete()
1137 	spin_unlock_irqrestore(&engine->hw_lock, flags);  in spacc_spacc_complete()
1149 	struct spacc_engine *engine = platform_get_drvdata(pdev);  in spacc_suspend()  local
1156 	clk_disable(engine->clk);  in spacc_suspend()
1164 	struct spacc_engine *engine = platform_get_drvdata(pdev);  in spacc_resume()  local
1166 	return clk_enable(engine->clk);  in spacc_resume()
1184 	struct spacc_engine *engine = spacc_dev_to_engine(dev);  in spacc_stat_irq_thresh_show()  local
1186 	return snprintf(buf, PAGE_SIZE, "%u\n", engine->stat_irq_thresh);  in spacc_stat_irq_thresh_show()
1193 	struct spacc_engine *engine = spacc_dev_to_engine(dev);  in spacc_stat_irq_thresh_store()  local
1199 	thresh = clamp(thresh, 1UL, engine->fifo_sz - 1);  in spacc_stat_irq_thresh_store()
1201 	engine->stat_irq_thresh = thresh;  in spacc_stat_irq_thresh_store()
1202 	writel(engine->stat_irq_thresh << SPA_IRQ_CTRL_STAT_CNT_OFFSET,  in spacc_stat_irq_thresh_store()
1203 	       engine->regs + SPA_IRQ_CTRL_REG_OFFSET);  in spacc_stat_irq_thresh_store()
1617 	struct spacc_engine *engine = devm_kzalloc(&pdev->dev, sizeof(*engine),  in spacc_probe()  local
1619 	if (!engine)  in spacc_probe()
1623 		engine->max_ctxs	= SPACC_CRYPTO_IPSEC_MAX_CTXS;  in spacc_probe()
1624 		engine->cipher_pg_sz	= SPACC_CRYPTO_IPSEC_CIPHER_PG_SZ;  in spacc_probe()
1625 		engine->hash_pg_sz	= SPACC_CRYPTO_IPSEC_HASH_PG_SZ;  in spacc_probe()
1626 		engine->fifo_sz		= SPACC_CRYPTO_IPSEC_FIFO_SZ;  in spacc_probe()
1627 		engine->algs		= ipsec_engine_algs;  in spacc_probe()
1628 		engine->num_algs	= ARRAY_SIZE(ipsec_engine_algs);  in spacc_probe()
1629 		engine->aeads		= ipsec_engine_aeads;  in spacc_probe()
1630 		engine->num_aeads	= ARRAY_SIZE(ipsec_engine_aeads);  in spacc_probe()
1632 		engine->max_ctxs	= SPACC_CRYPTO_L2_MAX_CTXS;  in spacc_probe()
1633 		engine->cipher_pg_sz	= SPACC_CRYPTO_L2_CIPHER_PG_SZ;  in spacc_probe()
1634 		engine->hash_pg_sz	= SPACC_CRYPTO_L2_HASH_PG_SZ;  in spacc_probe()
1635 		engine->fifo_sz		= SPACC_CRYPTO_L2_FIFO_SZ;  in spacc_probe()
1636 		engine->algs		= l2_engine_algs;  in spacc_probe()
1637 		engine->num_algs	= ARRAY_SIZE(l2_engine_algs);  in spacc_probe()
1642 	engine->name = dev_name(&pdev->dev);  in spacc_probe()
1645 	engine->regs = devm_ioremap_resource(&pdev->dev, mem);  in spacc_probe()
1646 	if (IS_ERR(engine->regs))  in spacc_probe()
1647 		return PTR_ERR(engine->regs);  in spacc_probe()
1656 			     engine->name, engine)) {  in spacc_probe()
1657 		dev_err(engine->dev, "failed to request IRQ\n");  in spacc_probe()
1661 	engine->dev		= &pdev->dev;  in spacc_probe()
1662 	engine->cipher_ctx_base = engine->regs + SPA_CIPH_KEY_BASE_REG_OFFSET;  in spacc_probe()
1663 	engine->hash_key_base	= engine->regs + SPA_HASH_KEY_BASE_REG_OFFSET;  in spacc_probe()
1665 	engine->req_pool = dmam_pool_create(engine->name, engine->dev,  in spacc_probe()
1667 	if (!engine->req_pool)  in spacc_probe()
1670 	spin_lock_init(&engine->hw_lock);  in spacc_probe()
1672 	engine->clk = clk_get(&pdev->dev, "ref");  in spacc_probe()
1673 	if (IS_ERR(engine->clk)) {  in spacc_probe()
1676 		return PTR_ERR(engine->clk);  in spacc_probe()
1679 	if (clk_prepare_enable(engine->clk)) {  in spacc_probe()
1681 		clk_put(engine->clk);  in spacc_probe()
1687 		clk_disable_unprepare(engine->clk);  in spacc_probe()
1688 		clk_put(engine->clk);  in spacc_probe()
1698 	engine->stat_irq_thresh = (engine->fifo_sz / 2);  in spacc_probe()
1705 	writel(engine->stat_irq_thresh << SPA_IRQ_CTRL_STAT_CNT_OFFSET,  in spacc_probe()
1706 	       engine->regs + SPA_IRQ_CTRL_REG_OFFSET);  in spacc_probe()
1708 	       engine->regs + SPA_IRQ_EN_REG_OFFSET);  in spacc_probe()
1710 	setup_timer(&engine->packet_timeout, spacc_packet_timeout,  in spacc_probe()
1711 		    (unsigned long)engine);  in spacc_probe()
1713 	INIT_LIST_HEAD(&engine->pending);  in spacc_probe()
1714 	INIT_LIST_HEAD(&engine->completed);  in spacc_probe()
1715 	INIT_LIST_HEAD(&engine->in_progress);  in spacc_probe()
1716 	engine->in_flight = 0;  in spacc_probe()
1717 	tasklet_init(&engine->complete, spacc_spacc_complete,  in spacc_probe()
1718 		     (unsigned long)engine);  in spacc_probe()
1720 	platform_set_drvdata(pdev, engine);  in spacc_probe()
1722 	INIT_LIST_HEAD(&engine->registered_algs);  in spacc_probe()
1723 	for (i = 0; i < engine->num_algs; ++i) {  in spacc_probe()
1724 		engine->algs[i].engine = engine;  in spacc_probe()
1725 		err = crypto_register_alg(&engine->algs[i].alg);  in spacc_probe()
1727 			list_add_tail(&engine->algs[i].entry,  in spacc_probe()
1728 				      &engine->registered_algs);  in spacc_probe()
1732 			dev_err(engine->dev, "failed to register alg \"%s\"\n",  in spacc_probe()
1733 				engine->algs[i].alg.cra_name);  in spacc_probe()
1735 			dev_dbg(engine->dev, "registered alg \"%s\"\n",  in spacc_probe()
1736 				engine->algs[i].alg.cra_name);  in spacc_probe()
1739 	INIT_LIST_HEAD(&engine->registered_aeads);  in spacc_probe()
1740 	for (i = 0; i < engine->num_aeads; ++i) {  in spacc_probe()
1741 		engine->aeads[i].engine = engine;  in spacc_probe()
1742 		err = crypto_register_aead(&engine->aeads[i].alg);  in spacc_probe()
1744 			list_add_tail(&engine->aeads[i].entry,  in spacc_probe()
1745 				      &engine->registered_aeads);  in spacc_probe()
1749 			dev_err(engine->dev, "failed to register alg \"%s\"\n",  in spacc_probe()
1750 				engine->aeads[i].alg.base.cra_name);  in spacc_probe()
1752 			dev_dbg(engine->dev, "registered alg \"%s\"\n",  in spacc_probe()
1753 				engine->aeads[i].alg.base.cra_name);  in spacc_probe()
1763 	struct spacc_engine *engine = platform_get_drvdata(pdev);  in spacc_remove()  local
1765 	del_timer_sync(&engine->packet_timeout);  in spacc_remove()
1768 	list_for_each_entry_safe(aead, an, &engine->registered_aeads, entry) {  in spacc_remove()
1773 	list_for_each_entry_safe(alg, next, &engine->registered_algs, entry) {  in spacc_remove()
1778 	clk_disable_unprepare(engine->clk);  in spacc_remove()
1779 	clk_put(engine->clk);  in spacc_remove()