93 	struct spacc_engine		*engine;  member
142 	struct spacc_engine		*engine;  member
150 	struct spacc_engine		*engine;  member
187 static inline int spacc_fifo_cmd_full(struct spacc_engine *engine)  in spacc_fifo_cmd_full()  argument
189 	u32 fifo_stat = readl(engine->regs + SPA_FIFO_STAT_REG_OFFSET);  in spacc_fifo_cmd_full()
205 	return is_cipher_ctx ? ctx->engine->cipher_ctx_base +  in spacc_ctx_page_addr()
206 			(indx * ctx->engine->cipher_pg_sz) :  in spacc_ctx_page_addr()
207 		ctx->engine->hash_key_base + (indx * ctx->engine->hash_pg_sz);  in spacc_ctx_page_addr()
241 	unsigned indx = ctx->engine->next_ctx++;  in spacc_load_ctx()
247 	ctx->engine->next_ctx &= ctx->engine->fifo_sz - 1;  in spacc_load_ctx()
252 	       ctx->engine->regs + SPA_KEY_SZ_REG_OFFSET);  in spacc_load_ctx()
257 		       ctx->engine->regs + SPA_KEY_SZ_REG_OFFSET);  in spacc_load_ctx()
289 static struct spacc_ddt *spacc_sg_to_ddt(struct spacc_engine *engine,  in spacc_sg_to_ddt()  argument
301 	mapped_ents = dma_map_sg(engine->dev, payload, nents, dir);  in spacc_sg_to_ddt()
306 	ddt = dma_pool_alloc(engine->req_pool, GFP_ATOMIC, ddt_phys);  in spacc_sg_to_ddt()
317 	dma_unmap_sg(engine->dev, payload, nents, dir);  in spacc_sg_to_ddt()
325 	struct spacc_engine *engine = req->engine;  in spacc_aead_make_ddts()  local
334 	src_ddt = dma_pool_alloc(engine->req_pool, GFP_ATOMIC, &req->src_addr);  in spacc_aead_make_ddts()
338 	dst_ddt = dma_pool_alloc(engine->req_pool, GFP_ATOMIC, &req->dst_addr);  in spacc_aead_make_ddts()
340 		dma_pool_free(engine->req_pool, src_ddt, req->src_addr);  in spacc_aead_make_ddts()
347 	assoc_ents = dma_map_sg(engine->dev, areq->assoc,  in spacc_aead_make_ddts()
350 		src_ents = dma_map_sg(engine->dev, areq->src, nents,  in spacc_aead_make_ddts()
352 		dst_ents = dma_map_sg(engine->dev, areq->dst, nents,  in spacc_aead_make_ddts()
355 		src_ents = dma_map_sg(engine->dev, areq->src, nents,  in spacc_aead_make_ddts()
364 	iv_addr = dma_map_single(engine->dev, iv, ivsize,  in spacc_aead_make_ddts()
410 	struct spacc_engine *engine = aead_ctx->generic.engine;  in spacc_aead_free_ddts()  local
415 		dma_unmap_sg(engine->dev, areq->src, nents, DMA_TO_DEVICE);  in spacc_aead_free_ddts()
416 		dma_unmap_sg(engine->dev, areq->dst,  in spacc_aead_free_ddts()
420 		dma_unmap_sg(engine->dev, areq->src, nents, DMA_BIDIRECTIONAL);  in spacc_aead_free_ddts()
422 	dma_unmap_sg(engine->dev, areq->assoc,  in spacc_aead_free_ddts()
425 	dma_unmap_single(engine->dev, req->giv_pa, ivsize, DMA_BIDIRECTIONAL);  in spacc_aead_free_ddts()
427 	dma_pool_free(engine->req_pool, req->src_ddt, req->src_addr);  in spacc_aead_free_ddts()
428 	dma_pool_free(engine->req_pool, req->dst_ddt, req->dst_addr);  in spacc_aead_free_ddts()
437 	dma_unmap_sg(req->engine->dev, payload, nents, dir);  in spacc_free_ddt()
438 	dma_pool_free(req->engine->req_pool, ddt, ddt_addr);  in spacc_free_ddt()
601 	struct spacc_engine *engine = ctx->generic.engine;  in spacc_aead_submit()  local
612 	writel(req->src_addr, engine->regs + SPA_SRC_PTR_REG_OFFSET);  in spacc_aead_submit()
613 	writel(req->dst_addr, engine->regs + SPA_DST_PTR_REG_OFFSET);  in spacc_aead_submit()
614 	writel(0, engine->regs + SPA_OFFSET_REG_OFFSET);  in spacc_aead_submit()
636 	writel(proc_len, engine->regs + SPA_PROC_LEN_REG_OFFSET);  in spacc_aead_submit()
637 	writel(assoc_len, engine->regs + SPA_AAD_LEN_REG_OFFSET);  in spacc_aead_submit()
638 	writel(ctx->auth_size, engine->regs + SPA_ICV_LEN_REG_OFFSET);  in spacc_aead_submit()
639 	writel(0, engine->regs + SPA_ICV_OFFSET_REG_OFFSET);  in spacc_aead_submit()
640 	writel(0, engine->regs + SPA_AUX_INFO_REG_OFFSET);  in spacc_aead_submit()
649 	mod_timer(&engine->packet_timeout, jiffies + PACKET_TIMEOUT);  in spacc_aead_submit()
651 	writel(ctrl, engine->regs + SPA_CTRL_REG_OFFSET);  in spacc_aead_submit()
658 static void spacc_push(struct spacc_engine *engine)  in spacc_push()  argument
662 	while (!list_empty(&engine->pending) &&  in spacc_push()
663 	       engine->in_flight + 1 <= engine->fifo_sz) {  in spacc_push()
665 		++engine->in_flight;  in spacc_push()
666 		req = list_first_entry(&engine->pending, struct spacc_req,  in spacc_push()
668 		list_move_tail(&req->list, &engine->in_progress);  in spacc_push()
685 	struct spacc_engine *engine = to_spacc_alg(alg)->engine;  in spacc_aead_setup()  local
696 	dev_req->engine		= engine;  in spacc_aead_setup()
705 	spin_lock_irqsave(&engine->hw_lock, flags);  in spacc_aead_setup()
706 	if (unlikely(spacc_fifo_cmd_full(engine)) ||  in spacc_aead_setup()
707 	    engine->in_flight + 1 > engine->fifo_sz) {  in spacc_aead_setup()
710 			spin_unlock_irqrestore(&engine->hw_lock, flags);  in spacc_aead_setup()
713 		list_add_tail(&dev_req->list, &engine->pending);  in spacc_aead_setup()
715 		list_add_tail(&dev_req->list, &engine->pending);  in spacc_aead_setup()
716 		spacc_push(engine);  in spacc_aead_setup()
718 	spin_unlock_irqrestore(&engine->hw_lock, flags);  in spacc_aead_setup()
776 	struct spacc_engine *engine = spacc_alg->engine;  in spacc_aead_cra_init()  local
779 	ctx->generic.engine = engine;  in spacc_aead_cra_init()
784 		dev_warn(engine->dev, "failed to allocate fallback for %s\n",  in spacc_aead_cra_init()
949 	struct spacc_engine *engine = ctx->generic.engine;  in spacc_ablk_submit()  local
956 	writel(req->src_addr, engine->regs + SPA_SRC_PTR_REG_OFFSET);  in spacc_ablk_submit()
957 	writel(req->dst_addr, engine->regs + SPA_DST_PTR_REG_OFFSET);  in spacc_ablk_submit()
958 	writel(0, engine->regs + SPA_OFFSET_REG_OFFSET);  in spacc_ablk_submit()
960 	writel(ablk_req->nbytes, engine->regs + SPA_PROC_LEN_REG_OFFSET);  in spacc_ablk_submit()
961 	writel(0, engine->regs + SPA_ICV_OFFSET_REG_OFFSET);  in spacc_ablk_submit()
962 	writel(0, engine->regs + SPA_AUX_INFO_REG_OFFSET);  in spacc_ablk_submit()
963 	writel(0, engine->regs + SPA_AAD_LEN_REG_OFFSET);  in spacc_ablk_submit()
969 	mod_timer(&engine->packet_timeout, jiffies + PACKET_TIMEOUT);  in spacc_ablk_submit()
971 	writel(ctrl, engine->regs + SPA_CTRL_REG_OFFSET);  in spacc_ablk_submit()
1004 	struct spacc_engine *engine = to_spacc_alg(alg)->engine;  in spacc_ablk_setup()  local
1011 	dev_req->engine		= engine;  in spacc_ablk_setup()
1023 		dev_req->src_ddt = spacc_sg_to_ddt(engine, req->src,  in spacc_ablk_setup()
1028 		dev_req->dst_ddt = spacc_sg_to_ddt(engine, req->dst,  in spacc_ablk_setup()
1033 		dev_req->dst_ddt = spacc_sg_to_ddt(engine, req->dst,  in spacc_ablk_setup()
1043 	spin_lock_irqsave(&engine->hw_lock, flags);  in spacc_ablk_setup()
1049 	if (unlikely(spacc_fifo_cmd_full(engine)) ||  in spacc_ablk_setup()
1050 	    engine->in_flight + 1 > engine->fifo_sz) {  in spacc_ablk_setup()
1053 			spin_unlock_irqrestore(&engine->hw_lock, flags);  in spacc_ablk_setup()
1056 		list_add_tail(&dev_req->list, &engine->pending);  in spacc_ablk_setup()
1058 		list_add_tail(&dev_req->list, &engine->pending);  in spacc_ablk_setup()
1059 		spacc_push(engine);  in spacc_ablk_setup()
1061 	spin_unlock_irqrestore(&engine->hw_lock, flags);  in spacc_ablk_setup()
1082 	struct spacc_engine *engine = spacc_alg->engine;  in spacc_ablk_cra_init()  local
1085 	ctx->generic.engine = engine;  in spacc_ablk_cra_init()
1090 			dev_warn(engine->dev, "failed to allocate fallback for %s\n",  in spacc_ablk_cra_init()
1130 static inline int spacc_fifo_stat_empty(struct spacc_engine *engine)  in spacc_fifo_stat_empty()  argument
1132 	return readl(engine->regs + SPA_FIFO_STAT_REG_OFFSET) &  in spacc_fifo_stat_empty()
1136 static void spacc_process_done(struct spacc_engine *engine)  in spacc_process_done()  argument
1141 	spin_lock_irqsave(&engine->hw_lock, flags);  in spacc_process_done()
1143 	while (!spacc_fifo_stat_empty(engine)) {  in spacc_process_done()
1144 		req = list_first_entry(&engine->in_progress, struct spacc_req,  in spacc_process_done()
1146 		list_move_tail(&req->list, &engine->completed);  in spacc_process_done()
1147 		--engine->in_flight;  in spacc_process_done()
1150 		writel(~0, engine->regs + SPA_STAT_POP_REG_OFFSET);  in spacc_process_done()
1151 		req->result = (readl(engine->regs + SPA_STATUS_REG_OFFSET) &  in spacc_process_done()
1165 				dev_warn(engine->dev,  in spacc_process_done()
1171 				dev_warn(engine->dev,  in spacc_process_done()
1179 	tasklet_schedule(&engine->complete);  in spacc_process_done()
1181 	spin_unlock_irqrestore(&engine->hw_lock, flags);  in spacc_process_done()
1186 	struct spacc_engine *engine = (struct spacc_engine *)dev;  in spacc_spacc_irq()  local
1187 	u32 spacc_irq_stat = readl(engine->regs + SPA_IRQ_STAT_REG_OFFSET);  in spacc_spacc_irq()
1189 	writel(spacc_irq_stat, engine->regs + SPA_IRQ_STAT_REG_OFFSET);  in spacc_spacc_irq()
1190 	spacc_process_done(engine);  in spacc_spacc_irq()
1197 	struct spacc_engine *engine = (struct spacc_engine *)data;  in spacc_packet_timeout()  local
1199 	spacc_process_done(engine);  in spacc_packet_timeout()
1214 	struct spacc_engine *engine = (struct spacc_engine *)data;  in spacc_spacc_complete()  local
1219 	spin_lock_irqsave(&engine->hw_lock, flags);  in spacc_spacc_complete()
1221 	list_splice_init(&engine->completed, &completed);  in spacc_spacc_complete()
1222 	spacc_push(engine);  in spacc_spacc_complete()
1223 	if (engine->in_flight)  in spacc_spacc_complete()
1224 		mod_timer(&engine->packet_timeout, jiffies + PACKET_TIMEOUT);  in spacc_spacc_complete()
1226 	spin_unlock_irqrestore(&engine->hw_lock, flags);  in spacc_spacc_complete()
1238 	struct spacc_engine *engine = platform_get_drvdata(pdev);  in spacc_suspend()  local
1245 	clk_disable(engine->clk);  in spacc_suspend()
1253 	struct spacc_engine *engine = platform_get_drvdata(pdev);  in spacc_resume()  local
1255 	return clk_enable(engine->clk);  in spacc_resume()
1273 	struct spacc_engine *engine = spacc_dev_to_engine(dev);  in spacc_stat_irq_thresh_show()  local
1275 	return snprintf(buf, PAGE_SIZE, "%u\n", engine->stat_irq_thresh);  in spacc_stat_irq_thresh_show()
1282 	struct spacc_engine *engine = spacc_dev_to_engine(dev);  in spacc_stat_irq_thresh_store()  local
1288 	thresh = clamp(thresh, 1UL, engine->fifo_sz - 1);  in spacc_stat_irq_thresh_store()
1290 	engine->stat_irq_thresh = thresh;  in spacc_stat_irq_thresh_store()
1291 	writel(engine->stat_irq_thresh << SPA_IRQ_CTRL_STAT_CNT_OFFSET,  in spacc_stat_irq_thresh_store()
1292 	       engine->regs + SPA_IRQ_CTRL_REG_OFFSET);  in spacc_stat_irq_thresh_store()
1697 	struct spacc_engine *engine = devm_kzalloc(&pdev->dev, sizeof(*engine),  in spacc_probe()  local
1699 	if (!engine)  in spacc_probe()
1703 		engine->max_ctxs	= SPACC_CRYPTO_IPSEC_MAX_CTXS;  in spacc_probe()
1704 		engine->cipher_pg_sz	= SPACC_CRYPTO_IPSEC_CIPHER_PG_SZ;  in spacc_probe()
1705 		engine->hash_pg_sz	= SPACC_CRYPTO_IPSEC_HASH_PG_SZ;  in spacc_probe()
1706 		engine->fifo_sz		= SPACC_CRYPTO_IPSEC_FIFO_SZ;  in spacc_probe()
1707 		engine->algs		= ipsec_engine_algs;  in spacc_probe()
1708 		engine->num_algs	= ARRAY_SIZE(ipsec_engine_algs);  in spacc_probe()
1710 		engine->max_ctxs	= SPACC_CRYPTO_L2_MAX_CTXS;  in spacc_probe()
1711 		engine->cipher_pg_sz	= SPACC_CRYPTO_L2_CIPHER_PG_SZ;  in spacc_probe()
1712 		engine->hash_pg_sz	= SPACC_CRYPTO_L2_HASH_PG_SZ;  in spacc_probe()
1713 		engine->fifo_sz		= SPACC_CRYPTO_L2_FIFO_SZ;  in spacc_probe()
1714 		engine->algs		= l2_engine_algs;  in spacc_probe()
1715 		engine->num_algs	= ARRAY_SIZE(l2_engine_algs);  in spacc_probe()
1720 	engine->name = dev_name(&pdev->dev);  in spacc_probe()
1723 	engine->regs = devm_ioremap_resource(&pdev->dev, mem);  in spacc_probe()
1724 	if (IS_ERR(engine->regs))  in spacc_probe()
1725 		return PTR_ERR(engine->regs);  in spacc_probe()
1734 			     engine->name, engine)) {  in spacc_probe()
1735 		dev_err(engine->dev, "failed to request IRQ\n");  in spacc_probe()
1739 	engine->dev		= &pdev->dev;  in spacc_probe()
1740 	engine->cipher_ctx_base = engine->regs + SPA_CIPH_KEY_BASE_REG_OFFSET;  in spacc_probe()
1741 	engine->hash_key_base	= engine->regs + SPA_HASH_KEY_BASE_REG_OFFSET;  in spacc_probe()
1743 	engine->req_pool = dmam_pool_create(engine->name, engine->dev,  in spacc_probe()
1745 	if (!engine->req_pool)  in spacc_probe()
1748 	spin_lock_init(&engine->hw_lock);  in spacc_probe()
1750 	engine->clk = clk_get(&pdev->dev, "ref");  in spacc_probe()
1751 	if (IS_ERR(engine->clk)) {  in spacc_probe()
1754 		return PTR_ERR(engine->clk);  in spacc_probe()
1757 	if (clk_enable(engine->clk)) {  in spacc_probe()
1759 		clk_put(engine->clk);  in spacc_probe()
1765 		clk_disable(engine->clk);  in spacc_probe()
1766 		clk_put(engine->clk);  in spacc_probe()
1776 	engine->stat_irq_thresh = (engine->fifo_sz / 2);  in spacc_probe()
1783 	writel(engine->stat_irq_thresh << SPA_IRQ_CTRL_STAT_CNT_OFFSET,  in spacc_probe()
1784 	       engine->regs + SPA_IRQ_CTRL_REG_OFFSET);  in spacc_probe()
1786 	       engine->regs + SPA_IRQ_EN_REG_OFFSET);  in spacc_probe()
1788 	setup_timer(&engine->packet_timeout, spacc_packet_timeout,  in spacc_probe()
1789 		    (unsigned long)engine);  in spacc_probe()
1791 	INIT_LIST_HEAD(&engine->pending);  in spacc_probe()
1792 	INIT_LIST_HEAD(&engine->completed);  in spacc_probe()
1793 	INIT_LIST_HEAD(&engine->in_progress);  in spacc_probe()
1794 	engine->in_flight = 0;  in spacc_probe()
1795 	tasklet_init(&engine->complete, spacc_spacc_complete,  in spacc_probe()
1796 		     (unsigned long)engine);  in spacc_probe()
1798 	platform_set_drvdata(pdev, engine);  in spacc_probe()
1800 	INIT_LIST_HEAD(&engine->registered_algs);  in spacc_probe()
1801 	for (i = 0; i < engine->num_algs; ++i) {  in spacc_probe()
1802 		engine->algs[i].engine = engine;  in spacc_probe()
1803 		err = crypto_register_alg(&engine->algs[i].alg);  in spacc_probe()
1805 			list_add_tail(&engine->algs[i].entry,  in spacc_probe()
1806 				      &engine->registered_algs);  in spacc_probe()
1810 			dev_err(engine->dev, "failed to register alg \"%s\"\n",  in spacc_probe()
1811 				engine->algs[i].alg.cra_name);  in spacc_probe()
1813 			dev_dbg(engine->dev, "registered alg \"%s\"\n",  in spacc_probe()
1814 				engine->algs[i].alg.cra_name);  in spacc_probe()
1823 	struct spacc_engine *engine = platform_get_drvdata(pdev);  in spacc_remove()  local
1825 	del_timer_sync(&engine->packet_timeout);  in spacc_remove()
1828 	list_for_each_entry_safe(alg, next, &engine->registered_algs, entry) {  in spacc_remove()
1833 	clk_disable(engine->clk);  in spacc_remove()
1834 	clk_put(engine->clk);  in spacc_remove()