47 void __ablkcipher_walk_complete(struct ablkcipher_walk *walk)  in __ablkcipher_walk_complete()  argument
51 	list_for_each_entry_safe(p, tmp, &walk->buffers, entry) {  in __ablkcipher_walk_complete()
59 static inline void ablkcipher_queue_write(struct ablkcipher_walk *walk,  in ablkcipher_queue_write()  argument
62 	p->dst = walk->out;  in ablkcipher_queue_write()
63 	list_add_tail(&p->entry, &walk->buffers);  in ablkcipher_queue_write()
76 static inline unsigned int ablkcipher_done_slow(struct ablkcipher_walk *walk,  in ablkcipher_done_slow()  argument
82 		unsigned int len_this_page = scatterwalk_pagelen(&walk->out);  in ablkcipher_done_slow()
86 		scatterwalk_advance(&walk->out, n);  in ablkcipher_done_slow()
90 		scatterwalk_start(&walk->out, sg_next(walk->out.sg));  in ablkcipher_done_slow()
96 static inline unsigned int ablkcipher_done_fast(struct ablkcipher_walk *walk,  in ablkcipher_done_fast()  argument
99 	scatterwalk_advance(&walk->in, n);  in ablkcipher_done_fast()
100 	scatterwalk_advance(&walk->out, n);  in ablkcipher_done_fast()
106 				struct ablkcipher_walk *walk);
109 			 struct ablkcipher_walk *walk, int err)  in ablkcipher_walk_done()  argument
115 		unsigned int n = walk->nbytes - err;  in ablkcipher_walk_done()
117 		if (likely(!(walk->flags & ABLKCIPHER_WALK_SLOW)))  in ablkcipher_walk_done()
118 			n = ablkcipher_done_fast(walk, n);  in ablkcipher_walk_done()
123 			n = ablkcipher_done_slow(walk, n);  in ablkcipher_walk_done()
125 		nbytes = walk->total - n;  in ablkcipher_walk_done()
129 	scatterwalk_done(&walk->in, 0, nbytes);  in ablkcipher_walk_done()
130 	scatterwalk_done(&walk->out, 1, nbytes);  in ablkcipher_walk_done()
133 	walk->total = nbytes;  in ablkcipher_walk_done()
134 	walk->nbytes = nbytes;  in ablkcipher_walk_done()
138 		return ablkcipher_walk_next(req, walk);  in ablkcipher_walk_done()
141 	if (walk->iv != req->info)  in ablkcipher_walk_done()
142 		memcpy(req->info, walk->iv, tfm->crt_ablkcipher.ivsize);  in ablkcipher_walk_done()
143 	kfree(walk->iv_buffer);  in ablkcipher_walk_done()
150 				       struct ablkcipher_walk *walk,  in ablkcipher_next_slow()  argument
166 		return ablkcipher_walk_done(req, walk, -ENOMEM);  in ablkcipher_next_slow()
176 	scatterwalk_copychunks(src, &walk->in, bsize, 0);  in ablkcipher_next_slow()
178 	ablkcipher_queue_write(walk, p);  in ablkcipher_next_slow()
180 	walk->nbytes = bsize;  in ablkcipher_next_slow()
181 	walk->flags |= ABLKCIPHER_WALK_SLOW;  in ablkcipher_next_slow()
189 static inline int ablkcipher_copy_iv(struct ablkcipher_walk *walk,  in ablkcipher_copy_iv()  argument
193 	unsigned bs = walk->blocksize;  in ablkcipher_copy_iv()
201 	walk->iv_buffer = kmalloc(size, GFP_ATOMIC);  in ablkcipher_copy_iv()
202 	if (!walk->iv_buffer)  in ablkcipher_copy_iv()
205 	iv = (u8 *)ALIGN((unsigned long)walk->iv_buffer, alignmask + 1);  in ablkcipher_copy_iv()
210 	walk->iv = memcpy(iv, walk->iv, ivsize);  in ablkcipher_copy_iv()
215 				       struct ablkcipher_walk *walk)  in ablkcipher_next_fast()  argument
217 	walk->src.page = scatterwalk_page(&walk->in);  in ablkcipher_next_fast()
218 	walk->src.offset = offset_in_page(walk->in.offset);  in ablkcipher_next_fast()
219 	walk->dst.page = scatterwalk_page(&walk->out);  in ablkcipher_next_fast()
220 	walk->dst.offset = offset_in_page(walk->out.offset);  in ablkcipher_next_fast()
226 				struct ablkcipher_walk *walk)  in ablkcipher_walk_next()  argument
234 	n = walk->total;  in ablkcipher_walk_next()
237 		return ablkcipher_walk_done(req, walk, -EINVAL);  in ablkcipher_walk_next()
240 	walk->flags &= ~ABLKCIPHER_WALK_SLOW;  in ablkcipher_walk_next()
243 	bsize = min(walk->blocksize, n);  in ablkcipher_walk_next()
244 	n = scatterwalk_clamp(&walk->in, n);  in ablkcipher_walk_next()
245 	n = scatterwalk_clamp(&walk->out, n);  in ablkcipher_walk_next()
248 	    !scatterwalk_aligned(&walk->in, alignmask) ||  in ablkcipher_walk_next()
249 	    !scatterwalk_aligned(&walk->out, alignmask)) {  in ablkcipher_walk_next()
250 		err = ablkcipher_next_slow(req, walk, bsize, alignmask,  in ablkcipher_walk_next()
255 	walk->nbytes = n;  in ablkcipher_walk_next()
257 	return ablkcipher_next_fast(req, walk);  in ablkcipher_walk_next()
261 		walk->src.page = virt_to_page(src);  in ablkcipher_walk_next()
262 		walk->dst.page = virt_to_page(dst);  in ablkcipher_walk_next()
263 		walk->src.offset = ((unsigned long)src & (PAGE_SIZE - 1));  in ablkcipher_walk_next()
264 		walk->dst.offset = ((unsigned long)dst & (PAGE_SIZE - 1));  in ablkcipher_walk_next()
271 				 struct ablkcipher_walk *walk)  in ablkcipher_walk_first()  argument
280 	walk->iv = req->info;  in ablkcipher_walk_first()
281 	walk->nbytes = walk->total;  in ablkcipher_walk_first()
282 	if (unlikely(!walk->total))  in ablkcipher_walk_first()
285 	walk->iv_buffer = NULL;  in ablkcipher_walk_first()
286 	if (unlikely(((unsigned long)walk->iv & alignmask))) {  in ablkcipher_walk_first()
287 		int err = ablkcipher_copy_iv(walk, tfm, alignmask);  in ablkcipher_walk_first()
293 	scatterwalk_start(&walk->in, walk->in.sg);  in ablkcipher_walk_first()
294 	scatterwalk_start(&walk->out, walk->out.sg);  in ablkcipher_walk_first()
296 	return ablkcipher_walk_next(req, walk);  in ablkcipher_walk_first()
300 			 struct ablkcipher_walk *walk)  in ablkcipher_walk_phys()  argument
302 	walk->blocksize = crypto_tfm_alg_blocksize(req->base.tfm);  in ablkcipher_walk_phys()
303 	return ablkcipher_walk_first(req, walk);  in ablkcipher_walk_phys()