1 /*
2  * Stream co-processor driver for the ETRAX FS
3  *
4  *    Copyright (C) 2003-2007  Axis Communications AB
5  */
6 
7 #include <linux/init.h>
8 #include <linux/sched.h>
9 #include <linux/module.h>
10 #include <linux/slab.h>
11 #include <linux/string.h>
12 #include <linux/fs.h>
13 #include <linux/mm.h>
14 #include <linux/spinlock.h>
15 #include <linux/stddef.h>
16 
17 #include <asm/uaccess.h>
18 #include <asm/io.h>
19 #include <linux/atomic.h>
20 
21 #include <linux/list.h>
22 #include <linux/interrupt.h>
23 
24 #include <asm/signal.h>
25 #include <asm/irq.h>
26 
27 #include <dma.h>
28 #include <hwregs/dma.h>
29 #include <hwregs/reg_map.h>
30 #include <hwregs/reg_rdwr.h>
31 #include <hwregs/intr_vect_defs.h>
32 
33 #include <hwregs/strcop.h>
34 #include <hwregs/strcop_defs.h>
35 #include <cryptocop.h>
36 
37 #ifdef CONFIG_ETRAXFS
38 #define IN_DMA 9
39 #define OUT_DMA 8
40 #define IN_DMA_INST regi_dma9
41 #define OUT_DMA_INST regi_dma8
42 #define DMA_IRQ DMA9_INTR_VECT
43 #else
44 #define IN_DMA 3
45 #define OUT_DMA 2
46 #define IN_DMA_INST regi_dma3
47 #define OUT_DMA_INST regi_dma2
48 #define DMA_IRQ DMA3_INTR_VECT
49 #endif
50 
51 #define DESCR_ALLOC_PAD  (31)
52 
53 struct cryptocop_dma_desc {
54 	char *free_buf; /* If non-null will be kfreed in free_cdesc() */
55 	dma_descr_data *dma_descr;
56 
57 	unsigned char dma_descr_buf[sizeof(dma_descr_data) + DESCR_ALLOC_PAD];
58 
59 	unsigned int from_pool:1; /* If 1 'allocated' from the descriptor pool. */
60 	struct cryptocop_dma_desc *next;
61 };
62 
63 
64 struct cryptocop_int_operation{
65 	void                        *alloc_ptr;
66 	cryptocop_session_id        sid;
67 
68 	dma_descr_context           ctx_out;
69 	dma_descr_context           ctx_in;
70 
71 	/* DMA descriptors allocated by driver. */
72 	struct cryptocop_dma_desc   *cdesc_out;
73 	struct cryptocop_dma_desc   *cdesc_in;
74 
75 	/* Strcop config to use. */
76 	cryptocop_3des_mode         tdes_mode;
77 	cryptocop_csum_type         csum_mode;
78 
79 	/* DMA descrs provided by consumer. */
80 	dma_descr_data              *ddesc_out;
81 	dma_descr_data              *ddesc_in;
82 };
83 
84 
85 struct cryptocop_tfrm_ctx {
86 	cryptocop_tfrm_id tid;
87 	unsigned int blocklength;
88 
89 	unsigned int start_ix;
90 
91 	struct cryptocop_tfrm_cfg *tcfg;
92 	struct cryptocop_transform_ctx *tctx;
93 
94 	unsigned char previous_src;
95 	unsigned char current_src;
96 
97 	/* Values to use in metadata out. */
98 	unsigned char hash_conf;
99 	unsigned char hash_mode;
100 	unsigned char ciph_conf;
101 	unsigned char cbcmode;
102 	unsigned char decrypt;
103 
104 	unsigned int requires_padding:1;
105 	unsigned int strict_block_length:1;
106 	unsigned int active:1;
107 	unsigned int done:1;
108 	size_t consumed;
109 	size_t produced;
110 
111 	/* Pad (input) descriptors to put in the DMA out list when the transform
112 	 * output is put on the DMA in list. */
113 	struct cryptocop_dma_desc *pad_descs;
114 
115 	struct cryptocop_tfrm_ctx *prev_src;
116 	struct cryptocop_tfrm_ctx *curr_src;
117 
118 	/* Mapping to HW. */
119 	unsigned char unit_no;
120 };
121 
122 
123 struct cryptocop_private{
124 	cryptocop_session_id sid;
125 	struct cryptocop_private *next;
126 };
127 
128 /* Session list. */
129 
130 struct cryptocop_transform_ctx{
131 	struct cryptocop_transform_init init;
132 	unsigned char dec_key[CRYPTOCOP_MAX_KEY_LENGTH];
133 	unsigned int dec_key_set:1;
134 
135 	struct cryptocop_transform_ctx *next;
136 };
137 
138 
139 struct cryptocop_session{
140 	cryptocop_session_id sid;
141 
142 	struct cryptocop_transform_ctx *tfrm_ctx;
143 
144 	struct cryptocop_session *next;
145 };
146 
147 /* Priority levels for jobs sent to the cryptocop.  Checksum operations from
148    kernel have highest priority since TCPIP stack processing must not
149    be a bottleneck. */
150 typedef enum {
151 	cryptocop_prio_kernel_csum = 0,
152 	cryptocop_prio_kernel = 1,
153 	cryptocop_prio_user = 2,
154 	cryptocop_prio_no_prios = 3
155 } cryptocop_queue_priority;
156 
157 struct cryptocop_prio_queue{
158 	struct list_head jobs;
159 	cryptocop_queue_priority prio;
160 };
161 
162 struct cryptocop_prio_job{
163 	struct list_head node;
164 	cryptocop_queue_priority prio;
165 
166 	struct cryptocop_operation *oper;
167 	struct cryptocop_int_operation *iop;
168 };
169 
170 struct ioctl_job_cb_ctx {
171 	unsigned int processed:1;
172 };
173 
174 
175 static struct cryptocop_session *cryptocop_sessions = NULL;
176 spinlock_t cryptocop_sessions_lock;
177 
178 /* Next Session ID to assign. */
179 static cryptocop_session_id next_sid = 1;
180 
181 /* Pad for checksum. */
182 static const char csum_zero_pad[1] = {0x00};
183 
184 /* Trash buffer for mem2mem operations. */
185 #define MEM2MEM_DISCARD_BUF_LENGTH  (512)
186 static unsigned char mem2mem_discard_buf[MEM2MEM_DISCARD_BUF_LENGTH];
187 
188 /* Descriptor pool. */
189 /* FIXME Tweak this value. */
190 #define CRYPTOCOP_DESCRIPTOR_POOL_SIZE   (100)
191 static struct cryptocop_dma_desc descr_pool[CRYPTOCOP_DESCRIPTOR_POOL_SIZE];
192 static struct cryptocop_dma_desc *descr_pool_free_list;
193 static int descr_pool_no_free;
194 static spinlock_t descr_pool_lock;
195 
196 /* Lock to stop cryptocop to start processing of a new operation. The holder
197    of this lock MUST call cryptocop_start_job() after it is unlocked. */
198 spinlock_t cryptocop_process_lock;
199 
200 static struct cryptocop_prio_queue cryptocop_job_queues[cryptocop_prio_no_prios];
201 static spinlock_t cryptocop_job_queue_lock;
202 static struct cryptocop_prio_job *cryptocop_running_job = NULL;
203 static spinlock_t running_job_lock;
204 
205 /* The interrupt handler appends completed jobs to this list. The scehduled
206  * tasklet removes them upon sending the response to the crypto consumer. */
207 static struct list_head cryptocop_completed_jobs;
208 static spinlock_t cryptocop_completed_jobs_lock;
209 
210 DECLARE_WAIT_QUEUE_HEAD(cryptocop_ioc_process_wq);
211 
212 
213 /** Local functions. **/
214 
215 static int cryptocop_open(struct inode *, struct file *);
216 
217 static int cryptocop_release(struct inode *, struct file *);
218 
219 static long cryptocop_ioctl(struct file *file,
220 			   unsigned int cmd, unsigned long arg);
221 
222 static void cryptocop_start_job(void);
223 
224 static int cryptocop_job_queue_insert(cryptocop_queue_priority prio, struct cryptocop_operation *operation);
225 static int cryptocop_job_setup(struct cryptocop_prio_job **pj, struct cryptocop_operation *operation);
226 
227 static int cryptocop_job_queue_init(void);
228 static void cryptocop_job_queue_close(void);
229 
230 static int create_md5_pad(int alloc_flag, unsigned long long hashed_length, char **pad, size_t *pad_length);
231 
232 static int create_sha1_pad(int alloc_flag, unsigned long long hashed_length, char **pad, size_t *pad_length);
233 
234 static int transform_ok(struct cryptocop_transform_init *tinit);
235 
236 static struct cryptocop_session *get_session(cryptocop_session_id sid);
237 
238 static struct cryptocop_transform_ctx *get_transform_ctx(struct cryptocop_session *sess, cryptocop_tfrm_id tid);
239 
240 static void delete_internal_operation(struct cryptocop_int_operation *iop);
241 
242 static void get_aes_decrypt_key(unsigned char *dec_key, const unsigned  char *key, unsigned int keylength);
243 
244 static int init_stream_coprocessor(void);
245 
246 static void __exit exit_stream_coprocessor(void);
247 
248 /*#define LDEBUG*/
249 #ifdef LDEBUG
250 #define DEBUG(s) s
251 #define DEBUG_API(s) s
252 static void print_cryptocop_operation(struct cryptocop_operation *cop);
253 static void print_dma_descriptors(struct cryptocop_int_operation *iop);
254 static void print_strcop_crypto_op(struct strcop_crypto_op *cop);
255 static void print_lock_status(void);
256 static void print_user_dma_lists(struct cryptocop_dma_list_operation *dma_op);
257 #define assert(s) do{if (!(s)) panic(#s);} while(0);
258 #else
259 #define DEBUG(s)
260 #define DEBUG_API(s)
261 #define assert(s)
262 #endif
263 
264 
265 /* Transform constants. */
266 #define DES_BLOCK_LENGTH   (8)
267 #define AES_BLOCK_LENGTH   (16)
268 #define MD5_BLOCK_LENGTH   (64)
269 #define SHA1_BLOCK_LENGTH  (64)
270 #define CSUM_BLOCK_LENGTH  (2)
271 #define MD5_STATE_LENGTH   (16)
272 #define SHA1_STATE_LENGTH  (20)
273 
274 /* The device number. */
275 #define CRYPTOCOP_MAJOR    (254)
276 #define CRYPTOCOP_MINOR    (0)
277 
278 
279 
280 const struct file_operations cryptocop_fops = {
281 	.owner		= THIS_MODULE,
282 	.open		= cryptocop_open,
283 	.release	= cryptocop_release,
284 	.unlocked_ioctl = cryptocop_ioctl,
285 	.llseek		= noop_llseek,
286 };
287 
288 
free_cdesc(struct cryptocop_dma_desc * cdesc)289 static void free_cdesc(struct cryptocop_dma_desc *cdesc)
290 {
291 	DEBUG(printk("free_cdesc: cdesc 0x%p, from_pool=%d\n", cdesc, cdesc->from_pool));
292 	kfree(cdesc->free_buf);
293 
294 	if (cdesc->from_pool) {
295 		unsigned long int flags;
296 		spin_lock_irqsave(&descr_pool_lock, flags);
297 		cdesc->next = descr_pool_free_list;
298 		descr_pool_free_list = cdesc;
299 		++descr_pool_no_free;
300 		spin_unlock_irqrestore(&descr_pool_lock, flags);
301 	} else {
302 		kfree(cdesc);
303 	}
304 }
305 
306 
alloc_cdesc(int alloc_flag)307 static struct cryptocop_dma_desc *alloc_cdesc(int alloc_flag)
308 {
309 	int use_pool = (alloc_flag & GFP_ATOMIC) ? 1 : 0;
310 	struct cryptocop_dma_desc *cdesc;
311 
312 	if (use_pool) {
313 		unsigned long int flags;
314 		spin_lock_irqsave(&descr_pool_lock, flags);
315 		if (!descr_pool_free_list) {
316 			spin_unlock_irqrestore(&descr_pool_lock, flags);
317 			DEBUG_API(printk("alloc_cdesc: pool is empty\n"));
318 			return NULL;
319 		}
320 		cdesc = descr_pool_free_list;
321 		descr_pool_free_list = descr_pool_free_list->next;
322 		--descr_pool_no_free;
323 		spin_unlock_irqrestore(&descr_pool_lock, flags);
324 		cdesc->from_pool = 1;
325 	} else {
326 		cdesc = kmalloc(sizeof(struct cryptocop_dma_desc), alloc_flag);
327 		if (!cdesc) {
328 			DEBUG_API(printk("alloc_cdesc: kmalloc\n"));
329 			return NULL;
330 		}
331 		cdesc->from_pool = 0;
332 	}
333 	cdesc->dma_descr = (dma_descr_data*)(((unsigned long int)cdesc + offsetof(struct cryptocop_dma_desc, dma_descr_buf) + DESCR_ALLOC_PAD) & ~0x0000001F);
334 
335 	cdesc->next = NULL;
336 
337 	cdesc->free_buf = NULL;
338 	cdesc->dma_descr->out_eop = 0;
339 	cdesc->dma_descr->in_eop = 0;
340 	cdesc->dma_descr->intr = 0;
341 	cdesc->dma_descr->eol = 0;
342 	cdesc->dma_descr->wait = 0;
343 	cdesc->dma_descr->buf = NULL;
344 	cdesc->dma_descr->after = NULL;
345 
346 	DEBUG_API(printk("alloc_cdesc: return 0x%p, cdesc->dma_descr=0x%p, from_pool=%d\n", cdesc, cdesc->dma_descr, cdesc->from_pool));
347 	return cdesc;
348 }
349 
350 
setup_descr_chain(struct cryptocop_dma_desc * cd)351 static void setup_descr_chain(struct cryptocop_dma_desc *cd)
352 {
353 	DEBUG(printk("setup_descr_chain: entering\n"));
354 	while (cd) {
355 		if (cd->next) {
356 			cd->dma_descr->next = (dma_descr_data*)virt_to_phys(cd->next->dma_descr);
357 		} else {
358 			cd->dma_descr->next = NULL;
359 		}
360 		cd = cd->next;
361 	}
362 	DEBUG(printk("setup_descr_chain: exit\n"));
363 }
364 
365 
366 /* Create a pad descriptor for the transform.
367  * Return -1 for error, 0 if pad created. */
create_pad_descriptor(struct cryptocop_tfrm_ctx * tc,struct cryptocop_dma_desc ** pad_desc,int alloc_flag)368 static int create_pad_descriptor(struct cryptocop_tfrm_ctx *tc, struct cryptocop_dma_desc **pad_desc, int alloc_flag)
369 {
370 	struct cryptocop_dma_desc        *cdesc = NULL;
371 	int                              error = 0;
372 	struct strcop_meta_out           mo = {
373 		.ciphsel = src_none,
374 		.hashsel = src_none,
375 		.csumsel = src_none
376 	};
377 	char                             *pad;
378 	size_t                           plen;
379 
380 	DEBUG(printk("create_pad_descriptor: start.\n"));
381 	/* Setup pad descriptor. */
382 
383 	DEBUG(printk("create_pad_descriptor: setting up padding.\n"));
384 	cdesc = alloc_cdesc(alloc_flag);
385 	if (!cdesc){
386 		DEBUG_API(printk("create_pad_descriptor: alloc pad desc\n"));
387 		goto error_cleanup;
388 	}
389 	switch (tc->unit_no) {
390 	case src_md5:
391 		error = create_md5_pad(alloc_flag, tc->consumed, &pad, &plen);
392 		if (error){
393 			DEBUG_API(printk("create_pad_descriptor: create_md5_pad_failed\n"));
394 			goto error_cleanup;
395 		}
396 		cdesc->free_buf = pad;
397 		mo.hashsel = src_dma;
398 		mo.hashconf = tc->hash_conf;
399 		mo.hashmode = tc->hash_mode;
400 		break;
401 	case src_sha1:
402 		error = create_sha1_pad(alloc_flag, tc->consumed, &pad, &plen);
403 		if (error){
404 			DEBUG_API(printk("create_pad_descriptor: create_sha1_pad_failed\n"));
405 			goto error_cleanup;
406 		}
407 		cdesc->free_buf = pad;
408 		mo.hashsel = src_dma;
409 		mo.hashconf = tc->hash_conf;
410 		mo.hashmode = tc->hash_mode;
411 		break;
412 	case src_csum:
413 		if (tc->consumed % tc->blocklength){
414 			pad = (char*)csum_zero_pad;
415 			plen = 1;
416 		} else {
417 			pad = (char*)cdesc; /* Use any pointer. */
418 			plen = 0;
419 		}
420 		mo.csumsel = src_dma;
421 		break;
422 	}
423 	cdesc->dma_descr->wait = 1;
424 	cdesc->dma_descr->out_eop = 1; /* Since this is a pad output is pushed.  EOP is ok here since the padded unit is the only one active. */
425 	cdesc->dma_descr->buf = (char*)virt_to_phys((char*)pad);
426 	cdesc->dma_descr->after = cdesc->dma_descr->buf + plen;
427 
428 	cdesc->dma_descr->md = REG_TYPE_CONV(unsigned short int, struct strcop_meta_out, mo);
429 	*pad_desc = cdesc;
430 
431 	return 0;
432 
433  error_cleanup:
434 	if (cdesc) free_cdesc(cdesc);
435 	return -1;
436 }
437 
438 
setup_key_dl_desc(struct cryptocop_tfrm_ctx * tc,struct cryptocop_dma_desc ** kd,int alloc_flag)439 static int setup_key_dl_desc(struct cryptocop_tfrm_ctx *tc, struct cryptocop_dma_desc **kd, int alloc_flag)
440 {
441 	struct cryptocop_dma_desc  *key_desc = alloc_cdesc(alloc_flag);
442 	struct strcop_meta_out     mo = {0};
443 
444 	DEBUG(printk("setup_key_dl_desc\n"));
445 
446 	if (!key_desc) {
447 		DEBUG_API(printk("setup_key_dl_desc: failed descriptor allocation.\n"));
448 		return -ENOMEM;
449 	}
450 
451 	/* Download key. */
452 	if ((tc->tctx->init.alg == cryptocop_alg_aes) && (tc->tcfg->flags & CRYPTOCOP_DECRYPT)) {
453 		/* Precook the AES decrypt key. */
454 		if (!tc->tctx->dec_key_set){
455 			get_aes_decrypt_key(tc->tctx->dec_key, tc->tctx->init.key, tc->tctx->init.keylen);
456 			tc->tctx->dec_key_set = 1;
457 		}
458 		key_desc->dma_descr->buf = (char*)virt_to_phys(tc->tctx->dec_key);
459 		key_desc->dma_descr->after = key_desc->dma_descr->buf + tc->tctx->init.keylen/8;
460 	} else {
461 		key_desc->dma_descr->buf = (char*)virt_to_phys(tc->tctx->init.key);
462 		key_desc->dma_descr->after = key_desc->dma_descr->buf + tc->tctx->init.keylen/8;
463 	}
464 	/* Setup metadata. */
465 	mo.dlkey = 1;
466 	switch (tc->tctx->init.keylen) {
467 	case 64:
468 		mo.decrypt = 0;
469 		mo.hashmode = 0;
470 		break;
471 	case 128:
472 		mo.decrypt = 0;
473 		mo.hashmode = 1;
474 		break;
475 	case 192:
476 		mo.decrypt = 1;
477 		mo.hashmode = 0;
478 		break;
479 	case 256:
480 		mo.decrypt = 1;
481 		mo.hashmode = 1;
482 		break;
483 	default:
484 		break;
485 	}
486 	mo.ciphsel = mo.hashsel = mo.csumsel = src_none;
487 	key_desc->dma_descr->md = REG_TYPE_CONV(unsigned short int, struct strcop_meta_out, mo);
488 
489 	key_desc->dma_descr->out_eop = 1;
490 	key_desc->dma_descr->wait = 1;
491 	key_desc->dma_descr->intr = 0;
492 
493 	*kd = key_desc;
494 	return 0;
495 }
496 
setup_cipher_iv_desc(struct cryptocop_tfrm_ctx * tc,struct cryptocop_dma_desc ** id,int alloc_flag)497 static int setup_cipher_iv_desc(struct cryptocop_tfrm_ctx *tc, struct cryptocop_dma_desc **id, int alloc_flag)
498 {
499 	struct cryptocop_dma_desc  *iv_desc = alloc_cdesc(alloc_flag);
500 	struct strcop_meta_out     mo = {0};
501 
502 	DEBUG(printk("setup_cipher_iv_desc\n"));
503 
504 	if (!iv_desc) {
505 		DEBUG_API(printk("setup_cipher_iv_desc: failed CBC IV descriptor allocation.\n"));
506 		return -ENOMEM;
507 	}
508 	/* Download IV. */
509 	iv_desc->dma_descr->buf = (char*)virt_to_phys(tc->tcfg->iv);
510 	iv_desc->dma_descr->after = iv_desc->dma_descr->buf + tc->blocklength;
511 
512 	/* Setup metadata. */
513 	mo.hashsel = mo.csumsel = src_none;
514 	mo.ciphsel = src_dma;
515 	mo.ciphconf = tc->ciph_conf;
516 	mo.cbcmode = tc->cbcmode;
517 
518 	iv_desc->dma_descr->md = REG_TYPE_CONV(unsigned short int, struct strcop_meta_out, mo);
519 
520 	iv_desc->dma_descr->out_eop = 0;
521 	iv_desc->dma_descr->wait = 1;
522 	iv_desc->dma_descr->intr = 0;
523 
524 	*id = iv_desc;
525 	return 0;
526 }
527 
528 /* Map the ouput length of the transform to operation output starting on the inject index. */
create_input_descriptors(struct cryptocop_operation * operation,struct cryptocop_tfrm_ctx * tc,struct cryptocop_dma_desc ** id,int alloc_flag)529 static int create_input_descriptors(struct cryptocop_operation *operation, struct cryptocop_tfrm_ctx *tc, struct cryptocop_dma_desc **id, int alloc_flag)
530 {
531 	int                        err = 0;
532 	struct cryptocop_dma_desc  head = {0};
533 	struct cryptocop_dma_desc  *outdesc = &head;
534 	size_t                     iov_offset = 0;
535 	size_t                     out_ix = 0;
536 	int                        outiov_ix = 0;
537 	struct strcop_meta_in      mi = {0};
538 
539 	size_t                     out_length = tc->produced;
540 	int                        rem_length;
541 	int                        dlength;
542 
543 	assert(out_length != 0);
544 	if (((tc->produced + tc->tcfg->inject_ix) > operation->tfrm_op.outlen) || (tc->produced && (operation->tfrm_op.outlen == 0))) {
545 		DEBUG_API(printk("create_input_descriptors: operation outdata too small\n"));
546 		return -EINVAL;
547 	}
548 	/* Traverse the out iovec until the result inject index is reached. */
549 	while ((outiov_ix < operation->tfrm_op.outcount) && ((out_ix + operation->tfrm_op.outdata[outiov_ix].iov_len) <= tc->tcfg->inject_ix)){
550 		out_ix += operation->tfrm_op.outdata[outiov_ix].iov_len;
551 		outiov_ix++;
552 	}
553 	if (outiov_ix >= operation->tfrm_op.outcount){
554 		DEBUG_API(printk("create_input_descriptors: operation outdata too small\n"));
555 		return -EINVAL;
556 	}
557 	iov_offset = tc->tcfg->inject_ix - out_ix;
558 	mi.dmasel = tc->unit_no;
559 
560 	/* Setup the output descriptors. */
561 	while ((out_length > 0) && (outiov_ix < operation->tfrm_op.outcount)) {
562 		outdesc->next = alloc_cdesc(alloc_flag);
563 		if (!outdesc->next) {
564 			DEBUG_API(printk("create_input_descriptors: alloc_cdesc\n"));
565 			err = -ENOMEM;
566 			goto error_cleanup;
567 		}
568 		outdesc = outdesc->next;
569 		rem_length = operation->tfrm_op.outdata[outiov_ix].iov_len - iov_offset;
570 		dlength = (out_length < rem_length) ? out_length : rem_length;
571 
572 		DEBUG(printk("create_input_descriptors:\n"
573 			     "outiov_ix=%d, rem_length=%d, dlength=%d\n"
574 			     "iov_offset=%d, outdata[outiov_ix].iov_len=%d\n"
575 			     "outcount=%d, outiov_ix=%d\n",
576 			     outiov_ix, rem_length, dlength, iov_offset, operation->tfrm_op.outdata[outiov_ix].iov_len, operation->tfrm_op.outcount, outiov_ix));
577 
578 		outdesc->dma_descr->buf = (char*)virt_to_phys(operation->tfrm_op.outdata[outiov_ix].iov_base + iov_offset);
579 		outdesc->dma_descr->after = outdesc->dma_descr->buf + dlength;
580 		outdesc->dma_descr->md = REG_TYPE_CONV(unsigned short int, struct strcop_meta_in, mi);
581 
582 		out_length -= dlength;
583 		iov_offset += dlength;
584 		if (iov_offset >= operation->tfrm_op.outdata[outiov_ix].iov_len) {
585 			iov_offset = 0;
586 			++outiov_ix;
587 		}
588 	}
589 	if (out_length > 0){
590 		DEBUG_API(printk("create_input_descriptors: not enough room for output, %d remained\n", out_length));
591 		err = -EINVAL;
592 		goto error_cleanup;
593 	}
594 	/* Set sync in last descriptor. */
595 	mi.sync = 1;
596 	outdesc->dma_descr->md = REG_TYPE_CONV(unsigned short int, struct strcop_meta_in, mi);
597 
598 	*id = head.next;
599 	return 0;
600 
601  error_cleanup:
602 	while (head.next) {
603 		outdesc = head.next->next;
604 		free_cdesc(head.next);
605 		head.next = outdesc;
606 	}
607 	return err;
608 }
609 
610 
create_output_descriptors(struct cryptocop_operation * operation,int * iniov_ix,int * iniov_offset,size_t desc_len,struct cryptocop_dma_desc ** current_out_cdesc,struct strcop_meta_out * meta_out,int alloc_flag)611 static int create_output_descriptors(struct cryptocop_operation *operation, int *iniov_ix, int *iniov_offset, size_t desc_len, struct cryptocop_dma_desc **current_out_cdesc, struct strcop_meta_out *meta_out, int alloc_flag)
612 {
613 	while (desc_len != 0) {
614 		struct cryptocop_dma_desc  *cdesc;
615 		int                        rem_length = operation->tfrm_op.indata[*iniov_ix].iov_len - *iniov_offset;
616 		int                        dlength = (desc_len < rem_length) ? desc_len : rem_length;
617 
618 		cdesc = alloc_cdesc(alloc_flag);
619 		if (!cdesc) {
620 			DEBUG_API(printk("create_output_descriptors: alloc_cdesc\n"));
621 			return -ENOMEM;
622 		}
623 		(*current_out_cdesc)->next = cdesc;
624 		(*current_out_cdesc) = cdesc;
625 
626 		cdesc->free_buf = NULL;
627 
628 		cdesc->dma_descr->buf = (char*)virt_to_phys(operation->tfrm_op.indata[*iniov_ix].iov_base + *iniov_offset);
629 		cdesc->dma_descr->after = cdesc->dma_descr->buf + dlength;
630 
631 		assert(desc_len >= dlength);
632 		desc_len -= dlength;
633 		*iniov_offset += dlength;
634 		if (*iniov_offset >= operation->tfrm_op.indata[*iniov_ix].iov_len) {
635 			*iniov_offset = 0;
636 			++(*iniov_ix);
637 			if (*iniov_ix > operation->tfrm_op.incount) {
638 				DEBUG_API(printk("create_output_descriptors: not enough indata in operation."));
639 				return  -EINVAL;
640 			}
641 		}
642 		cdesc->dma_descr->md = REG_TYPE_CONV(unsigned short int, struct strcop_meta_out, (*meta_out));
643 	} /* while (desc_len != 0) */
644 	/* Last DMA descriptor gets a 'wait' bit to signal expected change in metadata. */
645 	(*current_out_cdesc)->dma_descr->wait = 1; /* This will set extraneous WAIT in some situations, e.g. when padding hashes and checksums. */
646 
647 	return 0;
648 }
649 
650 
append_input_descriptors(struct cryptocop_operation * operation,struct cryptocop_dma_desc ** current_in_cdesc,struct cryptocop_dma_desc ** current_out_cdesc,struct cryptocop_tfrm_ctx * tc,int alloc_flag)651 static int append_input_descriptors(struct cryptocop_operation *operation, struct cryptocop_dma_desc **current_in_cdesc, struct cryptocop_dma_desc **current_out_cdesc, struct cryptocop_tfrm_ctx *tc, int alloc_flag)
652 {
653 	DEBUG(printk("append_input_descriptors, tc=0x%p, unit_no=%d\n", tc, tc->unit_no));
654 	if (tc->tcfg) {
655 		int                        failed = 0;
656 		struct cryptocop_dma_desc  *idescs = NULL;
657 		DEBUG(printk("append_input_descriptors: pushing output, consumed %d produced %d bytes.\n", tc->consumed, tc->produced));
658 		if (tc->pad_descs) {
659 			DEBUG(printk("append_input_descriptors: append pad descriptors to DMA out list.\n"));
660 			while (tc->pad_descs) {
661 				DEBUG(printk("append descriptor 0x%p\n", tc->pad_descs));
662 				(*current_out_cdesc)->next = tc->pad_descs;
663 				tc->pad_descs = tc->pad_descs->next;
664 				(*current_out_cdesc) = (*current_out_cdesc)->next;
665 			}
666 		}
667 
668 		/* Setup and append output descriptors to DMA in list. */
669 		if (tc->unit_no == src_dma){
670 			/* mem2mem.  Setup DMA in descriptors to discard all input prior to the requested mem2mem data. */
671 			struct strcop_meta_in mi = {.sync = 0, .dmasel = src_dma};
672 			unsigned int start_ix = tc->start_ix;
673 			while (start_ix){
674 				unsigned int desclen = start_ix < MEM2MEM_DISCARD_BUF_LENGTH ? start_ix : MEM2MEM_DISCARD_BUF_LENGTH;
675 				(*current_in_cdesc)->next = alloc_cdesc(alloc_flag);
676 				if (!(*current_in_cdesc)->next){
677 					DEBUG_API(printk("append_input_descriptors: alloc_cdesc mem2mem discard failed\n"));
678 					return -ENOMEM;
679 				}
680 				(*current_in_cdesc) = (*current_in_cdesc)->next;
681 				(*current_in_cdesc)->dma_descr->buf = (char*)virt_to_phys(mem2mem_discard_buf);
682 				(*current_in_cdesc)->dma_descr->after = (*current_in_cdesc)->dma_descr->buf + desclen;
683 				(*current_in_cdesc)->dma_descr->md = REG_TYPE_CONV(unsigned short int, struct strcop_meta_in, mi);
684 				start_ix -= desclen;
685 			}
686 			mi.sync = 1;
687 			(*current_in_cdesc)->dma_descr->md = REG_TYPE_CONV(unsigned short int, struct strcop_meta_in, mi);
688 		}
689 
690 		failed = create_input_descriptors(operation, tc, &idescs, alloc_flag);
691 		if (failed){
692 			DEBUG_API(printk("append_input_descriptors: output descriptor setup failed\n"));
693 			return failed;
694 		}
695 		DEBUG(printk("append_input_descriptors: append output descriptors to DMA in list.\n"));
696 		while (idescs) {
697 			DEBUG(printk("append descriptor 0x%p\n", idescs));
698 			(*current_in_cdesc)->next = idescs;
699 			idescs = idescs->next;
700 			(*current_in_cdesc) = (*current_in_cdesc)->next;
701 		}
702 	}
703 	return 0;
704 }
705 
706 
707 
cryptocop_setup_dma_list(struct cryptocop_operation * operation,struct cryptocop_int_operation ** int_op,int alloc_flag)708 static int cryptocop_setup_dma_list(struct cryptocop_operation *operation, struct cryptocop_int_operation **int_op, int alloc_flag)
709 {
710 	struct cryptocop_session *sess;
711 	struct cryptocop_transform_ctx *tctx;
712 
713 	struct cryptocop_tfrm_ctx digest_ctx = {
714 		.previous_src = src_none,
715 		.current_src = src_none,
716 		.start_ix = 0,
717 		.requires_padding = 1,
718 		.strict_block_length = 0,
719 		.hash_conf = 0,
720 		.hash_mode = 0,
721 		.ciph_conf = 0,
722 		.cbcmode = 0,
723 		.decrypt = 0,
724 		.consumed = 0,
725 		.produced = 0,
726 		.pad_descs = NULL,
727 		.active = 0,
728 		.done = 0,
729 		.prev_src = NULL,
730 		.curr_src = NULL,
731 		.tcfg = NULL};
732 	struct cryptocop_tfrm_ctx cipher_ctx = {
733 		.previous_src = src_none,
734 		.current_src = src_none,
735 		.start_ix = 0,
736 		.requires_padding = 0,
737 		.strict_block_length = 1,
738 		.hash_conf = 0,
739 		.hash_mode = 0,
740 		.ciph_conf = 0,
741 		.cbcmode = 0,
742 		.decrypt = 0,
743 		.consumed = 0,
744 		.produced = 0,
745 		.pad_descs = NULL,
746 		.active = 0,
747 		.done = 0,
748 		.prev_src = NULL,
749 		.curr_src = NULL,
750 		.tcfg = NULL};
751 	struct cryptocop_tfrm_ctx csum_ctx = {
752 		.previous_src = src_none,
753 		.current_src = src_none,
754 		.start_ix = 0,
755 		.blocklength = 2,
756 		.requires_padding = 1,
757 		.strict_block_length = 0,
758 		.hash_conf = 0,
759 		.hash_mode = 0,
760 		.ciph_conf = 0,
761 		.cbcmode = 0,
762 		.decrypt = 0,
763 		.consumed = 0,
764 		.produced = 0,
765 		.pad_descs = NULL,
766 		.active = 0,
767 		.done = 0,
768 		.tcfg = NULL,
769 		.prev_src = NULL,
770 		.curr_src = NULL,
771 		.unit_no = src_csum};
772 	struct cryptocop_tfrm_cfg *tcfg = operation->tfrm_op.tfrm_cfg;
773 
774 	unsigned int indata_ix = 0;
775 
776 	/* iovec accounting. */
777 	int iniov_ix = 0;
778 	int iniov_offset = 0;
779 
780 	/* Operation descriptor cfg traversal pointer. */
781 	struct cryptocop_desc *odsc;
782 
783 	int failed = 0;
784 	/* List heads for allocated descriptors. */
785 	struct cryptocop_dma_desc out_cdesc_head = {0};
786 	struct cryptocop_dma_desc in_cdesc_head = {0};
787 
788 	struct cryptocop_dma_desc *current_out_cdesc = &out_cdesc_head;
789 	struct cryptocop_dma_desc *current_in_cdesc = &in_cdesc_head;
790 
791 	struct cryptocop_tfrm_ctx *output_tc = NULL;
792 	void                      *iop_alloc_ptr;
793 
794 	assert(operation != NULL);
795 	assert(int_op != NULL);
796 
797 	DEBUG(printk("cryptocop_setup_dma_list: start\n"));
798 	DEBUG(print_cryptocop_operation(operation));
799 
800 	sess = get_session(operation->sid);
801 	if (!sess) {
802 		DEBUG_API(printk("cryptocop_setup_dma_list: no session found for operation.\n"));
803 		failed = -EINVAL;
804 		goto error_cleanup;
805 	}
806 	iop_alloc_ptr = kmalloc(DESCR_ALLOC_PAD + sizeof(struct cryptocop_int_operation), alloc_flag);
807 	if (!iop_alloc_ptr) {
808 		DEBUG_API(printk("cryptocop_setup_dma_list:  kmalloc cryptocop_int_operation\n"));
809 		failed = -ENOMEM;
810 		goto error_cleanup;
811 	}
812 	(*int_op) = (struct cryptocop_int_operation*)(((unsigned long int)(iop_alloc_ptr + DESCR_ALLOC_PAD + offsetof(struct cryptocop_int_operation, ctx_out)) & ~0x0000001F) - offsetof(struct cryptocop_int_operation, ctx_out));
813 	DEBUG(memset((*int_op), 0xff, sizeof(struct cryptocop_int_operation)));
814 	(*int_op)->alloc_ptr = iop_alloc_ptr;
815 	DEBUG(printk("cryptocop_setup_dma_list: *int_op=0x%p, alloc_ptr=0x%p\n", *int_op, (*int_op)->alloc_ptr));
816 
817 	(*int_op)->sid = operation->sid;
818 	(*int_op)->cdesc_out = NULL;
819 	(*int_op)->cdesc_in = NULL;
820 	(*int_op)->tdes_mode = cryptocop_3des_ede;
821 	(*int_op)->csum_mode = cryptocop_csum_le;
822 	(*int_op)->ddesc_out = NULL;
823 	(*int_op)->ddesc_in = NULL;
824 
825 	/* Scan operation->tfrm_op.tfrm_cfg for bad configuration and set up the local contexts. */
826 	if (!tcfg) {
827 		DEBUG_API(printk("cryptocop_setup_dma_list: no configured transforms in operation.\n"));
828 		failed = -EINVAL;
829 		goto error_cleanup;
830 	}
831 	while (tcfg) {
832 		tctx = get_transform_ctx(sess, tcfg->tid);
833 		if (!tctx) {
834 			DEBUG_API(printk("cryptocop_setup_dma_list: no transform id %d in session.\n", tcfg->tid));
835 			failed = -EINVAL;
836 			goto error_cleanup;
837 		}
838 		if (tcfg->inject_ix > operation->tfrm_op.outlen){
839 			DEBUG_API(printk("cryptocop_setup_dma_list: transform id %d inject_ix (%d) > operation->tfrm_op.outlen(%d)", tcfg->tid, tcfg->inject_ix, operation->tfrm_op.outlen));
840 			failed = -EINVAL;
841 			goto error_cleanup;
842 		}
843 		switch (tctx->init.alg){
844 		case cryptocop_alg_mem2mem:
845 			if (cipher_ctx.tcfg != NULL){
846 				DEBUG_API(printk("cryptocop_setup_dma_list: multiple ciphers in operation.\n"));
847 				failed = -EINVAL;
848 				goto error_cleanup;
849 			}
850 			/* mem2mem is handled as a NULL cipher. */
851 			cipher_ctx.cbcmode = 0;
852 			cipher_ctx.decrypt = 0;
853 			cipher_ctx.blocklength = 1;
854 			cipher_ctx.ciph_conf = 0;
855 			cipher_ctx.unit_no = src_dma;
856 			cipher_ctx.tcfg = tcfg;
857 			cipher_ctx.tctx = tctx;
858 			break;
859 		case cryptocop_alg_des:
860 		case cryptocop_alg_3des:
861 		case cryptocop_alg_aes:
862 			/* cipher */
863 			if (cipher_ctx.tcfg != NULL){
864 				DEBUG_API(printk("cryptocop_setup_dma_list: multiple ciphers in operation.\n"));
865 				failed = -EINVAL;
866 				goto error_cleanup;
867 			}
868 			cipher_ctx.tcfg = tcfg;
869 			cipher_ctx.tctx = tctx;
870 			if (cipher_ctx.tcfg->flags & CRYPTOCOP_DECRYPT){
871 				cipher_ctx.decrypt = 1;
872 			}
873 			switch (tctx->init.cipher_mode) {
874 			case cryptocop_cipher_mode_ecb:
875 				cipher_ctx.cbcmode = 0;
876 				break;
877 			case cryptocop_cipher_mode_cbc:
878 				cipher_ctx.cbcmode = 1;
879 				break;
880 			default:
881 				DEBUG_API(printk("cryptocop_setup_dma_list: cipher_ctx, bad cipher mode==%d\n", tctx->init.cipher_mode));
882 				failed = -EINVAL;
883 				goto error_cleanup;
884 			}
885 			DEBUG(printk("cryptocop_setup_dma_list: cipher_ctx, set CBC mode==%d\n", cipher_ctx.cbcmode));
886 			switch (tctx->init.alg){
887 			case cryptocop_alg_des:
888 				cipher_ctx.ciph_conf = 0;
889 				cipher_ctx.unit_no = src_des;
890 				cipher_ctx.blocklength = DES_BLOCK_LENGTH;
891 				break;
892 			case cryptocop_alg_3des:
893 				cipher_ctx.ciph_conf = 1;
894 				cipher_ctx.unit_no = src_des;
895 				cipher_ctx.blocklength = DES_BLOCK_LENGTH;
896 				break;
897 			case cryptocop_alg_aes:
898 				cipher_ctx.ciph_conf = 2;
899 				cipher_ctx.unit_no = src_aes;
900 				cipher_ctx.blocklength = AES_BLOCK_LENGTH;
901 				break;
902 			default:
903 				panic("cryptocop_setup_dma_list: impossible algorithm %d\n", tctx->init.alg);
904 			}
905 			(*int_op)->tdes_mode = tctx->init.tdes_mode;
906 			break;
907 		case cryptocop_alg_md5:
908 		case cryptocop_alg_sha1:
909 			/* digest */
910 			if (digest_ctx.tcfg != NULL){
911 				DEBUG_API(printk("cryptocop_setup_dma_list: multiple digests in operation.\n"));
912 				failed = -EINVAL;
913 				goto error_cleanup;
914 			}
915 			digest_ctx.tcfg = tcfg;
916 			digest_ctx.tctx = tctx;
917 			digest_ctx.hash_mode = 0; /* Don't use explicit IV in this API. */
918 			switch (tctx->init.alg){
919 			case cryptocop_alg_md5:
920 				digest_ctx.blocklength = MD5_BLOCK_LENGTH;
921 				digest_ctx.unit_no = src_md5;
922 				digest_ctx.hash_conf = 1; /* 1 => MD-5 */
923 				break;
924 			case cryptocop_alg_sha1:
925 				digest_ctx.blocklength = SHA1_BLOCK_LENGTH;
926 				digest_ctx.unit_no = src_sha1;
927 				digest_ctx.hash_conf = 0; /* 0 => SHA-1 */
928 				break;
929 			default:
930 				panic("cryptocop_setup_dma_list: impossible digest algorithm\n");
931 			}
932 			break;
933 		case cryptocop_alg_csum:
934 			/* digest */
935 			if (csum_ctx.tcfg != NULL){
936 				DEBUG_API(printk("cryptocop_setup_dma_list: multiple checksums in operation.\n"));
937 				failed = -EINVAL;
938 				goto error_cleanup;
939 			}
940 			(*int_op)->csum_mode = tctx->init.csum_mode;
941 			csum_ctx.tcfg = tcfg;
942 			csum_ctx.tctx = tctx;
943 			break;
944 		default:
945 			/* no algorithm. */
946 			DEBUG_API(printk("cryptocop_setup_dma_list: invalid algorithm %d specified in tfrm %d.\n", tctx->init.alg, tcfg->tid));
947 			failed = -EINVAL;
948 			goto error_cleanup;
949 		}
950 		tcfg = tcfg->next;
951 	}
952 	/* Download key if a cipher is used. */
953 	if (cipher_ctx.tcfg && (cipher_ctx.tctx->init.alg != cryptocop_alg_mem2mem)){
954 		struct cryptocop_dma_desc  *key_desc = NULL;
955 
956 		failed = setup_key_dl_desc(&cipher_ctx, &key_desc, alloc_flag);
957 		if (failed) {
958 			DEBUG_API(printk("cryptocop_setup_dma_list: setup key dl\n"));
959 			goto error_cleanup;
960 		}
961 		current_out_cdesc->next = key_desc;
962 		current_out_cdesc = key_desc;
963 		indata_ix += (unsigned int)(key_desc->dma_descr->after - key_desc->dma_descr->buf);
964 
965 		/* Download explicit IV if a cipher is used and CBC mode and explicit IV selected. */
966 		if ((cipher_ctx.tctx->init.cipher_mode == cryptocop_cipher_mode_cbc) && (cipher_ctx.tcfg->flags & CRYPTOCOP_EXPLICIT_IV)) {
967 			struct cryptocop_dma_desc  *iv_desc = NULL;
968 
969 			DEBUG(printk("cryptocop_setup_dma_list: setup cipher CBC IV descriptor.\n"));
970 
971 			failed = setup_cipher_iv_desc(&cipher_ctx, &iv_desc, alloc_flag);
972 			if (failed) {
973 				DEBUG_API(printk("cryptocop_setup_dma_list: CBC IV descriptor.\n"));
974 				goto error_cleanup;
975 			}
976 			current_out_cdesc->next = iv_desc;
977 			current_out_cdesc = iv_desc;
978 			indata_ix += (unsigned int)(iv_desc->dma_descr->after - iv_desc->dma_descr->buf);
979 		}
980 	}
981 
982 	/* Process descriptors. */
983 	odsc = operation->tfrm_op.desc;
984 	while (odsc) {
985 		struct cryptocop_desc_cfg   *dcfg = odsc->cfg;
986 		struct strcop_meta_out      meta_out = {0};
987 		size_t                      desc_len = odsc->length;
988 		int                         active_count, eop_needed_count;
989 
990 		output_tc = NULL;
991 
992 		DEBUG(printk("cryptocop_setup_dma_list: parsing an operation descriptor\n"));
993 
994 		while (dcfg) {
995 			struct cryptocop_tfrm_ctx  *tc = NULL;
996 
997 			DEBUG(printk("cryptocop_setup_dma_list: parsing an operation descriptor configuration.\n"));
998 			/* Get the local context for the transform and mark it as the output unit if it produces output. */
999 			if (digest_ctx.tcfg && (digest_ctx.tcfg->tid == dcfg->tid)){
1000 				tc = &digest_ctx;
1001 			} else if (cipher_ctx.tcfg && (cipher_ctx.tcfg->tid == dcfg->tid)){
1002 				tc = &cipher_ctx;
1003 			} else if (csum_ctx.tcfg && (csum_ctx.tcfg->tid == dcfg->tid)){
1004 				tc = &csum_ctx;
1005 			}
1006 			if (!tc) {
1007 				DEBUG_API(printk("cryptocop_setup_dma_list: invalid transform %d specified in descriptor.\n", dcfg->tid));
1008 				failed = -EINVAL;
1009 				goto error_cleanup;
1010 			}
1011 			if (tc->done) {
1012 				DEBUG_API(printk("cryptocop_setup_dma_list: completed transform %d reused.\n", dcfg->tid));
1013 				failed = -EINVAL;
1014 				goto error_cleanup;
1015 			}
1016 			if (!tc->active) {
1017 				tc->start_ix = indata_ix;
1018 				tc->active = 1;
1019 			}
1020 
1021 			tc->previous_src = tc->current_src;
1022 			tc->prev_src = tc->curr_src;
1023 			/* Map source unit id to DMA source config. */
1024 			switch (dcfg->src){
1025 			case cryptocop_source_dma:
1026 				tc->current_src = src_dma;
1027 				break;
1028 			case cryptocop_source_des:
1029 				tc->current_src = src_des;
1030 				break;
1031 			case cryptocop_source_3des:
1032 				tc->current_src = src_des;
1033 				break;
1034 			case cryptocop_source_aes:
1035 				tc->current_src = src_aes;
1036 				break;
1037 			case cryptocop_source_md5:
1038 			case cryptocop_source_sha1:
1039 			case cryptocop_source_csum:
1040 			case cryptocop_source_none:
1041 			default:
1042 				/* We do not allow using accumulating style units (SHA-1, MD5, checksum) as sources to other units.
1043 				 */
1044 				DEBUG_API(printk("cryptocop_setup_dma_list: bad unit source configured %d.\n", dcfg->src));
1045 				failed = -EINVAL;
1046 				goto error_cleanup;
1047 			}
1048 			if (tc->current_src != src_dma) {
1049 				/* Find the unit we are sourcing from. */
1050 				if (digest_ctx.unit_no == tc->current_src){
1051 					tc->curr_src = &digest_ctx;
1052 				} else if (cipher_ctx.unit_no == tc->current_src){
1053 					tc->curr_src = &cipher_ctx;
1054 				} else if (csum_ctx.unit_no == tc->current_src){
1055 					tc->curr_src = &csum_ctx;
1056 				}
1057 				if ((tc->curr_src == tc) && (tc->unit_no != src_dma)){
1058 					DEBUG_API(printk("cryptocop_setup_dma_list: unit %d configured to source from itself.\n", tc->unit_no));
1059 					failed = -EINVAL;
1060 					goto error_cleanup;
1061 				}
1062 			} else {
1063 				tc->curr_src = NULL;
1064 			}
1065 
1066 			/* Detect source switch. */
1067 			DEBUG(printk("cryptocop_setup_dma_list: tc->active=%d tc->unit_no=%d tc->current_src=%d tc->previous_src=%d, tc->curr_src=0x%p, tc->prev_srv=0x%p\n", tc->active, tc->unit_no, tc->current_src, tc->previous_src, tc->curr_src, tc->prev_src));
1068 			if (tc->active && (tc->current_src != tc->previous_src)) {
1069 				/* Only allow source switch when both the old source unit and the new one have
1070 				 * no pending data to process (i.e. the consumed length must be a multiple of the
1071 				 * transform blocklength). */
1072 				/* Note: if the src == NULL we are actually sourcing from DMA out. */
1073 				if (((tc->prev_src != NULL) && (tc->prev_src->consumed % tc->prev_src->blocklength)) ||
1074 				    ((tc->curr_src != NULL) && (tc->curr_src->consumed % tc->curr_src->blocklength)))
1075 				{
1076 					DEBUG_API(printk("cryptocop_setup_dma_list: can only disconnect from or connect to a unit on a multiple of the blocklength, old: cons=%d, prod=%d, block=%d, new: cons=%d prod=%d, block=%d.\n", tc->prev_src ? tc->prev_src->consumed : INT_MIN, tc->prev_src ? tc->prev_src->produced : INT_MIN, tc->prev_src ? tc->prev_src->blocklength : INT_MIN, tc->curr_src ? tc->curr_src->consumed : INT_MIN, tc->curr_src ? tc->curr_src->produced : INT_MIN, tc->curr_src ? tc->curr_src->blocklength : INT_MIN));
1077 					failed = -EINVAL;
1078 					goto error_cleanup;
1079 				}
1080 			}
1081 			/* Detect unit deactivation. */
1082 			if (dcfg->last) {
1083 				/* Length check of this is handled below. */
1084 				tc->done = 1;
1085 			}
1086 			dcfg = dcfg->next;
1087 		} /* while (dcfg) */
1088 		DEBUG(printk("cryptocop_setup_dma_list: parsing operation descriptor configuration complete.\n"));
1089 
1090 		if (cipher_ctx.active && (cipher_ctx.curr_src != NULL) && !cipher_ctx.curr_src->active){
1091 			DEBUG_API(printk("cryptocop_setup_dma_list: cipher source from inactive unit %d\n", cipher_ctx.curr_src->unit_no));
1092 			failed = -EINVAL;
1093 			goto error_cleanup;
1094 		}
1095 		if (digest_ctx.active && (digest_ctx.curr_src != NULL) && !digest_ctx.curr_src->active){
1096 			DEBUG_API(printk("cryptocop_setup_dma_list: digest source from inactive unit %d\n", digest_ctx.curr_src->unit_no));
1097 			failed = -EINVAL;
1098 			goto error_cleanup;
1099 		}
1100 		if (csum_ctx.active && (csum_ctx.curr_src != NULL) && !csum_ctx.curr_src->active){
1101 			DEBUG_API(printk("cryptocop_setup_dma_list: cipher source from inactive unit %d\n", csum_ctx.curr_src->unit_no));
1102 			failed = -EINVAL;
1103 			goto error_cleanup;
1104 		}
1105 
1106 		/* Update consumed and produced lengths.
1107 
1108 		   The consumed length accounting here is actually cheating.  If a unit source from DMA (or any
1109 		   other unit that process data in blocks of one octet) it is correct, but if it source from a
1110 		   block processing unit, i.e. a cipher, it will be temporarily incorrect at some times.  However
1111 		   since it is only allowed--by the HW--to change source to or from a block processing unit at times where that
1112 		   unit has processed an exact multiple of its block length the end result will be correct.
1113 		   Beware that if the source change restriction change this code will need to be (much) reworked.
1114 		*/
1115 		DEBUG(printk("cryptocop_setup_dma_list: desc->length=%d, desc_len=%d.\n", odsc->length, desc_len));
1116 
1117 		if (csum_ctx.active) {
1118 			csum_ctx.consumed += desc_len;
1119 			if (csum_ctx.done) {
1120 				csum_ctx.produced = 2;
1121 			}
1122 			DEBUG(printk("cryptocop_setup_dma_list: csum_ctx producing: consumed=%d, produced=%d, blocklength=%d.\n", csum_ctx.consumed, csum_ctx.produced, csum_ctx.blocklength));
1123 		}
1124 		if (digest_ctx.active) {
1125 			digest_ctx.consumed += desc_len;
1126 			if (digest_ctx.done) {
1127 				if (digest_ctx.unit_no == src_md5) {
1128 					digest_ctx.produced = MD5_STATE_LENGTH;
1129 				} else {
1130 					digest_ctx.produced = SHA1_STATE_LENGTH;
1131 				}
1132 			}
1133 			DEBUG(printk("cryptocop_setup_dma_list: digest_ctx producing: consumed=%d, produced=%d, blocklength=%d.\n", digest_ctx.consumed, digest_ctx.produced, digest_ctx.blocklength));
1134 		}
1135 		if (cipher_ctx.active) {
1136 			/* Ciphers are allowed only to source from DMA out.  That is filtered above. */
1137 			assert(cipher_ctx.current_src == src_dma);
1138 			cipher_ctx.consumed += desc_len;
1139 			cipher_ctx.produced = cipher_ctx.blocklength * (cipher_ctx.consumed / cipher_ctx.blocklength);
1140 			if (cipher_ctx.cbcmode && !(cipher_ctx.tcfg->flags & CRYPTOCOP_EXPLICIT_IV) && cipher_ctx.produced){
1141 				cipher_ctx.produced -= cipher_ctx.blocklength; /* Compensate for CBC iv. */
1142 			}
1143 			DEBUG(printk("cryptocop_setup_dma_list: cipher_ctx producing: consumed=%d, produced=%d, blocklength=%d.\n", cipher_ctx.consumed, cipher_ctx.produced, cipher_ctx.blocklength));
1144 		}
1145 
1146 		/* Setup the DMA out descriptors. */
1147 		/* Configure the metadata. */
1148 		active_count = 0;
1149 		eop_needed_count = 0;
1150 		if (cipher_ctx.active) {
1151 			++active_count;
1152 			if (cipher_ctx.unit_no == src_dma){
1153 				/* mem2mem */
1154 				meta_out.ciphsel = src_none;
1155 			} else {
1156 				meta_out.ciphsel = cipher_ctx.current_src;
1157 			}
1158 			meta_out.ciphconf = cipher_ctx.ciph_conf;
1159 			meta_out.cbcmode = cipher_ctx.cbcmode;
1160 			meta_out.decrypt = cipher_ctx.decrypt;
1161 			DEBUG(printk("set ciphsel=%d ciphconf=%d cbcmode=%d decrypt=%d\n", meta_out.ciphsel, meta_out.ciphconf, meta_out.cbcmode, meta_out.decrypt));
1162 			if (cipher_ctx.done) ++eop_needed_count;
1163 		} else {
1164 			meta_out.ciphsel = src_none;
1165 		}
1166 
1167 		if (digest_ctx.active) {
1168 			++active_count;
1169 			meta_out.hashsel = digest_ctx.current_src;
1170 			meta_out.hashconf = digest_ctx.hash_conf;
1171 			meta_out.hashmode = 0; /* Explicit mode is not used here. */
1172 			DEBUG(printk("set hashsel=%d hashconf=%d hashmode=%d\n", meta_out.hashsel, meta_out.hashconf, meta_out.hashmode));
1173 			if (digest_ctx.done) {
1174 				assert(digest_ctx.pad_descs == NULL);
1175 				failed = create_pad_descriptor(&digest_ctx, &digest_ctx.pad_descs, alloc_flag);
1176 				if (failed) {
1177 					DEBUG_API(printk("cryptocop_setup_dma_list: failed digest pad creation.\n"));
1178 					goto error_cleanup;
1179 				}
1180 			}
1181 		} else {
1182 			meta_out.hashsel = src_none;
1183 		}
1184 
1185 		if (csum_ctx.active) {
1186 			++active_count;
1187 			meta_out.csumsel = csum_ctx.current_src;
1188 			if (csum_ctx.done) {
1189 				assert(csum_ctx.pad_descs == NULL);
1190 				failed = create_pad_descriptor(&csum_ctx, &csum_ctx.pad_descs, alloc_flag);
1191 				if (failed) {
1192 					DEBUG_API(printk("cryptocop_setup_dma_list: failed csum pad creation.\n"));
1193 					goto error_cleanup;
1194 				}
1195 			}
1196 		} else {
1197 			meta_out.csumsel = src_none;
1198 		}
1199 		DEBUG(printk("cryptocop_setup_dma_list: %d eop needed, %d active units\n", eop_needed_count, active_count));
1200 		/* Setup DMA out descriptors for the indata. */
1201 		failed = create_output_descriptors(operation, &iniov_ix, &iniov_offset, desc_len, &current_out_cdesc, &meta_out, alloc_flag);
1202 		if (failed) {
1203 			DEBUG_API(printk("cryptocop_setup_dma_list: create_output_descriptors %d\n", failed));
1204 			goto error_cleanup;
1205 		}
1206 		/* Setup out EOP.  If there are active units that are not done here they cannot get an EOP
1207 		 * so we ust setup a zero length descriptor to DMA to signal EOP only to done units.
1208 		 * If there is a pad descriptor EOP for the padded unit will be EOPed by it.
1209 		 */
1210 		assert(active_count >= eop_needed_count);
1211 		assert((eop_needed_count == 0) || (eop_needed_count == 1));
1212 		if (eop_needed_count) {
1213 			/* This means that the bulk operation (cipeher/m2m) is terminated. */
1214 			if (active_count > 1) {
1215 				/* Use zero length EOP descriptor. */
1216 				struct cryptocop_dma_desc *ed = alloc_cdesc(alloc_flag);
1217 				struct strcop_meta_out    ed_mo = {0};
1218 				if (!ed) {
1219 					DEBUG_API(printk("cryptocop_setup_dma_list: alloc EOP descriptor for cipher\n"));
1220 					failed = -ENOMEM;
1221 					goto error_cleanup;
1222 				}
1223 
1224 				assert(cipher_ctx.active && cipher_ctx.done);
1225 
1226 				if (cipher_ctx.unit_no == src_dma){
1227 					/* mem2mem */
1228 					ed_mo.ciphsel = src_none;
1229 				} else {
1230 					ed_mo.ciphsel = cipher_ctx.current_src;
1231 				}
1232 				ed_mo.ciphconf = cipher_ctx.ciph_conf;
1233 				ed_mo.cbcmode = cipher_ctx.cbcmode;
1234 				ed_mo.decrypt = cipher_ctx.decrypt;
1235 
1236 				ed->free_buf = NULL;
1237 				ed->dma_descr->wait = 1;
1238 				ed->dma_descr->out_eop = 1;
1239 
1240 				ed->dma_descr->buf = (char*)virt_to_phys(&ed); /* Use any valid physical address for zero length descriptor. */
1241 				ed->dma_descr->after = ed->dma_descr->buf;
1242 				ed->dma_descr->md = REG_TYPE_CONV(unsigned short int, struct strcop_meta_out, ed_mo);
1243 				current_out_cdesc->next = ed;
1244 				current_out_cdesc = ed;
1245 			} else {
1246 				/* Set EOP in the current out descriptor since the only active module is
1247 				 * the one needing the EOP. */
1248 
1249 				current_out_cdesc->dma_descr->out_eop = 1;
1250 			}
1251 		}
1252 
1253 		if (cipher_ctx.done && cipher_ctx.active) cipher_ctx.active = 0;
1254 		if (digest_ctx.done && digest_ctx.active) digest_ctx.active = 0;
1255 		if (csum_ctx.done && csum_ctx.active) csum_ctx.active = 0;
1256 		indata_ix += odsc->length;
1257 		odsc = odsc->next;
1258 	} /* while (odsc) */ /* Process descriptors. */
1259 	DEBUG(printk("cryptocop_setup_dma_list: done parsing operation descriptors\n"));
1260 	if (cipher_ctx.tcfg && (cipher_ctx.active || !cipher_ctx.done)){
1261 		DEBUG_API(printk("cryptocop_setup_dma_list: cipher operation not terminated.\n"));
1262 		failed = -EINVAL;
1263 		goto error_cleanup;
1264 	}
1265 	if (digest_ctx.tcfg && (digest_ctx.active || !digest_ctx.done)){
1266 		DEBUG_API(printk("cryptocop_setup_dma_list: digest operation not terminated.\n"));
1267 		failed = -EINVAL;
1268 		goto error_cleanup;
1269 	}
1270 	if (csum_ctx.tcfg && (csum_ctx.active || !csum_ctx.done)){
1271 		DEBUG_API(printk("cryptocop_setup_dma_list: csum operation not terminated.\n"));
1272 		failed = -EINVAL;
1273 		goto error_cleanup;
1274 	}
1275 
1276 	failed = append_input_descriptors(operation, &current_in_cdesc, &current_out_cdesc, &cipher_ctx, alloc_flag);
1277 	if (failed){
1278 		DEBUG_API(printk("cryptocop_setup_dma_list: append_input_descriptors cipher_ctx %d\n", failed));
1279 		goto error_cleanup;
1280 	}
1281 	failed = append_input_descriptors(operation, &current_in_cdesc, &current_out_cdesc, &digest_ctx, alloc_flag);
1282 	if (failed){
1283 		DEBUG_API(printk("cryptocop_setup_dma_list: append_input_descriptors cipher_ctx %d\n", failed));
1284 		goto error_cleanup;
1285 	}
1286 	failed = append_input_descriptors(operation, &current_in_cdesc, &current_out_cdesc, &csum_ctx, alloc_flag);
1287 	if (failed){
1288 		DEBUG_API(printk("cryptocop_setup_dma_list: append_input_descriptors cipher_ctx %d\n", failed));
1289 		goto error_cleanup;
1290 	}
1291 
1292 	DEBUG(printk("cryptocop_setup_dma_list: int_op=0x%p, *int_op=0x%p\n", int_op, *int_op));
1293 	(*int_op)->cdesc_out = out_cdesc_head.next;
1294 	(*int_op)->cdesc_in = in_cdesc_head.next;
1295 	DEBUG(printk("cryptocop_setup_dma_list: out_cdesc_head=0x%p in_cdesc_head=0x%p\n", (*int_op)->cdesc_out, (*int_op)->cdesc_in));
1296 
1297 	setup_descr_chain(out_cdesc_head.next);
1298 	setup_descr_chain(in_cdesc_head.next);
1299 
1300 	/* Last but not least: mark the last DMA in descriptor for a INTR and EOL and the the
1301 	 * last DMA out descriptor for EOL.
1302 	 */
1303 	current_in_cdesc->dma_descr->intr = 1;
1304 	current_in_cdesc->dma_descr->eol = 1;
1305 	current_out_cdesc->dma_descr->eol = 1;
1306 
1307 	/* Setup DMA contexts. */
1308 	(*int_op)->ctx_out.next = NULL;
1309 	(*int_op)->ctx_out.eol = 1;
1310 	(*int_op)->ctx_out.intr = 0;
1311 	(*int_op)->ctx_out.store_mode = 0;
1312 	(*int_op)->ctx_out.en = 0;
1313 	(*int_op)->ctx_out.dis = 0;
1314 	(*int_op)->ctx_out.md0 = 0;
1315 	(*int_op)->ctx_out.md1 = 0;
1316 	(*int_op)->ctx_out.md2 = 0;
1317 	(*int_op)->ctx_out.md3 = 0;
1318 	(*int_op)->ctx_out.md4 = 0;
1319 	(*int_op)->ctx_out.saved_data = (dma_descr_data*)virt_to_phys((*int_op)->cdesc_out->dma_descr);
1320 	(*int_op)->ctx_out.saved_data_buf = (*int_op)->cdesc_out->dma_descr->buf; /* Already physical address. */
1321 
1322 	(*int_op)->ctx_in.next = NULL;
1323 	(*int_op)->ctx_in.eol = 1;
1324 	(*int_op)->ctx_in.intr = 0;
1325 	(*int_op)->ctx_in.store_mode = 0;
1326 	(*int_op)->ctx_in.en = 0;
1327 	(*int_op)->ctx_in.dis = 0;
1328 	(*int_op)->ctx_in.md0 = 0;
1329 	(*int_op)->ctx_in.md1 = 0;
1330 	(*int_op)->ctx_in.md2 = 0;
1331 	(*int_op)->ctx_in.md3 = 0;
1332 	(*int_op)->ctx_in.md4 = 0;
1333 
1334 	(*int_op)->ctx_in.saved_data = (dma_descr_data*)virt_to_phys((*int_op)->cdesc_in->dma_descr);
1335 	(*int_op)->ctx_in.saved_data_buf = (*int_op)->cdesc_in->dma_descr->buf; /* Already physical address. */
1336 
1337 	DEBUG(printk("cryptocop_setup_dma_list: done\n"));
1338 	return 0;
1339 
1340 error_cleanup:
1341 	{
1342 		/* Free all allocated resources. */
1343 		struct cryptocop_dma_desc *tmp_cdesc;
1344 		while (digest_ctx.pad_descs){
1345 			tmp_cdesc = digest_ctx.pad_descs->next;
1346 			free_cdesc(digest_ctx.pad_descs);
1347 			digest_ctx.pad_descs = tmp_cdesc;
1348 		}
1349 		while (csum_ctx.pad_descs){
1350 			tmp_cdesc = csum_ctx.pad_descs->next;
1351 			free_cdesc(csum_ctx.pad_descs);
1352 			csum_ctx.pad_descs = tmp_cdesc;
1353 		}
1354 		assert(cipher_ctx.pad_descs == NULL); /* The ciphers are never padded. */
1355 
1356 		if (*int_op != NULL) delete_internal_operation(*int_op);
1357 	}
1358 	DEBUG_API(printk("cryptocop_setup_dma_list: done with error %d\n", failed));
1359 	return failed;
1360 }
1361 
1362 
delete_internal_operation(struct cryptocop_int_operation * iop)1363 static void delete_internal_operation(struct cryptocop_int_operation *iop)
1364 {
1365 	void                      *ptr = iop->alloc_ptr;
1366 	struct cryptocop_dma_desc *cd = iop->cdesc_out;
1367 	struct cryptocop_dma_desc *next;
1368 
1369 	DEBUG(printk("delete_internal_operation: iop=0x%p, alloc_ptr=0x%p\n", iop, ptr));
1370 
1371 	while (cd) {
1372 		next = cd->next;
1373 		free_cdesc(cd);
1374 		cd = next;
1375 	}
1376 	cd = iop->cdesc_in;
1377 	while (cd) {
1378 		next = cd->next;
1379 		free_cdesc(cd);
1380 		cd = next;
1381 	}
1382 	kfree(ptr);
1383 }
1384 
1385 #define MD5_MIN_PAD_LENGTH (9)
1386 #define MD5_PAD_LENGTH_FIELD_LENGTH (8)
1387 
create_md5_pad(int alloc_flag,unsigned long long hashed_length,char ** pad,size_t * pad_length)1388 static int create_md5_pad(int alloc_flag, unsigned long long hashed_length, char **pad, size_t *pad_length)
1389 {
1390 	size_t                  padlen = MD5_BLOCK_LENGTH - (hashed_length % MD5_BLOCK_LENGTH);
1391 	unsigned char           *p;
1392 	int                     i;
1393 	unsigned long long int  bit_length = hashed_length << 3;
1394 
1395 	if (padlen < MD5_MIN_PAD_LENGTH) padlen += MD5_BLOCK_LENGTH;
1396 
1397 	p = kzalloc(padlen, alloc_flag);
1398 	if (!p) return -ENOMEM;
1399 
1400 	*p = 0x80;
1401 
1402 	DEBUG(printk("create_md5_pad: hashed_length=%lld bits == %lld bytes\n", bit_length, hashed_length));
1403 
1404 	i = padlen - MD5_PAD_LENGTH_FIELD_LENGTH;
1405 	while (bit_length != 0){
1406 		p[i++] = bit_length % 0x100;
1407 		bit_length >>= 8;
1408 	}
1409 
1410 	*pad = (char*)p;
1411 	*pad_length = padlen;
1412 
1413 	return 0;
1414 }
1415 
1416 #define SHA1_MIN_PAD_LENGTH (9)
1417 #define SHA1_PAD_LENGTH_FIELD_LENGTH (8)
1418 
create_sha1_pad(int alloc_flag,unsigned long long hashed_length,char ** pad,size_t * pad_length)1419 static int create_sha1_pad(int alloc_flag, unsigned long long hashed_length, char **pad, size_t *pad_length)
1420 {
1421 	size_t                  padlen = SHA1_BLOCK_LENGTH - (hashed_length % SHA1_BLOCK_LENGTH);
1422 	unsigned char           *p;
1423 	int                     i;
1424 	unsigned long long int  bit_length = hashed_length << 3;
1425 
1426 	if (padlen < SHA1_MIN_PAD_LENGTH) padlen += SHA1_BLOCK_LENGTH;
1427 
1428 	p = kzalloc(padlen, alloc_flag);
1429 	if (!p) return -ENOMEM;
1430 
1431 	*p = 0x80;
1432 
1433 	DEBUG(printk("create_sha1_pad: hashed_length=%lld bits == %lld bytes\n", bit_length, hashed_length));
1434 
1435 	i = padlen - 1;
1436 	while (bit_length != 0){
1437 		p[i--] = bit_length % 0x100;
1438 		bit_length >>= 8;
1439 	}
1440 
1441 	*pad = (char*)p;
1442 	*pad_length = padlen;
1443 
1444 	return 0;
1445 }
1446 
1447 
transform_ok(struct cryptocop_transform_init * tinit)1448 static int transform_ok(struct cryptocop_transform_init *tinit)
1449 {
1450 	switch (tinit->alg){
1451 	case cryptocop_alg_csum:
1452 		switch (tinit->csum_mode){
1453 		case cryptocop_csum_le:
1454 		case cryptocop_csum_be:
1455 			break;
1456 		default:
1457 			DEBUG_API(printk("transform_ok: Bad mode set for csum transform\n"));
1458 			return -EINVAL;
1459 		}
1460 	case cryptocop_alg_mem2mem:
1461 	case cryptocop_alg_md5:
1462 	case cryptocop_alg_sha1:
1463 		if (tinit->keylen != 0) {
1464 			DEBUG_API(printk("transform_ok: non-zero keylength, %d, for a digest/csum algorithm\n", tinit->keylen));
1465 			return -EINVAL; /* This check is a bit strict. */
1466 		}
1467 		break;
1468 	case cryptocop_alg_des:
1469 		if (tinit->keylen != 64) {
1470 			DEBUG_API(printk("transform_ok: keylen %d invalid for DES\n", tinit->keylen));
1471 			return -EINVAL;
1472 		}
1473 		break;
1474 	case cryptocop_alg_3des:
1475 		if (tinit->keylen != 192) {
1476 			DEBUG_API(printk("transform_ok: keylen %d invalid for 3DES\n", tinit->keylen));
1477 			return -EINVAL;
1478 		}
1479 		break;
1480 	case cryptocop_alg_aes:
1481 		if (tinit->keylen != 128 && tinit->keylen != 192 && tinit->keylen != 256) {
1482 			DEBUG_API(printk("transform_ok: keylen %d invalid for AES\n", tinit->keylen));
1483 			return -EINVAL;
1484 		}
1485 		break;
1486 	case cryptocop_no_alg:
1487 	default:
1488 		DEBUG_API(printk("transform_ok: no such algorithm %d\n", tinit->alg));
1489 		return -EINVAL;
1490 	}
1491 
1492 	switch (tinit->alg){
1493 	case cryptocop_alg_des:
1494 	case cryptocop_alg_3des:
1495 	case cryptocop_alg_aes:
1496 		if (tinit->cipher_mode != cryptocop_cipher_mode_ecb && tinit->cipher_mode != cryptocop_cipher_mode_cbc) return -EINVAL;
1497 	default:
1498 		 break;
1499 	}
1500 	return 0;
1501 }
1502 
1503 
cryptocop_new_session(cryptocop_session_id * sid,struct cryptocop_transform_init * tinit,int alloc_flag)1504 int cryptocop_new_session(cryptocop_session_id *sid, struct cryptocop_transform_init *tinit, int alloc_flag)
1505 {
1506 	struct cryptocop_session         *sess;
1507 	struct cryptocop_transform_init  *tfrm_in = tinit;
1508 	struct cryptocop_transform_init  *tmp_in;
1509 	int                              no_tfrms = 0;
1510 	int                              i;
1511 	unsigned long int                flags;
1512 
1513 	init_stream_coprocessor(); /* For safety if we are called early */
1514 
1515 	while (tfrm_in){
1516 		int err;
1517 		++no_tfrms;
1518 		if ((err = transform_ok(tfrm_in))) {
1519 			DEBUG_API(printk("cryptocop_new_session, bad transform\n"));
1520 			return err;
1521 		}
1522 		tfrm_in = tfrm_in->next;
1523 	}
1524 	if (0 == no_tfrms) {
1525 		DEBUG_API(printk("cryptocop_new_session, no transforms specified\n"));
1526 		return -EINVAL;
1527 	}
1528 
1529 	sess = kmalloc(sizeof(struct cryptocop_session), alloc_flag);
1530 	if (!sess){
1531 		DEBUG_API(printk("cryptocop_new_session, kmalloc cryptocop_session\n"));
1532 		return -ENOMEM;
1533 	}
1534 
1535 	sess->tfrm_ctx = kmalloc(no_tfrms * sizeof(struct cryptocop_transform_ctx), alloc_flag);
1536 	if (!sess->tfrm_ctx) {
1537 		DEBUG_API(printk("cryptocop_new_session, kmalloc cryptocop_transform_ctx\n"));
1538 		kfree(sess);
1539 		return -ENOMEM;
1540 	}
1541 
1542 	tfrm_in = tinit;
1543 	for (i = 0; i < no_tfrms; i++){
1544 		tmp_in = tfrm_in->next;
1545 		while (tmp_in){
1546 			if (tmp_in->tid == tfrm_in->tid) {
1547 				DEBUG_API(printk("cryptocop_new_session, duplicate transform ids\n"));
1548 				kfree(sess->tfrm_ctx);
1549 				kfree(sess);
1550 				return -EINVAL;
1551 			}
1552 			tmp_in = tmp_in->next;
1553 		}
1554 		memcpy(&sess->tfrm_ctx[i].init, tfrm_in, sizeof(struct cryptocop_transform_init));
1555 		sess->tfrm_ctx[i].dec_key_set = 0;
1556 		sess->tfrm_ctx[i].next = &sess->tfrm_ctx[i] + 1;
1557 
1558 		tfrm_in = tfrm_in->next;
1559 	}
1560 	sess->tfrm_ctx[i-1].next = NULL;
1561 
1562 	spin_lock_irqsave(&cryptocop_sessions_lock, flags);
1563 	sess->sid = next_sid;
1564 	next_sid++;
1565 	/* TODO If we are really paranoid we should do duplicate check to handle sid wraparound.
1566 	 *      OTOH 2^64 is a really large number of session. */
1567 	if (next_sid == 0) next_sid = 1;
1568 
1569 	/* Prepend to session list. */
1570 	sess->next = cryptocop_sessions;
1571 	cryptocop_sessions = sess;
1572 	spin_unlock_irqrestore(&cryptocop_sessions_lock, flags);
1573 	*sid = sess->sid;
1574 	return 0;
1575 }
1576 
1577 
cryptocop_free_session(cryptocop_session_id sid)1578 int cryptocop_free_session(cryptocop_session_id sid)
1579 {
1580 	struct cryptocop_transform_ctx    *tc;
1581 	struct cryptocop_session          *sess = NULL;
1582 	struct cryptocop_session          *psess = NULL;
1583 	unsigned long int                 flags;
1584 	int                               i;
1585 	LIST_HEAD(remove_list);
1586 	struct list_head                  *node, *tmp;
1587 	struct cryptocop_prio_job         *pj;
1588 
1589 	DEBUG(printk("cryptocop_free_session: sid=%lld\n", sid));
1590 
1591 	spin_lock_irqsave(&cryptocop_sessions_lock, flags);
1592 	sess = cryptocop_sessions;
1593 	while (sess && sess->sid != sid){
1594 		psess = sess;
1595 		sess = sess->next;
1596 	}
1597 	if (sess){
1598 		if (psess){
1599 			psess->next = sess->next;
1600 		} else {
1601 			cryptocop_sessions = sess->next;
1602 		}
1603 	}
1604 	spin_unlock_irqrestore(&cryptocop_sessions_lock, flags);
1605 
1606 	if (!sess) return -EINVAL;
1607 
1608 	/* Remove queued jobs. */
1609 	spin_lock_irqsave(&cryptocop_job_queue_lock, flags);
1610 
1611 	for (i = 0; i < cryptocop_prio_no_prios; i++){
1612 		if (!list_empty(&(cryptocop_job_queues[i].jobs))){
1613 			list_for_each_safe(node, tmp, &(cryptocop_job_queues[i].jobs)) {
1614 				pj = list_entry(node, struct cryptocop_prio_job, node);
1615 				if (pj->oper->sid == sid) {
1616 					list_move_tail(node, &remove_list);
1617 				}
1618 			}
1619 		}
1620 	}
1621 	spin_unlock_irqrestore(&cryptocop_job_queue_lock, flags);
1622 
1623 	list_for_each_safe(node, tmp, &remove_list) {
1624 		list_del(node);
1625 		pj = list_entry(node, struct cryptocop_prio_job, node);
1626 		pj->oper->operation_status = -EAGAIN;  /* EAGAIN is not ideal for job/session terminated but it's the best choice I know of. */
1627 		DEBUG(printk("cryptocop_free_session: pj=0x%p, pj->oper=0x%p, pj->iop=0x%p\n", pj, pj->oper, pj->iop));
1628 		pj->oper->cb(pj->oper, pj->oper->cb_data);
1629 		delete_internal_operation(pj->iop);
1630 		kfree(pj);
1631 	}
1632 
1633 	tc = sess->tfrm_ctx;
1634 	/* Erase keying data. */
1635 	while (tc){
1636 		DEBUG(printk("cryptocop_free_session: memset keys, tfrm id=%d\n", tc->init.tid));
1637 		memset(tc->init.key, 0xff, CRYPTOCOP_MAX_KEY_LENGTH);
1638 		memset(tc->dec_key, 0xff, CRYPTOCOP_MAX_KEY_LENGTH);
1639 		tc = tc->next;
1640 	}
1641 	kfree(sess->tfrm_ctx);
1642 	kfree(sess);
1643 
1644 	return 0;
1645 }
1646 
get_session(cryptocop_session_id sid)1647 static struct cryptocop_session *get_session(cryptocop_session_id sid)
1648 {
1649 	struct cryptocop_session    *sess;
1650 	unsigned long int           flags;
1651 
1652 	spin_lock_irqsave(&cryptocop_sessions_lock, flags);
1653 	sess = cryptocop_sessions;
1654 	while (sess && (sess->sid != sid)){
1655 		sess = sess->next;
1656 	}
1657 	spin_unlock_irqrestore(&cryptocop_sessions_lock, flags);
1658 
1659 	return sess;
1660 }
1661 
get_transform_ctx(struct cryptocop_session * sess,cryptocop_tfrm_id tid)1662 static struct cryptocop_transform_ctx *get_transform_ctx(struct cryptocop_session *sess, cryptocop_tfrm_id tid)
1663 {
1664 	struct cryptocop_transform_ctx *tc = sess->tfrm_ctx;
1665 
1666 	DEBUG(printk("get_transform_ctx, sess=0x%p, tid=%d\n", sess, tid));
1667 	assert(sess != NULL);
1668 	while (tc && tc->init.tid != tid){
1669 		DEBUG(printk("tc=0x%p, tc->next=0x%p\n", tc, tc->next));
1670 		tc = tc->next;
1671 	}
1672 	DEBUG(printk("get_transform_ctx, returning tc=0x%p\n", tc));
1673 	return tc;
1674 }
1675 
1676 
1677 
1678 /* The AES s-transform matrix (s-box). */
1679 static const u8 aes_sbox[256] = {
1680 	99,  124, 119, 123, 242, 107, 111, 197, 48,  1,   103, 43,  254, 215, 171, 118,
1681 	202, 130, 201, 125, 250, 89,  71,  240, 173, 212, 162, 175, 156, 164, 114, 192,
1682 	183, 253, 147, 38,  54,  63,  247, 204, 52,  165, 229, 241, 113, 216, 49,  21,
1683 	4,   199, 35,  195, 24,  150, 5,   154, 7,   18,  128, 226, 235, 39,  178, 117,
1684 	9,   131, 44,  26,  27,  110, 90,  160, 82,  59,  214, 179, 41,  227, 47,  132,
1685 	83,  209, 0,   237, 32,  252, 177, 91,  106, 203, 190, 57,  74,  76,  88,  207,
1686 	208, 239, 170, 251, 67,  77,  51,  133, 69,  249, 2,   127, 80,  60,  159, 168,
1687 	81,  163, 64,  143, 146, 157, 56,  245, 188, 182, 218, 33,  16,  255, 243, 210,
1688 	205, 12,  19,  236, 95,  151, 68,  23,  196, 167, 126, 61,  100, 93,  25,  115,
1689 	96,  129, 79,  220, 34,  42,  144, 136, 70,  238, 184, 20,  222, 94,  11,  219,
1690 	224, 50,  58,  10,  73,  6,   36,  92,  194, 211, 172, 98,  145, 149, 228, 121,
1691 	231, 200, 55,  109, 141, 213, 78,  169, 108, 86,  244, 234, 101, 122, 174, 8,
1692 	186, 120, 37,  46,  28,  166, 180, 198, 232, 221, 116, 31,  75,  189, 139, 138,
1693 	112, 62,  181, 102, 72,  3,   246, 14,  97,  53,  87,  185, 134, 193, 29,  158,
1694 	225, 248, 152, 17,  105, 217, 142, 148, 155, 30,  135, 233, 206, 85,  40,  223,
1695 	140, 161, 137, 13,  191, 230, 66,  104, 65,  153, 45,  15,  176, 84,  187, 22
1696 };
1697 
1698 /* AES has a 32 bit word round constants for each round in the
1699  * key schedule.  round_constant[i] is really Rcon[i+1] in FIPS187.
1700  */
1701 static u32 round_constant[11] = {
1702 	0x01000000, 0x02000000, 0x04000000, 0x08000000,
1703 	0x10000000, 0x20000000, 0x40000000, 0x80000000,
1704 	0x1B000000, 0x36000000, 0x6C000000
1705 };
1706 
1707 /* Apply the s-box to each of the four occtets in w. */
aes_ks_subword(const u32 w)1708 static u32 aes_ks_subword(const u32 w)
1709 {
1710 	u8 bytes[4];
1711 
1712 	*(u32*)(&bytes[0]) = w;
1713 	bytes[0] = aes_sbox[bytes[0]];
1714 	bytes[1] = aes_sbox[bytes[1]];
1715 	bytes[2] = aes_sbox[bytes[2]];
1716 	bytes[3] = aes_sbox[bytes[3]];
1717 	return *(u32*)(&bytes[0]);
1718 }
1719 
1720 /* The encrypt (forward) Rijndael key schedule algorithm pseudo code:
1721  * (Note that AES words are 32 bit long)
1722  *
1723  * KeyExpansion(byte key[4*Nk], word w[Nb*(Nr+1)], Nk){
1724  * word temp
1725  * i = 0
1726  * while (i < Nk) {
1727  *   w[i] = word(key[4*i, 4*i + 1, 4*i + 2, 4*i + 3])
1728  *   i = i + 1
1729  * }
1730  * i = Nk
1731  *
1732  * while (i < (Nb * (Nr + 1))) {
1733  *   temp = w[i - 1]
1734  *   if ((i mod Nk) == 0) {
1735  *     temp = SubWord(RotWord(temp)) xor Rcon[i/Nk]
1736  *   }
1737  *   else if ((Nk > 6) && ((i mod Nk) == 4)) {
1738  *     temp = SubWord(temp)
1739  *   }
1740  *   w[i] = w[i - Nk] xor temp
1741  * }
1742  * RotWord(t) does a 8 bit cyclic shift left on a 32 bit word.
1743  * SubWord(t) applies the AES s-box individually to each octet
1744  * in a 32 bit word.
1745  *
1746  * For AES Nk can have the values 4, 6, and 8 (corresponding to
1747  * values for Nr of 10, 12, and 14).  Nb is always 4.
1748  *
1749  * To construct w[i], w[i - 1] and w[i - Nk] must be
1750  * available.  Consequently we must keep a state of the last Nk words
1751  * to be able to create the last round keys.
1752  */
get_aes_decrypt_key(unsigned char * dec_key,const unsigned char * key,unsigned int keylength)1753 static void get_aes_decrypt_key(unsigned char *dec_key, const unsigned  char *key, unsigned int keylength)
1754 {
1755 	u32 temp;
1756 	u32 w_ring[8]; /* nk is max 8, use elements 0..(nk - 1) as a ringbuffer */
1757 	u8  w_last_ix;
1758 	int i;
1759 	u8  nr, nk;
1760 
1761 	switch (keylength){
1762 	case 128:
1763 		nk = 4;
1764 		nr = 10;
1765 		break;
1766 	case 192:
1767 		nk = 6;
1768 		nr = 12;
1769 		break;
1770 	case 256:
1771 		nk = 8;
1772 		nr = 14;
1773 		break;
1774 	default:
1775 		panic("stream co-processor: bad aes key length in get_aes_decrypt_key\n");
1776 	};
1777 
1778 	/* Need to do host byte order correction here since key is byte oriented and the
1779 	 * kx algorithm is word (u32) oriented. */
1780 	for (i = 0; i < nk; i+=1) {
1781 		w_ring[i] = be32_to_cpu(*(u32*)&key[4*i]);
1782 	}
1783 
1784 	i = (int)nk;
1785 	w_last_ix = i - 1;
1786 	while (i < (4 * (nr + 2))) {
1787 		temp = w_ring[w_last_ix];
1788 		if (!(i % nk)) {
1789 			/* RotWord(temp) */
1790 			temp = (temp << 8) | (temp >> 24);
1791 			temp = aes_ks_subword(temp);
1792 			temp ^= round_constant[i/nk - 1];
1793 		} else if ((nk > 6) && ((i % nk) == 4)) {
1794 			temp = aes_ks_subword(temp);
1795 		}
1796 		w_last_ix = (w_last_ix + 1) % nk; /* This is the same as (i-Nk) mod Nk */
1797 		temp ^= w_ring[w_last_ix];
1798 		w_ring[w_last_ix] = temp;
1799 
1800 		/* We need the round keys for round Nr+1 and Nr+2 (round key
1801 		 * Nr+2 is the round key beyond the last one used when
1802 		 * encrypting).  Rounds are numbered starting from 0, Nr=10
1803 		 * implies 11 rounds are used in encryption/decryption.
1804 		 */
1805 		if (i >= (4 * nr)) {
1806 			/* Need to do host byte order correction here, the key
1807 			 * is byte oriented. */
1808 			*(u32*)dec_key = cpu_to_be32(temp);
1809 			dec_key += 4;
1810 		}
1811 		++i;
1812 	}
1813 }
1814 
1815 
1816 /**** Job/operation management. ****/
1817 
cryptocop_job_queue_insert_csum(struct cryptocop_operation * operation)1818 int cryptocop_job_queue_insert_csum(struct cryptocop_operation *operation)
1819 {
1820 	return cryptocop_job_queue_insert(cryptocop_prio_kernel_csum, operation);
1821 }
1822 
cryptocop_job_queue_insert_crypto(struct cryptocop_operation * operation)1823 int cryptocop_job_queue_insert_crypto(struct cryptocop_operation *operation)
1824 {
1825 	return cryptocop_job_queue_insert(cryptocop_prio_kernel, operation);
1826 }
1827 
cryptocop_job_queue_insert_user_job(struct cryptocop_operation * operation)1828 int cryptocop_job_queue_insert_user_job(struct cryptocop_operation *operation)
1829 {
1830 	return cryptocop_job_queue_insert(cryptocop_prio_user, operation);
1831 }
1832 
cryptocop_job_queue_insert(cryptocop_queue_priority prio,struct cryptocop_operation * operation)1833 static int cryptocop_job_queue_insert(cryptocop_queue_priority prio, struct cryptocop_operation *operation)
1834 {
1835 	int                           ret;
1836 	struct cryptocop_prio_job     *pj = NULL;
1837 	unsigned long int             flags;
1838 
1839 	DEBUG(printk("cryptocop_job_queue_insert(%d, 0x%p)\n", prio, operation));
1840 
1841 	if (!operation || !operation->cb){
1842 		DEBUG_API(printk("cryptocop_job_queue_insert oper=0x%p, NULL operation or callback\n", operation));
1843 		return -EINVAL;
1844 	}
1845 
1846 	if ((ret = cryptocop_job_setup(&pj, operation)) != 0){
1847 		DEBUG_API(printk("cryptocop_job_queue_insert: job setup failed\n"));
1848 		return ret;
1849 	}
1850 	assert(pj != NULL);
1851 
1852 	spin_lock_irqsave(&cryptocop_job_queue_lock, flags);
1853 	list_add_tail(&pj->node, &cryptocop_job_queues[prio].jobs);
1854 	spin_unlock_irqrestore(&cryptocop_job_queue_lock, flags);
1855 
1856 	/* Make sure a job is running */
1857 	cryptocop_start_job();
1858 	return 0;
1859 }
1860 
1861 static void cryptocop_do_tasklet(unsigned long unused);
1862 DECLARE_TASKLET (cryptocop_tasklet, cryptocop_do_tasklet, 0);
1863 
cryptocop_do_tasklet(unsigned long unused)1864 static void cryptocop_do_tasklet(unsigned long unused)
1865 {
1866 	struct list_head             *node;
1867 	struct cryptocop_prio_job    *pj = NULL;
1868 	unsigned long                flags;
1869 
1870 	DEBUG(printk("cryptocop_do_tasklet: entering\n"));
1871 
1872 	do {
1873 		spin_lock_irqsave(&cryptocop_completed_jobs_lock, flags);
1874 		if (!list_empty(&cryptocop_completed_jobs)){
1875 			node = cryptocop_completed_jobs.next;
1876 			list_del(node);
1877 			pj = list_entry(node, struct cryptocop_prio_job, node);
1878 		} else {
1879 			pj = NULL;
1880 		}
1881 		spin_unlock_irqrestore(&cryptocop_completed_jobs_lock, flags);
1882 		if (pj) {
1883 			assert(pj->oper != NULL);
1884 
1885 			/* Notify consumer of operation completeness. */
1886 			DEBUG(printk("cryptocop_do_tasklet: callback 0x%p, data 0x%p\n", pj->oper->cb, pj->oper->cb_data));
1887 
1888 			pj->oper->operation_status = 0; /* Job is completed. */
1889 			pj->oper->cb(pj->oper, pj->oper->cb_data);
1890 			delete_internal_operation(pj->iop);
1891 			kfree(pj);
1892 		}
1893 	} while (pj != NULL);
1894 
1895 	DEBUG(printk("cryptocop_do_tasklet: exiting\n"));
1896 }
1897 
1898 static irqreturn_t
dma_done_interrupt(int irq,void * dev_id)1899 dma_done_interrupt(int irq, void *dev_id)
1900 {
1901 	struct cryptocop_prio_job *done_job;
1902 	reg_dma_rw_ack_intr ack_intr = {
1903 		.data = 1,
1904 	};
1905 
1906 	REG_WR(dma, IN_DMA_INST, rw_ack_intr, ack_intr);
1907 
1908 	DEBUG(printk("cryptocop DMA done\n"));
1909 
1910 	spin_lock(&running_job_lock);
1911 	if (cryptocop_running_job == NULL){
1912 		printk("stream co-processor got interrupt when not busy\n");
1913 		spin_unlock(&running_job_lock);
1914 		return IRQ_HANDLED;
1915 	}
1916 	done_job = cryptocop_running_job;
1917 	cryptocop_running_job = NULL;
1918 	spin_unlock(&running_job_lock);
1919 
1920 	/* Start processing a job. */
1921 	if (!spin_trylock(&cryptocop_process_lock)){
1922 		DEBUG(printk("cryptocop irq handler, not starting a job\n"));
1923 	} else {
1924 		cryptocop_start_job();
1925 		spin_unlock(&cryptocop_process_lock);
1926 	}
1927 
1928 	done_job->oper->operation_status = 0; /* Job is completed. */
1929 	if (done_job->oper->fast_callback){
1930 		/* This operation wants callback from interrupt. */
1931 		done_job->oper->cb(done_job->oper, done_job->oper->cb_data);
1932 		delete_internal_operation(done_job->iop);
1933 		kfree(done_job);
1934 	} else {
1935 		spin_lock(&cryptocop_completed_jobs_lock);
1936 		list_add_tail(&(done_job->node), &cryptocop_completed_jobs);
1937 		spin_unlock(&cryptocop_completed_jobs_lock);
1938 		tasklet_schedule(&cryptocop_tasklet);
1939 	}
1940 
1941 	DEBUG(printk("cryptocop leave irq handler\n"));
1942 	return IRQ_HANDLED;
1943 }
1944 
1945 
1946 /* Setup interrupts and DMA channels. */
init_cryptocop(void)1947 static int init_cryptocop(void)
1948 {
1949 	unsigned long          flags;
1950 	reg_dma_rw_cfg         dma_cfg = {.en = 1};
1951 	reg_dma_rw_intr_mask   intr_mask_in = {.data = regk_dma_yes}; /* Only want descriptor interrupts from the DMA in channel. */
1952 	reg_dma_rw_ack_intr    ack_intr = {.data = 1,.in_eop = 1 };
1953 	reg_strcop_rw_cfg      strcop_cfg = {
1954 		.ipend = regk_strcop_little,
1955 		.td1 = regk_strcop_e,
1956 		.td2 = regk_strcop_d,
1957 		.td3 = regk_strcop_e,
1958 		.ignore_sync = 0,
1959 		.en = 1
1960 	};
1961 
1962 	if (request_irq(DMA_IRQ, dma_done_interrupt, 0,
1963 			"stream co-processor DMA", NULL))
1964 		panic("request_irq stream co-processor irq dma9");
1965 
1966 	(void)crisv32_request_dma(OUT_DMA, "strcop", DMA_PANIC_ON_ERROR,
1967 		0, dma_strp);
1968 	(void)crisv32_request_dma(IN_DMA, "strcop", DMA_PANIC_ON_ERROR,
1969 		0, dma_strp);
1970 
1971 	local_irq_save(flags);
1972 
1973 	/* Reset and enable the cryptocop. */
1974 	strcop_cfg.en = 0;
1975 	REG_WR(strcop, regi_strcop, rw_cfg, strcop_cfg);
1976 	strcop_cfg.en = 1;
1977 	REG_WR(strcop, regi_strcop, rw_cfg, strcop_cfg);
1978 
1979 	/* Enable DMAs. */
1980 	REG_WR(dma, IN_DMA_INST, rw_cfg, dma_cfg); /* input DMA */
1981 	REG_WR(dma, OUT_DMA_INST, rw_cfg, dma_cfg); /* output DMA */
1982 
1983 	/* Set up wordsize = 4 for DMAs. */
1984 	DMA_WR_CMD(OUT_DMA_INST, regk_dma_set_w_size4);
1985 	DMA_WR_CMD(IN_DMA_INST, regk_dma_set_w_size4);
1986 
1987 	/* Enable interrupts. */
1988 	REG_WR(dma, IN_DMA_INST, rw_intr_mask, intr_mask_in);
1989 
1990 	/* Clear intr ack. */
1991 	REG_WR(dma, IN_DMA_INST, rw_ack_intr, ack_intr);
1992 
1993 	local_irq_restore(flags);
1994 
1995 	return 0;
1996 }
1997 
1998 /* Free used cryptocop hw resources (interrupt and DMA channels). */
release_cryptocop(void)1999 static void release_cryptocop(void)
2000 {
2001 	unsigned long          flags;
2002 	reg_dma_rw_cfg         dma_cfg = {.en = 0};
2003 	reg_dma_rw_intr_mask   intr_mask_in = {0};
2004 	reg_dma_rw_ack_intr    ack_intr = {.data = 1,.in_eop = 1 };
2005 
2006 	local_irq_save(flags);
2007 
2008 	/* Clear intr ack. */
2009 	REG_WR(dma, IN_DMA_INST, rw_ack_intr, ack_intr);
2010 
2011 	/* Disable DMAs. */
2012 	REG_WR(dma, IN_DMA_INST, rw_cfg, dma_cfg); /* input DMA */
2013 	REG_WR(dma, OUT_DMA_INST, rw_cfg, dma_cfg); /* output DMA */
2014 
2015 	/* Disable interrupts. */
2016 	REG_WR(dma, IN_DMA_INST, rw_intr_mask, intr_mask_in);
2017 
2018 	local_irq_restore(flags);
2019 
2020 	free_irq(DMA_IRQ, NULL);
2021 
2022 	(void)crisv32_free_dma(OUT_DMA);
2023 	(void)crisv32_free_dma(IN_DMA);
2024 }
2025 
2026 
2027 /* Init job queue. */
cryptocop_job_queue_init(void)2028 static int cryptocop_job_queue_init(void)
2029 {
2030 	int i;
2031 
2032 	INIT_LIST_HEAD(&cryptocop_completed_jobs);
2033 
2034 	for (i = 0; i < cryptocop_prio_no_prios; i++){
2035 		cryptocop_job_queues[i].prio = (cryptocop_queue_priority)i;
2036 		INIT_LIST_HEAD(&cryptocop_job_queues[i].jobs);
2037 	}
2038 	return 0;
2039 }
2040 
2041 
cryptocop_job_queue_close(void)2042 static void cryptocop_job_queue_close(void)
2043 {
2044 	struct list_head               *node, *tmp;
2045 	struct cryptocop_prio_job      *pj = NULL;
2046 	unsigned long int              process_flags, flags;
2047 	int                            i;
2048 
2049 	/* FIXME: This is as yet untested code. */
2050 
2051 	/* Stop strcop from getting an operation to process while we are closing the
2052 	   module. */
2053 	spin_lock_irqsave(&cryptocop_process_lock, process_flags);
2054 
2055 	/* Empty the job queue. */
2056 	for (i = 0; i < cryptocop_prio_no_prios; i++){
2057 		if (!list_empty(&(cryptocop_job_queues[i].jobs))){
2058 			list_for_each_safe(node, tmp, &(cryptocop_job_queues[i].jobs)) {
2059 				pj = list_entry(node, struct cryptocop_prio_job, node);
2060 				list_del(node);
2061 
2062 				/* Call callback to notify consumer of job removal. */
2063 				DEBUG(printk("cryptocop_job_queue_close: callback 0x%p, data 0x%p\n", pj->oper->cb, pj->oper->cb_data));
2064 				pj->oper->operation_status = -EINTR; /* Job is terminated without completion. */
2065 				pj->oper->cb(pj->oper, pj->oper->cb_data);
2066 
2067 				delete_internal_operation(pj->iop);
2068 				kfree(pj);
2069 			}
2070 		}
2071 	}
2072 	spin_unlock_irqrestore(&cryptocop_process_lock, process_flags);
2073 
2074 	/* Remove the running job, if any. */
2075 	spin_lock_irqsave(&running_job_lock, flags);
2076 	if (cryptocop_running_job){
2077 		reg_strcop_rw_cfg rw_cfg;
2078 		reg_dma_rw_cfg    dma_out_cfg, dma_in_cfg;
2079 
2080 		/* Stop DMA. */
2081 		dma_out_cfg = REG_RD(dma, OUT_DMA_INST, rw_cfg);
2082 		dma_out_cfg.en = regk_dma_no;
2083 		REG_WR(dma, OUT_DMA_INST, rw_cfg, dma_out_cfg);
2084 
2085 		dma_in_cfg = REG_RD(dma, IN_DMA_INST, rw_cfg);
2086 		dma_in_cfg.en = regk_dma_no;
2087 		REG_WR(dma, IN_DMA_INST, rw_cfg, dma_in_cfg);
2088 
2089 		/* Disble the cryptocop. */
2090 		rw_cfg = REG_RD(strcop, regi_strcop, rw_cfg);
2091 		rw_cfg.en = 0;
2092 		REG_WR(strcop, regi_strcop, rw_cfg, rw_cfg);
2093 
2094 		pj = cryptocop_running_job;
2095 		cryptocop_running_job = NULL;
2096 
2097 		/* Call callback to notify consumer of job removal. */
2098 		DEBUG(printk("cryptocop_job_queue_close: callback 0x%p, data 0x%p\n", pj->oper->cb, pj->oper->cb_data));
2099 		pj->oper->operation_status = -EINTR; /* Job is terminated without completion. */
2100 		pj->oper->cb(pj->oper, pj->oper->cb_data);
2101 
2102 		delete_internal_operation(pj->iop);
2103 		kfree(pj);
2104 	}
2105 	spin_unlock_irqrestore(&running_job_lock, flags);
2106 
2107 	/* Remove completed jobs, if any. */
2108 	spin_lock_irqsave(&cryptocop_completed_jobs_lock, flags);
2109 
2110 	list_for_each_safe(node, tmp, &cryptocop_completed_jobs) {
2111 		pj = list_entry(node, struct cryptocop_prio_job, node);
2112 		list_del(node);
2113 		/* Call callback to notify consumer of job removal. */
2114 		DEBUG(printk("cryptocop_job_queue_close: callback 0x%p, data 0x%p\n", pj->oper->cb, pj->oper->cb_data));
2115 		pj->oper->operation_status = -EINTR; /* Job is terminated without completion. */
2116 		pj->oper->cb(pj->oper, pj->oper->cb_data);
2117 
2118 		delete_internal_operation(pj->iop);
2119 		kfree(pj);
2120 	}
2121 	spin_unlock_irqrestore(&cryptocop_completed_jobs_lock, flags);
2122 }
2123 
2124 
cryptocop_start_job(void)2125 static void cryptocop_start_job(void)
2126 {
2127 	int                          i;
2128 	struct cryptocop_prio_job    *pj;
2129 	unsigned long int            flags;
2130 	unsigned long int            running_job_flags;
2131 	reg_strcop_rw_cfg            rw_cfg = {.en = 1, .ignore_sync = 0};
2132 
2133 	DEBUG(printk("cryptocop_start_job: entering\n"));
2134 
2135 	spin_lock_irqsave(&running_job_lock, running_job_flags);
2136 	if (cryptocop_running_job != NULL){
2137 		/* Already running. */
2138 		DEBUG(printk("cryptocop_start_job: already running, exit\n"));
2139 		spin_unlock_irqrestore(&running_job_lock, running_job_flags);
2140 		return;
2141 	}
2142 	spin_lock_irqsave(&cryptocop_job_queue_lock, flags);
2143 
2144 	/* Check the queues in priority order. */
2145 	for (i = cryptocop_prio_kernel_csum; (i < cryptocop_prio_no_prios) && list_empty(&cryptocop_job_queues[i].jobs); i++);
2146 	if (i == cryptocop_prio_no_prios) {
2147 		spin_unlock_irqrestore(&cryptocop_job_queue_lock, flags);
2148 		spin_unlock_irqrestore(&running_job_lock, running_job_flags);
2149 		DEBUG(printk("cryptocop_start_job: no jobs to run\n"));
2150 		return; /* No jobs to run */
2151 	}
2152 	DEBUG(printk("starting job for prio %d\n", i));
2153 
2154 	/* TODO: Do not starve lower priority jobs.  Let in a lower
2155 	 * prio job for every N-th processed higher prio job or some
2156 	 * other scheduling policy.  This could reasonably be
2157 	 * tweakable since the optimal balance would depend on the
2158 	 * type of load on the system. */
2159 
2160 	/* Pull the DMA lists from the job and start the DMA client. */
2161 	pj = list_entry(cryptocop_job_queues[i].jobs.next, struct cryptocop_prio_job, node);
2162 	list_del(&pj->node);
2163 	spin_unlock_irqrestore(&cryptocop_job_queue_lock, flags);
2164 	cryptocop_running_job = pj;
2165 
2166 	/* Set config register (3DES and CSUM modes). */
2167 	switch (pj->iop->tdes_mode){
2168 	case cryptocop_3des_eee:
2169 		rw_cfg.td1 = regk_strcop_e;
2170 		rw_cfg.td2 = regk_strcop_e;
2171 		rw_cfg.td3 = regk_strcop_e;
2172 		break;
2173 	case cryptocop_3des_eed:
2174 		rw_cfg.td1 = regk_strcop_e;
2175 		rw_cfg.td2 = regk_strcop_e;
2176 		rw_cfg.td3 = regk_strcop_d;
2177 		break;
2178 	case cryptocop_3des_ede:
2179 		rw_cfg.td1 = regk_strcop_e;
2180 		rw_cfg.td2 = regk_strcop_d;
2181 		rw_cfg.td3 = regk_strcop_e;
2182 		break;
2183 	case cryptocop_3des_edd:
2184 		rw_cfg.td1 = regk_strcop_e;
2185 		rw_cfg.td2 = regk_strcop_d;
2186 		rw_cfg.td3 = regk_strcop_d;
2187 		break;
2188 	case cryptocop_3des_dee:
2189 		rw_cfg.td1 = regk_strcop_d;
2190 		rw_cfg.td2 = regk_strcop_e;
2191 		rw_cfg.td3 = regk_strcop_e;
2192 		break;
2193 	case cryptocop_3des_ded:
2194 		rw_cfg.td1 = regk_strcop_d;
2195 		rw_cfg.td2 = regk_strcop_e;
2196 		rw_cfg.td3 = regk_strcop_d;
2197 		break;
2198 	case cryptocop_3des_dde:
2199 		rw_cfg.td1 = regk_strcop_d;
2200 		rw_cfg.td2 = regk_strcop_d;
2201 		rw_cfg.td3 = regk_strcop_e;
2202 		break;
2203 	case cryptocop_3des_ddd:
2204 		rw_cfg.td1 = regk_strcop_d;
2205 		rw_cfg.td2 = regk_strcop_d;
2206 		rw_cfg.td3 = regk_strcop_d;
2207 		break;
2208 	default:
2209 		DEBUG(printk("cryptocop_setup_dma_list: bad 3DES mode\n"));
2210 	}
2211 	switch (pj->iop->csum_mode){
2212 	case cryptocop_csum_le:
2213 		rw_cfg.ipend = regk_strcop_little;
2214 		break;
2215 	case cryptocop_csum_be:
2216 		rw_cfg.ipend = regk_strcop_big;
2217 		break;
2218 	default:
2219 		DEBUG(printk("cryptocop_setup_dma_list: bad checksum mode\n"));
2220 	}
2221 	REG_WR(strcop, regi_strcop, rw_cfg, rw_cfg);
2222 
2223 	DEBUG(printk("cryptocop_start_job: starting DMA, new cryptocop_running_job=0x%p\n"
2224 		     "ctx_in: 0x%p, phys: 0x%p\n"
2225 		     "ctx_out: 0x%p, phys: 0x%p\n",
2226 		     pj,
2227 		     &pj->iop->ctx_in, (char*)virt_to_phys(&pj->iop->ctx_in),
2228 		     &pj->iop->ctx_out, (char*)virt_to_phys(&pj->iop->ctx_out)));
2229 
2230 	/* Start input DMA. */
2231 	flush_dma_context(&pj->iop->ctx_in);
2232 	DMA_START_CONTEXT(IN_DMA_INST, virt_to_phys(&pj->iop->ctx_in));
2233 
2234 	/* Start output DMA. */
2235 	DMA_START_CONTEXT(OUT_DMA_INST, virt_to_phys(&pj->iop->ctx_out));
2236 
2237 	spin_unlock_irqrestore(&running_job_lock, running_job_flags);
2238 	DEBUG(printk("cryptocop_start_job: exiting\n"));
2239 }
2240 
2241 
cryptocop_job_setup(struct cryptocop_prio_job ** pj,struct cryptocop_operation * operation)2242 static int cryptocop_job_setup(struct cryptocop_prio_job **pj, struct cryptocop_operation *operation)
2243 {
2244 	int  err;
2245 	int  alloc_flag = operation->in_interrupt ? GFP_ATOMIC : GFP_KERNEL;
2246 	void *iop_alloc_ptr = NULL;
2247 
2248 	*pj = kmalloc(sizeof (struct cryptocop_prio_job), alloc_flag);
2249 	if (!*pj) return -ENOMEM;
2250 
2251 	DEBUG(printk("cryptocop_job_setup: operation=0x%p\n", operation));
2252 
2253 	(*pj)->oper = operation;
2254 	DEBUG(printk("cryptocop_job_setup, cb=0x%p cb_data=0x%p\n",  (*pj)->oper->cb, (*pj)->oper->cb_data));
2255 
2256 	if (operation->use_dmalists) {
2257 		DEBUG(print_user_dma_lists(&operation->list_op));
2258 		if (!operation->list_op.inlist || !operation->list_op.outlist || !operation->list_op.out_data_buf || !operation->list_op.in_data_buf){
2259 			DEBUG_API(printk("cryptocop_job_setup: bad indata (use_dmalists)\n"));
2260 			kfree(*pj);
2261 			return -EINVAL;
2262 		}
2263 		iop_alloc_ptr = kmalloc(DESCR_ALLOC_PAD + sizeof(struct cryptocop_int_operation), alloc_flag);
2264 		if (!iop_alloc_ptr) {
2265 			DEBUG_API(printk("cryptocop_job_setup: kmalloc cryptocop_int_operation\n"));
2266 			kfree(*pj);
2267 			return -ENOMEM;
2268 		}
2269 		(*pj)->iop = (struct cryptocop_int_operation*)(((unsigned long int)(iop_alloc_ptr + DESCR_ALLOC_PAD + offsetof(struct cryptocop_int_operation, ctx_out)) & ~0x0000001F) - offsetof(struct cryptocop_int_operation, ctx_out));
2270 		DEBUG(memset((*pj)->iop, 0xff, sizeof(struct cryptocop_int_operation)));
2271 		(*pj)->iop->alloc_ptr = iop_alloc_ptr;
2272 		(*pj)->iop->sid = operation->sid;
2273 		(*pj)->iop->cdesc_out = NULL;
2274 		(*pj)->iop->cdesc_in = NULL;
2275 		(*pj)->iop->tdes_mode = operation->list_op.tdes_mode;
2276 		(*pj)->iop->csum_mode = operation->list_op.csum_mode;
2277 		(*pj)->iop->ddesc_out = operation->list_op.outlist;
2278 		(*pj)->iop->ddesc_in = operation->list_op.inlist;
2279 
2280 		/* Setup DMA contexts. */
2281 		(*pj)->iop->ctx_out.next = NULL;
2282 		(*pj)->iop->ctx_out.eol = 1;
2283 		(*pj)->iop->ctx_out.saved_data = operation->list_op.outlist;
2284 		(*pj)->iop->ctx_out.saved_data_buf = operation->list_op.out_data_buf;
2285 
2286 		(*pj)->iop->ctx_in.next = NULL;
2287 		(*pj)->iop->ctx_in.eol = 1;
2288 		(*pj)->iop->ctx_in.saved_data = operation->list_op.inlist;
2289 		(*pj)->iop->ctx_in.saved_data_buf = operation->list_op.in_data_buf;
2290 	} else {
2291 		if ((err = cryptocop_setup_dma_list(operation, &(*pj)->iop, alloc_flag))) {
2292 			DEBUG_API(printk("cryptocop_job_setup: cryptocop_setup_dma_list failed %d\n", err));
2293 			kfree(*pj);
2294 			return err;
2295 		}
2296 	}
2297 	DEBUG(print_dma_descriptors((*pj)->iop));
2298 
2299 	DEBUG(printk("cryptocop_job_setup, DMA list setup successful\n"));
2300 
2301 	return 0;
2302 }
2303 
cryptocop_open(struct inode * inode,struct file * filp)2304 static int cryptocop_open(struct inode *inode, struct file *filp)
2305 {
2306 	int p = iminor(inode);
2307 
2308 	if (p != CRYPTOCOP_MINOR) return -EINVAL;
2309 
2310 	filp->private_data = NULL;
2311 	return 0;
2312 }
2313 
2314 
cryptocop_release(struct inode * inode,struct file * filp)2315 static int cryptocop_release(struct inode *inode, struct file *filp)
2316 {
2317 	struct cryptocop_private *dev = filp->private_data;
2318 	struct cryptocop_private *dev_next;
2319 
2320 	while (dev){
2321 		dev_next = dev->next;
2322 		if (dev->sid != CRYPTOCOP_SESSION_ID_NONE) {
2323 			(void)cryptocop_free_session(dev->sid);
2324 		}
2325 		kfree(dev);
2326 		dev = dev_next;
2327 	}
2328 
2329 	return 0;
2330 }
2331 
2332 
cryptocop_ioctl_close_session(struct inode * inode,struct file * filp,unsigned int cmd,unsigned long arg)2333 static int cryptocop_ioctl_close_session(struct inode *inode, struct file *filp,
2334 					 unsigned int cmd, unsigned long arg)
2335 {
2336 	struct cryptocop_private  *dev = filp->private_data;
2337 	struct cryptocop_private  *prev_dev = NULL;
2338 	struct strcop_session_op  *sess_op = (struct strcop_session_op *)arg;
2339 	struct strcop_session_op  sop;
2340 	int                       err;
2341 
2342 	DEBUG(printk("cryptocop_ioctl_close_session\n"));
2343 
2344 	if (!access_ok(VERIFY_READ, sess_op, sizeof(struct strcop_session_op)))
2345 		return -EFAULT;
2346 	err = copy_from_user(&sop, sess_op, sizeof(struct strcop_session_op));
2347 	if (err) return -EFAULT;
2348 
2349 	while (dev && (dev->sid != sop.ses_id)) {
2350 		prev_dev = dev;
2351 		dev = dev->next;
2352 	}
2353 	if (dev){
2354 		if (prev_dev){
2355 			prev_dev->next = dev->next;
2356 		} else {
2357 			filp->private_data = dev->next;
2358 		}
2359 		err = cryptocop_free_session(dev->sid);
2360 		if (err) return -EFAULT;
2361 	} else {
2362 		DEBUG_API(printk("cryptocop_ioctl_close_session: session %lld not found\n", sop.ses_id));
2363 		return -EINVAL;
2364 	}
2365 	return 0;
2366 }
2367 
2368 
ioctl_process_job_callback(struct cryptocop_operation * op,void * cb_data)2369 static void ioctl_process_job_callback(struct cryptocop_operation *op, void*cb_data)
2370 {
2371 	struct ioctl_job_cb_ctx *jc = (struct ioctl_job_cb_ctx *)cb_data;
2372 
2373 	DEBUG(printk("ioctl_process_job_callback: op=0x%p, cb_data=0x%p\n", op, cb_data));
2374 
2375 	jc->processed = 1;
2376 	wake_up(&cryptocop_ioc_process_wq);
2377 }
2378 
2379 
2380 #define CRYPTOCOP_IOCTL_CIPHER_TID  (1)
2381 #define CRYPTOCOP_IOCTL_DIGEST_TID  (2)
2382 #define CRYPTOCOP_IOCTL_CSUM_TID    (3)
2383 
first_cfg_change_ix(struct strcop_crypto_op * crp_op)2384 static size_t first_cfg_change_ix(struct strcop_crypto_op *crp_op)
2385 {
2386 	size_t ch_ix = 0;
2387 
2388 	if (crp_op->do_cipher) ch_ix = crp_op->cipher_start;
2389 	if (crp_op->do_digest && (crp_op->digest_start < ch_ix)) ch_ix = crp_op->digest_start;
2390 	if (crp_op->do_csum && (crp_op->csum_start < ch_ix)) ch_ix = crp_op->csum_start;
2391 
2392 	DEBUG(printk("first_cfg_change_ix: ix=%d\n", ch_ix));
2393 	return ch_ix;
2394 }
2395 
2396 
next_cfg_change_ix(struct strcop_crypto_op * crp_op,size_t ix)2397 static size_t next_cfg_change_ix(struct strcop_crypto_op *crp_op, size_t ix)
2398 {
2399 	size_t ch_ix = INT_MAX;
2400 	size_t tmp_ix = 0;
2401 
2402 	if (crp_op->do_cipher && ((crp_op->cipher_start + crp_op->cipher_len) > ix)){
2403 		if (crp_op->cipher_start > ix) {
2404 			ch_ix = crp_op->cipher_start;
2405 		} else {
2406 			ch_ix = crp_op->cipher_start + crp_op->cipher_len;
2407 		}
2408 	}
2409 	if (crp_op->do_digest && ((crp_op->digest_start + crp_op->digest_len) > ix)){
2410 		if (crp_op->digest_start > ix) {
2411 			tmp_ix = crp_op->digest_start;
2412 		} else {
2413 			tmp_ix = crp_op->digest_start + crp_op->digest_len;
2414 		}
2415 		if (tmp_ix < ch_ix) ch_ix = tmp_ix;
2416 	}
2417 	if (crp_op->do_csum && ((crp_op->csum_start + crp_op->csum_len) > ix)){
2418 		if (crp_op->csum_start > ix) {
2419 			tmp_ix = crp_op->csum_start;
2420 		} else {
2421 			tmp_ix = crp_op->csum_start + crp_op->csum_len;
2422 		}
2423 		if (tmp_ix < ch_ix) ch_ix = tmp_ix;
2424 	}
2425 	if (ch_ix == INT_MAX) ch_ix = ix;
2426 	DEBUG(printk("next_cfg_change_ix prev ix=%d, next ix=%d\n", ix, ch_ix));
2427 	return ch_ix;
2428 }
2429 
2430 
2431 /* Map map_length bytes from the pages starting on *pageix and *pageoffset to iovecs starting on *iovix.
2432  * Return -1 for ok, 0 for fail. */
map_pages_to_iovec(struct iovec * iov,int iovlen,int * iovix,struct page ** pages,int nopages,int * pageix,int * pageoffset,int map_length)2433 static int map_pages_to_iovec(struct iovec *iov, int iovlen, int *iovix, struct page **pages, int nopages, int *pageix, int *pageoffset, int map_length )
2434 {
2435 	int tmplen;
2436 
2437 	assert(iov != NULL);
2438 	assert(iovix != NULL);
2439 	assert(pages != NULL);
2440 	assert(pageix != NULL);
2441 	assert(pageoffset != NULL);
2442 
2443 	DEBUG(printk("map_pages_to_iovec, map_length=%d, iovlen=%d, *iovix=%d, nopages=%d, *pageix=%d, *pageoffset=%d\n", map_length, iovlen, *iovix, nopages, *pageix, *pageoffset));
2444 
2445 	while (map_length > 0){
2446 		DEBUG(printk("map_pages_to_iovec, map_length=%d, iovlen=%d, *iovix=%d, nopages=%d, *pageix=%d, *pageoffset=%d\n", map_length, iovlen, *iovix, nopages, *pageix, *pageoffset));
2447 		if (*iovix >= iovlen){
2448 			DEBUG_API(printk("map_page_to_iovec: *iovix=%d >= iovlen=%d\n", *iovix, iovlen));
2449 			return 0;
2450 		}
2451 		if (*pageix >= nopages){
2452 			DEBUG_API(printk("map_page_to_iovec: *pageix=%d >= nopages=%d\n", *pageix, nopages));
2453 			return 0;
2454 		}
2455 		iov[*iovix].iov_base = (unsigned char*)page_address(pages[*pageix]) + *pageoffset;
2456 		tmplen = PAGE_SIZE - *pageoffset;
2457 		if (tmplen < map_length){
2458 			(*pageoffset) = 0;
2459 			(*pageix)++;
2460 		} else {
2461 			tmplen = map_length;
2462 			(*pageoffset) += map_length;
2463 		}
2464 		DEBUG(printk("mapping %d bytes from page %d (or %d) to iovec %d\n", tmplen, *pageix, *pageix-1, *iovix));
2465 		iov[*iovix].iov_len = tmplen;
2466 		map_length -= tmplen;
2467 		(*iovix)++;
2468 	}
2469 	DEBUG(printk("map_page_to_iovec, exit, *iovix=%d\n", *iovix));
2470 	return -1;
2471 }
2472 
2473 
2474 
cryptocop_ioctl_process(struct inode * inode,struct file * filp,unsigned int cmd,unsigned long arg)2475 static int cryptocop_ioctl_process(struct inode *inode, struct file *filp, unsigned int cmd, unsigned long arg)
2476 {
2477 	int                             i;
2478 	struct cryptocop_private        *dev = filp->private_data;
2479 	struct strcop_crypto_op         *crp_oper = (struct strcop_crypto_op *)arg;
2480 	struct strcop_crypto_op         oper = {0};
2481 	int                             err = 0;
2482 	struct cryptocop_operation      *cop = NULL;
2483 
2484 	struct ioctl_job_cb_ctx         *jc = NULL;
2485 
2486 	struct page                     **inpages = NULL;
2487 	struct page                     **outpages = NULL;
2488 	int                             noinpages = 0;
2489 	int                             nooutpages = 0;
2490 
2491 	struct cryptocop_desc           descs[5]; /* Max 5 descriptors are needed, there are three transforms that
2492 						   * can get connected/disconnected on different places in the indata. */
2493 	struct cryptocop_desc_cfg       dcfgs[5*3];
2494 	int                             desc_ix = 0;
2495 	int                             dcfg_ix = 0;
2496 	struct cryptocop_tfrm_cfg       ciph_tcfg = {0};
2497 	struct cryptocop_tfrm_cfg       digest_tcfg = {0};
2498 	struct cryptocop_tfrm_cfg       csum_tcfg = {0};
2499 
2500 	unsigned char                   *digest_result = NULL;
2501 	int                             digest_length = 0;
2502 	int                             cblocklen = 0;
2503 	unsigned char                   csum_result[CSUM_BLOCK_LENGTH];
2504 	struct cryptocop_session        *sess;
2505 
2506 	int    iovlen = 0;
2507 	int    iovix = 0;
2508 	int    pageix = 0;
2509 	int    pageoffset = 0;
2510 
2511 	size_t prev_ix = 0;
2512 	size_t next_ix;
2513 
2514 	int    cipher_active, digest_active, csum_active;
2515 	int    end_digest, end_csum;
2516 	int    digest_done = 0;
2517 	int    cipher_done = 0;
2518 	int    csum_done = 0;
2519 
2520 	DEBUG(printk("cryptocop_ioctl_process\n"));
2521 
2522 	if (!access_ok(VERIFY_WRITE, crp_oper, sizeof(struct strcop_crypto_op))){
2523 		DEBUG_API(printk("cryptocop_ioctl_process: !access_ok crp_oper!\n"));
2524 		return -EFAULT;
2525 	}
2526 	if (copy_from_user(&oper, crp_oper, sizeof(struct strcop_crypto_op))) {
2527 		DEBUG_API(printk("cryptocop_ioctl_process: copy_from_user\n"));
2528 		return -EFAULT;
2529 	}
2530 	DEBUG(print_strcop_crypto_op(&oper));
2531 
2532 	while (dev && dev->sid != oper.ses_id) dev = dev->next;
2533 	if (!dev){
2534 		DEBUG_API(printk("cryptocop_ioctl_process: session %lld not found\n", oper.ses_id));
2535 		return -EINVAL;
2536 	}
2537 
2538 	/* Check buffers. */
2539 	if (((oper.indata + oper.inlen) < oper.indata) || ((oper.cipher_outdata + oper.cipher_outlen) < oper.cipher_outdata)){
2540 		DEBUG_API(printk("cryptocop_ioctl_process: user buffers wrapped around, bad user!\n"));
2541 		return -EINVAL;
2542 	}
2543 
2544 	if (!access_ok(VERIFY_WRITE, oper.cipher_outdata, oper.cipher_outlen)){
2545 		DEBUG_API(printk("cryptocop_ioctl_process: !access_ok out data!\n"));
2546 		return -EFAULT;
2547 	}
2548 	if (!access_ok(VERIFY_READ, oper.indata, oper.inlen)){
2549 		DEBUG_API(printk("cryptocop_ioctl_process: !access_ok in data!\n"));
2550 		return -EFAULT;
2551 	}
2552 
2553 	cop = kmalloc(sizeof(struct cryptocop_operation), GFP_KERNEL);
2554 	if (!cop) {
2555 		DEBUG_API(printk("cryptocop_ioctl_process: kmalloc\n"));
2556 		return -ENOMEM;
2557 	}
2558 	jc = kmalloc(sizeof(struct ioctl_job_cb_ctx), GFP_KERNEL);
2559 	if (!jc) {
2560 		DEBUG_API(printk("cryptocop_ioctl_process: kmalloc\n"));
2561 		err = -ENOMEM;
2562 		goto error_cleanup;
2563 	}
2564 	jc->processed = 0;
2565 
2566 	cop->cb_data = jc;
2567 	cop->cb = ioctl_process_job_callback;
2568 	cop->operation_status = 0;
2569 	cop->use_dmalists = 0;
2570 	cop->in_interrupt = 0;
2571 	cop->fast_callback = 0;
2572 	cop->tfrm_op.tfrm_cfg = NULL;
2573 	cop->tfrm_op.desc = NULL;
2574 	cop->tfrm_op.indata = NULL;
2575 	cop->tfrm_op.incount = 0;
2576 	cop->tfrm_op.inlen = 0;
2577 	cop->tfrm_op.outdata = NULL;
2578 	cop->tfrm_op.outcount = 0;
2579 	cop->tfrm_op.outlen = 0;
2580 
2581 	sess = get_session(oper.ses_id);
2582 	if (!sess){
2583 		DEBUG_API(printk("cryptocop_ioctl_process: bad session id.\n"));
2584 		kfree(cop);
2585 		kfree(jc);
2586 		return -EINVAL;
2587 	}
2588 
2589 	if (oper.do_cipher) {
2590 		unsigned int                    cipher_outlen = 0;
2591 		struct cryptocop_transform_ctx  *tc = get_transform_ctx(sess, CRYPTOCOP_IOCTL_CIPHER_TID);
2592 		if (!tc) {
2593 			DEBUG_API(printk("cryptocop_ioctl_process: no cipher transform in session.\n"));
2594 			err = -EINVAL;
2595 			goto error_cleanup;
2596 		}
2597 		ciph_tcfg.tid = CRYPTOCOP_IOCTL_CIPHER_TID;
2598 		ciph_tcfg.inject_ix = 0;
2599 		ciph_tcfg.flags = 0;
2600 		if ((oper.cipher_start < 0) || (oper.cipher_len <= 0) || (oper.cipher_start > oper.inlen) || ((oper.cipher_start + oper.cipher_len) > oper.inlen)){
2601 			DEBUG_API(printk("cryptocop_ioctl_process: bad cipher length\n"));
2602 			kfree(cop);
2603 			kfree(jc);
2604 			return -EINVAL;
2605 		}
2606 		cblocklen = tc->init.alg == cryptocop_alg_aes ? AES_BLOCK_LENGTH : DES_BLOCK_LENGTH;
2607 		if (oper.cipher_len % cblocklen) {
2608 			kfree(cop);
2609 			kfree(jc);
2610 			DEBUG_API(printk("cryptocop_ioctl_process: cipher inlength not multiple of block length.\n"));
2611 			return -EINVAL;
2612 		}
2613 		cipher_outlen = oper.cipher_len;
2614 		if (tc->init.cipher_mode == cryptocop_cipher_mode_cbc){
2615 			if (oper.cipher_explicit) {
2616 				ciph_tcfg.flags |= CRYPTOCOP_EXPLICIT_IV;
2617 				memcpy(ciph_tcfg.iv, oper.cipher_iv, cblocklen);
2618 			} else {
2619 				cipher_outlen = oper.cipher_len - cblocklen;
2620 			}
2621 		} else {
2622 			if (oper.cipher_explicit){
2623 				kfree(cop);
2624 				kfree(jc);
2625 				DEBUG_API(printk("cryptocop_ioctl_process: explicit_iv when not CBC mode\n"));
2626 				return -EINVAL;
2627 			}
2628 		}
2629 		if (oper.cipher_outlen != cipher_outlen) {
2630 			kfree(cop);
2631 			kfree(jc);
2632 			DEBUG_API(printk("cryptocop_ioctl_process: cipher_outlen incorrect, should be %d not %d.\n", cipher_outlen, oper.cipher_outlen));
2633 			return -EINVAL;
2634 		}
2635 
2636 		if (oper.decrypt){
2637 			ciph_tcfg.flags |= CRYPTOCOP_DECRYPT;
2638 		} else {
2639 			ciph_tcfg.flags |= CRYPTOCOP_ENCRYPT;
2640 		}
2641 		ciph_tcfg.next = cop->tfrm_op.tfrm_cfg;
2642 		cop->tfrm_op.tfrm_cfg = &ciph_tcfg;
2643 	}
2644 	if (oper.do_digest){
2645 		struct cryptocop_transform_ctx *tc = get_transform_ctx(sess, CRYPTOCOP_IOCTL_DIGEST_TID);
2646 		if (!tc) {
2647 			DEBUG_API(printk("cryptocop_ioctl_process: no digest transform in session.\n"));
2648 			err = -EINVAL;
2649 			goto error_cleanup;
2650 		}
2651 		digest_length = tc->init.alg == cryptocop_alg_md5 ? 16 : 20;
2652 		digest_result = kmalloc(digest_length, GFP_KERNEL);
2653 		if (!digest_result) {
2654 			DEBUG_API(printk("cryptocop_ioctl_process: kmalloc digest_result\n"));
2655 			err = -EINVAL;
2656 			goto error_cleanup;
2657 		}
2658 		DEBUG(memset(digest_result, 0xff, digest_length));
2659 
2660 		digest_tcfg.tid = CRYPTOCOP_IOCTL_DIGEST_TID;
2661 		digest_tcfg.inject_ix = 0;
2662 		ciph_tcfg.inject_ix += digest_length;
2663 		if ((oper.digest_start < 0) || (oper.digest_len <= 0) || (oper.digest_start > oper.inlen) || ((oper.digest_start + oper.digest_len) > oper.inlen)){
2664 			DEBUG_API(printk("cryptocop_ioctl_process: bad digest length\n"));
2665 			err = -EINVAL;
2666 			goto error_cleanup;
2667 		}
2668 
2669 		digest_tcfg.next = cop->tfrm_op.tfrm_cfg;
2670 		cop->tfrm_op.tfrm_cfg = &digest_tcfg;
2671 	}
2672 	if (oper.do_csum){
2673 		csum_tcfg.tid = CRYPTOCOP_IOCTL_CSUM_TID;
2674 		csum_tcfg.inject_ix = digest_length;
2675 		ciph_tcfg.inject_ix += 2;
2676 
2677 		if ((oper.csum_start < 0) || (oper.csum_len <= 0) || (oper.csum_start > oper.inlen) || ((oper.csum_start + oper.csum_len) > oper.inlen)){
2678 			DEBUG_API(printk("cryptocop_ioctl_process: bad csum length\n"));
2679 			kfree(cop);
2680 			kfree(jc);
2681 			return -EINVAL;
2682 		}
2683 
2684 		csum_tcfg.next = cop->tfrm_op.tfrm_cfg;
2685 		cop->tfrm_op.tfrm_cfg = &csum_tcfg;
2686 	}
2687 
2688 	prev_ix = first_cfg_change_ix(&oper);
2689 	if (prev_ix > oper.inlen) {
2690 		DEBUG_API(printk("cryptocop_ioctl_process: length mismatch\n"));
2691 		nooutpages = noinpages = 0;
2692 		err = -EINVAL;
2693 		goto error_cleanup;
2694 	}
2695 	DEBUG(printk("cryptocop_ioctl_process: inlen=%d, cipher_outlen=%d\n", oper.inlen, oper.cipher_outlen));
2696 
2697 	/* Map user pages for in and out data of the operation. */
2698 	noinpages = (((unsigned long int)(oper.indata + prev_ix) & ~PAGE_MASK) + oper.inlen - 1 - prev_ix + ~PAGE_MASK) >> PAGE_SHIFT;
2699 	DEBUG(printk("cryptocop_ioctl_process: noinpages=%d\n", noinpages));
2700 	inpages = kmalloc(noinpages * sizeof(struct page*), GFP_KERNEL);
2701 	if (!inpages){
2702 		DEBUG_API(printk("cryptocop_ioctl_process: kmalloc inpages\n"));
2703 		nooutpages = noinpages = 0;
2704 		err = -ENOMEM;
2705 		goto error_cleanup;
2706 	}
2707 	if (oper.do_cipher){
2708 		nooutpages = (((unsigned long int)oper.cipher_outdata & ~PAGE_MASK) + oper.cipher_outlen - 1 + ~PAGE_MASK) >> PAGE_SHIFT;
2709 		DEBUG(printk("cryptocop_ioctl_process: nooutpages=%d\n", nooutpages));
2710 		outpages = kmalloc(nooutpages * sizeof(struct page*), GFP_KERNEL);
2711 		if (!outpages){
2712 			DEBUG_API(printk("cryptocop_ioctl_process: kmalloc outpages\n"));
2713 			nooutpages = noinpages = 0;
2714 			err = -ENOMEM;
2715 			goto error_cleanup;
2716 		}
2717 	}
2718 
2719 	/* Acquire the mm page semaphore. */
2720 	down_read(&current->mm->mmap_sem);
2721 
2722 	err = get_user_pages(current,
2723 			     current->mm,
2724 			     (unsigned long int)(oper.indata + prev_ix),
2725 			     noinpages,
2726 			     0,  /* read access only for in data */
2727 			     0, /* no force */
2728 			     inpages,
2729 			     NULL);
2730 
2731 	if (err < 0) {
2732 		up_read(&current->mm->mmap_sem);
2733 		nooutpages = noinpages = 0;
2734 		DEBUG_API(printk("cryptocop_ioctl_process: get_user_pages indata\n"));
2735 		goto error_cleanup;
2736 	}
2737 	noinpages = err;
2738 	if (oper.do_cipher){
2739 		err = get_user_pages(current,
2740 				     current->mm,
2741 				     (unsigned long int)oper.cipher_outdata,
2742 				     nooutpages,
2743 				     1, /* write access for out data */
2744 				     0, /* no force */
2745 				     outpages,
2746 				     NULL);
2747 		up_read(&current->mm->mmap_sem);
2748 		if (err < 0) {
2749 			nooutpages = 0;
2750 			DEBUG_API(printk("cryptocop_ioctl_process: get_user_pages outdata\n"));
2751 			goto error_cleanup;
2752 		}
2753 		nooutpages = err;
2754 	} else {
2755 		up_read(&current->mm->mmap_sem);
2756 	}
2757 
2758 	/* Add 6 to nooutpages to make room for possibly inserted buffers for storing digest and
2759 	 * csum output and splits when units are (dis-)connected. */
2760 	cop->tfrm_op.indata = kmalloc((noinpages) * sizeof(struct iovec), GFP_KERNEL);
2761 	cop->tfrm_op.outdata = kmalloc((6 + nooutpages) * sizeof(struct iovec), GFP_KERNEL);
2762 	if (!cop->tfrm_op.indata || !cop->tfrm_op.outdata) {
2763 		DEBUG_API(printk("cryptocop_ioctl_process: kmalloc iovecs\n"));
2764 		err = -ENOMEM;
2765 		goto error_cleanup;
2766 	}
2767 
2768 	cop->tfrm_op.inlen = oper.inlen - prev_ix;
2769 	cop->tfrm_op.outlen = 0;
2770 	if (oper.do_cipher) cop->tfrm_op.outlen += oper.cipher_outlen;
2771 	if (oper.do_digest) cop->tfrm_op.outlen += digest_length;
2772 	if (oper.do_csum) cop->tfrm_op.outlen += 2;
2773 
2774 	/* Setup the in iovecs. */
2775 	cop->tfrm_op.incount = noinpages;
2776 	if (noinpages > 1){
2777 		size_t tmplen = cop->tfrm_op.inlen;
2778 
2779 		cop->tfrm_op.indata[0].iov_len = PAGE_SIZE - ((unsigned long int)(oper.indata + prev_ix) & ~PAGE_MASK);
2780 		cop->tfrm_op.indata[0].iov_base = (unsigned char*)page_address(inpages[0]) + ((unsigned long int)(oper.indata + prev_ix) & ~PAGE_MASK);
2781 		tmplen -= cop->tfrm_op.indata[0].iov_len;
2782 		for (i = 1; i<noinpages; i++){
2783 			cop->tfrm_op.indata[i].iov_len = tmplen < PAGE_SIZE ? tmplen : PAGE_SIZE;
2784 			cop->tfrm_op.indata[i].iov_base = (unsigned char*)page_address(inpages[i]);
2785 			tmplen -= PAGE_SIZE;
2786 		}
2787 	} else {
2788 		cop->tfrm_op.indata[0].iov_len = oper.inlen - prev_ix;
2789 		cop->tfrm_op.indata[0].iov_base = (unsigned char*)page_address(inpages[0]) + ((unsigned long int)(oper.indata + prev_ix) & ~PAGE_MASK);
2790 	}
2791 
2792 	iovlen = nooutpages + 6;
2793 	pageoffset = oper.do_cipher ? ((unsigned long int)oper.cipher_outdata & ~PAGE_MASK) : 0;
2794 
2795 	next_ix = next_cfg_change_ix(&oper, prev_ix);
2796 	if (prev_ix == next_ix){
2797 		DEBUG_API(printk("cryptocop_ioctl_process: length configuration broken.\n"));
2798 		err = -EINVAL;  /* This should be impossible barring bugs. */
2799 		goto error_cleanup;
2800 	}
2801 	while (prev_ix != next_ix){
2802 		end_digest = end_csum = cipher_active = digest_active = csum_active = 0;
2803 		descs[desc_ix].cfg = NULL;
2804 		descs[desc_ix].length = next_ix - prev_ix;
2805 
2806 		if (oper.do_cipher && (oper.cipher_start < next_ix) && (prev_ix < (oper.cipher_start + oper.cipher_len))) {
2807 			dcfgs[dcfg_ix].tid = CRYPTOCOP_IOCTL_CIPHER_TID;
2808 			dcfgs[dcfg_ix].src = cryptocop_source_dma;
2809 			cipher_active = 1;
2810 
2811 			if (next_ix == (oper.cipher_start + oper.cipher_len)){
2812 				cipher_done = 1;
2813 				dcfgs[dcfg_ix].last = 1;
2814 			} else {
2815 				dcfgs[dcfg_ix].last = 0;
2816 			}
2817 			dcfgs[dcfg_ix].next = descs[desc_ix].cfg;
2818 			descs[desc_ix].cfg = &dcfgs[dcfg_ix];
2819 			++dcfg_ix;
2820 		}
2821 		if (oper.do_digest && (oper.digest_start < next_ix) && (prev_ix < (oper.digest_start + oper.digest_len))) {
2822 			digest_active = 1;
2823 			dcfgs[dcfg_ix].tid = CRYPTOCOP_IOCTL_DIGEST_TID;
2824 			dcfgs[dcfg_ix].src = cryptocop_source_dma;
2825 			if (next_ix == (oper.digest_start + oper.digest_len)){
2826 				assert(!digest_done);
2827 				digest_done = 1;
2828 				dcfgs[dcfg_ix].last = 1;
2829 			} else {
2830 				dcfgs[dcfg_ix].last = 0;
2831 			}
2832 			dcfgs[dcfg_ix].next = descs[desc_ix].cfg;
2833 			descs[desc_ix].cfg = &dcfgs[dcfg_ix];
2834 			++dcfg_ix;
2835 		}
2836 		if (oper.do_csum && (oper.csum_start < next_ix) && (prev_ix < (oper.csum_start + oper.csum_len))){
2837 			csum_active = 1;
2838 			dcfgs[dcfg_ix].tid = CRYPTOCOP_IOCTL_CSUM_TID;
2839 			dcfgs[dcfg_ix].src = cryptocop_source_dma;
2840 			if (next_ix == (oper.csum_start + oper.csum_len)){
2841 				csum_done = 1;
2842 				dcfgs[dcfg_ix].last = 1;
2843 			} else {
2844 				dcfgs[dcfg_ix].last = 0;
2845 			}
2846 			dcfgs[dcfg_ix].next = descs[desc_ix].cfg;
2847 			descs[desc_ix].cfg = &dcfgs[dcfg_ix];
2848 			++dcfg_ix;
2849 		}
2850 		if (!descs[desc_ix].cfg){
2851 			DEBUG_API(printk("cryptocop_ioctl_process: data segment %d (%d to %d) had no active transforms\n", desc_ix, prev_ix, next_ix));
2852 			err = -EINVAL;
2853 			goto error_cleanup;
2854 		}
2855 		descs[desc_ix].next = &(descs[desc_ix]) + 1;
2856 		++desc_ix;
2857 		prev_ix = next_ix;
2858 		next_ix = next_cfg_change_ix(&oper, prev_ix);
2859 	}
2860 	if (desc_ix > 0){
2861 		descs[desc_ix-1].next = NULL;
2862 	} else {
2863 		descs[0].next = NULL;
2864 	}
2865 	if (oper.do_digest) {
2866 		DEBUG(printk("cryptocop_ioctl_process: mapping %d byte digest output to iovec %d\n", digest_length, iovix));
2867 		/* Add outdata iovec, length == <length of type of digest> */
2868 		cop->tfrm_op.outdata[iovix].iov_base = digest_result;
2869 		cop->tfrm_op.outdata[iovix].iov_len = digest_length;
2870 		++iovix;
2871 	}
2872 	if (oper.do_csum) {
2873 		/* Add outdata iovec, length == 2, the length of csum. */
2874 		DEBUG(printk("cryptocop_ioctl_process: mapping 2 byte csum output to iovec %d\n", iovix));
2875 		/* Add outdata iovec, length == <length of type of digest> */
2876 		cop->tfrm_op.outdata[iovix].iov_base = csum_result;
2877 		cop->tfrm_op.outdata[iovix].iov_len = 2;
2878 		++iovix;
2879 	}
2880 	if (oper.do_cipher) {
2881 		if (!map_pages_to_iovec(cop->tfrm_op.outdata, iovlen, &iovix, outpages, nooutpages, &pageix, &pageoffset, oper.cipher_outlen)){
2882 			DEBUG_API(printk("cryptocop_ioctl_process: failed to map pages to iovec.\n"));
2883 			err = -ENOSYS; /* This should be impossible barring bugs. */
2884 			goto error_cleanup;
2885 		}
2886 	}
2887 	DEBUG(printk("cryptocop_ioctl_process: setting cop->tfrm_op.outcount %d\n", iovix));
2888 	cop->tfrm_op.outcount = iovix;
2889 	assert(iovix <= (nooutpages + 6));
2890 
2891 	cop->sid = oper.ses_id;
2892 	cop->tfrm_op.desc = &descs[0];
2893 
2894 	DEBUG(printk("cryptocop_ioctl_process: inserting job, cb_data=0x%p\n", cop->cb_data));
2895 
2896 	if ((err = cryptocop_job_queue_insert_user_job(cop)) != 0) {
2897 		DEBUG_API(printk("cryptocop_ioctl_process: insert job %d\n", err));
2898 		err = -EINVAL;
2899 		goto error_cleanup;
2900 	}
2901 
2902 	DEBUG(printk("cryptocop_ioctl_process: begin wait for result\n"));
2903 
2904 	wait_event(cryptocop_ioc_process_wq, (jc->processed != 0));
2905 	DEBUG(printk("cryptocop_ioctl_process: end wait for result\n"));
2906         if (!jc->processed){
2907 		printk(KERN_WARNING "cryptocop_ioctl_process: job not processed at completion\n");
2908 		err = -EIO;
2909 		goto error_cleanup;
2910 	}
2911 
2912 	/* Job process done.  Cipher output should already be correct in job so no post processing of outdata. */
2913 	DEBUG(printk("cryptocop_ioctl_process: operation_status = %d\n", cop->operation_status));
2914 	if (cop->operation_status == 0){
2915 		if (oper.do_digest){
2916 			DEBUG(printk("cryptocop_ioctl_process: copy %d bytes digest to user\n", digest_length));
2917 			err = copy_to_user((unsigned char*)crp_oper + offsetof(struct strcop_crypto_op, digest), digest_result, digest_length);
2918 			if (0 != err){
2919 				DEBUG_API(printk("cryptocop_ioctl_process: copy_to_user, digest length %d, err %d\n", digest_length, err));
2920 				err = -EFAULT;
2921 				goto error_cleanup;
2922 			}
2923 		}
2924 		if (oper.do_csum){
2925 			DEBUG(printk("cryptocop_ioctl_process: copy 2 bytes checksum to user\n"));
2926 			err = copy_to_user((unsigned char*)crp_oper + offsetof(struct strcop_crypto_op, csum), csum_result, 2);
2927 			if (0 != err){
2928 				DEBUG_API(printk("cryptocop_ioctl_process: copy_to_user, csum, err %d\n", err));
2929 				err = -EFAULT;
2930 				goto error_cleanup;
2931 			}
2932 		}
2933 		err = 0;
2934 	} else {
2935 		DEBUG(printk("cryptocop_ioctl_process: returning err = operation_status = %d\n", cop->operation_status));
2936 		err = cop->operation_status;
2937 	}
2938 
2939  error_cleanup:
2940 	/* Release page caches. */
2941 	for (i = 0; i < noinpages; i++){
2942 		put_page(inpages[i]);
2943 	}
2944 	for (i = 0; i < nooutpages; i++){
2945 		int spdl_err;
2946 		/* Mark output pages dirty. */
2947 		spdl_err = set_page_dirty_lock(outpages[i]);
2948 		DEBUG(if (spdl_err < 0)printk("cryptocop_ioctl_process: set_page_dirty_lock returned %d\n", spdl_err));
2949 	}
2950 	for (i = 0; i < nooutpages; i++){
2951 		put_page(outpages[i]);
2952 	}
2953 
2954 	kfree(digest_result);
2955 	kfree(inpages);
2956 	kfree(outpages);
2957 	if (cop){
2958 		kfree(cop->tfrm_op.indata);
2959 		kfree(cop->tfrm_op.outdata);
2960 		kfree(cop);
2961 	}
2962 	kfree(jc);
2963 
2964 	DEBUG(print_lock_status());
2965 
2966 	return err;
2967 }
2968 
2969 
cryptocop_ioctl_create_session(struct inode * inode,struct file * filp,unsigned int cmd,unsigned long arg)2970 static int cryptocop_ioctl_create_session(struct inode *inode, struct file *filp, unsigned int cmd, unsigned long arg)
2971 {
2972 	cryptocop_session_id             sid;
2973 	int                              err;
2974 	struct cryptocop_private         *dev;
2975 	struct strcop_session_op         *sess_op = (struct strcop_session_op *)arg;
2976 	struct strcop_session_op         sop;
2977 	struct cryptocop_transform_init  *tis = NULL;
2978 	struct cryptocop_transform_init  ti_cipher = {0};
2979 	struct cryptocop_transform_init  ti_digest = {0};
2980 	struct cryptocop_transform_init  ti_csum = {0};
2981 
2982 	if (!access_ok(VERIFY_WRITE, sess_op, sizeof(struct strcop_session_op)))
2983 		return -EFAULT;
2984 	err = copy_from_user(&sop, sess_op, sizeof(struct strcop_session_op));
2985 	if (err) return -EFAULT;
2986 	if (sop.cipher != cryptocop_cipher_none) {
2987 		if (!access_ok(VERIFY_READ, sop.key, sop.keylen)) return -EFAULT;
2988 	}
2989 	DEBUG(printk("cryptocop_ioctl_create_session, sess_op:\n"));
2990 
2991 	DEBUG(printk("\tcipher:%d\n"
2992 		     "\tcipher_mode:%d\n"
2993 		     "\tdigest:%d\n"
2994 		     "\tcsum:%d\n",
2995 		     (int)sop.cipher,
2996 		     (int)sop.cmode,
2997 		     (int)sop.digest,
2998 		     (int)sop.csum));
2999 
3000 	if (sop.cipher != cryptocop_cipher_none){
3001 		/* Init the cipher. */
3002 		switch (sop.cipher){
3003 		case cryptocop_cipher_des:
3004 			ti_cipher.alg = cryptocop_alg_des;
3005 			break;
3006 		case cryptocop_cipher_3des:
3007 			ti_cipher.alg = cryptocop_alg_3des;
3008 			break;
3009 		case cryptocop_cipher_aes:
3010 			ti_cipher.alg = cryptocop_alg_aes;
3011 			break;
3012 		default:
3013 			DEBUG_API(printk("create session, bad cipher algorithm %d\n", sop.cipher));
3014 			return -EINVAL;
3015 		};
3016 		DEBUG(printk("setting cipher transform %d\n", ti_cipher.alg));
3017 		copy_from_user(ti_cipher.key, sop.key, sop.keylen/8);
3018 		ti_cipher.keylen = sop.keylen;
3019 		switch (sop.cmode){
3020 		case cryptocop_cipher_mode_cbc:
3021 		case cryptocop_cipher_mode_ecb:
3022 			ti_cipher.cipher_mode = sop.cmode;
3023 			break;
3024 		default:
3025 			DEBUG_API(printk("create session, bad cipher mode %d\n", sop.cmode));
3026 			return -EINVAL;
3027 		}
3028 		DEBUG(printk("cryptocop_ioctl_create_session: setting CBC mode %d\n", ti_cipher.cipher_mode));
3029 		switch (sop.des3_mode){
3030 		case cryptocop_3des_eee:
3031 		case cryptocop_3des_eed:
3032 		case cryptocop_3des_ede:
3033 		case cryptocop_3des_edd:
3034 		case cryptocop_3des_dee:
3035 		case cryptocop_3des_ded:
3036 		case cryptocop_3des_dde:
3037 		case cryptocop_3des_ddd:
3038 			ti_cipher.tdes_mode = sop.des3_mode;
3039 			break;
3040 		default:
3041 			DEBUG_API(printk("create session, bad 3DES mode %d\n", sop.des3_mode));
3042 			return -EINVAL;
3043 		}
3044 		ti_cipher.tid = CRYPTOCOP_IOCTL_CIPHER_TID;
3045 		ti_cipher.next = tis;
3046 		tis = &ti_cipher;
3047 	} /* if (sop.cipher != cryptocop_cipher_none) */
3048 	if (sop.digest != cryptocop_digest_none){
3049 		DEBUG(printk("setting digest transform\n"));
3050 		switch (sop.digest){
3051 		case cryptocop_digest_md5:
3052 			ti_digest.alg = cryptocop_alg_md5;
3053 			break;
3054 		case cryptocop_digest_sha1:
3055 			ti_digest.alg = cryptocop_alg_sha1;
3056 			break;
3057 		default:
3058 			DEBUG_API(printk("create session, bad digest algorithm %d\n", sop.digest));
3059 			return -EINVAL;
3060 		}
3061 		ti_digest.tid = CRYPTOCOP_IOCTL_DIGEST_TID;
3062 		ti_digest.next = tis;
3063 		tis = &ti_digest;
3064 	} /* if (sop.digest != cryptocop_digest_none) */
3065 	if (sop.csum != cryptocop_csum_none){
3066 		DEBUG(printk("setting csum transform\n"));
3067 		switch (sop.csum){
3068 		case cryptocop_csum_le:
3069 		case cryptocop_csum_be:
3070 			ti_csum.csum_mode = sop.csum;
3071 			break;
3072 		default:
3073 			DEBUG_API(printk("create session, bad checksum algorithm %d\n", sop.csum));
3074 			return -EINVAL;
3075 		}
3076 		ti_csum.alg = cryptocop_alg_csum;
3077 		ti_csum.tid = CRYPTOCOP_IOCTL_CSUM_TID;
3078 		ti_csum.next = tis;
3079 		tis = &ti_csum;
3080 	} /* (sop.csum != cryptocop_csum_none) */
3081 	dev = kmalloc(sizeof(struct cryptocop_private), GFP_KERNEL);
3082 	if (!dev){
3083 		DEBUG_API(printk("create session, alloc dev\n"));
3084 		return -ENOMEM;
3085 	}
3086 
3087 	err = cryptocop_new_session(&sid, tis, GFP_KERNEL);
3088 	DEBUG({ if (err) printk("create session, cryptocop_new_session %d\n", err);});
3089 
3090 	if (err) {
3091 		kfree(dev);
3092 		return err;
3093 	}
3094 	sess_op->ses_id = sid;
3095 	dev->sid = sid;
3096 	dev->next = filp->private_data;
3097 	filp->private_data = dev;
3098 
3099 	return 0;
3100 }
3101 
cryptocop_ioctl_unlocked(struct inode * inode,struct file * filp,unsigned int cmd,unsigned long arg)3102 static long cryptocop_ioctl_unlocked(struct inode *inode,
3103 	struct file *filp, unsigned int cmd, unsigned long arg)
3104 {
3105 	int err = 0;
3106 	if (_IOC_TYPE(cmd) != ETRAXCRYPTOCOP_IOCTYPE) {
3107 		DEBUG_API(printk("cryptocop_ioctl: wrong type\n"));
3108 		return -ENOTTY;
3109 	}
3110 	if (_IOC_NR(cmd) > CRYPTOCOP_IO_MAXNR){
3111 		return -ENOTTY;
3112 	}
3113 	/* Access check of the argument.  Some commands, e.g. create session and process op,
3114 	   needs additional checks.  Those are handled in the command handling functions. */
3115 	if (_IOC_DIR(cmd) & _IOC_READ)
3116 		err = !access_ok(VERIFY_WRITE, (void *)arg, _IOC_SIZE(cmd));
3117 	else if (_IOC_DIR(cmd) & _IOC_WRITE)
3118 		err = !access_ok(VERIFY_READ, (void *)arg, _IOC_SIZE(cmd));
3119 	if (err) return -EFAULT;
3120 
3121 	switch (cmd) {
3122 	case CRYPTOCOP_IO_CREATE_SESSION:
3123 		return cryptocop_ioctl_create_session(inode, filp, cmd, arg);
3124 	case CRYPTOCOP_IO_CLOSE_SESSION:
3125 		return cryptocop_ioctl_close_session(inode, filp, cmd, arg);
3126 	case CRYPTOCOP_IO_PROCESS_OP:
3127 		return cryptocop_ioctl_process(inode, filp, cmd, arg);
3128 	default:
3129 		DEBUG_API(printk("cryptocop_ioctl: unknown command\n"));
3130 		return -ENOTTY;
3131 	}
3132 	return 0;
3133 }
3134 
3135 static long
cryptocop_ioctl(struct file * filp,unsigned int cmd,unsigned long arg)3136 cryptocop_ioctl(struct file *filp, unsigned int cmd, unsigned long arg)
3137 {
3138        long ret;
3139 
3140        mutex_lock(&cryptocop_mutex);
3141        ret = cryptocop_ioctl_unlocked(file_inode(filp), filp, cmd, arg);
3142        mutex_unlock(&cryptocop_mutex);
3143 
3144        return ret;
3145 }
3146 
3147 
3148 #ifdef LDEBUG
print_dma_descriptors(struct cryptocop_int_operation * iop)3149 static void print_dma_descriptors(struct cryptocop_int_operation *iop)
3150 {
3151 	struct cryptocop_dma_desc *cdesc_out = iop->cdesc_out;
3152 	struct cryptocop_dma_desc *cdesc_in = iop->cdesc_in;
3153 	int                       i;
3154 
3155 	printk("print_dma_descriptors start\n");
3156 
3157 	printk("iop:\n");
3158 	printk("\tsid: 0x%lld\n", iop->sid);
3159 
3160 	printk("\tcdesc_out: 0x%p\n", iop->cdesc_out);
3161 	printk("\tcdesc_in: 0x%p\n", iop->cdesc_in);
3162 	printk("\tddesc_out: 0x%p\n", iop->ddesc_out);
3163 	printk("\tddesc_in: 0x%p\n", iop->ddesc_in);
3164 
3165 	printk("\niop->ctx_out: 0x%p phys: 0x%p\n", &iop->ctx_out, (char*)virt_to_phys(&iop->ctx_out));
3166 	printk("\tnext: 0x%p\n"
3167 	       "\tsaved_data: 0x%p\n"
3168 	       "\tsaved_data_buf: 0x%p\n",
3169 	       iop->ctx_out.next,
3170 	       iop->ctx_out.saved_data,
3171 	       iop->ctx_out.saved_data_buf);
3172 
3173 	printk("\niop->ctx_in: 0x%p phys: 0x%p\n", &iop->ctx_in, (char*)virt_to_phys(&iop->ctx_in));
3174 	printk("\tnext: 0x%p\n"
3175 	       "\tsaved_data: 0x%p\n"
3176 	       "\tsaved_data_buf: 0x%p\n",
3177 	       iop->ctx_in.next,
3178 	       iop->ctx_in.saved_data,
3179 	       iop->ctx_in.saved_data_buf);
3180 
3181 	i = 0;
3182 	while (cdesc_out) {
3183 		dma_descr_data *td;
3184 		printk("cdesc_out %d, desc=0x%p\n", i, cdesc_out->dma_descr);
3185 		printk("\n\tvirt_to_phys(desc): 0x%p\n", (char*)virt_to_phys(cdesc_out->dma_descr));
3186 		td = cdesc_out->dma_descr;
3187 		printk("\n\tbuf: 0x%p\n"
3188 		       "\tafter: 0x%p\n"
3189 		       "\tmd: 0x%04x\n"
3190 		       "\tnext: 0x%p\n",
3191 		       td->buf,
3192 		       td->after,
3193 		       td->md,
3194 		       td->next);
3195 		printk("flags:\n"
3196 		       "\twait:\t%d\n"
3197 		       "\teol:\t%d\n"
3198 		       "\touteop:\t%d\n"
3199 		       "\tineop:\t%d\n"
3200 		       "\tintr:\t%d\n",
3201 		       td->wait,
3202 		       td->eol,
3203 		       td->out_eop,
3204 		       td->in_eop,
3205 		       td->intr);
3206 		cdesc_out = cdesc_out->next;
3207 		i++;
3208 	}
3209 	i = 0;
3210 	while (cdesc_in) {
3211 		dma_descr_data *td;
3212 		printk("cdesc_in %d, desc=0x%p\n", i, cdesc_in->dma_descr);
3213 		printk("\n\tvirt_to_phys(desc): 0x%p\n", (char*)virt_to_phys(cdesc_in->dma_descr));
3214 		td = cdesc_in->dma_descr;
3215 		printk("\n\tbuf: 0x%p\n"
3216 		       "\tafter: 0x%p\n"
3217 		       "\tmd: 0x%04x\n"
3218 		       "\tnext: 0x%p\n",
3219 		       td->buf,
3220 		       td->after,
3221 		       td->md,
3222 		       td->next);
3223 		printk("flags:\n"
3224 		       "\twait:\t%d\n"
3225 		       "\teol:\t%d\n"
3226 		       "\touteop:\t%d\n"
3227 		       "\tineop:\t%d\n"
3228 		       "\tintr:\t%d\n",
3229 		       td->wait,
3230 		       td->eol,
3231 		       td->out_eop,
3232 		       td->in_eop,
3233 		       td->intr);
3234 		cdesc_in = cdesc_in->next;
3235 		i++;
3236 	}
3237 
3238 	printk("print_dma_descriptors end\n");
3239 }
3240 
3241 
print_strcop_crypto_op(struct strcop_crypto_op * cop)3242 static void print_strcop_crypto_op(struct strcop_crypto_op *cop)
3243 {
3244 	printk("print_strcop_crypto_op, 0x%p\n", cop);
3245 
3246 	/* Indata. */
3247 	printk("indata=0x%p\n"
3248 	       "inlen=%d\n"
3249 	       "do_cipher=%d\n"
3250 	       "decrypt=%d\n"
3251 	       "cipher_explicit=%d\n"
3252 	       "cipher_start=%d\n"
3253 	       "cipher_len=%d\n"
3254 	       "outdata=0x%p\n"
3255 	       "outlen=%d\n",
3256 	       cop->indata,
3257 	       cop->inlen,
3258 	       cop->do_cipher,
3259 	       cop->decrypt,
3260 	       cop->cipher_explicit,
3261 	       cop->cipher_start,
3262 	       cop->cipher_len,
3263 	       cop->cipher_outdata,
3264 	       cop->cipher_outlen);
3265 
3266 	printk("do_digest=%d\n"
3267 	       "digest_start=%d\n"
3268 	       "digest_len=%d\n",
3269 	       cop->do_digest,
3270 	       cop->digest_start,
3271 	       cop->digest_len);
3272 
3273 	printk("do_csum=%d\n"
3274 	       "csum_start=%d\n"
3275 	       "csum_len=%d\n",
3276 	       cop->do_csum,
3277 	       cop->csum_start,
3278 	       cop->csum_len);
3279 }
3280 
print_cryptocop_operation(struct cryptocop_operation * cop)3281 static void print_cryptocop_operation(struct cryptocop_operation *cop)
3282 {
3283 	struct cryptocop_desc      *d;
3284 	struct cryptocop_tfrm_cfg  *tc;
3285 	struct cryptocop_desc_cfg  *dc;
3286 	int                        i;
3287 
3288 	printk("print_cryptocop_operation, cop=0x%p\n\n", cop);
3289 	printk("sid: %lld\n", cop->sid);
3290 	printk("operation_status=%d\n"
3291 	       "use_dmalists=%d\n"
3292 	       "in_interrupt=%d\n"
3293 	       "fast_callback=%d\n",
3294 	       cop->operation_status,
3295 	       cop->use_dmalists,
3296 	       cop->in_interrupt,
3297 	       cop->fast_callback);
3298 
3299 	if (cop->use_dmalists){
3300 		print_user_dma_lists(&cop->list_op);
3301 	} else {
3302 		printk("cop->tfrm_op\n"
3303 		       "tfrm_cfg=0x%p\n"
3304 		       "desc=0x%p\n"
3305 		       "indata=0x%p\n"
3306 		       "incount=%d\n"
3307 		       "inlen=%d\n"
3308 		       "outdata=0x%p\n"
3309 		       "outcount=%d\n"
3310 		       "outlen=%d\n\n",
3311 		       cop->tfrm_op.tfrm_cfg,
3312 		       cop->tfrm_op.desc,
3313 		       cop->tfrm_op.indata,
3314 		       cop->tfrm_op.incount,
3315 		       cop->tfrm_op.inlen,
3316 		       cop->tfrm_op.outdata,
3317 		       cop->tfrm_op.outcount,
3318 		       cop->tfrm_op.outlen);
3319 
3320 		tc = cop->tfrm_op.tfrm_cfg;
3321 		while (tc){
3322 			printk("tfrm_cfg, 0x%p\n"
3323 			       "tid=%d\n"
3324 			       "flags=%d\n"
3325 			       "inject_ix=%d\n"
3326 			       "next=0x%p\n",
3327 			       tc,
3328 			       tc->tid,
3329 			       tc->flags,
3330 			       tc->inject_ix,
3331 			       tc->next);
3332 			tc = tc->next;
3333 		}
3334 		d = cop->tfrm_op.desc;
3335 		while (d){
3336 			printk("\n======================desc, 0x%p\n"
3337 			       "length=%d\n"
3338 			       "cfg=0x%p\n"
3339 			       "next=0x%p\n",
3340 			       d,
3341 			       d->length,
3342 			       d->cfg,
3343 			       d->next);
3344 			dc = d->cfg;
3345 			while (dc){
3346 				printk("=========desc_cfg, 0x%p\n"
3347 				       "tid=%d\n"
3348 				       "src=%d\n"
3349 				       "last=%d\n"
3350 				       "next=0x%p\n",
3351 				       dc,
3352 				       dc->tid,
3353 				       dc->src,
3354 				       dc->last,
3355 				       dc->next);
3356 				dc = dc->next;
3357 			}
3358 			d = d->next;
3359 		}
3360 		printk("\n====iniov\n");
3361 		for (i = 0; i < cop->tfrm_op.incount; i++){
3362 			printk("indata[%d]\n"
3363 			       "base=0x%p\n"
3364 			       "len=%d\n",
3365 			       i,
3366 			       cop->tfrm_op.indata[i].iov_base,
3367 			       cop->tfrm_op.indata[i].iov_len);
3368 		}
3369 		printk("\n====outiov\n");
3370 		for (i = 0; i < cop->tfrm_op.outcount; i++){
3371 			printk("outdata[%d]\n"
3372 			       "base=0x%p\n"
3373 			       "len=%d\n",
3374 			       i,
3375 			       cop->tfrm_op.outdata[i].iov_base,
3376 			       cop->tfrm_op.outdata[i].iov_len);
3377 		}
3378 	}
3379 	printk("------------end print_cryptocop_operation\n");
3380 }
3381 
3382 
print_user_dma_lists(struct cryptocop_dma_list_operation * dma_op)3383 static void print_user_dma_lists(struct cryptocop_dma_list_operation *dma_op)
3384 {
3385 	dma_descr_data *dd;
3386 	int i;
3387 
3388 	printk("print_user_dma_lists, dma_op=0x%p\n", dma_op);
3389 
3390 	printk("out_data_buf = 0x%p, phys_to_virt(out_data_buf) = 0x%p\n", dma_op->out_data_buf, phys_to_virt((unsigned long int)dma_op->out_data_buf));
3391 	printk("in_data_buf = 0x%p, phys_to_virt(in_data_buf) = 0x%p\n", dma_op->in_data_buf, phys_to_virt((unsigned long int)dma_op->in_data_buf));
3392 
3393 	printk("##############outlist\n");
3394 	dd = phys_to_virt((unsigned long int)dma_op->outlist);
3395 	i = 0;
3396 	while (dd != NULL) {
3397 		printk("#%d phys_to_virt(desc) 0x%p\n", i, dd);
3398 		printk("\n\tbuf: 0x%p\n"
3399 		       "\tafter: 0x%p\n"
3400 		       "\tmd: 0x%04x\n"
3401 		       "\tnext: 0x%p\n",
3402 		       dd->buf,
3403 		       dd->after,
3404 		       dd->md,
3405 		       dd->next);
3406 		printk("flags:\n"
3407 		       "\twait:\t%d\n"
3408 		       "\teol:\t%d\n"
3409 		       "\touteop:\t%d\n"
3410 		       "\tineop:\t%d\n"
3411 		       "\tintr:\t%d\n",
3412 		       dd->wait,
3413 		       dd->eol,
3414 		       dd->out_eop,
3415 		       dd->in_eop,
3416 		       dd->intr);
3417 		if (dd->eol)
3418 			dd = NULL;
3419 		else
3420 			dd = phys_to_virt((unsigned long int)dd->next);
3421 		++i;
3422 	}
3423 
3424 	printk("##############inlist\n");
3425 	dd = phys_to_virt((unsigned long int)dma_op->inlist);
3426 	i = 0;
3427 	while (dd != NULL) {
3428 		printk("#%d phys_to_virt(desc) 0x%p\n", i, dd);
3429 		printk("\n\tbuf: 0x%p\n"
3430 		       "\tafter: 0x%p\n"
3431 		       "\tmd: 0x%04x\n"
3432 		       "\tnext: 0x%p\n",
3433 		       dd->buf,
3434 		       dd->after,
3435 		       dd->md,
3436 		       dd->next);
3437 		printk("flags:\n"
3438 		       "\twait:\t%d\n"
3439 		       "\teol:\t%d\n"
3440 		       "\touteop:\t%d\n"
3441 		       "\tineop:\t%d\n"
3442 		       "\tintr:\t%d\n",
3443 		       dd->wait,
3444 		       dd->eol,
3445 		       dd->out_eop,
3446 		       dd->in_eop,
3447 		       dd->intr);
3448 		if (dd->eol)
3449 			dd = NULL;
3450 		else
3451 			dd = phys_to_virt((unsigned long int)dd->next);
3452 		++i;
3453 	}
3454 }
3455 
3456 
print_lock_status(void)3457 static void print_lock_status(void)
3458 {
3459 	printk("**********************print_lock_status\n");
3460 	printk("cryptocop_completed_jobs_lock %d\n", spin_is_locked(&cryptocop_completed_jobs_lock));
3461 	printk("cryptocop_job_queue_lock %d\n", spin_is_locked(&cryptocop_job_queue_lock));
3462 	printk("descr_pool_lock %d\n", spin_is_locked(&descr_pool_lock));
3463 	printk("cryptocop_sessions_lock %d\n", spin_is_locked(cryptocop_sessions_lock));
3464 	printk("running_job_lock %d\n", spin_is_locked(running_job_lock));
3465 	printk("cryptocop_process_lock %d\n", spin_is_locked(cryptocop_process_lock));
3466 }
3467 #endif /* LDEBUG */
3468 
3469 
3470 static const char cryptocop_name[] = "ETRAX FS stream co-processor";
3471 
init_stream_coprocessor(void)3472 static int init_stream_coprocessor(void)
3473 {
3474 	int err;
3475 	int i;
3476 	static int initialized = 0;
3477 
3478 	if (initialized)
3479 		return 0;
3480 
3481 	initialized = 1;
3482 
3483 	printk("ETRAX FS stream co-processor driver v0.01, (c) 2003 Axis Communications AB\n");
3484 
3485 	err = register_chrdev(CRYPTOCOP_MAJOR, cryptocop_name, &cryptocop_fops);
3486 	if (err < 0) {
3487 		printk(KERN_ERR "stream co-processor: could not get major number.\n");
3488 		return err;
3489 	}
3490 
3491 	err = init_cryptocop();
3492 	if (err) {
3493 		(void)unregister_chrdev(CRYPTOCOP_MAJOR, cryptocop_name);
3494 		return err;
3495 	}
3496 	err = cryptocop_job_queue_init();
3497 	if (err) {
3498 		release_cryptocop();
3499 		(void)unregister_chrdev(CRYPTOCOP_MAJOR, cryptocop_name);
3500 		return err;
3501 	}
3502 	/* Init the descriptor pool. */
3503 	for (i = 0; i < CRYPTOCOP_DESCRIPTOR_POOL_SIZE - 1; i++) {
3504 		descr_pool[i].from_pool = 1;
3505 		descr_pool[i].next = &descr_pool[i + 1];
3506 	}
3507 	descr_pool[i].from_pool = 1;
3508 	descr_pool[i].next = NULL;
3509 	descr_pool_free_list = &descr_pool[0];
3510 	descr_pool_no_free = CRYPTOCOP_DESCRIPTOR_POOL_SIZE;
3511 
3512 	spin_lock_init(&cryptocop_completed_jobs_lock);
3513 	spin_lock_init(&cryptocop_job_queue_lock);
3514 	spin_lock_init(&descr_pool_lock);
3515 	spin_lock_init(&cryptocop_sessions_lock);
3516 	spin_lock_init(&running_job_lock);
3517 	spin_lock_init(&cryptocop_process_lock);
3518 
3519 	cryptocop_sessions = NULL;
3520 	next_sid = 1;
3521 
3522 	cryptocop_running_job = NULL;
3523 
3524 	printk("stream co-processor: init done.\n");
3525 	return 0;
3526 }
3527 
exit_stream_coprocessor(void)3528 static void __exit exit_stream_coprocessor(void)
3529 {
3530 	release_cryptocop();
3531 	cryptocop_job_queue_close();
3532 }
3533 
3534 module_init(init_stream_coprocessor);
3535 module_exit(exit_stream_coprocessor);
3536 
3537