1 /* SPDX-License-Identifier: GPL-2.0 */
2 #ifndef __NITROX_REQ_H
3 #define __NITROX_REQ_H
4
5 #include <linux/dma-mapping.h>
6 #include <crypto/aes.h>
7
8 #include "nitrox_dev.h"
9
10 #define PENDING_SIG 0xFFFFFFFFFFFFFFFFUL
11 #define PRIO 4001
12
13 /**
14 * struct gphdr - General purpose Header
15 * @param0: first parameter.
16 * @param1: second parameter.
17 * @param2: third parameter.
18 * @param3: fourth parameter.
19 *
20 * Params tell the iv and enc/dec data offsets.
21 */
22 struct gphdr {
23 __be16 param0;
24 __be16 param1;
25 __be16 param2;
26 __be16 param3;
27 };
28
29 /**
30 * struct se_req_ctrl - SE request information.
31 * @arg: Minor number of the opcode
32 * @ctxc: Context control.
33 * @unca: Uncertainity enabled.
34 * @info: Additional information for SE cores.
35 * @ctxl: Context length in bytes.
36 * @uddl: User defined data length
37 */
38 union se_req_ctrl {
39 u64 value;
40 struct {
41 u64 raz : 22;
42 u64 arg : 8;
43 u64 ctxc : 2;
44 u64 unca : 1;
45 u64 info : 3;
46 u64 unc : 8;
47 u64 ctxl : 12;
48 u64 uddl : 8;
49 } s;
50 };
51
52 #define MAX_IV_LEN 16
53
54 /**
55 * struct se_crypto_request - SE crypto request structure.
56 * @opcode: Request opcode (enc/dec)
57 * @flags: flags from crypto subsystem
58 * @ctx_handle: Crypto context handle.
59 * @gph: GP Header
60 * @ctrl: Request Information.
61 * @orh: ORH address
62 * @comp: completion address
63 * @src: Input sglist
64 * @dst: Output sglist
65 */
66 struct se_crypto_request {
67 u8 opcode;
68 gfp_t gfp;
69 u32 flags;
70 u64 ctx_handle;
71
72 struct gphdr gph;
73 union se_req_ctrl ctrl;
74 u64 *orh;
75 u64 *comp;
76
77 struct scatterlist *src;
78 struct scatterlist *dst;
79 };
80
81 /* Crypto opcodes */
82 #define FLEXI_CRYPTO_ENCRYPT_HMAC 0x33
83 #define ENCRYPT 0
84 #define DECRYPT 1
85
86 /* IV from context */
87 #define IV_FROM_CTX 0
88 /* IV from Input data */
89 #define IV_FROM_DPTR 1
90
91 /**
92 * cipher opcodes for firmware
93 */
94 enum flexi_cipher {
95 CIPHER_NULL = 0,
96 CIPHER_3DES_CBC,
97 CIPHER_3DES_ECB,
98 CIPHER_AES_CBC,
99 CIPHER_AES_ECB,
100 CIPHER_AES_CFB,
101 CIPHER_AES_CTR,
102 CIPHER_AES_GCM,
103 CIPHER_AES_XTS,
104 CIPHER_AES_CCM,
105 CIPHER_AES_CBC_CTS,
106 CIPHER_AES_ECB_CTS,
107 CIPHER_INVALID
108 };
109
110 enum flexi_auth {
111 AUTH_NULL = 0,
112 AUTH_MD5,
113 AUTH_SHA1,
114 AUTH_SHA2_SHA224,
115 AUTH_SHA2_SHA256,
116 AUTH_SHA2_SHA384,
117 AUTH_SHA2_SHA512,
118 AUTH_GMAC,
119 AUTH_INVALID
120 };
121
122 /**
123 * struct crypto_keys - Crypto keys
124 * @key: Encryption key or KEY1 for AES-XTS
125 * @iv: Encryption IV or Tweak for AES-XTS
126 */
127 struct crypto_keys {
128 union {
129 u8 key[AES_MAX_KEY_SIZE];
130 u8 key1[AES_MAX_KEY_SIZE];
131 } u;
132 u8 iv[AES_BLOCK_SIZE];
133 };
134
135 /**
136 * struct auth_keys - Authentication keys
137 * @ipad: IPAD or KEY2 for AES-XTS
138 * @opad: OPAD or AUTH KEY if auth_input_type = 1
139 */
140 struct auth_keys {
141 union {
142 u8 ipad[64];
143 u8 key2[64];
144 } u;
145 u8 opad[64];
146 };
147
148 union fc_ctx_flags {
149 __be64 f;
150 struct {
151 #if defined(__BIG_ENDIAN_BITFIELD)
152 u64 cipher_type : 4;
153 u64 reserved_59 : 1;
154 u64 aes_keylen : 2;
155 u64 iv_source : 1;
156 u64 hash_type : 4;
157 u64 reserved_49_51 : 3;
158 u64 auth_input_type: 1;
159 u64 mac_len : 8;
160 u64 reserved_0_39 : 40;
161 #else
162 u64 reserved_0_39 : 40;
163 u64 mac_len : 8;
164 u64 auth_input_type: 1;
165 u64 reserved_49_51 : 3;
166 u64 hash_type : 4;
167 u64 iv_source : 1;
168 u64 aes_keylen : 2;
169 u64 reserved_59 : 1;
170 u64 cipher_type : 4;
171 #endif
172 } w0;
173 };
174 /**
175 * struct flexi_crypto_context - Crypto context
176 * @cipher_type: Encryption cipher type
177 * @aes_keylen: AES key length
178 * @iv_source: Encryption IV source
179 * @hash_type: Authentication type
180 * @auth_input_type: Authentication input type
181 * 1 - Authentication IV and KEY, microcode calculates OPAD/IPAD
182 * 0 - Authentication OPAD/IPAD
183 * @mac_len: mac length
184 * @crypto: Crypto keys
185 * @auth: Authentication keys
186 */
187 struct flexi_crypto_context {
188 union fc_ctx_flags flags;
189 struct crypto_keys crypto;
190 struct auth_keys auth;
191 };
192
193 struct crypto_ctx_hdr {
194 struct dma_pool *pool;
195 dma_addr_t dma;
196 void *vaddr;
197 };
198
199 struct nitrox_crypto_ctx {
200 struct nitrox_device *ndev;
201 union {
202 u64 ctx_handle;
203 struct flexi_crypto_context *fctx;
204 } u;
205 struct crypto_ctx_hdr *chdr;
206 };
207
208 struct nitrox_kcrypt_request {
209 struct se_crypto_request creq;
210 u8 *src;
211 u8 *dst;
212 };
213
214 /**
215 * struct nitrox_aead_rctx - AEAD request context
216 * @nkreq: Base request context
217 * @cryptlen: Encryption/Decryption data length
218 * @assoclen: AAD length
219 * @srclen: Input buffer length
220 * @dstlen: Output buffer length
221 * @iv: IV data
222 * @ivsize: IV data length
223 * @flags: AEAD req flags
224 * @ctx_handle: Device context handle
225 * @src: Source sglist
226 * @dst: Destination sglist
227 * @ctrl_arg: Identifies the request type (ENCRYPT/DECRYPT)
228 */
229 struct nitrox_aead_rctx {
230 struct nitrox_kcrypt_request nkreq;
231 unsigned int cryptlen;
232 unsigned int assoclen;
233 unsigned int srclen;
234 unsigned int dstlen;
235 u8 *iv;
236 int ivsize;
237 u32 flags;
238 u64 ctx_handle;
239 struct scatterlist *src;
240 struct scatterlist *dst;
241 u8 ctrl_arg;
242 };
243
244 /**
245 * struct nitrox_rfc4106_rctx - rfc4106 cipher request context
246 * @base: AEAD request context
247 * @src: Source sglist
248 * @dst: Destination sglist
249 * @assoc: AAD
250 */
251 struct nitrox_rfc4106_rctx {
252 struct nitrox_aead_rctx base;
253 struct scatterlist src[3];
254 struct scatterlist dst[3];
255 u8 assoc[20];
256 };
257
258 /**
259 * struct pkt_instr_hdr - Packet Instruction Header
260 * @g: Gather used
261 * When [G] is set and [GSZ] != 0, the instruction is
262 * indirect gather instruction.
263 * When [G] is set and [GSZ] = 0, the instruction is
264 * direct gather instruction.
265 * @gsz: Number of pointers in the indirect gather list
266 * @ihi: When set hardware duplicates the 1st 8 bytes of pkt_instr_hdr
267 * and adds them to the packet after the pkt_instr_hdr but before any UDD
268 * @ssz: Not used by the input hardware. But can become slc_store_int[SSZ]
269 * when [IHI] is set.
270 * @fsz: The number of front data bytes directly included in the
271 * PCIe instruction.
272 * @tlen: The length of the input packet in bytes, include:
273 * - 16B pkt_hdr
274 * - Inline context bytes if any,
275 * - UDD if any,
276 * - packet payload bytes
277 */
278 union pkt_instr_hdr {
279 u64 value;
280 struct {
281 #if defined(__BIG_ENDIAN_BITFIELD)
282 u64 raz_48_63 : 16;
283 u64 g : 1;
284 u64 gsz : 7;
285 u64 ihi : 1;
286 u64 ssz : 7;
287 u64 raz_30_31 : 2;
288 u64 fsz : 6;
289 u64 raz_16_23 : 8;
290 u64 tlen : 16;
291 #else
292 u64 tlen : 16;
293 u64 raz_16_23 : 8;
294 u64 fsz : 6;
295 u64 raz_30_31 : 2;
296 u64 ssz : 7;
297 u64 ihi : 1;
298 u64 gsz : 7;
299 u64 g : 1;
300 u64 raz_48_63 : 16;
301 #endif
302 } s;
303 };
304
305 /**
306 * struct pkt_hdr - Packet Input Header
307 * @opcode: Request opcode (Major)
308 * @arg: Request opcode (Minor)
309 * @ctxc: Context control.
310 * @unca: When set [UNC] is the uncertainty count for an input packet.
311 * The hardware uses uncertainty counts to predict
312 * output buffer use and avoid deadlock.
313 * @info: Not used by input hardware. Available for use
314 * during SE processing.
315 * @destport: The expected destination port/ring/channel for the packet.
316 * @unc: Uncertainty count for an input packet.
317 * @grp: SE group that will process the input packet.
318 * @ctxl: Context Length in 64-bit words.
319 * @uddl: User-defined data (UDD) length in bytes.
320 * @ctxp: Context pointer. CTXP<63,2:0> must be zero in all cases.
321 */
322 union pkt_hdr {
323 u64 value[2];
324 struct {
325 #if defined(__BIG_ENDIAN_BITFIELD)
326 u64 opcode : 8;
327 u64 arg : 8;
328 u64 ctxc : 2;
329 u64 unca : 1;
330 u64 raz_44 : 1;
331 u64 info : 3;
332 u64 destport : 9;
333 u64 unc : 8;
334 u64 raz_19_23 : 5;
335 u64 grp : 3;
336 u64 raz_15 : 1;
337 u64 ctxl : 7;
338 u64 uddl : 8;
339 #else
340 u64 uddl : 8;
341 u64 ctxl : 7;
342 u64 raz_15 : 1;
343 u64 grp : 3;
344 u64 raz_19_23 : 5;
345 u64 unc : 8;
346 u64 destport : 9;
347 u64 info : 3;
348 u64 raz_44 : 1;
349 u64 unca : 1;
350 u64 ctxc : 2;
351 u64 arg : 8;
352 u64 opcode : 8;
353 #endif
354 __be64 ctxp;
355 } s;
356 };
357
358 /**
359 * struct slc_store_info - Solicited Paceket Output Store Information.
360 * @ssz: The number of scatterlist pointers for the solicited output port
361 * packet.
362 * @rptr: The result pointer for the solicited output port packet.
363 * If [SSZ]=0, [RPTR] must point directly to a buffer on the remote
364 * host that is large enough to hold the entire output packet.
365 * If [SSZ]!=0, [RPTR] must point to an array of ([SSZ]+3)/4
366 * sglist components at [RPTR] on the remote host.
367 */
368 union slc_store_info {
369 u64 value[2];
370 struct {
371 #if defined(__BIG_ENDIAN_BITFIELD)
372 u64 raz_39_63 : 25;
373 u64 ssz : 7;
374 u64 raz_0_31 : 32;
375 #else
376 u64 raz_0_31 : 32;
377 u64 ssz : 7;
378 u64 raz_39_63 : 25;
379 #endif
380 __be64 rptr;
381 } s;
382 };
383
384 /**
385 * struct nps_pkt_instr - NPS Packet Instruction of SE cores.
386 * @dptr0 : Input pointer points to buffer in remote host.
387 * @ih: Packet Instruction Header (8 bytes)
388 * @irh: Packet Input Header (16 bytes)
389 * @slc: Solicited Packet Output Store Information (16 bytes)
390 * @fdata: Front data
391 *
392 * 64-Byte Instruction Format
393 */
394 struct nps_pkt_instr {
395 __be64 dptr0;
396 union pkt_instr_hdr ih;
397 union pkt_hdr irh;
398 union slc_store_info slc;
399 u64 fdata[2];
400 };
401
402 /**
403 * struct aqmq_command_s - The 32 byte command for AE processing.
404 * @opcode: Request opcode
405 * @param1: Request control parameter 1
406 * @param2: Request control parameter 2
407 * @dlen: Input length
408 * @dptr: Input pointer points to buffer in remote host
409 * @rptr: Result pointer points to buffer in remote host
410 * @grp: AQM Group (0..7)
411 * @cptr: Context pointer
412 */
413 struct aqmq_command_s {
414 __be16 opcode;
415 __be16 param1;
416 __be16 param2;
417 __be16 dlen;
418 __be64 dptr;
419 __be64 rptr;
420 union {
421 __be64 word3;
422 #if defined(__BIG_ENDIAN_BITFIELD)
423 u64 grp : 3;
424 u64 cptr : 61;
425 #else
426 u64 cptr : 61;
427 u64 grp : 3;
428 #endif
429 };
430 };
431
432 /**
433 * struct ctx_hdr - Book keeping data about the crypto context
434 * @pool: Pool used to allocate crypto context
435 * @dma: Base DMA address of the cypto context
436 * @ctx_dma: Actual usable crypto context for NITROX
437 */
438 struct ctx_hdr {
439 struct dma_pool *pool;
440 dma_addr_t dma;
441 dma_addr_t ctx_dma;
442 };
443
444 /*
445 * struct sglist_component - SG list component format
446 * @len0: The number of bytes at [PTR0] on the remote host.
447 * @len1: The number of bytes at [PTR1] on the remote host.
448 * @len2: The number of bytes at [PTR2] on the remote host.
449 * @len3: The number of bytes at [PTR3] on the remote host.
450 * @dma0: First pointer point to buffer in remote host.
451 * @dma1: Second pointer point to buffer in remote host.
452 * @dma2: Third pointer point to buffer in remote host.
453 * @dma3: Fourth pointer point to buffer in remote host.
454 */
455 struct nitrox_sgcomp {
456 __be16 len[4];
457 __be64 dma[4];
458 };
459
460 /*
461 * strutct nitrox_sgtable - SG list information
462 * @sgmap_cnt: Number of buffers mapped
463 * @total_bytes: Total bytes in sglist.
464 * @sgcomp_len: Total sglist components length.
465 * @sgcomp_dma: DMA address of sglist component.
466 * @sg: crypto request buffer.
467 * @sgcomp: sglist component for NITROX.
468 */
469 struct nitrox_sgtable {
470 u8 sgmap_cnt;
471 u16 total_bytes;
472 u32 sgcomp_len;
473 dma_addr_t sgcomp_dma;
474 struct scatterlist *sg;
475 struct nitrox_sgcomp *sgcomp;
476 };
477
478 /* Response Header Length */
479 #define ORH_HLEN 8
480 /* Completion bytes Length */
481 #define COMP_HLEN 8
482
483 struct resp_hdr {
484 u64 *orh;
485 u64 *completion;
486 };
487
488 typedef void (*completion_t)(void *arg, int err);
489
490 /**
491 * struct nitrox_softreq - Represents the NIROX Request.
492 * @response: response list entry
493 * @backlog: Backlog list entry
494 * @ndev: Device used to submit the request
495 * @cmdq: Command queue for submission
496 * @resp: Response headers
497 * @instr: 64B instruction
498 * @in: SG table for input
499 * @out SG table for output
500 * @tstamp: Request submitted time in jiffies
501 * @callback: callback after request completion/timeout
502 * @cb_arg: callback argument
503 */
504 struct nitrox_softreq {
505 struct list_head response;
506 struct list_head backlog;
507
508 u32 flags;
509 gfp_t gfp;
510 atomic_t status;
511
512 struct nitrox_device *ndev;
513 struct nitrox_cmdq *cmdq;
514
515 struct nps_pkt_instr instr;
516 struct resp_hdr resp;
517 struct nitrox_sgtable in;
518 struct nitrox_sgtable out;
519
520 unsigned long tstamp;
521
522 completion_t callback;
523 void *cb_arg;
524 };
525
526 static inline int flexi_aes_keylen(int keylen)
527 {
528 int aes_keylen;
529
530 switch (keylen) {
531 case AES_KEYSIZE_128:
532 aes_keylen = 1;
533 break;
534 case AES_KEYSIZE_192:
535 aes_keylen = 2;
536 break;
537 case AES_KEYSIZE_256:
538 aes_keylen = 3;
539 break;
540 default:
541 aes_keylen = -EINVAL;
542 break;
543 }
544 return aes_keylen;
545 }
546
547 static inline void *alloc_req_buf(int nents, int extralen, gfp_t gfp)
548 {
549 size_t size;
550
551 size = sizeof(struct scatterlist) * nents;
552 size += extralen;
553
554 return kzalloc(size, gfp);
555 }
556
557 /**
558 * create_single_sg - Point SG entry to the data
559 * @sg: Destination SG list
560 * @buf: Data
561 * @buflen: Data length
562 *
563 * Returns next free entry in the destination SG list
564 **/
565 static inline struct scatterlist *create_single_sg(struct scatterlist *sg,
566 void *buf, int buflen)
567 {
568 sg_set_buf(sg, buf, buflen);
569 sg++;
570 return sg;
571 }
572
573 /**
574 * create_multi_sg - Create multiple sg entries with buflen data length from
575 * source sglist
576 * @to_sg: Destination SG list
577 * @from_sg: Source SG list
578 * @buflen: Data length
579 *
580 * Returns next free entry in the destination SG list
581 **/
582 static inline struct scatterlist *create_multi_sg(struct scatterlist *to_sg,
583 struct scatterlist *from_sg,
584 int buflen)
585 {
586 struct scatterlist *sg = to_sg;
587 unsigned int sglen;
588
589 for (; buflen && from_sg; buflen -= sglen) {
590 sglen = from_sg->length;
591 if (sglen > buflen)
592 sglen = buflen;
593
594 sg_set_buf(sg, sg_virt(from_sg), sglen);
595 from_sg = sg_next(from_sg);
596 sg++;
597 }
598
599 return sg;
600 }
601
602 static inline void set_orh_value(u64 *orh)
603 {
604 WRITE_ONCE(*orh, PENDING_SIG);
605 }
606
607 static inline void set_comp_value(u64 *comp)
608 {
609 WRITE_ONCE(*comp, PENDING_SIG);
610 }
611
612 static inline int alloc_src_req_buf(struct nitrox_kcrypt_request *nkreq,
613 int nents, int ivsize)
614 {
615 struct se_crypto_request *creq = &nkreq->creq;
616
617 nkreq->src = alloc_req_buf(nents, ivsize, creq->gfp);
618 if (!nkreq->src)
619 return -ENOMEM;
620
621 return 0;
622 }
623
624 static inline void nitrox_creq_copy_iv(char *dst, char *src, int size)
625 {
626 memcpy(dst, src, size);
627 }
628
629 static inline struct scatterlist *nitrox_creq_src_sg(char *iv, int ivsize)
630 {
631 return (struct scatterlist *)(iv + ivsize);
632 }
633
634 static inline void nitrox_creq_set_src_sg(struct nitrox_kcrypt_request *nkreq,
635 int nents, int ivsize,
636 struct scatterlist *src, int buflen)
637 {
638 char *iv = nkreq->src;
639 struct scatterlist *sg;
640 struct se_crypto_request *creq = &nkreq->creq;
641
642 creq->src = nitrox_creq_src_sg(iv, ivsize);
643 sg = creq->src;
644 sg_init_table(sg, nents);
645
646 /* Input format:
647 * +----+----------------+
648 * | IV | SRC sg entries |
649 * +----+----------------+
650 */
651
652 /* IV */
653 sg = create_single_sg(sg, iv, ivsize);
654 /* SRC entries */
655 create_multi_sg(sg, src, buflen);
656 }
657
658 static inline int alloc_dst_req_buf(struct nitrox_kcrypt_request *nkreq,
659 int nents)
660 {
661 int extralen = ORH_HLEN + COMP_HLEN;
662 struct se_crypto_request *creq = &nkreq->creq;
663
664 nkreq->dst = alloc_req_buf(nents, extralen, creq->gfp);
665 if (!nkreq->dst)
666 return -ENOMEM;
667
668 return 0;
669 }
670
671 static inline void nitrox_creq_set_orh(struct nitrox_kcrypt_request *nkreq)
672 {
673 struct se_crypto_request *creq = &nkreq->creq;
674
675 creq->orh = (u64 *)(nkreq->dst);
676 set_orh_value(creq->orh);
677 }
678
679 static inline void nitrox_creq_set_comp(struct nitrox_kcrypt_request *nkreq)
680 {
681 struct se_crypto_request *creq = &nkreq->creq;
682
683 creq->comp = (u64 *)(nkreq->dst + ORH_HLEN);
684 set_comp_value(creq->comp);
685 }
686
687 static inline struct scatterlist *nitrox_creq_dst_sg(char *dst)
688 {
689 return (struct scatterlist *)(dst + ORH_HLEN + COMP_HLEN);
690 }
691
692 static inline void nitrox_creq_set_dst_sg(struct nitrox_kcrypt_request *nkreq,
693 int nents, int ivsize,
694 struct scatterlist *dst, int buflen)
695 {
696 struct se_crypto_request *creq = &nkreq->creq;
697 struct scatterlist *sg;
698 char *iv = nkreq->src;
699
700 creq->dst = nitrox_creq_dst_sg(nkreq->dst);
701 sg = creq->dst;
702 sg_init_table(sg, nents);
703
704 /* Output format:
705 * +-----+----+----------------+-----------------+
706 * | ORH | IV | DST sg entries | COMPLETION Bytes|
707 * +-----+----+----------------+-----------------+
708 */
709
710 /* ORH */
711 sg = create_single_sg(sg, creq->orh, ORH_HLEN);
712 /* IV */
713 sg = create_single_sg(sg, iv, ivsize);
714 /* DST entries */
715 sg = create_multi_sg(sg, dst, buflen);
716 /* COMPLETION Bytes */
717 create_single_sg(sg, creq->comp, COMP_HLEN);
718 }
719
720 #endif /* __NITROX_REQ_H */