This source file includes following definitions.
- qedr_inc_sw_cons
- qedr_inc_sw_prod
- qedr_get_dmac
- get_qedr_ucontext
- get_qedr_dev
- get_qedr_pd
- get_qedr_cq
- get_qedr_qp
- get_qedr_ah
- get_qedr_mr
- get_qedr_srq
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32 #ifndef __QEDR_H__
33 #define __QEDR_H__
34
35 #include <linux/pci.h>
36 #include <linux/xarray.h>
37 #include <rdma/ib_addr.h>
38 #include <linux/qed/qed_if.h>
39 #include <linux/qed/qed_chain.h>
40 #include <linux/qed/qed_rdma_if.h>
41 #include <linux/qed/qede_rdma.h>
42 #include <linux/qed/roce_common.h>
43 #include <linux/completion.h>
44 #include "qedr_hsi_rdma.h"
45
46 #define QEDR_NODE_DESC "QLogic 579xx RoCE HCA"
47 #define DP_NAME(_dev) dev_name(&(_dev)->ibdev.dev)
48 #define IS_IWARP(_dev) ((_dev)->rdma_type == QED_RDMA_TYPE_IWARP)
49 #define IS_ROCE(_dev) ((_dev)->rdma_type == QED_RDMA_TYPE_ROCE)
50
51 #define DP_DEBUG(dev, module, fmt, ...) \
52 pr_debug("(%s) " module ": " fmt, \
53 DP_NAME(dev) ? DP_NAME(dev) : "", ## __VA_ARGS__)
54
55 #define QEDR_MSG_INIT "INIT"
56 #define QEDR_MSG_MISC "MISC"
57 #define QEDR_MSG_CQ " CQ"
58 #define QEDR_MSG_MR " MR"
59 #define QEDR_MSG_RQ " RQ"
60 #define QEDR_MSG_SQ " SQ"
61 #define QEDR_MSG_QP " QP"
62 #define QEDR_MSG_SRQ " SRQ"
63 #define QEDR_MSG_GSI " GSI"
64 #define QEDR_MSG_IWARP " IW"
65
66 #define QEDR_CQ_MAGIC_NUMBER (0x11223344)
67
68 #define FW_PAGE_SIZE (RDMA_RING_PAGE_SIZE)
69 #define FW_PAGE_SHIFT (12)
70
71 struct qedr_dev;
72
73 struct qedr_cnq {
74 struct qedr_dev *dev;
75 struct qed_chain pbl;
76 struct qed_sb_info *sb;
77 char name[32];
78 u64 n_comp;
79 __le16 *hw_cons_ptr;
80 u8 index;
81 };
82
83 #define QEDR_MAX_SGID 128
84
85 struct qedr_device_attr {
86 u32 vendor_id;
87 u32 vendor_part_id;
88 u32 hw_ver;
89 u64 fw_ver;
90 u64 node_guid;
91 u64 sys_image_guid;
92 u8 max_cnq;
93 u8 max_sge;
94 u16 max_inline;
95 u32 max_sqe;
96 u32 max_rqe;
97 u8 max_qp_resp_rd_atomic_resc;
98 u8 max_qp_req_rd_atomic_resc;
99 u64 max_dev_resp_rd_atomic_resc;
100 u32 max_cq;
101 u32 max_qp;
102 u32 max_mr;
103 u64 max_mr_size;
104 u32 max_cqe;
105 u32 max_mw;
106 u32 max_fmr;
107 u32 max_mr_mw_fmr_pbl;
108 u64 max_mr_mw_fmr_size;
109 u32 max_pd;
110 u32 max_ah;
111 u8 max_pkey;
112 u32 max_srq;
113 u32 max_srq_wr;
114 u8 max_srq_sge;
115 u8 max_stats_queues;
116 u32 dev_caps;
117
118 u64 page_size_caps;
119 u8 dev_ack_delay;
120 u32 reserved_lkey;
121 u32 bad_pkey_counter;
122 struct qed_rdma_events events;
123 };
124
125 #define QEDR_ENET_STATE_BIT (0)
126
127 struct qedr_dev {
128 struct ib_device ibdev;
129 struct qed_dev *cdev;
130 struct pci_dev *pdev;
131 struct net_device *ndev;
132
133 enum ib_atomic_cap atomic_cap;
134
135 void *rdma_ctx;
136 struct qedr_device_attr attr;
137
138 const struct qed_rdma_ops *ops;
139 struct qed_int_info int_info;
140
141 struct qed_sb_info *sb_array;
142 struct qedr_cnq *cnq_array;
143 int num_cnq;
144 int sb_start;
145
146 void __iomem *db_addr;
147 u64 db_phys_addr;
148 u32 db_size;
149 u16 dpi;
150
151 union ib_gid *sgid_tbl;
152
153
154 spinlock_t sgid_lock;
155
156 u64 guid;
157
158 u32 dp_module;
159 u8 dp_level;
160 u8 num_hwfns;
161 #define QEDR_IS_CMT(dev) ((dev)->num_hwfns > 1)
162 u8 affin_hwfn_idx;
163 u8 gsi_ll2_handle;
164
165 uint wq_multiplier;
166 u8 gsi_ll2_mac_address[ETH_ALEN];
167 int gsi_qp_created;
168 struct qedr_cq *gsi_sqcq;
169 struct qedr_cq *gsi_rqcq;
170 struct qedr_qp *gsi_qp;
171 enum qed_rdma_type rdma_type;
172 struct xarray qps;
173 struct xarray srqs;
174 struct workqueue_struct *iwarp_wq;
175 u16 iwarp_max_mtu;
176
177 unsigned long enet_state;
178
179 u8 user_dpm_enabled;
180 };
181
182 #define QEDR_MAX_SQ_PBL (0x8000)
183 #define QEDR_MAX_SQ_PBL_ENTRIES (0x10000 / sizeof(void *))
184 #define QEDR_SQE_ELEMENT_SIZE (sizeof(struct rdma_sq_sge))
185 #define QEDR_MAX_SQE_ELEMENTS_PER_SQE (ROCE_REQ_MAX_SINGLE_SQ_WQE_SIZE / \
186 QEDR_SQE_ELEMENT_SIZE)
187 #define QEDR_MAX_SQE_ELEMENTS_PER_PAGE ((RDMA_RING_PAGE_SIZE) / \
188 QEDR_SQE_ELEMENT_SIZE)
189 #define QEDR_MAX_SQE ((QEDR_MAX_SQ_PBL_ENTRIES) *\
190 (RDMA_RING_PAGE_SIZE) / \
191 (QEDR_SQE_ELEMENT_SIZE) /\
192 (QEDR_MAX_SQE_ELEMENTS_PER_SQE))
193
194 #define QEDR_MAX_RQ_PBL (0x2000)
195 #define QEDR_MAX_RQ_PBL_ENTRIES (0x10000 / sizeof(void *))
196 #define QEDR_RQE_ELEMENT_SIZE (sizeof(struct rdma_rq_sge))
197 #define QEDR_MAX_RQE_ELEMENTS_PER_RQE (RDMA_MAX_SGE_PER_RQ_WQE)
198 #define QEDR_MAX_RQE_ELEMENTS_PER_PAGE ((RDMA_RING_PAGE_SIZE) / \
199 QEDR_RQE_ELEMENT_SIZE)
200 #define QEDR_MAX_RQE ((QEDR_MAX_RQ_PBL_ENTRIES) *\
201 (RDMA_RING_PAGE_SIZE) / \
202 (QEDR_RQE_ELEMENT_SIZE) /\
203 (QEDR_MAX_RQE_ELEMENTS_PER_RQE))
204
205 #define QEDR_CQE_SIZE (sizeof(union rdma_cqe))
206 #define QEDR_MAX_CQE_PBL_SIZE (512 * 1024)
207 #define QEDR_MAX_CQE_PBL_ENTRIES (((QEDR_MAX_CQE_PBL_SIZE) / \
208 sizeof(u64)) - 1)
209 #define QEDR_MAX_CQES ((u32)((QEDR_MAX_CQE_PBL_ENTRIES) * \
210 (QED_CHAIN_PAGE_SIZE) / QEDR_CQE_SIZE))
211
212 #define QEDR_ROCE_MAX_CNQ_SIZE (0x4000)
213
214 #define QEDR_MAX_PORT (1)
215 #define QEDR_PORT (1)
216
217 #define QEDR_UVERBS(CMD_NAME) (1ull << IB_USER_VERBS_CMD_##CMD_NAME)
218
219 #define QEDR_ROCE_PKEY_MAX 1
220 #define QEDR_ROCE_PKEY_TABLE_LEN 1
221 #define QEDR_ROCE_PKEY_DEFAULT 0xffff
222
223 struct qedr_pbl {
224 struct list_head list_entry;
225 void *va;
226 dma_addr_t pa;
227 };
228
229 struct qedr_ucontext {
230 struct ib_ucontext ibucontext;
231 struct qedr_dev *dev;
232 struct qedr_pd *pd;
233 void __iomem *dpi_addr;
234 u64 dpi_phys_addr;
235 u32 dpi_size;
236 u16 dpi;
237
238 struct list_head mm_head;
239
240
241 struct mutex mm_list_lock;
242 };
243
244 union db_prod64 {
245 struct rdma_pwm_val32_data data;
246 u64 raw;
247 };
248
249 enum qedr_cq_type {
250 QEDR_CQ_TYPE_GSI,
251 QEDR_CQ_TYPE_KERNEL,
252 QEDR_CQ_TYPE_USER,
253 };
254
255 struct qedr_pbl_info {
256 u32 num_pbls;
257 u32 num_pbes;
258 u32 pbl_size;
259 u32 pbe_size;
260 bool two_layered;
261 };
262
263 struct qedr_userq {
264 struct ib_umem *umem;
265 struct qedr_pbl_info pbl_info;
266 struct qedr_pbl *pbl_tbl;
267 u64 buf_addr;
268 size_t buf_len;
269 };
270
271 struct qedr_cq {
272 struct ib_cq ibcq;
273
274 enum qedr_cq_type cq_type;
275 u32 sig;
276
277 u16 icid;
278
279
280 spinlock_t cq_lock;
281 u8 arm_flags;
282 struct qed_chain pbl;
283
284 void __iomem *db_addr;
285 union db_prod64 db;
286
287 u8 pbl_toggle;
288 union rdma_cqe *latest_cqe;
289 union rdma_cqe *toggle_cqe;
290
291 u32 cq_cons;
292
293 struct qedr_userq q;
294 u8 destroyed;
295 u16 cnq_notif;
296 };
297
298 struct qedr_pd {
299 struct ib_pd ibpd;
300 u32 pd_id;
301 struct qedr_ucontext *uctx;
302 };
303
304 struct qedr_mm {
305 struct {
306 u64 phy_addr;
307 unsigned long len;
308 } key;
309 struct list_head entry;
310 };
311
312 union db_prod32 {
313 struct rdma_pwm_val16_data data;
314 u32 raw;
315 };
316
317 struct qedr_qp_hwq_info {
318
319 struct qed_chain pbl;
320 u64 p_phys_addr_tbl;
321 u32 max_sges;
322
323
324 u16 prod;
325 u16 cons;
326 u16 wqe_cons;
327 u16 gsi_cons;
328 u16 max_wr;
329
330
331 void __iomem *db;
332 union db_prod32 db_data;
333
334 void __iomem *iwarp_db2;
335 union db_prod32 iwarp_db2_data;
336 };
337
338 #define QEDR_INC_SW_IDX(p_info, index) \
339 do { \
340 p_info->index = (p_info->index + 1) & \
341 qed_chain_get_capacity(p_info->pbl) \
342 } while (0)
343
344 struct qedr_srq_hwq_info {
345 u32 max_sges;
346 u32 max_wr;
347 struct qed_chain pbl;
348 u64 p_phys_addr_tbl;
349 u32 wqe_prod;
350 u32 sge_prod;
351 u32 wr_prod_cnt;
352 u32 wr_cons_cnt;
353 u32 num_elems;
354
355 u32 *virt_prod_pair_addr;
356 dma_addr_t phy_prod_pair_addr;
357 };
358
359 struct qedr_srq {
360 struct ib_srq ibsrq;
361 struct qedr_dev *dev;
362
363 struct qedr_userq usrq;
364 struct qedr_srq_hwq_info hw_srq;
365 struct ib_umem *prod_umem;
366 u16 srq_id;
367 u32 srq_limit;
368
369 spinlock_t lock;
370 };
371
372 enum qedr_qp_err_bitmap {
373 QEDR_QP_ERR_SQ_FULL = 1,
374 QEDR_QP_ERR_RQ_FULL = 2,
375 QEDR_QP_ERR_BAD_SR = 4,
376 QEDR_QP_ERR_BAD_RR = 8,
377 QEDR_QP_ERR_SQ_PBL_FULL = 16,
378 QEDR_QP_ERR_RQ_PBL_FULL = 32,
379 };
380
381 enum qedr_qp_create_type {
382 QEDR_QP_CREATE_NONE,
383 QEDR_QP_CREATE_USER,
384 QEDR_QP_CREATE_KERNEL,
385 };
386
387 enum qedr_iwarp_cm_flags {
388 QEDR_IWARP_CM_WAIT_FOR_CONNECT = BIT(0),
389 QEDR_IWARP_CM_WAIT_FOR_DISCONNECT = BIT(1),
390 };
391
392 struct qedr_qp {
393 struct ib_qp ibqp;
394 struct qedr_dev *dev;
395 struct qedr_qp_hwq_info sq;
396 struct qedr_qp_hwq_info rq;
397
398 u32 max_inline_data;
399
400
401 spinlock_t q_lock;
402 struct qedr_cq *sq_cq;
403 struct qedr_cq *rq_cq;
404 struct qedr_srq *srq;
405 enum qed_roce_qp_state state;
406 u32 id;
407 struct qedr_pd *pd;
408 enum ib_qp_type qp_type;
409 enum qedr_qp_create_type create_type;
410 struct qed_rdma_qp *qed_qp;
411 u32 qp_id;
412 u16 icid;
413 u16 mtu;
414 int sgid_idx;
415 u32 rq_psn;
416 u32 sq_psn;
417 u32 qkey;
418 u32 dest_qp_num;
419
420
421 u8 prev_wqe_size;
422 u16 wqe_cons;
423 u32 err_bitmap;
424 bool signaled;
425
426
427 struct {
428 u64 wr_id;
429 enum ib_wc_opcode opcode;
430 u32 bytes_len;
431 u8 wqe_size;
432 bool signaled;
433 dma_addr_t icrc_mapping;
434 u32 *icrc;
435 struct qedr_mr *mr;
436 } *wqe_wr_id;
437
438
439 struct {
440 u64 wr_id;
441 struct ib_sge sg_list[RDMA_MAX_SGE_PER_RQ_WQE];
442 u8 wqe_size;
443
444 u8 smac[ETH_ALEN];
445 u16 vlan;
446 int rc;
447 } *rqe_wr_id;
448
449
450 struct qedr_userq usq;
451 struct qedr_userq urq;
452
453
454 struct kref refcnt;
455 struct completion iwarp_cm_comp;
456 unsigned long iwarp_cm_flags;
457 };
458
459 struct qedr_ah {
460 struct ib_ah ibah;
461 struct rdma_ah_attr attr;
462 };
463
464 enum qedr_mr_type {
465 QEDR_MR_USER,
466 QEDR_MR_KERNEL,
467 QEDR_MR_DMA,
468 QEDR_MR_FRMR,
469 };
470
471 struct mr_info {
472 struct qedr_pbl *pbl_table;
473 struct qedr_pbl_info pbl_info;
474 struct list_head free_pbl_list;
475 struct list_head inuse_pbl_list;
476 u32 completed;
477 u32 completed_handled;
478 };
479
480 struct qedr_mr {
481 struct ib_mr ibmr;
482 struct ib_umem *umem;
483
484 struct qed_rdma_register_tid_in_params hw_mr;
485 enum qedr_mr_type type;
486
487 struct qedr_dev *dev;
488 struct mr_info info;
489
490 u64 *pages;
491 u32 npages;
492 };
493
494 #define SET_FIELD2(value, name, flag) ((value) |= ((flag) << (name ## _SHIFT)))
495
496 #define QEDR_RESP_IMM (RDMA_CQE_RESPONDER_IMM_FLG_MASK << \
497 RDMA_CQE_RESPONDER_IMM_FLG_SHIFT)
498 #define QEDR_RESP_RDMA (RDMA_CQE_RESPONDER_RDMA_FLG_MASK << \
499 RDMA_CQE_RESPONDER_RDMA_FLG_SHIFT)
500 #define QEDR_RESP_INV (RDMA_CQE_RESPONDER_INV_FLG_MASK << \
501 RDMA_CQE_RESPONDER_INV_FLG_SHIFT)
502
503 static inline void qedr_inc_sw_cons(struct qedr_qp_hwq_info *info)
504 {
505 info->cons = (info->cons + 1) % info->max_wr;
506 info->wqe_cons++;
507 }
508
509 static inline void qedr_inc_sw_prod(struct qedr_qp_hwq_info *info)
510 {
511 info->prod = (info->prod + 1) % info->max_wr;
512 }
513
514 static inline int qedr_get_dmac(struct qedr_dev *dev,
515 struct rdma_ah_attr *ah_attr, u8 *mac_addr)
516 {
517 union ib_gid zero_sgid = { { 0 } };
518 struct in6_addr in6;
519 const struct ib_global_route *grh = rdma_ah_read_grh(ah_attr);
520 u8 *dmac;
521
522 if (!memcmp(&grh->dgid, &zero_sgid, sizeof(union ib_gid))) {
523 DP_ERR(dev, "Local port GID not supported\n");
524 eth_zero_addr(mac_addr);
525 return -EINVAL;
526 }
527
528 memcpy(&in6, grh->dgid.raw, sizeof(in6));
529 dmac = rdma_ah_retrieve_dmac(ah_attr);
530 if (!dmac)
531 return -EINVAL;
532 ether_addr_copy(mac_addr, dmac);
533
534 return 0;
535 }
536
537 struct qedr_iw_listener {
538 struct qedr_dev *dev;
539 struct iw_cm_id *cm_id;
540 int backlog;
541 void *qed_handle;
542 };
543
544 struct qedr_iw_ep {
545 struct qedr_dev *dev;
546 struct iw_cm_id *cm_id;
547 struct qedr_qp *qp;
548 void *qed_context;
549 struct kref refcnt;
550 };
551
552 static inline
553 struct qedr_ucontext *get_qedr_ucontext(struct ib_ucontext *ibucontext)
554 {
555 return container_of(ibucontext, struct qedr_ucontext, ibucontext);
556 }
557
558 static inline struct qedr_dev *get_qedr_dev(struct ib_device *ibdev)
559 {
560 return container_of(ibdev, struct qedr_dev, ibdev);
561 }
562
563 static inline struct qedr_pd *get_qedr_pd(struct ib_pd *ibpd)
564 {
565 return container_of(ibpd, struct qedr_pd, ibpd);
566 }
567
568 static inline struct qedr_cq *get_qedr_cq(struct ib_cq *ibcq)
569 {
570 return container_of(ibcq, struct qedr_cq, ibcq);
571 }
572
573 static inline struct qedr_qp *get_qedr_qp(struct ib_qp *ibqp)
574 {
575 return container_of(ibqp, struct qedr_qp, ibqp);
576 }
577
578 static inline struct qedr_ah *get_qedr_ah(struct ib_ah *ibah)
579 {
580 return container_of(ibah, struct qedr_ah, ibah);
581 }
582
583 static inline struct qedr_mr *get_qedr_mr(struct ib_mr *ibmr)
584 {
585 return container_of(ibmr, struct qedr_mr, ibmr);
586 }
587
588 static inline struct qedr_srq *get_qedr_srq(struct ib_srq *ibsrq)
589 {
590 return container_of(ibsrq, struct qedr_srq, ibsrq);
591 }
592 #endif