This source file includes following definitions.
- qedr_ib_copy_to_udata
- qedr_query_pkey
- qedr_iw_query_gid
- qedr_query_srq
- qedr_query_device
- get_link_speed_and_width
- qedr_query_port
- qedr_modify_port
- qedr_add_mmap
- qedr_search_mmap
- qedr_alloc_ucontext
- qedr_dealloc_ucontext
- qedr_mmap
- qedr_alloc_pd
- qedr_dealloc_pd
- qedr_free_pbl
- qedr_alloc_pbl_tbl
- qedr_prepare_pbl_tbl
- qedr_populate_pbls
- qedr_copy_cq_uresp
- consume_cqe
- qedr_align_cq_entries
- qedr_init_user_queue
- qedr_init_cq_params
- doorbell_cq
- qedr_arm_cq
- qedr_create_cq
- qedr_resize_cq
- qedr_destroy_cq
- get_gid_info_from_table
- qedr_check_qp_attrs
- qedr_copy_srq_uresp
- qedr_copy_rq_uresp
- qedr_copy_sq_uresp
- qedr_copy_qp_uresp
- qedr_set_common_qp_params
- qedr_set_roce_db_info
- qedr_check_srq_params
- qedr_free_srq_user_params
- qedr_free_srq_kernel_params
- qedr_init_srq_user_params
- qedr_alloc_srq_kernel_params
- qedr_create_srq
- qedr_destroy_srq
- qedr_modify_srq
- qedr_init_common_qp_in_params
- qedr_qp_user_print
- qedr_iwarp_populate_user_qp
- qedr_cleanup_user
- qedr_create_user_qp
- qedr_set_iwarp_db_info
- qedr_roce_create_kernel_qp
- qedr_iwarp_create_kernel_qp
- qedr_cleanup_kernel
- qedr_create_kernel_qp
- qedr_create_qp
- qedr_get_ibqp_state
- qedr_get_state_from_ibqp
- qedr_reset_qp_hwq_info
- qedr_update_qp_state
- qedr_modify_qp
- qedr_to_ib_qp_acc_flags
- qedr_query_qp
- qedr_free_qp_resources
- qedr_destroy_qp
- qedr_create_ah
- qedr_destroy_ah
- free_mr_info
- init_mr_info
- qedr_reg_user_mr
- qedr_dereg_mr
- __qedr_alloc_mr
- qedr_alloc_mr
- qedr_set_page
- handle_completed_mrs
- qedr_map_mr_sg
- qedr_get_dma_mr
- qedr_wq_is_full
- sge_data_len
- swap_wqe_data64
- qedr_prepare_sq_inline_data
- qedr_prepare_sq_sges
- qedr_prepare_sq_rdma_data
- qedr_prepare_sq_send_data
- qedr_prepare_reg
- qedr_ib_to_wc_opcode
- qedr_can_post_send
- __qedr_post_send
- qedr_post_send
- qedr_srq_elem_left
- qedr_post_srq_recv
- qedr_post_recv
- is_valid_cqe
- cqe_get_qp
- cqe_get_type
- get_cqe
- qedr_chk_if_fmr
- process_req
- qedr_poll_cq_req
- qedr_cqe_resp_status_to_ib
- qedr_set_ok_cqe_resp_wc
- __process_resp_one
- process_resp_one_srq
- process_resp_one
- process_resp_flush
- try_consume_resp_cqe
- qedr_poll_cq_resp_srq
- qedr_poll_cq_resp
- try_consume_req_cqe
- qedr_poll_cq
- qedr_process_mad
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32 #include <linux/dma-mapping.h>
33 #include <linux/crc32.h>
34 #include <net/ip.h>
35 #include <net/ipv6.h>
36 #include <net/udp.h>
37 #include <linux/iommu.h>
38
39 #include <rdma/ib_verbs.h>
40 #include <rdma/ib_user_verbs.h>
41 #include <rdma/iw_cm.h>
42 #include <rdma/ib_umem.h>
43 #include <rdma/ib_addr.h>
44 #include <rdma/ib_cache.h>
45 #include <rdma/uverbs_ioctl.h>
46
47 #include <linux/qed/common_hsi.h>
48 #include "qedr_hsi_rdma.h"
49 #include <linux/qed/qed_if.h>
50 #include "qedr.h"
51 #include "verbs.h"
52 #include <rdma/qedr-abi.h>
53 #include "qedr_roce_cm.h"
54 #include "qedr_iw_cm.h"
55
56 #define QEDR_SRQ_WQE_ELEM_SIZE sizeof(union rdma_srq_elm)
57 #define RDMA_MAX_SGE_PER_SRQ (4)
58 #define RDMA_MAX_SRQ_WQE_SIZE (RDMA_MAX_SGE_PER_SRQ + 1)
59
60 #define DB_ADDR_SHIFT(addr) ((addr) << DB_PWM_ADDR_OFFSET_SHIFT)
61
62 static inline int qedr_ib_copy_to_udata(struct ib_udata *udata, void *src,
63 size_t len)
64 {
65 size_t min_len = min_t(size_t, len, udata->outlen);
66
67 return ib_copy_to_udata(udata, src, min_len);
68 }
69
70 int qedr_query_pkey(struct ib_device *ibdev, u8 port, u16 index, u16 *pkey)
71 {
72 if (index >= QEDR_ROCE_PKEY_TABLE_LEN)
73 return -EINVAL;
74
75 *pkey = QEDR_ROCE_PKEY_DEFAULT;
76 return 0;
77 }
78
79 int qedr_iw_query_gid(struct ib_device *ibdev, u8 port,
80 int index, union ib_gid *sgid)
81 {
82 struct qedr_dev *dev = get_qedr_dev(ibdev);
83
84 memset(sgid->raw, 0, sizeof(sgid->raw));
85 ether_addr_copy(sgid->raw, dev->ndev->dev_addr);
86
87 DP_DEBUG(dev, QEDR_MSG_INIT, "QUERY sgid[%d]=%llx:%llx\n", index,
88 sgid->global.interface_id, sgid->global.subnet_prefix);
89
90 return 0;
91 }
92
93 int qedr_query_srq(struct ib_srq *ibsrq, struct ib_srq_attr *srq_attr)
94 {
95 struct qedr_dev *dev = get_qedr_dev(ibsrq->device);
96 struct qedr_device_attr *qattr = &dev->attr;
97 struct qedr_srq *srq = get_qedr_srq(ibsrq);
98
99 srq_attr->srq_limit = srq->srq_limit;
100 srq_attr->max_wr = qattr->max_srq_wr;
101 srq_attr->max_sge = qattr->max_sge;
102
103 return 0;
104 }
105
106 int qedr_query_device(struct ib_device *ibdev,
107 struct ib_device_attr *attr, struct ib_udata *udata)
108 {
109 struct qedr_dev *dev = get_qedr_dev(ibdev);
110 struct qedr_device_attr *qattr = &dev->attr;
111
112 if (!dev->rdma_ctx) {
113 DP_ERR(dev,
114 "qedr_query_device called with invalid params rdma_ctx=%p\n",
115 dev->rdma_ctx);
116 return -EINVAL;
117 }
118
119 memset(attr, 0, sizeof(*attr));
120
121 attr->fw_ver = qattr->fw_ver;
122 attr->sys_image_guid = qattr->sys_image_guid;
123 attr->max_mr_size = qattr->max_mr_size;
124 attr->page_size_cap = qattr->page_size_caps;
125 attr->vendor_id = qattr->vendor_id;
126 attr->vendor_part_id = qattr->vendor_part_id;
127 attr->hw_ver = qattr->hw_ver;
128 attr->max_qp = qattr->max_qp;
129 attr->max_qp_wr = max_t(u32, qattr->max_sqe, qattr->max_rqe);
130 attr->device_cap_flags = IB_DEVICE_CURR_QP_STATE_MOD |
131 IB_DEVICE_RC_RNR_NAK_GEN |
132 IB_DEVICE_LOCAL_DMA_LKEY | IB_DEVICE_MEM_MGT_EXTENSIONS;
133
134 attr->max_send_sge = qattr->max_sge;
135 attr->max_recv_sge = qattr->max_sge;
136 attr->max_sge_rd = qattr->max_sge;
137 attr->max_cq = qattr->max_cq;
138 attr->max_cqe = qattr->max_cqe;
139 attr->max_mr = qattr->max_mr;
140 attr->max_mw = qattr->max_mw;
141 attr->max_pd = qattr->max_pd;
142 attr->atomic_cap = dev->atomic_cap;
143 attr->max_fmr = qattr->max_fmr;
144 attr->max_map_per_fmr = 16;
145 attr->max_qp_init_rd_atom =
146 1 << (fls(qattr->max_qp_req_rd_atomic_resc) - 1);
147 attr->max_qp_rd_atom =
148 min(1 << (fls(qattr->max_qp_resp_rd_atomic_resc) - 1),
149 attr->max_qp_init_rd_atom);
150
151 attr->max_srq = qattr->max_srq;
152 attr->max_srq_sge = qattr->max_srq_sge;
153 attr->max_srq_wr = qattr->max_srq_wr;
154
155 attr->local_ca_ack_delay = qattr->dev_ack_delay;
156 attr->max_fast_reg_page_list_len = qattr->max_mr / 8;
157 attr->max_pkeys = QEDR_ROCE_PKEY_MAX;
158 attr->max_ah = qattr->max_ah;
159
160 return 0;
161 }
162
163 static inline void get_link_speed_and_width(int speed, u8 *ib_speed,
164 u8 *ib_width)
165 {
166 switch (speed) {
167 case 1000:
168 *ib_speed = IB_SPEED_SDR;
169 *ib_width = IB_WIDTH_1X;
170 break;
171 case 10000:
172 *ib_speed = IB_SPEED_QDR;
173 *ib_width = IB_WIDTH_1X;
174 break;
175
176 case 20000:
177 *ib_speed = IB_SPEED_DDR;
178 *ib_width = IB_WIDTH_4X;
179 break;
180
181 case 25000:
182 *ib_speed = IB_SPEED_EDR;
183 *ib_width = IB_WIDTH_1X;
184 break;
185
186 case 40000:
187 *ib_speed = IB_SPEED_QDR;
188 *ib_width = IB_WIDTH_4X;
189 break;
190
191 case 50000:
192 *ib_speed = IB_SPEED_HDR;
193 *ib_width = IB_WIDTH_1X;
194 break;
195
196 case 100000:
197 *ib_speed = IB_SPEED_EDR;
198 *ib_width = IB_WIDTH_4X;
199 break;
200
201 default:
202
203 *ib_speed = IB_SPEED_SDR;
204 *ib_width = IB_WIDTH_1X;
205 }
206 }
207
208 int qedr_query_port(struct ib_device *ibdev, u8 port, struct ib_port_attr *attr)
209 {
210 struct qedr_dev *dev;
211 struct qed_rdma_port *rdma_port;
212
213 dev = get_qedr_dev(ibdev);
214
215 if (!dev->rdma_ctx) {
216 DP_ERR(dev, "rdma_ctx is NULL\n");
217 return -EINVAL;
218 }
219
220 rdma_port = dev->ops->rdma_query_port(dev->rdma_ctx);
221
222
223 if (rdma_port->port_state == QED_RDMA_PORT_UP) {
224 attr->state = IB_PORT_ACTIVE;
225 attr->phys_state = IB_PORT_PHYS_STATE_LINK_UP;
226 } else {
227 attr->state = IB_PORT_DOWN;
228 attr->phys_state = IB_PORT_PHYS_STATE_DISABLED;
229 }
230 attr->max_mtu = IB_MTU_4096;
231 attr->active_mtu = iboe_get_mtu(dev->ndev->mtu);
232 attr->lid = 0;
233 attr->lmc = 0;
234 attr->sm_lid = 0;
235 attr->sm_sl = 0;
236 attr->ip_gids = true;
237 if (rdma_protocol_iwarp(&dev->ibdev, 1)) {
238 attr->gid_tbl_len = 1;
239 attr->pkey_tbl_len = 1;
240 } else {
241 attr->gid_tbl_len = QEDR_MAX_SGID;
242 attr->pkey_tbl_len = QEDR_ROCE_PKEY_TABLE_LEN;
243 }
244 attr->bad_pkey_cntr = rdma_port->pkey_bad_counter;
245 attr->qkey_viol_cntr = 0;
246 get_link_speed_and_width(rdma_port->link_speed,
247 &attr->active_speed, &attr->active_width);
248 attr->max_msg_sz = rdma_port->max_msg_size;
249 attr->max_vl_num = 4;
250
251 return 0;
252 }
253
254 int qedr_modify_port(struct ib_device *ibdev, u8 port, int mask,
255 struct ib_port_modify *props)
256 {
257 return 0;
258 }
259
260 static int qedr_add_mmap(struct qedr_ucontext *uctx, u64 phy_addr,
261 unsigned long len)
262 {
263 struct qedr_mm *mm;
264
265 mm = kzalloc(sizeof(*mm), GFP_KERNEL);
266 if (!mm)
267 return -ENOMEM;
268
269 mm->key.phy_addr = phy_addr;
270
271
272
273
274
275
276
277 mm->key.len = roundup(len, PAGE_SIZE);
278 INIT_LIST_HEAD(&mm->entry);
279
280 mutex_lock(&uctx->mm_list_lock);
281 list_add(&mm->entry, &uctx->mm_head);
282 mutex_unlock(&uctx->mm_list_lock);
283
284 DP_DEBUG(uctx->dev, QEDR_MSG_MISC,
285 "added (addr=0x%llx,len=0x%lx) for ctx=%p\n",
286 (unsigned long long)mm->key.phy_addr,
287 (unsigned long)mm->key.len, uctx);
288
289 return 0;
290 }
291
292 static bool qedr_search_mmap(struct qedr_ucontext *uctx, u64 phy_addr,
293 unsigned long len)
294 {
295 bool found = false;
296 struct qedr_mm *mm;
297
298 mutex_lock(&uctx->mm_list_lock);
299 list_for_each_entry(mm, &uctx->mm_head, entry) {
300 if (len != mm->key.len || phy_addr != mm->key.phy_addr)
301 continue;
302
303 found = true;
304 break;
305 }
306 mutex_unlock(&uctx->mm_list_lock);
307 DP_DEBUG(uctx->dev, QEDR_MSG_MISC,
308 "searched for (addr=0x%llx,len=0x%lx) for ctx=%p, result=%d\n",
309 mm->key.phy_addr, mm->key.len, uctx, found);
310
311 return found;
312 }
313
314 int qedr_alloc_ucontext(struct ib_ucontext *uctx, struct ib_udata *udata)
315 {
316 struct ib_device *ibdev = uctx->device;
317 int rc;
318 struct qedr_ucontext *ctx = get_qedr_ucontext(uctx);
319 struct qedr_alloc_ucontext_resp uresp = {};
320 struct qedr_dev *dev = get_qedr_dev(ibdev);
321 struct qed_rdma_add_user_out_params oparams;
322
323 if (!udata)
324 return -EFAULT;
325
326 rc = dev->ops->rdma_add_user(dev->rdma_ctx, &oparams);
327 if (rc) {
328 DP_ERR(dev,
329 "failed to allocate a DPI for a new RoCE application, rc=%d. To overcome this consider to increase the number of DPIs, increase the doorbell BAR size or just close unnecessary RoCE applications. In order to increase the number of DPIs consult the qedr readme\n",
330 rc);
331 return rc;
332 }
333
334 ctx->dpi = oparams.dpi;
335 ctx->dpi_addr = oparams.dpi_addr;
336 ctx->dpi_phys_addr = oparams.dpi_phys_addr;
337 ctx->dpi_size = oparams.dpi_size;
338 INIT_LIST_HEAD(&ctx->mm_head);
339 mutex_init(&ctx->mm_list_lock);
340
341 uresp.dpm_enabled = dev->user_dpm_enabled;
342 uresp.wids_enabled = 1;
343 uresp.wid_count = oparams.wid_count;
344 uresp.db_pa = ctx->dpi_phys_addr;
345 uresp.db_size = ctx->dpi_size;
346 uresp.max_send_wr = dev->attr.max_sqe;
347 uresp.max_recv_wr = dev->attr.max_rqe;
348 uresp.max_srq_wr = dev->attr.max_srq_wr;
349 uresp.sges_per_send_wr = QEDR_MAX_SQE_ELEMENTS_PER_SQE;
350 uresp.sges_per_recv_wr = QEDR_MAX_RQE_ELEMENTS_PER_RQE;
351 uresp.sges_per_srq_wr = dev->attr.max_srq_sge;
352 uresp.max_cqes = QEDR_MAX_CQES;
353
354 rc = qedr_ib_copy_to_udata(udata, &uresp, sizeof(uresp));
355 if (rc)
356 return rc;
357
358 ctx->dev = dev;
359
360 rc = qedr_add_mmap(ctx, ctx->dpi_phys_addr, ctx->dpi_size);
361 if (rc)
362 return rc;
363
364 DP_DEBUG(dev, QEDR_MSG_INIT, "Allocating user context %p\n",
365 &ctx->ibucontext);
366 return 0;
367 }
368
369 void qedr_dealloc_ucontext(struct ib_ucontext *ibctx)
370 {
371 struct qedr_ucontext *uctx = get_qedr_ucontext(ibctx);
372 struct qedr_mm *mm, *tmp;
373
374 DP_DEBUG(uctx->dev, QEDR_MSG_INIT, "Deallocating user context %p\n",
375 uctx);
376 uctx->dev->ops->rdma_remove_user(uctx->dev->rdma_ctx, uctx->dpi);
377
378 list_for_each_entry_safe(mm, tmp, &uctx->mm_head, entry) {
379 DP_DEBUG(uctx->dev, QEDR_MSG_MISC,
380 "deleted (addr=0x%llx,len=0x%lx) for ctx=%p\n",
381 mm->key.phy_addr, mm->key.len, uctx);
382 list_del(&mm->entry);
383 kfree(mm);
384 }
385 }
386
387 int qedr_mmap(struct ib_ucontext *context, struct vm_area_struct *vma)
388 {
389 struct qedr_ucontext *ucontext = get_qedr_ucontext(context);
390 struct qedr_dev *dev = get_qedr_dev(context->device);
391 unsigned long phys_addr = vma->vm_pgoff << PAGE_SHIFT;
392 unsigned long len = (vma->vm_end - vma->vm_start);
393 unsigned long dpi_start;
394
395 dpi_start = dev->db_phys_addr + (ucontext->dpi * ucontext->dpi_size);
396
397 DP_DEBUG(dev, QEDR_MSG_INIT,
398 "mmap invoked with vm_start=0x%pK, vm_end=0x%pK,vm_pgoff=0x%pK; dpi_start=0x%pK dpi_size=0x%x\n",
399 (void *)vma->vm_start, (void *)vma->vm_end,
400 (void *)vma->vm_pgoff, (void *)dpi_start, ucontext->dpi_size);
401
402 if ((vma->vm_start & (PAGE_SIZE - 1)) || (len & (PAGE_SIZE - 1))) {
403 DP_ERR(dev,
404 "failed mmap, addresses must be page aligned: start=0x%pK, end=0x%pK\n",
405 (void *)vma->vm_start, (void *)vma->vm_end);
406 return -EINVAL;
407 }
408
409 if (!qedr_search_mmap(ucontext, phys_addr, len)) {
410 DP_ERR(dev, "failed mmap, vm_pgoff=0x%lx is not authorized\n",
411 vma->vm_pgoff);
412 return -EINVAL;
413 }
414
415 if (phys_addr < dpi_start ||
416 ((phys_addr + len) > (dpi_start + ucontext->dpi_size))) {
417 DP_ERR(dev,
418 "failed mmap, pages are outside of dpi; page address=0x%pK, dpi_start=0x%pK, dpi_size=0x%x\n",
419 (void *)phys_addr, (void *)dpi_start,
420 ucontext->dpi_size);
421 return -EINVAL;
422 }
423
424 if (vma->vm_flags & VM_READ) {
425 DP_ERR(dev, "failed mmap, cannot map doorbell bar for read\n");
426 return -EINVAL;
427 }
428
429 vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot);
430 return io_remap_pfn_range(vma, vma->vm_start, vma->vm_pgoff, len,
431 vma->vm_page_prot);
432 }
433
434 int qedr_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)
435 {
436 struct ib_device *ibdev = ibpd->device;
437 struct qedr_dev *dev = get_qedr_dev(ibdev);
438 struct qedr_pd *pd = get_qedr_pd(ibpd);
439 u16 pd_id;
440 int rc;
441
442 DP_DEBUG(dev, QEDR_MSG_INIT, "Function called from: %s\n",
443 udata ? "User Lib" : "Kernel");
444
445 if (!dev->rdma_ctx) {
446 DP_ERR(dev, "invalid RDMA context\n");
447 return -EINVAL;
448 }
449
450 rc = dev->ops->rdma_alloc_pd(dev->rdma_ctx, &pd_id);
451 if (rc)
452 return rc;
453
454 pd->pd_id = pd_id;
455
456 if (udata) {
457 struct qedr_alloc_pd_uresp uresp = {
458 .pd_id = pd_id,
459 };
460 struct qedr_ucontext *context = rdma_udata_to_drv_context(
461 udata, struct qedr_ucontext, ibucontext);
462
463 rc = qedr_ib_copy_to_udata(udata, &uresp, sizeof(uresp));
464 if (rc) {
465 DP_ERR(dev, "copy error pd_id=0x%x.\n", pd_id);
466 dev->ops->rdma_dealloc_pd(dev->rdma_ctx, pd_id);
467 return rc;
468 }
469
470 pd->uctx = context;
471 pd->uctx->pd = pd;
472 }
473
474 return 0;
475 }
476
477 void qedr_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)
478 {
479 struct qedr_dev *dev = get_qedr_dev(ibpd->device);
480 struct qedr_pd *pd = get_qedr_pd(ibpd);
481
482 DP_DEBUG(dev, QEDR_MSG_INIT, "Deallocating PD %d\n", pd->pd_id);
483 dev->ops->rdma_dealloc_pd(dev->rdma_ctx, pd->pd_id);
484 }
485
486 static void qedr_free_pbl(struct qedr_dev *dev,
487 struct qedr_pbl_info *pbl_info, struct qedr_pbl *pbl)
488 {
489 struct pci_dev *pdev = dev->pdev;
490 int i;
491
492 for (i = 0; i < pbl_info->num_pbls; i++) {
493 if (!pbl[i].va)
494 continue;
495 dma_free_coherent(&pdev->dev, pbl_info->pbl_size,
496 pbl[i].va, pbl[i].pa);
497 }
498
499 kfree(pbl);
500 }
501
502 #define MIN_FW_PBL_PAGE_SIZE (4 * 1024)
503 #define MAX_FW_PBL_PAGE_SIZE (64 * 1024)
504
505 #define NUM_PBES_ON_PAGE(_page_size) (_page_size / sizeof(u64))
506 #define MAX_PBES_ON_PAGE NUM_PBES_ON_PAGE(MAX_FW_PBL_PAGE_SIZE)
507 #define MAX_PBES_TWO_LAYER (MAX_PBES_ON_PAGE * MAX_PBES_ON_PAGE)
508
509 static struct qedr_pbl *qedr_alloc_pbl_tbl(struct qedr_dev *dev,
510 struct qedr_pbl_info *pbl_info,
511 gfp_t flags)
512 {
513 struct pci_dev *pdev = dev->pdev;
514 struct qedr_pbl *pbl_table;
515 dma_addr_t *pbl_main_tbl;
516 dma_addr_t pa;
517 void *va;
518 int i;
519
520 pbl_table = kcalloc(pbl_info->num_pbls, sizeof(*pbl_table), flags);
521 if (!pbl_table)
522 return ERR_PTR(-ENOMEM);
523
524 for (i = 0; i < pbl_info->num_pbls; i++) {
525 va = dma_alloc_coherent(&pdev->dev, pbl_info->pbl_size, &pa,
526 flags);
527 if (!va)
528 goto err;
529
530 pbl_table[i].va = va;
531 pbl_table[i].pa = pa;
532 }
533
534
535
536
537 pbl_main_tbl = (dma_addr_t *)pbl_table[0].va;
538 for (i = 0; i < pbl_info->num_pbls - 1; i++)
539 pbl_main_tbl[i] = pbl_table[i + 1].pa;
540
541 return pbl_table;
542
543 err:
544 for (i--; i >= 0; i--)
545 dma_free_coherent(&pdev->dev, pbl_info->pbl_size,
546 pbl_table[i].va, pbl_table[i].pa);
547
548 qedr_free_pbl(dev, pbl_info, pbl_table);
549
550 return ERR_PTR(-ENOMEM);
551 }
552
553 static int qedr_prepare_pbl_tbl(struct qedr_dev *dev,
554 struct qedr_pbl_info *pbl_info,
555 u32 num_pbes, int two_layer_capable)
556 {
557 u32 pbl_capacity;
558 u32 pbl_size;
559 u32 num_pbls;
560
561 if ((num_pbes > MAX_PBES_ON_PAGE) && two_layer_capable) {
562 if (num_pbes > MAX_PBES_TWO_LAYER) {
563 DP_ERR(dev, "prepare pbl table: too many pages %d\n",
564 num_pbes);
565 return -EINVAL;
566 }
567
568
569 pbl_size = MIN_FW_PBL_PAGE_SIZE;
570 pbl_capacity = NUM_PBES_ON_PAGE(pbl_size) *
571 NUM_PBES_ON_PAGE(pbl_size);
572
573 while (pbl_capacity < num_pbes) {
574 pbl_size *= 2;
575 pbl_capacity = pbl_size / sizeof(u64);
576 pbl_capacity = pbl_capacity * pbl_capacity;
577 }
578
579 num_pbls = DIV_ROUND_UP(num_pbes, NUM_PBES_ON_PAGE(pbl_size));
580 num_pbls++;
581 pbl_info->two_layered = true;
582 } else {
583
584 num_pbls = 1;
585 pbl_size = max_t(u32, MIN_FW_PBL_PAGE_SIZE,
586 roundup_pow_of_two((num_pbes * sizeof(u64))));
587 pbl_info->two_layered = false;
588 }
589
590 pbl_info->num_pbls = num_pbls;
591 pbl_info->pbl_size = pbl_size;
592 pbl_info->num_pbes = num_pbes;
593
594 DP_DEBUG(dev, QEDR_MSG_MR,
595 "prepare pbl table: num_pbes=%d, num_pbls=%d, pbl_size=%d\n",
596 pbl_info->num_pbes, pbl_info->num_pbls, pbl_info->pbl_size);
597
598 return 0;
599 }
600
601 static void qedr_populate_pbls(struct qedr_dev *dev, struct ib_umem *umem,
602 struct qedr_pbl *pbl,
603 struct qedr_pbl_info *pbl_info, u32 pg_shift)
604 {
605 int pbe_cnt, total_num_pbes = 0;
606 u32 fw_pg_cnt, fw_pg_per_umem_pg;
607 struct qedr_pbl *pbl_tbl;
608 struct sg_dma_page_iter sg_iter;
609 struct regpair *pbe;
610 u64 pg_addr;
611
612 if (!pbl_info->num_pbes)
613 return;
614
615
616
617
618 if (pbl_info->two_layered)
619 pbl_tbl = &pbl[1];
620 else
621 pbl_tbl = pbl;
622
623 pbe = (struct regpair *)pbl_tbl->va;
624 if (!pbe) {
625 DP_ERR(dev, "cannot populate PBL due to a NULL PBE\n");
626 return;
627 }
628
629 pbe_cnt = 0;
630
631 fw_pg_per_umem_pg = BIT(PAGE_SHIFT - pg_shift);
632
633 for_each_sg_dma_page (umem->sg_head.sgl, &sg_iter, umem->nmap, 0) {
634 pg_addr = sg_page_iter_dma_address(&sg_iter);
635 for (fw_pg_cnt = 0; fw_pg_cnt < fw_pg_per_umem_pg;) {
636 pbe->lo = cpu_to_le32(pg_addr);
637 pbe->hi = cpu_to_le32(upper_32_bits(pg_addr));
638
639 pg_addr += BIT(pg_shift);
640 pbe_cnt++;
641 total_num_pbes++;
642 pbe++;
643
644 if (total_num_pbes == pbl_info->num_pbes)
645 return;
646
647
648
649
650 if (pbe_cnt == (pbl_info->pbl_size / sizeof(u64))) {
651 pbl_tbl++;
652 pbe = (struct regpair *)pbl_tbl->va;
653 pbe_cnt = 0;
654 }
655
656 fw_pg_cnt++;
657 }
658 }
659 }
660
661 static int qedr_copy_cq_uresp(struct qedr_dev *dev,
662 struct qedr_cq *cq, struct ib_udata *udata)
663 {
664 struct qedr_create_cq_uresp uresp;
665 int rc;
666
667 memset(&uresp, 0, sizeof(uresp));
668
669 uresp.db_offset = DB_ADDR_SHIFT(DQ_PWM_OFFSET_UCM_RDMA_CQ_CONS_32BIT);
670 uresp.icid = cq->icid;
671
672 rc = qedr_ib_copy_to_udata(udata, &uresp, sizeof(uresp));
673 if (rc)
674 DP_ERR(dev, "copy error cqid=0x%x.\n", cq->icid);
675
676 return rc;
677 }
678
679 static void consume_cqe(struct qedr_cq *cq)
680 {
681 if (cq->latest_cqe == cq->toggle_cqe)
682 cq->pbl_toggle ^= RDMA_CQE_REQUESTER_TOGGLE_BIT_MASK;
683
684 cq->latest_cqe = qed_chain_consume(&cq->pbl);
685 }
686
687 static inline int qedr_align_cq_entries(int entries)
688 {
689 u64 size, aligned_size;
690
691
692 size = (entries + 1) * QEDR_CQE_SIZE;
693 aligned_size = ALIGN(size, PAGE_SIZE);
694
695 return aligned_size / QEDR_CQE_SIZE;
696 }
697
698 static inline int qedr_init_user_queue(struct ib_udata *udata,
699 struct qedr_dev *dev,
700 struct qedr_userq *q, u64 buf_addr,
701 size_t buf_len, int access, int dmasync,
702 int alloc_and_init)
703 {
704 u32 fw_pages;
705 int rc;
706
707 q->buf_addr = buf_addr;
708 q->buf_len = buf_len;
709 q->umem = ib_umem_get(udata, q->buf_addr, q->buf_len, access, dmasync);
710 if (IS_ERR(q->umem)) {
711 DP_ERR(dev, "create user queue: failed ib_umem_get, got %ld\n",
712 PTR_ERR(q->umem));
713 return PTR_ERR(q->umem);
714 }
715
716 fw_pages = ib_umem_page_count(q->umem) <<
717 (PAGE_SHIFT - FW_PAGE_SHIFT);
718
719 rc = qedr_prepare_pbl_tbl(dev, &q->pbl_info, fw_pages, 0);
720 if (rc)
721 goto err0;
722
723 if (alloc_and_init) {
724 q->pbl_tbl = qedr_alloc_pbl_tbl(dev, &q->pbl_info, GFP_KERNEL);
725 if (IS_ERR(q->pbl_tbl)) {
726 rc = PTR_ERR(q->pbl_tbl);
727 goto err0;
728 }
729 qedr_populate_pbls(dev, q->umem, q->pbl_tbl, &q->pbl_info,
730 FW_PAGE_SHIFT);
731 } else {
732 q->pbl_tbl = kzalloc(sizeof(*q->pbl_tbl), GFP_KERNEL);
733 if (!q->pbl_tbl) {
734 rc = -ENOMEM;
735 goto err0;
736 }
737 }
738
739 return 0;
740
741 err0:
742 ib_umem_release(q->umem);
743 q->umem = NULL;
744
745 return rc;
746 }
747
748 static inline void qedr_init_cq_params(struct qedr_cq *cq,
749 struct qedr_ucontext *ctx,
750 struct qedr_dev *dev, int vector,
751 int chain_entries, int page_cnt,
752 u64 pbl_ptr,
753 struct qed_rdma_create_cq_in_params
754 *params)
755 {
756 memset(params, 0, sizeof(*params));
757 params->cq_handle_hi = upper_32_bits((uintptr_t)cq);
758 params->cq_handle_lo = lower_32_bits((uintptr_t)cq);
759 params->cnq_id = vector;
760 params->cq_size = chain_entries - 1;
761 params->dpi = (ctx) ? ctx->dpi : dev->dpi;
762 params->pbl_num_pages = page_cnt;
763 params->pbl_ptr = pbl_ptr;
764 params->pbl_two_level = 0;
765 }
766
767 static void doorbell_cq(struct qedr_cq *cq, u32 cons, u8 flags)
768 {
769 cq->db.data.agg_flags = flags;
770 cq->db.data.value = cpu_to_le32(cons);
771 writeq(cq->db.raw, cq->db_addr);
772 }
773
774 int qedr_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags)
775 {
776 struct qedr_cq *cq = get_qedr_cq(ibcq);
777 unsigned long sflags;
778 struct qedr_dev *dev;
779
780 dev = get_qedr_dev(ibcq->device);
781
782 if (cq->destroyed) {
783 DP_ERR(dev,
784 "warning: arm was invoked after destroy for cq %p (icid=%d)\n",
785 cq, cq->icid);
786 return -EINVAL;
787 }
788
789
790 if (cq->cq_type == QEDR_CQ_TYPE_GSI)
791 return 0;
792
793 spin_lock_irqsave(&cq->cq_lock, sflags);
794
795 cq->arm_flags = 0;
796
797 if (flags & IB_CQ_SOLICITED)
798 cq->arm_flags |= DQ_UCM_ROCE_CQ_ARM_SE_CF_CMD;
799
800 if (flags & IB_CQ_NEXT_COMP)
801 cq->arm_flags |= DQ_UCM_ROCE_CQ_ARM_CF_CMD;
802
803 doorbell_cq(cq, cq->cq_cons - 1, cq->arm_flags);
804
805 spin_unlock_irqrestore(&cq->cq_lock, sflags);
806
807 return 0;
808 }
809
810 int qedr_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
811 struct ib_udata *udata)
812 {
813 struct ib_device *ibdev = ibcq->device;
814 struct qedr_ucontext *ctx = rdma_udata_to_drv_context(
815 udata, struct qedr_ucontext, ibucontext);
816 struct qed_rdma_destroy_cq_out_params destroy_oparams;
817 struct qed_rdma_destroy_cq_in_params destroy_iparams;
818 struct qedr_dev *dev = get_qedr_dev(ibdev);
819 struct qed_rdma_create_cq_in_params params;
820 struct qedr_create_cq_ureq ureq = {};
821 int vector = attr->comp_vector;
822 int entries = attr->cqe;
823 struct qedr_cq *cq = get_qedr_cq(ibcq);
824 int chain_entries;
825 int page_cnt;
826 u64 pbl_ptr;
827 u16 icid;
828 int rc;
829
830 DP_DEBUG(dev, QEDR_MSG_INIT,
831 "create_cq: called from %s. entries=%d, vector=%d\n",
832 udata ? "User Lib" : "Kernel", entries, vector);
833
834 if (entries > QEDR_MAX_CQES) {
835 DP_ERR(dev,
836 "create cq: the number of entries %d is too high. Must be equal or below %d.\n",
837 entries, QEDR_MAX_CQES);
838 return -EINVAL;
839 }
840
841 chain_entries = qedr_align_cq_entries(entries);
842 chain_entries = min_t(int, chain_entries, QEDR_MAX_CQES);
843
844 if (udata) {
845 if (ib_copy_from_udata(&ureq, udata, sizeof(ureq))) {
846 DP_ERR(dev,
847 "create cq: problem copying data from user space\n");
848 goto err0;
849 }
850
851 if (!ureq.len) {
852 DP_ERR(dev,
853 "create cq: cannot create a cq with 0 entries\n");
854 goto err0;
855 }
856
857 cq->cq_type = QEDR_CQ_TYPE_USER;
858
859 rc = qedr_init_user_queue(udata, dev, &cq->q, ureq.addr,
860 ureq.len, IB_ACCESS_LOCAL_WRITE, 1,
861 1);
862 if (rc)
863 goto err0;
864
865 pbl_ptr = cq->q.pbl_tbl->pa;
866 page_cnt = cq->q.pbl_info.num_pbes;
867
868 cq->ibcq.cqe = chain_entries;
869 } else {
870 cq->cq_type = QEDR_CQ_TYPE_KERNEL;
871
872 rc = dev->ops->common->chain_alloc(dev->cdev,
873 QED_CHAIN_USE_TO_CONSUME,
874 QED_CHAIN_MODE_PBL,
875 QED_CHAIN_CNT_TYPE_U32,
876 chain_entries,
877 sizeof(union rdma_cqe),
878 &cq->pbl, NULL);
879 if (rc)
880 goto err1;
881
882 page_cnt = qed_chain_get_page_cnt(&cq->pbl);
883 pbl_ptr = qed_chain_get_pbl_phys(&cq->pbl);
884 cq->ibcq.cqe = cq->pbl.capacity;
885 }
886
887 qedr_init_cq_params(cq, ctx, dev, vector, chain_entries, page_cnt,
888 pbl_ptr, ¶ms);
889
890 rc = dev->ops->rdma_create_cq(dev->rdma_ctx, ¶ms, &icid);
891 if (rc)
892 goto err2;
893
894 cq->icid = icid;
895 cq->sig = QEDR_CQ_MAGIC_NUMBER;
896 spin_lock_init(&cq->cq_lock);
897
898 if (udata) {
899 rc = qedr_copy_cq_uresp(dev, cq, udata);
900 if (rc)
901 goto err3;
902 } else {
903
904 cq->db_addr = dev->db_addr +
905 DB_ADDR_SHIFT(DQ_PWM_OFFSET_UCM_RDMA_CQ_CONS_32BIT);
906 cq->db.data.icid = cq->icid;
907 cq->db.data.params = DB_AGG_CMD_SET <<
908 RDMA_PWM_VAL32_DATA_AGG_CMD_SHIFT;
909
910
911 cq->toggle_cqe = qed_chain_get_last_elem(&cq->pbl);
912 cq->pbl_toggle = RDMA_CQE_REQUESTER_TOGGLE_BIT_MASK;
913 cq->latest_cqe = NULL;
914 consume_cqe(cq);
915 cq->cq_cons = qed_chain_get_cons_idx_u32(&cq->pbl);
916 }
917
918 DP_DEBUG(dev, QEDR_MSG_CQ,
919 "create cq: icid=0x%0x, addr=%p, size(entries)=0x%0x\n",
920 cq->icid, cq, params.cq_size);
921
922 return 0;
923
924 err3:
925 destroy_iparams.icid = cq->icid;
926 dev->ops->rdma_destroy_cq(dev->rdma_ctx, &destroy_iparams,
927 &destroy_oparams);
928 err2:
929 if (udata)
930 qedr_free_pbl(dev, &cq->q.pbl_info, cq->q.pbl_tbl);
931 else
932 dev->ops->common->chain_free(dev->cdev, &cq->pbl);
933 err1:
934 if (udata)
935 ib_umem_release(cq->q.umem);
936 err0:
937 return -EINVAL;
938 }
939
940 int qedr_resize_cq(struct ib_cq *ibcq, int new_cnt, struct ib_udata *udata)
941 {
942 struct qedr_dev *dev = get_qedr_dev(ibcq->device);
943 struct qedr_cq *cq = get_qedr_cq(ibcq);
944
945 DP_ERR(dev, "cq %p RESIZE NOT SUPPORTED\n", cq);
946
947 return 0;
948 }
949
950 #define QEDR_DESTROY_CQ_MAX_ITERATIONS (10)
951 #define QEDR_DESTROY_CQ_ITER_DURATION (10)
952
953 void qedr_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata)
954 {
955 struct qedr_dev *dev = get_qedr_dev(ibcq->device);
956 struct qed_rdma_destroy_cq_out_params oparams;
957 struct qed_rdma_destroy_cq_in_params iparams;
958 struct qedr_cq *cq = get_qedr_cq(ibcq);
959 int iter;
960
961 DP_DEBUG(dev, QEDR_MSG_CQ, "destroy cq %p (icid=%d)\n", cq, cq->icid);
962
963 cq->destroyed = 1;
964
965
966 if (cq->cq_type == QEDR_CQ_TYPE_GSI)
967 return;
968
969 iparams.icid = cq->icid;
970 dev->ops->rdma_destroy_cq(dev->rdma_ctx, &iparams, &oparams);
971 dev->ops->common->chain_free(dev->cdev, &cq->pbl);
972
973 if (udata) {
974 qedr_free_pbl(dev, &cq->q.pbl_info, cq->q.pbl_tbl);
975 ib_umem_release(cq->q.umem);
976 }
977
978
979
980
981
982
983
984
985
986
987
988
989 iter = QEDR_DESTROY_CQ_MAX_ITERATIONS;
990 while (oparams.num_cq_notif != READ_ONCE(cq->cnq_notif) && iter) {
991 udelay(QEDR_DESTROY_CQ_ITER_DURATION);
992 iter--;
993 }
994
995 iter = QEDR_DESTROY_CQ_MAX_ITERATIONS;
996 while (oparams.num_cq_notif != READ_ONCE(cq->cnq_notif) && iter) {
997 msleep(QEDR_DESTROY_CQ_ITER_DURATION);
998 iter--;
999 }
1000
1001
1002
1003
1004
1005
1006 }
1007
1008 static inline int get_gid_info_from_table(struct ib_qp *ibqp,
1009 struct ib_qp_attr *attr,
1010 int attr_mask,
1011 struct qed_rdma_modify_qp_in_params
1012 *qp_params)
1013 {
1014 const struct ib_gid_attr *gid_attr;
1015 enum rdma_network_type nw_type;
1016 const struct ib_global_route *grh = rdma_ah_read_grh(&attr->ah_attr);
1017 u32 ipv4_addr;
1018 int ret;
1019 int i;
1020
1021 gid_attr = grh->sgid_attr;
1022 ret = rdma_read_gid_l2_fields(gid_attr, &qp_params->vlan_id, NULL);
1023 if (ret)
1024 return ret;
1025
1026 nw_type = rdma_gid_attr_network_type(gid_attr);
1027 switch (nw_type) {
1028 case RDMA_NETWORK_IPV6:
1029 memcpy(&qp_params->sgid.bytes[0], &gid_attr->gid.raw[0],
1030 sizeof(qp_params->sgid));
1031 memcpy(&qp_params->dgid.bytes[0],
1032 &grh->dgid,
1033 sizeof(qp_params->dgid));
1034 qp_params->roce_mode = ROCE_V2_IPV6;
1035 SET_FIELD(qp_params->modify_flags,
1036 QED_ROCE_MODIFY_QP_VALID_ROCE_MODE, 1);
1037 break;
1038 case RDMA_NETWORK_IB:
1039 memcpy(&qp_params->sgid.bytes[0], &gid_attr->gid.raw[0],
1040 sizeof(qp_params->sgid));
1041 memcpy(&qp_params->dgid.bytes[0],
1042 &grh->dgid,
1043 sizeof(qp_params->dgid));
1044 qp_params->roce_mode = ROCE_V1;
1045 break;
1046 case RDMA_NETWORK_IPV4:
1047 memset(&qp_params->sgid, 0, sizeof(qp_params->sgid));
1048 memset(&qp_params->dgid, 0, sizeof(qp_params->dgid));
1049 ipv4_addr = qedr_get_ipv4_from_gid(gid_attr->gid.raw);
1050 qp_params->sgid.ipv4_addr = ipv4_addr;
1051 ipv4_addr =
1052 qedr_get_ipv4_from_gid(grh->dgid.raw);
1053 qp_params->dgid.ipv4_addr = ipv4_addr;
1054 SET_FIELD(qp_params->modify_flags,
1055 QED_ROCE_MODIFY_QP_VALID_ROCE_MODE, 1);
1056 qp_params->roce_mode = ROCE_V2_IPV4;
1057 break;
1058 }
1059
1060 for (i = 0; i < 4; i++) {
1061 qp_params->sgid.dwords[i] = ntohl(qp_params->sgid.dwords[i]);
1062 qp_params->dgid.dwords[i] = ntohl(qp_params->dgid.dwords[i]);
1063 }
1064
1065 if (qp_params->vlan_id >= VLAN_CFI_MASK)
1066 qp_params->vlan_id = 0;
1067
1068 return 0;
1069 }
1070
1071 static int qedr_check_qp_attrs(struct ib_pd *ibpd, struct qedr_dev *dev,
1072 struct ib_qp_init_attr *attrs,
1073 struct ib_udata *udata)
1074 {
1075 struct qedr_device_attr *qattr = &dev->attr;
1076
1077
1078 if (attrs->qp_type != IB_QPT_RC && attrs->qp_type != IB_QPT_GSI) {
1079 DP_DEBUG(dev, QEDR_MSG_QP,
1080 "create qp: unsupported qp type=0x%x requested\n",
1081 attrs->qp_type);
1082 return -EINVAL;
1083 }
1084
1085 if (attrs->cap.max_send_wr > qattr->max_sqe) {
1086 DP_ERR(dev,
1087 "create qp: cannot create a SQ with %d elements (max_send_wr=0x%x)\n",
1088 attrs->cap.max_send_wr, qattr->max_sqe);
1089 return -EINVAL;
1090 }
1091
1092 if (attrs->cap.max_inline_data > qattr->max_inline) {
1093 DP_ERR(dev,
1094 "create qp: unsupported inline data size=0x%x requested (max_inline=0x%x)\n",
1095 attrs->cap.max_inline_data, qattr->max_inline);
1096 return -EINVAL;
1097 }
1098
1099 if (attrs->cap.max_send_sge > qattr->max_sge) {
1100 DP_ERR(dev,
1101 "create qp: unsupported send_sge=0x%x requested (max_send_sge=0x%x)\n",
1102 attrs->cap.max_send_sge, qattr->max_sge);
1103 return -EINVAL;
1104 }
1105
1106 if (attrs->cap.max_recv_sge > qattr->max_sge) {
1107 DP_ERR(dev,
1108 "create qp: unsupported recv_sge=0x%x requested (max_recv_sge=0x%x)\n",
1109 attrs->cap.max_recv_sge, qattr->max_sge);
1110 return -EINVAL;
1111 }
1112
1113
1114 if (udata && attrs->qp_type == IB_QPT_GSI) {
1115 DP_ERR(dev,
1116 "create qp: userspace can't create special QPs of type=0x%x\n",
1117 attrs->qp_type);
1118 return -EINVAL;
1119 }
1120
1121 return 0;
1122 }
1123
1124 static int qedr_copy_srq_uresp(struct qedr_dev *dev,
1125 struct qedr_srq *srq, struct ib_udata *udata)
1126 {
1127 struct qedr_create_srq_uresp uresp = {};
1128 int rc;
1129
1130 uresp.srq_id = srq->srq_id;
1131
1132 rc = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
1133 if (rc)
1134 DP_ERR(dev, "create srq: problem copying data to user space\n");
1135
1136 return rc;
1137 }
1138
1139 static void qedr_copy_rq_uresp(struct qedr_dev *dev,
1140 struct qedr_create_qp_uresp *uresp,
1141 struct qedr_qp *qp)
1142 {
1143
1144 if (rdma_protocol_iwarp(&dev->ibdev, 1)) {
1145 uresp->rq_db_offset =
1146 DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_IWARP_RQ_PROD);
1147 uresp->rq_db2_offset = DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_FLAGS);
1148 } else {
1149 uresp->rq_db_offset =
1150 DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_ROCE_RQ_PROD);
1151 }
1152
1153 uresp->rq_icid = qp->icid;
1154 }
1155
1156 static void qedr_copy_sq_uresp(struct qedr_dev *dev,
1157 struct qedr_create_qp_uresp *uresp,
1158 struct qedr_qp *qp)
1159 {
1160 uresp->sq_db_offset = DB_ADDR_SHIFT(DQ_PWM_OFFSET_XCM_RDMA_SQ_PROD);
1161
1162
1163 if (rdma_protocol_iwarp(&dev->ibdev, 1))
1164 uresp->sq_icid = qp->icid;
1165 else
1166 uresp->sq_icid = qp->icid + 1;
1167 }
1168
1169 static int qedr_copy_qp_uresp(struct qedr_dev *dev,
1170 struct qedr_qp *qp, struct ib_udata *udata)
1171 {
1172 struct qedr_create_qp_uresp uresp;
1173 int rc;
1174
1175 memset(&uresp, 0, sizeof(uresp));
1176 qedr_copy_sq_uresp(dev, &uresp, qp);
1177 qedr_copy_rq_uresp(dev, &uresp, qp);
1178
1179 uresp.atomic_supported = dev->atomic_cap != IB_ATOMIC_NONE;
1180 uresp.qp_id = qp->qp_id;
1181
1182 rc = qedr_ib_copy_to_udata(udata, &uresp, sizeof(uresp));
1183 if (rc)
1184 DP_ERR(dev,
1185 "create qp: failed a copy to user space with qp icid=0x%x.\n",
1186 qp->icid);
1187
1188 return rc;
1189 }
1190
1191 static void qedr_set_common_qp_params(struct qedr_dev *dev,
1192 struct qedr_qp *qp,
1193 struct qedr_pd *pd,
1194 struct ib_qp_init_attr *attrs)
1195 {
1196 spin_lock_init(&qp->q_lock);
1197 if (rdma_protocol_iwarp(&dev->ibdev, 1)) {
1198 kref_init(&qp->refcnt);
1199 init_completion(&qp->iwarp_cm_comp);
1200 }
1201 qp->pd = pd;
1202 qp->qp_type = attrs->qp_type;
1203 qp->max_inline_data = attrs->cap.max_inline_data;
1204 qp->sq.max_sges = attrs->cap.max_send_sge;
1205 qp->state = QED_ROCE_QP_STATE_RESET;
1206 qp->signaled = (attrs->sq_sig_type == IB_SIGNAL_ALL_WR) ? true : false;
1207 qp->sq_cq = get_qedr_cq(attrs->send_cq);
1208 qp->dev = dev;
1209
1210 if (attrs->srq) {
1211 qp->srq = get_qedr_srq(attrs->srq);
1212 } else {
1213 qp->rq_cq = get_qedr_cq(attrs->recv_cq);
1214 qp->rq.max_sges = attrs->cap.max_recv_sge;
1215 DP_DEBUG(dev, QEDR_MSG_QP,
1216 "RQ params:\trq_max_sges = %d, rq_cq_id = %d\n",
1217 qp->rq.max_sges, qp->rq_cq->icid);
1218 }
1219
1220 DP_DEBUG(dev, QEDR_MSG_QP,
1221 "QP params:\tpd = %d, qp_type = %d, max_inline_data = %d, state = %d, signaled = %d, use_srq=%d\n",
1222 pd->pd_id, qp->qp_type, qp->max_inline_data,
1223 qp->state, qp->signaled, (attrs->srq) ? 1 : 0);
1224 DP_DEBUG(dev, QEDR_MSG_QP,
1225 "SQ params:\tsq_max_sges = %d, sq_cq_id = %d\n",
1226 qp->sq.max_sges, qp->sq_cq->icid);
1227 }
1228
1229 static void qedr_set_roce_db_info(struct qedr_dev *dev, struct qedr_qp *qp)
1230 {
1231 qp->sq.db = dev->db_addr +
1232 DB_ADDR_SHIFT(DQ_PWM_OFFSET_XCM_RDMA_SQ_PROD);
1233 qp->sq.db_data.data.icid = qp->icid + 1;
1234 if (!qp->srq) {
1235 qp->rq.db = dev->db_addr +
1236 DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_ROCE_RQ_PROD);
1237 qp->rq.db_data.data.icid = qp->icid;
1238 }
1239 }
1240
1241 static int qedr_check_srq_params(struct qedr_dev *dev,
1242 struct ib_srq_init_attr *attrs,
1243 struct ib_udata *udata)
1244 {
1245 struct qedr_device_attr *qattr = &dev->attr;
1246
1247 if (attrs->attr.max_wr > qattr->max_srq_wr) {
1248 DP_ERR(dev,
1249 "create srq: unsupported srq_wr=0x%x requested (max_srq_wr=0x%x)\n",
1250 attrs->attr.max_wr, qattr->max_srq_wr);
1251 return -EINVAL;
1252 }
1253
1254 if (attrs->attr.max_sge > qattr->max_sge) {
1255 DP_ERR(dev,
1256 "create srq: unsupported sge=0x%x requested (max_srq_sge=0x%x)\n",
1257 attrs->attr.max_sge, qattr->max_sge);
1258 return -EINVAL;
1259 }
1260
1261 return 0;
1262 }
1263
1264 static void qedr_free_srq_user_params(struct qedr_srq *srq)
1265 {
1266 qedr_free_pbl(srq->dev, &srq->usrq.pbl_info, srq->usrq.pbl_tbl);
1267 ib_umem_release(srq->usrq.umem);
1268 ib_umem_release(srq->prod_umem);
1269 }
1270
1271 static void qedr_free_srq_kernel_params(struct qedr_srq *srq)
1272 {
1273 struct qedr_srq_hwq_info *hw_srq = &srq->hw_srq;
1274 struct qedr_dev *dev = srq->dev;
1275
1276 dev->ops->common->chain_free(dev->cdev, &hw_srq->pbl);
1277
1278 dma_free_coherent(&dev->pdev->dev, sizeof(struct rdma_srq_producers),
1279 hw_srq->virt_prod_pair_addr,
1280 hw_srq->phy_prod_pair_addr);
1281 }
1282
1283 static int qedr_init_srq_user_params(struct ib_udata *udata,
1284 struct qedr_srq *srq,
1285 struct qedr_create_srq_ureq *ureq,
1286 int access, int dmasync)
1287 {
1288 struct scatterlist *sg;
1289 int rc;
1290
1291 rc = qedr_init_user_queue(udata, srq->dev, &srq->usrq, ureq->srq_addr,
1292 ureq->srq_len, access, dmasync, 1);
1293 if (rc)
1294 return rc;
1295
1296 srq->prod_umem =
1297 ib_umem_get(udata, ureq->prod_pair_addr,
1298 sizeof(struct rdma_srq_producers), access, dmasync);
1299 if (IS_ERR(srq->prod_umem)) {
1300 qedr_free_pbl(srq->dev, &srq->usrq.pbl_info, srq->usrq.pbl_tbl);
1301 ib_umem_release(srq->usrq.umem);
1302 DP_ERR(srq->dev,
1303 "create srq: failed ib_umem_get for producer, got %ld\n",
1304 PTR_ERR(srq->prod_umem));
1305 return PTR_ERR(srq->prod_umem);
1306 }
1307
1308 sg = srq->prod_umem->sg_head.sgl;
1309 srq->hw_srq.phy_prod_pair_addr = sg_dma_address(sg);
1310
1311 return 0;
1312 }
1313
1314 static int qedr_alloc_srq_kernel_params(struct qedr_srq *srq,
1315 struct qedr_dev *dev,
1316 struct ib_srq_init_attr *init_attr)
1317 {
1318 struct qedr_srq_hwq_info *hw_srq = &srq->hw_srq;
1319 dma_addr_t phy_prod_pair_addr;
1320 u32 num_elems;
1321 void *va;
1322 int rc;
1323
1324 va = dma_alloc_coherent(&dev->pdev->dev,
1325 sizeof(struct rdma_srq_producers),
1326 &phy_prod_pair_addr, GFP_KERNEL);
1327 if (!va) {
1328 DP_ERR(dev,
1329 "create srq: failed to allocate dma memory for producer\n");
1330 return -ENOMEM;
1331 }
1332
1333 hw_srq->phy_prod_pair_addr = phy_prod_pair_addr;
1334 hw_srq->virt_prod_pair_addr = va;
1335
1336 num_elems = init_attr->attr.max_wr * RDMA_MAX_SRQ_WQE_SIZE;
1337 rc = dev->ops->common->chain_alloc(dev->cdev,
1338 QED_CHAIN_USE_TO_CONSUME_PRODUCE,
1339 QED_CHAIN_MODE_PBL,
1340 QED_CHAIN_CNT_TYPE_U32,
1341 num_elems,
1342 QEDR_SRQ_WQE_ELEM_SIZE,
1343 &hw_srq->pbl, NULL);
1344 if (rc)
1345 goto err0;
1346
1347 hw_srq->num_elems = num_elems;
1348
1349 return 0;
1350
1351 err0:
1352 dma_free_coherent(&dev->pdev->dev, sizeof(struct rdma_srq_producers),
1353 va, phy_prod_pair_addr);
1354 return rc;
1355 }
1356
1357 int qedr_create_srq(struct ib_srq *ibsrq, struct ib_srq_init_attr *init_attr,
1358 struct ib_udata *udata)
1359 {
1360 struct qed_rdma_destroy_srq_in_params destroy_in_params;
1361 struct qed_rdma_create_srq_in_params in_params = {};
1362 struct qedr_dev *dev = get_qedr_dev(ibsrq->device);
1363 struct qed_rdma_create_srq_out_params out_params;
1364 struct qedr_pd *pd = get_qedr_pd(ibsrq->pd);
1365 struct qedr_create_srq_ureq ureq = {};
1366 u64 pbl_base_addr, phy_prod_pair_addr;
1367 struct qedr_srq_hwq_info *hw_srq;
1368 u32 page_cnt, page_size;
1369 struct qedr_srq *srq = get_qedr_srq(ibsrq);
1370 int rc = 0;
1371
1372 DP_DEBUG(dev, QEDR_MSG_QP,
1373 "create SRQ called from %s (pd %p)\n",
1374 (udata) ? "User lib" : "kernel", pd);
1375
1376 rc = qedr_check_srq_params(dev, init_attr, udata);
1377 if (rc)
1378 return -EINVAL;
1379
1380 srq->dev = dev;
1381 hw_srq = &srq->hw_srq;
1382 spin_lock_init(&srq->lock);
1383
1384 hw_srq->max_wr = init_attr->attr.max_wr;
1385 hw_srq->max_sges = init_attr->attr.max_sge;
1386
1387 if (udata) {
1388 if (ib_copy_from_udata(&ureq, udata, sizeof(ureq))) {
1389 DP_ERR(dev,
1390 "create srq: problem copying data from user space\n");
1391 goto err0;
1392 }
1393
1394 rc = qedr_init_srq_user_params(udata, srq, &ureq, 0, 0);
1395 if (rc)
1396 goto err0;
1397
1398 page_cnt = srq->usrq.pbl_info.num_pbes;
1399 pbl_base_addr = srq->usrq.pbl_tbl->pa;
1400 phy_prod_pair_addr = hw_srq->phy_prod_pair_addr;
1401 page_size = PAGE_SIZE;
1402 } else {
1403 struct qed_chain *pbl;
1404
1405 rc = qedr_alloc_srq_kernel_params(srq, dev, init_attr);
1406 if (rc)
1407 goto err0;
1408
1409 pbl = &hw_srq->pbl;
1410 page_cnt = qed_chain_get_page_cnt(pbl);
1411 pbl_base_addr = qed_chain_get_pbl_phys(pbl);
1412 phy_prod_pair_addr = hw_srq->phy_prod_pair_addr;
1413 page_size = QED_CHAIN_PAGE_SIZE;
1414 }
1415
1416 in_params.pd_id = pd->pd_id;
1417 in_params.pbl_base_addr = pbl_base_addr;
1418 in_params.prod_pair_addr = phy_prod_pair_addr;
1419 in_params.num_pages = page_cnt;
1420 in_params.page_size = page_size;
1421
1422 rc = dev->ops->rdma_create_srq(dev->rdma_ctx, &in_params, &out_params);
1423 if (rc)
1424 goto err1;
1425
1426 srq->srq_id = out_params.srq_id;
1427
1428 if (udata) {
1429 rc = qedr_copy_srq_uresp(dev, srq, udata);
1430 if (rc)
1431 goto err2;
1432 }
1433
1434 rc = xa_insert_irq(&dev->srqs, srq->srq_id, srq, GFP_KERNEL);
1435 if (rc)
1436 goto err2;
1437
1438 DP_DEBUG(dev, QEDR_MSG_SRQ,
1439 "create srq: created srq with srq_id=0x%0x\n", srq->srq_id);
1440 return 0;
1441
1442 err2:
1443 destroy_in_params.srq_id = srq->srq_id;
1444
1445 dev->ops->rdma_destroy_srq(dev->rdma_ctx, &destroy_in_params);
1446 err1:
1447 if (udata)
1448 qedr_free_srq_user_params(srq);
1449 else
1450 qedr_free_srq_kernel_params(srq);
1451 err0:
1452 return -EFAULT;
1453 }
1454
1455 void qedr_destroy_srq(struct ib_srq *ibsrq, struct ib_udata *udata)
1456 {
1457 struct qed_rdma_destroy_srq_in_params in_params = {};
1458 struct qedr_dev *dev = get_qedr_dev(ibsrq->device);
1459 struct qedr_srq *srq = get_qedr_srq(ibsrq);
1460
1461 xa_erase_irq(&dev->srqs, srq->srq_id);
1462 in_params.srq_id = srq->srq_id;
1463 dev->ops->rdma_destroy_srq(dev->rdma_ctx, &in_params);
1464
1465 if (ibsrq->uobject)
1466 qedr_free_srq_user_params(srq);
1467 else
1468 qedr_free_srq_kernel_params(srq);
1469
1470 DP_DEBUG(dev, QEDR_MSG_SRQ,
1471 "destroy srq: destroyed srq with srq_id=0x%0x\n",
1472 srq->srq_id);
1473 }
1474
1475 int qedr_modify_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr,
1476 enum ib_srq_attr_mask attr_mask, struct ib_udata *udata)
1477 {
1478 struct qed_rdma_modify_srq_in_params in_params = {};
1479 struct qedr_dev *dev = get_qedr_dev(ibsrq->device);
1480 struct qedr_srq *srq = get_qedr_srq(ibsrq);
1481 int rc;
1482
1483 if (attr_mask & IB_SRQ_MAX_WR) {
1484 DP_ERR(dev,
1485 "modify srq: invalid attribute mask=0x%x specified for %p\n",
1486 attr_mask, srq);
1487 return -EINVAL;
1488 }
1489
1490 if (attr_mask & IB_SRQ_LIMIT) {
1491 if (attr->srq_limit >= srq->hw_srq.max_wr) {
1492 DP_ERR(dev,
1493 "modify srq: invalid srq_limit=0x%x (max_srq_limit=0x%x)\n",
1494 attr->srq_limit, srq->hw_srq.max_wr);
1495 return -EINVAL;
1496 }
1497
1498 in_params.srq_id = srq->srq_id;
1499 in_params.wqe_limit = attr->srq_limit;
1500 rc = dev->ops->rdma_modify_srq(dev->rdma_ctx, &in_params);
1501 if (rc)
1502 return rc;
1503 }
1504
1505 srq->srq_limit = attr->srq_limit;
1506
1507 DP_DEBUG(dev, QEDR_MSG_SRQ,
1508 "modify srq: modified srq with srq_id=0x%0x\n", srq->srq_id);
1509
1510 return 0;
1511 }
1512
1513 static inline void
1514 qedr_init_common_qp_in_params(struct qedr_dev *dev,
1515 struct qedr_pd *pd,
1516 struct qedr_qp *qp,
1517 struct ib_qp_init_attr *attrs,
1518 bool fmr_and_reserved_lkey,
1519 struct qed_rdma_create_qp_in_params *params)
1520 {
1521
1522 params->qp_handle_async_lo = lower_32_bits((uintptr_t) qp);
1523 params->qp_handle_async_hi = upper_32_bits((uintptr_t) qp);
1524
1525 params->signal_all = (attrs->sq_sig_type == IB_SIGNAL_ALL_WR);
1526 params->fmr_and_reserved_lkey = fmr_and_reserved_lkey;
1527 params->pd = pd->pd_id;
1528 params->dpi = pd->uctx ? pd->uctx->dpi : dev->dpi;
1529 params->sq_cq_id = get_qedr_cq(attrs->send_cq)->icid;
1530 params->stats_queue = 0;
1531 params->srq_id = 0;
1532 params->use_srq = false;
1533
1534 if (!qp->srq) {
1535 params->rq_cq_id = get_qedr_cq(attrs->recv_cq)->icid;
1536
1537 } else {
1538 params->rq_cq_id = get_qedr_cq(attrs->recv_cq)->icid;
1539 params->srq_id = qp->srq->srq_id;
1540 params->use_srq = true;
1541 }
1542 }
1543
1544 static inline void qedr_qp_user_print(struct qedr_dev *dev, struct qedr_qp *qp)
1545 {
1546 DP_DEBUG(dev, QEDR_MSG_QP, "create qp: successfully created user QP. "
1547 "qp=%p. "
1548 "sq_addr=0x%llx, "
1549 "sq_len=%zd, "
1550 "rq_addr=0x%llx, "
1551 "rq_len=%zd"
1552 "\n",
1553 qp,
1554 qp->usq.buf_addr,
1555 qp->usq.buf_len, qp->urq.buf_addr, qp->urq.buf_len);
1556 }
1557
1558 static inline void
1559 qedr_iwarp_populate_user_qp(struct qedr_dev *dev,
1560 struct qedr_qp *qp,
1561 struct qed_rdma_create_qp_out_params *out_params)
1562 {
1563 qp->usq.pbl_tbl->va = out_params->sq_pbl_virt;
1564 qp->usq.pbl_tbl->pa = out_params->sq_pbl_phys;
1565
1566 qedr_populate_pbls(dev, qp->usq.umem, qp->usq.pbl_tbl,
1567 &qp->usq.pbl_info, FW_PAGE_SHIFT);
1568 if (!qp->srq) {
1569 qp->urq.pbl_tbl->va = out_params->rq_pbl_virt;
1570 qp->urq.pbl_tbl->pa = out_params->rq_pbl_phys;
1571 }
1572
1573 qedr_populate_pbls(dev, qp->urq.umem, qp->urq.pbl_tbl,
1574 &qp->urq.pbl_info, FW_PAGE_SHIFT);
1575 }
1576
1577 static void qedr_cleanup_user(struct qedr_dev *dev, struct qedr_qp *qp)
1578 {
1579 ib_umem_release(qp->usq.umem);
1580 qp->usq.umem = NULL;
1581
1582 ib_umem_release(qp->urq.umem);
1583 qp->urq.umem = NULL;
1584
1585 if (rdma_protocol_roce(&dev->ibdev, 1)) {
1586 qedr_free_pbl(dev, &qp->usq.pbl_info, qp->usq.pbl_tbl);
1587 qedr_free_pbl(dev, &qp->urq.pbl_info, qp->urq.pbl_tbl);
1588 } else {
1589 kfree(qp->usq.pbl_tbl);
1590 kfree(qp->urq.pbl_tbl);
1591 }
1592 }
1593
1594 static int qedr_create_user_qp(struct qedr_dev *dev,
1595 struct qedr_qp *qp,
1596 struct ib_pd *ibpd,
1597 struct ib_udata *udata,
1598 struct ib_qp_init_attr *attrs)
1599 {
1600 struct qed_rdma_create_qp_in_params in_params;
1601 struct qed_rdma_create_qp_out_params out_params;
1602 struct qedr_pd *pd = get_qedr_pd(ibpd);
1603 struct qedr_create_qp_ureq ureq;
1604 int alloc_and_init = rdma_protocol_roce(&dev->ibdev, 1);
1605 int rc = -EINVAL;
1606
1607 qp->create_type = QEDR_QP_CREATE_USER;
1608 memset(&ureq, 0, sizeof(ureq));
1609 rc = ib_copy_from_udata(&ureq, udata, sizeof(ureq));
1610 if (rc) {
1611 DP_ERR(dev, "Problem copying data from user space\n");
1612 return rc;
1613 }
1614
1615
1616 rc = qedr_init_user_queue(udata, dev, &qp->usq, ureq.sq_addr,
1617 ureq.sq_len, 0, 0, alloc_and_init);
1618 if (rc)
1619 return rc;
1620
1621 if (!qp->srq) {
1622
1623 rc = qedr_init_user_queue(udata, dev, &qp->urq, ureq.rq_addr,
1624 ureq.rq_len, 0, 0, alloc_and_init);
1625 if (rc)
1626 return rc;
1627 }
1628
1629 memset(&in_params, 0, sizeof(in_params));
1630 qedr_init_common_qp_in_params(dev, pd, qp, attrs, false, &in_params);
1631 in_params.qp_handle_lo = ureq.qp_handle_lo;
1632 in_params.qp_handle_hi = ureq.qp_handle_hi;
1633 in_params.sq_num_pages = qp->usq.pbl_info.num_pbes;
1634 in_params.sq_pbl_ptr = qp->usq.pbl_tbl->pa;
1635 if (!qp->srq) {
1636 in_params.rq_num_pages = qp->urq.pbl_info.num_pbes;
1637 in_params.rq_pbl_ptr = qp->urq.pbl_tbl->pa;
1638 }
1639
1640 qp->qed_qp = dev->ops->rdma_create_qp(dev->rdma_ctx,
1641 &in_params, &out_params);
1642
1643 if (!qp->qed_qp) {
1644 rc = -ENOMEM;
1645 goto err1;
1646 }
1647
1648 if (rdma_protocol_iwarp(&dev->ibdev, 1))
1649 qedr_iwarp_populate_user_qp(dev, qp, &out_params);
1650
1651 qp->qp_id = out_params.qp_id;
1652 qp->icid = out_params.icid;
1653
1654 rc = qedr_copy_qp_uresp(dev, qp, udata);
1655 if (rc)
1656 goto err;
1657
1658 qedr_qp_user_print(dev, qp);
1659
1660 return 0;
1661 err:
1662 rc = dev->ops->rdma_destroy_qp(dev->rdma_ctx, qp->qed_qp);
1663 if (rc)
1664 DP_ERR(dev, "create qp: fatal fault. rc=%d", rc);
1665
1666 err1:
1667 qedr_cleanup_user(dev, qp);
1668 return rc;
1669 }
1670
1671 static void qedr_set_iwarp_db_info(struct qedr_dev *dev, struct qedr_qp *qp)
1672 {
1673 qp->sq.db = dev->db_addr +
1674 DB_ADDR_SHIFT(DQ_PWM_OFFSET_XCM_RDMA_SQ_PROD);
1675 qp->sq.db_data.data.icid = qp->icid;
1676
1677 qp->rq.db = dev->db_addr +
1678 DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_IWARP_RQ_PROD);
1679 qp->rq.db_data.data.icid = qp->icid;
1680 qp->rq.iwarp_db2 = dev->db_addr +
1681 DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_FLAGS);
1682 qp->rq.iwarp_db2_data.data.icid = qp->icid;
1683 qp->rq.iwarp_db2_data.data.value = DQ_TCM_IWARP_POST_RQ_CF_CMD;
1684 }
1685
1686 static int
1687 qedr_roce_create_kernel_qp(struct qedr_dev *dev,
1688 struct qedr_qp *qp,
1689 struct qed_rdma_create_qp_in_params *in_params,
1690 u32 n_sq_elems, u32 n_rq_elems)
1691 {
1692 struct qed_rdma_create_qp_out_params out_params;
1693 int rc;
1694
1695 rc = dev->ops->common->chain_alloc(dev->cdev,
1696 QED_CHAIN_USE_TO_PRODUCE,
1697 QED_CHAIN_MODE_PBL,
1698 QED_CHAIN_CNT_TYPE_U32,
1699 n_sq_elems,
1700 QEDR_SQE_ELEMENT_SIZE,
1701 &qp->sq.pbl, NULL);
1702
1703 if (rc)
1704 return rc;
1705
1706 in_params->sq_num_pages = qed_chain_get_page_cnt(&qp->sq.pbl);
1707 in_params->sq_pbl_ptr = qed_chain_get_pbl_phys(&qp->sq.pbl);
1708
1709 rc = dev->ops->common->chain_alloc(dev->cdev,
1710 QED_CHAIN_USE_TO_CONSUME_PRODUCE,
1711 QED_CHAIN_MODE_PBL,
1712 QED_CHAIN_CNT_TYPE_U32,
1713 n_rq_elems,
1714 QEDR_RQE_ELEMENT_SIZE,
1715 &qp->rq.pbl, NULL);
1716 if (rc)
1717 return rc;
1718
1719 in_params->rq_num_pages = qed_chain_get_page_cnt(&qp->rq.pbl);
1720 in_params->rq_pbl_ptr = qed_chain_get_pbl_phys(&qp->rq.pbl);
1721
1722 qp->qed_qp = dev->ops->rdma_create_qp(dev->rdma_ctx,
1723 in_params, &out_params);
1724
1725 if (!qp->qed_qp)
1726 return -EINVAL;
1727
1728 qp->qp_id = out_params.qp_id;
1729 qp->icid = out_params.icid;
1730
1731 qedr_set_roce_db_info(dev, qp);
1732 return rc;
1733 }
1734
1735 static int
1736 qedr_iwarp_create_kernel_qp(struct qedr_dev *dev,
1737 struct qedr_qp *qp,
1738 struct qed_rdma_create_qp_in_params *in_params,
1739 u32 n_sq_elems, u32 n_rq_elems)
1740 {
1741 struct qed_rdma_create_qp_out_params out_params;
1742 struct qed_chain_ext_pbl ext_pbl;
1743 int rc;
1744
1745 in_params->sq_num_pages = QED_CHAIN_PAGE_CNT(n_sq_elems,
1746 QEDR_SQE_ELEMENT_SIZE,
1747 QED_CHAIN_MODE_PBL);
1748 in_params->rq_num_pages = QED_CHAIN_PAGE_CNT(n_rq_elems,
1749 QEDR_RQE_ELEMENT_SIZE,
1750 QED_CHAIN_MODE_PBL);
1751
1752 qp->qed_qp = dev->ops->rdma_create_qp(dev->rdma_ctx,
1753 in_params, &out_params);
1754
1755 if (!qp->qed_qp)
1756 return -EINVAL;
1757
1758
1759 ext_pbl.p_pbl_virt = out_params.sq_pbl_virt;
1760 ext_pbl.p_pbl_phys = out_params.sq_pbl_phys;
1761
1762 rc = dev->ops->common->chain_alloc(dev->cdev,
1763 QED_CHAIN_USE_TO_PRODUCE,
1764 QED_CHAIN_MODE_PBL,
1765 QED_CHAIN_CNT_TYPE_U32,
1766 n_sq_elems,
1767 QEDR_SQE_ELEMENT_SIZE,
1768 &qp->sq.pbl, &ext_pbl);
1769
1770 if (rc)
1771 goto err;
1772
1773 ext_pbl.p_pbl_virt = out_params.rq_pbl_virt;
1774 ext_pbl.p_pbl_phys = out_params.rq_pbl_phys;
1775
1776 rc = dev->ops->common->chain_alloc(dev->cdev,
1777 QED_CHAIN_USE_TO_CONSUME_PRODUCE,
1778 QED_CHAIN_MODE_PBL,
1779 QED_CHAIN_CNT_TYPE_U32,
1780 n_rq_elems,
1781 QEDR_RQE_ELEMENT_SIZE,
1782 &qp->rq.pbl, &ext_pbl);
1783
1784 if (rc)
1785 goto err;
1786
1787 qp->qp_id = out_params.qp_id;
1788 qp->icid = out_params.icid;
1789
1790 qedr_set_iwarp_db_info(dev, qp);
1791 return rc;
1792
1793 err:
1794 dev->ops->rdma_destroy_qp(dev->rdma_ctx, qp->qed_qp);
1795
1796 return rc;
1797 }
1798
1799 static void qedr_cleanup_kernel(struct qedr_dev *dev, struct qedr_qp *qp)
1800 {
1801 dev->ops->common->chain_free(dev->cdev, &qp->sq.pbl);
1802 kfree(qp->wqe_wr_id);
1803
1804 dev->ops->common->chain_free(dev->cdev, &qp->rq.pbl);
1805 kfree(qp->rqe_wr_id);
1806 }
1807
1808 static int qedr_create_kernel_qp(struct qedr_dev *dev,
1809 struct qedr_qp *qp,
1810 struct ib_pd *ibpd,
1811 struct ib_qp_init_attr *attrs)
1812 {
1813 struct qed_rdma_create_qp_in_params in_params;
1814 struct qedr_pd *pd = get_qedr_pd(ibpd);
1815 int rc = -EINVAL;
1816 u32 n_rq_elems;
1817 u32 n_sq_elems;
1818 u32 n_sq_entries;
1819
1820 memset(&in_params, 0, sizeof(in_params));
1821 qp->create_type = QEDR_QP_CREATE_KERNEL;
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833 qp->sq.max_wr = min_t(u32, attrs->cap.max_send_wr * dev->wq_multiplier,
1834 dev->attr.max_sqe);
1835
1836 qp->wqe_wr_id = kcalloc(qp->sq.max_wr, sizeof(*qp->wqe_wr_id),
1837 GFP_KERNEL);
1838 if (!qp->wqe_wr_id) {
1839 DP_ERR(dev, "create qp: failed SQ shadow memory allocation\n");
1840 return -ENOMEM;
1841 }
1842
1843
1844 in_params.qp_handle_lo = lower_32_bits((uintptr_t) qp);
1845 in_params.qp_handle_hi = upper_32_bits((uintptr_t) qp);
1846
1847
1848
1849
1850
1851 qp->rq.max_wr = (u16) max_t(u32, attrs->cap.max_recv_wr, 1);
1852
1853
1854 qp->rqe_wr_id = kcalloc(qp->rq.max_wr, sizeof(*qp->rqe_wr_id),
1855 GFP_KERNEL);
1856 if (!qp->rqe_wr_id) {
1857 DP_ERR(dev,
1858 "create qp: failed RQ shadow memory allocation\n");
1859 kfree(qp->wqe_wr_id);
1860 return -ENOMEM;
1861 }
1862
1863 qedr_init_common_qp_in_params(dev, pd, qp, attrs, true, &in_params);
1864
1865 n_sq_entries = attrs->cap.max_send_wr;
1866 n_sq_entries = min_t(u32, n_sq_entries, dev->attr.max_sqe);
1867 n_sq_entries = max_t(u32, n_sq_entries, 1);
1868 n_sq_elems = n_sq_entries * QEDR_MAX_SQE_ELEMENTS_PER_SQE;
1869
1870 n_rq_elems = qp->rq.max_wr * QEDR_MAX_RQE_ELEMENTS_PER_RQE;
1871
1872 if (rdma_protocol_iwarp(&dev->ibdev, 1))
1873 rc = qedr_iwarp_create_kernel_qp(dev, qp, &in_params,
1874 n_sq_elems, n_rq_elems);
1875 else
1876 rc = qedr_roce_create_kernel_qp(dev, qp, &in_params,
1877 n_sq_elems, n_rq_elems);
1878 if (rc)
1879 qedr_cleanup_kernel(dev, qp);
1880
1881 return rc;
1882 }
1883
1884 struct ib_qp *qedr_create_qp(struct ib_pd *ibpd,
1885 struct ib_qp_init_attr *attrs,
1886 struct ib_udata *udata)
1887 {
1888 struct qedr_dev *dev = get_qedr_dev(ibpd->device);
1889 struct qedr_pd *pd = get_qedr_pd(ibpd);
1890 struct qedr_qp *qp;
1891 struct ib_qp *ibqp;
1892 int rc = 0;
1893
1894 DP_DEBUG(dev, QEDR_MSG_QP, "create qp: called from %s, pd=%p\n",
1895 udata ? "user library" : "kernel", pd);
1896
1897 rc = qedr_check_qp_attrs(ibpd, dev, attrs, udata);
1898 if (rc)
1899 return ERR_PTR(rc);
1900
1901 DP_DEBUG(dev, QEDR_MSG_QP,
1902 "create qp: called from %s, event_handler=%p, eepd=%p sq_cq=%p, sq_icid=%d, rq_cq=%p, rq_icid=%d\n",
1903 udata ? "user library" : "kernel", attrs->event_handler, pd,
1904 get_qedr_cq(attrs->send_cq),
1905 get_qedr_cq(attrs->send_cq)->icid,
1906 get_qedr_cq(attrs->recv_cq),
1907 attrs->recv_cq ? get_qedr_cq(attrs->recv_cq)->icid : 0);
1908
1909 qp = kzalloc(sizeof(*qp), GFP_KERNEL);
1910 if (!qp) {
1911 DP_ERR(dev, "create qp: failed allocating memory\n");
1912 return ERR_PTR(-ENOMEM);
1913 }
1914
1915 qedr_set_common_qp_params(dev, qp, pd, attrs);
1916
1917 if (attrs->qp_type == IB_QPT_GSI) {
1918 ibqp = qedr_create_gsi_qp(dev, attrs, qp);
1919 if (IS_ERR(ibqp))
1920 kfree(qp);
1921 return ibqp;
1922 }
1923
1924 if (udata)
1925 rc = qedr_create_user_qp(dev, qp, ibpd, udata, attrs);
1926 else
1927 rc = qedr_create_kernel_qp(dev, qp, ibpd, attrs);
1928
1929 if (rc)
1930 goto err;
1931
1932 qp->ibqp.qp_num = qp->qp_id;
1933
1934 if (rdma_protocol_iwarp(&dev->ibdev, 1)) {
1935 rc = xa_insert(&dev->qps, qp->qp_id, qp, GFP_KERNEL);
1936 if (rc)
1937 goto err;
1938 }
1939
1940 return &qp->ibqp;
1941
1942 err:
1943 kfree(qp);
1944
1945 return ERR_PTR(-EFAULT);
1946 }
1947
1948 static enum ib_qp_state qedr_get_ibqp_state(enum qed_roce_qp_state qp_state)
1949 {
1950 switch (qp_state) {
1951 case QED_ROCE_QP_STATE_RESET:
1952 return IB_QPS_RESET;
1953 case QED_ROCE_QP_STATE_INIT:
1954 return IB_QPS_INIT;
1955 case QED_ROCE_QP_STATE_RTR:
1956 return IB_QPS_RTR;
1957 case QED_ROCE_QP_STATE_RTS:
1958 return IB_QPS_RTS;
1959 case QED_ROCE_QP_STATE_SQD:
1960 return IB_QPS_SQD;
1961 case QED_ROCE_QP_STATE_ERR:
1962 return IB_QPS_ERR;
1963 case QED_ROCE_QP_STATE_SQE:
1964 return IB_QPS_SQE;
1965 }
1966 return IB_QPS_ERR;
1967 }
1968
1969 static enum qed_roce_qp_state qedr_get_state_from_ibqp(
1970 enum ib_qp_state qp_state)
1971 {
1972 switch (qp_state) {
1973 case IB_QPS_RESET:
1974 return QED_ROCE_QP_STATE_RESET;
1975 case IB_QPS_INIT:
1976 return QED_ROCE_QP_STATE_INIT;
1977 case IB_QPS_RTR:
1978 return QED_ROCE_QP_STATE_RTR;
1979 case IB_QPS_RTS:
1980 return QED_ROCE_QP_STATE_RTS;
1981 case IB_QPS_SQD:
1982 return QED_ROCE_QP_STATE_SQD;
1983 case IB_QPS_ERR:
1984 return QED_ROCE_QP_STATE_ERR;
1985 default:
1986 return QED_ROCE_QP_STATE_ERR;
1987 }
1988 }
1989
1990 static void qedr_reset_qp_hwq_info(struct qedr_qp_hwq_info *qph)
1991 {
1992 qed_chain_reset(&qph->pbl);
1993 qph->prod = 0;
1994 qph->cons = 0;
1995 qph->wqe_cons = 0;
1996 qph->db_data.data.value = cpu_to_le16(0);
1997 }
1998
1999 static int qedr_update_qp_state(struct qedr_dev *dev,
2000 struct qedr_qp *qp,
2001 enum qed_roce_qp_state cur_state,
2002 enum qed_roce_qp_state new_state)
2003 {
2004 int status = 0;
2005
2006 if (new_state == cur_state)
2007 return 0;
2008
2009 switch (cur_state) {
2010 case QED_ROCE_QP_STATE_RESET:
2011 switch (new_state) {
2012 case QED_ROCE_QP_STATE_INIT:
2013 qp->prev_wqe_size = 0;
2014 qedr_reset_qp_hwq_info(&qp->sq);
2015 qedr_reset_qp_hwq_info(&qp->rq);
2016 break;
2017 default:
2018 status = -EINVAL;
2019 break;
2020 }
2021 break;
2022 case QED_ROCE_QP_STATE_INIT:
2023 switch (new_state) {
2024 case QED_ROCE_QP_STATE_RTR:
2025
2026
2027
2028
2029 if (rdma_protocol_roce(&dev->ibdev, 1)) {
2030 writel(qp->rq.db_data.raw, qp->rq.db);
2031 }
2032 break;
2033 case QED_ROCE_QP_STATE_ERR:
2034 break;
2035 default:
2036
2037 status = -EINVAL;
2038 break;
2039 }
2040 break;
2041 case QED_ROCE_QP_STATE_RTR:
2042
2043 switch (new_state) {
2044 case QED_ROCE_QP_STATE_RTS:
2045 break;
2046 case QED_ROCE_QP_STATE_ERR:
2047 break;
2048 default:
2049
2050 status = -EINVAL;
2051 break;
2052 }
2053 break;
2054 case QED_ROCE_QP_STATE_RTS:
2055
2056 switch (new_state) {
2057 case QED_ROCE_QP_STATE_SQD:
2058 break;
2059 case QED_ROCE_QP_STATE_ERR:
2060 break;
2061 default:
2062
2063 status = -EINVAL;
2064 break;
2065 }
2066 break;
2067 case QED_ROCE_QP_STATE_SQD:
2068
2069 switch (new_state) {
2070 case QED_ROCE_QP_STATE_RTS:
2071 case QED_ROCE_QP_STATE_ERR:
2072 break;
2073 default:
2074
2075 status = -EINVAL;
2076 break;
2077 }
2078 break;
2079 case QED_ROCE_QP_STATE_ERR:
2080
2081 switch (new_state) {
2082 case QED_ROCE_QP_STATE_RESET:
2083 if ((qp->rq.prod != qp->rq.cons) ||
2084 (qp->sq.prod != qp->sq.cons)) {
2085 DP_NOTICE(dev,
2086 "Error->Reset with rq/sq not empty rq.prod=%x rq.cons=%x sq.prod=%x sq.cons=%x\n",
2087 qp->rq.prod, qp->rq.cons, qp->sq.prod,
2088 qp->sq.cons);
2089 status = -EINVAL;
2090 }
2091 break;
2092 default:
2093 status = -EINVAL;
2094 break;
2095 }
2096 break;
2097 default:
2098 status = -EINVAL;
2099 break;
2100 }
2101
2102 return status;
2103 }
2104
2105 int qedr_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
2106 int attr_mask, struct ib_udata *udata)
2107 {
2108 struct qedr_qp *qp = get_qedr_qp(ibqp);
2109 struct qed_rdma_modify_qp_in_params qp_params = { 0 };
2110 struct qedr_dev *dev = get_qedr_dev(&qp->dev->ibdev);
2111 const struct ib_global_route *grh = rdma_ah_read_grh(&attr->ah_attr);
2112 enum ib_qp_state old_qp_state, new_qp_state;
2113 enum qed_roce_qp_state cur_state;
2114 int rc = 0;
2115
2116 DP_DEBUG(dev, QEDR_MSG_QP,
2117 "modify qp: qp %p attr_mask=0x%x, state=%d", qp, attr_mask,
2118 attr->qp_state);
2119
2120 old_qp_state = qedr_get_ibqp_state(qp->state);
2121 if (attr_mask & IB_QP_STATE)
2122 new_qp_state = attr->qp_state;
2123 else
2124 new_qp_state = old_qp_state;
2125
2126 if (rdma_protocol_roce(&dev->ibdev, 1)) {
2127 if (!ib_modify_qp_is_ok(old_qp_state, new_qp_state,
2128 ibqp->qp_type, attr_mask)) {
2129 DP_ERR(dev,
2130 "modify qp: invalid attribute mask=0x%x specified for\n"
2131 "qpn=0x%x of type=0x%x old_qp_state=0x%x, new_qp_state=0x%x\n",
2132 attr_mask, qp->qp_id, ibqp->qp_type,
2133 old_qp_state, new_qp_state);
2134 rc = -EINVAL;
2135 goto err;
2136 }
2137 }
2138
2139
2140 if (attr_mask & IB_QP_STATE) {
2141 SET_FIELD(qp_params.modify_flags,
2142 QED_RDMA_MODIFY_QP_VALID_NEW_STATE, 1);
2143 qp_params.new_state = qedr_get_state_from_ibqp(attr->qp_state);
2144 }
2145
2146 if (attr_mask & IB_QP_EN_SQD_ASYNC_NOTIFY)
2147 qp_params.sqd_async = true;
2148
2149 if (attr_mask & IB_QP_PKEY_INDEX) {
2150 SET_FIELD(qp_params.modify_flags,
2151 QED_ROCE_MODIFY_QP_VALID_PKEY, 1);
2152 if (attr->pkey_index >= QEDR_ROCE_PKEY_TABLE_LEN) {
2153 rc = -EINVAL;
2154 goto err;
2155 }
2156
2157 qp_params.pkey = QEDR_ROCE_PKEY_DEFAULT;
2158 }
2159
2160 if (attr_mask & IB_QP_QKEY)
2161 qp->qkey = attr->qkey;
2162
2163 if (attr_mask & IB_QP_ACCESS_FLAGS) {
2164 SET_FIELD(qp_params.modify_flags,
2165 QED_RDMA_MODIFY_QP_VALID_RDMA_OPS_EN, 1);
2166 qp_params.incoming_rdma_read_en = attr->qp_access_flags &
2167 IB_ACCESS_REMOTE_READ;
2168 qp_params.incoming_rdma_write_en = attr->qp_access_flags &
2169 IB_ACCESS_REMOTE_WRITE;
2170 qp_params.incoming_atomic_en = attr->qp_access_flags &
2171 IB_ACCESS_REMOTE_ATOMIC;
2172 }
2173
2174 if (attr_mask & (IB_QP_AV | IB_QP_PATH_MTU)) {
2175 if (rdma_protocol_iwarp(&dev->ibdev, 1))
2176 return -EINVAL;
2177
2178 if (attr_mask & IB_QP_PATH_MTU) {
2179 if (attr->path_mtu < IB_MTU_256 ||
2180 attr->path_mtu > IB_MTU_4096) {
2181 pr_err("error: Only MTU sizes of 256, 512, 1024, 2048 and 4096 are supported by RoCE\n");
2182 rc = -EINVAL;
2183 goto err;
2184 }
2185 qp->mtu = min(ib_mtu_enum_to_int(attr->path_mtu),
2186 ib_mtu_enum_to_int(iboe_get_mtu
2187 (dev->ndev->mtu)));
2188 }
2189
2190 if (!qp->mtu) {
2191 qp->mtu =
2192 ib_mtu_enum_to_int(iboe_get_mtu(dev->ndev->mtu));
2193 pr_err("Fixing zeroed MTU to qp->mtu = %d\n", qp->mtu);
2194 }
2195
2196 SET_FIELD(qp_params.modify_flags,
2197 QED_ROCE_MODIFY_QP_VALID_ADDRESS_VECTOR, 1);
2198
2199 qp_params.traffic_class_tos = grh->traffic_class;
2200 qp_params.flow_label = grh->flow_label;
2201 qp_params.hop_limit_ttl = grh->hop_limit;
2202
2203 qp->sgid_idx = grh->sgid_index;
2204
2205 rc = get_gid_info_from_table(ibqp, attr, attr_mask, &qp_params);
2206 if (rc) {
2207 DP_ERR(dev,
2208 "modify qp: problems with GID index %d (rc=%d)\n",
2209 grh->sgid_index, rc);
2210 return rc;
2211 }
2212
2213 rc = qedr_get_dmac(dev, &attr->ah_attr,
2214 qp_params.remote_mac_addr);
2215 if (rc)
2216 return rc;
2217
2218 qp_params.use_local_mac = true;
2219 ether_addr_copy(qp_params.local_mac_addr, dev->ndev->dev_addr);
2220
2221 DP_DEBUG(dev, QEDR_MSG_QP, "dgid=%x:%x:%x:%x\n",
2222 qp_params.dgid.dwords[0], qp_params.dgid.dwords[1],
2223 qp_params.dgid.dwords[2], qp_params.dgid.dwords[3]);
2224 DP_DEBUG(dev, QEDR_MSG_QP, "sgid=%x:%x:%x:%x\n",
2225 qp_params.sgid.dwords[0], qp_params.sgid.dwords[1],
2226 qp_params.sgid.dwords[2], qp_params.sgid.dwords[3]);
2227 DP_DEBUG(dev, QEDR_MSG_QP, "remote_mac=[%pM]\n",
2228 qp_params.remote_mac_addr);
2229
2230 qp_params.mtu = qp->mtu;
2231 qp_params.lb_indication = false;
2232 }
2233
2234 if (!qp_params.mtu) {
2235
2236 if (qp->mtu)
2237 qp_params.mtu = qp->mtu;
2238 else
2239 qp_params.mtu =
2240 ib_mtu_enum_to_int(iboe_get_mtu(dev->ndev->mtu));
2241 }
2242
2243 if (attr_mask & IB_QP_TIMEOUT) {
2244 SET_FIELD(qp_params.modify_flags,
2245 QED_ROCE_MODIFY_QP_VALID_ACK_TIMEOUT, 1);
2246
2247
2248
2249
2250
2251
2252
2253
2254
2255
2256
2257 if (attr->timeout)
2258 qp_params.ack_timeout =
2259 1 << max_t(int, attr->timeout - 8, 0);
2260 else
2261 qp_params.ack_timeout = 0;
2262 }
2263
2264 if (attr_mask & IB_QP_RETRY_CNT) {
2265 SET_FIELD(qp_params.modify_flags,
2266 QED_ROCE_MODIFY_QP_VALID_RETRY_CNT, 1);
2267 qp_params.retry_cnt = attr->retry_cnt;
2268 }
2269
2270 if (attr_mask & IB_QP_RNR_RETRY) {
2271 SET_FIELD(qp_params.modify_flags,
2272 QED_ROCE_MODIFY_QP_VALID_RNR_RETRY_CNT, 1);
2273 qp_params.rnr_retry_cnt = attr->rnr_retry;
2274 }
2275
2276 if (attr_mask & IB_QP_RQ_PSN) {
2277 SET_FIELD(qp_params.modify_flags,
2278 QED_ROCE_MODIFY_QP_VALID_RQ_PSN, 1);
2279 qp_params.rq_psn = attr->rq_psn;
2280 qp->rq_psn = attr->rq_psn;
2281 }
2282
2283 if (attr_mask & IB_QP_MAX_QP_RD_ATOMIC) {
2284 if (attr->max_rd_atomic > dev->attr.max_qp_req_rd_atomic_resc) {
2285 rc = -EINVAL;
2286 DP_ERR(dev,
2287 "unsupported max_rd_atomic=%d, supported=%d\n",
2288 attr->max_rd_atomic,
2289 dev->attr.max_qp_req_rd_atomic_resc);
2290 goto err;
2291 }
2292
2293 SET_FIELD(qp_params.modify_flags,
2294 QED_RDMA_MODIFY_QP_VALID_MAX_RD_ATOMIC_REQ, 1);
2295 qp_params.max_rd_atomic_req = attr->max_rd_atomic;
2296 }
2297
2298 if (attr_mask & IB_QP_MIN_RNR_TIMER) {
2299 SET_FIELD(qp_params.modify_flags,
2300 QED_ROCE_MODIFY_QP_VALID_MIN_RNR_NAK_TIMER, 1);
2301 qp_params.min_rnr_nak_timer = attr->min_rnr_timer;
2302 }
2303
2304 if (attr_mask & IB_QP_SQ_PSN) {
2305 SET_FIELD(qp_params.modify_flags,
2306 QED_ROCE_MODIFY_QP_VALID_SQ_PSN, 1);
2307 qp_params.sq_psn = attr->sq_psn;
2308 qp->sq_psn = attr->sq_psn;
2309 }
2310
2311 if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) {
2312 if (attr->max_dest_rd_atomic >
2313 dev->attr.max_qp_resp_rd_atomic_resc) {
2314 DP_ERR(dev,
2315 "unsupported max_dest_rd_atomic=%d, supported=%d\n",
2316 attr->max_dest_rd_atomic,
2317 dev->attr.max_qp_resp_rd_atomic_resc);
2318
2319 rc = -EINVAL;
2320 goto err;
2321 }
2322
2323 SET_FIELD(qp_params.modify_flags,
2324 QED_RDMA_MODIFY_QP_VALID_MAX_RD_ATOMIC_RESP, 1);
2325 qp_params.max_rd_atomic_resp = attr->max_dest_rd_atomic;
2326 }
2327
2328 if (attr_mask & IB_QP_DEST_QPN) {
2329 SET_FIELD(qp_params.modify_flags,
2330 QED_ROCE_MODIFY_QP_VALID_DEST_QP, 1);
2331
2332 qp_params.dest_qp = attr->dest_qp_num;
2333 qp->dest_qp_num = attr->dest_qp_num;
2334 }
2335
2336 cur_state = qp->state;
2337
2338
2339
2340
2341
2342
2343 if ((attr_mask & IB_QP_STATE) && qp->qp_type != IB_QPT_GSI &&
2344 !udata && qp_params.new_state == QED_ROCE_QP_STATE_ERR)
2345 qp->state = QED_ROCE_QP_STATE_ERR;
2346
2347 if (qp->qp_type != IB_QPT_GSI)
2348 rc = dev->ops->rdma_modify_qp(dev->rdma_ctx,
2349 qp->qed_qp, &qp_params);
2350
2351 if (attr_mask & IB_QP_STATE) {
2352 if ((qp->qp_type != IB_QPT_GSI) && (!udata))
2353 rc = qedr_update_qp_state(dev, qp, cur_state,
2354 qp_params.new_state);
2355 qp->state = qp_params.new_state;
2356 }
2357
2358 err:
2359 return rc;
2360 }
2361
2362 static int qedr_to_ib_qp_acc_flags(struct qed_rdma_query_qp_out_params *params)
2363 {
2364 int ib_qp_acc_flags = 0;
2365
2366 if (params->incoming_rdma_write_en)
2367 ib_qp_acc_flags |= IB_ACCESS_REMOTE_WRITE;
2368 if (params->incoming_rdma_read_en)
2369 ib_qp_acc_flags |= IB_ACCESS_REMOTE_READ;
2370 if (params->incoming_atomic_en)
2371 ib_qp_acc_flags |= IB_ACCESS_REMOTE_ATOMIC;
2372 ib_qp_acc_flags |= IB_ACCESS_LOCAL_WRITE;
2373 return ib_qp_acc_flags;
2374 }
2375
2376 int qedr_query_qp(struct ib_qp *ibqp,
2377 struct ib_qp_attr *qp_attr,
2378 int attr_mask, struct ib_qp_init_attr *qp_init_attr)
2379 {
2380 struct qed_rdma_query_qp_out_params params;
2381 struct qedr_qp *qp = get_qedr_qp(ibqp);
2382 struct qedr_dev *dev = qp->dev;
2383 int rc = 0;
2384
2385 memset(¶ms, 0, sizeof(params));
2386
2387 rc = dev->ops->rdma_query_qp(dev->rdma_ctx, qp->qed_qp, ¶ms);
2388 if (rc)
2389 goto err;
2390
2391 memset(qp_attr, 0, sizeof(*qp_attr));
2392 memset(qp_init_attr, 0, sizeof(*qp_init_attr));
2393
2394 qp_attr->qp_state = qedr_get_ibqp_state(params.state);
2395 qp_attr->cur_qp_state = qedr_get_ibqp_state(params.state);
2396 qp_attr->path_mtu = ib_mtu_int_to_enum(params.mtu);
2397 qp_attr->path_mig_state = IB_MIG_MIGRATED;
2398 qp_attr->rq_psn = params.rq_psn;
2399 qp_attr->sq_psn = params.sq_psn;
2400 qp_attr->dest_qp_num = params.dest_qp;
2401
2402 qp_attr->qp_access_flags = qedr_to_ib_qp_acc_flags(¶ms);
2403
2404 qp_attr->cap.max_send_wr = qp->sq.max_wr;
2405 qp_attr->cap.max_recv_wr = qp->rq.max_wr;
2406 qp_attr->cap.max_send_sge = qp->sq.max_sges;
2407 qp_attr->cap.max_recv_sge = qp->rq.max_sges;
2408 qp_attr->cap.max_inline_data = ROCE_REQ_MAX_INLINE_DATA_SIZE;
2409 qp_init_attr->cap = qp_attr->cap;
2410
2411 qp_attr->ah_attr.type = RDMA_AH_ATTR_TYPE_ROCE;
2412 rdma_ah_set_grh(&qp_attr->ah_attr, NULL,
2413 params.flow_label, qp->sgid_idx,
2414 params.hop_limit_ttl, params.traffic_class_tos);
2415 rdma_ah_set_dgid_raw(&qp_attr->ah_attr, ¶ms.dgid.bytes[0]);
2416 rdma_ah_set_port_num(&qp_attr->ah_attr, 1);
2417 rdma_ah_set_sl(&qp_attr->ah_attr, 0);
2418 qp_attr->timeout = params.timeout;
2419 qp_attr->rnr_retry = params.rnr_retry;
2420 qp_attr->retry_cnt = params.retry_cnt;
2421 qp_attr->min_rnr_timer = params.min_rnr_nak_timer;
2422 qp_attr->pkey_index = params.pkey_index;
2423 qp_attr->port_num = 1;
2424 rdma_ah_set_path_bits(&qp_attr->ah_attr, 0);
2425 rdma_ah_set_static_rate(&qp_attr->ah_attr, 0);
2426 qp_attr->alt_pkey_index = 0;
2427 qp_attr->alt_port_num = 0;
2428 qp_attr->alt_timeout = 0;
2429 memset(&qp_attr->alt_ah_attr, 0, sizeof(qp_attr->alt_ah_attr));
2430
2431 qp_attr->sq_draining = (params.state == QED_ROCE_QP_STATE_SQD) ? 1 : 0;
2432 qp_attr->max_dest_rd_atomic = params.max_dest_rd_atomic;
2433 qp_attr->max_rd_atomic = params.max_rd_atomic;
2434 qp_attr->en_sqd_async_notify = (params.sqd_async) ? 1 : 0;
2435
2436 DP_DEBUG(dev, QEDR_MSG_QP, "QEDR_QUERY_QP: max_inline_data=%d\n",
2437 qp_attr->cap.max_inline_data);
2438
2439 err:
2440 return rc;
2441 }
2442
2443 static int qedr_free_qp_resources(struct qedr_dev *dev, struct qedr_qp *qp,
2444 struct ib_udata *udata)
2445 {
2446 int rc = 0;
2447
2448 if (qp->qp_type != IB_QPT_GSI) {
2449 rc = dev->ops->rdma_destroy_qp(dev->rdma_ctx, qp->qed_qp);
2450 if (rc)
2451 return rc;
2452 }
2453
2454 if (qp->create_type == QEDR_QP_CREATE_USER)
2455 qedr_cleanup_user(dev, qp);
2456 else
2457 qedr_cleanup_kernel(dev, qp);
2458
2459 return 0;
2460 }
2461
2462 int qedr_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata)
2463 {
2464 struct qedr_qp *qp = get_qedr_qp(ibqp);
2465 struct qedr_dev *dev = qp->dev;
2466 struct ib_qp_attr attr;
2467 int attr_mask = 0;
2468
2469 DP_DEBUG(dev, QEDR_MSG_QP, "destroy qp: destroying %p, qp type=%d\n",
2470 qp, qp->qp_type);
2471
2472 if (rdma_protocol_roce(&dev->ibdev, 1)) {
2473 if ((qp->state != QED_ROCE_QP_STATE_RESET) &&
2474 (qp->state != QED_ROCE_QP_STATE_ERR) &&
2475 (qp->state != QED_ROCE_QP_STATE_INIT)) {
2476
2477 attr.qp_state = IB_QPS_ERR;
2478 attr_mask |= IB_QP_STATE;
2479
2480
2481 qedr_modify_qp(ibqp, &attr, attr_mask, NULL);
2482 }
2483 } else {
2484
2485
2486
2487
2488 if (test_and_set_bit(QEDR_IWARP_CM_WAIT_FOR_CONNECT,
2489 &qp->iwarp_cm_flags))
2490 wait_for_completion(&qp->iwarp_cm_comp);
2491
2492
2493
2494
2495
2496
2497
2498
2499
2500
2501
2502 if (test_and_set_bit(QEDR_IWARP_CM_WAIT_FOR_DISCONNECT,
2503 &qp->iwarp_cm_flags))
2504 wait_for_completion(&qp->iwarp_cm_comp);
2505 }
2506
2507 if (qp->qp_type == IB_QPT_GSI)
2508 qedr_destroy_gsi_qp(dev);
2509
2510
2511
2512
2513
2514 if (rdma_protocol_iwarp(&dev->ibdev, 1))
2515 xa_erase(&dev->qps, qp->qp_id);
2516
2517 qedr_free_qp_resources(dev, qp, udata);
2518
2519 if (rdma_protocol_iwarp(&dev->ibdev, 1))
2520 qedr_iw_qp_rem_ref(&qp->ibqp);
2521
2522 return 0;
2523 }
2524
2525 int qedr_create_ah(struct ib_ah *ibah, struct rdma_ah_attr *attr, u32 flags,
2526 struct ib_udata *udata)
2527 {
2528 struct qedr_ah *ah = get_qedr_ah(ibah);
2529
2530 rdma_copy_ah_attr(&ah->attr, attr);
2531
2532 return 0;
2533 }
2534
2535 void qedr_destroy_ah(struct ib_ah *ibah, u32 flags)
2536 {
2537 struct qedr_ah *ah = get_qedr_ah(ibah);
2538
2539 rdma_destroy_ah_attr(&ah->attr);
2540 }
2541
2542 static void free_mr_info(struct qedr_dev *dev, struct mr_info *info)
2543 {
2544 struct qedr_pbl *pbl, *tmp;
2545
2546 if (info->pbl_table)
2547 list_add_tail(&info->pbl_table->list_entry,
2548 &info->free_pbl_list);
2549
2550 if (!list_empty(&info->inuse_pbl_list))
2551 list_splice(&info->inuse_pbl_list, &info->free_pbl_list);
2552
2553 list_for_each_entry_safe(pbl, tmp, &info->free_pbl_list, list_entry) {
2554 list_del(&pbl->list_entry);
2555 qedr_free_pbl(dev, &info->pbl_info, pbl);
2556 }
2557 }
2558
2559 static int init_mr_info(struct qedr_dev *dev, struct mr_info *info,
2560 size_t page_list_len, bool two_layered)
2561 {
2562 struct qedr_pbl *tmp;
2563 int rc;
2564
2565 INIT_LIST_HEAD(&info->free_pbl_list);
2566 INIT_LIST_HEAD(&info->inuse_pbl_list);
2567
2568 rc = qedr_prepare_pbl_tbl(dev, &info->pbl_info,
2569 page_list_len, two_layered);
2570 if (rc)
2571 goto done;
2572
2573 info->pbl_table = qedr_alloc_pbl_tbl(dev, &info->pbl_info, GFP_KERNEL);
2574 if (IS_ERR(info->pbl_table)) {
2575 rc = PTR_ERR(info->pbl_table);
2576 goto done;
2577 }
2578
2579 DP_DEBUG(dev, QEDR_MSG_MR, "pbl_table_pa = %pa\n",
2580 &info->pbl_table->pa);
2581
2582
2583
2584
2585 tmp = qedr_alloc_pbl_tbl(dev, &info->pbl_info, GFP_KERNEL);
2586 if (IS_ERR(tmp)) {
2587 DP_DEBUG(dev, QEDR_MSG_MR, "Extra PBL is not allocated\n");
2588 goto done;
2589 }
2590
2591 list_add_tail(&tmp->list_entry, &info->free_pbl_list);
2592
2593 DP_DEBUG(dev, QEDR_MSG_MR, "extra pbl_table_pa = %pa\n", &tmp->pa);
2594
2595 done:
2596 if (rc)
2597 free_mr_info(dev, info);
2598
2599 return rc;
2600 }
2601
2602 struct ib_mr *qedr_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 len,
2603 u64 usr_addr, int acc, struct ib_udata *udata)
2604 {
2605 struct qedr_dev *dev = get_qedr_dev(ibpd->device);
2606 struct qedr_mr *mr;
2607 struct qedr_pd *pd;
2608 int rc = -ENOMEM;
2609
2610 pd = get_qedr_pd(ibpd);
2611 DP_DEBUG(dev, QEDR_MSG_MR,
2612 "qedr_register user mr pd = %d start = %lld, len = %lld, usr_addr = %lld, acc = %d\n",
2613 pd->pd_id, start, len, usr_addr, acc);
2614
2615 if (acc & IB_ACCESS_REMOTE_WRITE && !(acc & IB_ACCESS_LOCAL_WRITE))
2616 return ERR_PTR(-EINVAL);
2617
2618 mr = kzalloc(sizeof(*mr), GFP_KERNEL);
2619 if (!mr)
2620 return ERR_PTR(rc);
2621
2622 mr->type = QEDR_MR_USER;
2623
2624 mr->umem = ib_umem_get(udata, start, len, acc, 0);
2625 if (IS_ERR(mr->umem)) {
2626 rc = -EFAULT;
2627 goto err0;
2628 }
2629
2630 rc = init_mr_info(dev, &mr->info, ib_umem_page_count(mr->umem), 1);
2631 if (rc)
2632 goto err1;
2633
2634 qedr_populate_pbls(dev, mr->umem, mr->info.pbl_table,
2635 &mr->info.pbl_info, PAGE_SHIFT);
2636
2637 rc = dev->ops->rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid);
2638 if (rc) {
2639 DP_ERR(dev, "roce alloc tid returned an error %d\n", rc);
2640 goto err1;
2641 }
2642
2643
2644 mr->hw_mr.tid_type = QED_RDMA_TID_REGISTERED_MR;
2645 mr->hw_mr.key = 0;
2646 mr->hw_mr.pd = pd->pd_id;
2647 mr->hw_mr.local_read = 1;
2648 mr->hw_mr.local_write = (acc & IB_ACCESS_LOCAL_WRITE) ? 1 : 0;
2649 mr->hw_mr.remote_read = (acc & IB_ACCESS_REMOTE_READ) ? 1 : 0;
2650 mr->hw_mr.remote_write = (acc & IB_ACCESS_REMOTE_WRITE) ? 1 : 0;
2651 mr->hw_mr.remote_atomic = (acc & IB_ACCESS_REMOTE_ATOMIC) ? 1 : 0;
2652 mr->hw_mr.mw_bind = false;
2653 mr->hw_mr.pbl_ptr = mr->info.pbl_table[0].pa;
2654 mr->hw_mr.pbl_two_level = mr->info.pbl_info.two_layered;
2655 mr->hw_mr.pbl_page_size_log = ilog2(mr->info.pbl_info.pbl_size);
2656 mr->hw_mr.page_size_log = PAGE_SHIFT;
2657 mr->hw_mr.fbo = ib_umem_offset(mr->umem);
2658 mr->hw_mr.length = len;
2659 mr->hw_mr.vaddr = usr_addr;
2660 mr->hw_mr.zbva = false;
2661 mr->hw_mr.phy_mr = false;
2662 mr->hw_mr.dma_mr = false;
2663
2664 rc = dev->ops->rdma_register_tid(dev->rdma_ctx, &mr->hw_mr);
2665 if (rc) {
2666 DP_ERR(dev, "roce register tid returned an error %d\n", rc);
2667 goto err2;
2668 }
2669
2670 mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
2671 if (mr->hw_mr.remote_write || mr->hw_mr.remote_read ||
2672 mr->hw_mr.remote_atomic)
2673 mr->ibmr.rkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
2674
2675 DP_DEBUG(dev, QEDR_MSG_MR, "register user mr lkey: %x\n",
2676 mr->ibmr.lkey);
2677 return &mr->ibmr;
2678
2679 err2:
2680 dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid);
2681 err1:
2682 qedr_free_pbl(dev, &mr->info.pbl_info, mr->info.pbl_table);
2683 err0:
2684 kfree(mr);
2685 return ERR_PTR(rc);
2686 }
2687
2688 int qedr_dereg_mr(struct ib_mr *ib_mr, struct ib_udata *udata)
2689 {
2690 struct qedr_mr *mr = get_qedr_mr(ib_mr);
2691 struct qedr_dev *dev = get_qedr_dev(ib_mr->device);
2692 int rc = 0;
2693
2694 rc = dev->ops->rdma_deregister_tid(dev->rdma_ctx, mr->hw_mr.itid);
2695 if (rc)
2696 return rc;
2697
2698 dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid);
2699
2700 if (mr->type != QEDR_MR_DMA)
2701 free_mr_info(dev, &mr->info);
2702
2703
2704 ib_umem_release(mr->umem);
2705
2706 kfree(mr);
2707
2708 return rc;
2709 }
2710
2711 static struct qedr_mr *__qedr_alloc_mr(struct ib_pd *ibpd,
2712 int max_page_list_len)
2713 {
2714 struct qedr_pd *pd = get_qedr_pd(ibpd);
2715 struct qedr_dev *dev = get_qedr_dev(ibpd->device);
2716 struct qedr_mr *mr;
2717 int rc = -ENOMEM;
2718
2719 DP_DEBUG(dev, QEDR_MSG_MR,
2720 "qedr_alloc_frmr pd = %d max_page_list_len= %d\n", pd->pd_id,
2721 max_page_list_len);
2722
2723 mr = kzalloc(sizeof(*mr), GFP_KERNEL);
2724 if (!mr)
2725 return ERR_PTR(rc);
2726
2727 mr->dev = dev;
2728 mr->type = QEDR_MR_FRMR;
2729
2730 rc = init_mr_info(dev, &mr->info, max_page_list_len, 1);
2731 if (rc)
2732 goto err0;
2733
2734 rc = dev->ops->rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid);
2735 if (rc) {
2736 DP_ERR(dev, "roce alloc tid returned an error %d\n", rc);
2737 goto err0;
2738 }
2739
2740
2741 mr->hw_mr.tid_type = QED_RDMA_TID_FMR;
2742 mr->hw_mr.key = 0;
2743 mr->hw_mr.pd = pd->pd_id;
2744 mr->hw_mr.local_read = 1;
2745 mr->hw_mr.local_write = 0;
2746 mr->hw_mr.remote_read = 0;
2747 mr->hw_mr.remote_write = 0;
2748 mr->hw_mr.remote_atomic = 0;
2749 mr->hw_mr.mw_bind = false;
2750 mr->hw_mr.pbl_ptr = 0;
2751 mr->hw_mr.pbl_two_level = mr->info.pbl_info.two_layered;
2752 mr->hw_mr.pbl_page_size_log = ilog2(mr->info.pbl_info.pbl_size);
2753 mr->hw_mr.fbo = 0;
2754 mr->hw_mr.length = 0;
2755 mr->hw_mr.vaddr = 0;
2756 mr->hw_mr.zbva = false;
2757 mr->hw_mr.phy_mr = true;
2758 mr->hw_mr.dma_mr = false;
2759
2760 rc = dev->ops->rdma_register_tid(dev->rdma_ctx, &mr->hw_mr);
2761 if (rc) {
2762 DP_ERR(dev, "roce register tid returned an error %d\n", rc);
2763 goto err1;
2764 }
2765
2766 mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
2767 mr->ibmr.rkey = mr->ibmr.lkey;
2768
2769 DP_DEBUG(dev, QEDR_MSG_MR, "alloc frmr: %x\n", mr->ibmr.lkey);
2770 return mr;
2771
2772 err1:
2773 dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid);
2774 err0:
2775 kfree(mr);
2776 return ERR_PTR(rc);
2777 }
2778
2779 struct ib_mr *qedr_alloc_mr(struct ib_pd *ibpd, enum ib_mr_type mr_type,
2780 u32 max_num_sg, struct ib_udata *udata)
2781 {
2782 struct qedr_mr *mr;
2783
2784 if (mr_type != IB_MR_TYPE_MEM_REG)
2785 return ERR_PTR(-EINVAL);
2786
2787 mr = __qedr_alloc_mr(ibpd, max_num_sg);
2788
2789 if (IS_ERR(mr))
2790 return ERR_PTR(-EINVAL);
2791
2792 return &mr->ibmr;
2793 }
2794
2795 static int qedr_set_page(struct ib_mr *ibmr, u64 addr)
2796 {
2797 struct qedr_mr *mr = get_qedr_mr(ibmr);
2798 struct qedr_pbl *pbl_table;
2799 struct regpair *pbe;
2800 u32 pbes_in_page;
2801
2802 if (unlikely(mr->npages == mr->info.pbl_info.num_pbes)) {
2803 DP_ERR(mr->dev, "qedr_set_page fails when %d\n", mr->npages);
2804 return -ENOMEM;
2805 }
2806
2807 DP_DEBUG(mr->dev, QEDR_MSG_MR, "qedr_set_page pages[%d] = 0x%llx\n",
2808 mr->npages, addr);
2809
2810 pbes_in_page = mr->info.pbl_info.pbl_size / sizeof(u64);
2811 pbl_table = mr->info.pbl_table + (mr->npages / pbes_in_page);
2812 pbe = (struct regpair *)pbl_table->va;
2813 pbe += mr->npages % pbes_in_page;
2814 pbe->lo = cpu_to_le32((u32)addr);
2815 pbe->hi = cpu_to_le32((u32)upper_32_bits(addr));
2816
2817 mr->npages++;
2818
2819 return 0;
2820 }
2821
2822 static void handle_completed_mrs(struct qedr_dev *dev, struct mr_info *info)
2823 {
2824 int work = info->completed - info->completed_handled - 1;
2825
2826 DP_DEBUG(dev, QEDR_MSG_MR, "Special FMR work = %d\n", work);
2827 while (work-- > 0 && !list_empty(&info->inuse_pbl_list)) {
2828 struct qedr_pbl *pbl;
2829
2830
2831
2832
2833
2834
2835 pbl = list_first_entry(&info->inuse_pbl_list,
2836 struct qedr_pbl, list_entry);
2837 list_move_tail(&pbl->list_entry, &info->free_pbl_list);
2838 info->completed_handled++;
2839 }
2840 }
2841
2842 int qedr_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg,
2843 int sg_nents, unsigned int *sg_offset)
2844 {
2845 struct qedr_mr *mr = get_qedr_mr(ibmr);
2846
2847 mr->npages = 0;
2848
2849 handle_completed_mrs(mr->dev, &mr->info);
2850 return ib_sg_to_pages(ibmr, sg, sg_nents, NULL, qedr_set_page);
2851 }
2852
2853 struct ib_mr *qedr_get_dma_mr(struct ib_pd *ibpd, int acc)
2854 {
2855 struct qedr_dev *dev = get_qedr_dev(ibpd->device);
2856 struct qedr_pd *pd = get_qedr_pd(ibpd);
2857 struct qedr_mr *mr;
2858 int rc;
2859
2860 mr = kzalloc(sizeof(*mr), GFP_KERNEL);
2861 if (!mr)
2862 return ERR_PTR(-ENOMEM);
2863
2864 mr->type = QEDR_MR_DMA;
2865
2866 rc = dev->ops->rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid);
2867 if (rc) {
2868 DP_ERR(dev, "roce alloc tid returned an error %d\n", rc);
2869 goto err1;
2870 }
2871
2872
2873 mr->hw_mr.tid_type = QED_RDMA_TID_REGISTERED_MR;
2874 mr->hw_mr.pd = pd->pd_id;
2875 mr->hw_mr.local_read = 1;
2876 mr->hw_mr.local_write = (acc & IB_ACCESS_LOCAL_WRITE) ? 1 : 0;
2877 mr->hw_mr.remote_read = (acc & IB_ACCESS_REMOTE_READ) ? 1 : 0;
2878 mr->hw_mr.remote_write = (acc & IB_ACCESS_REMOTE_WRITE) ? 1 : 0;
2879 mr->hw_mr.remote_atomic = (acc & IB_ACCESS_REMOTE_ATOMIC) ? 1 : 0;
2880 mr->hw_mr.dma_mr = true;
2881
2882 rc = dev->ops->rdma_register_tid(dev->rdma_ctx, &mr->hw_mr);
2883 if (rc) {
2884 DP_ERR(dev, "roce register tid returned an error %d\n", rc);
2885 goto err2;
2886 }
2887
2888 mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
2889 if (mr->hw_mr.remote_write || mr->hw_mr.remote_read ||
2890 mr->hw_mr.remote_atomic)
2891 mr->ibmr.rkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
2892
2893 DP_DEBUG(dev, QEDR_MSG_MR, "get dma mr: lkey = %x\n", mr->ibmr.lkey);
2894 return &mr->ibmr;
2895
2896 err2:
2897 dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid);
2898 err1:
2899 kfree(mr);
2900 return ERR_PTR(rc);
2901 }
2902
2903 static inline int qedr_wq_is_full(struct qedr_qp_hwq_info *wq)
2904 {
2905 return (((wq->prod + 1) % wq->max_wr) == wq->cons);
2906 }
2907
2908 static int sge_data_len(struct ib_sge *sg_list, int num_sge)
2909 {
2910 int i, len = 0;
2911
2912 for (i = 0; i < num_sge; i++)
2913 len += sg_list[i].length;
2914
2915 return len;
2916 }
2917
2918 static void swap_wqe_data64(u64 *p)
2919 {
2920 int i;
2921
2922 for (i = 0; i < QEDR_SQE_ELEMENT_SIZE / sizeof(u64); i++, p++)
2923 *p = cpu_to_be64(cpu_to_le64(*p));
2924 }
2925
2926 static u32 qedr_prepare_sq_inline_data(struct qedr_dev *dev,
2927 struct qedr_qp *qp, u8 *wqe_size,
2928 const struct ib_send_wr *wr,
2929 const struct ib_send_wr **bad_wr,
2930 u8 *bits, u8 bit)
2931 {
2932 u32 data_size = sge_data_len(wr->sg_list, wr->num_sge);
2933 char *seg_prt, *wqe;
2934 int i, seg_siz;
2935
2936 if (data_size > ROCE_REQ_MAX_INLINE_DATA_SIZE) {
2937 DP_ERR(dev, "Too much inline data in WR: %d\n", data_size);
2938 *bad_wr = wr;
2939 return 0;
2940 }
2941
2942 if (!data_size)
2943 return data_size;
2944
2945 *bits |= bit;
2946
2947 seg_prt = NULL;
2948 wqe = NULL;
2949 seg_siz = 0;
2950
2951
2952 for (i = 0; i < wr->num_sge; i++) {
2953 u32 len = wr->sg_list[i].length;
2954 void *src = (void *)(uintptr_t)wr->sg_list[i].addr;
2955
2956 while (len > 0) {
2957 u32 cur;
2958
2959
2960 if (!seg_siz) {
2961 wqe = (char *)qed_chain_produce(&qp->sq.pbl);
2962 seg_prt = wqe;
2963 seg_siz = sizeof(struct rdma_sq_common_wqe);
2964 (*wqe_size)++;
2965 }
2966
2967
2968 cur = min_t(u32, len, seg_siz);
2969 memcpy(seg_prt, src, cur);
2970
2971
2972 seg_prt += cur;
2973 seg_siz -= cur;
2974
2975
2976 src += cur;
2977 len -= cur;
2978
2979
2980 if (!seg_siz)
2981 swap_wqe_data64((u64 *)wqe);
2982 }
2983 }
2984
2985
2986 if (seg_siz)
2987 swap_wqe_data64((u64 *)wqe);
2988
2989 return data_size;
2990 }
2991
2992 #define RQ_SGE_SET(sge, vaddr, vlength, vflags) \
2993 do { \
2994 DMA_REGPAIR_LE(sge->addr, vaddr); \
2995 (sge)->length = cpu_to_le32(vlength); \
2996 (sge)->flags = cpu_to_le32(vflags); \
2997 } while (0)
2998
2999 #define SRQ_HDR_SET(hdr, vwr_id, num_sge) \
3000 do { \
3001 DMA_REGPAIR_LE(hdr->wr_id, vwr_id); \
3002 (hdr)->num_sges = num_sge; \
3003 } while (0)
3004
3005 #define SRQ_SGE_SET(sge, vaddr, vlength, vlkey) \
3006 do { \
3007 DMA_REGPAIR_LE(sge->addr, vaddr); \
3008 (sge)->length = cpu_to_le32(vlength); \
3009 (sge)->l_key = cpu_to_le32(vlkey); \
3010 } while (0)
3011
3012 static u32 qedr_prepare_sq_sges(struct qedr_qp *qp, u8 *wqe_size,
3013 const struct ib_send_wr *wr)
3014 {
3015 u32 data_size = 0;
3016 int i;
3017
3018 for (i = 0; i < wr->num_sge; i++) {
3019 struct rdma_sq_sge *sge = qed_chain_produce(&qp->sq.pbl);
3020
3021 DMA_REGPAIR_LE(sge->addr, wr->sg_list[i].addr);
3022 sge->l_key = cpu_to_le32(wr->sg_list[i].lkey);
3023 sge->length = cpu_to_le32(wr->sg_list[i].length);
3024 data_size += wr->sg_list[i].length;
3025 }
3026
3027 if (wqe_size)
3028 *wqe_size += wr->num_sge;
3029
3030 return data_size;
3031 }
3032
3033 static u32 qedr_prepare_sq_rdma_data(struct qedr_dev *dev,
3034 struct qedr_qp *qp,
3035 struct rdma_sq_rdma_wqe_1st *rwqe,
3036 struct rdma_sq_rdma_wqe_2nd *rwqe2,
3037 const struct ib_send_wr *wr,
3038 const struct ib_send_wr **bad_wr)
3039 {
3040 rwqe2->r_key = cpu_to_le32(rdma_wr(wr)->rkey);
3041 DMA_REGPAIR_LE(rwqe2->remote_va, rdma_wr(wr)->remote_addr);
3042
3043 if (wr->send_flags & IB_SEND_INLINE &&
3044 (wr->opcode == IB_WR_RDMA_WRITE_WITH_IMM ||
3045 wr->opcode == IB_WR_RDMA_WRITE)) {
3046 u8 flags = 0;
3047
3048 SET_FIELD2(flags, RDMA_SQ_RDMA_WQE_1ST_INLINE_FLG, 1);
3049 return qedr_prepare_sq_inline_data(dev, qp, &rwqe->wqe_size, wr,
3050 bad_wr, &rwqe->flags, flags);
3051 }
3052
3053 return qedr_prepare_sq_sges(qp, &rwqe->wqe_size, wr);
3054 }
3055
3056 static u32 qedr_prepare_sq_send_data(struct qedr_dev *dev,
3057 struct qedr_qp *qp,
3058 struct rdma_sq_send_wqe_1st *swqe,
3059 struct rdma_sq_send_wqe_2st *swqe2,
3060 const struct ib_send_wr *wr,
3061 const struct ib_send_wr **bad_wr)
3062 {
3063 memset(swqe2, 0, sizeof(*swqe2));
3064 if (wr->send_flags & IB_SEND_INLINE) {
3065 u8 flags = 0;
3066
3067 SET_FIELD2(flags, RDMA_SQ_SEND_WQE_INLINE_FLG, 1);
3068 return qedr_prepare_sq_inline_data(dev, qp, &swqe->wqe_size, wr,
3069 bad_wr, &swqe->flags, flags);
3070 }
3071
3072 return qedr_prepare_sq_sges(qp, &swqe->wqe_size, wr);
3073 }
3074
3075 static int qedr_prepare_reg(struct qedr_qp *qp,
3076 struct rdma_sq_fmr_wqe_1st *fwqe1,
3077 const struct ib_reg_wr *wr)
3078 {
3079 struct qedr_mr *mr = get_qedr_mr(wr->mr);
3080 struct rdma_sq_fmr_wqe_2nd *fwqe2;
3081
3082 fwqe2 = (struct rdma_sq_fmr_wqe_2nd *)qed_chain_produce(&qp->sq.pbl);
3083 fwqe1->addr.hi = upper_32_bits(mr->ibmr.iova);
3084 fwqe1->addr.lo = lower_32_bits(mr->ibmr.iova);
3085 fwqe1->l_key = wr->key;
3086
3087 fwqe2->access_ctrl = 0;
3088
3089 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_REMOTE_READ,
3090 !!(wr->access & IB_ACCESS_REMOTE_READ));
3091 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_REMOTE_WRITE,
3092 !!(wr->access & IB_ACCESS_REMOTE_WRITE));
3093 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_ENABLE_ATOMIC,
3094 !!(wr->access & IB_ACCESS_REMOTE_ATOMIC));
3095 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_LOCAL_READ, 1);
3096 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_LOCAL_WRITE,
3097 !!(wr->access & IB_ACCESS_LOCAL_WRITE));
3098 fwqe2->fmr_ctrl = 0;
3099
3100 SET_FIELD2(fwqe2->fmr_ctrl, RDMA_SQ_FMR_WQE_2ND_PAGE_SIZE_LOG,
3101 ilog2(mr->ibmr.page_size) - 12);
3102
3103 fwqe2->length_hi = 0;
3104 fwqe2->length_lo = mr->ibmr.length;
3105 fwqe2->pbl_addr.hi = upper_32_bits(mr->info.pbl_table->pa);
3106 fwqe2->pbl_addr.lo = lower_32_bits(mr->info.pbl_table->pa);
3107
3108 qp->wqe_wr_id[qp->sq.prod].mr = mr;
3109
3110 return 0;
3111 }
3112
3113 static enum ib_wc_opcode qedr_ib_to_wc_opcode(enum ib_wr_opcode opcode)
3114 {
3115 switch (opcode) {
3116 case IB_WR_RDMA_WRITE:
3117 case IB_WR_RDMA_WRITE_WITH_IMM:
3118 return IB_WC_RDMA_WRITE;
3119 case IB_WR_SEND_WITH_IMM:
3120 case IB_WR_SEND:
3121 case IB_WR_SEND_WITH_INV:
3122 return IB_WC_SEND;
3123 case IB_WR_RDMA_READ:
3124 case IB_WR_RDMA_READ_WITH_INV:
3125 return IB_WC_RDMA_READ;
3126 case IB_WR_ATOMIC_CMP_AND_SWP:
3127 return IB_WC_COMP_SWAP;
3128 case IB_WR_ATOMIC_FETCH_AND_ADD:
3129 return IB_WC_FETCH_ADD;
3130 case IB_WR_REG_MR:
3131 return IB_WC_REG_MR;
3132 case IB_WR_LOCAL_INV:
3133 return IB_WC_LOCAL_INV;
3134 default:
3135 return IB_WC_SEND;
3136 }
3137 }
3138
3139 static inline bool qedr_can_post_send(struct qedr_qp *qp,
3140 const struct ib_send_wr *wr)
3141 {
3142 int wq_is_full, err_wr, pbl_is_full;
3143 struct qedr_dev *dev = qp->dev;
3144
3145
3146 err_wr = wr->num_sge > qp->sq.max_sges;
3147 wq_is_full = qedr_wq_is_full(&qp->sq);
3148 pbl_is_full = qed_chain_get_elem_left_u32(&qp->sq.pbl) <
3149 QEDR_MAX_SQE_ELEMENTS_PER_SQE;
3150 if (wq_is_full || err_wr || pbl_is_full) {
3151 if (wq_is_full && !(qp->err_bitmap & QEDR_QP_ERR_SQ_FULL)) {
3152 DP_ERR(dev,
3153 "error: WQ is full. Post send on QP %p failed (this error appears only once)\n",
3154 qp);
3155 qp->err_bitmap |= QEDR_QP_ERR_SQ_FULL;
3156 }
3157
3158 if (err_wr && !(qp->err_bitmap & QEDR_QP_ERR_BAD_SR)) {
3159 DP_ERR(dev,
3160 "error: WR is bad. Post send on QP %p failed (this error appears only once)\n",
3161 qp);
3162 qp->err_bitmap |= QEDR_QP_ERR_BAD_SR;
3163 }
3164
3165 if (pbl_is_full &&
3166 !(qp->err_bitmap & QEDR_QP_ERR_SQ_PBL_FULL)) {
3167 DP_ERR(dev,
3168 "error: WQ PBL is full. Post send on QP %p failed (this error appears only once)\n",
3169 qp);
3170 qp->err_bitmap |= QEDR_QP_ERR_SQ_PBL_FULL;
3171 }
3172 return false;
3173 }
3174 return true;
3175 }
3176
3177 static int __qedr_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr,
3178 const struct ib_send_wr **bad_wr)
3179 {
3180 struct qedr_dev *dev = get_qedr_dev(ibqp->device);
3181 struct qedr_qp *qp = get_qedr_qp(ibqp);
3182 struct rdma_sq_atomic_wqe_1st *awqe1;
3183 struct rdma_sq_atomic_wqe_2nd *awqe2;
3184 struct rdma_sq_atomic_wqe_3rd *awqe3;
3185 struct rdma_sq_send_wqe_2st *swqe2;
3186 struct rdma_sq_local_inv_wqe *iwqe;
3187 struct rdma_sq_rdma_wqe_2nd *rwqe2;
3188 struct rdma_sq_send_wqe_1st *swqe;
3189 struct rdma_sq_rdma_wqe_1st *rwqe;
3190 struct rdma_sq_fmr_wqe_1st *fwqe1;
3191 struct rdma_sq_common_wqe *wqe;
3192 u32 length;
3193 int rc = 0;
3194 bool comp;
3195
3196 if (!qedr_can_post_send(qp, wr)) {
3197 *bad_wr = wr;
3198 return -ENOMEM;
3199 }
3200
3201 wqe = qed_chain_produce(&qp->sq.pbl);
3202 qp->wqe_wr_id[qp->sq.prod].signaled =
3203 !!(wr->send_flags & IB_SEND_SIGNALED) || qp->signaled;
3204
3205 wqe->flags = 0;
3206 SET_FIELD2(wqe->flags, RDMA_SQ_SEND_WQE_SE_FLG,
3207 !!(wr->send_flags & IB_SEND_SOLICITED));
3208 comp = (!!(wr->send_flags & IB_SEND_SIGNALED)) || qp->signaled;
3209 SET_FIELD2(wqe->flags, RDMA_SQ_SEND_WQE_COMP_FLG, comp);
3210 SET_FIELD2(wqe->flags, RDMA_SQ_SEND_WQE_RD_FENCE_FLG,
3211 !!(wr->send_flags & IB_SEND_FENCE));
3212 wqe->prev_wqe_size = qp->prev_wqe_size;
3213
3214 qp->wqe_wr_id[qp->sq.prod].opcode = qedr_ib_to_wc_opcode(wr->opcode);
3215
3216 switch (wr->opcode) {
3217 case IB_WR_SEND_WITH_IMM:
3218 if (unlikely(rdma_protocol_iwarp(&dev->ibdev, 1))) {
3219 rc = -EINVAL;
3220 *bad_wr = wr;
3221 break;
3222 }
3223 wqe->req_type = RDMA_SQ_REQ_TYPE_SEND_WITH_IMM;
3224 swqe = (struct rdma_sq_send_wqe_1st *)wqe;
3225 swqe->wqe_size = 2;
3226 swqe2 = qed_chain_produce(&qp->sq.pbl);
3227
3228 swqe->inv_key_or_imm_data = cpu_to_le32(be32_to_cpu(wr->ex.imm_data));
3229 length = qedr_prepare_sq_send_data(dev, qp, swqe, swqe2,
3230 wr, bad_wr);
3231 swqe->length = cpu_to_le32(length);
3232 qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size;
3233 qp->prev_wqe_size = swqe->wqe_size;
3234 qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length;
3235 break;
3236 case IB_WR_SEND:
3237 wqe->req_type = RDMA_SQ_REQ_TYPE_SEND;
3238 swqe = (struct rdma_sq_send_wqe_1st *)wqe;
3239
3240 swqe->wqe_size = 2;
3241 swqe2 = qed_chain_produce(&qp->sq.pbl);
3242 length = qedr_prepare_sq_send_data(dev, qp, swqe, swqe2,
3243 wr, bad_wr);
3244 swqe->length = cpu_to_le32(length);
3245 qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size;
3246 qp->prev_wqe_size = swqe->wqe_size;
3247 qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length;
3248 break;
3249 case IB_WR_SEND_WITH_INV:
3250 wqe->req_type = RDMA_SQ_REQ_TYPE_SEND_WITH_INVALIDATE;
3251 swqe = (struct rdma_sq_send_wqe_1st *)wqe;
3252 swqe2 = qed_chain_produce(&qp->sq.pbl);
3253 swqe->wqe_size = 2;
3254 swqe->inv_key_or_imm_data = cpu_to_le32(wr->ex.invalidate_rkey);
3255 length = qedr_prepare_sq_send_data(dev, qp, swqe, swqe2,
3256 wr, bad_wr);
3257 swqe->length = cpu_to_le32(length);
3258 qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size;
3259 qp->prev_wqe_size = swqe->wqe_size;
3260 qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length;
3261 break;
3262
3263 case IB_WR_RDMA_WRITE_WITH_IMM:
3264 if (unlikely(rdma_protocol_iwarp(&dev->ibdev, 1))) {
3265 rc = -EINVAL;
3266 *bad_wr = wr;
3267 break;
3268 }
3269 wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_WR_WITH_IMM;
3270 rwqe = (struct rdma_sq_rdma_wqe_1st *)wqe;
3271
3272 rwqe->wqe_size = 2;
3273 rwqe->imm_data = htonl(cpu_to_le32(wr->ex.imm_data));
3274 rwqe2 = qed_chain_produce(&qp->sq.pbl);
3275 length = qedr_prepare_sq_rdma_data(dev, qp, rwqe, rwqe2,
3276 wr, bad_wr);
3277 rwqe->length = cpu_to_le32(length);
3278 qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size;
3279 qp->prev_wqe_size = rwqe->wqe_size;
3280 qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length;
3281 break;
3282 case IB_WR_RDMA_WRITE:
3283 wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_WR;
3284 rwqe = (struct rdma_sq_rdma_wqe_1st *)wqe;
3285
3286 rwqe->wqe_size = 2;
3287 rwqe2 = qed_chain_produce(&qp->sq.pbl);
3288 length = qedr_prepare_sq_rdma_data(dev, qp, rwqe, rwqe2,
3289 wr, bad_wr);
3290 rwqe->length = cpu_to_le32(length);
3291 qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size;
3292 qp->prev_wqe_size = rwqe->wqe_size;
3293 qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length;
3294 break;
3295 case IB_WR_RDMA_READ_WITH_INV:
3296 SET_FIELD2(wqe->flags, RDMA_SQ_RDMA_WQE_1ST_READ_INV_FLG, 1);
3297
3298
3299 case IB_WR_RDMA_READ:
3300 wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_RD;
3301 rwqe = (struct rdma_sq_rdma_wqe_1st *)wqe;
3302
3303 rwqe->wqe_size = 2;
3304 rwqe2 = qed_chain_produce(&qp->sq.pbl);
3305 length = qedr_prepare_sq_rdma_data(dev, qp, rwqe, rwqe2,
3306 wr, bad_wr);
3307 rwqe->length = cpu_to_le32(length);
3308 qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size;
3309 qp->prev_wqe_size = rwqe->wqe_size;
3310 qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length;
3311 break;
3312
3313 case IB_WR_ATOMIC_CMP_AND_SWP:
3314 case IB_WR_ATOMIC_FETCH_AND_ADD:
3315 awqe1 = (struct rdma_sq_atomic_wqe_1st *)wqe;
3316 awqe1->wqe_size = 4;
3317
3318 awqe2 = qed_chain_produce(&qp->sq.pbl);
3319 DMA_REGPAIR_LE(awqe2->remote_va, atomic_wr(wr)->remote_addr);
3320 awqe2->r_key = cpu_to_le32(atomic_wr(wr)->rkey);
3321
3322 awqe3 = qed_chain_produce(&qp->sq.pbl);
3323
3324 if (wr->opcode == IB_WR_ATOMIC_FETCH_AND_ADD) {
3325 wqe->req_type = RDMA_SQ_REQ_TYPE_ATOMIC_ADD;
3326 DMA_REGPAIR_LE(awqe3->swap_data,
3327 atomic_wr(wr)->compare_add);
3328 } else {
3329 wqe->req_type = RDMA_SQ_REQ_TYPE_ATOMIC_CMP_AND_SWAP;
3330 DMA_REGPAIR_LE(awqe3->swap_data,
3331 atomic_wr(wr)->swap);
3332 DMA_REGPAIR_LE(awqe3->cmp_data,
3333 atomic_wr(wr)->compare_add);
3334 }
3335
3336 qedr_prepare_sq_sges(qp, NULL, wr);
3337
3338 qp->wqe_wr_id[qp->sq.prod].wqe_size = awqe1->wqe_size;
3339 qp->prev_wqe_size = awqe1->wqe_size;
3340 break;
3341
3342 case IB_WR_LOCAL_INV:
3343 iwqe = (struct rdma_sq_local_inv_wqe *)wqe;
3344 iwqe->wqe_size = 1;
3345
3346 iwqe->req_type = RDMA_SQ_REQ_TYPE_LOCAL_INVALIDATE;
3347 iwqe->inv_l_key = wr->ex.invalidate_rkey;
3348 qp->wqe_wr_id[qp->sq.prod].wqe_size = iwqe->wqe_size;
3349 qp->prev_wqe_size = iwqe->wqe_size;
3350 break;
3351 case IB_WR_REG_MR:
3352 DP_DEBUG(dev, QEDR_MSG_CQ, "REG_MR\n");
3353 wqe->req_type = RDMA_SQ_REQ_TYPE_FAST_MR;
3354 fwqe1 = (struct rdma_sq_fmr_wqe_1st *)wqe;
3355 fwqe1->wqe_size = 2;
3356
3357 rc = qedr_prepare_reg(qp, fwqe1, reg_wr(wr));
3358 if (rc) {
3359 DP_ERR(dev, "IB_REG_MR failed rc=%d\n", rc);
3360 *bad_wr = wr;
3361 break;
3362 }
3363
3364 qp->wqe_wr_id[qp->sq.prod].wqe_size = fwqe1->wqe_size;
3365 qp->prev_wqe_size = fwqe1->wqe_size;
3366 break;
3367 default:
3368 DP_ERR(dev, "invalid opcode 0x%x!\n", wr->opcode);
3369 rc = -EINVAL;
3370 *bad_wr = wr;
3371 break;
3372 }
3373
3374 if (*bad_wr) {
3375 u16 value;
3376
3377
3378
3379
3380 value = le16_to_cpu(qp->sq.db_data.data.value);
3381 qed_chain_set_prod(&qp->sq.pbl, value, wqe);
3382
3383
3384 qp->prev_wqe_size = wqe->prev_wqe_size;
3385 rc = -EINVAL;
3386 DP_ERR(dev, "POST SEND FAILED\n");
3387 }
3388
3389 return rc;
3390 }
3391
3392 int qedr_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr,
3393 const struct ib_send_wr **bad_wr)
3394 {
3395 struct qedr_dev *dev = get_qedr_dev(ibqp->device);
3396 struct qedr_qp *qp = get_qedr_qp(ibqp);
3397 unsigned long flags;
3398 int rc = 0;
3399
3400 *bad_wr = NULL;
3401
3402 if (qp->qp_type == IB_QPT_GSI)
3403 return qedr_gsi_post_send(ibqp, wr, bad_wr);
3404
3405 spin_lock_irqsave(&qp->q_lock, flags);
3406
3407 if (rdma_protocol_roce(&dev->ibdev, 1)) {
3408 if ((qp->state != QED_ROCE_QP_STATE_RTS) &&
3409 (qp->state != QED_ROCE_QP_STATE_ERR) &&
3410 (qp->state != QED_ROCE_QP_STATE_SQD)) {
3411 spin_unlock_irqrestore(&qp->q_lock, flags);
3412 *bad_wr = wr;
3413 DP_DEBUG(dev, QEDR_MSG_CQ,
3414 "QP in wrong state! QP icid=0x%x state %d\n",
3415 qp->icid, qp->state);
3416 return -EINVAL;
3417 }
3418 }
3419
3420 while (wr) {
3421 rc = __qedr_post_send(ibqp, wr, bad_wr);
3422 if (rc)
3423 break;
3424
3425 qp->wqe_wr_id[qp->sq.prod].wr_id = wr->wr_id;
3426
3427 qedr_inc_sw_prod(&qp->sq);
3428
3429 qp->sq.db_data.data.value++;
3430
3431 wr = wr->next;
3432 }
3433
3434
3435
3436
3437
3438
3439
3440
3441
3442
3443
3444
3445
3446
3447 smp_wmb();
3448 writel(qp->sq.db_data.raw, qp->sq.db);
3449
3450 spin_unlock_irqrestore(&qp->q_lock, flags);
3451
3452 return rc;
3453 }
3454
3455 static u32 qedr_srq_elem_left(struct qedr_srq_hwq_info *hw_srq)
3456 {
3457 u32 used;
3458
3459
3460
3461
3462
3463 used = hw_srq->wr_prod_cnt - hw_srq->wr_cons_cnt;
3464
3465 return hw_srq->max_wr - used;
3466 }
3467
3468 int qedr_post_srq_recv(struct ib_srq *ibsrq, const struct ib_recv_wr *wr,
3469 const struct ib_recv_wr **bad_wr)
3470 {
3471 struct qedr_srq *srq = get_qedr_srq(ibsrq);
3472 struct qedr_srq_hwq_info *hw_srq;
3473 struct qedr_dev *dev = srq->dev;
3474 struct qed_chain *pbl;
3475 unsigned long flags;
3476 int status = 0;
3477 u32 num_sge;
3478 u32 offset;
3479
3480 spin_lock_irqsave(&srq->lock, flags);
3481
3482 hw_srq = &srq->hw_srq;
3483 pbl = &srq->hw_srq.pbl;
3484 while (wr) {
3485 struct rdma_srq_wqe_header *hdr;
3486 int i;
3487
3488 if (!qedr_srq_elem_left(hw_srq) ||
3489 wr->num_sge > srq->hw_srq.max_sges) {
3490 DP_ERR(dev, "Can't post WR (%d,%d) || (%d > %d)\n",
3491 hw_srq->wr_prod_cnt, hw_srq->wr_cons_cnt,
3492 wr->num_sge, srq->hw_srq.max_sges);
3493 status = -ENOMEM;
3494 *bad_wr = wr;
3495 break;
3496 }
3497
3498 hdr = qed_chain_produce(pbl);
3499 num_sge = wr->num_sge;
3500
3501 SRQ_HDR_SET(hdr, wr->wr_id, num_sge);
3502
3503 srq->hw_srq.wr_prod_cnt++;
3504 hw_srq->wqe_prod++;
3505 hw_srq->sge_prod++;
3506
3507 DP_DEBUG(dev, QEDR_MSG_SRQ,
3508 "SRQ WR: SGEs: %d with wr_id[%d] = %llx\n",
3509 wr->num_sge, hw_srq->wqe_prod, wr->wr_id);
3510
3511 for (i = 0; i < wr->num_sge; i++) {
3512 struct rdma_srq_sge *srq_sge = qed_chain_produce(pbl);
3513
3514
3515 SRQ_SGE_SET(srq_sge, wr->sg_list[i].addr,
3516 wr->sg_list[i].length, wr->sg_list[i].lkey);
3517
3518 DP_DEBUG(dev, QEDR_MSG_SRQ,
3519 "[%d]: len %d key %x addr %x:%x\n",
3520 i, srq_sge->length, srq_sge->l_key,
3521 srq_sge->addr.hi, srq_sge->addr.lo);
3522 hw_srq->sge_prod++;
3523 }
3524
3525
3526
3527
3528 wmb();
3529
3530
3531
3532
3533
3534 *srq->hw_srq.virt_prod_pair_addr = hw_srq->sge_prod;
3535 offset = offsetof(struct rdma_srq_producers, wqe_prod);
3536 *((u8 *)srq->hw_srq.virt_prod_pair_addr + offset) =
3537 hw_srq->wqe_prod;
3538
3539
3540 wmb();
3541 wr = wr->next;
3542 }
3543
3544 DP_DEBUG(dev, QEDR_MSG_SRQ, "POST: Elements in S-RQ: %d\n",
3545 qed_chain_get_elem_left(pbl));
3546 spin_unlock_irqrestore(&srq->lock, flags);
3547
3548 return status;
3549 }
3550
3551 int qedr_post_recv(struct ib_qp *ibqp, const struct ib_recv_wr *wr,
3552 const struct ib_recv_wr **bad_wr)
3553 {
3554 struct qedr_qp *qp = get_qedr_qp(ibqp);
3555 struct qedr_dev *dev = qp->dev;
3556 unsigned long flags;
3557 int status = 0;
3558
3559 if (qp->qp_type == IB_QPT_GSI)
3560 return qedr_gsi_post_recv(ibqp, wr, bad_wr);
3561
3562 spin_lock_irqsave(&qp->q_lock, flags);
3563
3564 if (qp->state == QED_ROCE_QP_STATE_RESET) {
3565 spin_unlock_irqrestore(&qp->q_lock, flags);
3566 *bad_wr = wr;
3567 return -EINVAL;
3568 }
3569
3570 while (wr) {
3571 int i;
3572
3573 if (qed_chain_get_elem_left_u32(&qp->rq.pbl) <
3574 QEDR_MAX_RQE_ELEMENTS_PER_RQE ||
3575 wr->num_sge > qp->rq.max_sges) {
3576 DP_ERR(dev, "Can't post WR (%d < %d) || (%d > %d)\n",
3577 qed_chain_get_elem_left_u32(&qp->rq.pbl),
3578 QEDR_MAX_RQE_ELEMENTS_PER_RQE, wr->num_sge,
3579 qp->rq.max_sges);
3580 status = -ENOMEM;
3581 *bad_wr = wr;
3582 break;
3583 }
3584 for (i = 0; i < wr->num_sge; i++) {
3585 u32 flags = 0;
3586 struct rdma_rq_sge *rqe =
3587 qed_chain_produce(&qp->rq.pbl);
3588
3589
3590
3591
3592 if (!i)
3593 SET_FIELD(flags, RDMA_RQ_SGE_NUM_SGES,
3594 wr->num_sge);
3595
3596 SET_FIELD(flags, RDMA_RQ_SGE_L_KEY_LO,
3597 wr->sg_list[i].lkey);
3598
3599 RQ_SGE_SET(rqe, wr->sg_list[i].addr,
3600 wr->sg_list[i].length, flags);
3601 }
3602
3603
3604
3605
3606
3607 if (!wr->num_sge) {
3608 u32 flags = 0;
3609 struct rdma_rq_sge *rqe =
3610 qed_chain_produce(&qp->rq.pbl);
3611
3612
3613
3614
3615 SET_FIELD(flags, RDMA_RQ_SGE_L_KEY_LO, 0);
3616 SET_FIELD(flags, RDMA_RQ_SGE_NUM_SGES, 1);
3617
3618 RQ_SGE_SET(rqe, 0, 0, flags);
3619 i = 1;
3620 }
3621
3622 qp->rqe_wr_id[qp->rq.prod].wr_id = wr->wr_id;
3623 qp->rqe_wr_id[qp->rq.prod].wqe_size = i;
3624
3625 qedr_inc_sw_prod(&qp->rq);
3626
3627
3628
3629
3630
3631
3632
3633
3634 smp_wmb();
3635
3636 qp->rq.db_data.data.value++;
3637
3638 writel(qp->rq.db_data.raw, qp->rq.db);
3639
3640 if (rdma_protocol_iwarp(&dev->ibdev, 1)) {
3641 writel(qp->rq.iwarp_db2_data.raw, qp->rq.iwarp_db2);
3642 }
3643
3644 wr = wr->next;
3645 }
3646
3647 spin_unlock_irqrestore(&qp->q_lock, flags);
3648
3649 return status;
3650 }
3651
3652 static int is_valid_cqe(struct qedr_cq *cq, union rdma_cqe *cqe)
3653 {
3654 struct rdma_cqe_requester *resp_cqe = &cqe->req;
3655
3656 return (resp_cqe->flags & RDMA_CQE_REQUESTER_TOGGLE_BIT_MASK) ==
3657 cq->pbl_toggle;
3658 }
3659
3660 static struct qedr_qp *cqe_get_qp(union rdma_cqe *cqe)
3661 {
3662 struct rdma_cqe_requester *resp_cqe = &cqe->req;
3663 struct qedr_qp *qp;
3664
3665 qp = (struct qedr_qp *)(uintptr_t)HILO_GEN(resp_cqe->qp_handle.hi,
3666 resp_cqe->qp_handle.lo,
3667 u64);
3668 return qp;
3669 }
3670
3671 static enum rdma_cqe_type cqe_get_type(union rdma_cqe *cqe)
3672 {
3673 struct rdma_cqe_requester *resp_cqe = &cqe->req;
3674
3675 return GET_FIELD(resp_cqe->flags, RDMA_CQE_REQUESTER_TYPE);
3676 }
3677
3678
3679 static union rdma_cqe *get_cqe(struct qedr_cq *cq)
3680 {
3681 return cq->latest_cqe;
3682 }
3683
3684
3685
3686
3687
3688
3689
3690
3691 static inline void qedr_chk_if_fmr(struct qedr_qp *qp)
3692 {
3693 if (qp->wqe_wr_id[qp->sq.cons].opcode == IB_WC_REG_MR)
3694 qp->wqe_wr_id[qp->sq.cons].mr->info.completed++;
3695 }
3696
3697 static int process_req(struct qedr_dev *dev, struct qedr_qp *qp,
3698 struct qedr_cq *cq, int num_entries,
3699 struct ib_wc *wc, u16 hw_cons, enum ib_wc_status status,
3700 int force)
3701 {
3702 u16 cnt = 0;
3703
3704 while (num_entries && qp->sq.wqe_cons != hw_cons) {
3705 if (!qp->wqe_wr_id[qp->sq.cons].signaled && !force) {
3706 qedr_chk_if_fmr(qp);
3707
3708 goto next_cqe;
3709 }
3710
3711
3712 wc->status = status;
3713 wc->vendor_err = 0;
3714 wc->wc_flags = 0;
3715 wc->src_qp = qp->id;
3716 wc->qp = &qp->ibqp;
3717
3718 wc->wr_id = qp->wqe_wr_id[qp->sq.cons].wr_id;
3719 wc->opcode = qp->wqe_wr_id[qp->sq.cons].opcode;
3720
3721 switch (wc->opcode) {
3722 case IB_WC_RDMA_WRITE:
3723 wc->byte_len = qp->wqe_wr_id[qp->sq.cons].bytes_len;
3724 break;
3725 case IB_WC_COMP_SWAP:
3726 case IB_WC_FETCH_ADD:
3727 wc->byte_len = 8;
3728 break;
3729 case IB_WC_REG_MR:
3730 qp->wqe_wr_id[qp->sq.cons].mr->info.completed++;
3731 break;
3732 case IB_WC_RDMA_READ:
3733 case IB_WC_SEND:
3734 wc->byte_len = qp->wqe_wr_id[qp->sq.cons].bytes_len;
3735 break;
3736 default:
3737 break;
3738 }
3739
3740 num_entries--;
3741 wc++;
3742 cnt++;
3743 next_cqe:
3744 while (qp->wqe_wr_id[qp->sq.cons].wqe_size--)
3745 qed_chain_consume(&qp->sq.pbl);
3746 qedr_inc_sw_cons(&qp->sq);
3747 }
3748
3749 return cnt;
3750 }
3751
3752 static int qedr_poll_cq_req(struct qedr_dev *dev,
3753 struct qedr_qp *qp, struct qedr_cq *cq,
3754 int num_entries, struct ib_wc *wc,
3755 struct rdma_cqe_requester *req)
3756 {
3757 int cnt = 0;
3758
3759 switch (req->status) {
3760 case RDMA_CQE_REQ_STS_OK:
3761 cnt = process_req(dev, qp, cq, num_entries, wc, req->sq_cons,
3762 IB_WC_SUCCESS, 0);
3763 break;
3764 case RDMA_CQE_REQ_STS_WORK_REQUEST_FLUSHED_ERR:
3765 if (qp->state != QED_ROCE_QP_STATE_ERR)
3766 DP_DEBUG(dev, QEDR_MSG_CQ,
3767 "Error: POLL CQ with RDMA_CQE_REQ_STS_WORK_REQUEST_FLUSHED_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3768 cq->icid, qp->icid);
3769 cnt = process_req(dev, qp, cq, num_entries, wc, req->sq_cons,
3770 IB_WC_WR_FLUSH_ERR, 1);
3771 break;
3772 default:
3773
3774 qp->state = QED_ROCE_QP_STATE_ERR;
3775 cnt = process_req(dev, qp, cq, num_entries, wc,
3776 req->sq_cons - 1, IB_WC_SUCCESS, 0);
3777 wc += cnt;
3778
3779 if (cnt < num_entries) {
3780 enum ib_wc_status wc_status;
3781
3782 switch (req->status) {
3783 case RDMA_CQE_REQ_STS_BAD_RESPONSE_ERR:
3784 DP_ERR(dev,
3785 "Error: POLL CQ with RDMA_CQE_REQ_STS_BAD_RESPONSE_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3786 cq->icid, qp->icid);
3787 wc_status = IB_WC_BAD_RESP_ERR;
3788 break;
3789 case RDMA_CQE_REQ_STS_LOCAL_LENGTH_ERR:
3790 DP_ERR(dev,
3791 "Error: POLL CQ with RDMA_CQE_REQ_STS_LOCAL_LENGTH_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3792 cq->icid, qp->icid);
3793 wc_status = IB_WC_LOC_LEN_ERR;
3794 break;
3795 case RDMA_CQE_REQ_STS_LOCAL_QP_OPERATION_ERR:
3796 DP_ERR(dev,
3797 "Error: POLL CQ with RDMA_CQE_REQ_STS_LOCAL_QP_OPERATION_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3798 cq->icid, qp->icid);
3799 wc_status = IB_WC_LOC_QP_OP_ERR;
3800 break;
3801 case RDMA_CQE_REQ_STS_LOCAL_PROTECTION_ERR:
3802 DP_ERR(dev,
3803 "Error: POLL CQ with RDMA_CQE_REQ_STS_LOCAL_PROTECTION_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3804 cq->icid, qp->icid);
3805 wc_status = IB_WC_LOC_PROT_ERR;
3806 break;
3807 case RDMA_CQE_REQ_STS_MEMORY_MGT_OPERATION_ERR:
3808 DP_ERR(dev,
3809 "Error: POLL CQ with RDMA_CQE_REQ_STS_MEMORY_MGT_OPERATION_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3810 cq->icid, qp->icid);
3811 wc_status = IB_WC_MW_BIND_ERR;
3812 break;
3813 case RDMA_CQE_REQ_STS_REMOTE_INVALID_REQUEST_ERR:
3814 DP_ERR(dev,
3815 "Error: POLL CQ with RDMA_CQE_REQ_STS_REMOTE_INVALID_REQUEST_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3816 cq->icid, qp->icid);
3817 wc_status = IB_WC_REM_INV_REQ_ERR;
3818 break;
3819 case RDMA_CQE_REQ_STS_REMOTE_ACCESS_ERR:
3820 DP_ERR(dev,
3821 "Error: POLL CQ with RDMA_CQE_REQ_STS_REMOTE_ACCESS_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3822 cq->icid, qp->icid);
3823 wc_status = IB_WC_REM_ACCESS_ERR;
3824 break;
3825 case RDMA_CQE_REQ_STS_REMOTE_OPERATION_ERR:
3826 DP_ERR(dev,
3827 "Error: POLL CQ with RDMA_CQE_REQ_STS_REMOTE_OPERATION_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3828 cq->icid, qp->icid);
3829 wc_status = IB_WC_REM_OP_ERR;
3830 break;
3831 case RDMA_CQE_REQ_STS_RNR_NAK_RETRY_CNT_ERR:
3832 DP_ERR(dev,
3833 "Error: POLL CQ with RDMA_CQE_REQ_STS_RNR_NAK_RETRY_CNT_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3834 cq->icid, qp->icid);
3835 wc_status = IB_WC_RNR_RETRY_EXC_ERR;
3836 break;
3837 case RDMA_CQE_REQ_STS_TRANSPORT_RETRY_CNT_ERR:
3838 DP_ERR(dev,
3839 "Error: POLL CQ with ROCE_CQE_REQ_STS_TRANSPORT_RETRY_CNT_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3840 cq->icid, qp->icid);
3841 wc_status = IB_WC_RETRY_EXC_ERR;
3842 break;
3843 default:
3844 DP_ERR(dev,
3845 "Error: POLL CQ with IB_WC_GENERAL_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3846 cq->icid, qp->icid);
3847 wc_status = IB_WC_GENERAL_ERR;
3848 }
3849 cnt += process_req(dev, qp, cq, 1, wc, req->sq_cons,
3850 wc_status, 1);
3851 }
3852 }
3853
3854 return cnt;
3855 }
3856
3857 static inline int qedr_cqe_resp_status_to_ib(u8 status)
3858 {
3859 switch (status) {
3860 case RDMA_CQE_RESP_STS_LOCAL_ACCESS_ERR:
3861 return IB_WC_LOC_ACCESS_ERR;
3862 case RDMA_CQE_RESP_STS_LOCAL_LENGTH_ERR:
3863 return IB_WC_LOC_LEN_ERR;
3864 case RDMA_CQE_RESP_STS_LOCAL_QP_OPERATION_ERR:
3865 return IB_WC_LOC_QP_OP_ERR;
3866 case RDMA_CQE_RESP_STS_LOCAL_PROTECTION_ERR:
3867 return IB_WC_LOC_PROT_ERR;
3868 case RDMA_CQE_RESP_STS_MEMORY_MGT_OPERATION_ERR:
3869 return IB_WC_MW_BIND_ERR;
3870 case RDMA_CQE_RESP_STS_REMOTE_INVALID_REQUEST_ERR:
3871 return IB_WC_REM_INV_RD_REQ_ERR;
3872 case RDMA_CQE_RESP_STS_OK:
3873 return IB_WC_SUCCESS;
3874 default:
3875 return IB_WC_GENERAL_ERR;
3876 }
3877 }
3878
3879 static inline int qedr_set_ok_cqe_resp_wc(struct rdma_cqe_responder *resp,
3880 struct ib_wc *wc)
3881 {
3882 wc->status = IB_WC_SUCCESS;
3883 wc->byte_len = le32_to_cpu(resp->length);
3884
3885 if (resp->flags & QEDR_RESP_IMM) {
3886 wc->ex.imm_data = cpu_to_be32(le32_to_cpu(resp->imm_data_or_inv_r_Key));
3887 wc->wc_flags |= IB_WC_WITH_IMM;
3888
3889 if (resp->flags & QEDR_RESP_RDMA)
3890 wc->opcode = IB_WC_RECV_RDMA_WITH_IMM;
3891
3892 if (resp->flags & QEDR_RESP_INV)
3893 return -EINVAL;
3894
3895 } else if (resp->flags & QEDR_RESP_INV) {
3896 wc->ex.imm_data = le32_to_cpu(resp->imm_data_or_inv_r_Key);
3897 wc->wc_flags |= IB_WC_WITH_INVALIDATE;
3898
3899 if (resp->flags & QEDR_RESP_RDMA)
3900 return -EINVAL;
3901
3902 } else if (resp->flags & QEDR_RESP_RDMA) {
3903 return -EINVAL;
3904 }
3905
3906 return 0;
3907 }
3908
3909 static void __process_resp_one(struct qedr_dev *dev, struct qedr_qp *qp,
3910 struct qedr_cq *cq, struct ib_wc *wc,
3911 struct rdma_cqe_responder *resp, u64 wr_id)
3912 {
3913
3914 wc->opcode = IB_WC_RECV;
3915 wc->wc_flags = 0;
3916
3917 if (likely(resp->status == RDMA_CQE_RESP_STS_OK)) {
3918 if (qedr_set_ok_cqe_resp_wc(resp, wc))
3919 DP_ERR(dev,
3920 "CQ %p (icid=%d) has invalid CQE responder flags=0x%x\n",
3921 cq, cq->icid, resp->flags);
3922
3923 } else {
3924 wc->status = qedr_cqe_resp_status_to_ib(resp->status);
3925 if (wc->status == IB_WC_GENERAL_ERR)
3926 DP_ERR(dev,
3927 "CQ %p (icid=%d) contains an invalid CQE status %d\n",
3928 cq, cq->icid, resp->status);
3929 }
3930
3931
3932 wc->vendor_err = 0;
3933 wc->src_qp = qp->id;
3934 wc->qp = &qp->ibqp;
3935 wc->wr_id = wr_id;
3936 }
3937
3938 static int process_resp_one_srq(struct qedr_dev *dev, struct qedr_qp *qp,
3939 struct qedr_cq *cq, struct ib_wc *wc,
3940 struct rdma_cqe_responder *resp)
3941 {
3942 struct qedr_srq *srq = qp->srq;
3943 u64 wr_id;
3944
3945 wr_id = HILO_GEN(le32_to_cpu(resp->srq_wr_id.hi),
3946 le32_to_cpu(resp->srq_wr_id.lo), u64);
3947
3948 if (resp->status == RDMA_CQE_RESP_STS_WORK_REQUEST_FLUSHED_ERR) {
3949 wc->status = IB_WC_WR_FLUSH_ERR;
3950 wc->vendor_err = 0;
3951 wc->wr_id = wr_id;
3952 wc->byte_len = 0;
3953 wc->src_qp = qp->id;
3954 wc->qp = &qp->ibqp;
3955 wc->wr_id = wr_id;
3956 } else {
3957 __process_resp_one(dev, qp, cq, wc, resp, wr_id);
3958 }
3959 srq->hw_srq.wr_cons_cnt++;
3960
3961 return 1;
3962 }
3963 static int process_resp_one(struct qedr_dev *dev, struct qedr_qp *qp,
3964 struct qedr_cq *cq, struct ib_wc *wc,
3965 struct rdma_cqe_responder *resp)
3966 {
3967 u64 wr_id = qp->rqe_wr_id[qp->rq.cons].wr_id;
3968
3969 __process_resp_one(dev, qp, cq, wc, resp, wr_id);
3970
3971 while (qp->rqe_wr_id[qp->rq.cons].wqe_size--)
3972 qed_chain_consume(&qp->rq.pbl);
3973 qedr_inc_sw_cons(&qp->rq);
3974
3975 return 1;
3976 }
3977
3978 static int process_resp_flush(struct qedr_qp *qp, struct qedr_cq *cq,
3979 int num_entries, struct ib_wc *wc, u16 hw_cons)
3980 {
3981 u16 cnt = 0;
3982
3983 while (num_entries && qp->rq.wqe_cons != hw_cons) {
3984
3985 wc->status = IB_WC_WR_FLUSH_ERR;
3986 wc->vendor_err = 0;
3987 wc->wc_flags = 0;
3988 wc->src_qp = qp->id;
3989 wc->byte_len = 0;
3990 wc->wr_id = qp->rqe_wr_id[qp->rq.cons].wr_id;
3991 wc->qp = &qp->ibqp;
3992 num_entries--;
3993 wc++;
3994 cnt++;
3995 while (qp->rqe_wr_id[qp->rq.cons].wqe_size--)
3996 qed_chain_consume(&qp->rq.pbl);
3997 qedr_inc_sw_cons(&qp->rq);
3998 }
3999
4000 return cnt;
4001 }
4002
4003 static void try_consume_resp_cqe(struct qedr_cq *cq, struct qedr_qp *qp,
4004 struct rdma_cqe_responder *resp, int *update)
4005 {
4006 if (le16_to_cpu(resp->rq_cons_or_srq_id) == qp->rq.wqe_cons) {
4007 consume_cqe(cq);
4008 *update |= 1;
4009 }
4010 }
4011
4012 static int qedr_poll_cq_resp_srq(struct qedr_dev *dev, struct qedr_qp *qp,
4013 struct qedr_cq *cq, int num_entries,
4014 struct ib_wc *wc,
4015 struct rdma_cqe_responder *resp)
4016 {
4017 int cnt;
4018
4019 cnt = process_resp_one_srq(dev, qp, cq, wc, resp);
4020 consume_cqe(cq);
4021
4022 return cnt;
4023 }
4024
4025 static int qedr_poll_cq_resp(struct qedr_dev *dev, struct qedr_qp *qp,
4026 struct qedr_cq *cq, int num_entries,
4027 struct ib_wc *wc, struct rdma_cqe_responder *resp,
4028 int *update)
4029 {
4030 int cnt;
4031
4032 if (resp->status == RDMA_CQE_RESP_STS_WORK_REQUEST_FLUSHED_ERR) {
4033 cnt = process_resp_flush(qp, cq, num_entries, wc,
4034 resp->rq_cons_or_srq_id);
4035 try_consume_resp_cqe(cq, qp, resp, update);
4036 } else {
4037 cnt = process_resp_one(dev, qp, cq, wc, resp);
4038 consume_cqe(cq);
4039 *update |= 1;
4040 }
4041
4042 return cnt;
4043 }
4044
4045 static void try_consume_req_cqe(struct qedr_cq *cq, struct qedr_qp *qp,
4046 struct rdma_cqe_requester *req, int *update)
4047 {
4048 if (le16_to_cpu(req->sq_cons) == qp->sq.wqe_cons) {
4049 consume_cqe(cq);
4050 *update |= 1;
4051 }
4052 }
4053
4054 int qedr_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc)
4055 {
4056 struct qedr_dev *dev = get_qedr_dev(ibcq->device);
4057 struct qedr_cq *cq = get_qedr_cq(ibcq);
4058 union rdma_cqe *cqe;
4059 u32 old_cons, new_cons;
4060 unsigned long flags;
4061 int update = 0;
4062 int done = 0;
4063
4064 if (cq->destroyed) {
4065 DP_ERR(dev,
4066 "warning: poll was invoked after destroy for cq %p (icid=%d)\n",
4067 cq, cq->icid);
4068 return 0;
4069 }
4070
4071 if (cq->cq_type == QEDR_CQ_TYPE_GSI)
4072 return qedr_gsi_poll_cq(ibcq, num_entries, wc);
4073
4074 spin_lock_irqsave(&cq->cq_lock, flags);
4075 cqe = cq->latest_cqe;
4076 old_cons = qed_chain_get_cons_idx_u32(&cq->pbl);
4077 while (num_entries && is_valid_cqe(cq, cqe)) {
4078 struct qedr_qp *qp;
4079 int cnt = 0;
4080
4081
4082 rmb();
4083
4084 qp = cqe_get_qp(cqe);
4085 if (!qp) {
4086 WARN(1, "Error: CQE QP pointer is NULL. CQE=%p\n", cqe);
4087 break;
4088 }
4089
4090 wc->qp = &qp->ibqp;
4091
4092 switch (cqe_get_type(cqe)) {
4093 case RDMA_CQE_TYPE_REQUESTER:
4094 cnt = qedr_poll_cq_req(dev, qp, cq, num_entries, wc,
4095 &cqe->req);
4096 try_consume_req_cqe(cq, qp, &cqe->req, &update);
4097 break;
4098 case RDMA_CQE_TYPE_RESPONDER_RQ:
4099 cnt = qedr_poll_cq_resp(dev, qp, cq, num_entries, wc,
4100 &cqe->resp, &update);
4101 break;
4102 case RDMA_CQE_TYPE_RESPONDER_SRQ:
4103 cnt = qedr_poll_cq_resp_srq(dev, qp, cq, num_entries,
4104 wc, &cqe->resp);
4105 update = 1;
4106 break;
4107 case RDMA_CQE_TYPE_INVALID:
4108 default:
4109 DP_ERR(dev, "Error: invalid CQE type = %d\n",
4110 cqe_get_type(cqe));
4111 }
4112 num_entries -= cnt;
4113 wc += cnt;
4114 done += cnt;
4115
4116 cqe = get_cqe(cq);
4117 }
4118 new_cons = qed_chain_get_cons_idx_u32(&cq->pbl);
4119
4120 cq->cq_cons += new_cons - old_cons;
4121
4122 if (update)
4123
4124
4125
4126 doorbell_cq(cq, cq->cq_cons - 1, cq->arm_flags);
4127
4128 spin_unlock_irqrestore(&cq->cq_lock, flags);
4129 return done;
4130 }
4131
4132 int qedr_process_mad(struct ib_device *ibdev, int process_mad_flags,
4133 u8 port_num,
4134 const struct ib_wc *in_wc,
4135 const struct ib_grh *in_grh,
4136 const struct ib_mad_hdr *mad_hdr,
4137 size_t in_mad_size, struct ib_mad_hdr *out_mad,
4138 size_t *out_mad_size, u16 *out_mad_pkey_index)
4139 {
4140 struct qedr_dev *dev = get_qedr_dev(ibdev);
4141
4142 DP_DEBUG(dev, QEDR_MSG_GSI,
4143 "QEDR_PROCESS_MAD in_mad %x %x %x %x %x %x %x %x\n",
4144 mad_hdr->attr_id, mad_hdr->base_version, mad_hdr->attr_mod,
4145 mad_hdr->class_specific, mad_hdr->class_version,
4146 mad_hdr->method, mad_hdr->mgmt_class, mad_hdr->status);
4147 return IB_MAD_RESULT_SUCCESS;
4148 }