This source file includes following definitions.
- siw_create_uobj
- siw_get_uobj
- siw_mmap
- siw_alloc_ucontext
- siw_dealloc_ucontext
- siw_query_device
- siw_query_port
- siw_get_port_immutable
- siw_query_pkey
- siw_query_gid
- siw_alloc_pd
- siw_dealloc_pd
- siw_qp_get_ref
- siw_qp_put_ref
- siw_create_qp
- siw_query_qp
- siw_verbs_modify_qp
- siw_destroy_qp
- siw_copy_inline_sgl
- siw_sq_flush_wr
- siw_rq_flush_wr
- siw_post_send
- siw_post_receive
- siw_destroy_cq
- siw_create_cq
- siw_poll_cq
- siw_req_notify_cq
- siw_dereg_mr
- siw_reg_user_mr
- siw_alloc_mr
- siw_set_pbl_page
- siw_map_mr_sg
- siw_get_dma_mr
- siw_create_srq
- siw_modify_srq
- siw_query_srq
- siw_destroy_srq
- siw_post_srq_recv
- siw_qp_event
- siw_cq_event
- siw_srq_event
- siw_port_event
1
2
3
4
5
6 #include <linux/errno.h>
7 #include <linux/types.h>
8 #include <linux/uaccess.h>
9 #include <linux/vmalloc.h>
10 #include <linux/xarray.h>
11
12 #include <rdma/iw_cm.h>
13 #include <rdma/ib_verbs.h>
14 #include <rdma/ib_user_verbs.h>
15 #include <rdma/uverbs_ioctl.h>
16
17 #include "siw.h"
18 #include "siw_verbs.h"
19 #include "siw_mem.h"
20
21 static int ib_qp_state_to_siw_qp_state[IB_QPS_ERR + 1] = {
22 [IB_QPS_RESET] = SIW_QP_STATE_IDLE,
23 [IB_QPS_INIT] = SIW_QP_STATE_IDLE,
24 [IB_QPS_RTR] = SIW_QP_STATE_RTR,
25 [IB_QPS_RTS] = SIW_QP_STATE_RTS,
26 [IB_QPS_SQD] = SIW_QP_STATE_CLOSING,
27 [IB_QPS_SQE] = SIW_QP_STATE_TERMINATE,
28 [IB_QPS_ERR] = SIW_QP_STATE_ERROR
29 };
30
31 static char ib_qp_state_to_string[IB_QPS_ERR + 1][sizeof("RESET")] = {
32 [IB_QPS_RESET] = "RESET", [IB_QPS_INIT] = "INIT", [IB_QPS_RTR] = "RTR",
33 [IB_QPS_RTS] = "RTS", [IB_QPS_SQD] = "SQD", [IB_QPS_SQE] = "SQE",
34 [IB_QPS_ERR] = "ERR"
35 };
36
37 static u32 siw_create_uobj(struct siw_ucontext *uctx, void *vaddr, u32 size)
38 {
39 struct siw_uobj *uobj;
40 struct xa_limit limit = XA_LIMIT(0, SIW_UOBJ_MAX_KEY);
41 u32 key;
42
43 uobj = kzalloc(sizeof(*uobj), GFP_KERNEL);
44 if (!uobj)
45 return SIW_INVAL_UOBJ_KEY;
46
47 if (xa_alloc_cyclic(&uctx->xa, &key, uobj, limit, &uctx->uobj_nextkey,
48 GFP_KERNEL) < 0) {
49 kfree(uobj);
50 return SIW_INVAL_UOBJ_KEY;
51 }
52 uobj->size = PAGE_ALIGN(size);
53 uobj->addr = vaddr;
54
55 return key;
56 }
57
58 static struct siw_uobj *siw_get_uobj(struct siw_ucontext *uctx,
59 unsigned long off, u32 size)
60 {
61 struct siw_uobj *uobj = xa_load(&uctx->xa, off);
62
63 if (uobj && uobj->size == size)
64 return uobj;
65
66 return NULL;
67 }
68
69 int siw_mmap(struct ib_ucontext *ctx, struct vm_area_struct *vma)
70 {
71 struct siw_ucontext *uctx = to_siw_ctx(ctx);
72 struct siw_uobj *uobj;
73 unsigned long off = vma->vm_pgoff;
74 int size = vma->vm_end - vma->vm_start;
75 int rv = -EINVAL;
76
77
78
79
80 if (vma->vm_start & (PAGE_SIZE - 1)) {
81 pr_warn("siw: mmap not page aligned\n");
82 goto out;
83 }
84 uobj = siw_get_uobj(uctx, off, size);
85 if (!uobj) {
86 siw_dbg(&uctx->sdev->base_dev, "mmap lookup failed: %lu, %u\n",
87 off, size);
88 goto out;
89 }
90 rv = remap_vmalloc_range(vma, uobj->addr, 0);
91 if (rv)
92 pr_warn("remap_vmalloc_range failed: %lu, %u\n", off, size);
93 out:
94 return rv;
95 }
96
97 int siw_alloc_ucontext(struct ib_ucontext *base_ctx, struct ib_udata *udata)
98 {
99 struct siw_device *sdev = to_siw_dev(base_ctx->device);
100 struct siw_ucontext *ctx = to_siw_ctx(base_ctx);
101 struct siw_uresp_alloc_ctx uresp = {};
102 int rv;
103
104 if (atomic_inc_return(&sdev->num_ctx) > SIW_MAX_CONTEXT) {
105 rv = -ENOMEM;
106 goto err_out;
107 }
108 xa_init_flags(&ctx->xa, XA_FLAGS_ALLOC);
109 ctx->uobj_nextkey = 0;
110 ctx->sdev = sdev;
111
112 uresp.dev_id = sdev->vendor_part_id;
113
114 if (udata->outlen < sizeof(uresp)) {
115 rv = -EINVAL;
116 goto err_out;
117 }
118 rv = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
119 if (rv)
120 goto err_out;
121
122 siw_dbg(base_ctx->device, "success. now %d context(s)\n",
123 atomic_read(&sdev->num_ctx));
124
125 return 0;
126
127 err_out:
128 atomic_dec(&sdev->num_ctx);
129 siw_dbg(base_ctx->device, "failure %d. now %d context(s)\n", rv,
130 atomic_read(&sdev->num_ctx));
131
132 return rv;
133 }
134
135 void siw_dealloc_ucontext(struct ib_ucontext *base_ctx)
136 {
137 struct siw_ucontext *uctx = to_siw_ctx(base_ctx);
138 void *entry;
139 unsigned long index;
140
141
142
143
144
145
146 xa_for_each(&uctx->xa, index, entry) {
147 kfree(xa_erase(&uctx->xa, index));
148 pr_warn("siw: dropping orphaned uobj at %lu\n", index);
149 }
150 xa_destroy(&uctx->xa);
151 atomic_dec(&uctx->sdev->num_ctx);
152 }
153
154 int siw_query_device(struct ib_device *base_dev, struct ib_device_attr *attr,
155 struct ib_udata *udata)
156 {
157 struct siw_device *sdev = to_siw_dev(base_dev);
158
159 if (udata->inlen || udata->outlen)
160 return -EINVAL;
161
162 memset(attr, 0, sizeof(*attr));
163
164
165 attr->atomic_cap = 0;
166 attr->device_cap_flags =
167 IB_DEVICE_MEM_MGT_EXTENSIONS | IB_DEVICE_ALLOW_USER_UNREG;
168 attr->max_cq = sdev->attrs.max_cq;
169 attr->max_cqe = sdev->attrs.max_cqe;
170 attr->max_fast_reg_page_list_len = SIW_MAX_SGE_PBL;
171 attr->max_fmr = sdev->attrs.max_fmr;
172 attr->max_mr = sdev->attrs.max_mr;
173 attr->max_mw = sdev->attrs.max_mw;
174 attr->max_mr_size = ~0ull;
175 attr->max_pd = sdev->attrs.max_pd;
176 attr->max_qp = sdev->attrs.max_qp;
177 attr->max_qp_init_rd_atom = sdev->attrs.max_ird;
178 attr->max_qp_rd_atom = sdev->attrs.max_ord;
179 attr->max_qp_wr = sdev->attrs.max_qp_wr;
180 attr->max_recv_sge = sdev->attrs.max_sge;
181 attr->max_res_rd_atom = sdev->attrs.max_qp * sdev->attrs.max_ird;
182 attr->max_send_sge = sdev->attrs.max_sge;
183 attr->max_sge_rd = sdev->attrs.max_sge_rd;
184 attr->max_srq = sdev->attrs.max_srq;
185 attr->max_srq_sge = sdev->attrs.max_srq_sge;
186 attr->max_srq_wr = sdev->attrs.max_srq_wr;
187 attr->page_size_cap = PAGE_SIZE;
188 attr->vendor_id = SIW_VENDOR_ID;
189 attr->vendor_part_id = sdev->vendor_part_id;
190
191 memcpy(&attr->sys_image_guid, sdev->netdev->dev_addr, 6);
192
193 return 0;
194 }
195
196 int siw_query_port(struct ib_device *base_dev, u8 port,
197 struct ib_port_attr *attr)
198 {
199 struct siw_device *sdev = to_siw_dev(base_dev);
200
201 memset(attr, 0, sizeof(*attr));
202
203 attr->active_mtu = attr->max_mtu;
204 attr->active_speed = 2;
205 attr->active_width = 2;
206 attr->gid_tbl_len = 1;
207 attr->max_msg_sz = -1;
208 attr->max_mtu = ib_mtu_int_to_enum(sdev->netdev->mtu);
209 attr->phys_state = sdev->state == IB_PORT_ACTIVE ?
210 IB_PORT_PHYS_STATE_LINK_UP : IB_PORT_PHYS_STATE_DISABLED;
211 attr->pkey_tbl_len = 1;
212 attr->port_cap_flags = IB_PORT_CM_SUP | IB_PORT_DEVICE_MGMT_SUP;
213 attr->state = sdev->state;
214
215
216
217
218
219
220
221
222
223
224
225
226
227 return 0;
228 }
229
230 int siw_get_port_immutable(struct ib_device *base_dev, u8 port,
231 struct ib_port_immutable *port_immutable)
232 {
233 struct ib_port_attr attr;
234 int rv = siw_query_port(base_dev, port, &attr);
235
236 if (rv)
237 return rv;
238
239 port_immutable->pkey_tbl_len = attr.pkey_tbl_len;
240 port_immutable->gid_tbl_len = attr.gid_tbl_len;
241 port_immutable->core_cap_flags = RDMA_CORE_PORT_IWARP;
242
243 return 0;
244 }
245
246 int siw_query_pkey(struct ib_device *base_dev, u8 port, u16 idx, u16 *pkey)
247 {
248
249 *pkey = 0xffff;
250 return 0;
251 }
252
253 int siw_query_gid(struct ib_device *base_dev, u8 port, int idx,
254 union ib_gid *gid)
255 {
256 struct siw_device *sdev = to_siw_dev(base_dev);
257
258
259 memset(gid, 0, sizeof(*gid));
260 memcpy(&gid->raw[0], sdev->netdev->dev_addr, 6);
261
262 return 0;
263 }
264
265 int siw_alloc_pd(struct ib_pd *pd, struct ib_udata *udata)
266 {
267 struct siw_device *sdev = to_siw_dev(pd->device);
268
269 if (atomic_inc_return(&sdev->num_pd) > SIW_MAX_PD) {
270 atomic_dec(&sdev->num_pd);
271 return -ENOMEM;
272 }
273 siw_dbg_pd(pd, "now %d PD's(s)\n", atomic_read(&sdev->num_pd));
274
275 return 0;
276 }
277
278 void siw_dealloc_pd(struct ib_pd *pd, struct ib_udata *udata)
279 {
280 struct siw_device *sdev = to_siw_dev(pd->device);
281
282 siw_dbg_pd(pd, "free PD\n");
283 atomic_dec(&sdev->num_pd);
284 }
285
286 void siw_qp_get_ref(struct ib_qp *base_qp)
287 {
288 siw_qp_get(to_siw_qp(base_qp));
289 }
290
291 void siw_qp_put_ref(struct ib_qp *base_qp)
292 {
293 siw_qp_put(to_siw_qp(base_qp));
294 }
295
296
297
298
299
300
301
302
303
304
305
306 struct ib_qp *siw_create_qp(struct ib_pd *pd,
307 struct ib_qp_init_attr *attrs,
308 struct ib_udata *udata)
309 {
310 struct siw_qp *qp = NULL;
311 struct siw_base_qp *siw_base_qp = NULL;
312 struct ib_device *base_dev = pd->device;
313 struct siw_device *sdev = to_siw_dev(base_dev);
314 struct siw_ucontext *uctx =
315 rdma_udata_to_drv_context(udata, struct siw_ucontext,
316 base_ucontext);
317 struct siw_cq *scq = NULL, *rcq = NULL;
318 unsigned long flags;
319 int num_sqe, num_rqe, rv = 0;
320
321 siw_dbg(base_dev, "create new QP\n");
322
323 if (atomic_inc_return(&sdev->num_qp) > SIW_MAX_QP) {
324 siw_dbg(base_dev, "too many QP's\n");
325 rv = -ENOMEM;
326 goto err_out;
327 }
328 if (attrs->qp_type != IB_QPT_RC) {
329 siw_dbg(base_dev, "only RC QP's supported\n");
330 rv = -EINVAL;
331 goto err_out;
332 }
333 if ((attrs->cap.max_send_wr > SIW_MAX_QP_WR) ||
334 (attrs->cap.max_recv_wr > SIW_MAX_QP_WR) ||
335 (attrs->cap.max_send_sge > SIW_MAX_SGE) ||
336 (attrs->cap.max_recv_sge > SIW_MAX_SGE)) {
337 siw_dbg(base_dev, "QP size error\n");
338 rv = -EINVAL;
339 goto err_out;
340 }
341 if (attrs->cap.max_inline_data > SIW_MAX_INLINE) {
342 siw_dbg(base_dev, "max inline send: %d > %d\n",
343 attrs->cap.max_inline_data, (int)SIW_MAX_INLINE);
344 rv = -EINVAL;
345 goto err_out;
346 }
347
348
349
350
351 if (attrs->cap.max_send_wr + attrs->cap.max_recv_wr == 0) {
352 siw_dbg(base_dev, "QP must have send or receive queue\n");
353 rv = -EINVAL;
354 goto err_out;
355 }
356 scq = to_siw_cq(attrs->send_cq);
357 rcq = to_siw_cq(attrs->recv_cq);
358
359 if (!scq || (!rcq && !attrs->srq)) {
360 siw_dbg(base_dev, "send CQ or receive CQ invalid\n");
361 rv = -EINVAL;
362 goto err_out;
363 }
364 siw_base_qp = kzalloc(sizeof(*siw_base_qp), GFP_KERNEL);
365 if (!siw_base_qp) {
366 rv = -ENOMEM;
367 goto err_out;
368 }
369 qp = kzalloc(sizeof(*qp), GFP_KERNEL);
370 if (!qp) {
371 rv = -ENOMEM;
372 goto err_out;
373 }
374 siw_base_qp->qp = qp;
375 qp->ib_qp = &siw_base_qp->base_qp;
376
377 init_rwsem(&qp->state_lock);
378 spin_lock_init(&qp->sq_lock);
379 spin_lock_init(&qp->rq_lock);
380 spin_lock_init(&qp->orq_lock);
381
382 qp->kernel_verbs = !udata;
383 qp->xa_sq_index = SIW_INVAL_UOBJ_KEY;
384 qp->xa_rq_index = SIW_INVAL_UOBJ_KEY;
385
386 rv = siw_qp_add(sdev, qp);
387 if (rv)
388 goto err_out;
389
390
391
392
393
394
395 num_sqe = roundup_pow_of_two(attrs->cap.max_send_wr);
396 num_rqe = roundup_pow_of_two(attrs->cap.max_recv_wr);
397
398 if (qp->kernel_verbs)
399 qp->sendq = vzalloc(num_sqe * sizeof(struct siw_sqe));
400 else
401 qp->sendq = vmalloc_user(num_sqe * sizeof(struct siw_sqe));
402
403 if (qp->sendq == NULL) {
404 siw_dbg(base_dev, "SQ size %d alloc failed\n", num_sqe);
405 rv = -ENOMEM;
406 goto err_out_xa;
407 }
408 if (attrs->sq_sig_type != IB_SIGNAL_REQ_WR) {
409 if (attrs->sq_sig_type == IB_SIGNAL_ALL_WR)
410 qp->attrs.flags |= SIW_SIGNAL_ALL_WR;
411 else {
412 rv = -EINVAL;
413 goto err_out_xa;
414 }
415 }
416 qp->pd = pd;
417 qp->scq = scq;
418 qp->rcq = rcq;
419
420 if (attrs->srq) {
421
422
423
424
425
426 qp->srq = to_siw_srq(attrs->srq);
427 qp->attrs.rq_size = 0;
428 siw_dbg(base_dev, "QP [%u]: SRQ attached\n", qp->qp_num);
429 } else if (num_rqe) {
430 if (qp->kernel_verbs)
431 qp->recvq = vzalloc(num_rqe * sizeof(struct siw_rqe));
432 else
433 qp->recvq =
434 vmalloc_user(num_rqe * sizeof(struct siw_rqe));
435
436 if (qp->recvq == NULL) {
437 siw_dbg(base_dev, "RQ size %d alloc failed\n", num_rqe);
438 rv = -ENOMEM;
439 goto err_out_xa;
440 }
441 qp->attrs.rq_size = num_rqe;
442 }
443 qp->attrs.sq_size = num_sqe;
444 qp->attrs.sq_max_sges = attrs->cap.max_send_sge;
445 qp->attrs.rq_max_sges = attrs->cap.max_recv_sge;
446
447
448 qp->tx_ctx.gso_seg_limit = 1;
449 qp->tx_ctx.zcopy_tx = zcopy_tx;
450
451 qp->attrs.state = SIW_QP_STATE_IDLE;
452
453 if (udata) {
454 struct siw_uresp_create_qp uresp = {};
455
456 uresp.num_sqe = num_sqe;
457 uresp.num_rqe = num_rqe;
458 uresp.qp_id = qp_id(qp);
459
460 if (qp->sendq) {
461 qp->xa_sq_index =
462 siw_create_uobj(uctx, qp->sendq,
463 num_sqe * sizeof(struct siw_sqe));
464 }
465 if (qp->recvq) {
466 qp->xa_rq_index =
467 siw_create_uobj(uctx, qp->recvq,
468 num_rqe * sizeof(struct siw_rqe));
469 }
470 if (qp->xa_sq_index == SIW_INVAL_UOBJ_KEY ||
471 qp->xa_rq_index == SIW_INVAL_UOBJ_KEY) {
472 rv = -ENOMEM;
473 goto err_out_xa;
474 }
475 uresp.sq_key = qp->xa_sq_index << PAGE_SHIFT;
476 uresp.rq_key = qp->xa_rq_index << PAGE_SHIFT;
477
478 if (udata->outlen < sizeof(uresp)) {
479 rv = -EINVAL;
480 goto err_out_xa;
481 }
482 rv = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
483 if (rv)
484 goto err_out_xa;
485 }
486 qp->tx_cpu = siw_get_tx_cpu(sdev);
487 if (qp->tx_cpu < 0) {
488 rv = -EINVAL;
489 goto err_out_xa;
490 }
491 INIT_LIST_HEAD(&qp->devq);
492 spin_lock_irqsave(&sdev->lock, flags);
493 list_add_tail(&qp->devq, &sdev->qp_list);
494 spin_unlock_irqrestore(&sdev->lock, flags);
495
496 return qp->ib_qp;
497
498 err_out_xa:
499 xa_erase(&sdev->qp_xa, qp_id(qp));
500 err_out:
501 kfree(siw_base_qp);
502
503 if (qp) {
504 if (qp->xa_sq_index != SIW_INVAL_UOBJ_KEY)
505 kfree(xa_erase(&uctx->xa, qp->xa_sq_index));
506 if (qp->xa_rq_index != SIW_INVAL_UOBJ_KEY)
507 kfree(xa_erase(&uctx->xa, qp->xa_rq_index));
508
509 vfree(qp->sendq);
510 vfree(qp->recvq);
511 kfree(qp);
512 }
513 atomic_dec(&sdev->num_qp);
514
515 return ERR_PTR(rv);
516 }
517
518
519
520
521
522
523 int siw_query_qp(struct ib_qp *base_qp, struct ib_qp_attr *qp_attr,
524 int qp_attr_mask, struct ib_qp_init_attr *qp_init_attr)
525 {
526 struct siw_qp *qp;
527 struct siw_device *sdev;
528
529 if (base_qp && qp_attr && qp_init_attr) {
530 qp = to_siw_qp(base_qp);
531 sdev = to_siw_dev(base_qp->device);
532 } else {
533 return -EINVAL;
534 }
535 qp_attr->cap.max_inline_data = SIW_MAX_INLINE;
536 qp_attr->cap.max_send_wr = qp->attrs.sq_size;
537 qp_attr->cap.max_send_sge = qp->attrs.sq_max_sges;
538 qp_attr->cap.max_recv_wr = qp->attrs.rq_size;
539 qp_attr->cap.max_recv_sge = qp->attrs.rq_max_sges;
540 qp_attr->path_mtu = ib_mtu_int_to_enum(sdev->netdev->mtu);
541 qp_attr->max_rd_atomic = qp->attrs.irq_size;
542 qp_attr->max_dest_rd_atomic = qp->attrs.orq_size;
543
544 qp_attr->qp_access_flags = IB_ACCESS_LOCAL_WRITE |
545 IB_ACCESS_REMOTE_WRITE |
546 IB_ACCESS_REMOTE_READ;
547
548 qp_init_attr->qp_type = base_qp->qp_type;
549 qp_init_attr->send_cq = base_qp->send_cq;
550 qp_init_attr->recv_cq = base_qp->recv_cq;
551 qp_init_attr->srq = base_qp->srq;
552
553 qp_init_attr->cap = qp_attr->cap;
554
555 return 0;
556 }
557
558 int siw_verbs_modify_qp(struct ib_qp *base_qp, struct ib_qp_attr *attr,
559 int attr_mask, struct ib_udata *udata)
560 {
561 struct siw_qp_attrs new_attrs;
562 enum siw_qp_attr_mask siw_attr_mask = 0;
563 struct siw_qp *qp = to_siw_qp(base_qp);
564 int rv = 0;
565
566 if (!attr_mask)
567 return 0;
568
569 memset(&new_attrs, 0, sizeof(new_attrs));
570
571 if (attr_mask & IB_QP_ACCESS_FLAGS) {
572 siw_attr_mask = SIW_QP_ATTR_ACCESS_FLAGS;
573
574 if (attr->qp_access_flags & IB_ACCESS_REMOTE_READ)
575 new_attrs.flags |= SIW_RDMA_READ_ENABLED;
576 if (attr->qp_access_flags & IB_ACCESS_REMOTE_WRITE)
577 new_attrs.flags |= SIW_RDMA_WRITE_ENABLED;
578 if (attr->qp_access_flags & IB_ACCESS_MW_BIND)
579 new_attrs.flags |= SIW_RDMA_BIND_ENABLED;
580 }
581 if (attr_mask & IB_QP_STATE) {
582 siw_dbg_qp(qp, "desired IB QP state: %s\n",
583 ib_qp_state_to_string[attr->qp_state]);
584
585 new_attrs.state = ib_qp_state_to_siw_qp_state[attr->qp_state];
586
587 if (new_attrs.state > SIW_QP_STATE_RTS)
588 qp->tx_ctx.tx_suspend = 1;
589
590 siw_attr_mask |= SIW_QP_ATTR_STATE;
591 }
592 if (!siw_attr_mask)
593 goto out;
594
595 down_write(&qp->state_lock);
596
597 rv = siw_qp_modify(qp, &new_attrs, siw_attr_mask);
598
599 up_write(&qp->state_lock);
600 out:
601 return rv;
602 }
603
604 int siw_destroy_qp(struct ib_qp *base_qp, struct ib_udata *udata)
605 {
606 struct siw_qp *qp = to_siw_qp(base_qp);
607 struct siw_ucontext *uctx =
608 rdma_udata_to_drv_context(udata, struct siw_ucontext,
609 base_ucontext);
610 struct siw_qp_attrs qp_attrs;
611
612 siw_dbg_qp(qp, "state %d\n", qp->attrs.state);
613
614
615
616
617
618 qp->attrs.flags |= SIW_QP_IN_DESTROY;
619 qp->rx_stream.rx_suspend = 1;
620
621 if (uctx && qp->xa_sq_index != SIW_INVAL_UOBJ_KEY)
622 kfree(xa_erase(&uctx->xa, qp->xa_sq_index));
623 if (uctx && qp->xa_rq_index != SIW_INVAL_UOBJ_KEY)
624 kfree(xa_erase(&uctx->xa, qp->xa_rq_index));
625
626 down_write(&qp->state_lock);
627
628 qp_attrs.state = SIW_QP_STATE_ERROR;
629 siw_qp_modify(qp, &qp_attrs, SIW_QP_ATTR_STATE);
630
631 if (qp->cep) {
632 siw_cep_put(qp->cep);
633 qp->cep = NULL;
634 }
635 up_write(&qp->state_lock);
636
637 kfree(qp->tx_ctx.mpa_crc_hd);
638 kfree(qp->rx_stream.mpa_crc_hd);
639
640 qp->scq = qp->rcq = NULL;
641
642 siw_qp_put(qp);
643
644 return 0;
645 }
646
647
648
649
650
651
652
653
654
655
656 static int siw_copy_inline_sgl(const struct ib_send_wr *core_wr,
657 struct siw_sqe *sqe)
658 {
659 struct ib_sge *core_sge = core_wr->sg_list;
660 void *kbuf = &sqe->sge[1];
661 int num_sge = core_wr->num_sge, bytes = 0;
662
663 sqe->sge[0].laddr = (uintptr_t)kbuf;
664 sqe->sge[0].lkey = 0;
665
666 while (num_sge--) {
667 if (!core_sge->length) {
668 core_sge++;
669 continue;
670 }
671 bytes += core_sge->length;
672 if (bytes > SIW_MAX_INLINE) {
673 bytes = -EINVAL;
674 break;
675 }
676 memcpy(kbuf, (void *)(uintptr_t)core_sge->addr,
677 core_sge->length);
678
679 kbuf += core_sge->length;
680 core_sge++;
681 }
682 sqe->sge[0].length = bytes > 0 ? bytes : 0;
683 sqe->num_sge = bytes > 0 ? 1 : 0;
684
685 return bytes;
686 }
687
688
689 static int siw_sq_flush_wr(struct siw_qp *qp, const struct ib_send_wr *wr,
690 const struct ib_send_wr **bad_wr)
691 {
692 struct siw_sqe sqe = {};
693 int rv = 0;
694
695 while (wr) {
696 sqe.id = wr->wr_id;
697 sqe.opcode = wr->opcode;
698 rv = siw_sqe_complete(qp, &sqe, 0, SIW_WC_WR_FLUSH_ERR);
699 if (rv) {
700 if (bad_wr)
701 *bad_wr = wr;
702 break;
703 }
704 wr = wr->next;
705 }
706 return rv;
707 }
708
709
710 static int siw_rq_flush_wr(struct siw_qp *qp, const struct ib_recv_wr *wr,
711 const struct ib_recv_wr **bad_wr)
712 {
713 struct siw_rqe rqe = {};
714 int rv = 0;
715
716 while (wr) {
717 rqe.id = wr->wr_id;
718 rv = siw_rqe_complete(qp, &rqe, 0, 0, SIW_WC_WR_FLUSH_ERR);
719 if (rv) {
720 if (bad_wr)
721 *bad_wr = wr;
722 break;
723 }
724 wr = wr->next;
725 }
726 return rv;
727 }
728
729
730
731
732
733
734
735
736
737
738 int siw_post_send(struct ib_qp *base_qp, const struct ib_send_wr *wr,
739 const struct ib_send_wr **bad_wr)
740 {
741 struct siw_qp *qp = to_siw_qp(base_qp);
742 struct siw_wqe *wqe = tx_wqe(qp);
743
744 unsigned long flags;
745 int rv = 0;
746
747 if (wr && !qp->kernel_verbs) {
748 siw_dbg_qp(qp, "wr must be empty for user mapped sq\n");
749 *bad_wr = wr;
750 return -EINVAL;
751 }
752
753
754
755
756
757 if (!down_read_trylock(&qp->state_lock)) {
758 if (qp->attrs.state == SIW_QP_STATE_ERROR) {
759
760
761
762
763
764
765
766
767
768 rv = siw_sq_flush_wr(qp, wr, bad_wr);
769 } else {
770 siw_dbg_qp(qp, "QP locked, state %d\n",
771 qp->attrs.state);
772 *bad_wr = wr;
773 rv = -ENOTCONN;
774 }
775 return rv;
776 }
777 if (unlikely(qp->attrs.state != SIW_QP_STATE_RTS)) {
778 if (qp->attrs.state == SIW_QP_STATE_ERROR) {
779
780
781
782
783
784
785
786 rv = siw_sq_flush_wr(qp, wr, bad_wr);
787 } else {
788 siw_dbg_qp(qp, "QP out of state %d\n",
789 qp->attrs.state);
790 *bad_wr = wr;
791 rv = -ENOTCONN;
792 }
793 up_read(&qp->state_lock);
794 return rv;
795 }
796 spin_lock_irqsave(&qp->sq_lock, flags);
797
798 while (wr) {
799 u32 idx = qp->sq_put % qp->attrs.sq_size;
800 struct siw_sqe *sqe = &qp->sendq[idx];
801
802 if (sqe->flags) {
803 siw_dbg_qp(qp, "sq full\n");
804 rv = -ENOMEM;
805 break;
806 }
807 if (wr->num_sge > qp->attrs.sq_max_sges) {
808 siw_dbg_qp(qp, "too many sge's: %d\n", wr->num_sge);
809 rv = -EINVAL;
810 break;
811 }
812 sqe->id = wr->wr_id;
813
814 if ((wr->send_flags & IB_SEND_SIGNALED) ||
815 (qp->attrs.flags & SIW_SIGNAL_ALL_WR))
816 sqe->flags |= SIW_WQE_SIGNALLED;
817
818 if (wr->send_flags & IB_SEND_FENCE)
819 sqe->flags |= SIW_WQE_READ_FENCE;
820
821 switch (wr->opcode) {
822 case IB_WR_SEND:
823 case IB_WR_SEND_WITH_INV:
824 if (wr->send_flags & IB_SEND_SOLICITED)
825 sqe->flags |= SIW_WQE_SOLICITED;
826
827 if (!(wr->send_flags & IB_SEND_INLINE)) {
828 siw_copy_sgl(wr->sg_list, sqe->sge,
829 wr->num_sge);
830 sqe->num_sge = wr->num_sge;
831 } else {
832 rv = siw_copy_inline_sgl(wr, sqe);
833 if (rv <= 0) {
834 rv = -EINVAL;
835 break;
836 }
837 sqe->flags |= SIW_WQE_INLINE;
838 sqe->num_sge = 1;
839 }
840 if (wr->opcode == IB_WR_SEND)
841 sqe->opcode = SIW_OP_SEND;
842 else {
843 sqe->opcode = SIW_OP_SEND_REMOTE_INV;
844 sqe->rkey = wr->ex.invalidate_rkey;
845 }
846 break;
847
848 case IB_WR_RDMA_READ_WITH_INV:
849 case IB_WR_RDMA_READ:
850
851
852
853
854
855
856
857 if (unlikely(wr->num_sge != 1)) {
858 rv = -EINVAL;
859 break;
860 }
861 siw_copy_sgl(wr->sg_list, &sqe->sge[0], 1);
862
863
864
865 sqe->raddr = rdma_wr(wr)->remote_addr;
866 sqe->rkey = rdma_wr(wr)->rkey;
867 sqe->num_sge = 1;
868
869 if (wr->opcode == IB_WR_RDMA_READ)
870 sqe->opcode = SIW_OP_READ;
871 else
872 sqe->opcode = SIW_OP_READ_LOCAL_INV;
873 break;
874
875 case IB_WR_RDMA_WRITE:
876 if (!(wr->send_flags & IB_SEND_INLINE)) {
877 siw_copy_sgl(wr->sg_list, &sqe->sge[0],
878 wr->num_sge);
879 sqe->num_sge = wr->num_sge;
880 } else {
881 rv = siw_copy_inline_sgl(wr, sqe);
882 if (unlikely(rv < 0)) {
883 rv = -EINVAL;
884 break;
885 }
886 sqe->flags |= SIW_WQE_INLINE;
887 sqe->num_sge = 1;
888 }
889 sqe->raddr = rdma_wr(wr)->remote_addr;
890 sqe->rkey = rdma_wr(wr)->rkey;
891 sqe->opcode = SIW_OP_WRITE;
892 break;
893
894 case IB_WR_REG_MR:
895 sqe->base_mr = (uintptr_t)reg_wr(wr)->mr;
896 sqe->rkey = reg_wr(wr)->key;
897 sqe->access = reg_wr(wr)->access & IWARP_ACCESS_MASK;
898 sqe->opcode = SIW_OP_REG_MR;
899 break;
900
901 case IB_WR_LOCAL_INV:
902 sqe->rkey = wr->ex.invalidate_rkey;
903 sqe->opcode = SIW_OP_INVAL_STAG;
904 break;
905
906 default:
907 siw_dbg_qp(qp, "ib wr type %d unsupported\n",
908 wr->opcode);
909 rv = -EINVAL;
910 break;
911 }
912 siw_dbg_qp(qp, "opcode %d, flags 0x%x, wr_id 0x%pK\n",
913 sqe->opcode, sqe->flags,
914 (void *)(uintptr_t)sqe->id);
915
916 if (unlikely(rv < 0))
917 break;
918
919
920 smp_wmb();
921 sqe->flags |= SIW_WQE_VALID;
922
923 qp->sq_put++;
924 wr = wr->next;
925 }
926
927
928
929
930
931
932
933
934 if (wqe->wr_status != SIW_WR_IDLE) {
935 spin_unlock_irqrestore(&qp->sq_lock, flags);
936 goto skip_direct_sending;
937 }
938 rv = siw_activate_tx(qp);
939 spin_unlock_irqrestore(&qp->sq_lock, flags);
940
941 if (rv <= 0)
942 goto skip_direct_sending;
943
944 if (qp->kernel_verbs) {
945 rv = siw_sq_start(qp);
946 } else {
947 qp->tx_ctx.in_syscall = 1;
948
949 if (siw_qp_sq_process(qp) != 0 && !(qp->tx_ctx.tx_suspend))
950 siw_qp_cm_drop(qp, 0);
951
952 qp->tx_ctx.in_syscall = 0;
953 }
954 skip_direct_sending:
955
956 up_read(&qp->state_lock);
957
958 if (rv >= 0)
959 return 0;
960
961
962
963 siw_dbg_qp(qp, "error %d\n", rv);
964
965 *bad_wr = wr;
966 return rv;
967 }
968
969
970
971
972
973
974
975
976
977
978 int siw_post_receive(struct ib_qp *base_qp, const struct ib_recv_wr *wr,
979 const struct ib_recv_wr **bad_wr)
980 {
981 struct siw_qp *qp = to_siw_qp(base_qp);
982 unsigned long flags;
983 int rv = 0;
984
985 if (qp->srq) {
986 *bad_wr = wr;
987 return -EOPNOTSUPP;
988 }
989 if (!qp->kernel_verbs) {
990 siw_dbg_qp(qp, "no kernel post_recv for user mapped sq\n");
991 *bad_wr = wr;
992 return -EINVAL;
993 }
994
995
996
997
998
999 if (!down_read_trylock(&qp->state_lock)) {
1000 if (qp->attrs.state == SIW_QP_STATE_ERROR) {
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010 rv = siw_rq_flush_wr(qp, wr, bad_wr);
1011 } else {
1012 siw_dbg_qp(qp, "QP locked, state %d\n",
1013 qp->attrs.state);
1014 *bad_wr = wr;
1015 rv = -ENOTCONN;
1016 }
1017 return rv;
1018 }
1019 if (qp->attrs.state > SIW_QP_STATE_RTS) {
1020 if (qp->attrs.state == SIW_QP_STATE_ERROR) {
1021
1022
1023
1024
1025
1026
1027
1028 rv = siw_rq_flush_wr(qp, wr, bad_wr);
1029 } else {
1030 siw_dbg_qp(qp, "QP out of state %d\n",
1031 qp->attrs.state);
1032 *bad_wr = wr;
1033 rv = -ENOTCONN;
1034 }
1035 up_read(&qp->state_lock);
1036 return rv;
1037 }
1038
1039
1040
1041
1042 spin_lock_irqsave(&qp->rq_lock, flags);
1043
1044 while (wr) {
1045 u32 idx = qp->rq_put % qp->attrs.rq_size;
1046 struct siw_rqe *rqe = &qp->recvq[idx];
1047
1048 if (rqe->flags) {
1049 siw_dbg_qp(qp, "RQ full\n");
1050 rv = -ENOMEM;
1051 break;
1052 }
1053 if (wr->num_sge > qp->attrs.rq_max_sges) {
1054 siw_dbg_qp(qp, "too many sge's: %d\n", wr->num_sge);
1055 rv = -EINVAL;
1056 break;
1057 }
1058 rqe->id = wr->wr_id;
1059 rqe->num_sge = wr->num_sge;
1060 siw_copy_sgl(wr->sg_list, rqe->sge, wr->num_sge);
1061
1062
1063 smp_wmb();
1064
1065 rqe->flags = SIW_WQE_VALID;
1066
1067 qp->rq_put++;
1068 wr = wr->next;
1069 }
1070 spin_unlock_irqrestore(&qp->rq_lock, flags);
1071
1072 up_read(&qp->state_lock);
1073
1074 if (rv < 0) {
1075 siw_dbg_qp(qp, "error %d\n", rv);
1076 *bad_wr = wr;
1077 }
1078 return rv > 0 ? 0 : rv;
1079 }
1080
1081 void siw_destroy_cq(struct ib_cq *base_cq, struct ib_udata *udata)
1082 {
1083 struct siw_cq *cq = to_siw_cq(base_cq);
1084 struct siw_device *sdev = to_siw_dev(base_cq->device);
1085 struct siw_ucontext *ctx =
1086 rdma_udata_to_drv_context(udata, struct siw_ucontext,
1087 base_ucontext);
1088
1089 siw_dbg_cq(cq, "free CQ resources\n");
1090
1091 siw_cq_flush(cq);
1092
1093 if (ctx && cq->xa_cq_index != SIW_INVAL_UOBJ_KEY)
1094 kfree(xa_erase(&ctx->xa, cq->xa_cq_index));
1095
1096 atomic_dec(&sdev->num_cq);
1097
1098 vfree(cq->queue);
1099 }
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111 int siw_create_cq(struct ib_cq *base_cq, const struct ib_cq_init_attr *attr,
1112 struct ib_udata *udata)
1113 {
1114 struct siw_device *sdev = to_siw_dev(base_cq->device);
1115 struct siw_cq *cq = to_siw_cq(base_cq);
1116 int rv, size = attr->cqe;
1117
1118 if (atomic_inc_return(&sdev->num_cq) > SIW_MAX_CQ) {
1119 siw_dbg(base_cq->device, "too many CQ's\n");
1120 rv = -ENOMEM;
1121 goto err_out;
1122 }
1123 if (size < 1 || size > sdev->attrs.max_cqe) {
1124 siw_dbg(base_cq->device, "CQ size error: %d\n", size);
1125 rv = -EINVAL;
1126 goto err_out;
1127 }
1128 size = roundup_pow_of_two(size);
1129 cq->base_cq.cqe = size;
1130 cq->num_cqe = size;
1131 cq->xa_cq_index = SIW_INVAL_UOBJ_KEY;
1132
1133 if (!udata) {
1134 cq->kernel_verbs = 1;
1135 cq->queue = vzalloc(size * sizeof(struct siw_cqe) +
1136 sizeof(struct siw_cq_ctrl));
1137 } else {
1138 cq->queue = vmalloc_user(size * sizeof(struct siw_cqe) +
1139 sizeof(struct siw_cq_ctrl));
1140 }
1141 if (cq->queue == NULL) {
1142 rv = -ENOMEM;
1143 goto err_out;
1144 }
1145 get_random_bytes(&cq->id, 4);
1146 siw_dbg(base_cq->device, "new CQ [%u]\n", cq->id);
1147
1148 spin_lock_init(&cq->lock);
1149
1150 cq->notify = (struct siw_cq_ctrl *)&cq->queue[size];
1151
1152 if (udata) {
1153 struct siw_uresp_create_cq uresp = {};
1154 struct siw_ucontext *ctx =
1155 rdma_udata_to_drv_context(udata, struct siw_ucontext,
1156 base_ucontext);
1157
1158 cq->xa_cq_index =
1159 siw_create_uobj(ctx, cq->queue,
1160 size * sizeof(struct siw_cqe) +
1161 sizeof(struct siw_cq_ctrl));
1162 if (cq->xa_cq_index == SIW_INVAL_UOBJ_KEY) {
1163 rv = -ENOMEM;
1164 goto err_out;
1165 }
1166 uresp.cq_key = cq->xa_cq_index << PAGE_SHIFT;
1167 uresp.cq_id = cq->id;
1168 uresp.num_cqe = size;
1169
1170 if (udata->outlen < sizeof(uresp)) {
1171 rv = -EINVAL;
1172 goto err_out;
1173 }
1174 rv = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
1175 if (rv)
1176 goto err_out;
1177 }
1178 return 0;
1179
1180 err_out:
1181 siw_dbg(base_cq->device, "CQ creation failed: %d", rv);
1182
1183 if (cq && cq->queue) {
1184 struct siw_ucontext *ctx =
1185 rdma_udata_to_drv_context(udata, struct siw_ucontext,
1186 base_ucontext);
1187 if (cq->xa_cq_index != SIW_INVAL_UOBJ_KEY)
1188 kfree(xa_erase(&ctx->xa, cq->xa_cq_index));
1189 vfree(cq->queue);
1190 }
1191 atomic_dec(&sdev->num_cq);
1192
1193 return rv;
1194 }
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206 int siw_poll_cq(struct ib_cq *base_cq, int num_cqe, struct ib_wc *wc)
1207 {
1208 struct siw_cq *cq = to_siw_cq(base_cq);
1209 int i;
1210
1211 for (i = 0; i < num_cqe; i++) {
1212 if (!siw_reap_cqe(cq, wc))
1213 break;
1214 wc++;
1215 }
1216 return i;
1217 }
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235 int siw_req_notify_cq(struct ib_cq *base_cq, enum ib_cq_notify_flags flags)
1236 {
1237 struct siw_cq *cq = to_siw_cq(base_cq);
1238
1239 siw_dbg_cq(cq, "flags: 0x%02x\n", flags);
1240
1241 if ((flags & IB_CQ_SOLICITED_MASK) == IB_CQ_SOLICITED)
1242
1243
1244
1245
1246 smp_store_mb(cq->notify->flags, SIW_NOTIFY_SOLICITED);
1247 else
1248
1249
1250
1251
1252 smp_store_mb(cq->notify->flags, SIW_NOTIFY_ALL);
1253
1254 if (flags & IB_CQ_REPORT_MISSED_EVENTS)
1255 return cq->cq_put - cq->cq_get;
1256
1257 return 0;
1258 }
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268 int siw_dereg_mr(struct ib_mr *base_mr, struct ib_udata *udata)
1269 {
1270 struct siw_mr *mr = to_siw_mr(base_mr);
1271 struct siw_device *sdev = to_siw_dev(base_mr->device);
1272
1273 siw_dbg_mem(mr->mem, "deregister MR\n");
1274
1275 atomic_dec(&sdev->num_mr);
1276
1277 siw_mr_drop_mem(mr);
1278 kfree_rcu(mr, rcu);
1279
1280 return 0;
1281 }
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295 struct ib_mr *siw_reg_user_mr(struct ib_pd *pd, u64 start, u64 len,
1296 u64 rnic_va, int rights, struct ib_udata *udata)
1297 {
1298 struct siw_mr *mr = NULL;
1299 struct siw_umem *umem = NULL;
1300 struct siw_ureq_reg_mr ureq;
1301 struct siw_device *sdev = to_siw_dev(pd->device);
1302
1303 unsigned long mem_limit = rlimit(RLIMIT_MEMLOCK);
1304 int rv;
1305
1306 siw_dbg_pd(pd, "start: 0x%pK, va: 0x%pK, len: %llu\n",
1307 (void *)(uintptr_t)start, (void *)(uintptr_t)rnic_va,
1308 (unsigned long long)len);
1309
1310 if (atomic_inc_return(&sdev->num_mr) > SIW_MAX_MR) {
1311 siw_dbg_pd(pd, "too many mr's\n");
1312 rv = -ENOMEM;
1313 goto err_out;
1314 }
1315 if (!len) {
1316 rv = -EINVAL;
1317 goto err_out;
1318 }
1319 if (mem_limit != RLIM_INFINITY) {
1320 unsigned long num_pages =
1321 (PAGE_ALIGN(len + (start & ~PAGE_MASK))) >> PAGE_SHIFT;
1322 mem_limit >>= PAGE_SHIFT;
1323
1324 if (num_pages > mem_limit - current->mm->locked_vm) {
1325 siw_dbg_pd(pd, "pages req %lu, max %lu, lock %lu\n",
1326 num_pages, mem_limit,
1327 current->mm->locked_vm);
1328 rv = -ENOMEM;
1329 goto err_out;
1330 }
1331 }
1332 umem = siw_umem_get(start, len, ib_access_writable(rights));
1333 if (IS_ERR(umem)) {
1334 rv = PTR_ERR(umem);
1335 siw_dbg_pd(pd, "getting user memory failed: %d\n", rv);
1336 umem = NULL;
1337 goto err_out;
1338 }
1339 mr = kzalloc(sizeof(*mr), GFP_KERNEL);
1340 if (!mr) {
1341 rv = -ENOMEM;
1342 goto err_out;
1343 }
1344 rv = siw_mr_add_mem(mr, pd, umem, start, len, rights);
1345 if (rv)
1346 goto err_out;
1347
1348 if (udata) {
1349 struct siw_uresp_reg_mr uresp = {};
1350 struct siw_mem *mem = mr->mem;
1351
1352 if (udata->inlen < sizeof(ureq)) {
1353 rv = -EINVAL;
1354 goto err_out;
1355 }
1356 rv = ib_copy_from_udata(&ureq, udata, sizeof(ureq));
1357 if (rv)
1358 goto err_out;
1359
1360 mr->base_mr.lkey |= ureq.stag_key;
1361 mr->base_mr.rkey |= ureq.stag_key;
1362 mem->stag |= ureq.stag_key;
1363 uresp.stag = mem->stag;
1364
1365 if (udata->outlen < sizeof(uresp)) {
1366 rv = -EINVAL;
1367 goto err_out;
1368 }
1369 rv = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
1370 if (rv)
1371 goto err_out;
1372 }
1373 mr->mem->stag_valid = 1;
1374
1375 return &mr->base_mr;
1376
1377 err_out:
1378 atomic_dec(&sdev->num_mr);
1379 if (mr) {
1380 if (mr->mem)
1381 siw_mr_drop_mem(mr);
1382 kfree_rcu(mr, rcu);
1383 } else {
1384 if (umem)
1385 siw_umem_release(umem, false);
1386 }
1387 return ERR_PTR(rv);
1388 }
1389
1390 struct ib_mr *siw_alloc_mr(struct ib_pd *pd, enum ib_mr_type mr_type,
1391 u32 max_sge, struct ib_udata *udata)
1392 {
1393 struct siw_device *sdev = to_siw_dev(pd->device);
1394 struct siw_mr *mr = NULL;
1395 struct siw_pbl *pbl = NULL;
1396 int rv;
1397
1398 if (atomic_inc_return(&sdev->num_mr) > SIW_MAX_MR) {
1399 siw_dbg_pd(pd, "too many mr's\n");
1400 rv = -ENOMEM;
1401 goto err_out;
1402 }
1403 if (mr_type != IB_MR_TYPE_MEM_REG) {
1404 siw_dbg_pd(pd, "mr type %d unsupported\n", mr_type);
1405 rv = -EOPNOTSUPP;
1406 goto err_out;
1407 }
1408 if (max_sge > SIW_MAX_SGE_PBL) {
1409 siw_dbg_pd(pd, "too many sge's: %d\n", max_sge);
1410 rv = -ENOMEM;
1411 goto err_out;
1412 }
1413 pbl = siw_pbl_alloc(max_sge);
1414 if (IS_ERR(pbl)) {
1415 rv = PTR_ERR(pbl);
1416 siw_dbg_pd(pd, "pbl allocation failed: %d\n", rv);
1417 pbl = NULL;
1418 goto err_out;
1419 }
1420 mr = kzalloc(sizeof(*mr), GFP_KERNEL);
1421 if (!mr) {
1422 rv = -ENOMEM;
1423 goto err_out;
1424 }
1425 rv = siw_mr_add_mem(mr, pd, pbl, 0, max_sge * PAGE_SIZE, 0);
1426 if (rv)
1427 goto err_out;
1428
1429 mr->mem->is_pbl = 1;
1430
1431 siw_dbg_pd(pd, "[MEM %u]: success\n", mr->mem->stag);
1432
1433 return &mr->base_mr;
1434
1435 err_out:
1436 atomic_dec(&sdev->num_mr);
1437
1438 if (!mr) {
1439 kfree(pbl);
1440 } else {
1441 if (mr->mem)
1442 siw_mr_drop_mem(mr);
1443 kfree_rcu(mr, rcu);
1444 }
1445 siw_dbg_pd(pd, "failed: %d\n", rv);
1446
1447 return ERR_PTR(rv);
1448 }
1449
1450
1451 static int siw_set_pbl_page(struct ib_mr *base_mr, u64 buf_addr)
1452 {
1453 return 0;
1454 }
1455
1456 int siw_map_mr_sg(struct ib_mr *base_mr, struct scatterlist *sl, int num_sle,
1457 unsigned int *sg_off)
1458 {
1459 struct scatterlist *slp;
1460 struct siw_mr *mr = to_siw_mr(base_mr);
1461 struct siw_mem *mem = mr->mem;
1462 struct siw_pbl *pbl = mem->pbl;
1463 struct siw_pble *pble;
1464 unsigned long pbl_size;
1465 int i, rv;
1466
1467 if (!pbl) {
1468 siw_dbg_mem(mem, "no PBL allocated\n");
1469 return -EINVAL;
1470 }
1471 pble = pbl->pbe;
1472
1473 if (pbl->max_buf < num_sle) {
1474 siw_dbg_mem(mem, "too many SGE's: %d > %d\n",
1475 mem->pbl->max_buf, num_sle);
1476 return -ENOMEM;
1477 }
1478 for_each_sg(sl, slp, num_sle, i) {
1479 if (sg_dma_len(slp) == 0) {
1480 siw_dbg_mem(mem, "empty SGE\n");
1481 return -EINVAL;
1482 }
1483 if (i == 0) {
1484 pble->addr = sg_dma_address(slp);
1485 pble->size = sg_dma_len(slp);
1486 pble->pbl_off = 0;
1487 pbl_size = pble->size;
1488 pbl->num_buf = 1;
1489 } else {
1490
1491 if (pble->addr + pble->size == sg_dma_address(slp)) {
1492 pble->size += sg_dma_len(slp);
1493 } else {
1494 pble++;
1495 pbl->num_buf++;
1496 pble->addr = sg_dma_address(slp);
1497 pble->size = sg_dma_len(slp);
1498 pble->pbl_off = pbl_size;
1499 }
1500 pbl_size += sg_dma_len(slp);
1501 }
1502 siw_dbg_mem(mem,
1503 "sge[%d], size %u, addr 0x%p, total %lu\n",
1504 i, pble->size, (void *)(uintptr_t)pble->addr,
1505 pbl_size);
1506 }
1507 rv = ib_sg_to_pages(base_mr, sl, num_sle, sg_off, siw_set_pbl_page);
1508 if (rv > 0) {
1509 mem->len = base_mr->length;
1510 mem->va = base_mr->iova;
1511 siw_dbg_mem(mem,
1512 "%llu bytes, start 0x%pK, %u SLE to %u entries\n",
1513 mem->len, (void *)(uintptr_t)mem->va, num_sle,
1514 pbl->num_buf);
1515 }
1516 return rv;
1517 }
1518
1519
1520
1521
1522
1523
1524 struct ib_mr *siw_get_dma_mr(struct ib_pd *pd, int rights)
1525 {
1526 struct siw_device *sdev = to_siw_dev(pd->device);
1527 struct siw_mr *mr = NULL;
1528 int rv;
1529
1530 if (atomic_inc_return(&sdev->num_mr) > SIW_MAX_MR) {
1531 siw_dbg_pd(pd, "too many mr's\n");
1532 rv = -ENOMEM;
1533 goto err_out;
1534 }
1535 mr = kzalloc(sizeof(*mr), GFP_KERNEL);
1536 if (!mr) {
1537 rv = -ENOMEM;
1538 goto err_out;
1539 }
1540 rv = siw_mr_add_mem(mr, pd, NULL, 0, ULONG_MAX, rights);
1541 if (rv)
1542 goto err_out;
1543
1544 mr->mem->stag_valid = 1;
1545
1546 siw_dbg_pd(pd, "[MEM %u]: success\n", mr->mem->stag);
1547
1548 return &mr->base_mr;
1549
1550 err_out:
1551 if (rv)
1552 kfree(mr);
1553
1554 atomic_dec(&sdev->num_mr);
1555
1556 return ERR_PTR(rv);
1557 }
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569 int siw_create_srq(struct ib_srq *base_srq,
1570 struct ib_srq_init_attr *init_attrs, struct ib_udata *udata)
1571 {
1572 struct siw_srq *srq = to_siw_srq(base_srq);
1573 struct ib_srq_attr *attrs = &init_attrs->attr;
1574 struct siw_device *sdev = to_siw_dev(base_srq->device);
1575 struct siw_ucontext *ctx =
1576 rdma_udata_to_drv_context(udata, struct siw_ucontext,
1577 base_ucontext);
1578 int rv;
1579
1580 if (atomic_inc_return(&sdev->num_srq) > SIW_MAX_SRQ) {
1581 siw_dbg_pd(base_srq->pd, "too many SRQ's\n");
1582 rv = -ENOMEM;
1583 goto err_out;
1584 }
1585 if (attrs->max_wr == 0 || attrs->max_wr > SIW_MAX_SRQ_WR ||
1586 attrs->max_sge > SIW_MAX_SGE || attrs->srq_limit > attrs->max_wr) {
1587 rv = -EINVAL;
1588 goto err_out;
1589 }
1590 srq->max_sge = attrs->max_sge;
1591 srq->num_rqe = roundup_pow_of_two(attrs->max_wr);
1592 srq->xa_srq_index = SIW_INVAL_UOBJ_KEY;
1593 srq->limit = attrs->srq_limit;
1594 if (srq->limit)
1595 srq->armed = 1;
1596
1597 srq->kernel_verbs = !udata;
1598
1599 if (udata)
1600 srq->recvq =
1601 vmalloc_user(srq->num_rqe * sizeof(struct siw_rqe));
1602 else
1603 srq->recvq = vzalloc(srq->num_rqe * sizeof(struct siw_rqe));
1604
1605 if (srq->recvq == NULL) {
1606 rv = -ENOMEM;
1607 goto err_out;
1608 }
1609 if (udata) {
1610 struct siw_uresp_create_srq uresp = {};
1611
1612 srq->xa_srq_index = siw_create_uobj(
1613 ctx, srq->recvq, srq->num_rqe * sizeof(struct siw_rqe));
1614
1615 if (srq->xa_srq_index == SIW_INVAL_UOBJ_KEY) {
1616 rv = -ENOMEM;
1617 goto err_out;
1618 }
1619 uresp.srq_key = srq->xa_srq_index;
1620 uresp.num_rqe = srq->num_rqe;
1621
1622 if (udata->outlen < sizeof(uresp)) {
1623 rv = -EINVAL;
1624 goto err_out;
1625 }
1626 rv = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
1627 if (rv)
1628 goto err_out;
1629 }
1630 spin_lock_init(&srq->lock);
1631
1632 siw_dbg_pd(base_srq->pd, "[SRQ]: success\n");
1633
1634 return 0;
1635
1636 err_out:
1637 if (srq->recvq) {
1638 if (ctx && srq->xa_srq_index != SIW_INVAL_UOBJ_KEY)
1639 kfree(xa_erase(&ctx->xa, srq->xa_srq_index));
1640 vfree(srq->recvq);
1641 }
1642 atomic_dec(&sdev->num_srq);
1643
1644 return rv;
1645 }
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656 int siw_modify_srq(struct ib_srq *base_srq, struct ib_srq_attr *attrs,
1657 enum ib_srq_attr_mask attr_mask, struct ib_udata *udata)
1658 {
1659 struct siw_srq *srq = to_siw_srq(base_srq);
1660 unsigned long flags;
1661 int rv = 0;
1662
1663 spin_lock_irqsave(&srq->lock, flags);
1664
1665 if (attr_mask & IB_SRQ_MAX_WR) {
1666
1667 rv = -EOPNOTSUPP;
1668 goto out;
1669 }
1670 if (attr_mask & IB_SRQ_LIMIT) {
1671 if (attrs->srq_limit) {
1672 if (unlikely(attrs->srq_limit > srq->num_rqe)) {
1673 rv = -EINVAL;
1674 goto out;
1675 }
1676 srq->armed = 1;
1677 } else {
1678 srq->armed = 0;
1679 }
1680 srq->limit = attrs->srq_limit;
1681 }
1682 out:
1683 spin_unlock_irqrestore(&srq->lock, flags);
1684
1685 return rv;
1686 }
1687
1688
1689
1690
1691
1692
1693 int siw_query_srq(struct ib_srq *base_srq, struct ib_srq_attr *attrs)
1694 {
1695 struct siw_srq *srq = to_siw_srq(base_srq);
1696 unsigned long flags;
1697
1698 spin_lock_irqsave(&srq->lock, flags);
1699
1700 attrs->max_wr = srq->num_rqe;
1701 attrs->max_sge = srq->max_sge;
1702 attrs->srq_limit = srq->limit;
1703
1704 spin_unlock_irqrestore(&srq->lock, flags);
1705
1706 return 0;
1707 }
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717 void siw_destroy_srq(struct ib_srq *base_srq, struct ib_udata *udata)
1718 {
1719 struct siw_srq *srq = to_siw_srq(base_srq);
1720 struct siw_device *sdev = to_siw_dev(base_srq->device);
1721 struct siw_ucontext *ctx =
1722 rdma_udata_to_drv_context(udata, struct siw_ucontext,
1723 base_ucontext);
1724
1725 if (ctx && srq->xa_srq_index != SIW_INVAL_UOBJ_KEY)
1726 kfree(xa_erase(&ctx->xa, srq->xa_srq_index));
1727
1728 vfree(srq->recvq);
1729 atomic_dec(&sdev->num_srq);
1730 }
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744 int siw_post_srq_recv(struct ib_srq *base_srq, const struct ib_recv_wr *wr,
1745 const struct ib_recv_wr **bad_wr)
1746 {
1747 struct siw_srq *srq = to_siw_srq(base_srq);
1748 unsigned long flags;
1749 int rv = 0;
1750
1751 if (unlikely(!srq->kernel_verbs)) {
1752 siw_dbg_pd(base_srq->pd,
1753 "[SRQ]: no kernel post_recv for mapped srq\n");
1754 rv = -EINVAL;
1755 goto out;
1756 }
1757
1758
1759
1760
1761
1762 spin_lock_irqsave(&srq->lock, flags);
1763
1764 while (wr) {
1765 u32 idx = srq->rq_put % srq->num_rqe;
1766 struct siw_rqe *rqe = &srq->recvq[idx];
1767
1768 if (rqe->flags) {
1769 siw_dbg_pd(base_srq->pd, "SRQ full\n");
1770 rv = -ENOMEM;
1771 break;
1772 }
1773 if (unlikely(wr->num_sge > srq->max_sge)) {
1774 siw_dbg_pd(base_srq->pd,
1775 "[SRQ]: too many sge's: %d\n", wr->num_sge);
1776 rv = -EINVAL;
1777 break;
1778 }
1779 rqe->id = wr->wr_id;
1780 rqe->num_sge = wr->num_sge;
1781 siw_copy_sgl(wr->sg_list, rqe->sge, wr->num_sge);
1782
1783
1784 smp_wmb();
1785
1786 rqe->flags = SIW_WQE_VALID;
1787
1788 srq->rq_put++;
1789 wr = wr->next;
1790 }
1791 spin_unlock_irqrestore(&srq->lock, flags);
1792 out:
1793 if (unlikely(rv < 0)) {
1794 siw_dbg_pd(base_srq->pd, "[SRQ]: error %d\n", rv);
1795 *bad_wr = wr;
1796 }
1797 return rv;
1798 }
1799
1800 void siw_qp_event(struct siw_qp *qp, enum ib_event_type etype)
1801 {
1802 struct ib_event event;
1803 struct ib_qp *base_qp = qp->ib_qp;
1804
1805
1806
1807
1808
1809 if (qp->attrs.flags & SIW_QP_IN_DESTROY)
1810 return;
1811
1812 event.event = etype;
1813 event.device = base_qp->device;
1814 event.element.qp = base_qp;
1815
1816 if (base_qp->event_handler) {
1817 siw_dbg_qp(qp, "reporting event %d\n", etype);
1818 base_qp->event_handler(&event, base_qp->qp_context);
1819 }
1820 }
1821
1822 void siw_cq_event(struct siw_cq *cq, enum ib_event_type etype)
1823 {
1824 struct ib_event event;
1825 struct ib_cq *base_cq = &cq->base_cq;
1826
1827 event.event = etype;
1828 event.device = base_cq->device;
1829 event.element.cq = base_cq;
1830
1831 if (base_cq->event_handler) {
1832 siw_dbg_cq(cq, "reporting CQ event %d\n", etype);
1833 base_cq->event_handler(&event, base_cq->cq_context);
1834 }
1835 }
1836
1837 void siw_srq_event(struct siw_srq *srq, enum ib_event_type etype)
1838 {
1839 struct ib_event event;
1840 struct ib_srq *base_srq = &srq->base_srq;
1841
1842 event.event = etype;
1843 event.device = base_srq->device;
1844 event.element.srq = base_srq;
1845
1846 if (base_srq->event_handler) {
1847 siw_dbg_pd(srq->base_srq.pd,
1848 "reporting SRQ event %d\n", etype);
1849 base_srq->event_handler(&event, base_srq->srq_context);
1850 }
1851 }
1852
1853 void siw_port_event(struct siw_device *sdev, u8 port, enum ib_event_type etype)
1854 {
1855 struct ib_event event;
1856
1857 event.event = etype;
1858 event.device = &sdev->base_dev;
1859 event.element.port_num = port;
1860
1861 siw_dbg(&sdev->base_dev, "reporting port event %d\n", etype);
1862
1863 ib_dispatch_event(&event);
1864 }