1 /*
2 * Copyright (c) 2006, 2007, 2008 QLogic Corporation. All rights reserved.
3 * Copyright (c) 2005, 2006 PathScale, Inc. All rights reserved.
4 *
5 * This software is available to you under a choice of one of two
6 * licenses. You may choose to be licensed under the terms of the GNU
7 * General Public License (GPL) Version 2, available from the file
8 * COPYING in the main directory of this source tree, or the
9 * OpenIB.org BSD license below:
10 *
11 * Redistribution and use in source and binary forms, with or
12 * without modification, are permitted provided that the following
13 * conditions are met:
14 *
15 * - Redistributions of source code must retain the above
16 * copyright notice, this list of conditions and the following
17 * disclaimer.
18 *
19 * - Redistributions in binary form must reproduce the above
20 * copyright notice, this list of conditions and the following
21 * disclaimer in the documentation and/or other materials
22 * provided with the distribution.
23 *
24 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
25 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
26 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
27 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
28 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
29 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
30 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
31 * SOFTWARE.
32 */
33
34 #include <linux/spinlock.h>
35
36 #include "ipath_verbs.h"
37 #include "ipath_kernel.h"
38
39 /*
40 * Convert the AETH RNR timeout code into the number of milliseconds.
41 */
42 const u32 ib_ipath_rnr_table[32] = {
43 656, /* 0 */
44 1, /* 1 */
45 1, /* 2 */
46 1, /* 3 */
47 1, /* 4 */
48 1, /* 5 */
49 1, /* 6 */
50 1, /* 7 */
51 1, /* 8 */
52 1, /* 9 */
53 1, /* A */
54 1, /* B */
55 1, /* C */
56 1, /* D */
57 2, /* E */
58 2, /* F */
59 3, /* 10 */
60 4, /* 11 */
61 6, /* 12 */
62 8, /* 13 */
63 11, /* 14 */
64 16, /* 15 */
65 21, /* 16 */
66 31, /* 17 */
67 41, /* 18 */
68 62, /* 19 */
69 82, /* 1A */
70 123, /* 1B */
71 164, /* 1C */
72 246, /* 1D */
73 328, /* 1E */
74 492 /* 1F */
75 };
76
77 /**
78 * ipath_insert_rnr_queue - put QP on the RNR timeout list for the device
79 * @qp: the QP
80 *
81 * Called with the QP s_lock held and interrupts disabled.
82 * XXX Use a simple list for now. We might need a priority
83 * queue if we have lots of QPs waiting for RNR timeouts
84 * but that should be rare.
85 */
ipath_insert_rnr_queue(struct ipath_qp * qp)86 void ipath_insert_rnr_queue(struct ipath_qp *qp)
87 {
88 struct ipath_ibdev *dev = to_idev(qp->ibqp.device);
89
90 /* We already did a spin_lock_irqsave(), so just use spin_lock */
91 spin_lock(&dev->pending_lock);
92 if (list_empty(&dev->rnrwait))
93 list_add(&qp->timerwait, &dev->rnrwait);
94 else {
95 struct list_head *l = &dev->rnrwait;
96 struct ipath_qp *nqp = list_entry(l->next, struct ipath_qp,
97 timerwait);
98
99 while (qp->s_rnr_timeout >= nqp->s_rnr_timeout) {
100 qp->s_rnr_timeout -= nqp->s_rnr_timeout;
101 l = l->next;
102 if (l->next == &dev->rnrwait) {
103 nqp = NULL;
104 break;
105 }
106 nqp = list_entry(l->next, struct ipath_qp,
107 timerwait);
108 }
109 if (nqp)
110 nqp->s_rnr_timeout -= qp->s_rnr_timeout;
111 list_add(&qp->timerwait, l);
112 }
113 spin_unlock(&dev->pending_lock);
114 }
115
116 /**
117 * ipath_init_sge - Validate a RWQE and fill in the SGE state
118 * @qp: the QP
119 *
120 * Return 1 if OK.
121 */
ipath_init_sge(struct ipath_qp * qp,struct ipath_rwqe * wqe,u32 * lengthp,struct ipath_sge_state * ss)122 int ipath_init_sge(struct ipath_qp *qp, struct ipath_rwqe *wqe,
123 u32 *lengthp, struct ipath_sge_state *ss)
124 {
125 int i, j, ret;
126 struct ib_wc wc;
127
128 *lengthp = 0;
129 for (i = j = 0; i < wqe->num_sge; i++) {
130 if (wqe->sg_list[i].length == 0)
131 continue;
132 /* Check LKEY */
133 if (!ipath_lkey_ok(qp, j ? &ss->sg_list[j - 1] : &ss->sge,
134 &wqe->sg_list[i], IB_ACCESS_LOCAL_WRITE))
135 goto bad_lkey;
136 *lengthp += wqe->sg_list[i].length;
137 j++;
138 }
139 ss->num_sge = j;
140 ret = 1;
141 goto bail;
142
143 bad_lkey:
144 memset(&wc, 0, sizeof(wc));
145 wc.wr_id = wqe->wr_id;
146 wc.status = IB_WC_LOC_PROT_ERR;
147 wc.opcode = IB_WC_RECV;
148 wc.qp = &qp->ibqp;
149 /* Signal solicited completion event. */
150 ipath_cq_enter(to_icq(qp->ibqp.recv_cq), &wc, 1);
151 ret = 0;
152 bail:
153 return ret;
154 }
155
156 /**
157 * ipath_get_rwqe - copy the next RWQE into the QP's RWQE
158 * @qp: the QP
159 * @wr_id_only: update qp->r_wr_id only, not qp->r_sge
160 *
161 * Return 0 if no RWQE is available, otherwise return 1.
162 *
163 * Can be called from interrupt level.
164 */
ipath_get_rwqe(struct ipath_qp * qp,int wr_id_only)165 int ipath_get_rwqe(struct ipath_qp *qp, int wr_id_only)
166 {
167 unsigned long flags;
168 struct ipath_rq *rq;
169 struct ipath_rwq *wq;
170 struct ipath_srq *srq;
171 struct ipath_rwqe *wqe;
172 void (*handler)(struct ib_event *, void *);
173 u32 tail;
174 int ret;
175
176 if (qp->ibqp.srq) {
177 srq = to_isrq(qp->ibqp.srq);
178 handler = srq->ibsrq.event_handler;
179 rq = &srq->rq;
180 } else {
181 srq = NULL;
182 handler = NULL;
183 rq = &qp->r_rq;
184 }
185
186 spin_lock_irqsave(&rq->lock, flags);
187 if (!(ib_ipath_state_ops[qp->state] & IPATH_PROCESS_RECV_OK)) {
188 ret = 0;
189 goto unlock;
190 }
191
192 wq = rq->wq;
193 tail = wq->tail;
194 /* Validate tail before using it since it is user writable. */
195 if (tail >= rq->size)
196 tail = 0;
197 do {
198 if (unlikely(tail == wq->head)) {
199 ret = 0;
200 goto unlock;
201 }
202 /* Make sure entry is read after head index is read. */
203 smp_rmb();
204 wqe = get_rwqe_ptr(rq, tail);
205 if (++tail >= rq->size)
206 tail = 0;
207 if (wr_id_only)
208 break;
209 qp->r_sge.sg_list = qp->r_sg_list;
210 } while (!ipath_init_sge(qp, wqe, &qp->r_len, &qp->r_sge));
211 qp->r_wr_id = wqe->wr_id;
212 wq->tail = tail;
213
214 ret = 1;
215 set_bit(IPATH_R_WRID_VALID, &qp->r_aflags);
216 if (handler) {
217 u32 n;
218
219 /*
220 * validate head pointer value and compute
221 * the number of remaining WQEs.
222 */
223 n = wq->head;
224 if (n >= rq->size)
225 n = 0;
226 if (n < tail)
227 n += rq->size - tail;
228 else
229 n -= tail;
230 if (n < srq->limit) {
231 struct ib_event ev;
232
233 srq->limit = 0;
234 spin_unlock_irqrestore(&rq->lock, flags);
235 ev.device = qp->ibqp.device;
236 ev.element.srq = qp->ibqp.srq;
237 ev.event = IB_EVENT_SRQ_LIMIT_REACHED;
238 handler(&ev, srq->ibsrq.srq_context);
239 goto bail;
240 }
241 }
242 unlock:
243 spin_unlock_irqrestore(&rq->lock, flags);
244 bail:
245 return ret;
246 }
247
248 /**
249 * ipath_ruc_loopback - handle UC and RC lookback requests
250 * @sqp: the sending QP
251 *
252 * This is called from ipath_do_send() to
253 * forward a WQE addressed to the same HCA.
254 * Note that although we are single threaded due to the tasklet, we still
255 * have to protect against post_send(). We don't have to worry about
256 * receive interrupts since this is a connected protocol and all packets
257 * will pass through here.
258 */
ipath_ruc_loopback(struct ipath_qp * sqp)259 static void ipath_ruc_loopback(struct ipath_qp *sqp)
260 {
261 struct ipath_ibdev *dev = to_idev(sqp->ibqp.device);
262 struct ipath_qp *qp;
263 struct ipath_swqe *wqe;
264 struct ipath_sge *sge;
265 unsigned long flags;
266 struct ib_wc wc;
267 u64 sdata;
268 atomic64_t *maddr;
269 enum ib_wc_status send_status;
270
271 /*
272 * Note that we check the responder QP state after
273 * checking the requester's state.
274 */
275 qp = ipath_lookup_qpn(&dev->qp_table, sqp->remote_qpn);
276
277 spin_lock_irqsave(&sqp->s_lock, flags);
278
279 /* Return if we are already busy processing a work request. */
280 if ((sqp->s_flags & (IPATH_S_BUSY | IPATH_S_ANY_WAIT)) ||
281 !(ib_ipath_state_ops[sqp->state] & IPATH_PROCESS_OR_FLUSH_SEND))
282 goto unlock;
283
284 sqp->s_flags |= IPATH_S_BUSY;
285
286 again:
287 if (sqp->s_last == sqp->s_head)
288 goto clr_busy;
289 wqe = get_swqe_ptr(sqp, sqp->s_last);
290
291 /* Return if it is not OK to start a new work reqeust. */
292 if (!(ib_ipath_state_ops[sqp->state] & IPATH_PROCESS_NEXT_SEND_OK)) {
293 if (!(ib_ipath_state_ops[sqp->state] & IPATH_FLUSH_SEND))
294 goto clr_busy;
295 /* We are in the error state, flush the work request. */
296 send_status = IB_WC_WR_FLUSH_ERR;
297 goto flush_send;
298 }
299
300 /*
301 * We can rely on the entry not changing without the s_lock
302 * being held until we update s_last.
303 * We increment s_cur to indicate s_last is in progress.
304 */
305 if (sqp->s_last == sqp->s_cur) {
306 if (++sqp->s_cur >= sqp->s_size)
307 sqp->s_cur = 0;
308 }
309 spin_unlock_irqrestore(&sqp->s_lock, flags);
310
311 if (!qp || !(ib_ipath_state_ops[qp->state] & IPATH_PROCESS_RECV_OK)) {
312 dev->n_pkt_drops++;
313 /*
314 * For RC, the requester would timeout and retry so
315 * shortcut the timeouts and just signal too many retries.
316 */
317 if (sqp->ibqp.qp_type == IB_QPT_RC)
318 send_status = IB_WC_RETRY_EXC_ERR;
319 else
320 send_status = IB_WC_SUCCESS;
321 goto serr;
322 }
323
324 memset(&wc, 0, sizeof wc);
325 send_status = IB_WC_SUCCESS;
326
327 sqp->s_sge.sge = wqe->sg_list[0];
328 sqp->s_sge.sg_list = wqe->sg_list + 1;
329 sqp->s_sge.num_sge = wqe->wr.num_sge;
330 sqp->s_len = wqe->length;
331 switch (wqe->wr.opcode) {
332 case IB_WR_SEND_WITH_IMM:
333 wc.wc_flags = IB_WC_WITH_IMM;
334 wc.ex.imm_data = wqe->wr.ex.imm_data;
335 /* FALLTHROUGH */
336 case IB_WR_SEND:
337 if (!ipath_get_rwqe(qp, 0))
338 goto rnr_nak;
339 break;
340
341 case IB_WR_RDMA_WRITE_WITH_IMM:
342 if (unlikely(!(qp->qp_access_flags & IB_ACCESS_REMOTE_WRITE)))
343 goto inv_err;
344 wc.wc_flags = IB_WC_WITH_IMM;
345 wc.ex.imm_data = wqe->wr.ex.imm_data;
346 if (!ipath_get_rwqe(qp, 1))
347 goto rnr_nak;
348 /* FALLTHROUGH */
349 case IB_WR_RDMA_WRITE:
350 if (unlikely(!(qp->qp_access_flags & IB_ACCESS_REMOTE_WRITE)))
351 goto inv_err;
352 if (wqe->length == 0)
353 break;
354 if (unlikely(!ipath_rkey_ok(qp, &qp->r_sge, wqe->length,
355 wqe->rdma_wr.remote_addr,
356 wqe->rdma_wr.rkey,
357 IB_ACCESS_REMOTE_WRITE)))
358 goto acc_err;
359 break;
360
361 case IB_WR_RDMA_READ:
362 if (unlikely(!(qp->qp_access_flags & IB_ACCESS_REMOTE_READ)))
363 goto inv_err;
364 if (unlikely(!ipath_rkey_ok(qp, &sqp->s_sge, wqe->length,
365 wqe->rdma_wr.remote_addr,
366 wqe->rdma_wr.rkey,
367 IB_ACCESS_REMOTE_READ)))
368 goto acc_err;
369 qp->r_sge.sge = wqe->sg_list[0];
370 qp->r_sge.sg_list = wqe->sg_list + 1;
371 qp->r_sge.num_sge = wqe->wr.num_sge;
372 break;
373
374 case IB_WR_ATOMIC_CMP_AND_SWP:
375 case IB_WR_ATOMIC_FETCH_AND_ADD:
376 if (unlikely(!(qp->qp_access_flags & IB_ACCESS_REMOTE_ATOMIC)))
377 goto inv_err;
378 if (unlikely(!ipath_rkey_ok(qp, &qp->r_sge, sizeof(u64),
379 wqe->atomic_wr.remote_addr,
380 wqe->atomic_wr.rkey,
381 IB_ACCESS_REMOTE_ATOMIC)))
382 goto acc_err;
383 /* Perform atomic OP and save result. */
384 maddr = (atomic64_t *) qp->r_sge.sge.vaddr;
385 sdata = wqe->atomic_wr.compare_add;
386 *(u64 *) sqp->s_sge.sge.vaddr =
387 (wqe->wr.opcode == IB_WR_ATOMIC_FETCH_AND_ADD) ?
388 (u64) atomic64_add_return(sdata, maddr) - sdata :
389 (u64) cmpxchg((u64 *) qp->r_sge.sge.vaddr,
390 sdata, wqe->atomic_wr.swap);
391 goto send_comp;
392
393 default:
394 send_status = IB_WC_LOC_QP_OP_ERR;
395 goto serr;
396 }
397
398 sge = &sqp->s_sge.sge;
399 while (sqp->s_len) {
400 u32 len = sqp->s_len;
401
402 if (len > sge->length)
403 len = sge->length;
404 if (len > sge->sge_length)
405 len = sge->sge_length;
406 BUG_ON(len == 0);
407 ipath_copy_sge(&qp->r_sge, sge->vaddr, len);
408 sge->vaddr += len;
409 sge->length -= len;
410 sge->sge_length -= len;
411 if (sge->sge_length == 0) {
412 if (--sqp->s_sge.num_sge)
413 *sge = *sqp->s_sge.sg_list++;
414 } else if (sge->length == 0 && sge->mr != NULL) {
415 if (++sge->n >= IPATH_SEGSZ) {
416 if (++sge->m >= sge->mr->mapsz)
417 break;
418 sge->n = 0;
419 }
420 sge->vaddr =
421 sge->mr->map[sge->m]->segs[sge->n].vaddr;
422 sge->length =
423 sge->mr->map[sge->m]->segs[sge->n].length;
424 }
425 sqp->s_len -= len;
426 }
427
428 if (!test_and_clear_bit(IPATH_R_WRID_VALID, &qp->r_aflags))
429 goto send_comp;
430
431 if (wqe->wr.opcode == IB_WR_RDMA_WRITE_WITH_IMM)
432 wc.opcode = IB_WC_RECV_RDMA_WITH_IMM;
433 else
434 wc.opcode = IB_WC_RECV;
435 wc.wr_id = qp->r_wr_id;
436 wc.status = IB_WC_SUCCESS;
437 wc.byte_len = wqe->length;
438 wc.qp = &qp->ibqp;
439 wc.src_qp = qp->remote_qpn;
440 wc.slid = qp->remote_ah_attr.dlid;
441 wc.sl = qp->remote_ah_attr.sl;
442 wc.port_num = 1;
443 /* Signal completion event if the solicited bit is set. */
444 ipath_cq_enter(to_icq(qp->ibqp.recv_cq), &wc,
445 wqe->wr.send_flags & IB_SEND_SOLICITED);
446
447 send_comp:
448 spin_lock_irqsave(&sqp->s_lock, flags);
449 flush_send:
450 sqp->s_rnr_retry = sqp->s_rnr_retry_cnt;
451 ipath_send_complete(sqp, wqe, send_status);
452 goto again;
453
454 rnr_nak:
455 /* Handle RNR NAK */
456 if (qp->ibqp.qp_type == IB_QPT_UC)
457 goto send_comp;
458 /*
459 * Note: we don't need the s_lock held since the BUSY flag
460 * makes this single threaded.
461 */
462 if (sqp->s_rnr_retry == 0) {
463 send_status = IB_WC_RNR_RETRY_EXC_ERR;
464 goto serr;
465 }
466 if (sqp->s_rnr_retry_cnt < 7)
467 sqp->s_rnr_retry--;
468 spin_lock_irqsave(&sqp->s_lock, flags);
469 if (!(ib_ipath_state_ops[sqp->state] & IPATH_PROCESS_RECV_OK))
470 goto clr_busy;
471 sqp->s_flags |= IPATH_S_WAITING;
472 dev->n_rnr_naks++;
473 sqp->s_rnr_timeout = ib_ipath_rnr_table[qp->r_min_rnr_timer];
474 ipath_insert_rnr_queue(sqp);
475 goto clr_busy;
476
477 inv_err:
478 send_status = IB_WC_REM_INV_REQ_ERR;
479 wc.status = IB_WC_LOC_QP_OP_ERR;
480 goto err;
481
482 acc_err:
483 send_status = IB_WC_REM_ACCESS_ERR;
484 wc.status = IB_WC_LOC_PROT_ERR;
485 err:
486 /* responder goes to error state */
487 ipath_rc_error(qp, wc.status);
488
489 serr:
490 spin_lock_irqsave(&sqp->s_lock, flags);
491 ipath_send_complete(sqp, wqe, send_status);
492 if (sqp->ibqp.qp_type == IB_QPT_RC) {
493 int lastwqe = ipath_error_qp(sqp, IB_WC_WR_FLUSH_ERR);
494
495 sqp->s_flags &= ~IPATH_S_BUSY;
496 spin_unlock_irqrestore(&sqp->s_lock, flags);
497 if (lastwqe) {
498 struct ib_event ev;
499
500 ev.device = sqp->ibqp.device;
501 ev.element.qp = &sqp->ibqp;
502 ev.event = IB_EVENT_QP_LAST_WQE_REACHED;
503 sqp->ibqp.event_handler(&ev, sqp->ibqp.qp_context);
504 }
505 goto done;
506 }
507 clr_busy:
508 sqp->s_flags &= ~IPATH_S_BUSY;
509 unlock:
510 spin_unlock_irqrestore(&sqp->s_lock, flags);
511 done:
512 if (qp && atomic_dec_and_test(&qp->refcount))
513 wake_up(&qp->wait);
514 }
515
want_buffer(struct ipath_devdata * dd,struct ipath_qp * qp)516 static void want_buffer(struct ipath_devdata *dd, struct ipath_qp *qp)
517 {
518 if (!(dd->ipath_flags & IPATH_HAS_SEND_DMA) ||
519 qp->ibqp.qp_type == IB_QPT_SMI) {
520 unsigned long flags;
521
522 spin_lock_irqsave(&dd->ipath_sendctrl_lock, flags);
523 dd->ipath_sendctrl |= INFINIPATH_S_PIOINTBUFAVAIL;
524 ipath_write_kreg(dd, dd->ipath_kregs->kr_sendctrl,
525 dd->ipath_sendctrl);
526 ipath_read_kreg64(dd, dd->ipath_kregs->kr_scratch);
527 spin_unlock_irqrestore(&dd->ipath_sendctrl_lock, flags);
528 }
529 }
530
531 /**
532 * ipath_no_bufs_available - tell the layer driver we need buffers
533 * @qp: the QP that caused the problem
534 * @dev: the device we ran out of buffers on
535 *
536 * Called when we run out of PIO buffers.
537 * If we are now in the error state, return zero to flush the
538 * send work request.
539 */
ipath_no_bufs_available(struct ipath_qp * qp,struct ipath_ibdev * dev)540 static int ipath_no_bufs_available(struct ipath_qp *qp,
541 struct ipath_ibdev *dev)
542 {
543 unsigned long flags;
544 int ret = 1;
545
546 /*
547 * Note that as soon as want_buffer() is called and
548 * possibly before it returns, ipath_ib_piobufavail()
549 * could be called. Therefore, put QP on the piowait list before
550 * enabling the PIO avail interrupt.
551 */
552 spin_lock_irqsave(&qp->s_lock, flags);
553 if (ib_ipath_state_ops[qp->state] & IPATH_PROCESS_SEND_OK) {
554 dev->n_piowait++;
555 qp->s_flags |= IPATH_S_WAITING;
556 qp->s_flags &= ~IPATH_S_BUSY;
557 spin_lock(&dev->pending_lock);
558 if (list_empty(&qp->piowait))
559 list_add_tail(&qp->piowait, &dev->piowait);
560 spin_unlock(&dev->pending_lock);
561 } else
562 ret = 0;
563 spin_unlock_irqrestore(&qp->s_lock, flags);
564 if (ret)
565 want_buffer(dev->dd, qp);
566 return ret;
567 }
568
569 /**
570 * ipath_make_grh - construct a GRH header
571 * @dev: a pointer to the ipath device
572 * @hdr: a pointer to the GRH header being constructed
573 * @grh: the global route address to send to
574 * @hwords: the number of 32 bit words of header being sent
575 * @nwords: the number of 32 bit words of data being sent
576 *
577 * Return the size of the header in 32 bit words.
578 */
ipath_make_grh(struct ipath_ibdev * dev,struct ib_grh * hdr,struct ib_global_route * grh,u32 hwords,u32 nwords)579 u32 ipath_make_grh(struct ipath_ibdev *dev, struct ib_grh *hdr,
580 struct ib_global_route *grh, u32 hwords, u32 nwords)
581 {
582 hdr->version_tclass_flow =
583 cpu_to_be32((6 << 28) |
584 (grh->traffic_class << 20) |
585 grh->flow_label);
586 hdr->paylen = cpu_to_be16((hwords - 2 + nwords + SIZE_OF_CRC) << 2);
587 /* next_hdr is defined by C8-7 in ch. 8.4.1 */
588 hdr->next_hdr = 0x1B;
589 hdr->hop_limit = grh->hop_limit;
590 /* The SGID is 32-bit aligned. */
591 hdr->sgid.global.subnet_prefix = dev->gid_prefix;
592 hdr->sgid.global.interface_id = dev->dd->ipath_guid;
593 hdr->dgid = grh->dgid;
594
595 /* GRH header size in 32-bit words. */
596 return sizeof(struct ib_grh) / sizeof(u32);
597 }
598
ipath_make_ruc_header(struct ipath_ibdev * dev,struct ipath_qp * qp,struct ipath_other_headers * ohdr,u32 bth0,u32 bth2)599 void ipath_make_ruc_header(struct ipath_ibdev *dev, struct ipath_qp *qp,
600 struct ipath_other_headers *ohdr,
601 u32 bth0, u32 bth2)
602 {
603 u16 lrh0;
604 u32 nwords;
605 u32 extra_bytes;
606
607 /* Construct the header. */
608 extra_bytes = -qp->s_cur_size & 3;
609 nwords = (qp->s_cur_size + extra_bytes) >> 2;
610 lrh0 = IPATH_LRH_BTH;
611 if (unlikely(qp->remote_ah_attr.ah_flags & IB_AH_GRH)) {
612 qp->s_hdrwords += ipath_make_grh(dev, &qp->s_hdr.u.l.grh,
613 &qp->remote_ah_attr.grh,
614 qp->s_hdrwords, nwords);
615 lrh0 = IPATH_LRH_GRH;
616 }
617 lrh0 |= qp->remote_ah_attr.sl << 4;
618 qp->s_hdr.lrh[0] = cpu_to_be16(lrh0);
619 qp->s_hdr.lrh[1] = cpu_to_be16(qp->remote_ah_attr.dlid);
620 qp->s_hdr.lrh[2] = cpu_to_be16(qp->s_hdrwords + nwords + SIZE_OF_CRC);
621 qp->s_hdr.lrh[3] = cpu_to_be16(dev->dd->ipath_lid |
622 qp->remote_ah_attr.src_path_bits);
623 bth0 |= ipath_get_pkey(dev->dd, qp->s_pkey_index);
624 bth0 |= extra_bytes << 20;
625 ohdr->bth[0] = cpu_to_be32(bth0 | (1 << 22));
626 ohdr->bth[1] = cpu_to_be32(qp->remote_qpn);
627 ohdr->bth[2] = cpu_to_be32(bth2);
628 }
629
630 /**
631 * ipath_do_send - perform a send on a QP
632 * @data: contains a pointer to the QP
633 *
634 * Process entries in the send work queue until credit or queue is
635 * exhausted. Only allow one CPU to send a packet per QP (tasklet).
636 * Otherwise, two threads could send packets out of order.
637 */
ipath_do_send(unsigned long data)638 void ipath_do_send(unsigned long data)
639 {
640 struct ipath_qp *qp = (struct ipath_qp *)data;
641 struct ipath_ibdev *dev = to_idev(qp->ibqp.device);
642 int (*make_req)(struct ipath_qp *qp);
643 unsigned long flags;
644
645 if ((qp->ibqp.qp_type == IB_QPT_RC ||
646 qp->ibqp.qp_type == IB_QPT_UC) &&
647 qp->remote_ah_attr.dlid == dev->dd->ipath_lid) {
648 ipath_ruc_loopback(qp);
649 goto bail;
650 }
651
652 if (qp->ibqp.qp_type == IB_QPT_RC)
653 make_req = ipath_make_rc_req;
654 else if (qp->ibqp.qp_type == IB_QPT_UC)
655 make_req = ipath_make_uc_req;
656 else
657 make_req = ipath_make_ud_req;
658
659 spin_lock_irqsave(&qp->s_lock, flags);
660
661 /* Return if we are already busy processing a work request. */
662 if ((qp->s_flags & (IPATH_S_BUSY | IPATH_S_ANY_WAIT)) ||
663 !(ib_ipath_state_ops[qp->state] & IPATH_PROCESS_OR_FLUSH_SEND)) {
664 spin_unlock_irqrestore(&qp->s_lock, flags);
665 goto bail;
666 }
667
668 qp->s_flags |= IPATH_S_BUSY;
669
670 spin_unlock_irqrestore(&qp->s_lock, flags);
671
672 again:
673 /* Check for a constructed packet to be sent. */
674 if (qp->s_hdrwords != 0) {
675 /*
676 * If no PIO bufs are available, return. An interrupt will
677 * call ipath_ib_piobufavail() when one is available.
678 */
679 if (ipath_verbs_send(qp, &qp->s_hdr, qp->s_hdrwords,
680 qp->s_cur_sge, qp->s_cur_size)) {
681 if (ipath_no_bufs_available(qp, dev))
682 goto bail;
683 }
684 dev->n_unicast_xmit++;
685 /* Record that we sent the packet and s_hdr is empty. */
686 qp->s_hdrwords = 0;
687 }
688
689 if (make_req(qp))
690 goto again;
691
692 bail:;
693 }
694
695 /*
696 * This should be called with s_lock held.
697 */
ipath_send_complete(struct ipath_qp * qp,struct ipath_swqe * wqe,enum ib_wc_status status)698 void ipath_send_complete(struct ipath_qp *qp, struct ipath_swqe *wqe,
699 enum ib_wc_status status)
700 {
701 u32 old_last, last;
702
703 if (!(ib_ipath_state_ops[qp->state] & IPATH_PROCESS_OR_FLUSH_SEND))
704 return;
705
706 /* See ch. 11.2.4.1 and 10.7.3.1 */
707 if (!(qp->s_flags & IPATH_S_SIGNAL_REQ_WR) ||
708 (wqe->wr.send_flags & IB_SEND_SIGNALED) ||
709 status != IB_WC_SUCCESS) {
710 struct ib_wc wc;
711
712 memset(&wc, 0, sizeof wc);
713 wc.wr_id = wqe->wr.wr_id;
714 wc.status = status;
715 wc.opcode = ib_ipath_wc_opcode[wqe->wr.opcode];
716 wc.qp = &qp->ibqp;
717 if (status == IB_WC_SUCCESS)
718 wc.byte_len = wqe->length;
719 ipath_cq_enter(to_icq(qp->ibqp.send_cq), &wc,
720 status != IB_WC_SUCCESS);
721 }
722
723 old_last = last = qp->s_last;
724 if (++last >= qp->s_size)
725 last = 0;
726 qp->s_last = last;
727 if (qp->s_cur == old_last)
728 qp->s_cur = last;
729 if (qp->s_tail == old_last)
730 qp->s_tail = last;
731 if (qp->state == IB_QPS_SQD && last == qp->s_cur)
732 qp->s_draining = 0;
733 }
734