1 /*
2 * Copyright(c) 2016 Intel Corporation.
3 *
4 * This file is provided under a dual BSD/GPLv2 license. When using or
5 * redistributing this file, you may do so under either license.
6 *
7 * GPL LICENSE SUMMARY
8 *
9 * This program is free software; you can redistribute it and/or modify
10 * it under the terms of version 2 of the GNU General Public License as
11 * published by the Free Software Foundation.
12 *
13 * This program is distributed in the hope that it will be useful, but
14 * WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
16 * General Public License for more details.
17 *
18 * BSD LICENSE
19 *
20 * Redistribution and use in source and binary forms, with or without
21 * modification, are permitted provided that the following conditions
22 * are met:
23 *
24 * - Redistributions of source code must retain the above copyright
25 * notice, this list of conditions and the following disclaimer.
26 * - Redistributions in binary form must reproduce the above copyright
27 * notice, this list of conditions and the following disclaimer in
28 * the documentation and/or other materials provided with the
29 * distribution.
30 * - Neither the name of Intel Corporation nor the names of its
31 * contributors may be used to endorse or promote products derived
32 * from this software without specific prior written permission.
33 *
34 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
35 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
36 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
37 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
38 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
39 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
40 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
41 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
42 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
43 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
44 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
45 *
46 */
47
48 #include <rdma/rdmavt_qp.h>
49 #include <rdma/ib_hdrs.h>
50
51 /*
52 * Convert the AETH credit code into the number of credits.
53 */
54 static const u16 credit_table[31] = {
55 0, /* 0 */
56 1, /* 1 */
57 2, /* 2 */
58 3, /* 3 */
59 4, /* 4 */
60 6, /* 5 */
61 8, /* 6 */
62 12, /* 7 */
63 16, /* 8 */
64 24, /* 9 */
65 32, /* A */
66 48, /* B */
67 64, /* C */
68 96, /* D */
69 128, /* E */
70 192, /* F */
71 256, /* 10 */
72 384, /* 11 */
73 512, /* 12 */
74 768, /* 13 */
75 1024, /* 14 */
76 1536, /* 15 */
77 2048, /* 16 */
78 3072, /* 17 */
79 4096, /* 18 */
80 6144, /* 19 */
81 8192, /* 1A */
82 12288, /* 1B */
83 16384, /* 1C */
84 24576, /* 1D */
85 32768 /* 1E */
86 };
87
88 /**
89 * rvt_compute_aeth - compute the AETH (syndrome + MSN)
90 * @qp: the queue pair to compute the AETH for
91 *
92 * Returns the AETH.
93 */
94 __be32 rvt_compute_aeth(struct rvt_qp *qp)
95 {
96 u32 aeth = qp->r_msn & IB_MSN_MASK;
97
98 if (qp->ibqp.srq) {
99 /*
100 * Shared receive queues don't generate credits.
101 * Set the credit field to the invalid value.
102 */
103 aeth |= IB_AETH_CREDIT_INVAL << IB_AETH_CREDIT_SHIFT;
104 } else {
105 u32 min, max, x;
106 u32 credits;
107 u32 head;
108 u32 tail;
109
110 credits = READ_ONCE(qp->r_rq.kwq->count);
111 if (credits == 0) {
112 /* sanity check pointers before trusting them */
113 if (qp->ip) {
114 head = RDMA_READ_UAPI_ATOMIC(qp->r_rq.wq->head);
115 tail = RDMA_READ_UAPI_ATOMIC(qp->r_rq.wq->tail);
116 } else {
117 head = READ_ONCE(qp->r_rq.kwq->head);
118 tail = READ_ONCE(qp->r_rq.kwq->tail);
119 }
120 if (head >= qp->r_rq.size)
121 head = 0;
122 if (tail >= qp->r_rq.size)
123 tail = 0;
124 /*
125 * Compute the number of credits available (RWQEs).
126 * There is a small chance that the pair of reads are
127 * not atomic, which is OK, since the fuzziness is
128 * resolved as further ACKs go out.
129 */
130 credits = head - tail;
131 if ((int)credits < 0)
132 credits += qp->r_rq.size;
133 }
134 /*
135 * Binary search the credit table to find the code to
136 * use.
137 */
138 min = 0;
139 max = 31;
140 for (;;) {
141 x = (min + max) / 2;
142 if (credit_table[x] == credits)
143 break;
144 if (credit_table[x] > credits) {
145 max = x;
146 } else {
147 if (min == x)
148 break;
149 min = x;
150 }
151 }
152 aeth |= x << IB_AETH_CREDIT_SHIFT;
153 }
154 return cpu_to_be32(aeth);
155 }
156 EXPORT_SYMBOL(rvt_compute_aeth);
157
158 /**
159 * rvt_get_credit - flush the send work queue of a QP
160 * @qp: the qp who's send work queue to flush
161 * @aeth: the Acknowledge Extended Transport Header
162 *
163 * The QP s_lock should be held.
164 */
165 void rvt_get_credit(struct rvt_qp *qp, u32 aeth)
166 {
167 struct rvt_dev_info *rdi = ib_to_rvt(qp->ibqp.device);
168 u32 credit = (aeth >> IB_AETH_CREDIT_SHIFT) & IB_AETH_CREDIT_MASK;
169
170 lockdep_assert_held(&qp->s_lock);
171 /*
172 * If the credit is invalid, we can send
173 * as many packets as we like. Otherwise, we have to
174 * honor the credit field.
175 */
176 if (credit == IB_AETH_CREDIT_INVAL) {
177 if (!(qp->s_flags & RVT_S_UNLIMITED_CREDIT)) {
178 qp->s_flags |= RVT_S_UNLIMITED_CREDIT;
179 if (qp->s_flags & RVT_S_WAIT_SSN_CREDIT) {
180 qp->s_flags &= ~RVT_S_WAIT_SSN_CREDIT;
181 rdi->driver_f.schedule_send(qp);
182 }
183 }
184 } else if (!(qp->s_flags & RVT_S_UNLIMITED_CREDIT)) {
185 /* Compute new LSN (i.e., MSN + credit) */
186 credit = (aeth + credit_table[credit]) & IB_MSN_MASK;
187 if (rvt_cmp_msn(credit, qp->s_lsn) > 0) {
188 qp->s_lsn = credit;
189 if (qp->s_flags & RVT_S_WAIT_SSN_CREDIT) {
190 qp->s_flags &= ~RVT_S_WAIT_SSN_CREDIT;
191 rdi->driver_f.schedule_send(qp);
192 }
193 }
194 }
195 }
196 EXPORT_SYMBOL(rvt_get_credit);
197
198 /* rvt_restart_sge - rewind the sge state for a wqe */
199 u32 rvt_restart_sge(struct rvt_sge_state *ss, struct rvt_swqe *wqe, u32 len)
200 {
201 ss->sge = wqe->sg_list[0];
202 ss->sg_list = wqe->sg_list + 1;
203 ss->num_sge = wqe->wr.num_sge;
204 ss->total_len = wqe->length;
205 rvt_skip_sge(ss, len, false);
206 return wqe->length - len;
207 }
208 EXPORT_SYMBOL(rvt_restart_sge);
209