1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43 #ifndef SVC_RDMA_H
44 #define SVC_RDMA_H
45 #include <linux/llist.h>
46 #include <linux/sunrpc/xdr.h>
47 #include <linux/sunrpc/svcsock.h>
48 #include <linux/sunrpc/rpc_rdma.h>
49 #include <rdma/ib_verbs.h>
50 #include <rdma/rdma_cm.h>
51 #define SVCRDMA_DEBUG
52
53
54 enum {
55 RPCRDMA_DEF_INLINE_THRESH = 4096,
56 RPCRDMA_MAX_INLINE_THRESH = 65536
57 };
58
59
60 extern unsigned int svcrdma_ord;
61 extern unsigned int svcrdma_max_requests;
62 extern unsigned int svcrdma_max_bc_requests;
63 extern unsigned int svcrdma_max_req_size;
64
65 extern atomic_t rdma_stat_recv;
66 extern atomic_t rdma_stat_read;
67 extern atomic_t rdma_stat_write;
68 extern atomic_t rdma_stat_sq_starve;
69 extern atomic_t rdma_stat_rq_starve;
70 extern atomic_t rdma_stat_rq_poll;
71 extern atomic_t rdma_stat_rq_prod;
72 extern atomic_t rdma_stat_sq_poll;
73 extern atomic_t rdma_stat_sq_prod;
74
75 struct svcxprt_rdma {
76 struct svc_xprt sc_xprt;
77 struct rdma_cm_id *sc_cm_id;
78 struct list_head sc_accept_q;
79 int sc_ord;
80 int sc_max_send_sges;
81 bool sc_snd_w_inv;
82
83 atomic_t sc_sq_avail;
84 unsigned int sc_sq_depth;
85 __be32 sc_fc_credits;
86 u32 sc_max_requests;
87 u32 sc_max_bc_requests;
88 int sc_max_req_size;
89 u8 sc_port_num;
90
91 struct ib_pd *sc_pd;
92
93 spinlock_t sc_send_lock;
94 struct list_head sc_send_ctxts;
95 spinlock_t sc_rw_ctxt_lock;
96 struct list_head sc_rw_ctxts;
97
98 struct list_head sc_rq_dto_q;
99 spinlock_t sc_rq_dto_lock;
100 struct ib_qp *sc_qp;
101 struct ib_cq *sc_rq_cq;
102 struct ib_cq *sc_sq_cq;
103
104 spinlock_t sc_lock;
105
106 wait_queue_head_t sc_send_wait;
107 unsigned long sc_flags;
108 struct list_head sc_read_complete_q;
109 struct work_struct sc_work;
110
111 struct llist_head sc_recv_ctxts;
112 };
113
114 #define RDMAXPRT_CONN_PENDING 3
115
116
117
118
119 enum {
120 RPCRDMA_LISTEN_BACKLOG = 10,
121 RPCRDMA_MAX_REQUESTS = 64,
122 RPCRDMA_MAX_BC_REQUESTS = 2,
123 };
124
125 #define RPCSVC_MAXPAYLOAD_RDMA RPCSVC_MAXPAYLOAD
126
127 struct svc_rdma_recv_ctxt {
128 struct llist_node rc_node;
129 struct list_head rc_list;
130 struct ib_recv_wr rc_recv_wr;
131 struct ib_cqe rc_cqe;
132 struct ib_sge rc_recv_sge;
133 void *rc_recv_buf;
134 struct xdr_buf rc_arg;
135 bool rc_temp;
136 u32 rc_byte_len;
137 unsigned int rc_page_count;
138 unsigned int rc_hdr_count;
139 u32 rc_inv_rkey;
140 struct page *rc_pages[RPCSVC_MAXPAGES];
141 };
142
143 struct svc_rdma_send_ctxt {
144 struct list_head sc_list;
145 struct ib_send_wr sc_send_wr;
146 struct ib_cqe sc_cqe;
147 void *sc_xprt_buf;
148 int sc_page_count;
149 int sc_cur_sge_no;
150 struct page *sc_pages[RPCSVC_MAXPAGES];
151 struct ib_sge sc_sges[];
152 };
153
154
155 extern int svc_rdma_handle_bc_reply(struct rpc_xprt *xprt,
156 __be32 *rdma_resp,
157 struct xdr_buf *rcvbuf);
158
159
160 extern void svc_rdma_recv_ctxts_destroy(struct svcxprt_rdma *rdma);
161 extern bool svc_rdma_post_recvs(struct svcxprt_rdma *rdma);
162 extern void svc_rdma_recv_ctxt_put(struct svcxprt_rdma *rdma,
163 struct svc_rdma_recv_ctxt *ctxt);
164 extern void svc_rdma_flush_recv_queues(struct svcxprt_rdma *rdma);
165 extern void svc_rdma_release_rqst(struct svc_rqst *rqstp);
166 extern int svc_rdma_recvfrom(struct svc_rqst *);
167
168
169 extern void svc_rdma_destroy_rw_ctxts(struct svcxprt_rdma *rdma);
170 extern int svc_rdma_recv_read_chunk(struct svcxprt_rdma *rdma,
171 struct svc_rqst *rqstp,
172 struct svc_rdma_recv_ctxt *head, __be32 *p);
173 extern int svc_rdma_send_write_chunk(struct svcxprt_rdma *rdma,
174 __be32 *wr_ch, struct xdr_buf *xdr);
175 extern int svc_rdma_send_reply_chunk(struct svcxprt_rdma *rdma,
176 __be32 *rp_ch, bool writelist,
177 struct xdr_buf *xdr);
178
179
180 extern void svc_rdma_send_ctxts_destroy(struct svcxprt_rdma *rdma);
181 extern struct svc_rdma_send_ctxt *
182 svc_rdma_send_ctxt_get(struct svcxprt_rdma *rdma);
183 extern void svc_rdma_send_ctxt_put(struct svcxprt_rdma *rdma,
184 struct svc_rdma_send_ctxt *ctxt);
185 extern int svc_rdma_send(struct svcxprt_rdma *rdma, struct ib_send_wr *wr);
186 extern void svc_rdma_sync_reply_hdr(struct svcxprt_rdma *rdma,
187 struct svc_rdma_send_ctxt *ctxt,
188 unsigned int len);
189 extern int svc_rdma_map_reply_msg(struct svcxprt_rdma *rdma,
190 struct svc_rdma_send_ctxt *ctxt,
191 struct xdr_buf *xdr, __be32 *wr_lst);
192 extern int svc_rdma_sendto(struct svc_rqst *);
193
194
195 extern int svc_rdma_create_listen(struct svc_serv *, int, struct sockaddr *);
196 extern void svc_sq_reap(struct svcxprt_rdma *);
197 extern void svc_rq_reap(struct svcxprt_rdma *);
198
199 extern struct svc_xprt_class svc_rdma_class;
200 #ifdef CONFIG_SUNRPC_BACKCHANNEL
201 extern struct svc_xprt_class svc_rdma_bc_class;
202 #endif
203
204
205 extern int svc_rdma_init(void);
206 extern void svc_rdma_cleanup(void);
207
208 #endif