This source file includes following definitions.
- mlx5_fpga_conn_map_buf
- mlx5_fpga_conn_unmap_buf
- mlx5_fpga_conn_post_recv
- mlx5_fpga_conn_notify_hw
- mlx5_fpga_conn_post_send
- mlx5_fpga_conn_send
- mlx5_fpga_conn_post_recv_buf
- mlx5_fpga_conn_create_mkey
- mlx5_fpga_conn_rq_cqe
- mlx5_fpga_conn_sq_cqe
- mlx5_fpga_conn_handle_cqe
- mlx5_fpga_conn_arm_cq
- mlx5_fpga_conn_cq_event
- mlx5_fpga_conn_event
- mlx5_fpga_conn_cqes
- mlx5_fpga_conn_cq_tasklet
- mlx5_fpga_conn_cq_complete
- mlx5_fpga_conn_create_cq
- mlx5_fpga_conn_destroy_cq
- mlx5_fpga_conn_create_wq
- mlx5_fpga_conn_create_qp
- mlx5_fpga_conn_free_recv_bufs
- mlx5_fpga_conn_flush_send_bufs
- mlx5_fpga_conn_destroy_qp
- mlx5_fpga_conn_reset_qp
- mlx5_fpga_conn_init_qp
- mlx5_fpga_conn_rtr_qp
- mlx5_fpga_conn_rts_qp
- mlx5_fpga_conn_connect
- mlx5_fpga_conn_create
- mlx5_fpga_conn_destroy
- mlx5_fpga_conn_device_init
- mlx5_fpga_conn_device_cleanup
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34 #include <net/addrconf.h>
35 #include <linux/etherdevice.h>
36 #include <linux/mlx5/vport.h>
37
38 #include "mlx5_core.h"
39 #include "lib/mlx5.h"
40 #include "fpga/conn.h"
41
42 #define MLX5_FPGA_PKEY 0xFFFF
43 #define MLX5_FPGA_PKEY_INDEX 0
44 #define MLX5_FPGA_RECV_SIZE 2048
45 #define MLX5_FPGA_PORT_NUM 1
46 #define MLX5_FPGA_CQ_BUDGET 64
47
48 static int mlx5_fpga_conn_map_buf(struct mlx5_fpga_conn *conn,
49 struct mlx5_fpga_dma_buf *buf)
50 {
51 struct device *dma_device;
52 int err = 0;
53
54 if (unlikely(!buf->sg[0].data))
55 goto out;
56
57 dma_device = &conn->fdev->mdev->pdev->dev;
58 buf->sg[0].dma_addr = dma_map_single(dma_device, buf->sg[0].data,
59 buf->sg[0].size, buf->dma_dir);
60 err = dma_mapping_error(dma_device, buf->sg[0].dma_addr);
61 if (unlikely(err)) {
62 mlx5_fpga_warn(conn->fdev, "DMA error on sg 0: %d\n", err);
63 err = -ENOMEM;
64 goto out;
65 }
66
67 if (!buf->sg[1].data)
68 goto out;
69
70 buf->sg[1].dma_addr = dma_map_single(dma_device, buf->sg[1].data,
71 buf->sg[1].size, buf->dma_dir);
72 err = dma_mapping_error(dma_device, buf->sg[1].dma_addr);
73 if (unlikely(err)) {
74 mlx5_fpga_warn(conn->fdev, "DMA error on sg 1: %d\n", err);
75 dma_unmap_single(dma_device, buf->sg[0].dma_addr,
76 buf->sg[0].size, buf->dma_dir);
77 err = -ENOMEM;
78 }
79
80 out:
81 return err;
82 }
83
84 static void mlx5_fpga_conn_unmap_buf(struct mlx5_fpga_conn *conn,
85 struct mlx5_fpga_dma_buf *buf)
86 {
87 struct device *dma_device;
88
89 dma_device = &conn->fdev->mdev->pdev->dev;
90 if (buf->sg[1].data)
91 dma_unmap_single(dma_device, buf->sg[1].dma_addr,
92 buf->sg[1].size, buf->dma_dir);
93
94 if (likely(buf->sg[0].data))
95 dma_unmap_single(dma_device, buf->sg[0].dma_addr,
96 buf->sg[0].size, buf->dma_dir);
97 }
98
99 static int mlx5_fpga_conn_post_recv(struct mlx5_fpga_conn *conn,
100 struct mlx5_fpga_dma_buf *buf)
101 {
102 struct mlx5_wqe_data_seg *data;
103 unsigned int ix;
104 int err = 0;
105
106 err = mlx5_fpga_conn_map_buf(conn, buf);
107 if (unlikely(err))
108 goto out;
109
110 if (unlikely(conn->qp.rq.pc - conn->qp.rq.cc >= conn->qp.rq.size)) {
111 mlx5_fpga_conn_unmap_buf(conn, buf);
112 return -EBUSY;
113 }
114
115 ix = conn->qp.rq.pc & (conn->qp.rq.size - 1);
116 data = mlx5_wq_cyc_get_wqe(&conn->qp.wq.rq, ix);
117 data->byte_count = cpu_to_be32(buf->sg[0].size);
118 data->lkey = cpu_to_be32(conn->fdev->conn_res.mkey.key);
119 data->addr = cpu_to_be64(buf->sg[0].dma_addr);
120
121 conn->qp.rq.pc++;
122 conn->qp.rq.bufs[ix] = buf;
123
124
125 dma_wmb();
126 *conn->qp.wq.rq.db = cpu_to_be32(conn->qp.rq.pc & 0xffff);
127 out:
128 return err;
129 }
130
131 static void mlx5_fpga_conn_notify_hw(struct mlx5_fpga_conn *conn, void *wqe)
132 {
133
134 dma_wmb();
135 *conn->qp.wq.sq.db = cpu_to_be32(conn->qp.sq.pc);
136
137 wmb();
138 mlx5_write64(wqe, conn->fdev->conn_res.uar->map + MLX5_BF_OFFSET);
139 }
140
141 static void mlx5_fpga_conn_post_send(struct mlx5_fpga_conn *conn,
142 struct mlx5_fpga_dma_buf *buf)
143 {
144 struct mlx5_wqe_ctrl_seg *ctrl;
145 struct mlx5_wqe_data_seg *data;
146 unsigned int ix, sgi;
147 int size = 1;
148
149 ix = conn->qp.sq.pc & (conn->qp.sq.size - 1);
150
151 ctrl = mlx5_wq_cyc_get_wqe(&conn->qp.wq.sq, ix);
152 data = (void *)(ctrl + 1);
153
154 for (sgi = 0; sgi < ARRAY_SIZE(buf->sg); sgi++) {
155 if (!buf->sg[sgi].data)
156 break;
157 data->byte_count = cpu_to_be32(buf->sg[sgi].size);
158 data->lkey = cpu_to_be32(conn->fdev->conn_res.mkey.key);
159 data->addr = cpu_to_be64(buf->sg[sgi].dma_addr);
160 data++;
161 size++;
162 }
163
164 ctrl->imm = 0;
165 ctrl->fm_ce_se = MLX5_WQE_CTRL_CQ_UPDATE;
166 ctrl->opmod_idx_opcode = cpu_to_be32(((conn->qp.sq.pc & 0xffff) << 8) |
167 MLX5_OPCODE_SEND);
168 ctrl->qpn_ds = cpu_to_be32(size | (conn->qp.mqp.qpn << 8));
169
170 conn->qp.sq.pc++;
171 conn->qp.sq.bufs[ix] = buf;
172 mlx5_fpga_conn_notify_hw(conn, ctrl);
173 }
174
175 int mlx5_fpga_conn_send(struct mlx5_fpga_conn *conn,
176 struct mlx5_fpga_dma_buf *buf)
177 {
178 unsigned long flags;
179 int err;
180
181 if (!conn->qp.active)
182 return -ENOTCONN;
183
184 buf->dma_dir = DMA_TO_DEVICE;
185 err = mlx5_fpga_conn_map_buf(conn, buf);
186 if (err)
187 return err;
188
189 spin_lock_irqsave(&conn->qp.sq.lock, flags);
190
191 if (conn->qp.sq.pc - conn->qp.sq.cc >= conn->qp.sq.size) {
192 list_add_tail(&buf->list, &conn->qp.sq.backlog);
193 goto out_unlock;
194 }
195
196 mlx5_fpga_conn_post_send(conn, buf);
197
198 out_unlock:
199 spin_unlock_irqrestore(&conn->qp.sq.lock, flags);
200 return err;
201 }
202
203 static int mlx5_fpga_conn_post_recv_buf(struct mlx5_fpga_conn *conn)
204 {
205 struct mlx5_fpga_dma_buf *buf;
206 int err;
207
208 buf = kzalloc(sizeof(*buf) + MLX5_FPGA_RECV_SIZE, 0);
209 if (!buf)
210 return -ENOMEM;
211
212 buf->sg[0].data = (void *)(buf + 1);
213 buf->sg[0].size = MLX5_FPGA_RECV_SIZE;
214 buf->dma_dir = DMA_FROM_DEVICE;
215
216 err = mlx5_fpga_conn_post_recv(conn, buf);
217 if (err)
218 kfree(buf);
219
220 return err;
221 }
222
223 static int mlx5_fpga_conn_create_mkey(struct mlx5_core_dev *mdev, u32 pdn,
224 struct mlx5_core_mkey *mkey)
225 {
226 int inlen = MLX5_ST_SZ_BYTES(create_mkey_in);
227 void *mkc;
228 u32 *in;
229 int err;
230
231 in = kvzalloc(inlen, GFP_KERNEL);
232 if (!in)
233 return -ENOMEM;
234
235 mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry);
236 MLX5_SET(mkc, mkc, access_mode_1_0, MLX5_MKC_ACCESS_MODE_PA);
237 MLX5_SET(mkc, mkc, lw, 1);
238 MLX5_SET(mkc, mkc, lr, 1);
239
240 MLX5_SET(mkc, mkc, pd, pdn);
241 MLX5_SET(mkc, mkc, length64, 1);
242 MLX5_SET(mkc, mkc, qpn, 0xffffff);
243
244 err = mlx5_core_create_mkey(mdev, mkey, in, inlen);
245
246 kvfree(in);
247 return err;
248 }
249
250 static void mlx5_fpga_conn_rq_cqe(struct mlx5_fpga_conn *conn,
251 struct mlx5_cqe64 *cqe, u8 status)
252 {
253 struct mlx5_fpga_dma_buf *buf;
254 int ix, err;
255
256 ix = be16_to_cpu(cqe->wqe_counter) & (conn->qp.rq.size - 1);
257 buf = conn->qp.rq.bufs[ix];
258 conn->qp.rq.bufs[ix] = NULL;
259 conn->qp.rq.cc++;
260
261 if (unlikely(status && (status != MLX5_CQE_SYNDROME_WR_FLUSH_ERR)))
262 mlx5_fpga_warn(conn->fdev, "RQ buf %p on FPGA QP %u completion status %d\n",
263 buf, conn->fpga_qpn, status);
264 else
265 mlx5_fpga_dbg(conn->fdev, "RQ buf %p on FPGA QP %u completion status %d\n",
266 buf, conn->fpga_qpn, status);
267
268 mlx5_fpga_conn_unmap_buf(conn, buf);
269
270 if (unlikely(status || !conn->qp.active)) {
271 conn->qp.active = false;
272 kfree(buf);
273 return;
274 }
275
276 buf->sg[0].size = be32_to_cpu(cqe->byte_cnt);
277 mlx5_fpga_dbg(conn->fdev, "Message with %u bytes received successfully\n",
278 buf->sg[0].size);
279 conn->recv_cb(conn->cb_arg, buf);
280
281 buf->sg[0].size = MLX5_FPGA_RECV_SIZE;
282 err = mlx5_fpga_conn_post_recv(conn, buf);
283 if (unlikely(err)) {
284 mlx5_fpga_warn(conn->fdev,
285 "Failed to re-post recv buf: %d\n", err);
286 kfree(buf);
287 }
288 }
289
290 static void mlx5_fpga_conn_sq_cqe(struct mlx5_fpga_conn *conn,
291 struct mlx5_cqe64 *cqe, u8 status)
292 {
293 struct mlx5_fpga_dma_buf *buf, *nextbuf;
294 unsigned long flags;
295 int ix;
296
297 spin_lock_irqsave(&conn->qp.sq.lock, flags);
298
299 ix = be16_to_cpu(cqe->wqe_counter) & (conn->qp.sq.size - 1);
300 buf = conn->qp.sq.bufs[ix];
301 conn->qp.sq.bufs[ix] = NULL;
302 conn->qp.sq.cc++;
303
304
305 if (unlikely(!list_empty(&conn->qp.sq.backlog))) {
306 if (likely(conn->qp.active)) {
307 nextbuf = list_first_entry(&conn->qp.sq.backlog,
308 struct mlx5_fpga_dma_buf, list);
309 list_del(&nextbuf->list);
310 mlx5_fpga_conn_post_send(conn, nextbuf);
311 }
312 }
313
314 spin_unlock_irqrestore(&conn->qp.sq.lock, flags);
315
316 if (unlikely(status && (status != MLX5_CQE_SYNDROME_WR_FLUSH_ERR)))
317 mlx5_fpga_warn(conn->fdev, "SQ buf %p on FPGA QP %u completion status %d\n",
318 buf, conn->fpga_qpn, status);
319 else
320 mlx5_fpga_dbg(conn->fdev, "SQ buf %p on FPGA QP %u completion status %d\n",
321 buf, conn->fpga_qpn, status);
322
323 mlx5_fpga_conn_unmap_buf(conn, buf);
324
325 if (likely(buf->complete))
326 buf->complete(conn, conn->fdev, buf, status);
327
328 if (unlikely(status))
329 conn->qp.active = false;
330 }
331
332 static void mlx5_fpga_conn_handle_cqe(struct mlx5_fpga_conn *conn,
333 struct mlx5_cqe64 *cqe)
334 {
335 u8 opcode, status = 0;
336
337 opcode = get_cqe_opcode(cqe);
338
339 switch (opcode) {
340 case MLX5_CQE_REQ_ERR:
341 status = ((struct mlx5_err_cqe *)cqe)->syndrome;
342
343 case MLX5_CQE_REQ:
344 mlx5_fpga_conn_sq_cqe(conn, cqe, status);
345 break;
346
347 case MLX5_CQE_RESP_ERR:
348 status = ((struct mlx5_err_cqe *)cqe)->syndrome;
349
350 case MLX5_CQE_RESP_SEND:
351 mlx5_fpga_conn_rq_cqe(conn, cqe, status);
352 break;
353 default:
354 mlx5_fpga_warn(conn->fdev, "Unexpected cqe opcode %u\n",
355 opcode);
356 }
357 }
358
359 static void mlx5_fpga_conn_arm_cq(struct mlx5_fpga_conn *conn)
360 {
361 mlx5_cq_arm(&conn->cq.mcq, MLX5_CQ_DB_REQ_NOT,
362 conn->fdev->conn_res.uar->map, conn->cq.wq.cc);
363 }
364
365 static void mlx5_fpga_conn_cq_event(struct mlx5_core_cq *mcq,
366 enum mlx5_event event)
367 {
368 struct mlx5_fpga_conn *conn;
369
370 conn = container_of(mcq, struct mlx5_fpga_conn, cq.mcq);
371 mlx5_fpga_warn(conn->fdev, "CQ event %u on CQ #%u\n", event, mcq->cqn);
372 }
373
374 static void mlx5_fpga_conn_event(struct mlx5_core_qp *mqp, int event)
375 {
376 struct mlx5_fpga_conn *conn;
377
378 conn = container_of(mqp, struct mlx5_fpga_conn, qp.mqp);
379 mlx5_fpga_warn(conn->fdev, "QP event %u on QP #%u\n", event, mqp->qpn);
380 }
381
382 static inline void mlx5_fpga_conn_cqes(struct mlx5_fpga_conn *conn,
383 unsigned int budget)
384 {
385 struct mlx5_cqe64 *cqe;
386
387 while (budget) {
388 cqe = mlx5_cqwq_get_cqe(&conn->cq.wq);
389 if (!cqe)
390 break;
391
392 budget--;
393 mlx5_cqwq_pop(&conn->cq.wq);
394 mlx5_fpga_conn_handle_cqe(conn, cqe);
395 mlx5_cqwq_update_db_record(&conn->cq.wq);
396 }
397 if (!budget) {
398 tasklet_schedule(&conn->cq.tasklet);
399 return;
400 }
401
402 mlx5_fpga_dbg(conn->fdev, "Re-arming CQ with cc# %u\n", conn->cq.wq.cc);
403
404 wmb();
405 mlx5_fpga_conn_arm_cq(conn);
406 }
407
408 static void mlx5_fpga_conn_cq_tasklet(unsigned long data)
409 {
410 struct mlx5_fpga_conn *conn = (void *)data;
411
412 if (unlikely(!conn->qp.active))
413 return;
414 mlx5_fpga_conn_cqes(conn, MLX5_FPGA_CQ_BUDGET);
415 }
416
417 static void mlx5_fpga_conn_cq_complete(struct mlx5_core_cq *mcq,
418 struct mlx5_eqe *eqe)
419 {
420 struct mlx5_fpga_conn *conn;
421
422 conn = container_of(mcq, struct mlx5_fpga_conn, cq.mcq);
423 if (unlikely(!conn->qp.active))
424 return;
425 mlx5_fpga_conn_cqes(conn, MLX5_FPGA_CQ_BUDGET);
426 }
427
428 static int mlx5_fpga_conn_create_cq(struct mlx5_fpga_conn *conn, int cq_size)
429 {
430 struct mlx5_fpga_device *fdev = conn->fdev;
431 struct mlx5_core_dev *mdev = fdev->mdev;
432 u32 temp_cqc[MLX5_ST_SZ_DW(cqc)] = {0};
433 u32 out[MLX5_ST_SZ_DW(create_cq_out)];
434 struct mlx5_wq_param wqp;
435 struct mlx5_cqe64 *cqe;
436 int inlen, err, eqn;
437 unsigned int irqn;
438 void *cqc, *in;
439 __be64 *pas;
440 u32 i;
441
442 cq_size = roundup_pow_of_two(cq_size);
443 MLX5_SET(cqc, temp_cqc, log_cq_size, ilog2(cq_size));
444
445 wqp.buf_numa_node = mdev->priv.numa_node;
446 wqp.db_numa_node = mdev->priv.numa_node;
447
448 err = mlx5_cqwq_create(mdev, &wqp, temp_cqc, &conn->cq.wq,
449 &conn->cq.wq_ctrl);
450 if (err)
451 return err;
452
453 for (i = 0; i < mlx5_cqwq_get_size(&conn->cq.wq); i++) {
454 cqe = mlx5_cqwq_get_wqe(&conn->cq.wq, i);
455 cqe->op_own = MLX5_CQE_INVALID << 4 | MLX5_CQE_OWNER_MASK;
456 }
457
458 inlen = MLX5_ST_SZ_BYTES(create_cq_in) +
459 sizeof(u64) * conn->cq.wq_ctrl.buf.npages;
460 in = kvzalloc(inlen, GFP_KERNEL);
461 if (!in) {
462 err = -ENOMEM;
463 goto err_cqwq;
464 }
465
466 err = mlx5_vector2eqn(mdev, smp_processor_id(), &eqn, &irqn);
467 if (err) {
468 kvfree(in);
469 goto err_cqwq;
470 }
471
472 cqc = MLX5_ADDR_OF(create_cq_in, in, cq_context);
473 MLX5_SET(cqc, cqc, log_cq_size, ilog2(cq_size));
474 MLX5_SET(cqc, cqc, c_eqn, eqn);
475 MLX5_SET(cqc, cqc, uar_page, fdev->conn_res.uar->index);
476 MLX5_SET(cqc, cqc, log_page_size, conn->cq.wq_ctrl.buf.page_shift -
477 MLX5_ADAPTER_PAGE_SHIFT);
478 MLX5_SET64(cqc, cqc, dbr_addr, conn->cq.wq_ctrl.db.dma);
479
480 pas = (__be64 *)MLX5_ADDR_OF(create_cq_in, in, pas);
481 mlx5_fill_page_frag_array(&conn->cq.wq_ctrl.buf, pas);
482
483 err = mlx5_core_create_cq(mdev, &conn->cq.mcq, in, inlen, out, sizeof(out));
484 kvfree(in);
485
486 if (err)
487 goto err_cqwq;
488
489 conn->cq.mcq.cqe_sz = 64;
490 conn->cq.mcq.set_ci_db = conn->cq.wq_ctrl.db.db;
491 conn->cq.mcq.arm_db = conn->cq.wq_ctrl.db.db + 1;
492 *conn->cq.mcq.set_ci_db = 0;
493 *conn->cq.mcq.arm_db = 0;
494 conn->cq.mcq.vector = 0;
495 conn->cq.mcq.comp = mlx5_fpga_conn_cq_complete;
496 conn->cq.mcq.event = mlx5_fpga_conn_cq_event;
497 conn->cq.mcq.irqn = irqn;
498 conn->cq.mcq.uar = fdev->conn_res.uar;
499 tasklet_init(&conn->cq.tasklet, mlx5_fpga_conn_cq_tasklet,
500 (unsigned long)conn);
501
502 mlx5_fpga_dbg(fdev, "Created CQ #0x%x\n", conn->cq.mcq.cqn);
503
504 goto out;
505
506 err_cqwq:
507 mlx5_wq_destroy(&conn->cq.wq_ctrl);
508 out:
509 return err;
510 }
511
512 static void mlx5_fpga_conn_destroy_cq(struct mlx5_fpga_conn *conn)
513 {
514 tasklet_disable(&conn->cq.tasklet);
515 tasklet_kill(&conn->cq.tasklet);
516 mlx5_core_destroy_cq(conn->fdev->mdev, &conn->cq.mcq);
517 mlx5_wq_destroy(&conn->cq.wq_ctrl);
518 }
519
520 static int mlx5_fpga_conn_create_wq(struct mlx5_fpga_conn *conn, void *qpc)
521 {
522 struct mlx5_fpga_device *fdev = conn->fdev;
523 struct mlx5_core_dev *mdev = fdev->mdev;
524 struct mlx5_wq_param wqp;
525
526 wqp.buf_numa_node = mdev->priv.numa_node;
527 wqp.db_numa_node = mdev->priv.numa_node;
528
529 return mlx5_wq_qp_create(mdev, &wqp, qpc, &conn->qp.wq,
530 &conn->qp.wq_ctrl);
531 }
532
533 static int mlx5_fpga_conn_create_qp(struct mlx5_fpga_conn *conn,
534 unsigned int tx_size, unsigned int rx_size)
535 {
536 struct mlx5_fpga_device *fdev = conn->fdev;
537 struct mlx5_core_dev *mdev = fdev->mdev;
538 u32 temp_qpc[MLX5_ST_SZ_DW(qpc)] = {0};
539 void *in = NULL, *qpc;
540 int err, inlen;
541
542 conn->qp.rq.pc = 0;
543 conn->qp.rq.cc = 0;
544 conn->qp.rq.size = roundup_pow_of_two(rx_size);
545 conn->qp.sq.pc = 0;
546 conn->qp.sq.cc = 0;
547 conn->qp.sq.size = roundup_pow_of_two(tx_size);
548
549 MLX5_SET(qpc, temp_qpc, log_rq_stride, ilog2(MLX5_SEND_WQE_DS) - 4);
550 MLX5_SET(qpc, temp_qpc, log_rq_size, ilog2(conn->qp.rq.size));
551 MLX5_SET(qpc, temp_qpc, log_sq_size, ilog2(conn->qp.sq.size));
552 err = mlx5_fpga_conn_create_wq(conn, temp_qpc);
553 if (err)
554 goto out;
555
556 conn->qp.rq.bufs = kvcalloc(conn->qp.rq.size,
557 sizeof(conn->qp.rq.bufs[0]),
558 GFP_KERNEL);
559 if (!conn->qp.rq.bufs) {
560 err = -ENOMEM;
561 goto err_wq;
562 }
563
564 conn->qp.sq.bufs = kvcalloc(conn->qp.sq.size,
565 sizeof(conn->qp.sq.bufs[0]),
566 GFP_KERNEL);
567 if (!conn->qp.sq.bufs) {
568 err = -ENOMEM;
569 goto err_rq_bufs;
570 }
571
572 inlen = MLX5_ST_SZ_BYTES(create_qp_in) +
573 MLX5_FLD_SZ_BYTES(create_qp_in, pas[0]) *
574 conn->qp.wq_ctrl.buf.npages;
575 in = kvzalloc(inlen, GFP_KERNEL);
576 if (!in) {
577 err = -ENOMEM;
578 goto err_sq_bufs;
579 }
580
581 qpc = MLX5_ADDR_OF(create_qp_in, in, qpc);
582 MLX5_SET(qpc, qpc, uar_page, fdev->conn_res.uar->index);
583 MLX5_SET(qpc, qpc, log_page_size,
584 conn->qp.wq_ctrl.buf.page_shift - MLX5_ADAPTER_PAGE_SHIFT);
585 MLX5_SET(qpc, qpc, fre, 1);
586 MLX5_SET(qpc, qpc, rlky, 1);
587 MLX5_SET(qpc, qpc, st, MLX5_QP_ST_RC);
588 MLX5_SET(qpc, qpc, pm_state, MLX5_QP_PM_MIGRATED);
589 MLX5_SET(qpc, qpc, pd, fdev->conn_res.pdn);
590 MLX5_SET(qpc, qpc, log_rq_stride, ilog2(MLX5_SEND_WQE_DS) - 4);
591 MLX5_SET(qpc, qpc, log_rq_size, ilog2(conn->qp.rq.size));
592 MLX5_SET(qpc, qpc, rq_type, MLX5_NON_ZERO_RQ);
593 MLX5_SET(qpc, qpc, log_sq_size, ilog2(conn->qp.sq.size));
594 MLX5_SET(qpc, qpc, cqn_snd, conn->cq.mcq.cqn);
595 MLX5_SET(qpc, qpc, cqn_rcv, conn->cq.mcq.cqn);
596 MLX5_SET64(qpc, qpc, dbr_addr, conn->qp.wq_ctrl.db.dma);
597 if (MLX5_CAP_GEN(mdev, cqe_version) == 1)
598 MLX5_SET(qpc, qpc, user_index, 0xFFFFFF);
599
600 mlx5_fill_page_frag_array(&conn->qp.wq_ctrl.buf,
601 (__be64 *)MLX5_ADDR_OF(create_qp_in, in, pas));
602
603 err = mlx5_core_create_qp(mdev, &conn->qp.mqp, in, inlen);
604 if (err)
605 goto err_sq_bufs;
606
607 conn->qp.mqp.event = mlx5_fpga_conn_event;
608 mlx5_fpga_dbg(fdev, "Created QP #0x%x\n", conn->qp.mqp.qpn);
609
610 goto out;
611
612 err_sq_bufs:
613 kvfree(conn->qp.sq.bufs);
614 err_rq_bufs:
615 kvfree(conn->qp.rq.bufs);
616 err_wq:
617 mlx5_wq_destroy(&conn->qp.wq_ctrl);
618 out:
619 kvfree(in);
620 return err;
621 }
622
623 static void mlx5_fpga_conn_free_recv_bufs(struct mlx5_fpga_conn *conn)
624 {
625 int ix;
626
627 for (ix = 0; ix < conn->qp.rq.size; ix++) {
628 if (!conn->qp.rq.bufs[ix])
629 continue;
630 mlx5_fpga_conn_unmap_buf(conn, conn->qp.rq.bufs[ix]);
631 kfree(conn->qp.rq.bufs[ix]);
632 conn->qp.rq.bufs[ix] = NULL;
633 }
634 }
635
636 static void mlx5_fpga_conn_flush_send_bufs(struct mlx5_fpga_conn *conn)
637 {
638 struct mlx5_fpga_dma_buf *buf, *temp;
639 int ix;
640
641 for (ix = 0; ix < conn->qp.sq.size; ix++) {
642 buf = conn->qp.sq.bufs[ix];
643 if (!buf)
644 continue;
645 conn->qp.sq.bufs[ix] = NULL;
646 mlx5_fpga_conn_unmap_buf(conn, buf);
647 if (!buf->complete)
648 continue;
649 buf->complete(conn, conn->fdev, buf, MLX5_CQE_SYNDROME_WR_FLUSH_ERR);
650 }
651 list_for_each_entry_safe(buf, temp, &conn->qp.sq.backlog, list) {
652 mlx5_fpga_conn_unmap_buf(conn, buf);
653 if (!buf->complete)
654 continue;
655 buf->complete(conn, conn->fdev, buf, MLX5_CQE_SYNDROME_WR_FLUSH_ERR);
656 }
657 }
658
659 static void mlx5_fpga_conn_destroy_qp(struct mlx5_fpga_conn *conn)
660 {
661 mlx5_core_destroy_qp(conn->fdev->mdev, &conn->qp.mqp);
662 mlx5_fpga_conn_free_recv_bufs(conn);
663 mlx5_fpga_conn_flush_send_bufs(conn);
664 kvfree(conn->qp.sq.bufs);
665 kvfree(conn->qp.rq.bufs);
666 mlx5_wq_destroy(&conn->qp.wq_ctrl);
667 }
668
669 static inline int mlx5_fpga_conn_reset_qp(struct mlx5_fpga_conn *conn)
670 {
671 struct mlx5_core_dev *mdev = conn->fdev->mdev;
672
673 mlx5_fpga_dbg(conn->fdev, "Modifying QP %u to RST\n", conn->qp.mqp.qpn);
674
675 return mlx5_core_qp_modify(mdev, MLX5_CMD_OP_2RST_QP, 0, NULL,
676 &conn->qp.mqp);
677 }
678
679 static inline int mlx5_fpga_conn_init_qp(struct mlx5_fpga_conn *conn)
680 {
681 struct mlx5_fpga_device *fdev = conn->fdev;
682 struct mlx5_core_dev *mdev = fdev->mdev;
683 u32 *qpc = NULL;
684 int err;
685
686 mlx5_fpga_dbg(conn->fdev, "Modifying QP %u to INIT\n", conn->qp.mqp.qpn);
687
688 qpc = kzalloc(MLX5_ST_SZ_BYTES(qpc), GFP_KERNEL);
689 if (!qpc) {
690 err = -ENOMEM;
691 goto out;
692 }
693
694 MLX5_SET(qpc, qpc, st, MLX5_QP_ST_RC);
695 MLX5_SET(qpc, qpc, pm_state, MLX5_QP_PM_MIGRATED);
696 MLX5_SET(qpc, qpc, primary_address_path.pkey_index, MLX5_FPGA_PKEY_INDEX);
697 MLX5_SET(qpc, qpc, primary_address_path.vhca_port_num, MLX5_FPGA_PORT_NUM);
698 MLX5_SET(qpc, qpc, pd, conn->fdev->conn_res.pdn);
699 MLX5_SET(qpc, qpc, cqn_snd, conn->cq.mcq.cqn);
700 MLX5_SET(qpc, qpc, cqn_rcv, conn->cq.mcq.cqn);
701 MLX5_SET64(qpc, qpc, dbr_addr, conn->qp.wq_ctrl.db.dma);
702
703 err = mlx5_core_qp_modify(mdev, MLX5_CMD_OP_RST2INIT_QP, 0, qpc,
704 &conn->qp.mqp);
705 if (err) {
706 mlx5_fpga_warn(fdev, "qp_modify RST2INIT failed: %d\n", err);
707 goto out;
708 }
709
710 out:
711 kfree(qpc);
712 return err;
713 }
714
715 static inline int mlx5_fpga_conn_rtr_qp(struct mlx5_fpga_conn *conn)
716 {
717 struct mlx5_fpga_device *fdev = conn->fdev;
718 struct mlx5_core_dev *mdev = fdev->mdev;
719 u32 *qpc = NULL;
720 int err;
721
722 mlx5_fpga_dbg(conn->fdev, "QP RTR\n");
723
724 qpc = kzalloc(MLX5_ST_SZ_BYTES(qpc), GFP_KERNEL);
725 if (!qpc) {
726 err = -ENOMEM;
727 goto out;
728 }
729
730 MLX5_SET(qpc, qpc, mtu, MLX5_QPC_MTU_1K_BYTES);
731 MLX5_SET(qpc, qpc, log_msg_max, (u8)MLX5_CAP_GEN(mdev, log_max_msg));
732 MLX5_SET(qpc, qpc, remote_qpn, conn->fpga_qpn);
733 MLX5_SET(qpc, qpc, next_rcv_psn,
734 MLX5_GET(fpga_qpc, conn->fpga_qpc, next_send_psn));
735 MLX5_SET(qpc, qpc, primary_address_path.pkey_index, MLX5_FPGA_PKEY_INDEX);
736 MLX5_SET(qpc, qpc, primary_address_path.vhca_port_num, MLX5_FPGA_PORT_NUM);
737 ether_addr_copy(MLX5_ADDR_OF(qpc, qpc, primary_address_path.rmac_47_32),
738 MLX5_ADDR_OF(fpga_qpc, conn->fpga_qpc, fpga_mac_47_32));
739 MLX5_SET(qpc, qpc, primary_address_path.udp_sport,
740 MLX5_CAP_ROCE(mdev, r_roce_min_src_udp_port));
741 MLX5_SET(qpc, qpc, primary_address_path.src_addr_index,
742 conn->qp.sgid_index);
743 MLX5_SET(qpc, qpc, primary_address_path.hop_limit, 0);
744 memcpy(MLX5_ADDR_OF(qpc, qpc, primary_address_path.rgid_rip),
745 MLX5_ADDR_OF(fpga_qpc, conn->fpga_qpc, fpga_ip),
746 MLX5_FLD_SZ_BYTES(qpc, primary_address_path.rgid_rip));
747
748 err = mlx5_core_qp_modify(mdev, MLX5_CMD_OP_INIT2RTR_QP, 0, qpc,
749 &conn->qp.mqp);
750 if (err) {
751 mlx5_fpga_warn(fdev, "qp_modify RST2INIT failed: %d\n", err);
752 goto out;
753 }
754
755 out:
756 kfree(qpc);
757 return err;
758 }
759
760 static inline int mlx5_fpga_conn_rts_qp(struct mlx5_fpga_conn *conn)
761 {
762 struct mlx5_fpga_device *fdev = conn->fdev;
763 struct mlx5_core_dev *mdev = fdev->mdev;
764 u32 *qpc = NULL;
765 u32 opt_mask;
766 int err;
767
768 mlx5_fpga_dbg(conn->fdev, "QP RTS\n");
769
770 qpc = kzalloc(MLX5_ST_SZ_BYTES(qpc), GFP_KERNEL);
771 if (!qpc) {
772 err = -ENOMEM;
773 goto out;
774 }
775
776 MLX5_SET(qpc, qpc, log_ack_req_freq, 8);
777 MLX5_SET(qpc, qpc, min_rnr_nak, 0x12);
778 MLX5_SET(qpc, qpc, primary_address_path.ack_timeout, 0x12);
779 MLX5_SET(qpc, qpc, next_send_psn,
780 MLX5_GET(fpga_qpc, conn->fpga_qpc, next_rcv_psn));
781 MLX5_SET(qpc, qpc, retry_count, 7);
782 MLX5_SET(qpc, qpc, rnr_retry, 7);
783
784 opt_mask = MLX5_QP_OPTPAR_RNR_TIMEOUT;
785 err = mlx5_core_qp_modify(mdev, MLX5_CMD_OP_RTR2RTS_QP, opt_mask, qpc,
786 &conn->qp.mqp);
787 if (err) {
788 mlx5_fpga_warn(fdev, "qp_modify RST2INIT failed: %d\n", err);
789 goto out;
790 }
791
792 out:
793 kfree(qpc);
794 return err;
795 }
796
797 static int mlx5_fpga_conn_connect(struct mlx5_fpga_conn *conn)
798 {
799 struct mlx5_fpga_device *fdev = conn->fdev;
800 int err;
801
802 MLX5_SET(fpga_qpc, conn->fpga_qpc, state, MLX5_FPGA_QPC_STATE_ACTIVE);
803 err = mlx5_fpga_modify_qp(conn->fdev->mdev, conn->fpga_qpn,
804 MLX5_FPGA_QPC_STATE, &conn->fpga_qpc);
805 if (err) {
806 mlx5_fpga_err(fdev, "Failed to activate FPGA RC QP: %d\n", err);
807 goto out;
808 }
809
810 err = mlx5_fpga_conn_reset_qp(conn);
811 if (err) {
812 mlx5_fpga_err(fdev, "Failed to change QP state to reset\n");
813 goto err_fpga_qp;
814 }
815
816 err = mlx5_fpga_conn_init_qp(conn);
817 if (err) {
818 mlx5_fpga_err(fdev, "Failed to modify QP from RESET to INIT\n");
819 goto err_fpga_qp;
820 }
821 conn->qp.active = true;
822
823 while (!mlx5_fpga_conn_post_recv_buf(conn))
824 ;
825
826 err = mlx5_fpga_conn_rtr_qp(conn);
827 if (err) {
828 mlx5_fpga_err(fdev, "Failed to change QP state from INIT to RTR\n");
829 goto err_recv_bufs;
830 }
831
832 err = mlx5_fpga_conn_rts_qp(conn);
833 if (err) {
834 mlx5_fpga_err(fdev, "Failed to change QP state from RTR to RTS\n");
835 goto err_recv_bufs;
836 }
837 goto out;
838
839 err_recv_bufs:
840 mlx5_fpga_conn_free_recv_bufs(conn);
841 err_fpga_qp:
842 MLX5_SET(fpga_qpc, conn->fpga_qpc, state, MLX5_FPGA_QPC_STATE_INIT);
843 if (mlx5_fpga_modify_qp(conn->fdev->mdev, conn->fpga_qpn,
844 MLX5_FPGA_QPC_STATE, &conn->fpga_qpc))
845 mlx5_fpga_err(fdev, "Failed to revert FPGA QP to INIT\n");
846 out:
847 return err;
848 }
849
850 struct mlx5_fpga_conn *mlx5_fpga_conn_create(struct mlx5_fpga_device *fdev,
851 struct mlx5_fpga_conn_attr *attr,
852 enum mlx5_ifc_fpga_qp_type qp_type)
853 {
854 struct mlx5_fpga_conn *ret, *conn;
855 u8 *remote_mac, *remote_ip;
856 int err;
857
858 if (!attr->recv_cb)
859 return ERR_PTR(-EINVAL);
860
861 conn = kzalloc(sizeof(*conn), GFP_KERNEL);
862 if (!conn)
863 return ERR_PTR(-ENOMEM);
864
865 conn->fdev = fdev;
866 INIT_LIST_HEAD(&conn->qp.sq.backlog);
867
868 spin_lock_init(&conn->qp.sq.lock);
869
870 conn->recv_cb = attr->recv_cb;
871 conn->cb_arg = attr->cb_arg;
872
873 remote_mac = MLX5_ADDR_OF(fpga_qpc, conn->fpga_qpc, remote_mac_47_32);
874 err = mlx5_query_mac_address(fdev->mdev, remote_mac);
875 if (err) {
876 mlx5_fpga_err(fdev, "Failed to query local MAC: %d\n", err);
877 ret = ERR_PTR(err);
878 goto err;
879 }
880
881
882 remote_ip = MLX5_ADDR_OF(fpga_qpc, conn->fpga_qpc, remote_ip);
883 remote_ip[0] = 0xfe;
884 remote_ip[1] = 0x80;
885 addrconf_addr_eui48(&remote_ip[8], remote_mac);
886
887 err = mlx5_core_reserved_gid_alloc(fdev->mdev, &conn->qp.sgid_index);
888 if (err) {
889 mlx5_fpga_err(fdev, "Failed to allocate SGID: %d\n", err);
890 ret = ERR_PTR(err);
891 goto err;
892 }
893
894 err = mlx5_core_roce_gid_set(fdev->mdev, conn->qp.sgid_index,
895 MLX5_ROCE_VERSION_2,
896 MLX5_ROCE_L3_TYPE_IPV6,
897 remote_ip, remote_mac, true, 0,
898 MLX5_FPGA_PORT_NUM);
899 if (err) {
900 mlx5_fpga_err(fdev, "Failed to set SGID: %d\n", err);
901 ret = ERR_PTR(err);
902 goto err_rsvd_gid;
903 }
904 mlx5_fpga_dbg(fdev, "Reserved SGID index %u\n", conn->qp.sgid_index);
905
906
907
908
909 err = mlx5_fpga_conn_create_cq(conn,
910 (attr->tx_size + attr->rx_size) * 2);
911 if (err) {
912 mlx5_fpga_err(fdev, "Failed to create CQ: %d\n", err);
913 ret = ERR_PTR(err);
914 goto err_gid;
915 }
916
917 mlx5_fpga_conn_arm_cq(conn);
918
919 err = mlx5_fpga_conn_create_qp(conn, attr->tx_size, attr->rx_size);
920 if (err) {
921 mlx5_fpga_err(fdev, "Failed to create QP: %d\n", err);
922 ret = ERR_PTR(err);
923 goto err_cq;
924 }
925
926 MLX5_SET(fpga_qpc, conn->fpga_qpc, state, MLX5_FPGA_QPC_STATE_INIT);
927 MLX5_SET(fpga_qpc, conn->fpga_qpc, qp_type, qp_type);
928 MLX5_SET(fpga_qpc, conn->fpga_qpc, st, MLX5_FPGA_QPC_ST_RC);
929 MLX5_SET(fpga_qpc, conn->fpga_qpc, ether_type, ETH_P_8021Q);
930 MLX5_SET(fpga_qpc, conn->fpga_qpc, vid, 0);
931 MLX5_SET(fpga_qpc, conn->fpga_qpc, next_rcv_psn, 1);
932 MLX5_SET(fpga_qpc, conn->fpga_qpc, next_send_psn, 0);
933 MLX5_SET(fpga_qpc, conn->fpga_qpc, pkey, MLX5_FPGA_PKEY);
934 MLX5_SET(fpga_qpc, conn->fpga_qpc, remote_qpn, conn->qp.mqp.qpn);
935 MLX5_SET(fpga_qpc, conn->fpga_qpc, rnr_retry, 7);
936 MLX5_SET(fpga_qpc, conn->fpga_qpc, retry_count, 7);
937
938 err = mlx5_fpga_create_qp(fdev->mdev, &conn->fpga_qpc,
939 &conn->fpga_qpn);
940 if (err) {
941 mlx5_fpga_err(fdev, "Failed to create FPGA RC QP: %d\n", err);
942 ret = ERR_PTR(err);
943 goto err_qp;
944 }
945
946 err = mlx5_fpga_conn_connect(conn);
947 if (err) {
948 ret = ERR_PTR(err);
949 goto err_conn;
950 }
951
952 mlx5_fpga_dbg(fdev, "FPGA QPN is %u\n", conn->fpga_qpn);
953 ret = conn;
954 goto out;
955
956 err_conn:
957 mlx5_fpga_destroy_qp(conn->fdev->mdev, conn->fpga_qpn);
958 err_qp:
959 mlx5_fpga_conn_destroy_qp(conn);
960 err_cq:
961 mlx5_fpga_conn_destroy_cq(conn);
962 err_gid:
963 mlx5_core_roce_gid_set(fdev->mdev, conn->qp.sgid_index, 0, 0, NULL,
964 NULL, false, 0, MLX5_FPGA_PORT_NUM);
965 err_rsvd_gid:
966 mlx5_core_reserved_gid_free(fdev->mdev, conn->qp.sgid_index);
967 err:
968 kfree(conn);
969 out:
970 return ret;
971 }
972
973 void mlx5_fpga_conn_destroy(struct mlx5_fpga_conn *conn)
974 {
975 struct mlx5_fpga_device *fdev = conn->fdev;
976 struct mlx5_core_dev *mdev = fdev->mdev;
977 int err = 0;
978
979 conn->qp.active = false;
980 tasklet_disable(&conn->cq.tasklet);
981 synchronize_irq(conn->cq.mcq.irqn);
982
983 mlx5_fpga_destroy_qp(conn->fdev->mdev, conn->fpga_qpn);
984 err = mlx5_core_qp_modify(mdev, MLX5_CMD_OP_2ERR_QP, 0, NULL,
985 &conn->qp.mqp);
986 if (err)
987 mlx5_fpga_warn(fdev, "qp_modify 2ERR failed: %d\n", err);
988 mlx5_fpga_conn_destroy_qp(conn);
989 mlx5_fpga_conn_destroy_cq(conn);
990
991 mlx5_core_roce_gid_set(conn->fdev->mdev, conn->qp.sgid_index, 0, 0,
992 NULL, NULL, false, 0, MLX5_FPGA_PORT_NUM);
993 mlx5_core_reserved_gid_free(conn->fdev->mdev, conn->qp.sgid_index);
994 kfree(conn);
995 }
996
997 int mlx5_fpga_conn_device_init(struct mlx5_fpga_device *fdev)
998 {
999 int err;
1000
1001 err = mlx5_nic_vport_enable_roce(fdev->mdev);
1002 if (err) {
1003 mlx5_fpga_err(fdev, "Failed to enable RoCE: %d\n", err);
1004 goto out;
1005 }
1006
1007 fdev->conn_res.uar = mlx5_get_uars_page(fdev->mdev);
1008 if (IS_ERR(fdev->conn_res.uar)) {
1009 err = PTR_ERR(fdev->conn_res.uar);
1010 mlx5_fpga_err(fdev, "get_uars_page failed, %d\n", err);
1011 goto err_roce;
1012 }
1013 mlx5_fpga_dbg(fdev, "Allocated UAR index %u\n",
1014 fdev->conn_res.uar->index);
1015
1016 err = mlx5_core_alloc_pd(fdev->mdev, &fdev->conn_res.pdn);
1017 if (err) {
1018 mlx5_fpga_err(fdev, "alloc pd failed, %d\n", err);
1019 goto err_uar;
1020 }
1021 mlx5_fpga_dbg(fdev, "Allocated PD %u\n", fdev->conn_res.pdn);
1022
1023 err = mlx5_fpga_conn_create_mkey(fdev->mdev, fdev->conn_res.pdn,
1024 &fdev->conn_res.mkey);
1025 if (err) {
1026 mlx5_fpga_err(fdev, "create mkey failed, %d\n", err);
1027 goto err_dealloc_pd;
1028 }
1029 mlx5_fpga_dbg(fdev, "Created mkey 0x%x\n", fdev->conn_res.mkey.key);
1030
1031 return 0;
1032
1033 err_dealloc_pd:
1034 mlx5_core_dealloc_pd(fdev->mdev, fdev->conn_res.pdn);
1035 err_uar:
1036 mlx5_put_uars_page(fdev->mdev, fdev->conn_res.uar);
1037 err_roce:
1038 mlx5_nic_vport_disable_roce(fdev->mdev);
1039 out:
1040 return err;
1041 }
1042
1043 void mlx5_fpga_conn_device_cleanup(struct mlx5_fpga_device *fdev)
1044 {
1045 mlx5_core_destroy_mkey(fdev->mdev, &fdev->conn_res.mkey);
1046 mlx5_core_dealloc_pd(fdev->mdev, fdev->conn_res.pdn);
1047 mlx5_put_uars_page(fdev->mdev, fdev->conn_res.uar);
1048 mlx5_nic_vport_disable_roce(fdev->mdev);
1049 }