This source file includes following definitions.
- scif_rma_ep_init
- scif_rma_ep_can_uninit
- scif_create_pinned_pages
- scif_destroy_pinned_pages
- scif_create_window
- scif_destroy_incomplete_window
- scif_unmap_window
- __scif_acquire_mm
- __scif_release_mm
- __scif_dec_pinned_vm_lock
- __scif_check_inc_pinned_vm
- scif_destroy_window
- scif_create_remote_lookup
- scif_destroy_remote_lookup
- scif_create_remote_window
- scif_destroy_remote_window
- scif_iommu_map
- scif_map_window
- scif_send_scif_unregister
- scif_unregister_window
- scif_send_alloc_request
- scif_prep_remote_window
- scif_send_scif_register
- scif_get_window_offset
- scif_free_window_offset
- scif_alloc_req
- scif_alloc_gnt_rej
- scif_free_virt
- scif_fixup_aper_base
- scif_recv_reg
- scif_recv_unreg
- scif_recv_reg_ack
- scif_recv_reg_nack
- scif_recv_unreg_ack
- scif_recv_unreg_nack
- __scif_pin_pages
- scif_pin_pages
- scif_unpin_pages
- scif_insert_local_window
- scif_register_pinned_pages
- scif_register
- scif_unregister
1
2
3
4
5
6
7
8
9 #include <linux/intel-iommu.h>
10 #include <linux/pagemap.h>
11 #include <linux/sched/mm.h>
12 #include <linux/sched/signal.h>
13
14 #include "scif_main.h"
15 #include "scif_map.h"
16
17
18 #define SCIF_MAP_ULIMIT 0x40
19
20 bool scif_ulimit_check = 1;
21
22
23
24
25
26
27
28 void scif_rma_ep_init(struct scif_endpt *ep)
29 {
30 struct scif_endpt_rma_info *rma = &ep->rma_info;
31
32 mutex_init(&rma->rma_lock);
33 init_iova_domain(&rma->iovad, PAGE_SIZE, SCIF_IOVA_START_PFN);
34 spin_lock_init(&rma->tc_lock);
35 mutex_init(&rma->mmn_lock);
36 INIT_LIST_HEAD(&rma->reg_list);
37 INIT_LIST_HEAD(&rma->remote_reg_list);
38 atomic_set(&rma->tw_refcount, 0);
39 atomic_set(&rma->tcw_refcount, 0);
40 atomic_set(&rma->tcw_total_pages, 0);
41 atomic_set(&rma->fence_refcount, 0);
42
43 rma->async_list_del = 0;
44 rma->dma_chan = NULL;
45 INIT_LIST_HEAD(&rma->mmn_list);
46 INIT_LIST_HEAD(&rma->vma_list);
47 init_waitqueue_head(&rma->markwq);
48 }
49
50
51
52
53
54
55
56 int scif_rma_ep_can_uninit(struct scif_endpt *ep)
57 {
58 int ret = 0;
59
60 mutex_lock(&ep->rma_info.rma_lock);
61
62 if (list_empty(&ep->rma_info.reg_list) &&
63 list_empty(&ep->rma_info.remote_reg_list) &&
64 list_empty(&ep->rma_info.mmn_list) &&
65 !atomic_read(&ep->rma_info.tw_refcount) &&
66 !atomic_read(&ep->rma_info.tcw_refcount) &&
67 !atomic_read(&ep->rma_info.fence_refcount))
68 ret = 1;
69 mutex_unlock(&ep->rma_info.rma_lock);
70 return ret;
71 }
72
73
74
75
76
77
78
79
80 static struct scif_pinned_pages *
81 scif_create_pinned_pages(int nr_pages, int prot)
82 {
83 struct scif_pinned_pages *pin;
84
85 might_sleep();
86 pin = scif_zalloc(sizeof(*pin));
87 if (!pin)
88 goto error;
89
90 pin->pages = scif_zalloc(nr_pages * sizeof(*pin->pages));
91 if (!pin->pages)
92 goto error_free_pinned_pages;
93
94 pin->prot = prot;
95 pin->magic = SCIFEP_MAGIC;
96 return pin;
97
98 error_free_pinned_pages:
99 scif_free(pin, sizeof(*pin));
100 error:
101 return NULL;
102 }
103
104
105
106
107
108
109
110 static int scif_destroy_pinned_pages(struct scif_pinned_pages *pin)
111 {
112 int j;
113 int writeable = pin->prot & SCIF_PROT_WRITE;
114 int kernel = SCIF_MAP_KERNEL & pin->map_flags;
115
116 for (j = 0; j < pin->nr_pages; j++) {
117 if (pin->pages[j] && !kernel) {
118 if (writeable)
119 SetPageDirty(pin->pages[j]);
120 put_page(pin->pages[j]);
121 }
122 }
123
124 scif_free(pin->pages,
125 pin->nr_pages * sizeof(*pin->pages));
126 scif_free(pin, sizeof(*pin));
127 return 0;
128 }
129
130
131
132
133
134
135
136
137
138
139 struct scif_window *scif_create_window(struct scif_endpt *ep, int nr_pages,
140 s64 offset, bool temp)
141 {
142 struct scif_window *window;
143
144 might_sleep();
145 window = scif_zalloc(sizeof(*window));
146 if (!window)
147 goto error;
148
149 window->dma_addr = scif_zalloc(nr_pages * sizeof(*window->dma_addr));
150 if (!window->dma_addr)
151 goto error_free_window;
152
153 window->num_pages = scif_zalloc(nr_pages * sizeof(*window->num_pages));
154 if (!window->num_pages)
155 goto error_free_window;
156
157 window->offset = offset;
158 window->ep = (u64)ep;
159 window->magic = SCIFEP_MAGIC;
160 window->reg_state = OP_IDLE;
161 init_waitqueue_head(&window->regwq);
162 window->unreg_state = OP_IDLE;
163 init_waitqueue_head(&window->unregwq);
164 INIT_LIST_HEAD(&window->list);
165 window->type = SCIF_WINDOW_SELF;
166 window->temp = temp;
167 return window;
168
169 error_free_window:
170 scif_free(window->dma_addr,
171 nr_pages * sizeof(*window->dma_addr));
172 scif_free(window, sizeof(*window));
173 error:
174 return NULL;
175 }
176
177
178
179
180
181
182
183
184 static void scif_destroy_incomplete_window(struct scif_endpt *ep,
185 struct scif_window *window)
186 {
187 int err;
188 int nr_pages = window->nr_pages;
189 struct scif_allocmsg *alloc = &window->alloc_handle;
190 struct scifmsg msg;
191
192 retry:
193
194 err = wait_event_timeout(alloc->allocwq,
195 alloc->state != OP_IN_PROGRESS,
196 SCIF_NODE_ALIVE_TIMEOUT);
197 if (!err && scifdev_alive(ep))
198 goto retry;
199
200 mutex_lock(&ep->rma_info.rma_lock);
201 if (alloc->state == OP_COMPLETED) {
202 msg.uop = SCIF_FREE_VIRT;
203 msg.src = ep->port;
204 msg.payload[0] = ep->remote_ep;
205 msg.payload[1] = window->alloc_handle.vaddr;
206 msg.payload[2] = (u64)window;
207 msg.payload[3] = SCIF_REGISTER;
208 _scif_nodeqp_send(ep->remote_dev, &msg);
209 }
210 mutex_unlock(&ep->rma_info.rma_lock);
211
212 scif_free_window_offset(ep, window, window->offset);
213 scif_free(window->dma_addr, nr_pages * sizeof(*window->dma_addr));
214 scif_free(window->num_pages, nr_pages * sizeof(*window->num_pages));
215 scif_free(window, sizeof(*window));
216 }
217
218
219
220
221
222
223
224
225 void scif_unmap_window(struct scif_dev *remote_dev, struct scif_window *window)
226 {
227 int j;
228
229 if (scif_is_iommu_enabled() && !scifdev_self(remote_dev)) {
230 if (window->st) {
231 dma_unmap_sg(&remote_dev->sdev->dev,
232 window->st->sgl, window->st->nents,
233 DMA_BIDIRECTIONAL);
234 sg_free_table(window->st);
235 kfree(window->st);
236 window->st = NULL;
237 }
238 } else {
239 for (j = 0; j < window->nr_contig_chunks; j++) {
240 if (window->dma_addr[j]) {
241 scif_unmap_single(window->dma_addr[j],
242 remote_dev,
243 window->num_pages[j] <<
244 PAGE_SHIFT);
245 window->dma_addr[j] = 0x0;
246 }
247 }
248 }
249 }
250
251 static inline struct mm_struct *__scif_acquire_mm(void)
252 {
253 if (scif_ulimit_check)
254 return get_task_mm(current);
255 return NULL;
256 }
257
258 static inline void __scif_release_mm(struct mm_struct *mm)
259 {
260 if (mm)
261 mmput(mm);
262 }
263
264 static inline int
265 __scif_dec_pinned_vm_lock(struct mm_struct *mm,
266 int nr_pages)
267 {
268 if (!mm || !nr_pages || !scif_ulimit_check)
269 return 0;
270
271 atomic64_sub(nr_pages, &mm->pinned_vm);
272 return 0;
273 }
274
275 static inline int __scif_check_inc_pinned_vm(struct mm_struct *mm,
276 int nr_pages)
277 {
278 unsigned long locked, lock_limit;
279
280 if (!mm || !nr_pages || !scif_ulimit_check)
281 return 0;
282
283 lock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT;
284 locked = atomic64_add_return(nr_pages, &mm->pinned_vm);
285
286 if ((locked > lock_limit) && !capable(CAP_IPC_LOCK)) {
287 atomic64_sub(nr_pages, &mm->pinned_vm);
288 dev_err(scif_info.mdev.this_device,
289 "locked(%lu) > lock_limit(%lu)\n",
290 locked, lock_limit);
291 return -ENOMEM;
292 }
293 return 0;
294 }
295
296
297
298
299
300
301
302
303 int scif_destroy_window(struct scif_endpt *ep, struct scif_window *window)
304 {
305 int j;
306 struct scif_pinned_pages *pinned_pages = window->pinned_pages;
307 int nr_pages = window->nr_pages;
308
309 might_sleep();
310 if (!window->temp && window->mm) {
311 __scif_dec_pinned_vm_lock(window->mm, window->nr_pages);
312 __scif_release_mm(window->mm);
313 window->mm = NULL;
314 }
315
316 scif_free_window_offset(ep, window, window->offset);
317 scif_unmap_window(ep->remote_dev, window);
318
319
320
321
322 j = atomic_sub_return(1, &pinned_pages->ref_count);
323 if (j < 0)
324 dev_err(scif_info.mdev.this_device,
325 "%s %d incorrect ref count %d\n",
326 __func__, __LINE__, j);
327
328
329
330
331
332 if (!j)
333 scif_destroy_pinned_pages(window->pinned_pages);
334 scif_free(window->dma_addr, nr_pages * sizeof(*window->dma_addr));
335 scif_free(window->num_pages, nr_pages * sizeof(*window->num_pages));
336 window->magic = 0;
337 scif_free(window, sizeof(*window));
338 return 0;
339 }
340
341
342
343
344
345
346
347
348
349
350 static int scif_create_remote_lookup(struct scif_dev *remote_dev,
351 struct scif_window *window)
352 {
353 int i, j, err = 0;
354 int nr_pages = window->nr_pages;
355 bool vmalloc_dma_phys, vmalloc_num_pages;
356
357 might_sleep();
358
359 err = scif_map_single(&window->mapped_offset,
360 window, remote_dev, sizeof(*window));
361 if (err)
362 goto error_window;
363
364
365 window->nr_lookup = ALIGN(nr_pages * PAGE_SIZE,
366 ((2) * 1024 * 1024)) >> 21;
367
368 window->dma_addr_lookup.lookup =
369 scif_alloc_coherent(&window->dma_addr_lookup.offset,
370 remote_dev, window->nr_lookup *
371 sizeof(*window->dma_addr_lookup.lookup),
372 GFP_KERNEL | __GFP_ZERO);
373 if (!window->dma_addr_lookup.lookup) {
374 err = -ENOMEM;
375 goto error_window;
376 }
377
378 window->num_pages_lookup.lookup =
379 scif_alloc_coherent(&window->num_pages_lookup.offset,
380 remote_dev, window->nr_lookup *
381 sizeof(*window->num_pages_lookup.lookup),
382 GFP_KERNEL | __GFP_ZERO);
383 if (!window->num_pages_lookup.lookup) {
384 err = -ENOMEM;
385 goto error_window;
386 }
387
388 vmalloc_dma_phys = is_vmalloc_addr(&window->dma_addr[0]);
389 vmalloc_num_pages = is_vmalloc_addr(&window->num_pages[0]);
390
391
392 for (i = 0, j = 0; i < nr_pages; i += SCIF_NR_ADDR_IN_PAGE, j++) {
393 err = scif_map_page(&window->dma_addr_lookup.lookup[j],
394 vmalloc_dma_phys ?
395 vmalloc_to_page(&window->dma_addr[i]) :
396 virt_to_page(&window->dma_addr[i]),
397 remote_dev);
398 if (err)
399 goto error_window;
400 err = scif_map_page(&window->num_pages_lookup.lookup[j],
401 vmalloc_num_pages ?
402 vmalloc_to_page(&window->num_pages[i]) :
403 virt_to_page(&window->num_pages[i]),
404 remote_dev);
405 if (err)
406 goto error_window;
407 }
408 return 0;
409 error_window:
410 return err;
411 }
412
413
414
415
416
417
418
419
420
421 static void scif_destroy_remote_lookup(struct scif_dev *remote_dev,
422 struct scif_window *window)
423 {
424 int i, j;
425
426 if (window->nr_lookup) {
427 struct scif_rma_lookup *lup = &window->dma_addr_lookup;
428 struct scif_rma_lookup *npup = &window->num_pages_lookup;
429
430 for (i = 0, j = 0; i < window->nr_pages;
431 i += SCIF_NR_ADDR_IN_PAGE, j++) {
432 if (lup->lookup && lup->lookup[j])
433 scif_unmap_single(lup->lookup[j],
434 remote_dev,
435 PAGE_SIZE);
436 if (npup->lookup && npup->lookup[j])
437 scif_unmap_single(npup->lookup[j],
438 remote_dev,
439 PAGE_SIZE);
440 }
441 if (lup->lookup)
442 scif_free_coherent(lup->lookup, lup->offset,
443 remote_dev, window->nr_lookup *
444 sizeof(*lup->lookup));
445 if (npup->lookup)
446 scif_free_coherent(npup->lookup, npup->offset,
447 remote_dev, window->nr_lookup *
448 sizeof(*npup->lookup));
449 if (window->mapped_offset)
450 scif_unmap_single(window->mapped_offset,
451 remote_dev, sizeof(*window));
452 window->nr_lookup = 0;
453 }
454 }
455
456
457
458
459
460
461
462
463 static struct scif_window *
464 scif_create_remote_window(struct scif_dev *scifdev, int nr_pages)
465 {
466 struct scif_window *window;
467
468 might_sleep();
469 window = scif_zalloc(sizeof(*window));
470 if (!window)
471 goto error_ret;
472
473 window->magic = SCIFEP_MAGIC;
474 window->nr_pages = nr_pages;
475
476 window->dma_addr = scif_zalloc(nr_pages * sizeof(*window->dma_addr));
477 if (!window->dma_addr)
478 goto error_window;
479
480 window->num_pages = scif_zalloc(nr_pages *
481 sizeof(*window->num_pages));
482 if (!window->num_pages)
483 goto error_window;
484
485 if (scif_create_remote_lookup(scifdev, window))
486 goto error_window;
487
488 window->type = SCIF_WINDOW_PEER;
489 window->unreg_state = OP_IDLE;
490 INIT_LIST_HEAD(&window->list);
491 return window;
492 error_window:
493 scif_destroy_remote_window(window);
494 error_ret:
495 return NULL;
496 }
497
498
499
500
501
502
503
504
505 void
506 scif_destroy_remote_window(struct scif_window *window)
507 {
508 scif_free(window->dma_addr, window->nr_pages *
509 sizeof(*window->dma_addr));
510 scif_free(window->num_pages, window->nr_pages *
511 sizeof(*window->num_pages));
512 window->magic = 0;
513 scif_free(window, sizeof(*window));
514 }
515
516
517
518
519
520
521
522
523
524 static int scif_iommu_map(struct scif_dev *remote_dev,
525 struct scif_window *window)
526 {
527 struct scatterlist *sg;
528 int i, err;
529 scif_pinned_pages_t pin = window->pinned_pages;
530
531 window->st = kzalloc(sizeof(*window->st), GFP_KERNEL);
532 if (!window->st)
533 return -ENOMEM;
534
535 err = sg_alloc_table(window->st, window->nr_pages, GFP_KERNEL);
536 if (err)
537 return err;
538
539 for_each_sg(window->st->sgl, sg, window->st->nents, i)
540 sg_set_page(sg, pin->pages[i], PAGE_SIZE, 0x0);
541
542 err = dma_map_sg(&remote_dev->sdev->dev, window->st->sgl,
543 window->st->nents, DMA_BIDIRECTIONAL);
544 if (!err)
545 return -ENOMEM;
546
547 sg = window->st->sgl;
548 for (i = 0; sg; i++) {
549 dma_addr_t last_da;
550
551 window->dma_addr[i] = sg_dma_address(sg);
552 window->num_pages[i] = sg_dma_len(sg) >> PAGE_SHIFT;
553 last_da = sg_dma_address(sg) + sg_dma_len(sg);
554 while ((sg = sg_next(sg)) && sg_dma_address(sg) == last_da) {
555 window->num_pages[i] +=
556 (sg_dma_len(sg) >> PAGE_SHIFT);
557 last_da = window->dma_addr[i] +
558 sg_dma_len(sg);
559 }
560 window->nr_contig_chunks++;
561 }
562 return 0;
563 }
564
565
566
567
568
569
570
571
572
573 int
574 scif_map_window(struct scif_dev *remote_dev, struct scif_window *window)
575 {
576 int i, j, k, err = 0, nr_contig_pages;
577 scif_pinned_pages_t pin;
578 phys_addr_t phys_prev, phys_curr;
579
580 might_sleep();
581
582 pin = window->pinned_pages;
583
584 if (intel_iommu_enabled && !scifdev_self(remote_dev))
585 return scif_iommu_map(remote_dev, window);
586
587 for (i = 0, j = 0; i < window->nr_pages; i += nr_contig_pages, j++) {
588 phys_prev = page_to_phys(pin->pages[i]);
589 nr_contig_pages = 1;
590
591
592 for (k = i + 1; k < window->nr_pages; k++) {
593 phys_curr = page_to_phys(pin->pages[k]);
594 if (phys_curr != (phys_prev + PAGE_SIZE))
595 break;
596 phys_prev = phys_curr;
597 nr_contig_pages++;
598 }
599 window->num_pages[j] = nr_contig_pages;
600 window->nr_contig_chunks++;
601 if (scif_is_mgmt_node()) {
602
603
604
605
606 err = scif_map_single(&window->dma_addr[j],
607 phys_to_virt(page_to_phys(
608 pin->pages[i])),
609 remote_dev,
610 nr_contig_pages << PAGE_SHIFT);
611 if (err)
612 return err;
613 } else {
614 window->dma_addr[j] = page_to_phys(pin->pages[i]);
615 }
616 }
617 return err;
618 }
619
620
621
622
623
624
625
626
627 static int scif_send_scif_unregister(struct scif_endpt *ep,
628 struct scif_window *window)
629 {
630 struct scifmsg msg;
631
632 msg.uop = SCIF_UNREGISTER;
633 msg.src = ep->port;
634 msg.payload[0] = window->alloc_handle.vaddr;
635 msg.payload[1] = (u64)window;
636 return scif_nodeqp_send(ep->remote_dev, &msg);
637 }
638
639
640
641
642
643
644
645 int scif_unregister_window(struct scif_window *window)
646 {
647 int err = 0;
648 struct scif_endpt *ep = (struct scif_endpt *)window->ep;
649 bool send_msg = false;
650
651 might_sleep();
652 switch (window->unreg_state) {
653 case OP_IDLE:
654 {
655 window->unreg_state = OP_IN_PROGRESS;
656 send_msg = true;
657 }
658
659 case OP_IN_PROGRESS:
660 {
661 scif_get_window(window, 1);
662 mutex_unlock(&ep->rma_info.rma_lock);
663 if (send_msg) {
664 err = scif_send_scif_unregister(ep, window);
665 if (err) {
666 window->unreg_state = OP_COMPLETED;
667 goto done;
668 }
669 } else {
670
671 mutex_lock(&ep->rma_info.rma_lock);
672 return -ENXIO;
673 }
674 retry:
675
676 err = wait_event_timeout(window->unregwq,
677 window->unreg_state != OP_IN_PROGRESS,
678 SCIF_NODE_ALIVE_TIMEOUT);
679 if (!err && scifdev_alive(ep))
680 goto retry;
681 if (!err) {
682 err = -ENODEV;
683 window->unreg_state = OP_COMPLETED;
684 dev_err(scif_info.mdev.this_device,
685 "%s %d err %d\n", __func__, __LINE__, err);
686 }
687 if (err > 0)
688 err = 0;
689 done:
690 mutex_lock(&ep->rma_info.rma_lock);
691 scif_put_window(window, 1);
692 break;
693 }
694 case OP_FAILED:
695 {
696 if (!scifdev_alive(ep)) {
697 err = -ENODEV;
698 window->unreg_state = OP_COMPLETED;
699 }
700 break;
701 }
702 case OP_COMPLETED:
703 break;
704 default:
705 err = -ENODEV;
706 }
707
708 if (window->unreg_state == OP_COMPLETED && window->ref_count)
709 scif_put_window(window, window->nr_pages);
710
711 if (!window->ref_count) {
712 atomic_inc(&ep->rma_info.tw_refcount);
713 list_del_init(&window->list);
714 scif_free_window_offset(ep, window, window->offset);
715 mutex_unlock(&ep->rma_info.rma_lock);
716 if ((!!(window->pinned_pages->map_flags & SCIF_MAP_KERNEL)) &&
717 scifdev_alive(ep)) {
718 scif_drain_dma_intr(ep->remote_dev->sdev,
719 ep->rma_info.dma_chan);
720 } else {
721 if (!__scif_dec_pinned_vm_lock(window->mm,
722 window->nr_pages)) {
723 __scif_release_mm(window->mm);
724 window->mm = NULL;
725 }
726 }
727 scif_queue_for_cleanup(window, &scif_info.rma);
728 mutex_lock(&ep->rma_info.rma_lock);
729 }
730 return err;
731 }
732
733
734
735
736
737
738
739
740 static int scif_send_alloc_request(struct scif_endpt *ep,
741 struct scif_window *window)
742 {
743 struct scifmsg msg;
744 struct scif_allocmsg *alloc = &window->alloc_handle;
745
746
747 alloc->state = OP_IN_PROGRESS;
748 init_waitqueue_head(&alloc->allocwq);
749
750
751 msg.uop = SCIF_ALLOC_REQ;
752 msg.payload[1] = window->nr_pages;
753 msg.payload[2] = (u64)&window->alloc_handle;
754 return _scif_nodeqp_send(ep->remote_dev, &msg);
755 }
756
757
758
759
760
761
762
763
764
765 static int scif_prep_remote_window(struct scif_endpt *ep,
766 struct scif_window *window)
767 {
768 struct scifmsg msg;
769 struct scif_window *remote_window;
770 struct scif_allocmsg *alloc = &window->alloc_handle;
771 dma_addr_t *dma_phys_lookup, *tmp, *num_pages_lookup, *tmp1;
772 int i = 0, j = 0;
773 int nr_contig_chunks, loop_nr_contig_chunks;
774 int remaining_nr_contig_chunks, nr_lookup;
775 int err, map_err;
776
777 map_err = scif_map_window(ep->remote_dev, window);
778 if (map_err)
779 dev_err(&ep->remote_dev->sdev->dev,
780 "%s %d map_err %d\n", __func__, __LINE__, map_err);
781 remaining_nr_contig_chunks = window->nr_contig_chunks;
782 nr_contig_chunks = window->nr_contig_chunks;
783 retry:
784
785 err = wait_event_timeout(alloc->allocwq,
786 alloc->state != OP_IN_PROGRESS,
787 SCIF_NODE_ALIVE_TIMEOUT);
788 mutex_lock(&ep->rma_info.rma_lock);
789
790 mutex_unlock(&ep->rma_info.rma_lock);
791 if (!err && scifdev_alive(ep))
792 goto retry;
793
794 if (!err)
795 err = -ENODEV;
796
797 if (err > 0)
798 err = 0;
799 else
800 return err;
801
802
803 if (alloc->state == OP_FAILED)
804 return -ENOMEM;
805
806 if (map_err) {
807 dev_err(&ep->remote_dev->sdev->dev,
808 "%s %d err %d\n", __func__, __LINE__, map_err);
809 msg.uop = SCIF_FREE_VIRT;
810 msg.src = ep->port;
811 msg.payload[0] = ep->remote_ep;
812 msg.payload[1] = window->alloc_handle.vaddr;
813 msg.payload[2] = (u64)window;
814 msg.payload[3] = SCIF_REGISTER;
815 spin_lock(&ep->lock);
816 if (ep->state == SCIFEP_CONNECTED)
817 err = _scif_nodeqp_send(ep->remote_dev, &msg);
818 else
819 err = -ENOTCONN;
820 spin_unlock(&ep->lock);
821 return err;
822 }
823
824 remote_window = scif_ioremap(alloc->phys_addr, sizeof(*window),
825 ep->remote_dev);
826
827
828 nr_lookup = ALIGN(nr_contig_chunks, SCIF_NR_ADDR_IN_PAGE)
829 >> ilog2(SCIF_NR_ADDR_IN_PAGE);
830
831 dma_phys_lookup =
832 scif_ioremap(remote_window->dma_addr_lookup.offset,
833 nr_lookup *
834 sizeof(*remote_window->dma_addr_lookup.lookup),
835 ep->remote_dev);
836 num_pages_lookup =
837 scif_ioremap(remote_window->num_pages_lookup.offset,
838 nr_lookup *
839 sizeof(*remote_window->num_pages_lookup.lookup),
840 ep->remote_dev);
841
842 while (remaining_nr_contig_chunks) {
843 loop_nr_contig_chunks = min_t(int, remaining_nr_contig_chunks,
844 (int)SCIF_NR_ADDR_IN_PAGE);
845
846
847
848
849
850
851
852
853 tmp = scif_ioremap(dma_phys_lookup[j],
854 loop_nr_contig_chunks *
855 sizeof(*window->dma_addr),
856 ep->remote_dev);
857 tmp1 = scif_ioremap(num_pages_lookup[j],
858 loop_nr_contig_chunks *
859 sizeof(*window->num_pages),
860 ep->remote_dev);
861 if (scif_is_mgmt_node()) {
862 memcpy_toio((void __force __iomem *)tmp,
863 &window->dma_addr[i], loop_nr_contig_chunks
864 * sizeof(*window->dma_addr));
865 memcpy_toio((void __force __iomem *)tmp1,
866 &window->num_pages[i], loop_nr_contig_chunks
867 * sizeof(*window->num_pages));
868 } else {
869 if (scifdev_is_p2p(ep->remote_dev)) {
870
871
872
873
874 int m;
875 dma_addr_t dma_addr;
876
877 for (m = 0; m < loop_nr_contig_chunks; m++) {
878 dma_addr = window->dma_addr[i + m] +
879 ep->remote_dev->base_addr;
880 writeq(dma_addr,
881 (void __force __iomem *)&tmp[m]);
882 }
883 memcpy_toio((void __force __iomem *)tmp1,
884 &window->num_pages[i],
885 loop_nr_contig_chunks
886 * sizeof(*window->num_pages));
887 } else {
888
889
890
891
892
893
894
895 memcpy_toio((void __force __iomem *)tmp,
896 &window->dma_addr[i],
897 loop_nr_contig_chunks *
898 sizeof(*window->dma_addr));
899 memcpy_toio((void __force __iomem *)tmp1,
900 &window->num_pages[i],
901 loop_nr_contig_chunks *
902 sizeof(*window->num_pages));
903 }
904 }
905 remaining_nr_contig_chunks -= loop_nr_contig_chunks;
906 i += loop_nr_contig_chunks;
907 j++;
908 scif_iounmap(tmp, loop_nr_contig_chunks *
909 sizeof(*window->dma_addr), ep->remote_dev);
910 scif_iounmap(tmp1, loop_nr_contig_chunks *
911 sizeof(*window->num_pages), ep->remote_dev);
912 }
913
914
915 remote_window->peer_window = (u64)window;
916 remote_window->offset = window->offset;
917 remote_window->prot = window->prot;
918 remote_window->nr_contig_chunks = nr_contig_chunks;
919 remote_window->ep = ep->remote_ep;
920 scif_iounmap(num_pages_lookup,
921 nr_lookup *
922 sizeof(*remote_window->num_pages_lookup.lookup),
923 ep->remote_dev);
924 scif_iounmap(dma_phys_lookup,
925 nr_lookup *
926 sizeof(*remote_window->dma_addr_lookup.lookup),
927 ep->remote_dev);
928 scif_iounmap(remote_window, sizeof(*remote_window), ep->remote_dev);
929 window->peer_window = alloc->vaddr;
930 return err;
931 }
932
933
934
935
936
937
938
939
940
941
942 static int scif_send_scif_register(struct scif_endpt *ep,
943 struct scif_window *window)
944 {
945 int err = 0;
946 struct scifmsg msg;
947
948 msg.src = ep->port;
949 msg.payload[0] = ep->remote_ep;
950 msg.payload[1] = window->alloc_handle.vaddr;
951 msg.payload[2] = (u64)window;
952 spin_lock(&ep->lock);
953 if (ep->state == SCIFEP_CONNECTED) {
954 msg.uop = SCIF_REGISTER;
955 window->reg_state = OP_IN_PROGRESS;
956 err = _scif_nodeqp_send(ep->remote_dev, &msg);
957 spin_unlock(&ep->lock);
958 if (!err) {
959 retry:
960
961 err = wait_event_timeout(window->regwq,
962 window->reg_state !=
963 OP_IN_PROGRESS,
964 SCIF_NODE_ALIVE_TIMEOUT);
965 if (!err && scifdev_alive(ep))
966 goto retry;
967 err = !err ? -ENODEV : 0;
968 if (window->reg_state == OP_FAILED)
969 err = -ENOTCONN;
970 }
971 } else {
972 msg.uop = SCIF_FREE_VIRT;
973 msg.payload[3] = SCIF_REGISTER;
974 err = _scif_nodeqp_send(ep->remote_dev, &msg);
975 spin_unlock(&ep->lock);
976 if (!err)
977 err = -ENOTCONN;
978 }
979 return err;
980 }
981
982
983
984
985
986
987
988
989
990
991
992 int scif_get_window_offset(struct scif_endpt *ep, int flags, s64 offset,
993 int num_pages, s64 *out_offset)
994 {
995 s64 page_index;
996 struct iova *iova_ptr;
997 int err = 0;
998
999 if (flags & SCIF_MAP_FIXED) {
1000 page_index = SCIF_IOVA_PFN(offset);
1001 iova_ptr = reserve_iova(&ep->rma_info.iovad, page_index,
1002 page_index + num_pages - 1);
1003 if (!iova_ptr)
1004 err = -EADDRINUSE;
1005 } else {
1006 iova_ptr = alloc_iova(&ep->rma_info.iovad, num_pages,
1007 SCIF_DMA_63BIT_PFN - 1, 0);
1008 if (!iova_ptr)
1009 err = -ENOMEM;
1010 }
1011 if (!err)
1012 *out_offset = (iova_ptr->pfn_lo) << PAGE_SHIFT;
1013 return err;
1014 }
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025 void scif_free_window_offset(struct scif_endpt *ep,
1026 struct scif_window *window, s64 offset)
1027 {
1028 if ((window && !window->offset_freed) || !window) {
1029 free_iova(&ep->rma_info.iovad, offset >> PAGE_SHIFT);
1030 if (window)
1031 window->offset_freed = true;
1032 }
1033 }
1034
1035
1036
1037
1038
1039
1040
1041 void scif_alloc_req(struct scif_dev *scifdev, struct scifmsg *msg)
1042 {
1043 int err;
1044 struct scif_window *window = NULL;
1045 int nr_pages = msg->payload[1];
1046
1047 window = scif_create_remote_window(scifdev, nr_pages);
1048 if (!window) {
1049 err = -ENOMEM;
1050 goto error;
1051 }
1052
1053
1054 msg->uop = SCIF_ALLOC_GNT;
1055 msg->payload[0] = (u64)window;
1056 msg->payload[1] = window->mapped_offset;
1057 err = scif_nodeqp_send(scifdev, msg);
1058 if (err)
1059 scif_destroy_remote_window(window);
1060 return;
1061 error:
1062
1063 dev_err(&scifdev->sdev->dev,
1064 "%s %d error %d alloc_ptr %p nr_pages 0x%x\n",
1065 __func__, __LINE__, err, window, nr_pages);
1066 msg->uop = SCIF_ALLOC_REJ;
1067 scif_nodeqp_send(scifdev, msg);
1068 }
1069
1070
1071
1072
1073
1074
1075
1076 void scif_alloc_gnt_rej(struct scif_dev *scifdev, struct scifmsg *msg)
1077 {
1078 struct scif_allocmsg *handle = (struct scif_allocmsg *)msg->payload[2];
1079 struct scif_window *window = container_of(handle, struct scif_window,
1080 alloc_handle);
1081 struct scif_endpt *ep = (struct scif_endpt *)window->ep;
1082
1083 mutex_lock(&ep->rma_info.rma_lock);
1084 handle->vaddr = msg->payload[0];
1085 handle->phys_addr = msg->payload[1];
1086 if (msg->uop == SCIF_ALLOC_GNT)
1087 handle->state = OP_COMPLETED;
1088 else
1089 handle->state = OP_FAILED;
1090 wake_up(&handle->allocwq);
1091 mutex_unlock(&ep->rma_info.rma_lock);
1092 }
1093
1094
1095
1096
1097
1098
1099
1100 void scif_free_virt(struct scif_dev *scifdev, struct scifmsg *msg)
1101 {
1102 struct scif_window *window = (struct scif_window *)msg->payload[1];
1103
1104 scif_destroy_remote_window(window);
1105 }
1106
1107 static void
1108 scif_fixup_aper_base(struct scif_dev *dev, struct scif_window *window)
1109 {
1110 int j;
1111 struct scif_hw_dev *sdev = dev->sdev;
1112 phys_addr_t apt_base = 0;
1113
1114
1115
1116
1117
1118 if (!scifdev_self(dev) && window->type == SCIF_WINDOW_PEER &&
1119 sdev->aper && !sdev->card_rel_da)
1120 apt_base = sdev->aper->pa;
1121 else
1122 return;
1123
1124 for (j = 0; j < window->nr_contig_chunks; j++) {
1125 if (window->num_pages[j])
1126 window->dma_addr[j] += apt_base;
1127 else
1128 break;
1129 }
1130 }
1131
1132
1133
1134
1135
1136
1137
1138 void scif_recv_reg(struct scif_dev *scifdev, struct scifmsg *msg)
1139 {
1140 struct scif_endpt *ep = (struct scif_endpt *)msg->payload[0];
1141 struct scif_window *window =
1142 (struct scif_window *)msg->payload[1];
1143
1144 mutex_lock(&ep->rma_info.rma_lock);
1145 spin_lock(&ep->lock);
1146 if (ep->state == SCIFEP_CONNECTED) {
1147 msg->uop = SCIF_REGISTER_ACK;
1148 scif_nodeqp_send(ep->remote_dev, msg);
1149 scif_fixup_aper_base(ep->remote_dev, window);
1150
1151 scif_insert_window(window, &ep->rma_info.remote_reg_list);
1152 } else {
1153 msg->uop = SCIF_REGISTER_NACK;
1154 scif_nodeqp_send(ep->remote_dev, msg);
1155 }
1156 spin_unlock(&ep->lock);
1157 mutex_unlock(&ep->rma_info.rma_lock);
1158
1159 scif_destroy_remote_lookup(ep->remote_dev, window);
1160
1161
1162
1163
1164 if (msg->uop == SCIF_REGISTER_NACK)
1165 scif_destroy_remote_window(window);
1166 }
1167
1168
1169
1170
1171
1172
1173
1174 void scif_recv_unreg(struct scif_dev *scifdev, struct scifmsg *msg)
1175 {
1176 struct scif_rma_req req;
1177 struct scif_window *window = NULL;
1178 struct scif_window *recv_window =
1179 (struct scif_window *)msg->payload[0];
1180 struct scif_endpt *ep;
1181 int del_window = 0;
1182
1183 ep = (struct scif_endpt *)recv_window->ep;
1184 req.out_window = &window;
1185 req.offset = recv_window->offset;
1186 req.prot = 0;
1187 req.nr_bytes = recv_window->nr_pages << PAGE_SHIFT;
1188 req.type = SCIF_WINDOW_FULL;
1189 req.head = &ep->rma_info.remote_reg_list;
1190 msg->payload[0] = ep->remote_ep;
1191
1192 mutex_lock(&ep->rma_info.rma_lock);
1193
1194 if (scif_query_window(&req)) {
1195 dev_err(&scifdev->sdev->dev,
1196 "%s %d -ENXIO\n", __func__, __LINE__);
1197 msg->uop = SCIF_UNREGISTER_ACK;
1198 goto error;
1199 }
1200 if (window) {
1201 if (window->ref_count)
1202 scif_put_window(window, window->nr_pages);
1203 else
1204 dev_err(&scifdev->sdev->dev,
1205 "%s %d ref count should be +ve\n",
1206 __func__, __LINE__);
1207 window->unreg_state = OP_COMPLETED;
1208 if (!window->ref_count) {
1209 msg->uop = SCIF_UNREGISTER_ACK;
1210 atomic_inc(&ep->rma_info.tw_refcount);
1211 ep->rma_info.async_list_del = 1;
1212 list_del_init(&window->list);
1213 del_window = 1;
1214 } else {
1215
1216 msg->uop = SCIF_UNREGISTER_NACK;
1217 }
1218 } else {
1219
1220 msg->uop = SCIF_UNREGISTER_ACK;
1221 scif_destroy_remote_window(recv_window);
1222 }
1223 error:
1224 mutex_unlock(&ep->rma_info.rma_lock);
1225 if (del_window)
1226 scif_drain_dma_intr(ep->remote_dev->sdev,
1227 ep->rma_info.dma_chan);
1228 scif_nodeqp_send(ep->remote_dev, msg);
1229 if (del_window)
1230 scif_queue_for_cleanup(window, &scif_info.rma);
1231 }
1232
1233
1234
1235
1236
1237
1238
1239 void scif_recv_reg_ack(struct scif_dev *scifdev, struct scifmsg *msg)
1240 {
1241 struct scif_window *window =
1242 (struct scif_window *)msg->payload[2];
1243 struct scif_endpt *ep = (struct scif_endpt *)window->ep;
1244
1245 mutex_lock(&ep->rma_info.rma_lock);
1246 window->reg_state = OP_COMPLETED;
1247 wake_up(&window->regwq);
1248 mutex_unlock(&ep->rma_info.rma_lock);
1249 }
1250
1251
1252
1253
1254
1255
1256
1257
1258 void scif_recv_reg_nack(struct scif_dev *scifdev, struct scifmsg *msg)
1259 {
1260 struct scif_window *window =
1261 (struct scif_window *)msg->payload[2];
1262 struct scif_endpt *ep = (struct scif_endpt *)window->ep;
1263
1264 mutex_lock(&ep->rma_info.rma_lock);
1265 window->reg_state = OP_FAILED;
1266 wake_up(&window->regwq);
1267 mutex_unlock(&ep->rma_info.rma_lock);
1268 }
1269
1270
1271
1272
1273
1274
1275
1276 void scif_recv_unreg_ack(struct scif_dev *scifdev, struct scifmsg *msg)
1277 {
1278 struct scif_window *window =
1279 (struct scif_window *)msg->payload[1];
1280 struct scif_endpt *ep = (struct scif_endpt *)window->ep;
1281
1282 mutex_lock(&ep->rma_info.rma_lock);
1283 window->unreg_state = OP_COMPLETED;
1284 wake_up(&window->unregwq);
1285 mutex_unlock(&ep->rma_info.rma_lock);
1286 }
1287
1288
1289
1290
1291
1292
1293
1294
1295 void scif_recv_unreg_nack(struct scif_dev *scifdev, struct scifmsg *msg)
1296 {
1297 struct scif_window *window =
1298 (struct scif_window *)msg->payload[1];
1299 struct scif_endpt *ep = (struct scif_endpt *)window->ep;
1300
1301 mutex_lock(&ep->rma_info.rma_lock);
1302 window->unreg_state = OP_FAILED;
1303 wake_up(&window->unregwq);
1304 mutex_unlock(&ep->rma_info.rma_lock);
1305 }
1306
1307 int __scif_pin_pages(void *addr, size_t len, int *out_prot,
1308 int map_flags, scif_pinned_pages_t *pages)
1309 {
1310 struct scif_pinned_pages *pinned_pages;
1311 int nr_pages, err = 0, i;
1312 bool vmalloc_addr = false;
1313 bool try_upgrade = false;
1314 int prot = *out_prot;
1315 int ulimit = 0;
1316 struct mm_struct *mm = NULL;
1317
1318
1319 if (map_flags & ~(SCIF_MAP_KERNEL | SCIF_MAP_ULIMIT))
1320 return -EINVAL;
1321 ulimit = !!(map_flags & SCIF_MAP_ULIMIT);
1322
1323
1324 if (prot & ~(SCIF_PROT_READ | SCIF_PROT_WRITE))
1325 return -EINVAL;
1326
1327
1328 if (!len ||
1329 (ALIGN((u64)addr, PAGE_SIZE) != (u64)addr) ||
1330 (ALIGN((u64)len, PAGE_SIZE) != (u64)len))
1331 return -EINVAL;
1332
1333 might_sleep();
1334
1335 nr_pages = len >> PAGE_SHIFT;
1336
1337
1338 pinned_pages = scif_create_pinned_pages(nr_pages, prot);
1339 if (!pinned_pages)
1340 return -ENOMEM;
1341
1342 if (map_flags & SCIF_MAP_KERNEL) {
1343 if (is_vmalloc_addr(addr))
1344 vmalloc_addr = true;
1345
1346 for (i = 0; i < nr_pages; i++) {
1347 if (vmalloc_addr)
1348 pinned_pages->pages[i] =
1349 vmalloc_to_page(addr + (i * PAGE_SIZE));
1350 else
1351 pinned_pages->pages[i] =
1352 virt_to_page(addr + (i * PAGE_SIZE));
1353 }
1354 pinned_pages->nr_pages = nr_pages;
1355 pinned_pages->map_flags = SCIF_MAP_KERNEL;
1356 } else {
1357
1358
1359
1360
1361
1362
1363
1364
1365 if (prot == SCIF_PROT_READ)
1366 try_upgrade = true;
1367 prot |= SCIF_PROT_WRITE;
1368 retry:
1369 mm = current->mm;
1370 if (ulimit) {
1371 err = __scif_check_inc_pinned_vm(mm, nr_pages);
1372 if (err) {
1373 pinned_pages->nr_pages = 0;
1374 goto error_unmap;
1375 }
1376 }
1377
1378 pinned_pages->nr_pages = get_user_pages_fast(
1379 (u64)addr,
1380 nr_pages,
1381 (prot & SCIF_PROT_WRITE) ? FOLL_WRITE : 0,
1382 pinned_pages->pages);
1383 if (nr_pages != pinned_pages->nr_pages) {
1384 if (try_upgrade) {
1385 if (ulimit)
1386 __scif_dec_pinned_vm_lock(mm, nr_pages);
1387
1388 for (i = 0; i < pinned_pages->nr_pages; i++) {
1389 if (pinned_pages->pages[i])
1390 put_page(
1391 pinned_pages->pages[i]);
1392 }
1393 prot &= ~SCIF_PROT_WRITE;
1394 try_upgrade = false;
1395 goto retry;
1396 }
1397 }
1398 pinned_pages->map_flags = 0;
1399 }
1400
1401 if (pinned_pages->nr_pages < nr_pages) {
1402 err = -EFAULT;
1403 pinned_pages->nr_pages = nr_pages;
1404 goto dec_pinned;
1405 }
1406
1407 *out_prot = prot;
1408 atomic_set(&pinned_pages->ref_count, 1);
1409 *pages = pinned_pages;
1410 return err;
1411 dec_pinned:
1412 if (ulimit)
1413 __scif_dec_pinned_vm_lock(mm, nr_pages);
1414
1415 error_unmap:
1416 pinned_pages->nr_pages = nr_pages;
1417 scif_destroy_pinned_pages(pinned_pages);
1418 *pages = NULL;
1419 dev_dbg(scif_info.mdev.this_device,
1420 "%s %d err %d len 0x%lx\n", __func__, __LINE__, err, len);
1421 return err;
1422 }
1423
1424 int scif_pin_pages(void *addr, size_t len, int prot,
1425 int map_flags, scif_pinned_pages_t *pages)
1426 {
1427 return __scif_pin_pages(addr, len, &prot, map_flags, pages);
1428 }
1429 EXPORT_SYMBOL_GPL(scif_pin_pages);
1430
1431 int scif_unpin_pages(scif_pinned_pages_t pinned_pages)
1432 {
1433 int err = 0, ret;
1434
1435 if (!pinned_pages || SCIFEP_MAGIC != pinned_pages->magic)
1436 return -EINVAL;
1437
1438 ret = atomic_sub_return(1, &pinned_pages->ref_count);
1439 if (ret < 0) {
1440 dev_err(scif_info.mdev.this_device,
1441 "%s %d scif_unpin_pages called without pinning? rc %d\n",
1442 __func__, __LINE__, ret);
1443 return -EINVAL;
1444 }
1445
1446
1447
1448
1449
1450
1451 if (!ret)
1452 err = scif_destroy_pinned_pages(pinned_pages);
1453
1454 return err;
1455 }
1456 EXPORT_SYMBOL_GPL(scif_unpin_pages);
1457
1458 static inline void
1459 scif_insert_local_window(struct scif_window *window, struct scif_endpt *ep)
1460 {
1461 mutex_lock(&ep->rma_info.rma_lock);
1462 scif_insert_window(window, &ep->rma_info.reg_list);
1463 mutex_unlock(&ep->rma_info.rma_lock);
1464 }
1465
1466 off_t scif_register_pinned_pages(scif_epd_t epd,
1467 scif_pinned_pages_t pinned_pages,
1468 off_t offset, int map_flags)
1469 {
1470 struct scif_endpt *ep = (struct scif_endpt *)epd;
1471 s64 computed_offset;
1472 struct scif_window *window;
1473 int err;
1474 size_t len;
1475 struct device *spdev;
1476
1477
1478 if (map_flags & ~SCIF_MAP_FIXED)
1479 return -EINVAL;
1480
1481 len = pinned_pages->nr_pages << PAGE_SHIFT;
1482
1483
1484
1485
1486
1487 if ((map_flags & SCIF_MAP_FIXED) &&
1488 ((ALIGN(offset, PAGE_SIZE) != offset) ||
1489 (offset < 0) ||
1490 (len > LONG_MAX - offset)))
1491 return -EINVAL;
1492
1493 might_sleep();
1494
1495 err = scif_verify_epd(ep);
1496 if (err)
1497 return err;
1498
1499
1500
1501
1502 if (!atomic_add_unless(&pinned_pages->ref_count, 1, 0))
1503 return -EINVAL;
1504
1505
1506 err = scif_get_window_offset(ep, map_flags, offset,
1507 len, &computed_offset);
1508 if (err) {
1509 atomic_sub(1, &pinned_pages->ref_count);
1510 return err;
1511 }
1512
1513
1514 window = scif_create_window(ep, pinned_pages->nr_pages,
1515 computed_offset, false);
1516 if (!window) {
1517 atomic_sub(1, &pinned_pages->ref_count);
1518 scif_free_window_offset(ep, NULL, computed_offset);
1519 return -ENOMEM;
1520 }
1521
1522 window->pinned_pages = pinned_pages;
1523 window->nr_pages = pinned_pages->nr_pages;
1524 window->prot = pinned_pages->prot;
1525
1526 spdev = scif_get_peer_dev(ep->remote_dev);
1527 if (IS_ERR(spdev)) {
1528 err = PTR_ERR(spdev);
1529 scif_destroy_window(ep, window);
1530 return err;
1531 }
1532 err = scif_send_alloc_request(ep, window);
1533 if (err) {
1534 dev_err(&ep->remote_dev->sdev->dev,
1535 "%s %d err %d\n", __func__, __LINE__, err);
1536 goto error_unmap;
1537 }
1538
1539
1540 err = scif_prep_remote_window(ep, window);
1541 if (err) {
1542 dev_err(&ep->remote_dev->sdev->dev,
1543 "%s %d err %d\n", __func__, __LINE__, err);
1544 goto error_unmap;
1545 }
1546
1547
1548 err = scif_send_scif_register(ep, window);
1549 if (err) {
1550 dev_err(&ep->remote_dev->sdev->dev,
1551 "%s %d err %d\n", __func__, __LINE__, err);
1552 goto error_unmap;
1553 }
1554
1555 scif_put_peer_dev(spdev);
1556
1557 scif_insert_local_window(window, ep);
1558 return computed_offset;
1559 error_unmap:
1560 scif_destroy_window(ep, window);
1561 scif_put_peer_dev(spdev);
1562 dev_err(&ep->remote_dev->sdev->dev,
1563 "%s %d err %d\n", __func__, __LINE__, err);
1564 return err;
1565 }
1566 EXPORT_SYMBOL_GPL(scif_register_pinned_pages);
1567
1568 off_t scif_register(scif_epd_t epd, void *addr, size_t len, off_t offset,
1569 int prot, int map_flags)
1570 {
1571 scif_pinned_pages_t pinned_pages;
1572 off_t err;
1573 struct scif_endpt *ep = (struct scif_endpt *)epd;
1574 s64 computed_offset;
1575 struct scif_window *window;
1576 struct mm_struct *mm = NULL;
1577 struct device *spdev;
1578
1579 dev_dbg(scif_info.mdev.this_device,
1580 "SCIFAPI register: ep %p addr %p len 0x%lx offset 0x%lx prot 0x%x map_flags 0x%x\n",
1581 epd, addr, len, offset, prot, map_flags);
1582
1583 if (map_flags & ~(SCIF_MAP_FIXED | SCIF_MAP_KERNEL))
1584 return -EINVAL;
1585
1586
1587
1588
1589
1590 if ((map_flags & SCIF_MAP_FIXED) &&
1591 ((ALIGN(offset, PAGE_SIZE) != offset) ||
1592 (offset < 0) ||
1593 (len > LONG_MAX - offset)))
1594 return -EINVAL;
1595
1596
1597 if (prot & ~(SCIF_PROT_READ | SCIF_PROT_WRITE))
1598 return -EINVAL;
1599
1600
1601 if (!len || (ALIGN((u64)addr, PAGE_SIZE) != (u64)addr) ||
1602 (ALIGN(len, PAGE_SIZE) != len))
1603 return -EINVAL;
1604
1605 might_sleep();
1606
1607 err = scif_verify_epd(ep);
1608 if (err)
1609 return err;
1610
1611
1612 err = scif_get_window_offset(ep, map_flags, offset,
1613 len >> PAGE_SHIFT, &computed_offset);
1614 if (err)
1615 return err;
1616
1617 spdev = scif_get_peer_dev(ep->remote_dev);
1618 if (IS_ERR(spdev)) {
1619 err = PTR_ERR(spdev);
1620 scif_free_window_offset(ep, NULL, computed_offset);
1621 return err;
1622 }
1623
1624 window = scif_create_window(ep, len >> PAGE_SHIFT,
1625 computed_offset, false);
1626 if (!window) {
1627 scif_free_window_offset(ep, NULL, computed_offset);
1628 scif_put_peer_dev(spdev);
1629 return -ENOMEM;
1630 }
1631
1632 window->nr_pages = len >> PAGE_SHIFT;
1633
1634 err = scif_send_alloc_request(ep, window);
1635 if (err) {
1636 scif_destroy_incomplete_window(ep, window);
1637 scif_put_peer_dev(spdev);
1638 return err;
1639 }
1640
1641 if (!(map_flags & SCIF_MAP_KERNEL)) {
1642 mm = __scif_acquire_mm();
1643 map_flags |= SCIF_MAP_ULIMIT;
1644 }
1645
1646 err = __scif_pin_pages(addr, len, &prot,
1647 map_flags & (SCIF_MAP_KERNEL | SCIF_MAP_ULIMIT),
1648 &pinned_pages);
1649 if (err) {
1650 scif_destroy_incomplete_window(ep, window);
1651 __scif_release_mm(mm);
1652 goto error;
1653 }
1654
1655 window->pinned_pages = pinned_pages;
1656 window->prot = pinned_pages->prot;
1657 window->mm = mm;
1658
1659
1660 err = scif_prep_remote_window(ep, window);
1661 if (err) {
1662 dev_err(&ep->remote_dev->sdev->dev,
1663 "%s %d err %ld\n", __func__, __LINE__, err);
1664 goto error_unmap;
1665 }
1666
1667
1668 err = scif_send_scif_register(ep, window);
1669 if (err) {
1670 dev_err(&ep->remote_dev->sdev->dev,
1671 "%s %d err %ld\n", __func__, __LINE__, err);
1672 goto error_unmap;
1673 }
1674
1675 scif_put_peer_dev(spdev);
1676
1677 scif_insert_local_window(window, ep);
1678 dev_dbg(&ep->remote_dev->sdev->dev,
1679 "SCIFAPI register: ep %p addr %p len 0x%lx computed_offset 0x%llx\n",
1680 epd, addr, len, computed_offset);
1681 return computed_offset;
1682 error_unmap:
1683 scif_destroy_window(ep, window);
1684 error:
1685 scif_put_peer_dev(spdev);
1686 dev_err(&ep->remote_dev->sdev->dev,
1687 "%s %d err %ld\n", __func__, __LINE__, err);
1688 return err;
1689 }
1690 EXPORT_SYMBOL_GPL(scif_register);
1691
1692 int
1693 scif_unregister(scif_epd_t epd, off_t offset, size_t len)
1694 {
1695 struct scif_endpt *ep = (struct scif_endpt *)epd;
1696 struct scif_window *window = NULL;
1697 struct scif_rma_req req;
1698 int nr_pages, err;
1699 struct device *spdev;
1700
1701 dev_dbg(scif_info.mdev.this_device,
1702 "SCIFAPI unregister: ep %p offset 0x%lx len 0x%lx\n",
1703 ep, offset, len);
1704
1705 if (!len ||
1706 (ALIGN((u64)len, PAGE_SIZE) != (u64)len))
1707 return -EINVAL;
1708
1709
1710 if ((ALIGN(offset, PAGE_SIZE) != offset) ||
1711 (offset < 0) ||
1712 (len > LONG_MAX - offset))
1713 return -EINVAL;
1714
1715 err = scif_verify_epd(ep);
1716 if (err)
1717 return err;
1718
1719 might_sleep();
1720 nr_pages = len >> PAGE_SHIFT;
1721
1722 req.out_window = &window;
1723 req.offset = offset;
1724 req.prot = 0;
1725 req.nr_bytes = len;
1726 req.type = SCIF_WINDOW_FULL;
1727 req.head = &ep->rma_info.reg_list;
1728
1729 spdev = scif_get_peer_dev(ep->remote_dev);
1730 if (IS_ERR(spdev)) {
1731 err = PTR_ERR(spdev);
1732 return err;
1733 }
1734 mutex_lock(&ep->rma_info.rma_lock);
1735
1736 err = scif_query_window(&req);
1737 if (err) {
1738 dev_err(&ep->remote_dev->sdev->dev,
1739 "%s %d err %d\n", __func__, __LINE__, err);
1740 goto error;
1741 }
1742
1743 err = scif_rma_list_unregister(window, offset, nr_pages);
1744 if (err)
1745 dev_err(&ep->remote_dev->sdev->dev,
1746 "%s %d err %d\n", __func__, __LINE__, err);
1747 error:
1748 mutex_unlock(&ep->rma_info.rma_lock);
1749 scif_put_peer_dev(spdev);
1750 return err;
1751 }
1752 EXPORT_SYMBOL_GPL(scif_unregister);