This source file includes following definitions.
- vmci_transport_notify_waiting_write
- vmci_transport_notify_waiting_read
- vmci_transport_handle_waiting_read
- vmci_transport_handle_waiting_write
- vmci_transport_handle_read
- send_waiting_read
- send_waiting_write
- vmci_transport_send_read_notification
- vmci_transport_handle_wrote
- vmci_transport_notify_pkt_socket_init
- vmci_transport_notify_pkt_socket_destruct
- vmci_transport_notify_pkt_poll_in
- vmci_transport_notify_pkt_poll_out
- vmci_transport_notify_pkt_recv_init
- vmci_transport_notify_pkt_recv_pre_block
- vmci_transport_notify_pkt_recv_pre_dequeue
- vmci_transport_notify_pkt_recv_post_dequeue
- vmci_transport_notify_pkt_send_init
- vmci_transport_notify_pkt_send_pre_block
- vmci_transport_notify_pkt_send_pre_enqueue
- vmci_transport_notify_pkt_send_post_enqueue
- vmci_transport_notify_pkt_handle_pkt
- vmci_transport_notify_pkt_process_request
- vmci_transport_notify_pkt_process_negotiate
1
2
3
4
5
6
7
8 #include <linux/types.h>
9 #include <linux/socket.h>
10 #include <linux/stddef.h>
11 #include <net/sock.h>
12
13 #include "vmci_transport_notify.h"
14
15 #define PKT_FIELD(vsk, field_name) (vmci_trans(vsk)->notify.pkt.field_name)
16
17 static bool vmci_transport_notify_waiting_write(struct vsock_sock *vsk)
18 {
19 #if defined(VSOCK_OPTIMIZATION_WAITING_NOTIFY)
20 bool retval;
21 u64 notify_limit;
22
23 if (!PKT_FIELD(vsk, peer_waiting_write))
24 return false;
25
26 #ifdef VSOCK_OPTIMIZATION_FLOW_CONTROL
27
28
29
30
31
32
33
34 if (!PKT_FIELD(vsk, peer_waiting_write_detected)) {
35 PKT_FIELD(vsk, peer_waiting_write_detected) = true;
36 if (PKT_FIELD(vsk, write_notify_window) < PAGE_SIZE) {
37 PKT_FIELD(vsk, write_notify_window) =
38 PKT_FIELD(vsk, write_notify_min_window);
39 } else {
40 PKT_FIELD(vsk, write_notify_window) -= PAGE_SIZE;
41 if (PKT_FIELD(vsk, write_notify_window) <
42 PKT_FIELD(vsk, write_notify_min_window))
43 PKT_FIELD(vsk, write_notify_window) =
44 PKT_FIELD(vsk, write_notify_min_window);
45
46 }
47 }
48 notify_limit = vmci_trans(vsk)->consume_size -
49 PKT_FIELD(vsk, write_notify_window);
50 #else
51 notify_limit = 0;
52 #endif
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68 retval = vmci_qpair_consume_free_space(vmci_trans(vsk)->qpair) >
69 notify_limit;
70 #ifdef VSOCK_OPTIMIZATION_FLOW_CONTROL
71 if (retval) {
72
73
74
75
76
77 PKT_FIELD(vsk, peer_waiting_write_detected) = false;
78 }
79 #endif
80 return retval;
81 #else
82 return true;
83 #endif
84 }
85
86 static bool vmci_transport_notify_waiting_read(struct vsock_sock *vsk)
87 {
88 #if defined(VSOCK_OPTIMIZATION_WAITING_NOTIFY)
89 if (!PKT_FIELD(vsk, peer_waiting_read))
90 return false;
91
92
93
94
95
96
97
98 return vmci_qpair_produce_buf_ready(vmci_trans(vsk)->qpair) > 0;
99 #else
100 return true;
101 #endif
102 }
103
104 static void
105 vmci_transport_handle_waiting_read(struct sock *sk,
106 struct vmci_transport_packet *pkt,
107 bool bottom_half,
108 struct sockaddr_vm *dst,
109 struct sockaddr_vm *src)
110 {
111 #if defined(VSOCK_OPTIMIZATION_WAITING_NOTIFY)
112 struct vsock_sock *vsk;
113
114 vsk = vsock_sk(sk);
115
116 PKT_FIELD(vsk, peer_waiting_read) = true;
117 memcpy(&PKT_FIELD(vsk, peer_waiting_read_info), &pkt->u.wait,
118 sizeof(PKT_FIELD(vsk, peer_waiting_read_info)));
119
120 if (vmci_transport_notify_waiting_read(vsk)) {
121 bool sent;
122
123 if (bottom_half)
124 sent = vmci_transport_send_wrote_bh(dst, src) > 0;
125 else
126 sent = vmci_transport_send_wrote(sk) > 0;
127
128 if (sent)
129 PKT_FIELD(vsk, peer_waiting_read) = false;
130 }
131 #endif
132 }
133
134 static void
135 vmci_transport_handle_waiting_write(struct sock *sk,
136 struct vmci_transport_packet *pkt,
137 bool bottom_half,
138 struct sockaddr_vm *dst,
139 struct sockaddr_vm *src)
140 {
141 #if defined(VSOCK_OPTIMIZATION_WAITING_NOTIFY)
142 struct vsock_sock *vsk;
143
144 vsk = vsock_sk(sk);
145
146 PKT_FIELD(vsk, peer_waiting_write) = true;
147 memcpy(&PKT_FIELD(vsk, peer_waiting_write_info), &pkt->u.wait,
148 sizeof(PKT_FIELD(vsk, peer_waiting_write_info)));
149
150 if (vmci_transport_notify_waiting_write(vsk)) {
151 bool sent;
152
153 if (bottom_half)
154 sent = vmci_transport_send_read_bh(dst, src) > 0;
155 else
156 sent = vmci_transport_send_read(sk) > 0;
157
158 if (sent)
159 PKT_FIELD(vsk, peer_waiting_write) = false;
160 }
161 #endif
162 }
163
164 static void
165 vmci_transport_handle_read(struct sock *sk,
166 struct vmci_transport_packet *pkt,
167 bool bottom_half,
168 struct sockaddr_vm *dst, struct sockaddr_vm *src)
169 {
170 #if defined(VSOCK_OPTIMIZATION_WAITING_NOTIFY)
171 struct vsock_sock *vsk;
172
173 vsk = vsock_sk(sk);
174 PKT_FIELD(vsk, sent_waiting_write) = false;
175 #endif
176
177 sk->sk_write_space(sk);
178 }
179
180 static bool send_waiting_read(struct sock *sk, u64 room_needed)
181 {
182 #if defined(VSOCK_OPTIMIZATION_WAITING_NOTIFY)
183 struct vsock_sock *vsk;
184 struct vmci_transport_waiting_info waiting_info;
185 u64 tail;
186 u64 head;
187 u64 room_left;
188 bool ret;
189
190 vsk = vsock_sk(sk);
191
192 if (PKT_FIELD(vsk, sent_waiting_read))
193 return true;
194
195 if (PKT_FIELD(vsk, write_notify_window) <
196 vmci_trans(vsk)->consume_size)
197 PKT_FIELD(vsk, write_notify_window) =
198 min(PKT_FIELD(vsk, write_notify_window) + PAGE_SIZE,
199 vmci_trans(vsk)->consume_size);
200
201 vmci_qpair_get_consume_indexes(vmci_trans(vsk)->qpair, &tail, &head);
202 room_left = vmci_trans(vsk)->consume_size - head;
203 if (room_needed >= room_left) {
204 waiting_info.offset = room_needed - room_left;
205 waiting_info.generation =
206 PKT_FIELD(vsk, consume_q_generation) + 1;
207 } else {
208 waiting_info.offset = head + room_needed;
209 waiting_info.generation = PKT_FIELD(vsk, consume_q_generation);
210 }
211
212 ret = vmci_transport_send_waiting_read(sk, &waiting_info) > 0;
213 if (ret)
214 PKT_FIELD(vsk, sent_waiting_read) = true;
215
216 return ret;
217 #else
218 return true;
219 #endif
220 }
221
222 static bool send_waiting_write(struct sock *sk, u64 room_needed)
223 {
224 #if defined(VSOCK_OPTIMIZATION_WAITING_NOTIFY)
225 struct vsock_sock *vsk;
226 struct vmci_transport_waiting_info waiting_info;
227 u64 tail;
228 u64 head;
229 u64 room_left;
230 bool ret;
231
232 vsk = vsock_sk(sk);
233
234 if (PKT_FIELD(vsk, sent_waiting_write))
235 return true;
236
237 vmci_qpair_get_produce_indexes(vmci_trans(vsk)->qpair, &tail, &head);
238 room_left = vmci_trans(vsk)->produce_size - tail;
239 if (room_needed + 1 >= room_left) {
240
241 waiting_info.offset = room_needed + 1 - room_left;
242 waiting_info.generation = PKT_FIELD(vsk, produce_q_generation);
243 } else {
244 waiting_info.offset = tail + room_needed + 1;
245 waiting_info.generation =
246 PKT_FIELD(vsk, produce_q_generation) - 1;
247 }
248
249 ret = vmci_transport_send_waiting_write(sk, &waiting_info) > 0;
250 if (ret)
251 PKT_FIELD(vsk, sent_waiting_write) = true;
252
253 return ret;
254 #else
255 return true;
256 #endif
257 }
258
259 static int vmci_transport_send_read_notification(struct sock *sk)
260 {
261 struct vsock_sock *vsk;
262 bool sent_read;
263 unsigned int retries;
264 int err;
265
266 vsk = vsock_sk(sk);
267 sent_read = false;
268 retries = 0;
269 err = 0;
270
271 if (vmci_transport_notify_waiting_write(vsk)) {
272
273
274
275
276
277
278
279 while (!(vsk->peer_shutdown & RCV_SHUTDOWN) &&
280 !sent_read &&
281 retries < VMCI_TRANSPORT_MAX_DGRAM_RESENDS) {
282 err = vmci_transport_send_read(sk);
283 if (err >= 0)
284 sent_read = true;
285
286 retries++;
287 }
288
289 if (retries >= VMCI_TRANSPORT_MAX_DGRAM_RESENDS)
290 pr_err("%p unable to send read notify to peer\n", sk);
291 else
292 #if defined(VSOCK_OPTIMIZATION_WAITING_NOTIFY)
293 PKT_FIELD(vsk, peer_waiting_write) = false;
294 #endif
295
296 }
297 return err;
298 }
299
300 static void
301 vmci_transport_handle_wrote(struct sock *sk,
302 struct vmci_transport_packet *pkt,
303 bool bottom_half,
304 struct sockaddr_vm *dst, struct sockaddr_vm *src)
305 {
306 #if defined(VSOCK_OPTIMIZATION_WAITING_NOTIFY)
307 struct vsock_sock *vsk = vsock_sk(sk);
308 PKT_FIELD(vsk, sent_waiting_read) = false;
309 #endif
310 sk->sk_data_ready(sk);
311 }
312
313 static void vmci_transport_notify_pkt_socket_init(struct sock *sk)
314 {
315 struct vsock_sock *vsk = vsock_sk(sk);
316
317 PKT_FIELD(vsk, write_notify_window) = PAGE_SIZE;
318 PKT_FIELD(vsk, write_notify_min_window) = PAGE_SIZE;
319 PKT_FIELD(vsk, peer_waiting_read) = false;
320 PKT_FIELD(vsk, peer_waiting_write) = false;
321 PKT_FIELD(vsk, peer_waiting_write_detected) = false;
322 PKT_FIELD(vsk, sent_waiting_read) = false;
323 PKT_FIELD(vsk, sent_waiting_write) = false;
324 PKT_FIELD(vsk, produce_q_generation) = 0;
325 PKT_FIELD(vsk, consume_q_generation) = 0;
326
327 memset(&PKT_FIELD(vsk, peer_waiting_read_info), 0,
328 sizeof(PKT_FIELD(vsk, peer_waiting_read_info)));
329 memset(&PKT_FIELD(vsk, peer_waiting_write_info), 0,
330 sizeof(PKT_FIELD(vsk, peer_waiting_write_info)));
331 }
332
333 static void vmci_transport_notify_pkt_socket_destruct(struct vsock_sock *vsk)
334 {
335 }
336
337 static int
338 vmci_transport_notify_pkt_poll_in(struct sock *sk,
339 size_t target, bool *data_ready_now)
340 {
341 struct vsock_sock *vsk = vsock_sk(sk);
342
343 if (vsock_stream_has_data(vsk)) {
344 *data_ready_now = true;
345 } else {
346
347
348
349
350 if (sk->sk_state == TCP_ESTABLISHED) {
351 if (!send_waiting_read(sk, 1))
352 return -1;
353
354 }
355 *data_ready_now = false;
356 }
357
358 return 0;
359 }
360
361 static int
362 vmci_transport_notify_pkt_poll_out(struct sock *sk,
363 size_t target, bool *space_avail_now)
364 {
365 s64 produce_q_free_space;
366 struct vsock_sock *vsk = vsock_sk(sk);
367
368 produce_q_free_space = vsock_stream_has_space(vsk);
369 if (produce_q_free_space > 0) {
370 *space_avail_now = true;
371 return 0;
372 } else if (produce_q_free_space == 0) {
373
374
375
376
377
378
379
380
381 if (!send_waiting_write(sk, 1))
382 return -1;
383
384 *space_avail_now = false;
385 }
386
387 return 0;
388 }
389
390 static int
391 vmci_transport_notify_pkt_recv_init(
392 struct sock *sk,
393 size_t target,
394 struct vmci_transport_recv_notify_data *data)
395 {
396 struct vsock_sock *vsk = vsock_sk(sk);
397
398 #ifdef VSOCK_OPTIMIZATION_WAITING_NOTIFY
399 data->consume_head = 0;
400 data->produce_tail = 0;
401 #ifdef VSOCK_OPTIMIZATION_FLOW_CONTROL
402 data->notify_on_block = false;
403
404 if (PKT_FIELD(vsk, write_notify_min_window) < target + 1) {
405 PKT_FIELD(vsk, write_notify_min_window) = target + 1;
406 if (PKT_FIELD(vsk, write_notify_window) <
407 PKT_FIELD(vsk, write_notify_min_window)) {
408
409
410
411
412
413
414
415 PKT_FIELD(vsk, write_notify_window) =
416 PKT_FIELD(vsk, write_notify_min_window);
417 data->notify_on_block = true;
418 }
419 }
420 #endif
421 #endif
422
423 return 0;
424 }
425
426 static int
427 vmci_transport_notify_pkt_recv_pre_block(
428 struct sock *sk,
429 size_t target,
430 struct vmci_transport_recv_notify_data *data)
431 {
432 int err = 0;
433
434
435 if (!send_waiting_read(sk, target)) {
436 err = -EHOSTUNREACH;
437 return err;
438 }
439 #ifdef VSOCK_OPTIMIZATION_FLOW_CONTROL
440 if (data->notify_on_block) {
441 err = vmci_transport_send_read_notification(sk);
442 if (err < 0)
443 return err;
444
445 data->notify_on_block = false;
446 }
447 #endif
448
449 return err;
450 }
451
452 static int
453 vmci_transport_notify_pkt_recv_pre_dequeue(
454 struct sock *sk,
455 size_t target,
456 struct vmci_transport_recv_notify_data *data)
457 {
458 struct vsock_sock *vsk = vsock_sk(sk);
459
460
461
462
463 #if defined(VSOCK_OPTIMIZATION_WAITING_NOTIFY)
464 vmci_qpair_get_consume_indexes(vmci_trans(vsk)->qpair,
465 &data->produce_tail,
466 &data->consume_head);
467 #endif
468
469 return 0;
470 }
471
472 static int
473 vmci_transport_notify_pkt_recv_post_dequeue(
474 struct sock *sk,
475 size_t target,
476 ssize_t copied,
477 bool data_read,
478 struct vmci_transport_recv_notify_data *data)
479 {
480 struct vsock_sock *vsk;
481 int err;
482
483 vsk = vsock_sk(sk);
484 err = 0;
485
486 if (data_read) {
487 #if defined(VSOCK_OPTIMIZATION_WAITING_NOTIFY)
488
489
490
491
492 if (copied >=
493 vmci_trans(vsk)->consume_size - data->consume_head)
494 PKT_FIELD(vsk, consume_q_generation)++;
495 #endif
496
497 err = vmci_transport_send_read_notification(sk);
498 if (err < 0)
499 return err;
500
501 }
502 return err;
503 }
504
505 static int
506 vmci_transport_notify_pkt_send_init(
507 struct sock *sk,
508 struct vmci_transport_send_notify_data *data)
509 {
510 #ifdef VSOCK_OPTIMIZATION_WAITING_NOTIFY
511 data->consume_head = 0;
512 data->produce_tail = 0;
513 #endif
514
515 return 0;
516 }
517
518 static int
519 vmci_transport_notify_pkt_send_pre_block(
520 struct sock *sk,
521 struct vmci_transport_send_notify_data *data)
522 {
523
524 if (!send_waiting_write(sk, 1))
525 return -EHOSTUNREACH;
526
527 return 0;
528 }
529
530 static int
531 vmci_transport_notify_pkt_send_pre_enqueue(
532 struct sock *sk,
533 struct vmci_transport_send_notify_data *data)
534 {
535 struct vsock_sock *vsk = vsock_sk(sk);
536
537 #if defined(VSOCK_OPTIMIZATION_WAITING_NOTIFY)
538 vmci_qpair_get_produce_indexes(vmci_trans(vsk)->qpair,
539 &data->produce_tail,
540 &data->consume_head);
541 #endif
542
543 return 0;
544 }
545
546 static int
547 vmci_transport_notify_pkt_send_post_enqueue(
548 struct sock *sk,
549 ssize_t written,
550 struct vmci_transport_send_notify_data *data)
551 {
552 int err = 0;
553 struct vsock_sock *vsk;
554 bool sent_wrote = false;
555 int retries = 0;
556
557 vsk = vsock_sk(sk);
558
559 #if defined(VSOCK_OPTIMIZATION_WAITING_NOTIFY)
560
561
562
563
564 if (written >= vmci_trans(vsk)->produce_size - data->produce_tail)
565 PKT_FIELD(vsk, produce_q_generation)++;
566
567 #endif
568
569 if (vmci_transport_notify_waiting_read(vsk)) {
570
571
572
573
574
575 while (!(vsk->peer_shutdown & RCV_SHUTDOWN) &&
576 !sent_wrote &&
577 retries < VMCI_TRANSPORT_MAX_DGRAM_RESENDS) {
578 err = vmci_transport_send_wrote(sk);
579 if (err >= 0)
580 sent_wrote = true;
581
582 retries++;
583 }
584
585 if (retries >= VMCI_TRANSPORT_MAX_DGRAM_RESENDS) {
586 pr_err("%p unable to send wrote notify to peer\n", sk);
587 return err;
588 } else {
589 #if defined(VSOCK_OPTIMIZATION_WAITING_NOTIFY)
590 PKT_FIELD(vsk, peer_waiting_read) = false;
591 #endif
592 }
593 }
594 return err;
595 }
596
597 static void
598 vmci_transport_notify_pkt_handle_pkt(
599 struct sock *sk,
600 struct vmci_transport_packet *pkt,
601 bool bottom_half,
602 struct sockaddr_vm *dst,
603 struct sockaddr_vm *src, bool *pkt_processed)
604 {
605 bool processed = false;
606
607 switch (pkt->type) {
608 case VMCI_TRANSPORT_PACKET_TYPE_WROTE:
609 vmci_transport_handle_wrote(sk, pkt, bottom_half, dst, src);
610 processed = true;
611 break;
612 case VMCI_TRANSPORT_PACKET_TYPE_READ:
613 vmci_transport_handle_read(sk, pkt, bottom_half, dst, src);
614 processed = true;
615 break;
616 case VMCI_TRANSPORT_PACKET_TYPE_WAITING_WRITE:
617 vmci_transport_handle_waiting_write(sk, pkt, bottom_half,
618 dst, src);
619 processed = true;
620 break;
621
622 case VMCI_TRANSPORT_PACKET_TYPE_WAITING_READ:
623 vmci_transport_handle_waiting_read(sk, pkt, bottom_half,
624 dst, src);
625 processed = true;
626 break;
627 }
628
629 if (pkt_processed)
630 *pkt_processed = processed;
631 }
632
633 static void vmci_transport_notify_pkt_process_request(struct sock *sk)
634 {
635 struct vsock_sock *vsk = vsock_sk(sk);
636
637 PKT_FIELD(vsk, write_notify_window) = vmci_trans(vsk)->consume_size;
638 if (vmci_trans(vsk)->consume_size <
639 PKT_FIELD(vsk, write_notify_min_window))
640 PKT_FIELD(vsk, write_notify_min_window) =
641 vmci_trans(vsk)->consume_size;
642 }
643
644 static void vmci_transport_notify_pkt_process_negotiate(struct sock *sk)
645 {
646 struct vsock_sock *vsk = vsock_sk(sk);
647
648 PKT_FIELD(vsk, write_notify_window) = vmci_trans(vsk)->consume_size;
649 if (vmci_trans(vsk)->consume_size <
650 PKT_FIELD(vsk, write_notify_min_window))
651 PKT_FIELD(vsk, write_notify_min_window) =
652 vmci_trans(vsk)->consume_size;
653 }
654
655
656 const struct vmci_transport_notify_ops vmci_transport_notify_pkt_ops = {
657 .socket_init = vmci_transport_notify_pkt_socket_init,
658 .socket_destruct = vmci_transport_notify_pkt_socket_destruct,
659 .poll_in = vmci_transport_notify_pkt_poll_in,
660 .poll_out = vmci_transport_notify_pkt_poll_out,
661 .handle_notify_pkt = vmci_transport_notify_pkt_handle_pkt,
662 .recv_init = vmci_transport_notify_pkt_recv_init,
663 .recv_pre_block = vmci_transport_notify_pkt_recv_pre_block,
664 .recv_pre_dequeue = vmci_transport_notify_pkt_recv_pre_dequeue,
665 .recv_post_dequeue = vmci_transport_notify_pkt_recv_post_dequeue,
666 .send_init = vmci_transport_notify_pkt_send_init,
667 .send_pre_block = vmci_transport_notify_pkt_send_pre_block,
668 .send_pre_enqueue = vmci_transport_notify_pkt_send_pre_enqueue,
669 .send_post_enqueue = vmci_transport_notify_pkt_send_post_enqueue,
670 .process_request = vmci_transport_notify_pkt_process_request,
671 .process_negotiate = vmci_transport_notify_pkt_process_negotiate,
672 };