This source file includes following definitions.
- hv_signal_on_write
- hv_get_next_write_location
- hv_set_next_write_location
- hv_set_next_read_location
- hv_get_ring_buffersize
- hv_get_ring_bufferindices
- hv_copyto_ringbuffer
- hv_get_ringbuffer_availbytes
- hv_ringbuffer_get_debuginfo
- hv_ringbuffer_pre_init
- hv_ringbuffer_init
- hv_ringbuffer_cleanup
- hv_ringbuffer_write
- hv_ringbuffer_read
- hv_pkt_iter_avail
- hv_pkt_iter_first
- __hv_pkt_iter_next
- hv_pkt_iter_bytes_read
- hv_pkt_iter_close
1
2
3
4
5
6
7
8
9
10
11 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
12
13 #include <linux/kernel.h>
14 #include <linux/mm.h>
15 #include <linux/hyperv.h>
16 #include <linux/uio.h>
17 #include <linux/vmalloc.h>
18 #include <linux/slab.h>
19 #include <linux/prefetch.h>
20
21 #include "hyperv_vmbus.h"
22
23 #define VMBUS_PKT_TRAILER 8
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50 static void hv_signal_on_write(u32 old_write, struct vmbus_channel *channel)
51 {
52 struct hv_ring_buffer_info *rbi = &channel->outbound;
53
54 virt_mb();
55 if (READ_ONCE(rbi->ring_buffer->interrupt_mask))
56 return;
57
58
59 virt_rmb();
60
61
62
63
64 if (old_write == READ_ONCE(rbi->ring_buffer->read_index)) {
65 ++channel->intr_out_empty;
66 vmbus_setevent(channel);
67 }
68 }
69
70
71 static inline u32
72 hv_get_next_write_location(struct hv_ring_buffer_info *ring_info)
73 {
74 u32 next = ring_info->ring_buffer->write_index;
75
76 return next;
77 }
78
79
80 static inline void
81 hv_set_next_write_location(struct hv_ring_buffer_info *ring_info,
82 u32 next_write_location)
83 {
84 ring_info->ring_buffer->write_index = next_write_location;
85 }
86
87
88 static inline void
89 hv_set_next_read_location(struct hv_ring_buffer_info *ring_info,
90 u32 next_read_location)
91 {
92 ring_info->ring_buffer->read_index = next_read_location;
93 ring_info->priv_read_index = next_read_location;
94 }
95
96
97 static inline u32
98 hv_get_ring_buffersize(const struct hv_ring_buffer_info *ring_info)
99 {
100 return ring_info->ring_datasize;
101 }
102
103
104 static inline u64
105 hv_get_ring_bufferindices(struct hv_ring_buffer_info *ring_info)
106 {
107 return (u64)ring_info->ring_buffer->write_index << 32;
108 }
109
110
111
112
113
114 static u32 hv_copyto_ringbuffer(
115 struct hv_ring_buffer_info *ring_info,
116 u32 start_write_offset,
117 const void *src,
118 u32 srclen)
119 {
120 void *ring_buffer = hv_get_ring_buffer(ring_info);
121 u32 ring_buffer_size = hv_get_ring_buffersize(ring_info);
122
123 memcpy(ring_buffer + start_write_offset, src, srclen);
124
125 start_write_offset += srclen;
126 if (start_write_offset >= ring_buffer_size)
127 start_write_offset -= ring_buffer_size;
128
129 return start_write_offset;
130 }
131
132
133
134
135
136
137
138
139 static void
140 hv_get_ringbuffer_availbytes(const struct hv_ring_buffer_info *rbi,
141 u32 *read, u32 *write)
142 {
143 u32 read_loc, write_loc, dsize;
144
145
146 read_loc = READ_ONCE(rbi->ring_buffer->read_index);
147 write_loc = READ_ONCE(rbi->ring_buffer->write_index);
148 dsize = rbi->ring_datasize;
149
150 *write = write_loc >= read_loc ? dsize - (write_loc - read_loc) :
151 read_loc - write_loc;
152 *read = dsize - *write;
153 }
154
155
156 int hv_ringbuffer_get_debuginfo(struct hv_ring_buffer_info *ring_info,
157 struct hv_ring_buffer_debug_info *debug_info)
158 {
159 u32 bytes_avail_towrite;
160 u32 bytes_avail_toread;
161
162 mutex_lock(&ring_info->ring_buffer_mutex);
163
164 if (!ring_info->ring_buffer) {
165 mutex_unlock(&ring_info->ring_buffer_mutex);
166 return -EINVAL;
167 }
168
169 hv_get_ringbuffer_availbytes(ring_info,
170 &bytes_avail_toread,
171 &bytes_avail_towrite);
172 debug_info->bytes_avail_toread = bytes_avail_toread;
173 debug_info->bytes_avail_towrite = bytes_avail_towrite;
174 debug_info->current_read_index = ring_info->ring_buffer->read_index;
175 debug_info->current_write_index = ring_info->ring_buffer->write_index;
176 debug_info->current_interrupt_mask
177 = ring_info->ring_buffer->interrupt_mask;
178 mutex_unlock(&ring_info->ring_buffer_mutex);
179
180 return 0;
181 }
182 EXPORT_SYMBOL_GPL(hv_ringbuffer_get_debuginfo);
183
184
185 void hv_ringbuffer_pre_init(struct vmbus_channel *channel)
186 {
187 mutex_init(&channel->inbound.ring_buffer_mutex);
188 mutex_init(&channel->outbound.ring_buffer_mutex);
189 }
190
191
192 int hv_ringbuffer_init(struct hv_ring_buffer_info *ring_info,
193 struct page *pages, u32 page_cnt)
194 {
195 int i;
196 struct page **pages_wraparound;
197
198 BUILD_BUG_ON((sizeof(struct hv_ring_buffer) != PAGE_SIZE));
199
200
201
202
203
204 pages_wraparound = kcalloc(page_cnt * 2 - 1, sizeof(struct page *),
205 GFP_KERNEL);
206 if (!pages_wraparound)
207 return -ENOMEM;
208
209 pages_wraparound[0] = pages;
210 for (i = 0; i < 2 * (page_cnt - 1); i++)
211 pages_wraparound[i + 1] = &pages[i % (page_cnt - 1) + 1];
212
213 ring_info->ring_buffer = (struct hv_ring_buffer *)
214 vmap(pages_wraparound, page_cnt * 2 - 1, VM_MAP, PAGE_KERNEL);
215
216 kfree(pages_wraparound);
217
218
219 if (!ring_info->ring_buffer)
220 return -ENOMEM;
221
222 ring_info->ring_buffer->read_index =
223 ring_info->ring_buffer->write_index = 0;
224
225
226 ring_info->ring_buffer->feature_bits.value = 1;
227
228 ring_info->ring_size = page_cnt << PAGE_SHIFT;
229 ring_info->ring_size_div10_reciprocal =
230 reciprocal_value(ring_info->ring_size / 10);
231 ring_info->ring_datasize = ring_info->ring_size -
232 sizeof(struct hv_ring_buffer);
233 ring_info->priv_read_index = 0;
234
235 spin_lock_init(&ring_info->ring_lock);
236
237 return 0;
238 }
239
240
241 void hv_ringbuffer_cleanup(struct hv_ring_buffer_info *ring_info)
242 {
243 mutex_lock(&ring_info->ring_buffer_mutex);
244 vunmap(ring_info->ring_buffer);
245 ring_info->ring_buffer = NULL;
246 mutex_unlock(&ring_info->ring_buffer_mutex);
247 }
248
249
250 int hv_ringbuffer_write(struct vmbus_channel *channel,
251 const struct kvec *kv_list, u32 kv_count)
252 {
253 int i;
254 u32 bytes_avail_towrite;
255 u32 totalbytes_towrite = sizeof(u64);
256 u32 next_write_location;
257 u32 old_write;
258 u64 prev_indices;
259 unsigned long flags;
260 struct hv_ring_buffer_info *outring_info = &channel->outbound;
261
262 if (channel->rescind)
263 return -ENODEV;
264
265 for (i = 0; i < kv_count; i++)
266 totalbytes_towrite += kv_list[i].iov_len;
267
268 spin_lock_irqsave(&outring_info->ring_lock, flags);
269
270 bytes_avail_towrite = hv_get_bytes_to_write(outring_info);
271
272
273
274
275
276
277 if (bytes_avail_towrite <= totalbytes_towrite) {
278 ++channel->out_full_total;
279
280 if (!channel->out_full_flag) {
281 ++channel->out_full_first;
282 channel->out_full_flag = true;
283 }
284
285 spin_unlock_irqrestore(&outring_info->ring_lock, flags);
286 return -EAGAIN;
287 }
288
289 channel->out_full_flag = false;
290
291
292 next_write_location = hv_get_next_write_location(outring_info);
293
294 old_write = next_write_location;
295
296 for (i = 0; i < kv_count; i++) {
297 next_write_location = hv_copyto_ringbuffer(outring_info,
298 next_write_location,
299 kv_list[i].iov_base,
300 kv_list[i].iov_len);
301 }
302
303
304 prev_indices = hv_get_ring_bufferindices(outring_info);
305
306 next_write_location = hv_copyto_ringbuffer(outring_info,
307 next_write_location,
308 &prev_indices,
309 sizeof(u64));
310
311
312 virt_mb();
313
314
315 hv_set_next_write_location(outring_info, next_write_location);
316
317
318 spin_unlock_irqrestore(&outring_info->ring_lock, flags);
319
320 hv_signal_on_write(old_write, channel);
321
322 if (channel->rescind)
323 return -ENODEV;
324
325 return 0;
326 }
327
328 int hv_ringbuffer_read(struct vmbus_channel *channel,
329 void *buffer, u32 buflen, u32 *buffer_actual_len,
330 u64 *requestid, bool raw)
331 {
332 struct vmpacket_descriptor *desc;
333 u32 packetlen, offset;
334
335 if (unlikely(buflen == 0))
336 return -EINVAL;
337
338 *buffer_actual_len = 0;
339 *requestid = 0;
340
341
342 desc = hv_pkt_iter_first(channel);
343 if (desc == NULL) {
344
345
346
347
348 return 0;
349 }
350
351 offset = raw ? 0 : (desc->offset8 << 3);
352 packetlen = (desc->len8 << 3) - offset;
353 *buffer_actual_len = packetlen;
354 *requestid = desc->trans_id;
355
356 if (unlikely(packetlen > buflen))
357 return -ENOBUFS;
358
359
360 memcpy(buffer, (const char *)desc + offset, packetlen);
361
362
363 __hv_pkt_iter_next(channel, desc);
364
365
366 hv_pkt_iter_close(channel);
367
368 return 0;
369 }
370
371
372
373
374
375
376
377
378 static u32 hv_pkt_iter_avail(const struct hv_ring_buffer_info *rbi)
379 {
380 u32 priv_read_loc = rbi->priv_read_index;
381 u32 write_loc = READ_ONCE(rbi->ring_buffer->write_index);
382
383 if (write_loc >= priv_read_loc)
384 return write_loc - priv_read_loc;
385 else
386 return (rbi->ring_datasize - priv_read_loc) + write_loc;
387 }
388
389
390
391
392
393
394 struct vmpacket_descriptor *hv_pkt_iter_first(struct vmbus_channel *channel)
395 {
396 struct hv_ring_buffer_info *rbi = &channel->inbound;
397 struct vmpacket_descriptor *desc;
398
399 if (hv_pkt_iter_avail(rbi) < sizeof(struct vmpacket_descriptor))
400 return NULL;
401
402 desc = hv_get_ring_buffer(rbi) + rbi->priv_read_index;
403 if (desc)
404 prefetch((char *)desc + (desc->len8 << 3));
405
406 return desc;
407 }
408 EXPORT_SYMBOL_GPL(hv_pkt_iter_first);
409
410
411
412
413
414
415
416 struct vmpacket_descriptor *
417 __hv_pkt_iter_next(struct vmbus_channel *channel,
418 const struct vmpacket_descriptor *desc)
419 {
420 struct hv_ring_buffer_info *rbi = &channel->inbound;
421 u32 packetlen = desc->len8 << 3;
422 u32 dsize = rbi->ring_datasize;
423
424
425 rbi->priv_read_index += packetlen + VMBUS_PKT_TRAILER;
426 if (rbi->priv_read_index >= dsize)
427 rbi->priv_read_index -= dsize;
428
429
430 return hv_pkt_iter_first(channel);
431 }
432 EXPORT_SYMBOL_GPL(__hv_pkt_iter_next);
433
434
435 static u32 hv_pkt_iter_bytes_read(const struct hv_ring_buffer_info *rbi,
436 u32 start_read_index)
437 {
438 if (rbi->priv_read_index >= start_read_index)
439 return rbi->priv_read_index - start_read_index;
440 else
441 return rbi->ring_datasize - start_read_index +
442 rbi->priv_read_index;
443 }
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465 void hv_pkt_iter_close(struct vmbus_channel *channel)
466 {
467 struct hv_ring_buffer_info *rbi = &channel->inbound;
468 u32 curr_write_sz, pending_sz, bytes_read, start_read_index;
469
470
471
472
473
474
475 virt_rmb();
476 start_read_index = rbi->ring_buffer->read_index;
477 rbi->ring_buffer->read_index = rbi->priv_read_index;
478
479
480
481
482
483
484 if (!rbi->ring_buffer->feature_bits.feat_pending_send_sz)
485 return;
486
487
488
489
490
491
492
493
494
495
496 virt_mb();
497
498
499
500
501
502
503 pending_sz = READ_ONCE(rbi->ring_buffer->pending_send_sz);
504 if (!pending_sz)
505 return;
506
507
508
509
510
511 virt_rmb();
512 curr_write_sz = hv_get_bytes_to_write(rbi);
513 bytes_read = hv_pkt_iter_bytes_read(rbi, start_read_index);
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535 if (curr_write_sz - bytes_read > pending_sz)
536 return;
537
538
539
540
541
542 if (curr_write_sz <= pending_sz)
543 return;
544
545 ++channel->intr_in_full;
546 vmbus_setevent(channel);
547 }
548 EXPORT_SYMBOL_GPL(hv_pkt_iter_close);