35 	struct rchan_buf *buf = vma->vm_private_data;  in relay_file_mmap_close()  local
36 	buf->chan->cb->buf_unmapped(buf, vma->vm_file);  in relay_file_mmap_close()
45 	struct rchan_buf *buf = vma->vm_private_data;  in relay_buf_fault()  local
48 	if (!buf)  in relay_buf_fault()
51 	page = vmalloc_to_page(buf->start + (pgoff << PAGE_SHIFT));  in relay_buf_fault()
99 static int relay_mmap_buf(struct rchan_buf *buf, struct vm_area_struct *vma)  in relay_mmap_buf()  argument
104 	if (!buf)  in relay_mmap_buf()
107 	if (length != (unsigned long)buf->chan->alloc_size)  in relay_mmap_buf()
112 	vma->vm_private_data = buf;  in relay_mmap_buf()
113 	buf->chan->cb->buf_mapped(buf, filp);  in relay_mmap_buf()
126 static void *relay_alloc_buf(struct rchan_buf *buf, size_t *size)  in relay_alloc_buf()  argument
134 	buf->page_array = relay_alloc_page_array(n_pages);  in relay_alloc_buf()
135 	if (!buf->page_array)  in relay_alloc_buf()
139 		buf->page_array[i] = alloc_page(GFP_KERNEL);  in relay_alloc_buf()
140 		if (unlikely(!buf->page_array[i]))  in relay_alloc_buf()
142 		set_page_private(buf->page_array[i], (unsigned long)buf);  in relay_alloc_buf()
144 	mem = vmap(buf->page_array, n_pages, VM_MAP, PAGE_KERNEL);  in relay_alloc_buf()
149 	buf->page_count = n_pages;  in relay_alloc_buf()
154 		__free_page(buf->page_array[j]);  in relay_alloc_buf()
155 	relay_free_page_array(buf->page_array);  in relay_alloc_buf()
167 	struct rchan_buf *buf;  in relay_create_buf()  local
172 	buf = kzalloc(sizeof(struct rchan_buf), GFP_KERNEL);  in relay_create_buf()
173 	if (!buf)  in relay_create_buf()
175 	buf->padding = kmalloc(chan->n_subbufs * sizeof(size_t *), GFP_KERNEL);  in relay_create_buf()
176 	if (!buf->padding)  in relay_create_buf()
179 	buf->start = relay_alloc_buf(buf, &chan->alloc_size);  in relay_create_buf()
180 	if (!buf->start)  in relay_create_buf()
183 	buf->chan = chan;  in relay_create_buf()
184 	kref_get(&buf->chan->kref);  in relay_create_buf()
185 	return buf;  in relay_create_buf()
188 	kfree(buf->padding);  in relay_create_buf()
189 	kfree(buf);  in relay_create_buf()
209 static void relay_destroy_buf(struct rchan_buf *buf)  in relay_destroy_buf()  argument
211 	struct rchan *chan = buf->chan;  in relay_destroy_buf()
214 	if (likely(buf->start)) {  in relay_destroy_buf()
215 		vunmap(buf->start);  in relay_destroy_buf()
216 		for (i = 0; i < buf->page_count; i++)  in relay_destroy_buf()
217 			__free_page(buf->page_array[i]);  in relay_destroy_buf()
218 		relay_free_page_array(buf->page_array);  in relay_destroy_buf()
220 	chan->buf[buf->cpu] = NULL;  in relay_destroy_buf()
221 	kfree(buf->padding);  in relay_destroy_buf()
222 	kfree(buf);  in relay_destroy_buf()
236 	struct rchan_buf *buf = container_of(kref, struct rchan_buf, kref);  in relay_remove_buf()  local
237 	relay_destroy_buf(buf);  in relay_remove_buf()
246 static int relay_buf_empty(struct rchan_buf *buf)  in relay_buf_empty()  argument
248 	return (buf->subbufs_produced - buf->subbufs_consumed) ? 0 : 1;  in relay_buf_empty()
257 int relay_buf_full(struct rchan_buf *buf)  in relay_buf_full()  argument
259 	size_t ready = buf->subbufs_produced - buf->subbufs_consumed;  in relay_buf_full()
260 	return (ready >= buf->chan->n_subbufs) ? 1 : 0;  in relay_buf_full()
276 static int subbuf_start_default_callback (struct rchan_buf *buf,  in subbuf_start_default_callback()  argument
281 	if (relay_buf_full(buf))  in subbuf_start_default_callback()
290 static void buf_mapped_default_callback(struct rchan_buf *buf,  in buf_mapped_default_callback()  argument
298 static void buf_unmapped_default_callback(struct rchan_buf *buf,  in buf_unmapped_default_callback()  argument
309 						       struct rchan_buf *buf,  in create_buf_file_default_callback()  argument
340 	struct rchan_buf *buf = (struct rchan_buf *)data;  in wakeup_readers()  local
341 	wake_up_interruptible(&buf->read_wait);  in wakeup_readers()
351 static void __relay_reset(struct rchan_buf *buf, unsigned int init)  in __relay_reset()  argument
356 		init_waitqueue_head(&buf->read_wait);  in __relay_reset()
357 		kref_init(&buf->kref);  in __relay_reset()
358 		setup_timer(&buf->timer, wakeup_readers, (unsigned long)buf);  in __relay_reset()
360 		del_timer_sync(&buf->timer);  in __relay_reset()
362 	buf->subbufs_produced = 0;  in __relay_reset()
363 	buf->subbufs_consumed = 0;  in __relay_reset()
364 	buf->bytes_consumed = 0;  in __relay_reset()
365 	buf->finalized = 0;  in __relay_reset()
366 	buf->data = buf->start;  in __relay_reset()
367 	buf->offset = 0;  in __relay_reset()
369 	for (i = 0; i < buf->chan->n_subbufs; i++)  in __relay_reset()
370 		buf->padding[i] = 0;  in __relay_reset()
372 	buf->chan->cb->subbuf_start(buf, buf->data, NULL, 0);  in __relay_reset()
393 	if (chan->is_global && chan->buf[0]) {  in relay_reset()
394 		__relay_reset(chan->buf[0], 0);  in relay_reset()
400 		if (chan->buf[i])  in relay_reset()
401 			__relay_reset(chan->buf[i], 0);  in relay_reset()
406 static inline void relay_set_buf_dentry(struct rchan_buf *buf,  in relay_set_buf_dentry()  argument
409 	buf->dentry = dentry;  in relay_set_buf_dentry()
410 	d_inode(buf->dentry)->i_size = buf->early_bytes;  in relay_set_buf_dentry()
414 					    struct rchan_buf *buf,  in relay_create_buf_file()  argument
427 					   S_IRUSR, buf,  in relay_create_buf_file()
442  	struct rchan_buf *buf = NULL;  in relay_open_buf()  local
446 		return chan->buf[0];  in relay_open_buf()
448 	buf = relay_create_buf(chan);  in relay_open_buf()
449 	if (!buf)  in relay_open_buf()
453 		dentry = relay_create_buf_file(chan, buf, cpu);  in relay_open_buf()
456 		relay_set_buf_dentry(buf, dentry);  in relay_open_buf()
459  	buf->cpu = cpu;  in relay_open_buf()
460  	__relay_reset(buf, 1);  in relay_open_buf()
463  		chan->buf[0] = buf;  in relay_open_buf()
464  		buf->cpu = 0;  in relay_open_buf()
467 	return buf;  in relay_open_buf()
470  	relay_destroy_buf(buf);  in relay_open_buf()
482 static void relay_close_buf(struct rchan_buf *buf)  in relay_close_buf()  argument
484 	buf->finalized = 1;  in relay_close_buf()
485 	del_timer_sync(&buf->timer);  in relay_close_buf()
486 	buf->chan->cb->remove_buf_file(buf->dentry);  in relay_close_buf()
487 	kref_put(&buf->kref, relay_remove_buf);  in relay_close_buf()
531 			if (chan->buf[hotcpu])  in relay_hotcpu_callback()
533 			chan->buf[hotcpu] = relay_open_buf(chan, hotcpu);  in relay_hotcpu_callback()
534 			if(!chan->buf[hotcpu]) {  in relay_hotcpu_callback()
603 		chan->buf[i] = relay_open_buf(chan, i);  in relay_open()
604 		if (!chan->buf[i])  in relay_open()
614 		if (chan->buf[i])  in relay_open()
615 			relay_close_buf(chan->buf[i]);  in relay_open()
625 	struct rchan_buf *buf;  member
634 	relay_set_buf_dentry(p->buf, p->dentry);  in __relay_set_buf_dentry()
678 		if (unlikely(!chan->buf[i])) {  in relay_late_setup_files()
684 		dentry = relay_create_buf_file(chan, chan->buf[i], i);  in relay_late_setup_files()
692 			relay_set_buf_dentry(chan->buf[i], dentry);  in relay_late_setup_files()
695 			disp.buf = chan->buf[i];  in relay_late_setup_files()
722 size_t relay_switch_subbuf(struct rchan_buf *buf, size_t length)  in relay_switch_subbuf()  argument
727 	if (unlikely(length > buf->chan->subbuf_size))  in relay_switch_subbuf()
730 	if (buf->offset != buf->chan->subbuf_size + 1) {  in relay_switch_subbuf()
731 		buf->prev_padding = buf->chan->subbuf_size - buf->offset;  in relay_switch_subbuf()
732 		old_subbuf = buf->subbufs_produced % buf->chan->n_subbufs;  in relay_switch_subbuf()
733 		buf->padding[old_subbuf] = buf->prev_padding;  in relay_switch_subbuf()
734 		buf->subbufs_produced++;  in relay_switch_subbuf()
735 		if (buf->dentry)  in relay_switch_subbuf()
736 			d_inode(buf->dentry)->i_size +=  in relay_switch_subbuf()
737 				buf->chan->subbuf_size -  in relay_switch_subbuf()
738 				buf->padding[old_subbuf];  in relay_switch_subbuf()
740 			buf->early_bytes += buf->chan->subbuf_size -  in relay_switch_subbuf()
741 					    buf->padding[old_subbuf];  in relay_switch_subbuf()
743 		if (waitqueue_active(&buf->read_wait))  in relay_switch_subbuf()
750 			mod_timer(&buf->timer, jiffies + 1);  in relay_switch_subbuf()
753 	old = buf->data;  in relay_switch_subbuf()
754 	new_subbuf = buf->subbufs_produced % buf->chan->n_subbufs;  in relay_switch_subbuf()
755 	new = buf->start + new_subbuf * buf->chan->subbuf_size;  in relay_switch_subbuf()
756 	buf->offset = 0;  in relay_switch_subbuf()
757 	if (!buf->chan->cb->subbuf_start(buf, new, old, buf->prev_padding)) {  in relay_switch_subbuf()
758 		buf->offset = buf->chan->subbuf_size + 1;  in relay_switch_subbuf()
761 	buf->data = new;  in relay_switch_subbuf()
762 	buf->padding[new_subbuf] = 0;  in relay_switch_subbuf()
764 	if (unlikely(length + buf->offset > buf->chan->subbuf_size))  in relay_switch_subbuf()
770 	buf->chan->last_toobig = length;  in relay_switch_subbuf()
792 	struct rchan_buf *buf;  in relay_subbufs_consumed()  local
797 	if (cpu >= NR_CPUS || !chan->buf[cpu] ||  in relay_subbufs_consumed()
801 	buf = chan->buf[cpu];  in relay_subbufs_consumed()
802 	if (subbufs_consumed > buf->subbufs_produced - buf->subbufs_consumed)  in relay_subbufs_consumed()
803 		buf->subbufs_consumed = buf->subbufs_produced;  in relay_subbufs_consumed()
805 		buf->subbufs_consumed += subbufs_consumed;  in relay_subbufs_consumed()
823 	if (chan->is_global && chan->buf[0])  in relay_close()
824 		relay_close_buf(chan->buf[0]);  in relay_close()
827 			if (chan->buf[i])  in relay_close()
828 				relay_close_buf(chan->buf[i]);  in relay_close()
854 	if (chan->is_global && chan->buf[0]) {  in relay_flush()
855 		relay_switch_subbuf(chan->buf[0], 0);  in relay_flush()
861 		if (chan->buf[i])  in relay_flush()
862 			relay_switch_subbuf(chan->buf[i], 0);  in relay_flush()
876 	struct rchan_buf *buf = inode->i_private;  in relay_file_open()  local
877 	kref_get(&buf->kref);  in relay_file_open()
878 	filp->private_data = buf;  in relay_file_open()
892 	struct rchan_buf *buf = filp->private_data;  in relay_file_mmap()  local
893 	return relay_mmap_buf(buf, vma);  in relay_file_mmap()
906 	struct rchan_buf *buf = filp->private_data;  in relay_file_poll()  local
908 	if (buf->finalized)  in relay_file_poll()
912 		poll_wait(filp, &buf->read_wait, wait);  in relay_file_poll()
913 		if (!relay_buf_empty(buf))  in relay_file_poll()
930 	struct rchan_buf *buf = filp->private_data;  in relay_file_release()  local
931 	kref_put(&buf->kref, relay_remove_buf);  in relay_file_release()
939 static void relay_file_read_consume(struct rchan_buf *buf,  in relay_file_read_consume()  argument
943 	size_t subbuf_size = buf->chan->subbuf_size;  in relay_file_read_consume()
944 	size_t n_subbufs = buf->chan->n_subbufs;  in relay_file_read_consume()
947 	if (buf->subbufs_produced == buf->subbufs_consumed &&  in relay_file_read_consume()
948 	    buf->offset == buf->bytes_consumed)  in relay_file_read_consume()
951 	if (buf->bytes_consumed + bytes_consumed > subbuf_size) {  in relay_file_read_consume()
952 		relay_subbufs_consumed(buf->chan, buf->cpu, 1);  in relay_file_read_consume()
953 		buf->bytes_consumed = 0;  in relay_file_read_consume()
956 	buf->bytes_consumed += bytes_consumed;  in relay_file_read_consume()
958 		read_subbuf = buf->subbufs_consumed % n_subbufs;  in relay_file_read_consume()
960 		read_subbuf = read_pos / buf->chan->subbuf_size;  in relay_file_read_consume()
961 	if (buf->bytes_consumed + buf->padding[read_subbuf] == subbuf_size) {  in relay_file_read_consume()
962 		if ((read_subbuf == buf->subbufs_produced % n_subbufs) &&  in relay_file_read_consume()
963 		    (buf->offset == subbuf_size))  in relay_file_read_consume()
965 		relay_subbufs_consumed(buf->chan, buf->cpu, 1);  in relay_file_read_consume()
966 		buf->bytes_consumed = 0;  in relay_file_read_consume()
973 static int relay_file_read_avail(struct rchan_buf *buf, size_t read_pos)  in relay_file_read_avail()  argument
975 	size_t subbuf_size = buf->chan->subbuf_size;  in relay_file_read_avail()
976 	size_t n_subbufs = buf->chan->n_subbufs;  in relay_file_read_avail()
977 	size_t produced = buf->subbufs_produced;  in relay_file_read_avail()
978 	size_t consumed = buf->subbufs_consumed;  in relay_file_read_avail()
980 	relay_file_read_consume(buf, read_pos, 0);  in relay_file_read_avail()
982 	consumed = buf->subbufs_consumed;  in relay_file_read_avail()
984 	if (unlikely(buf->offset > subbuf_size)) {  in relay_file_read_avail()
992 		buf->subbufs_consumed = consumed;  in relay_file_read_avail()
993 		buf->bytes_consumed = 0;  in relay_file_read_avail()
996 	produced = (produced % n_subbufs) * subbuf_size + buf->offset;  in relay_file_read_avail()
997 	consumed = (consumed % n_subbufs) * subbuf_size + buf->bytes_consumed;  in relay_file_read_avail()
1003 		if (buf->offset == subbuf_size &&  in relay_file_read_avail()
1004 		    buf->subbufs_produced > buf->subbufs_consumed)  in relay_file_read_avail()
1018 					   struct rchan_buf *buf)  in relay_file_read_subbuf_avail()  argument
1022 	size_t subbuf_size = buf->chan->subbuf_size;  in relay_file_read_subbuf_avail()
1024 	write_subbuf = (buf->data - buf->start) / subbuf_size;  in relay_file_read_subbuf_avail()
1025 	write_offset = buf->offset > subbuf_size ? subbuf_size : buf->offset;  in relay_file_read_subbuf_avail()
1028 	padding = buf->padding[read_subbuf];  in relay_file_read_subbuf_avail()
1049 					struct rchan_buf *buf)  in relay_file_read_start_pos()  argument
1052 	size_t subbuf_size = buf->chan->subbuf_size;  in relay_file_read_start_pos()
1053 	size_t n_subbufs = buf->chan->n_subbufs;  in relay_file_read_start_pos()
1054 	size_t consumed = buf->subbufs_consumed % n_subbufs;  in relay_file_read_start_pos()
1057 		read_pos = consumed * subbuf_size + buf->bytes_consumed;  in relay_file_read_start_pos()
1059 	padding = buf->padding[read_subbuf];  in relay_file_read_start_pos()
1076 static size_t relay_file_read_end_pos(struct rchan_buf *buf,  in relay_file_read_end_pos()  argument
1081 	size_t subbuf_size = buf->chan->subbuf_size;  in relay_file_read_end_pos()
1082 	size_t n_subbufs = buf->chan->n_subbufs;  in relay_file_read_end_pos()
1085 	padding = buf->padding[read_subbuf];  in relay_file_read_end_pos()
1100 			     struct rchan_buf *buf,  in subbuf_read_actor()  argument
1107 	from = buf->start + read_start;  in subbuf_read_actor()
1109 	if (copy_to_user(desc->arg.buf, from, avail)) {  in subbuf_read_actor()
1121 			       struct rchan_buf *buf,
1132 	struct rchan_buf *buf = filp->private_data;  in relay_file_read_subbufs()  local
1141 		if (!relay_file_read_avail(buf, *ppos))  in relay_file_read_subbufs()
1144 		read_start = relay_file_read_start_pos(*ppos, buf);  in relay_file_read_subbufs()
1145 		avail = relay_file_read_subbuf_avail(read_start, buf);  in relay_file_read_subbufs()
1150 		ret = subbuf_actor(read_start, buf, avail, desc);  in relay_file_read_subbufs()
1155 			relay_file_read_consume(buf, read_start, ret);  in relay_file_read_subbufs()
1156 			*ppos = relay_file_read_end_pos(buf, read_start, ret);  in relay_file_read_subbufs()
1172 	desc.arg.buf = buffer;  in relay_file_read()
1188 				   struct pipe_buffer *buf)  in relay_pipe_buf_release()  argument
1192 	rbuf = (struct rchan_buf *)page_private(buf->page);  in relay_pipe_buf_release()
1193 	relay_consume_bytes(rbuf, buf->private);  in relay_pipe_buf_release()