35 	struct rchan_buf *buf = vma->vm_private_data;  in relay_file_mmap_close()  local
36 	buf->chan->cb->buf_unmapped(buf, vma->vm_file);  in relay_file_mmap_close()
45 	struct rchan_buf *buf = vma->vm_private_data;  in relay_buf_fault()  local
48 	if (!buf)  in relay_buf_fault()
51 	page = vmalloc_to_page(buf->start + (pgoff << PAGE_SHIFT));  in relay_buf_fault()
96 static int relay_mmap_buf(struct rchan_buf *buf, struct vm_area_struct *vma)  in relay_mmap_buf()  argument
101 	if (!buf)  in relay_mmap_buf()
104 	if (length != (unsigned long)buf->chan->alloc_size)  in relay_mmap_buf()
109 	vma->vm_private_data = buf;  in relay_mmap_buf()
110 	buf->chan->cb->buf_mapped(buf, filp);  in relay_mmap_buf()
123 static void *relay_alloc_buf(struct rchan_buf *buf, size_t *size)  in relay_alloc_buf()  argument
131 	buf->page_array = relay_alloc_page_array(n_pages);  in relay_alloc_buf()
132 	if (!buf->page_array)  in relay_alloc_buf()
136 		buf->page_array[i] = alloc_page(GFP_KERNEL);  in relay_alloc_buf()
137 		if (unlikely(!buf->page_array[i]))  in relay_alloc_buf()
139 		set_page_private(buf->page_array[i], (unsigned long)buf);  in relay_alloc_buf()
141 	mem = vmap(buf->page_array, n_pages, VM_MAP, PAGE_KERNEL);  in relay_alloc_buf()
146 	buf->page_count = n_pages;  in relay_alloc_buf()
151 		__free_page(buf->page_array[j]);  in relay_alloc_buf()
152 	relay_free_page_array(buf->page_array);  in relay_alloc_buf()
164 	struct rchan_buf *buf;  in relay_create_buf()  local
169 	buf = kzalloc(sizeof(struct rchan_buf), GFP_KERNEL);  in relay_create_buf()
170 	if (!buf)  in relay_create_buf()
172 	buf->padding = kmalloc(chan->n_subbufs * sizeof(size_t *), GFP_KERNEL);  in relay_create_buf()
173 	if (!buf->padding)  in relay_create_buf()
176 	buf->start = relay_alloc_buf(buf, &chan->alloc_size);  in relay_create_buf()
177 	if (!buf->start)  in relay_create_buf()
180 	buf->chan = chan;  in relay_create_buf()
181 	kref_get(&buf->chan->kref);  in relay_create_buf()
182 	return buf;  in relay_create_buf()
185 	kfree(buf->padding);  in relay_create_buf()
186 	kfree(buf);  in relay_create_buf()
206 static void relay_destroy_buf(struct rchan_buf *buf)  in relay_destroy_buf()  argument
208 	struct rchan *chan = buf->chan;  in relay_destroy_buf()
211 	if (likely(buf->start)) {  in relay_destroy_buf()
212 		vunmap(buf->start);  in relay_destroy_buf()
213 		for (i = 0; i < buf->page_count; i++)  in relay_destroy_buf()
214 			__free_page(buf->page_array[i]);  in relay_destroy_buf()
215 		relay_free_page_array(buf->page_array);  in relay_destroy_buf()
217 	chan->buf[buf->cpu] = NULL;  in relay_destroy_buf()
218 	kfree(buf->padding);  in relay_destroy_buf()
219 	kfree(buf);  in relay_destroy_buf()
233 	struct rchan_buf *buf = container_of(kref, struct rchan_buf, kref);  in relay_remove_buf()  local
234 	relay_destroy_buf(buf);  in relay_remove_buf()
243 static int relay_buf_empty(struct rchan_buf *buf)  in relay_buf_empty()  argument
245 	return (buf->subbufs_produced - buf->subbufs_consumed) ? 0 : 1;  in relay_buf_empty()
254 int relay_buf_full(struct rchan_buf *buf)  in relay_buf_full()  argument
256 	size_t ready = buf->subbufs_produced - buf->subbufs_consumed;  in relay_buf_full()
257 	return (ready >= buf->chan->n_subbufs) ? 1 : 0;  in relay_buf_full()
273 static int subbuf_start_default_callback (struct rchan_buf *buf,  in subbuf_start_default_callback()  argument
278 	if (relay_buf_full(buf))  in subbuf_start_default_callback()
287 static void buf_mapped_default_callback(struct rchan_buf *buf,  in buf_mapped_default_callback()  argument
295 static void buf_unmapped_default_callback(struct rchan_buf *buf,  in buf_unmapped_default_callback()  argument
306 						       struct rchan_buf *buf,  in create_buf_file_default_callback()  argument
337 	struct rchan_buf *buf = (struct rchan_buf *)data;  in wakeup_readers()  local
338 	wake_up_interruptible(&buf->read_wait);  in wakeup_readers()
348 static void __relay_reset(struct rchan_buf *buf, unsigned int init)  in __relay_reset()  argument
353 		init_waitqueue_head(&buf->read_wait);  in __relay_reset()
354 		kref_init(&buf->kref);  in __relay_reset()
355 		setup_timer(&buf->timer, wakeup_readers, (unsigned long)buf);  in __relay_reset()
357 		del_timer_sync(&buf->timer);  in __relay_reset()
359 	buf->subbufs_produced = 0;  in __relay_reset()
360 	buf->subbufs_consumed = 0;  in __relay_reset()
361 	buf->bytes_consumed = 0;  in __relay_reset()
362 	buf->finalized = 0;  in __relay_reset()
363 	buf->data = buf->start;  in __relay_reset()
364 	buf->offset = 0;  in __relay_reset()
366 	for (i = 0; i < buf->chan->n_subbufs; i++)  in __relay_reset()
367 		buf->padding[i] = 0;  in __relay_reset()
369 	buf->chan->cb->subbuf_start(buf, buf->data, NULL, 0);  in __relay_reset()
390 	if (chan->is_global && chan->buf[0]) {  in relay_reset()
391 		__relay_reset(chan->buf[0], 0);  in relay_reset()
397 		if (chan->buf[i])  in relay_reset()
398 			__relay_reset(chan->buf[i], 0);  in relay_reset()
403 static inline void relay_set_buf_dentry(struct rchan_buf *buf,  in relay_set_buf_dentry()  argument
406 	buf->dentry = dentry;  in relay_set_buf_dentry()
407 	d_inode(buf->dentry)->i_size = buf->early_bytes;  in relay_set_buf_dentry()
411 					    struct rchan_buf *buf,  in relay_create_buf_file()  argument
424 					   S_IRUSR, buf,  in relay_create_buf_file()
439  	struct rchan_buf *buf = NULL;  in relay_open_buf()  local
443 		return chan->buf[0];  in relay_open_buf()
445 	buf = relay_create_buf(chan);  in relay_open_buf()
446 	if (!buf)  in relay_open_buf()
450 		dentry = relay_create_buf_file(chan, buf, cpu);  in relay_open_buf()
453 		relay_set_buf_dentry(buf, dentry);  in relay_open_buf()
456  	buf->cpu = cpu;  in relay_open_buf()
457  	__relay_reset(buf, 1);  in relay_open_buf()
460  		chan->buf[0] = buf;  in relay_open_buf()
461  		buf->cpu = 0;  in relay_open_buf()
464 	return buf;  in relay_open_buf()
467  	relay_destroy_buf(buf);  in relay_open_buf()
479 static void relay_close_buf(struct rchan_buf *buf)  in relay_close_buf()  argument
481 	buf->finalized = 1;  in relay_close_buf()
482 	del_timer_sync(&buf->timer);  in relay_close_buf()
483 	buf->chan->cb->remove_buf_file(buf->dentry);  in relay_close_buf()
484 	kref_put(&buf->kref, relay_remove_buf);  in relay_close_buf()
528 			if (chan->buf[hotcpu])  in relay_hotcpu_callback()
530 			chan->buf[hotcpu] = relay_open_buf(chan, hotcpu);  in relay_hotcpu_callback()
531 			if(!chan->buf[hotcpu]) {  in relay_hotcpu_callback()
600 		chan->buf[i] = relay_open_buf(chan, i);  in relay_open()
601 		if (!chan->buf[i])  in relay_open()
611 		if (chan->buf[i])  in relay_open()
612 			relay_close_buf(chan->buf[i]);  in relay_open()
622 	struct rchan_buf *buf;  member
631 	relay_set_buf_dentry(p->buf, p->dentry);  in __relay_set_buf_dentry()
675 		if (unlikely(!chan->buf[i])) {  in relay_late_setup_files()
681 		dentry = relay_create_buf_file(chan, chan->buf[i], i);  in relay_late_setup_files()
689 			relay_set_buf_dentry(chan->buf[i], dentry);  in relay_late_setup_files()
692 			disp.buf = chan->buf[i];  in relay_late_setup_files()
719 size_t relay_switch_subbuf(struct rchan_buf *buf, size_t length)  in relay_switch_subbuf()  argument
724 	if (unlikely(length > buf->chan->subbuf_size))  in relay_switch_subbuf()
727 	if (buf->offset != buf->chan->subbuf_size + 1) {  in relay_switch_subbuf()
728 		buf->prev_padding = buf->chan->subbuf_size - buf->offset;  in relay_switch_subbuf()
729 		old_subbuf = buf->subbufs_produced % buf->chan->n_subbufs;  in relay_switch_subbuf()
730 		buf->padding[old_subbuf] = buf->prev_padding;  in relay_switch_subbuf()
731 		buf->subbufs_produced++;  in relay_switch_subbuf()
732 		if (buf->dentry)  in relay_switch_subbuf()
733 			d_inode(buf->dentry)->i_size +=  in relay_switch_subbuf()
734 				buf->chan->subbuf_size -  in relay_switch_subbuf()
735 				buf->padding[old_subbuf];  in relay_switch_subbuf()
737 			buf->early_bytes += buf->chan->subbuf_size -  in relay_switch_subbuf()
738 					    buf->padding[old_subbuf];  in relay_switch_subbuf()
740 		if (waitqueue_active(&buf->read_wait))  in relay_switch_subbuf()
747 			mod_timer(&buf->timer, jiffies + 1);  in relay_switch_subbuf()
750 	old = buf->data;  in relay_switch_subbuf()
751 	new_subbuf = buf->subbufs_produced % buf->chan->n_subbufs;  in relay_switch_subbuf()
752 	new = buf->start + new_subbuf * buf->chan->subbuf_size;  in relay_switch_subbuf()
753 	buf->offset = 0;  in relay_switch_subbuf()
754 	if (!buf->chan->cb->subbuf_start(buf, new, old, buf->prev_padding)) {  in relay_switch_subbuf()
755 		buf->offset = buf->chan->subbuf_size + 1;  in relay_switch_subbuf()
758 	buf->data = new;  in relay_switch_subbuf()
759 	buf->padding[new_subbuf] = 0;  in relay_switch_subbuf()
761 	if (unlikely(length + buf->offset > buf->chan->subbuf_size))  in relay_switch_subbuf()
767 	buf->chan->last_toobig = length;  in relay_switch_subbuf()
789 	struct rchan_buf *buf;  in relay_subbufs_consumed()  local
794 	if (cpu >= NR_CPUS || !chan->buf[cpu] ||  in relay_subbufs_consumed()
798 	buf = chan->buf[cpu];  in relay_subbufs_consumed()
799 	if (subbufs_consumed > buf->subbufs_produced - buf->subbufs_consumed)  in relay_subbufs_consumed()
800 		buf->subbufs_consumed = buf->subbufs_produced;  in relay_subbufs_consumed()
802 		buf->subbufs_consumed += subbufs_consumed;  in relay_subbufs_consumed()
820 	if (chan->is_global && chan->buf[0])  in relay_close()
821 		relay_close_buf(chan->buf[0]);  in relay_close()
824 			if (chan->buf[i])  in relay_close()
825 				relay_close_buf(chan->buf[i]);  in relay_close()
851 	if (chan->is_global && chan->buf[0]) {  in relay_flush()
852 		relay_switch_subbuf(chan->buf[0], 0);  in relay_flush()
858 		if (chan->buf[i])  in relay_flush()
859 			relay_switch_subbuf(chan->buf[i], 0);  in relay_flush()
873 	struct rchan_buf *buf = inode->i_private;  in relay_file_open()  local
874 	kref_get(&buf->kref);  in relay_file_open()
875 	filp->private_data = buf;  in relay_file_open()
889 	struct rchan_buf *buf = filp->private_data;  in relay_file_mmap()  local
890 	return relay_mmap_buf(buf, vma);  in relay_file_mmap()
903 	struct rchan_buf *buf = filp->private_data;  in relay_file_poll()  local
905 	if (buf->finalized)  in relay_file_poll()
909 		poll_wait(filp, &buf->read_wait, wait);  in relay_file_poll()
910 		if (!relay_buf_empty(buf))  in relay_file_poll()
927 	struct rchan_buf *buf = filp->private_data;  in relay_file_release()  local
928 	kref_put(&buf->kref, relay_remove_buf);  in relay_file_release()
936 static void relay_file_read_consume(struct rchan_buf *buf,  in relay_file_read_consume()  argument
940 	size_t subbuf_size = buf->chan->subbuf_size;  in relay_file_read_consume()
941 	size_t n_subbufs = buf->chan->n_subbufs;  in relay_file_read_consume()
944 	if (buf->subbufs_produced == buf->subbufs_consumed &&  in relay_file_read_consume()
945 	    buf->offset == buf->bytes_consumed)  in relay_file_read_consume()
948 	if (buf->bytes_consumed + bytes_consumed > subbuf_size) {  in relay_file_read_consume()
949 		relay_subbufs_consumed(buf->chan, buf->cpu, 1);  in relay_file_read_consume()
950 		buf->bytes_consumed = 0;  in relay_file_read_consume()
953 	buf->bytes_consumed += bytes_consumed;  in relay_file_read_consume()
955 		read_subbuf = buf->subbufs_consumed % n_subbufs;  in relay_file_read_consume()
957 		read_subbuf = read_pos / buf->chan->subbuf_size;  in relay_file_read_consume()
958 	if (buf->bytes_consumed + buf->padding[read_subbuf] == subbuf_size) {  in relay_file_read_consume()
959 		if ((read_subbuf == buf->subbufs_produced % n_subbufs) &&  in relay_file_read_consume()
960 		    (buf->offset == subbuf_size))  in relay_file_read_consume()
962 		relay_subbufs_consumed(buf->chan, buf->cpu, 1);  in relay_file_read_consume()
963 		buf->bytes_consumed = 0;  in relay_file_read_consume()
970 static int relay_file_read_avail(struct rchan_buf *buf, size_t read_pos)  in relay_file_read_avail()  argument
972 	size_t subbuf_size = buf->chan->subbuf_size;  in relay_file_read_avail()
973 	size_t n_subbufs = buf->chan->n_subbufs;  in relay_file_read_avail()
974 	size_t produced = buf->subbufs_produced;  in relay_file_read_avail()
975 	size_t consumed = buf->subbufs_consumed;  in relay_file_read_avail()
977 	relay_file_read_consume(buf, read_pos, 0);  in relay_file_read_avail()
979 	consumed = buf->subbufs_consumed;  in relay_file_read_avail()
981 	if (unlikely(buf->offset > subbuf_size)) {  in relay_file_read_avail()
989 		buf->subbufs_consumed = consumed;  in relay_file_read_avail()
990 		buf->bytes_consumed = 0;  in relay_file_read_avail()
993 	produced = (produced % n_subbufs) * subbuf_size + buf->offset;  in relay_file_read_avail()
994 	consumed = (consumed % n_subbufs) * subbuf_size + buf->bytes_consumed;  in relay_file_read_avail()
1000 		if (buf->offset == subbuf_size &&  in relay_file_read_avail()
1001 		    buf->subbufs_produced > buf->subbufs_consumed)  in relay_file_read_avail()
1015 					   struct rchan_buf *buf)  in relay_file_read_subbuf_avail()  argument
1019 	size_t subbuf_size = buf->chan->subbuf_size;  in relay_file_read_subbuf_avail()
1021 	write_subbuf = (buf->data - buf->start) / subbuf_size;  in relay_file_read_subbuf_avail()
1022 	write_offset = buf->offset > subbuf_size ? subbuf_size : buf->offset;  in relay_file_read_subbuf_avail()
1025 	padding = buf->padding[read_subbuf];  in relay_file_read_subbuf_avail()
1046 					struct rchan_buf *buf)  in relay_file_read_start_pos()  argument
1049 	size_t subbuf_size = buf->chan->subbuf_size;  in relay_file_read_start_pos()
1050 	size_t n_subbufs = buf->chan->n_subbufs;  in relay_file_read_start_pos()
1051 	size_t consumed = buf->subbufs_consumed % n_subbufs;  in relay_file_read_start_pos()
1054 		read_pos = consumed * subbuf_size + buf->bytes_consumed;  in relay_file_read_start_pos()
1056 	padding = buf->padding[read_subbuf];  in relay_file_read_start_pos()
1073 static size_t relay_file_read_end_pos(struct rchan_buf *buf,  in relay_file_read_end_pos()  argument
1078 	size_t subbuf_size = buf->chan->subbuf_size;  in relay_file_read_end_pos()
1079 	size_t n_subbufs = buf->chan->n_subbufs;  in relay_file_read_end_pos()
1082 	padding = buf->padding[read_subbuf];  in relay_file_read_end_pos()
1097 			     struct rchan_buf *buf,  in subbuf_read_actor()  argument
1104 	from = buf->start + read_start;  in subbuf_read_actor()
1106 	if (copy_to_user(desc->arg.buf, from, avail)) {  in subbuf_read_actor()
1118 			       struct rchan_buf *buf,
1129 	struct rchan_buf *buf = filp->private_data;  in relay_file_read_subbufs()  local
1138 		if (!relay_file_read_avail(buf, *ppos))  in relay_file_read_subbufs()
1141 		read_start = relay_file_read_start_pos(*ppos, buf);  in relay_file_read_subbufs()
1142 		avail = relay_file_read_subbuf_avail(read_start, buf);  in relay_file_read_subbufs()
1147 		ret = subbuf_actor(read_start, buf, avail, desc);  in relay_file_read_subbufs()
1152 			relay_file_read_consume(buf, read_start, ret);  in relay_file_read_subbufs()
1153 			*ppos = relay_file_read_end_pos(buf, read_start, ret);  in relay_file_read_subbufs()
1169 	desc.arg.buf = buffer;  in relay_file_read()
1185 				   struct pipe_buffer *buf)  in relay_pipe_buf_release()  argument
1189 	rbuf = (struct rchan_buf *)page_private(buf->page);  in relay_pipe_buf_release()
1190 	relay_consume_bytes(rbuf, buf->private);  in relay_pipe_buf_release()