36 	buf->chan->cb->buf_unmapped(buf, vma->vm_file);  in relay_file_mmap_close()
104 	if (length != (unsigned long)buf->chan->alloc_size)  in relay_mmap_buf()
110 	buf->chan->cb->buf_mapped(buf, filp);  in relay_mmap_buf()
162 static struct rchan_buf *relay_create_buf(struct rchan *chan)  in relay_create_buf()  argument
166 	if (chan->n_subbufs > UINT_MAX / sizeof(size_t *))  in relay_create_buf()
172 	buf->padding = kmalloc(chan->n_subbufs * sizeof(size_t *), GFP_KERNEL);  in relay_create_buf()
176 	buf->start = relay_alloc_buf(buf, &chan->alloc_size);  in relay_create_buf()
180 	buf->chan = chan;  in relay_create_buf()
181 	kref_get(&buf->chan->kref);  in relay_create_buf()
198 	struct rchan *chan = container_of(kref, struct rchan, kref);  in relay_destroy_channel()  local
199 	kfree(chan);  in relay_destroy_channel()
208 	struct rchan *chan = buf->chan;  in relay_destroy_buf()  local
217 	chan->buf[buf->cpu] = NULL;  in relay_destroy_buf()
220 	kref_put(&chan->kref, relay_destroy_channel);  in relay_destroy_buf()
257 	return (ready >= buf->chan->n_subbufs) ? 1 : 0;  in relay_buf_full()
366 	for (i = 0; i < buf->chan->n_subbufs; i++)  in __relay_reset()
369 	buf->chan->cb->subbuf_start(buf, buf->data, NULL, 0);  in __relay_reset()
383 void relay_reset(struct rchan *chan)  in relay_reset()  argument
387 	if (!chan)  in relay_reset()
390 	if (chan->is_global && chan->buf[0]) {  in relay_reset()
391 		__relay_reset(chan->buf[0], 0);  in relay_reset()
397 		if (chan->buf[i])  in relay_reset()
398 			__relay_reset(chan->buf[i], 0);  in relay_reset()
410 static struct dentry *relay_create_buf_file(struct rchan *chan,  in relay_create_buf_file()  argument
420 	snprintf(tmpname, NAME_MAX, "%s%d", chan->base_filename, cpu);  in relay_create_buf_file()
423 	dentry = chan->cb->create_buf_file(tmpname, chan->parent,  in relay_create_buf_file()
425 					   &chan->is_global);  in relay_create_buf_file()
437 static struct rchan_buf *relay_open_buf(struct rchan *chan, unsigned int cpu)  in relay_open_buf()  argument
442  	if (chan->is_global)  in relay_open_buf()
443 		return chan->buf[0];  in relay_open_buf()
445 	buf = relay_create_buf(chan);  in relay_open_buf()
449 	if (chan->has_base_filename) {  in relay_open_buf()
450 		dentry = relay_create_buf_file(chan, buf, cpu);  in relay_open_buf()
459  	if(chan->is_global) {  in relay_open_buf()
460  		chan->buf[0] = buf;  in relay_open_buf()
483 	buf->chan->cb->remove_buf_file(buf->dentry);  in relay_close_buf()
487 static void setup_callbacks(struct rchan *chan,  in setup_callbacks()  argument
491 		chan->cb = &default_channel_callbacks;  in setup_callbacks()
505 	chan->cb = cb;  in setup_callbacks()
521 	struct rchan *chan;  in relay_hotcpu_callback()  local
527 		list_for_each_entry(chan, &relay_channels, list) {  in relay_hotcpu_callback()
528 			if (chan->buf[hotcpu])  in relay_hotcpu_callback()
530 			chan->buf[hotcpu] = relay_open_buf(chan, hotcpu);  in relay_hotcpu_callback()
531 			if(!chan->buf[hotcpu]) {  in relay_hotcpu_callback()
574 	struct rchan *chan;  in relay_open()  local
581 	chan = kzalloc(sizeof(struct rchan), GFP_KERNEL);  in relay_open()
582 	if (!chan)  in relay_open()
585 	chan->version = RELAYFS_CHANNEL_VERSION;  in relay_open()
586 	chan->n_subbufs = n_subbufs;  in relay_open()
587 	chan->subbuf_size = subbuf_size;  in relay_open()
588 	chan->alloc_size = PAGE_ALIGN(subbuf_size * n_subbufs);  in relay_open()
589 	chan->parent = parent;  in relay_open()
590 	chan->private_data = private_data;  in relay_open()
592 		chan->has_base_filename = 1;  in relay_open()
593 		strlcpy(chan->base_filename, base_filename, NAME_MAX);  in relay_open()
595 	setup_callbacks(chan, cb);  in relay_open()
596 	kref_init(&chan->kref);  in relay_open()
600 		chan->buf[i] = relay_open_buf(chan, i);  in relay_open()
601 		if (!chan->buf[i])  in relay_open()
604 	list_add(&chan->list, &relay_channels);  in relay_open()
607 	return chan;  in relay_open()
611 		if (chan->buf[i])  in relay_open()
612 			relay_close_buf(chan->buf[i]);  in relay_open()
615 	kref_put(&chan->kref, relay_destroy_channel);  in relay_open()
645 int relay_late_setup_files(struct rchan *chan,  in relay_late_setup_files()  argument
655 	if (!chan || !base_filename)  in relay_late_setup_files()
658 	strlcpy(chan->base_filename, base_filename, NAME_MAX);  in relay_late_setup_files()
662 	if (unlikely(chan->has_base_filename)) {  in relay_late_setup_files()
666 	chan->has_base_filename = 1;  in relay_late_setup_files()
667 	chan->parent = parent;  in relay_late_setup_files()
675 		if (unlikely(!chan->buf[i])) {  in relay_late_setup_files()
681 		dentry = relay_create_buf_file(chan, chan->buf[i], i);  in relay_late_setup_files()
689 			relay_set_buf_dentry(chan->buf[i], dentry);  in relay_late_setup_files()
692 			disp.buf = chan->buf[i];  in relay_late_setup_files()
724 	if (unlikely(length > buf->chan->subbuf_size))  in relay_switch_subbuf()
727 	if (buf->offset != buf->chan->subbuf_size + 1) {  in relay_switch_subbuf()
728 		buf->prev_padding = buf->chan->subbuf_size - buf->offset;  in relay_switch_subbuf()
729 		old_subbuf = buf->subbufs_produced % buf->chan->n_subbufs;  in relay_switch_subbuf()
734 				buf->chan->subbuf_size -  in relay_switch_subbuf()
737 			buf->early_bytes += buf->chan->subbuf_size -  in relay_switch_subbuf()
751 	new_subbuf = buf->subbufs_produced % buf->chan->n_subbufs;  in relay_switch_subbuf()
752 	new = buf->start + new_subbuf * buf->chan->subbuf_size;  in relay_switch_subbuf()
754 	if (!buf->chan->cb->subbuf_start(buf, new, old, buf->prev_padding)) {  in relay_switch_subbuf()
755 		buf->offset = buf->chan->subbuf_size + 1;  in relay_switch_subbuf()
761 	if (unlikely(length + buf->offset > buf->chan->subbuf_size))  in relay_switch_subbuf()
767 	buf->chan->last_toobig = length;  in relay_switch_subbuf()
785 void relay_subbufs_consumed(struct rchan *chan,  in relay_subbufs_consumed()  argument
791 	if (!chan)  in relay_subbufs_consumed()
794 	if (cpu >= NR_CPUS || !chan->buf[cpu] ||  in relay_subbufs_consumed()
795 					subbufs_consumed > chan->n_subbufs)  in relay_subbufs_consumed()
798 	buf = chan->buf[cpu];  in relay_subbufs_consumed()
812 void relay_close(struct rchan *chan)  in relay_close()  argument
816 	if (!chan)  in relay_close()
820 	if (chan->is_global && chan->buf[0])  in relay_close()
821 		relay_close_buf(chan->buf[0]);  in relay_close()
824 			if (chan->buf[i])  in relay_close()
825 				relay_close_buf(chan->buf[i]);  in relay_close()
827 	if (chan->last_toobig)  in relay_close()
830 		       chan->last_toobig, chan->subbuf_size);  in relay_close()
832 	list_del(&chan->list);  in relay_close()
833 	kref_put(&chan->kref, relay_destroy_channel);  in relay_close()
844 void relay_flush(struct rchan *chan)  in relay_flush()  argument
848 	if (!chan)  in relay_flush()
851 	if (chan->is_global && chan->buf[0]) {  in relay_flush()
852 		relay_switch_subbuf(chan->buf[0], 0);  in relay_flush()
858 		if (chan->buf[i])  in relay_flush()
859 			relay_switch_subbuf(chan->buf[i], 0);  in relay_flush()
940 	size_t subbuf_size = buf->chan->subbuf_size;  in relay_file_read_consume()
941 	size_t n_subbufs = buf->chan->n_subbufs;  in relay_file_read_consume()
949 		relay_subbufs_consumed(buf->chan, buf->cpu, 1);  in relay_file_read_consume()
957 		read_subbuf = read_pos / buf->chan->subbuf_size;  in relay_file_read_consume()
962 		relay_subbufs_consumed(buf->chan, buf->cpu, 1);  in relay_file_read_consume()
972 	size_t subbuf_size = buf->chan->subbuf_size;  in relay_file_read_avail()
973 	size_t n_subbufs = buf->chan->n_subbufs;  in relay_file_read_avail()
1019 	size_t subbuf_size = buf->chan->subbuf_size;  in relay_file_read_subbuf_avail()
1049 	size_t subbuf_size = buf->chan->subbuf_size;  in relay_file_read_start_pos()
1050 	size_t n_subbufs = buf->chan->n_subbufs;  in relay_file_read_start_pos()
1078 	size_t subbuf_size = buf->chan->subbuf_size;  in relay_file_read_end_pos()
1079 	size_t n_subbufs = buf->chan->n_subbufs;  in relay_file_read_end_pos()
1178 	if (rbuf->bytes_consumed >= rbuf->chan->subbuf_size) {  in relay_consume_bytes()
1179 		relay_subbufs_consumed(rbuf->chan, rbuf->cpu, 1);  in relay_consume_bytes()
1180 		rbuf->bytes_consumed %= rbuf->chan->subbuf_size;  in relay_consume_bytes()
1217 	unsigned int subbuf_size = rbuf->chan->subbuf_size;  in subbuf_splice_actor()
1219 	uint32_t alloc_size = (uint32_t) rbuf->chan->alloc_size;  in subbuf_splice_actor()
1248 	subbuf_pages = rbuf->chan->alloc_size >> PAGE_SHIFT;  in subbuf_splice_actor()