323 #define for_each_buffer_cpu(buffer, cpu)		\  argument
324 	for_each_cpu(cpu, buffer->cpumask)
460 	struct ring_buffer		*buffer;  member
550 int ring_buffer_wait(struct ring_buffer *buffer, int cpu, bool full)  in ring_buffer_wait()  argument
563 		work = &buffer->irq_work;  in ring_buffer_wait()
567 		if (!cpumask_test_cpu(cpu, buffer->cpumask))  in ring_buffer_wait()
569 		cpu_buffer = buffer->buffers[cpu];  in ring_buffer_wait()
610 		if (cpu == RING_BUFFER_ALL_CPUS && !ring_buffer_empty(buffer))  in ring_buffer_wait()
614 		    !ring_buffer_empty_cpu(buffer, cpu)) {  in ring_buffer_wait()
654 int ring_buffer_poll_wait(struct ring_buffer *buffer, int cpu,  in ring_buffer_poll_wait()  argument
661 		work = &buffer->irq_work;  in ring_buffer_poll_wait()
663 		if (!cpumask_test_cpu(cpu, buffer->cpumask))  in ring_buffer_poll_wait()
666 		cpu_buffer = buffer->buffers[cpu];  in ring_buffer_poll_wait()
687 	if ((cpu == RING_BUFFER_ALL_CPUS && !ring_buffer_empty(buffer)) ||  in ring_buffer_poll_wait()
688 	    (cpu != RING_BUFFER_ALL_CPUS && !ring_buffer_empty_cpu(buffer, cpu)))  in ring_buffer_poll_wait()
701 				atomic_inc(&__b->buffer->record_disabled); \
712 static inline u64 rb_time_stamp(struct ring_buffer *buffer)  in rb_time_stamp()  argument
715 	return buffer->clock() << DEBUG_SHIFT;  in rb_time_stamp()
718 u64 ring_buffer_time_stamp(struct ring_buffer *buffer, int cpu)  in ring_buffer_time_stamp()  argument
723 	time = rb_time_stamp(buffer);  in ring_buffer_time_stamp()
730 void ring_buffer_normalize_time_stamp(struct ring_buffer *buffer,  in ring_buffer_normalize_time_stamp()  argument
1229 rb_allocate_cpu_buffer(struct ring_buffer *buffer, long nr_pages, int cpu)  in rb_allocate_cpu_buffer()  argument
1242 	cpu_buffer->buffer = buffer;  in rb_allocate_cpu_buffer()
1244 	lockdep_set_class(&cpu_buffer->reader_lock, buffer->reader_lock_key);  in rb_allocate_cpu_buffer()
1328 	struct ring_buffer *buffer;  in __ring_buffer_alloc()  local
1334 	buffer = kzalloc(ALIGN(sizeof(*buffer), cache_line_size()),  in __ring_buffer_alloc()
1336 	if (!buffer)  in __ring_buffer_alloc()
1339 	if (!alloc_cpumask_var(&buffer->cpumask, GFP_KERNEL))  in __ring_buffer_alloc()
1343 	buffer->flags = flags;  in __ring_buffer_alloc()
1344 	buffer->clock = trace_clock_local;  in __ring_buffer_alloc()
1345 	buffer->reader_lock_key = key;  in __ring_buffer_alloc()
1347 	init_irq_work(&buffer->irq_work.work, rb_wake_up_waiters);  in __ring_buffer_alloc()
1348 	init_waitqueue_head(&buffer->irq_work.waiters);  in __ring_buffer_alloc()
1361 	cpumask_copy(buffer->cpumask, cpu_online_mask);  in __ring_buffer_alloc()
1363 	cpumask_copy(buffer->cpumask, cpu_possible_mask);  in __ring_buffer_alloc()
1365 	buffer->cpus = nr_cpu_ids;  in __ring_buffer_alloc()
1368 	buffer->buffers = kzalloc(ALIGN(bsize, cache_line_size()),  in __ring_buffer_alloc()
1370 	if (!buffer->buffers)  in __ring_buffer_alloc()
1373 	for_each_buffer_cpu(buffer, cpu) {  in __ring_buffer_alloc()
1374 		buffer->buffers[cpu] =  in __ring_buffer_alloc()
1375 			rb_allocate_cpu_buffer(buffer, nr_pages, cpu);  in __ring_buffer_alloc()
1376 		if (!buffer->buffers[cpu])  in __ring_buffer_alloc()
1381 	buffer->cpu_notify.notifier_call = rb_cpu_notify;  in __ring_buffer_alloc()
1382 	buffer->cpu_notify.priority = 0;  in __ring_buffer_alloc()
1383 	__register_cpu_notifier(&buffer->cpu_notify);  in __ring_buffer_alloc()
1387 	mutex_init(&buffer->mutex);  in __ring_buffer_alloc()
1389 	return buffer;  in __ring_buffer_alloc()
1392 	for_each_buffer_cpu(buffer, cpu) {  in __ring_buffer_alloc()
1393 		if (buffer->buffers[cpu])  in __ring_buffer_alloc()
1394 			rb_free_cpu_buffer(buffer->buffers[cpu]);  in __ring_buffer_alloc()
1396 	kfree(buffer->buffers);  in __ring_buffer_alloc()
1399 	free_cpumask_var(buffer->cpumask);  in __ring_buffer_alloc()
1405 	kfree(buffer);  in __ring_buffer_alloc()
1415 ring_buffer_free(struct ring_buffer *buffer)  in ring_buffer_free()  argument
1421 	__unregister_cpu_notifier(&buffer->cpu_notify);  in ring_buffer_free()
1424 	for_each_buffer_cpu(buffer, cpu)  in ring_buffer_free()
1425 		rb_free_cpu_buffer(buffer->buffers[cpu]);  in ring_buffer_free()
1431 	kfree(buffer->buffers);  in ring_buffer_free()
1432 	free_cpumask_var(buffer->cpumask);  in ring_buffer_free()
1434 	kfree(buffer);  in ring_buffer_free()
1438 void ring_buffer_set_clock(struct ring_buffer *buffer,  in ring_buffer_set_clock()  argument
1441 	buffer->clock = clock;  in ring_buffer_set_clock()
1675 int ring_buffer_resize(struct ring_buffer *buffer, unsigned long size,  in ring_buffer_resize()  argument
1685 	if (!buffer)  in ring_buffer_resize()
1690 	    !cpumask_test_cpu(cpu_id, buffer->cpumask))  in ring_buffer_resize()
1706 	if (atomic_read(&buffer->resize_disabled))  in ring_buffer_resize()
1710 	mutex_lock(&buffer->mutex);  in ring_buffer_resize()
1714 		for_each_buffer_cpu(buffer, cpu) {  in ring_buffer_resize()
1715 			cpu_buffer = buffer->buffers[cpu];  in ring_buffer_resize()
1743 		for_each_buffer_cpu(buffer, cpu) {  in ring_buffer_resize()
1744 			cpu_buffer = buffer->buffers[cpu];  in ring_buffer_resize()
1759 		for_each_buffer_cpu(buffer, cpu) {  in ring_buffer_resize()
1760 			cpu_buffer = buffer->buffers[cpu];  in ring_buffer_resize()
1772 		if (!cpumask_test_cpu(cpu_id, buffer->cpumask))  in ring_buffer_resize()
1775 		cpu_buffer = buffer->buffers[cpu_id];  in ring_buffer_resize()
1814 	if (atomic_read(&buffer->record_disabled)) {  in ring_buffer_resize()
1815 		atomic_inc(&buffer->record_disabled);  in ring_buffer_resize()
1823 		for_each_buffer_cpu(buffer, cpu) {  in ring_buffer_resize()
1824 			cpu_buffer = buffer->buffers[cpu];  in ring_buffer_resize()
1827 		atomic_dec(&buffer->record_disabled);  in ring_buffer_resize()
1830 	mutex_unlock(&buffer->mutex);  in ring_buffer_resize()
1834 	for_each_buffer_cpu(buffer, cpu) {  in ring_buffer_resize()
1837 		cpu_buffer = buffer->buffers[cpu];  in ring_buffer_resize()
1849 	mutex_unlock(&buffer->mutex);  in ring_buffer_resize()
1854 void ring_buffer_change_overwrite(struct ring_buffer *buffer, int val)  in ring_buffer_change_overwrite()  argument
1856 	mutex_lock(&buffer->mutex);  in ring_buffer_change_overwrite()
1858 		buffer->flags |= RB_FL_OVERWRITE;  in ring_buffer_change_overwrite()
1860 		buffer->flags &= ~RB_FL_OVERWRITE;  in ring_buffer_change_overwrite()
1861 	mutex_unlock(&buffer->mutex);  in ring_buffer_change_overwrite()
2318 	struct ring_buffer *buffer = cpu_buffer->buffer;  in rb_move_tail()  local
2361 			if (!(buffer->flags & RB_FL_OVERWRITE)) {  in rb_move_tail()
2400 		ts = rb_time_stamp(buffer);  in rb_move_tail()
2554 rb_reserve_next_event(struct ring_buffer *buffer,  in rb_reserve_next_event()  argument
2574 	if (unlikely(ACCESS_ONCE(cpu_buffer->buffer) != buffer)) {  in rb_reserve_next_event()
2598 	ts = rb_time_stamp(cpu_buffer->buffer);  in rb_reserve_next_event()
2734 ring_buffer_lock_reserve(struct ring_buffer *buffer, unsigned long length)  in ring_buffer_lock_reserve()  argument
2746 	if (unlikely(atomic_read(&buffer->record_disabled)))  in ring_buffer_lock_reserve()
2751 	if (unlikely(!cpumask_test_cpu(cpu, buffer->cpumask)))  in ring_buffer_lock_reserve()
2754 	cpu_buffer = buffer->buffers[cpu];  in ring_buffer_lock_reserve()
2765 	event = rb_reserve_next_event(buffer, cpu_buffer, length);  in ring_buffer_lock_reserve()
2816 rb_wakeups(struct ring_buffer *buffer, struct ring_buffer_per_cpu *cpu_buffer)  in rb_wakeups()  argument
2820 	if (buffer->irq_work.waiters_pending) {  in rb_wakeups()
2821 		buffer->irq_work.waiters_pending = false;  in rb_wakeups()
2823 		irq_work_queue(&buffer->irq_work.work);  in rb_wakeups()
2851 int ring_buffer_unlock_commit(struct ring_buffer *buffer,  in ring_buffer_unlock_commit()  argument
2857 	cpu_buffer = buffer->buffers[cpu];  in ring_buffer_unlock_commit()
2861 	rb_wakeups(buffer, cpu_buffer);  in ring_buffer_unlock_commit()
2943 void ring_buffer_discard_commit(struct ring_buffer *buffer,  in ring_buffer_discard_commit()  argument
2953 	cpu_buffer = buffer->buffers[cpu];  in ring_buffer_discard_commit()
2960 	RB_WARN_ON(buffer, !local_read(&cpu_buffer->committing));  in ring_buffer_discard_commit()
2994 int ring_buffer_write(struct ring_buffer *buffer,  in ring_buffer_write()  argument
3009 	if (atomic_read(&buffer->record_disabled))  in ring_buffer_write()
3014 	if (!cpumask_test_cpu(cpu, buffer->cpumask))  in ring_buffer_write()
3017 	cpu_buffer = buffer->buffers[cpu];  in ring_buffer_write()
3025 	event = rb_reserve_next_event(buffer, cpu_buffer, length);  in ring_buffer_write()
3035 	rb_wakeups(buffer, cpu_buffer);  in ring_buffer_write()
3070 void ring_buffer_record_disable(struct ring_buffer *buffer)  in ring_buffer_record_disable()  argument
3072 	atomic_inc(&buffer->record_disabled);  in ring_buffer_record_disable()
3083 void ring_buffer_record_enable(struct ring_buffer *buffer)  in ring_buffer_record_enable()  argument
3085 	atomic_dec(&buffer->record_disabled);  in ring_buffer_record_enable()
3100 void ring_buffer_record_off(struct ring_buffer *buffer)  in ring_buffer_record_off()  argument
3106 		rd = atomic_read(&buffer->record_disabled);  in ring_buffer_record_off()
3108 	} while (atomic_cmpxchg(&buffer->record_disabled, rd, new_rd) != rd);  in ring_buffer_record_off()
3123 void ring_buffer_record_on(struct ring_buffer *buffer)  in ring_buffer_record_on()  argument
3129 		rd = atomic_read(&buffer->record_disabled);  in ring_buffer_record_on()
3131 	} while (atomic_cmpxchg(&buffer->record_disabled, rd, new_rd) != rd);  in ring_buffer_record_on()
3141 int ring_buffer_record_is_on(struct ring_buffer *buffer)  in ring_buffer_record_is_on()  argument
3143 	return !atomic_read(&buffer->record_disabled);  in ring_buffer_record_is_on()
3156 void ring_buffer_record_disable_cpu(struct ring_buffer *buffer, int cpu)  in ring_buffer_record_disable_cpu()  argument
3160 	if (!cpumask_test_cpu(cpu, buffer->cpumask))  in ring_buffer_record_disable_cpu()
3163 	cpu_buffer = buffer->buffers[cpu];  in ring_buffer_record_disable_cpu()
3176 void ring_buffer_record_enable_cpu(struct ring_buffer *buffer, int cpu)  in ring_buffer_record_enable_cpu()  argument
3180 	if (!cpumask_test_cpu(cpu, buffer->cpumask))  in ring_buffer_record_enable_cpu()
3183 	cpu_buffer = buffer->buffers[cpu];  in ring_buffer_record_enable_cpu()
3206 u64 ring_buffer_oldest_event_ts(struct ring_buffer *buffer, int cpu)  in ring_buffer_oldest_event_ts()  argument
3213 	if (!cpumask_test_cpu(cpu, buffer->cpumask))  in ring_buffer_oldest_event_ts()
3216 	cpu_buffer = buffer->buffers[cpu];  in ring_buffer_oldest_event_ts()
3239 unsigned long ring_buffer_bytes_cpu(struct ring_buffer *buffer, int cpu)  in ring_buffer_bytes_cpu()  argument
3244 	if (!cpumask_test_cpu(cpu, buffer->cpumask))  in ring_buffer_bytes_cpu()
3247 	cpu_buffer = buffer->buffers[cpu];  in ring_buffer_bytes_cpu()
3259 unsigned long ring_buffer_entries_cpu(struct ring_buffer *buffer, int cpu)  in ring_buffer_entries_cpu()  argument
3263 	if (!cpumask_test_cpu(cpu, buffer->cpumask))  in ring_buffer_entries_cpu()
3266 	cpu_buffer = buffer->buffers[cpu];  in ring_buffer_entries_cpu()
3278 unsigned long ring_buffer_overrun_cpu(struct ring_buffer *buffer, int cpu)  in ring_buffer_overrun_cpu()  argument
3283 	if (!cpumask_test_cpu(cpu, buffer->cpumask))  in ring_buffer_overrun_cpu()
3286 	cpu_buffer = buffer->buffers[cpu];  in ring_buffer_overrun_cpu()
3301 ring_buffer_commit_overrun_cpu(struct ring_buffer *buffer, int cpu)  in ring_buffer_commit_overrun_cpu()  argument
3306 	if (!cpumask_test_cpu(cpu, buffer->cpumask))  in ring_buffer_commit_overrun_cpu()
3309 	cpu_buffer = buffer->buffers[cpu];  in ring_buffer_commit_overrun_cpu()
3323 ring_buffer_dropped_events_cpu(struct ring_buffer *buffer, int cpu)  in ring_buffer_dropped_events_cpu()  argument
3328 	if (!cpumask_test_cpu(cpu, buffer->cpumask))  in ring_buffer_dropped_events_cpu()
3331 	cpu_buffer = buffer->buffers[cpu];  in ring_buffer_dropped_events_cpu()
3344 ring_buffer_read_events_cpu(struct ring_buffer *buffer, int cpu)  in ring_buffer_read_events_cpu()  argument
3348 	if (!cpumask_test_cpu(cpu, buffer->cpumask))  in ring_buffer_read_events_cpu()
3351 	cpu_buffer = buffer->buffers[cpu];  in ring_buffer_read_events_cpu()
3363 unsigned long ring_buffer_entries(struct ring_buffer *buffer)  in ring_buffer_entries()  argument
3370 	for_each_buffer_cpu(buffer, cpu) {  in ring_buffer_entries()
3371 		cpu_buffer = buffer->buffers[cpu];  in ring_buffer_entries()
3386 unsigned long ring_buffer_overruns(struct ring_buffer *buffer)  in ring_buffer_overruns()  argument
3393 	for_each_buffer_cpu(buffer, cpu) {  in ring_buffer_overruns()
3394 		cpu_buffer = buffer->buffers[cpu];  in ring_buffer_overruns()
3769 			ring_buffer_normalize_time_stamp(cpu_buffer->buffer,  in rb_buffer_peek()
3787 	struct ring_buffer *buffer;  in rb_iter_peek()  local
3793 	buffer = cpu_buffer->buffer;  in rb_iter_peek()
3851 			ring_buffer_normalize_time_stamp(buffer,  in rb_iter_peek()
3890 ring_buffer_peek(struct ring_buffer *buffer, int cpu, u64 *ts,  in ring_buffer_peek()  argument
3893 	struct ring_buffer_per_cpu *cpu_buffer = buffer->buffers[cpu];  in ring_buffer_peek()
3898 	if (!cpumask_test_cpu(cpu, buffer->cpumask))  in ring_buffer_peek()
3957 ring_buffer_consume(struct ring_buffer *buffer, int cpu, u64 *ts,  in ring_buffer_consume()  argument
3971 	if (!cpumask_test_cpu(cpu, buffer->cpumask))  in ring_buffer_consume()
3974 	cpu_buffer = buffer->buffers[cpu];  in ring_buffer_consume()
4020 ring_buffer_read_prepare(struct ring_buffer *buffer, int cpu)  in ring_buffer_read_prepare()  argument
4025 	if (!cpumask_test_cpu(cpu, buffer->cpumask))  in ring_buffer_read_prepare()
4032 	cpu_buffer = buffer->buffers[cpu];  in ring_buffer_read_prepare()
4036 	atomic_inc(&buffer->resize_disabled);  in ring_buffer_read_prepare()
4111 	atomic_dec(&cpu_buffer->buffer->resize_disabled);  in ring_buffer_read_finish()
4151 unsigned long ring_buffer_size(struct ring_buffer *buffer, int cpu)  in ring_buffer_size()  argument
4159 	if (!cpumask_test_cpu(cpu, buffer->cpumask))  in ring_buffer_size()
4162 	return BUF_PAGE_SIZE * buffer->buffers[cpu]->nr_pages;  in ring_buffer_size()
4213 void ring_buffer_reset_cpu(struct ring_buffer *buffer, int cpu)  in ring_buffer_reset_cpu()  argument
4215 	struct ring_buffer_per_cpu *cpu_buffer = buffer->buffers[cpu];  in ring_buffer_reset_cpu()
4218 	if (!cpumask_test_cpu(cpu, buffer->cpumask))  in ring_buffer_reset_cpu()
4221 	atomic_inc(&buffer->resize_disabled);  in ring_buffer_reset_cpu()
4242 	atomic_dec(&buffer->resize_disabled);  in ring_buffer_reset_cpu()
4250 void ring_buffer_reset(struct ring_buffer *buffer)  in ring_buffer_reset()  argument
4254 	for_each_buffer_cpu(buffer, cpu)  in ring_buffer_reset()
4255 		ring_buffer_reset_cpu(buffer, cpu);  in ring_buffer_reset()
4263 int ring_buffer_empty(struct ring_buffer *buffer)  in ring_buffer_empty()  argument
4274 	for_each_buffer_cpu(buffer, cpu) {  in ring_buffer_empty()
4275 		cpu_buffer = buffer->buffers[cpu];  in ring_buffer_empty()
4297 int ring_buffer_empty_cpu(struct ring_buffer *buffer, int cpu)  in ring_buffer_empty_cpu()  argument
4304 	if (!cpumask_test_cpu(cpu, buffer->cpumask))  in ring_buffer_empty_cpu()
4309 	cpu_buffer = buffer->buffers[cpu];  in ring_buffer_empty_cpu()
4386 	cpu_buffer_b->buffer = buffer_a;  in ring_buffer_swap_cpu()
4387 	cpu_buffer_a->buffer = buffer_b;  in ring_buffer_swap_cpu()
4416 void *ring_buffer_alloc_read_page(struct ring_buffer *buffer, int cpu)  in ring_buffer_alloc_read_page()  argument
4441 void ring_buffer_free_read_page(struct ring_buffer *buffer, void *data)  in ring_buffer_free_read_page()  argument
4480 int ring_buffer_read_page(struct ring_buffer *buffer,  in ring_buffer_read_page()  argument
4483 	struct ring_buffer_per_cpu *cpu_buffer = buffer->buffers[cpu];  in ring_buffer_read_page()
4494 	if (!cpumask_test_cpu(cpu, buffer->cpumask))  in ring_buffer_read_page()
4648 	struct ring_buffer *buffer =  in rb_cpu_notify()  local
4658 		if (cpumask_test_cpu(cpu, buffer->cpumask))  in rb_cpu_notify()
4664 		for_each_buffer_cpu(buffer, cpu_i) {  in rb_cpu_notify()
4667 				nr_pages = buffer->buffers[cpu_i]->nr_pages;  in rb_cpu_notify()
4668 			if (nr_pages != buffer->buffers[cpu_i]->nr_pages) {  in rb_cpu_notify()
4676 		buffer->buffers[cpu] =  in rb_cpu_notify()
4677 			rb_allocate_cpu_buffer(buffer, nr_pages, cpu);  in rb_cpu_notify()
4678 		if (!buffer->buffers[cpu]) {  in rb_cpu_notify()
4684 		cpumask_set_cpu(cpu, buffer->cpumask);  in rb_cpu_notify()
4720 	struct ring_buffer	*buffer;  member
4776 	event = ring_buffer_lock_reserve(data->buffer, len);  in rb_write_something()
4790 	if (RB_WARN_ON(data->buffer, event_len < len))  in rb_write_something()
4816 	ring_buffer_unlock_commit(data->buffer, event);  in rb_write_something()
4862 	struct ring_buffer *buffer;  in test_ringbuffer()  local
4868 	buffer = ring_buffer_alloc(RB_TEST_BUFFER_SIZE, RB_FL_OVERWRITE);  in test_ringbuffer()
4869 	if (WARN_ON(!buffer))  in test_ringbuffer()
4873 	ring_buffer_record_off(buffer);  in test_ringbuffer()
4876 		rb_data[cpu].buffer = buffer;  in test_ringbuffer()
4899 	ring_buffer_record_on(buffer);  in test_ringbuffer()
4925 		ring_buffer_free(buffer);  in test_ringbuffer()
4965 		if (RB_WARN_ON(buffer, total_dropped))  in test_ringbuffer()
4970 		while ((event = ring_buffer_consume(buffer, cpu, NULL, &lost))) {  in test_ringbuffer()
4979 				RB_WARN_ON(buffer, 1);  in test_ringbuffer()
4999 			if (RB_WARN_ON(buffer, total_len != total_alloc ||  in test_ringbuffer()
5003 		if (RB_WARN_ON(buffer, total_lost + total_read != total_events))  in test_ringbuffer()
5011 	ring_buffer_free(buffer);  in test_ringbuffer()