20 static void perf_output_wakeup(struct perf_output_handle *handle)  in perf_output_wakeup()  argument
22 	atomic_set(&handle->rb->poll, POLLIN);  in perf_output_wakeup()
24 	handle->event->pending_wakeup = 1;  in perf_output_wakeup()
25 	irq_work_queue(&handle->event->pending);  in perf_output_wakeup()
36 static void perf_output_get_handle(struct perf_output_handle *handle)  in perf_output_get_handle()  argument
38 	struct ring_buffer *rb = handle->rb;  in perf_output_get_handle()
42 	handle->wakeup = local_read(&rb->wakeup);  in perf_output_get_handle()
45 static void perf_output_put_handle(struct perf_output_handle *handle)  in perf_output_put_handle()  argument
47 	struct ring_buffer *rb = handle->rb;  in perf_output_put_handle()
98 	if (handle->wakeup != local_read(&rb->wakeup))  in perf_output_put_handle()
99 		perf_output_wakeup(handle);  in perf_output_put_handle()
105 int perf_output_begin(struct perf_output_handle *handle,  in perf_output_begin()  argument
131 	handle->rb    = rb;  in perf_output_begin()
132 	handle->event = event;  in perf_output_begin()
141 	perf_output_get_handle(handle);  in perf_output_begin()
175 	handle->page = (offset >> page_shift) & (rb->nr_pages - 1);  in perf_output_begin()
177 	handle->addr = rb->data_pages[handle->page] + offset;  in perf_output_begin()
178 	handle->size = (1UL << page_shift) - offset;  in perf_output_begin()
191 		perf_output_put(handle, lost_event);  in perf_output_begin()
192 		perf_event__output_id_sample(event, handle, &sample_data);  in perf_output_begin()
199 	perf_output_put_handle(handle);  in perf_output_begin()
206 unsigned int perf_output_copy(struct perf_output_handle *handle,  in perf_output_copy()  argument
209 	return __output_copy(handle, buf, len);  in perf_output_copy()
212 unsigned int perf_output_skip(struct perf_output_handle *handle,  in perf_output_skip()  argument
215 	return __output_skip(handle, NULL, len);  in perf_output_skip()
218 void perf_output_end(struct perf_output_handle *handle)  in perf_output_end()  argument
220 	perf_output_put_handle(handle);  in perf_output_end()
268 void *perf_aux_output_begin(struct perf_output_handle *handle,  in perf_aux_output_begin()  argument
299 	handle->rb = rb;  in perf_aux_output_begin()
300 	handle->event = event;  in perf_aux_output_begin()
301 	handle->head = aux_head;  in perf_aux_output_begin()
302 	handle->size = 0;  in perf_aux_output_begin()
311 		handle->wakeup = local_read(&rb->aux_wakeup) + rb->aux_watermark;  in perf_aux_output_begin()
313 			handle->size = CIRC_SPACE(aux_head, aux_tail, perf_aux_size(rb));  in perf_aux_output_begin()
320 		if (!handle->size) { /* A, matches D */  in perf_aux_output_begin()
322 			perf_output_wakeup(handle);  in perf_aux_output_begin()
328 	return handle->rb->aux_priv;  in perf_aux_output_begin()
335 	handle->event = NULL;  in perf_aux_output_begin()
346 void perf_aux_output_end(struct perf_output_handle *handle, unsigned long size,  in perf_aux_output_end()  argument
349 	struct ring_buffer *rb = handle->rb;  in perf_aux_output_end()
361 		aux_head = handle->head;  in perf_aux_output_end()
373 		perf_event_aux_event(handle->event, aux_head, size, flags);  in perf_aux_output_end()
385 			handle->event->pending_disable = 1;  in perf_aux_output_end()
386 		perf_output_wakeup(handle);  in perf_aux_output_end()
389 	handle->event = NULL;  in perf_aux_output_end()
400 int perf_aux_output_skip(struct perf_output_handle *handle, unsigned long size)  in perf_aux_output_skip()  argument
402 	struct ring_buffer *rb = handle->rb;  in perf_aux_output_skip()
405 	if (size > handle->size)  in perf_aux_output_skip()
412 		perf_output_wakeup(handle);  in perf_aux_output_skip()
414 		handle->wakeup = local_read(&rb->aux_wakeup) +  in perf_aux_output_skip()
418 	handle->head = aux_head;  in perf_aux_output_skip()
419 	handle->size -= size;  in perf_aux_output_skip()
424 void *perf_get_aux(struct perf_output_handle *handle)  in perf_get_aux()  argument
427 	if (!handle->event)  in perf_get_aux()
430 	return handle->rb->aux_priv;  in perf_get_aux()