34 static DEFINE_PER_CPU(struct pt, pt_ctx);
465 static void pt_update_head(struct pt *pt)  in pt_update_head()  argument
467 	struct pt_buffer *buf = perf_get_aux(&pt->handle);  in pt_update_head()
511 static void pt_handle_status(struct pt *pt)  in pt_handle_status()  argument
513 	struct pt_buffer *buf = perf_get_aux(&pt->handle);  in pt_handle_status()
850 static bool pt_buffer_is_full(struct pt_buffer *buf, struct pt *pt)  in pt_buffer_is_full()  argument
855 	if (local_read(&buf->data_size) >= pt->handle.size)  in pt_buffer_is_full()
866 	struct pt *pt = this_cpu_ptr(&pt_ctx);  in intel_pt_interrupt()  local
868 	struct perf_event *event = pt->handle.event;  in intel_pt_interrupt()
875 	if (!ACCESS_ONCE(pt->handle_nmi))  in intel_pt_interrupt()
883 	buf = perf_get_aux(&pt->handle);  in intel_pt_interrupt()
889 	pt_handle_status(pt);  in intel_pt_interrupt()
891 	pt_update_head(pt);  in intel_pt_interrupt()
893 	perf_aux_output_end(&pt->handle, local_xchg(&buf->data_size, 0),  in intel_pt_interrupt()
899 		buf = perf_aux_output_begin(&pt->handle, event);  in intel_pt_interrupt()
905 		pt_buffer_reset_offsets(buf, pt->handle.head);  in intel_pt_interrupt()
906 		ret = pt_buffer_reset_markers(buf, &pt->handle);  in intel_pt_interrupt()
908 			perf_aux_output_end(&pt->handle, 0, true);  in intel_pt_interrupt()
925 	struct pt *pt = this_cpu_ptr(&pt_ctx);  in pt_event_start()  local
926 	struct pt_buffer *buf = perf_get_aux(&pt->handle);  in pt_event_start()
928 	if (pt_is_running() || !buf || pt_buffer_is_full(buf, pt)) {  in pt_event_start()
933 	ACCESS_ONCE(pt->handle_nmi) = 1;  in pt_event_start()
944 	struct pt *pt = this_cpu_ptr(&pt_ctx);  in pt_event_stop()  local
950 	ACCESS_ONCE(pt->handle_nmi) = 0;  in pt_event_stop()
959 		struct pt *pt = this_cpu_ptr(&pt_ctx);  in pt_event_stop()  local
960 		struct pt_buffer *buf = perf_get_aux(&pt->handle);  in pt_event_stop()
965 		if (WARN_ON_ONCE(pt->handle.event != event))  in pt_event_stop()
970 		pt_handle_status(pt);  in pt_event_stop()
972 		pt_update_head(pt);  in pt_event_stop()
978 	struct pt *pt = this_cpu_ptr(&pt_ctx);  in pt_event_del()  local
983 	buf = perf_get_aux(&pt->handle);  in pt_event_del()
987 			pt->handle.head =  in pt_event_del()
990 		perf_aux_output_end(&pt->handle, local_xchg(&buf->data_size, 0),  in pt_event_del()
998 	struct pt *pt = this_cpu_ptr(&pt_ctx);  in pt_event_add()  local
1002 	if (pt->handle.event)  in pt_event_add()
1005 	buf = perf_aux_output_begin(&pt->handle, event);  in pt_event_add()
1010 	pt_buffer_reset_offsets(buf, pt->handle.head);  in pt_event_add()
1012 		ret = pt_buffer_reset_markers(buf, &pt->handle);  in pt_event_add()
1029 	perf_aux_output_end(&pt->handle, 0, true);  in pt_event_add()