120 static u64 precise_datala_hsw(struct perf_event *event, u64 status)  in precise_datala_hsw()  argument
126 	if (event->hw.flags & PERF_X86_EVENT_PEBS_ST_HSW)  in precise_datala_hsw()
128 	else if (event->hw.flags & PERF_X86_EVENT_PEBS_LD_HSW)  in precise_datala_hsw()
139 	if (event->hw.flags & PERF_X86_EVENT_PEBS_ST_HSW) {  in precise_datala_hsw()
521 	struct perf_event *event = cpuc->events[INTEL_PMC_IDX_FIXED_BTS];  in intel_pmu_drain_bts_buffer()  local
529 	if (!event)  in intel_pmu_drain_bts_buffer()
545 	perf_sample_data_init(&data, 0, event->hw.last_period);  in intel_pmu_drain_bts_buffer()
563 		if (event->attr.exclude_kernel &&  in intel_pmu_drain_bts_buffer()
573 	perf_prepare_sample(&header, &data, event, &regs);  in intel_pmu_drain_bts_buffer()
575 	if (perf_output_begin(&handle, event, header.size *  in intel_pmu_drain_bts_buffer()
581 		if (event->attr.exclude_kernel &&  in intel_pmu_drain_bts_buffer()
588 		perf_output_sample(&handle, &header, &data, event);  in intel_pmu_drain_bts_buffer()
594 	event->hw.interrupts++;  in intel_pmu_drain_bts_buffer()
595 	event->pending_kill = POLL_IN;  in intel_pmu_drain_bts_buffer()
750 struct event_constraint *intel_pebs_constraints(struct perf_event *event)  in intel_pebs_constraints()  argument
754 	if (!event->attr.precise_ip)  in intel_pebs_constraints()
759 			if ((event->hw.config & c->cmask) == c->code) {  in intel_pebs_constraints()
760 				event->hw.flags |= c->flags;  in intel_pebs_constraints()
774 void intel_pmu_pebs_enable(struct perf_event *event)  in intel_pmu_pebs_enable()  argument
777 	struct hw_perf_event *hwc = &event->hw;  in intel_pmu_pebs_enable()
787 	if (event->hw.flags & PERF_X86_EVENT_PEBS_LDLAT)  in intel_pmu_pebs_enable()
789 	else if (event->hw.flags & PERF_X86_EVENT_PEBS_ST)  in intel_pmu_pebs_enable()
801 			perf_sched_cb_inc(event->ctx->pmu);  in intel_pmu_pebs_enable()
811 			perf_sched_cb_dec(event->ctx->pmu);  in intel_pmu_pebs_enable()
824 void intel_pmu_pebs_disable(struct perf_event *event)  in intel_pmu_pebs_disable()  argument
827 	struct hw_perf_event *hwc = &event->hw;  in intel_pmu_pebs_disable()
837 	if (event->hw.flags & PERF_X86_EVENT_PEBS_LDLAT)  in intel_pmu_pebs_disable()
839 	else if (event->hw.flags & PERF_X86_EVENT_PEBS_ST)  in intel_pmu_pebs_disable()
843 		perf_sched_cb_dec(event->ctx->pmu);  in intel_pmu_pebs_disable()
980 static void setup_pebs_sample_data(struct perf_event *event,  in setup_pebs_sample_data()  argument
997 	int fl = event->hw.flags;  in setup_pebs_sample_data()
1002 	sample_type = event->attr.sample_type;  in setup_pebs_sample_data()
1008 	perf_sample_data_init(data, 0, event->hw.last_period);  in setup_pebs_sample_data()
1010 	data->period = event->hw.last_period;  in setup_pebs_sample_data()
1026 			val = precise_datala_hsw(event, pebs->dse);  in setup_pebs_sample_data()
1071 	if (event->attr.precise_ip > 1 && x86_pmu.intel_cap.pebs_format >= 2) {  in setup_pebs_sample_data()
1074 	} else if (event->attr.precise_ip > 1 && intel_pmu_pebs_fixup_ip(regs))  in setup_pebs_sample_data()
1099 		event->attr.use_clockid == 0)  in setup_pebs_sample_data()
1102 	if (has_branch_stack(event))  in setup_pebs_sample_data()
1137 static void __intel_pmu_pebs_event(struct perf_event *event,  in __intel_pmu_pebs_event()  argument
1146 	if (!intel_pmu_save_and_restart(event) &&  in __intel_pmu_pebs_event()
1147 	    !(event->hw.flags & PERF_X86_EVENT_AUTO_RELOAD))  in __intel_pmu_pebs_event()
1151 		setup_pebs_sample_data(event, iregs, at, &data, &regs);  in __intel_pmu_pebs_event()
1152 		perf_event_output(event, &data, &regs);  in __intel_pmu_pebs_event()
1158 	setup_pebs_sample_data(event, iregs, at, &data, &regs);  in __intel_pmu_pebs_event()
1164 	if (perf_event_overflow(event, &data, &regs)) {  in __intel_pmu_pebs_event()
1165 		x86_pmu_stop(event, 0);  in __intel_pmu_pebs_event()
1175 	struct perf_event *event = cpuc->events[0]; /* PMC0 only */  in intel_pmu_drain_pebs_core()  local
1193 	WARN_ON_ONCE(!event);  in intel_pmu_drain_pebs_core()
1195 	if (!event->attr.precise_ip)  in intel_pmu_drain_pebs_core()
1202 	__intel_pmu_pebs_event(event, iregs, at, top, 0, n);  in intel_pmu_drain_pebs_core()
1209 	struct perf_event *event;  in intel_pmu_drain_pebs_nhm()  local
1279 		event = cpuc->events[bit];  in intel_pmu_drain_pebs_nhm()
1280 		WARN_ON_ONCE(!event);  in intel_pmu_drain_pebs_nhm()
1281 		WARN_ON_ONCE(!event->attr.precise_ip);  in intel_pmu_drain_pebs_nhm()
1285 			perf_log_lost_samples(event, error[bit]);  in intel_pmu_drain_pebs_nhm()
1288 			__intel_pmu_pebs_event(event, iregs, base,  in intel_pmu_drain_pebs_nhm()