1279 		struct perf_event *event =  in __intel_pmu_enable_all()  local
1282 		if (WARN_ON_ONCE(!event))  in __intel_pmu_enable_all()
1285 		intel_pmu_enable_bts(event->hw.config);  in __intel_pmu_enable_all()
1318 	struct perf_event *event;  in intel_pmu_nhm_workaround()  local
1344 		event = cpuc->events[i];  in intel_pmu_nhm_workaround()
1345 		if (event)  in intel_pmu_nhm_workaround()
1346 			x86_perf_event_update(event);  in intel_pmu_nhm_workaround()
1358 		event = cpuc->events[i];  in intel_pmu_nhm_workaround()
1360 		if (event) {  in intel_pmu_nhm_workaround()
1361 			x86_perf_event_set_period(event);  in intel_pmu_nhm_workaround()
1362 			__x86_pmu_enable_event(&event->hw,  in intel_pmu_nhm_workaround()
1402 static inline bool event_is_checkpointed(struct perf_event *event)  in event_is_checkpointed()  argument
1404 	return (event->hw.config & HSW_IN_TX_CHECKPOINTED) != 0;  in event_is_checkpointed()
1407 static void intel_pmu_disable_event(struct perf_event *event)  in intel_pmu_disable_event()  argument
1409 	struct hw_perf_event *hwc = &event->hw;  in intel_pmu_disable_event()
1426 	if (needs_branch_stack(event))  in intel_pmu_disable_event()
1427 		intel_pmu_lbr_disable(event);  in intel_pmu_disable_event()
1434 	x86_pmu_disable_event(event);  in intel_pmu_disable_event()
1436 	if (unlikely(event->attr.precise_ip))  in intel_pmu_disable_event()
1437 		intel_pmu_pebs_disable(event);  in intel_pmu_disable_event()
1471 static void intel_pmu_enable_event(struct perf_event *event)  in intel_pmu_enable_event()  argument
1473 	struct hw_perf_event *hwc = &event->hw;  in intel_pmu_enable_event()
1487 	if (needs_branch_stack(event))  in intel_pmu_enable_event()
1488 		intel_pmu_lbr_enable(event);  in intel_pmu_enable_event()
1490 	if (event->attr.exclude_host)  in intel_pmu_enable_event()
1492 	if (event->attr.exclude_guest)  in intel_pmu_enable_event()
1495 	if (unlikely(event_is_checkpointed(event)))  in intel_pmu_enable_event()
1503 	if (unlikely(event->attr.precise_ip))  in intel_pmu_enable_event()
1504 		intel_pmu_pebs_enable(event);  in intel_pmu_enable_event()
1513 int intel_pmu_save_and_restart(struct perf_event *event)  in intel_pmu_save_and_restart()  argument
1515 	x86_perf_event_update(event);  in intel_pmu_save_and_restart()
1522 	if (unlikely(event_is_checkpointed(event))) {  in intel_pmu_save_and_restart()
1524 		wrmsrl(event->hw.event_base, 0);  in intel_pmu_save_and_restart()
1525 		local64_set(&event->hw.prev_count, 0);  in intel_pmu_save_and_restart()
1527 	return x86_perf_event_set_period(event);  in intel_pmu_save_and_restart()
1646 		struct perf_event *event = cpuc->events[bit];  in intel_pmu_handle_irq()  local
1653 		if (!intel_pmu_save_and_restart(event))  in intel_pmu_handle_irq()
1656 		perf_sample_data_init(&data, 0, event->hw.last_period);  in intel_pmu_handle_irq()
1658 		if (has_branch_stack(event))  in intel_pmu_handle_irq()
1661 		if (perf_event_overflow(event, &data, regs))  in intel_pmu_handle_irq()
1662 			x86_pmu_stop(event, 0);  in intel_pmu_handle_irq()
1685 intel_bts_constraints(struct perf_event *event)  in intel_bts_constraints()  argument
1687 	struct hw_perf_event *hwc = &event->hw;  in intel_bts_constraints()
1690 	if (event->attr.freq)  in intel_bts_constraints()
1716 static void intel_fixup_er(struct perf_event *event, int idx)  in intel_fixup_er()  argument
1718 	event->hw.extra_reg.idx = idx;  in intel_fixup_er()
1721 		event->hw.config &= ~INTEL_ARCH_EVENT_MASK;  in intel_fixup_er()
1722 		event->hw.config |= x86_pmu.extra_regs[EXTRA_REG_RSP_0].event;  in intel_fixup_er()
1723 		event->hw.extra_reg.reg = MSR_OFFCORE_RSP_0;  in intel_fixup_er()
1725 		event->hw.config &= ~INTEL_ARCH_EVENT_MASK;  in intel_fixup_er()
1726 		event->hw.config |= x86_pmu.extra_regs[EXTRA_REG_RSP_1].event;  in intel_fixup_er()
1727 		event->hw.extra_reg.reg = MSR_OFFCORE_RSP_1;  in intel_fixup_er()
1740 				   struct perf_event *event,  in __intel_shared_reg_get_constraints()  argument
1778 				intel_fixup_er(event, idx);  in __intel_shared_reg_get_constraints()
1841 			      struct perf_event *event)  in intel_shared_regs_constraints()  argument
1846 	xreg = &event->hw.extra_reg;  in intel_shared_regs_constraints()
1848 		c = __intel_shared_reg_get_constraints(cpuc, event, xreg);  in intel_shared_regs_constraints()
1852 	breg = &event->hw.branch_reg;  in intel_shared_regs_constraints()
1854 		d = __intel_shared_reg_get_constraints(cpuc, event, breg);  in intel_shared_regs_constraints()
1865 			  struct perf_event *event)  in x86_get_event_constraints()  argument
1871 			if ((event->hw.config & c->cmask) == c->code) {  in x86_get_event_constraints()
1872 				event->hw.flags |= c->flags;  in x86_get_event_constraints()
1883 			    struct perf_event *event)  in __intel_get_event_constraints()  argument
1887 	c = intel_bts_constraints(event);  in __intel_get_event_constraints()
1891 	c = intel_shared_regs_constraints(cpuc, event);  in __intel_get_event_constraints()
1895 	c = intel_pebs_constraints(event);  in __intel_get_event_constraints()
1899 	return x86_get_event_constraints(cpuc, idx, event);  in __intel_get_event_constraints()
1975 intel_get_excl_constraints(struct cpu_hw_events *cpuc, struct perf_event *event,  in intel_get_excl_constraints()  argument
2002 	if (is_excl && !(event->hw.flags & PERF_X86_EVENT_EXCL_ACCT)) {  in intel_get_excl_constraints()
2003 		event->hw.flags |= PERF_X86_EVENT_EXCL_ACCT;  in intel_get_excl_constraints()
2099 			    struct perf_event *event)  in intel_get_event_constraints()  argument
2112 	c2 = __intel_get_event_constraints(cpuc, idx, event);  in intel_get_event_constraints()
2120 		return intel_get_excl_constraints(cpuc, event, idx, c2);  in intel_get_event_constraints()
2126 		struct perf_event *event)  in intel_put_excl_constraints()  argument
2128 	struct hw_perf_event *hwc = &event->hw;  in intel_put_excl_constraints()
2175 					struct perf_event *event)  in intel_put_shared_regs_event_constraints()  argument
2179 	reg = &event->hw.extra_reg;  in intel_put_shared_regs_event_constraints()
2183 	reg = &event->hw.branch_reg;  in intel_put_shared_regs_event_constraints()
2189 					struct perf_event *event)  in intel_put_event_constraints()  argument
2191 	intel_put_shared_regs_event_constraints(cpuc, event);  in intel_put_event_constraints()
2199 		intel_put_excl_constraints(cpuc, event);  in intel_put_event_constraints()
2237 static void intel_pebs_aliases_core2(struct perf_event *event)  in intel_pebs_aliases_core2()  argument
2239 	if ((event->hw.config & X86_RAW_EVENT_MASK) == 0x003c) {  in intel_pebs_aliases_core2()
2258 		u64 alt_config = X86_CONFIG(.event=0xc0, .inv=1, .cmask=16);  in intel_pebs_aliases_core2()
2260 		alt_config |= (event->hw.config & ~X86_RAW_EVENT_MASK);  in intel_pebs_aliases_core2()
2261 		event->hw.config = alt_config;  in intel_pebs_aliases_core2()
2265 static void intel_pebs_aliases_snb(struct perf_event *event)  in intel_pebs_aliases_snb()  argument
2267 	if ((event->hw.config & X86_RAW_EVENT_MASK) == 0x003c) {  in intel_pebs_aliases_snb()
2286 		u64 alt_config = X86_CONFIG(.event=0xc2, .umask=0x01, .inv=1, .cmask=16);  in intel_pebs_aliases_snb()
2288 		alt_config |= (event->hw.config & ~X86_RAW_EVENT_MASK);  in intel_pebs_aliases_snb()
2289 		event->hw.config = alt_config;  in intel_pebs_aliases_snb()
2293 static int intel_pmu_hw_config(struct perf_event *event)  in intel_pmu_hw_config()  argument
2295 	int ret = x86_pmu_hw_config(event);  in intel_pmu_hw_config()
2300 	if (event->attr.precise_ip && x86_pmu.pebs_aliases)  in intel_pmu_hw_config()
2301 		x86_pmu.pebs_aliases(event);  in intel_pmu_hw_config()
2303 	if (needs_branch_stack(event)) {  in intel_pmu_hw_config()
2304 		ret = intel_pmu_setup_lbr_filter(event);  in intel_pmu_hw_config()
2311 		if (!intel_pmu_has_bts(event)) {  in intel_pmu_hw_config()
2316 			event->destroy = hw_perf_lbr_event_destroy;  in intel_pmu_hw_config()
2320 	if (event->attr.type != PERF_TYPE_RAW)  in intel_pmu_hw_config()
2323 	if (!(event->attr.config & ARCH_PERFMON_EVENTSEL_ANY))  in intel_pmu_hw_config()
2332 	event->hw.config |= ARCH_PERFMON_EVENTSEL_ANY;  in intel_pmu_hw_config()
2374 		struct perf_event *event = cpuc->events[idx];  in core_guest_get_msrs()  local
2383 			event->hw.config | ARCH_PERFMON_EVENTSEL_ENABLE;  in core_guest_get_msrs()
2385 		if (event->attr.exclude_host)  in core_guest_get_msrs()
2387 		else if (event->attr.exclude_guest)  in core_guest_get_msrs()
2395 static void core_pmu_enable_event(struct perf_event *event)  in core_pmu_enable_event()  argument
2397 	if (!event->attr.exclude_host)  in core_pmu_enable_event()
2398 		x86_pmu_enable_event(event);  in core_pmu_enable_event()
2417 static int hsw_hw_config(struct perf_event *event)  in hsw_hw_config()  argument
2419 	int ret = intel_pmu_hw_config(event);  in hsw_hw_config()
2425 	event->hw.config |= event->attr.config & (HSW_IN_TX|HSW_IN_TX_CHECKPOINTED);  in hsw_hw_config()
2432 	if ((event->hw.config & (HSW_IN_TX|HSW_IN_TX_CHECKPOINTED)) &&  in hsw_hw_config()
2433 	     ((event->hw.config & ARCH_PERFMON_EVENTSEL_ANY) ||  in hsw_hw_config()
2434 	      event->attr.precise_ip > 0))  in hsw_hw_config()
2437 	if (event_is_checkpointed(event)) {  in hsw_hw_config()
2447 		if (event->attr.sample_period > 0 &&  in hsw_hw_config()
2448 		    event->attr.sample_period < 0x7fffffff)  in hsw_hw_config()
2459 			  struct perf_event *event)  in hsw_get_event_constraints()  argument
2463 	c = intel_get_event_constraints(cpuc, idx, event);  in hsw_get_event_constraints()
2466 	if (event->hw.config & HSW_IN_TX_CHECKPOINTED) {  in hsw_get_event_constraints()
2490 static unsigned bdw_limit_period(struct perf_event *event, unsigned left)  in bdw_limit_period()  argument
2492 	if ((event->hw.config & INTEL_ARCH_EVENT_MASK) ==  in bdw_limit_period()
2493 			X86_CONFIG(.event=0xc0, .umask=0x01)) {  in bdw_limit_period()
2501 PMU_FORMAT_ATTR(event,	"config:0-7"	);
2523 	u64 event = (config & ARCH_PERFMON_EVENTSEL_EVENT);  in intel_event_sysfs_show()  local
2525 	return x86_event_sysfs_show(page, config, event);  in intel_event_sysfs_show()
3091 			X86_CONFIG(.event=0x0e, .umask=0x01, .inv=1, .cmask=1);  in intel_pmu_init()
3094 			X86_CONFIG(.event=0xb1, .umask=0x3f, .inv=1, .cmask=1);  in intel_pmu_init()
3153 			X86_CONFIG(.event=0x0e, .umask=0x01, .inv=1, .cmask=1);  in intel_pmu_init()
3156 			X86_CONFIG(.event=0xb1, .umask=0x3f, .inv=1, .cmask=1);  in intel_pmu_init()
3189 			X86_CONFIG(.event=0x0e, .umask=0x01, .inv=1, .cmask=1);  in intel_pmu_init()
3192 			X86_CONFIG(.event=0xb1, .umask=0x01, .inv=1, .cmask=1);  in intel_pmu_init()
3225 			X86_CONFIG(.event=0x0e, .umask=0x01, .inv=1, .cmask=1);  in intel_pmu_init()