1473 	struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);  in __intel_pmu_disable_all()  local
1477 	if (test_bit(INTEL_PMC_IDX_FIXED_BTS, cpuc->active_mask))  in __intel_pmu_disable_all()
1493 	struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);  in __intel_pmu_enable_all()  local
1498 			x86_pmu.intel_ctrl & ~cpuc->intel_ctrl_guest_mask);  in __intel_pmu_enable_all()
1500 	if (test_bit(INTEL_PMC_IDX_FIXED_BTS, cpuc->active_mask)) {  in __intel_pmu_enable_all()
1502 			cpuc->events[INTEL_PMC_IDX_FIXED_BTS];  in __intel_pmu_enable_all()
1533 	struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);  in intel_pmu_nhm_workaround()  local
1566 		event = cpuc->events[i];  in intel_pmu_nhm_workaround()
1580 		event = cpuc->events[i];  in intel_pmu_nhm_workaround()
1632 	struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);  in intel_pmu_disable_event()  local
1640 	cpuc->intel_ctrl_guest_mask &= ~(1ull << hwc->idx);  in intel_pmu_disable_event()
1641 	cpuc->intel_ctrl_host_mask &= ~(1ull << hwc->idx);  in intel_pmu_disable_event()
1642 	cpuc->intel_cp_status &= ~(1ull << hwc->idx);  in intel_pmu_disable_event()
1696 	struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);  in intel_pmu_enable_event()  local
1713 		cpuc->intel_ctrl_guest_mask |= (1ull << hwc->idx);  in intel_pmu_enable_event()
1715 		cpuc->intel_ctrl_host_mask |= (1ull << hwc->idx);  in intel_pmu_enable_event()
1718 		cpuc->intel_cp_status |= (1ull << hwc->idx);  in intel_pmu_enable_event()
1797 	struct cpu_hw_events *cpuc;  in intel_pmu_handle_irq()  local
1802 	cpuc = this_cpu_ptr(&cpu_hw_events);  in intel_pmu_handle_irq()
1859 		status &= ~cpuc->pebs_enabled;  in intel_pmu_handle_irq()
1876 	status |= cpuc->intel_cp_status;  in intel_pmu_handle_irq()
1879 		struct perf_event *event = cpuc->events[bit];  in intel_pmu_handle_irq()
1883 		if (!test_bit(bit, cpuc->active_mask))  in intel_pmu_handle_irq()
1892 			data.br_stack = &cpuc->lbr_stack;  in intel_pmu_handle_irq()
1907 	if (cpuc->enabled)  in intel_pmu_handle_irq()
1979 __intel_shared_reg_get_constraints(struct cpu_hw_events *cpuc,  in __intel_shared_reg_get_constraints()  argument
1993 	if (reg->alloc && !cpuc->is_fake)  in __intel_shared_reg_get_constraints()
1997 	era = &cpuc->shared_regs->regs[idx];  in __intel_shared_reg_get_constraints()
2016 		if (!cpuc->is_fake) {  in __intel_shared_reg_get_constraints()
2054 __intel_shared_reg_put_constraints(struct cpu_hw_events *cpuc,  in __intel_shared_reg_put_constraints()  argument
2067 	if (!reg->alloc || cpuc->is_fake)  in __intel_shared_reg_put_constraints()
2070 	era = &cpuc->shared_regs->regs[reg->idx];  in __intel_shared_reg_put_constraints()
2080 intel_shared_regs_constraints(struct cpu_hw_events *cpuc,  in intel_shared_regs_constraints()  argument
2088 		c = __intel_shared_reg_get_constraints(cpuc, event, xreg);  in intel_shared_regs_constraints()
2094 		d = __intel_shared_reg_get_constraints(cpuc, event, breg);  in intel_shared_regs_constraints()
2096 			__intel_shared_reg_put_constraints(cpuc, xreg);  in intel_shared_regs_constraints()
2104 x86_get_event_constraints(struct cpu_hw_events *cpuc, int idx,  in x86_get_event_constraints()  argument
2122 __intel_get_event_constraints(struct cpu_hw_events *cpuc, int idx,  in __intel_get_event_constraints()  argument
2131 	c = intel_shared_regs_constraints(cpuc, event);  in __intel_get_event_constraints()
2139 	return x86_get_event_constraints(cpuc, idx, event);  in __intel_get_event_constraints()
2143 intel_start_scheduling(struct cpu_hw_events *cpuc)  in intel_start_scheduling()  argument
2145 	struct intel_excl_cntrs *excl_cntrs = cpuc->excl_cntrs;  in intel_start_scheduling()
2147 	int tid = cpuc->excl_thread_id;  in intel_start_scheduling()
2152 	if (cpuc->is_fake || !is_ht_workaround_enabled())  in intel_start_scheduling()
2172 static void intel_commit_scheduling(struct cpu_hw_events *cpuc, int idx, int cntr)  in intel_commit_scheduling()  argument
2174 	struct intel_excl_cntrs *excl_cntrs = cpuc->excl_cntrs;  in intel_commit_scheduling()
2175 	struct event_constraint *c = cpuc->event_constraint[idx];  in intel_commit_scheduling()
2177 	int tid = cpuc->excl_thread_id;  in intel_commit_scheduling()
2179 	if (cpuc->is_fake || !is_ht_workaround_enabled())  in intel_commit_scheduling()
2199 intel_stop_scheduling(struct cpu_hw_events *cpuc)  in intel_stop_scheduling()  argument
2201 	struct intel_excl_cntrs *excl_cntrs = cpuc->excl_cntrs;  in intel_stop_scheduling()
2203 	int tid = cpuc->excl_thread_id;  in intel_stop_scheduling()
2208 	if (cpuc->is_fake || !is_ht_workaround_enabled())  in intel_stop_scheduling()
2226 intel_get_excl_constraints(struct cpu_hw_events *cpuc, struct perf_event *event,  in intel_get_excl_constraints()  argument
2229 	struct intel_excl_cntrs *excl_cntrs = cpuc->excl_cntrs;  in intel_get_excl_constraints()
2231 	int tid = cpuc->excl_thread_id;  in intel_get_excl_constraints()
2238 	if (cpuc->is_fake || !is_ht_workaround_enabled())  in intel_get_excl_constraints()
2261 		cx = &cpuc->constraint_list[idx];  in intel_get_excl_constraints()
2296 		if (!cpuc->n_excl++)  in intel_get_excl_constraints()
2342 intel_get_event_constraints(struct cpu_hw_events *cpuc, int idx,  in intel_get_event_constraints()  argument
2349 		c1 = cpuc->event_constraint[idx];  in intel_get_event_constraints()
2356 	c2 = __intel_get_event_constraints(cpuc, idx, event);  in intel_get_event_constraints()
2363 	if (cpuc->excl_cntrs)  in intel_get_event_constraints()
2364 		return intel_get_excl_constraints(cpuc, event, idx, c2);  in intel_get_event_constraints()
2369 static void intel_put_excl_constraints(struct cpu_hw_events *cpuc,  in intel_put_excl_constraints()  argument
2373 	struct intel_excl_cntrs *excl_cntrs = cpuc->excl_cntrs;  in intel_put_excl_constraints()
2374 	int tid = cpuc->excl_thread_id;  in intel_put_excl_constraints()
2380 	if (cpuc->is_fake)  in intel_put_excl_constraints()
2388 		if (!--cpuc->n_excl)  in intel_put_excl_constraints()
2415 intel_put_shared_regs_event_constraints(struct cpu_hw_events *cpuc,  in intel_put_shared_regs_event_constraints()  argument
2422 		__intel_shared_reg_put_constraints(cpuc, reg);  in intel_put_shared_regs_event_constraints()
2426 		__intel_shared_reg_put_constraints(cpuc, reg);  in intel_put_shared_regs_event_constraints()
2429 static void intel_put_event_constraints(struct cpu_hw_events *cpuc,  in intel_put_event_constraints()  argument
2432 	intel_put_shared_regs_event_constraints(cpuc, event);  in intel_put_event_constraints()
2439 	if (cpuc->excl_cntrs)  in intel_put_event_constraints()
2440 		intel_put_excl_constraints(cpuc, event);  in intel_put_event_constraints()
2571 	struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);  in intel_guest_get_msrs()  local
2572 	struct perf_guest_switch_msr *arr = cpuc->guest_switch_msrs;  in intel_guest_get_msrs()
2575 	arr[0].host = x86_pmu.intel_ctrl & ~cpuc->intel_ctrl_guest_mask;  in intel_guest_get_msrs()
2576 	arr[0].guest = x86_pmu.intel_ctrl & ~cpuc->intel_ctrl_host_mask;  in intel_guest_get_msrs()
2583 	arr[1].host = cpuc->pebs_enabled;  in intel_guest_get_msrs()
2592 	struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);  in core_guest_get_msrs()  local
2593 	struct perf_guest_switch_msr *arr = cpuc->guest_switch_msrs;  in core_guest_get_msrs()
2597 		struct perf_event *event = cpuc->events[idx];  in core_guest_get_msrs()
2602 		if (!test_bit(idx, cpuc->active_mask))  in core_guest_get_msrs()
2626 	struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);  in core_pmu_enable_all()  local
2630 		struct hw_perf_event *hwc = &cpuc->events[idx]->hw;  in core_pmu_enable_all()
2632 		if (!test_bit(idx, cpuc->active_mask) ||  in core_pmu_enable_all()
2633 				cpuc->events[idx]->attr.exclude_host)  in core_pmu_enable_all()
2681 hsw_get_event_constraints(struct cpu_hw_events *cpuc, int idx,  in hsw_get_event_constraints()  argument
2686 	c = intel_get_event_constraints(cpuc, idx, event);  in hsw_get_event_constraints()
2785 	struct cpu_hw_events *cpuc = &per_cpu(cpu_hw_events, cpu);  in intel_pmu_cpu_prepare()  local
2788 		cpuc->shared_regs = allocate_shared_regs(cpu);  in intel_pmu_cpu_prepare()
2789 		if (!cpuc->shared_regs)  in intel_pmu_cpu_prepare()
2796 		cpuc->constraint_list = kzalloc(sz, GFP_KERNEL);  in intel_pmu_cpu_prepare()
2797 		if (!cpuc->constraint_list)  in intel_pmu_cpu_prepare()
2800 		cpuc->excl_cntrs = allocate_excl_cntrs(cpu);  in intel_pmu_cpu_prepare()
2801 		if (!cpuc->excl_cntrs)  in intel_pmu_cpu_prepare()
2804 		cpuc->excl_thread_id = 0;  in intel_pmu_cpu_prepare()
2810 	kfree(cpuc->constraint_list);  in intel_pmu_cpu_prepare()
2811 	cpuc->constraint_list = NULL;  in intel_pmu_cpu_prepare()
2814 	kfree(cpuc->shared_regs);  in intel_pmu_cpu_prepare()
2815 	cpuc->shared_regs = NULL;  in intel_pmu_cpu_prepare()
2823 	struct cpu_hw_events *cpuc = &per_cpu(cpu_hw_events, cpu);  in intel_pmu_cpu_starting()  local
2833 	cpuc->lbr_sel = NULL;  in intel_pmu_cpu_starting()
2835 	if (!cpuc->shared_regs)  in intel_pmu_cpu_starting()
2839 		void **onln = &cpuc->kfree_on_online[X86_PERF_KFREE_SHARED];  in intel_pmu_cpu_starting()
2846 				*onln = cpuc->shared_regs;  in intel_pmu_cpu_starting()
2847 				cpuc->shared_regs = pc;  in intel_pmu_cpu_starting()
2851 		cpuc->shared_regs->core_id = core_id;  in intel_pmu_cpu_starting()
2852 		cpuc->shared_regs->refcnt++;  in intel_pmu_cpu_starting()
2856 		cpuc->lbr_sel = &cpuc->shared_regs->regs[EXTRA_REG_LBR];  in intel_pmu_cpu_starting()
2864 				cpuc->kfree_on_online[1] = cpuc->excl_cntrs;  in intel_pmu_cpu_starting()
2865 				cpuc->excl_cntrs = c;  in intel_pmu_cpu_starting()
2866 				cpuc->excl_thread_id = 1;  in intel_pmu_cpu_starting()
2870 		cpuc->excl_cntrs->core_id = core_id;  in intel_pmu_cpu_starting()
2871 		cpuc->excl_cntrs->refcnt++;  in intel_pmu_cpu_starting()
2877 	struct cpu_hw_events *cpuc = &per_cpu(cpu_hw_events, cpu);  in free_excl_cntrs()  local
2880 	c = cpuc->excl_cntrs;  in free_excl_cntrs()
2884 		cpuc->excl_cntrs = NULL;  in free_excl_cntrs()
2885 		kfree(cpuc->constraint_list);  in free_excl_cntrs()
2886 		cpuc->constraint_list = NULL;  in free_excl_cntrs()
2892 	struct cpu_hw_events *cpuc = &per_cpu(cpu_hw_events, cpu);  in intel_pmu_cpu_dying()  local
2895 	pc = cpuc->shared_regs;  in intel_pmu_cpu_dying()
2899 		cpuc->shared_regs = NULL;  in intel_pmu_cpu_dying()