131 static void intel_pmu_lbr_filter(struct cpu_hw_events *cpuc);
140 	struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);  in __intel_pmu_lbr_enable()  local
154 	if (cpuc->lbr_sel)  in __intel_pmu_lbr_enable()
155 		lbr_select = cpuc->lbr_sel->config;  in __intel_pmu_lbr_enable()
280 	struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);  in intel_pmu_lbr_sched_task()  local
292 			cpuc->lbr_context = ctx;  in intel_pmu_lbr_sched_task()
313 		cpuc->lbr_context = ctx;  in intel_pmu_lbr_sched_task()
324 	struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);  in intel_pmu_lbr_enable()  local
334 	if (event->ctx->task && cpuc->lbr_context != event->ctx) {  in intel_pmu_lbr_enable()
336 		cpuc->lbr_context = event->ctx;  in intel_pmu_lbr_enable()
338 	cpuc->br_sel = event->hw.branch_reg.reg;  in intel_pmu_lbr_enable()
340 	if (branch_user_callstack(cpuc->br_sel) && event->ctx &&  in intel_pmu_lbr_enable()
346 	cpuc->lbr_users++;  in intel_pmu_lbr_enable()
352 	struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);  in intel_pmu_lbr_disable()  local
358 	if (branch_user_callstack(cpuc->br_sel) && event->ctx &&  in intel_pmu_lbr_disable()
364 	cpuc->lbr_users--;  in intel_pmu_lbr_disable()
365 	WARN_ON_ONCE(cpuc->lbr_users < 0);  in intel_pmu_lbr_disable()
368 	if (cpuc->enabled && !cpuc->lbr_users) {  in intel_pmu_lbr_disable()
371 		cpuc->lbr_context = NULL;  in intel_pmu_lbr_disable()
377 	struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);  in intel_pmu_lbr_enable_all()  local
379 	if (cpuc->lbr_users)  in intel_pmu_lbr_enable_all()
385 	struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);  in intel_pmu_lbr_disable_all()  local
387 	if (cpuc->lbr_users)  in intel_pmu_lbr_disable_all()
391 static void intel_pmu_lbr_read_32(struct cpu_hw_events *cpuc)  in intel_pmu_lbr_read_32()  argument
409 		cpuc->lbr_entries[i].from	= msr_lastbranch.from;  in intel_pmu_lbr_read_32()
410 		cpuc->lbr_entries[i].to		= msr_lastbranch.to;  in intel_pmu_lbr_read_32()
411 		cpuc->lbr_entries[i].mispred	= 0;  in intel_pmu_lbr_read_32()
412 		cpuc->lbr_entries[i].predicted	= 0;  in intel_pmu_lbr_read_32()
413 		cpuc->lbr_entries[i].reserved	= 0;  in intel_pmu_lbr_read_32()
415 	cpuc->lbr_stack.nr = i;  in intel_pmu_lbr_read_32()
423 static void intel_pmu_lbr_read_64(struct cpu_hw_events *cpuc)  in intel_pmu_lbr_read_64()  argument
432 	if (cpuc->lbr_sel->config & LBR_CALL_STACK)  in intel_pmu_lbr_read_64()
478 		cpuc->lbr_entries[out].from	 = from;  in intel_pmu_lbr_read_64()
479 		cpuc->lbr_entries[out].to	 = to;  in intel_pmu_lbr_read_64()
480 		cpuc->lbr_entries[out].mispred	 = mis;  in intel_pmu_lbr_read_64()
481 		cpuc->lbr_entries[out].predicted = pred;  in intel_pmu_lbr_read_64()
482 		cpuc->lbr_entries[out].in_tx	 = in_tx;  in intel_pmu_lbr_read_64()
483 		cpuc->lbr_entries[out].abort	 = abort;  in intel_pmu_lbr_read_64()
484 		cpuc->lbr_entries[out].cycles	 = cycles;  in intel_pmu_lbr_read_64()
485 		cpuc->lbr_entries[out].reserved	 = 0;  in intel_pmu_lbr_read_64()
488 	cpuc->lbr_stack.nr = out;  in intel_pmu_lbr_read_64()
493 	struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);  in intel_pmu_lbr_read()  local
495 	if (!cpuc->lbr_users)  in intel_pmu_lbr_read()
499 		intel_pmu_lbr_read_32(cpuc);  in intel_pmu_lbr_read()
501 		intel_pmu_lbr_read_64(cpuc);  in intel_pmu_lbr_read()
503 	intel_pmu_lbr_filter(cpuc);  in intel_pmu_lbr_read()
815 intel_pmu_lbr_filter(struct cpu_hw_events *cpuc)  in intel_pmu_lbr_filter()  argument
818 	int br_sel = cpuc->br_sel;  in intel_pmu_lbr_filter()
826 	for (i = 0; i < cpuc->lbr_stack.nr; i++) {  in intel_pmu_lbr_filter()
828 		from = cpuc->lbr_entries[i].from;  in intel_pmu_lbr_filter()
829 		to = cpuc->lbr_entries[i].to;  in intel_pmu_lbr_filter()
831 		type = branch_type(from, to, cpuc->lbr_entries[i].abort);  in intel_pmu_lbr_filter()
833 			if (cpuc->lbr_entries[i].in_tx)  in intel_pmu_lbr_filter()
841 			cpuc->lbr_entries[i].from = 0;  in intel_pmu_lbr_filter()
850 	for (i = 0; i < cpuc->lbr_stack.nr; ) {  in intel_pmu_lbr_filter()
851 		if (!cpuc->lbr_entries[i].from) {  in intel_pmu_lbr_filter()
853 			while (++j < cpuc->lbr_stack.nr)  in intel_pmu_lbr_filter()
854 				cpuc->lbr_entries[j-1] = cpuc->lbr_entries[j];  in intel_pmu_lbr_filter()
855 			cpuc->lbr_stack.nr--;  in intel_pmu_lbr_filter()
856 			if (!cpuc->lbr_entries[i].from)  in intel_pmu_lbr_filter()