108 static int sf_buffer_available(struct cpu_hw_sf *cpuhw)  in sf_buffer_available()  argument
110 	return !!cpuhw->sfb.sdbt;  in sf_buffer_available()
360 static void deallocate_buffers(struct cpu_hw_sf *cpuhw)  in deallocate_buffers()  argument
362 	if (cpuhw->sfb.sdbt)  in deallocate_buffers()
363 		free_sampling_buffer(&cpuhw->sfb);  in deallocate_buffers()
366 static int allocate_buffers(struct cpu_hw_sf *cpuhw, struct hw_perf_event *hwc)  in allocate_buffers()  argument
388 	sfr_size = ALIGN((sizeof(*sfr) - sizeof(sfr->diag) + cpuhw->qsi.dsdes) +  in allocate_buffers()
395 	sfr->bsdes = cpuhw->qsi.bsdes;  in allocate_buffers()
396 	sfr->dsdes = cpuhw->qsi.dsdes;  in allocate_buffers()
424 	freq = sample_rate_to_freq(&cpuhw->qsi, SAMPL_RATE(hwc));  in allocate_buffers()
439 	if (sf_buffer_available(cpuhw))  in allocate_buffers()
446 			    sample_size, cpuhw);  in allocate_buffers()
448 	return alloc_sampling_buffer(&cpuhw->sfb,  in allocate_buffers()
449 				     sfb_pending_allocs(&cpuhw->sfb, hwc));  in allocate_buffers()
481 static void sfb_account_overflows(struct cpu_hw_sf *cpuhw,  in sfb_account_overflows()  argument
496 	ratio = DIV_ROUND_UP(100 * OVERFLOW_REG(hwc) * cpuhw->sfb.num_sdb,  in sfb_account_overflows()
497 			     sample_rate_to_freq(&cpuhw->qsi, SAMPL_RATE(hwc)));  in sfb_account_overflows()
500 	num = compute_sfb_extent(ratio, cpuhw->sfb.num_sdb);  in sfb_account_overflows()
670 	struct cpu_hw_sf *cpuhw;  in __hw_perf_event_init()  local
702 	cpuhw = NULL;  in __hw_perf_event_init()
709 		cpuhw = &per_cpu(cpu_hw_sf, event->cpu);  in __hw_perf_event_init()
710 		si = cpuhw->qsi;  in __hw_perf_event_init()
783 	if (cpuhw)  in __hw_perf_event_init()
785 		err = allocate_buffers(cpuhw, hwc);  in __hw_perf_event_init()
791 			cpuhw = &per_cpu(cpu_hw_sf, cpu);  in __hw_perf_event_init()
792 			err = allocate_buffers(cpuhw, hwc);  in __hw_perf_event_init()
852 	struct cpu_hw_sf *cpuhw = this_cpu_ptr(&cpu_hw_sf);  in cpumsf_pmu_enable()  local
856 	if (cpuhw->flags & PMU_F_ENABLED)  in cpumsf_pmu_enable()
859 	if (cpuhw->flags & PMU_F_ERR_MASK)  in cpumsf_pmu_enable()
873 	if (cpuhw->event) {  in cpumsf_pmu_enable()
874 		hwc = &cpuhw->event->hw;  in cpumsf_pmu_enable()
876 		sfb_account_overflows(cpuhw, hwc);  in cpumsf_pmu_enable()
877 		if (sfb_has_pending_allocs(&cpuhw->sfb, hwc))  in cpumsf_pmu_enable()
878 			extend_sampling_buffer(&cpuhw->sfb, hwc);  in cpumsf_pmu_enable()
882 	cpuhw->flags |= PMU_F_ENABLED;  in cpumsf_pmu_enable()
885 	err = lsctl(&cpuhw->lsctl);  in cpumsf_pmu_enable()
887 		cpuhw->flags &= ~PMU_F_ENABLED;  in cpumsf_pmu_enable()
894 			    "tear=%p dear=%p\n", cpuhw->lsctl.es, cpuhw->lsctl.cs,  in cpumsf_pmu_enable()
895 			    cpuhw->lsctl.ed, cpuhw->lsctl.cd,  in cpumsf_pmu_enable()
896 			    (void *) cpuhw->lsctl.tear, (void *) cpuhw->lsctl.dear);  in cpumsf_pmu_enable()
901 	struct cpu_hw_sf *cpuhw = this_cpu_ptr(&cpu_hw_sf);  in cpumsf_pmu_disable()  local
906 	if (!(cpuhw->flags & PMU_F_ENABLED))  in cpumsf_pmu_disable()
909 	if (cpuhw->flags & PMU_F_ERR_MASK)  in cpumsf_pmu_disable()
913 	inactive = cpuhw->lsctl;  in cpumsf_pmu_disable()
932 			cpuhw->lsctl.tear = si.tear;  in cpumsf_pmu_disable()
933 			cpuhw->lsctl.dear = si.dear;  in cpumsf_pmu_disable()
939 	cpuhw->flags &= ~PMU_F_ENABLED;  in cpumsf_pmu_disable()
1308 	struct cpu_hw_sf *cpuhw = this_cpu_ptr(&cpu_hw_sf);  in cpumsf_pmu_start()  local
1318 	cpuhw->lsctl.cs = 1;  in cpumsf_pmu_start()
1320 		cpuhw->lsctl.cd = 1;  in cpumsf_pmu_start()
1329 	struct cpu_hw_sf *cpuhw = this_cpu_ptr(&cpu_hw_sf);  in cpumsf_pmu_stop()  local
1335 	cpuhw->lsctl.cs = 0;  in cpumsf_pmu_stop()
1336 	cpuhw->lsctl.cd = 0;  in cpumsf_pmu_stop()
1348 	struct cpu_hw_sf *cpuhw = this_cpu_ptr(&cpu_hw_sf);  in cpumsf_pmu_add()  local
1351 	if (cpuhw->flags & PMU_F_IN_USE)  in cpumsf_pmu_add()
1354 	if (!cpuhw->sfb.sdbt)  in cpumsf_pmu_add()
1367 	cpuhw->lsctl.s = 0;  in cpumsf_pmu_add()
1368 	cpuhw->lsctl.h = 1;  in cpumsf_pmu_add()
1369 	cpuhw->lsctl.tear = (unsigned long) cpuhw->sfb.sdbt;  in cpumsf_pmu_add()
1370 	cpuhw->lsctl.dear = *(unsigned long *) cpuhw->sfb.sdbt;  in cpumsf_pmu_add()
1371 	cpuhw->lsctl.interval = SAMPL_RATE(&event->hw);  in cpumsf_pmu_add()
1372 	hw_reset_registers(&event->hw, cpuhw->sfb.sdbt);  in cpumsf_pmu_add()
1376 	if (WARN_ON_ONCE(cpuhw->lsctl.es == 1 || cpuhw->lsctl.ed == 1)) {  in cpumsf_pmu_add()
1380 	cpuhw->lsctl.es = 1;  in cpumsf_pmu_add()
1382 		cpuhw->lsctl.ed = 1;  in cpumsf_pmu_add()
1385 	cpuhw->event = event;  in cpumsf_pmu_add()
1386 	cpuhw->flags |= PMU_F_IN_USE;  in cpumsf_pmu_add()
1398 	struct cpu_hw_sf *cpuhw = this_cpu_ptr(&cpu_hw_sf);  in cpumsf_pmu_del()  local
1403 	cpuhw->lsctl.es = 0;  in cpumsf_pmu_del()
1404 	cpuhw->lsctl.ed = 0;  in cpumsf_pmu_del()
1405 	cpuhw->flags &= ~PMU_F_IN_USE;  in cpumsf_pmu_del()
1406 	cpuhw->event = NULL;  in cpumsf_pmu_del()
1460 	struct cpu_hw_sf *cpuhw;  in cpumf_measurement_alert()  local
1465 	cpuhw = this_cpu_ptr(&cpu_hw_sf);  in cpumf_measurement_alert()
1469 	if (!(cpuhw->flags & PMU_F_RESERVED))  in cpumf_measurement_alert()
1477 		if (cpuhw->flags & PMU_F_IN_USE)  in cpumf_measurement_alert()
1478 			hw_perf_event_update(cpuhw->event, 0);  in cpumf_measurement_alert()
1480 			WARN_ON_ONCE(!(cpuhw->flags & PMU_F_IN_USE));  in cpumf_measurement_alert()
1489 		qsi(&cpuhw->qsi);  in cpumf_measurement_alert()
1494 		cpuhw->flags |= PMU_F_ERR_LSDA;  in cpumf_measurement_alert()
1502 		cpuhw->flags |= PMU_F_ERR_IBE;  in cpumf_measurement_alert()