308 static unsigned long sfb_max_limit(struct hw_perf_event *hwc)  in sfb_max_limit()  argument
310 	return SAMPL_DIAG_MODE(hwc) ? CPUM_SF_MAX_SDB * CPUM_SF_SDB_DIAG_FACTOR  in sfb_max_limit()
315 					struct hw_perf_event *hwc)  in sfb_pending_allocs()  argument
318 		return SFB_ALLOC_REG(hwc);  in sfb_pending_allocs()
319 	if (SFB_ALLOC_REG(hwc) > sfb->num_sdb)  in sfb_pending_allocs()
320 		return SFB_ALLOC_REG(hwc) - sfb->num_sdb;  in sfb_pending_allocs()
325 				   struct hw_perf_event *hwc)  in sfb_has_pending_allocs()  argument
327 	return sfb_pending_allocs(sfb, hwc) > 0;  in sfb_has_pending_allocs()
330 static void sfb_account_allocs(unsigned long num, struct hw_perf_event *hwc)  in sfb_account_allocs()  argument
333 	num = min_t(unsigned long, num, sfb_max_limit(hwc) - SFB_ALLOC_REG(hwc));  in sfb_account_allocs()
335 		SFB_ALLOC_REG(hwc) += num;  in sfb_account_allocs()
338 static void sfb_init_allocs(unsigned long num, struct hw_perf_event *hwc)  in sfb_init_allocs()  argument
340 	SFB_ALLOC_REG(hwc) = 0;  in sfb_init_allocs()
341 	sfb_account_allocs(num, hwc);  in sfb_init_allocs()
344 static size_t event_sample_size(struct hw_perf_event *hwc)  in event_sample_size()  argument
346 	struct sf_raw_sample *sfr = (struct sf_raw_sample *) RAWSAMPLE_REG(hwc);  in event_sample_size()
354 	if (SAMPL_DIAG_MODE(hwc))  in event_sample_size()
366 static int allocate_buffers(struct cpu_hw_sf *cpuhw, struct hw_perf_event *hwc)  in allocate_buffers()  argument
397 	RAWSAMPLE_REG(hwc) = (unsigned long) sfr;  in allocate_buffers()
423 	sample_size = event_sample_size(hwc);  in allocate_buffers()
424 	freq = sample_rate_to_freq(&cpuhw->qsi, SAMPL_RATE(hwc));  in allocate_buffers()
438 	sfb_init_allocs(n_sdb, hwc);  in allocate_buffers()
445 			    SAMPL_RATE(hwc), freq, n_sdb, sfb_max_limit(hwc),  in allocate_buffers()
449 				     sfb_pending_allocs(&cpuhw->sfb, hwc));  in allocate_buffers()
482 				  struct hw_perf_event *hwc)  in sfb_account_overflows()  argument
486 	if (!OVERFLOW_REG(hwc))  in sfb_account_overflows()
496 	ratio = DIV_ROUND_UP(100 * OVERFLOW_REG(hwc) * cpuhw->sfb.num_sdb,  in sfb_account_overflows()
497 			     sample_rate_to_freq(&cpuhw->qsi, SAMPL_RATE(hwc)));  in sfb_account_overflows()
502 		sfb_account_allocs(num, hwc);  in sfb_account_overflows()
505 			    " num=%lu\n", OVERFLOW_REG(hwc), ratio, num);  in sfb_account_overflows()
506 	OVERFLOW_REG(hwc) = 0;  in sfb_account_overflows()
521 				   struct hw_perf_event *hwc)  in extend_sampling_buffer()  argument
526 	num = sfb_pending_allocs(sfb, hwc);  in extend_sampling_buffer()
546 	if (sfb_has_pending_allocs(sfb, hwc))  in extend_sampling_buffer()
550 				    sfb_pending_allocs(sfb, hwc));  in extend_sampling_buffer()
640 static void hw_init_period(struct hw_perf_event *hwc, u64 period)  in hw_init_period()  argument
642 	hwc->sample_period = period;  in hw_init_period()
643 	hwc->last_period = hwc->sample_period;  in hw_init_period()
644 	local64_set(&hwc->period_left, hwc->sample_period);  in hw_init_period()
647 static void hw_reset_registers(struct hw_perf_event *hwc,  in hw_reset_registers()  argument
653 	TEAR_REG(hwc) = (unsigned long) sdbt_origin;  in hw_reset_registers()
656 	sfr = (struct sf_raw_sample *) RAWSAMPLE_REG(hwc);  in hw_reset_registers()
673 	struct hw_perf_event *hwc = &event->hw;  in __hw_perf_event_init()  local
723 	SAMPL_FLAGS(hwc) = PERF_CPUM_SF_BASIC_MODE;  in __hw_perf_event_init()
733 		SAMPL_FLAGS(hwc) |= PERF_CPUM_SF_DIAG_MODE;  in __hw_perf_event_init()
738 		SAMPL_FLAGS(hwc) |= PERF_CPUM_SF_FULL_BLOCKS;  in __hw_perf_event_init()
756 		rate = hw_limit_rate(&si, hwc->sample_period);  in __hw_perf_event_init()
771 	SAMPL_RATE(hwc) = rate;  in __hw_perf_event_init()
772 	hw_init_period(hwc, SAMPL_RATE(hwc));  in __hw_perf_event_init()
775 	hwc->extra_reg.reg = REG_OVERFLOW;  in __hw_perf_event_init()
776 	OVERFLOW_REG(hwc) = 0;  in __hw_perf_event_init()
785 		err = allocate_buffers(cpuhw, hwc);  in __hw_perf_event_init()
792 			err = allocate_buffers(cpuhw, hwc);  in __hw_perf_event_init()
853 	struct hw_perf_event *hwc;  in cpumsf_pmu_enable()  local
874 		hwc = &cpuhw->event->hw;  in cpumsf_pmu_enable()
876 		sfb_account_overflows(cpuhw, hwc);  in cpumsf_pmu_enable()
877 		if (sfb_has_pending_allocs(&cpuhw->sfb, hwc))  in cpumsf_pmu_enable()
878 			extend_sampling_buffer(&cpuhw->sfb, hwc);  in cpumsf_pmu_enable()
1217 	struct hw_perf_event *hwc = &event->hw;  in hw_perf_event_update()  local
1223 	if (flush_all && SDB_FULL_BLOCKS(hwc))  in hw_perf_event_update()
1226 	sdbt = (unsigned long *) TEAR_REG(hwc);  in hw_perf_event_update()
1274 		TEAR_REG(hwc) = (unsigned long) sdbt;  in hw_perf_event_update()
1291 		OVERFLOW_REG(hwc) = DIV_ROUND_UP(OVERFLOW_REG(hwc) +  in hw_perf_event_update()