sg_policy          49 kernel/sched/cpufreq_schedutil.c 	struct sugov_policy	*sg_policy;
sg_policy          69 kernel/sched/cpufreq_schedutil.c static bool sugov_should_update_freq(struct sugov_policy *sg_policy, u64 time)
sg_policy          88 kernel/sched/cpufreq_schedutil.c 	if (!cpufreq_this_cpu_can_update(sg_policy->policy))
sg_policy          91 kernel/sched/cpufreq_schedutil.c 	if (unlikely(sg_policy->limits_changed)) {
sg_policy          92 kernel/sched/cpufreq_schedutil.c 		sg_policy->limits_changed = false;
sg_policy          93 kernel/sched/cpufreq_schedutil.c 		sg_policy->need_freq_update = true;
sg_policy          97 kernel/sched/cpufreq_schedutil.c 	delta_ns = time - sg_policy->last_freq_update_time;
sg_policy          99 kernel/sched/cpufreq_schedutil.c 	return delta_ns >= sg_policy->freq_update_delay_ns;
sg_policy         102 kernel/sched/cpufreq_schedutil.c static bool sugov_update_next_freq(struct sugov_policy *sg_policy, u64 time,
sg_policy         105 kernel/sched/cpufreq_schedutil.c 	if (sg_policy->next_freq == next_freq)
sg_policy         108 kernel/sched/cpufreq_schedutil.c 	sg_policy->next_freq = next_freq;
sg_policy         109 kernel/sched/cpufreq_schedutil.c 	sg_policy->last_freq_update_time = time;
sg_policy         114 kernel/sched/cpufreq_schedutil.c static void sugov_fast_switch(struct sugov_policy *sg_policy, u64 time,
sg_policy         117 kernel/sched/cpufreq_schedutil.c 	struct cpufreq_policy *policy = sg_policy->policy;
sg_policy         120 kernel/sched/cpufreq_schedutil.c 	if (!sugov_update_next_freq(sg_policy, time, next_freq))
sg_policy         135 kernel/sched/cpufreq_schedutil.c static void sugov_deferred_update(struct sugov_policy *sg_policy, u64 time,
sg_policy         138 kernel/sched/cpufreq_schedutil.c 	if (!sugov_update_next_freq(sg_policy, time, next_freq))
sg_policy         141 kernel/sched/cpufreq_schedutil.c 	if (!sg_policy->work_in_progress) {
sg_policy         142 kernel/sched/cpufreq_schedutil.c 		sg_policy->work_in_progress = true;
sg_policy         143 kernel/sched/cpufreq_schedutil.c 		irq_work_queue(&sg_policy->irq_work);
sg_policy         169 kernel/sched/cpufreq_schedutil.c static unsigned int get_next_freq(struct sugov_policy *sg_policy,
sg_policy         172 kernel/sched/cpufreq_schedutil.c 	struct cpufreq_policy *policy = sg_policy->policy;
sg_policy         178 kernel/sched/cpufreq_schedutil.c 	if (freq == sg_policy->cached_raw_freq && !sg_policy->need_freq_update)
sg_policy         179 kernel/sched/cpufreq_schedutil.c 		return sg_policy->next_freq;
sg_policy         181 kernel/sched/cpufreq_schedutil.c 	sg_policy->need_freq_update = false;
sg_policy         182 kernel/sched/cpufreq_schedutil.c 	sg_policy->cached_raw_freq = freq;
sg_policy         444 kernel/sched/cpufreq_schedutil.c static inline void ignore_dl_rate_limit(struct sugov_cpu *sg_cpu, struct sugov_policy *sg_policy)
sg_policy         447 kernel/sched/cpufreq_schedutil.c 		sg_policy->limits_changed = true;
sg_policy         454 kernel/sched/cpufreq_schedutil.c 	struct sugov_policy *sg_policy = sg_cpu->sg_policy;
sg_policy         462 kernel/sched/cpufreq_schedutil.c 	ignore_dl_rate_limit(sg_cpu, sg_policy);
sg_policy         464 kernel/sched/cpufreq_schedutil.c 	if (!sugov_should_update_freq(sg_policy, time))
sg_policy         468 kernel/sched/cpufreq_schedutil.c 	busy = !sg_policy->need_freq_update && sugov_cpu_is_busy(sg_cpu);
sg_policy         473 kernel/sched/cpufreq_schedutil.c 	next_f = get_next_freq(sg_policy, util, max);
sg_policy         478 kernel/sched/cpufreq_schedutil.c 	if (busy && next_f < sg_policy->next_freq) {
sg_policy         479 kernel/sched/cpufreq_schedutil.c 		next_f = sg_policy->next_freq;
sg_policy         482 kernel/sched/cpufreq_schedutil.c 		sg_policy->cached_raw_freq = 0;
sg_policy         490 kernel/sched/cpufreq_schedutil.c 	if (sg_policy->policy->fast_switch_enabled) {
sg_policy         491 kernel/sched/cpufreq_schedutil.c 		sugov_fast_switch(sg_policy, time, next_f);
sg_policy         493 kernel/sched/cpufreq_schedutil.c 		raw_spin_lock(&sg_policy->update_lock);
sg_policy         494 kernel/sched/cpufreq_schedutil.c 		sugov_deferred_update(sg_policy, time, next_f);
sg_policy         495 kernel/sched/cpufreq_schedutil.c 		raw_spin_unlock(&sg_policy->update_lock);
sg_policy         501 kernel/sched/cpufreq_schedutil.c 	struct sugov_policy *sg_policy = sg_cpu->sg_policy;
sg_policy         502 kernel/sched/cpufreq_schedutil.c 	struct cpufreq_policy *policy = sg_policy->policy;
sg_policy         520 kernel/sched/cpufreq_schedutil.c 	return get_next_freq(sg_policy, util, max);
sg_policy         527 kernel/sched/cpufreq_schedutil.c 	struct sugov_policy *sg_policy = sg_cpu->sg_policy;
sg_policy         530 kernel/sched/cpufreq_schedutil.c 	raw_spin_lock(&sg_policy->update_lock);
sg_policy         535 kernel/sched/cpufreq_schedutil.c 	ignore_dl_rate_limit(sg_cpu, sg_policy);
sg_policy         537 kernel/sched/cpufreq_schedutil.c 	if (sugov_should_update_freq(sg_policy, time)) {
sg_policy         540 kernel/sched/cpufreq_schedutil.c 		if (sg_policy->policy->fast_switch_enabled)
sg_policy         541 kernel/sched/cpufreq_schedutil.c 			sugov_fast_switch(sg_policy, time, next_f);
sg_policy         543 kernel/sched/cpufreq_schedutil.c 			sugov_deferred_update(sg_policy, time, next_f);
sg_policy         546 kernel/sched/cpufreq_schedutil.c 	raw_spin_unlock(&sg_policy->update_lock);
sg_policy         551 kernel/sched/cpufreq_schedutil.c 	struct sugov_policy *sg_policy = container_of(work, struct sugov_policy, work);
sg_policy         565 kernel/sched/cpufreq_schedutil.c 	raw_spin_lock_irqsave(&sg_policy->update_lock, flags);
sg_policy         566 kernel/sched/cpufreq_schedutil.c 	freq = sg_policy->next_freq;
sg_policy         567 kernel/sched/cpufreq_schedutil.c 	sg_policy->work_in_progress = false;
sg_policy         568 kernel/sched/cpufreq_schedutil.c 	raw_spin_unlock_irqrestore(&sg_policy->update_lock, flags);
sg_policy         570 kernel/sched/cpufreq_schedutil.c 	mutex_lock(&sg_policy->work_lock);
sg_policy         571 kernel/sched/cpufreq_schedutil.c 	__cpufreq_driver_target(sg_policy->policy, freq, CPUFREQ_RELATION_L);
sg_policy         572 kernel/sched/cpufreq_schedutil.c 	mutex_unlock(&sg_policy->work_lock);
sg_policy         577 kernel/sched/cpufreq_schedutil.c 	struct sugov_policy *sg_policy;
sg_policy         579 kernel/sched/cpufreq_schedutil.c 	sg_policy = container_of(irq_work, struct sugov_policy, irq_work);
sg_policy         581 kernel/sched/cpufreq_schedutil.c 	kthread_queue_work(&sg_policy->worker, &sg_policy->work);
sg_policy         605 kernel/sched/cpufreq_schedutil.c 	struct sugov_policy *sg_policy;
sg_policy         613 kernel/sched/cpufreq_schedutil.c 	list_for_each_entry(sg_policy, &attr_set->policy_list, tunables_hook)
sg_policy         614 kernel/sched/cpufreq_schedutil.c 		sg_policy->freq_update_delay_ns = rate_limit_us * NSEC_PER_USEC;
sg_policy         638 kernel/sched/cpufreq_schedutil.c 	struct sugov_policy *sg_policy;
sg_policy         640 kernel/sched/cpufreq_schedutil.c 	sg_policy = kzalloc(sizeof(*sg_policy), GFP_KERNEL);
sg_policy         641 kernel/sched/cpufreq_schedutil.c 	if (!sg_policy)
sg_policy         644 kernel/sched/cpufreq_schedutil.c 	sg_policy->policy = policy;
sg_policy         645 kernel/sched/cpufreq_schedutil.c 	raw_spin_lock_init(&sg_policy->update_lock);
sg_policy         646 kernel/sched/cpufreq_schedutil.c 	return sg_policy;
sg_policy         649 kernel/sched/cpufreq_schedutil.c static void sugov_policy_free(struct sugov_policy *sg_policy)
sg_policy         651 kernel/sched/cpufreq_schedutil.c 	kfree(sg_policy);
sg_policy         654 kernel/sched/cpufreq_schedutil.c static int sugov_kthread_create(struct sugov_policy *sg_policy)
sg_policy         671 kernel/sched/cpufreq_schedutil.c 	struct cpufreq_policy *policy = sg_policy->policy;
sg_policy         678 kernel/sched/cpufreq_schedutil.c 	kthread_init_work(&sg_policy->work, sugov_work);
sg_policy         679 kernel/sched/cpufreq_schedutil.c 	kthread_init_worker(&sg_policy->worker);
sg_policy         680 kernel/sched/cpufreq_schedutil.c 	thread = kthread_create(kthread_worker_fn, &sg_policy->worker,
sg_policy         695 kernel/sched/cpufreq_schedutil.c 	sg_policy->thread = thread;
sg_policy         697 kernel/sched/cpufreq_schedutil.c 	init_irq_work(&sg_policy->irq_work, sugov_irq_work);
sg_policy         698 kernel/sched/cpufreq_schedutil.c 	mutex_init(&sg_policy->work_lock);
sg_policy         705 kernel/sched/cpufreq_schedutil.c static void sugov_kthread_stop(struct sugov_policy *sg_policy)
sg_policy         708 kernel/sched/cpufreq_schedutil.c 	if (sg_policy->policy->fast_switch_enabled)
sg_policy         711 kernel/sched/cpufreq_schedutil.c 	kthread_flush_worker(&sg_policy->worker);
sg_policy         712 kernel/sched/cpufreq_schedutil.c 	kthread_stop(sg_policy->thread);
sg_policy         713 kernel/sched/cpufreq_schedutil.c 	mutex_destroy(&sg_policy->work_lock);
sg_policy         716 kernel/sched/cpufreq_schedutil.c static struct sugov_tunables *sugov_tunables_alloc(struct sugov_policy *sg_policy)
sg_policy         722 kernel/sched/cpufreq_schedutil.c 		gov_attr_set_init(&tunables->attr_set, &sg_policy->tunables_hook);
sg_policy         739 kernel/sched/cpufreq_schedutil.c 	struct sugov_policy *sg_policy;
sg_policy         749 kernel/sched/cpufreq_schedutil.c 	sg_policy = sugov_policy_alloc(policy);
sg_policy         750 kernel/sched/cpufreq_schedutil.c 	if (!sg_policy) {
sg_policy         755 kernel/sched/cpufreq_schedutil.c 	ret = sugov_kthread_create(sg_policy);
sg_policy         766 kernel/sched/cpufreq_schedutil.c 		policy->governor_data = sg_policy;
sg_policy         767 kernel/sched/cpufreq_schedutil.c 		sg_policy->tunables = global_tunables;
sg_policy         769 kernel/sched/cpufreq_schedutil.c 		gov_attr_set_get(&global_tunables->attr_set, &sg_policy->tunables_hook);
sg_policy         773 kernel/sched/cpufreq_schedutil.c 	tunables = sugov_tunables_alloc(sg_policy);
sg_policy         781 kernel/sched/cpufreq_schedutil.c 	policy->governor_data = sg_policy;
sg_policy         782 kernel/sched/cpufreq_schedutil.c 	sg_policy->tunables = tunables;
sg_policy         800 kernel/sched/cpufreq_schedutil.c 	sugov_kthread_stop(sg_policy);
sg_policy         804 kernel/sched/cpufreq_schedutil.c 	sugov_policy_free(sg_policy);
sg_policy         815 kernel/sched/cpufreq_schedutil.c 	struct sugov_policy *sg_policy = policy->governor_data;
sg_policy         816 kernel/sched/cpufreq_schedutil.c 	struct sugov_tunables *tunables = sg_policy->tunables;
sg_policy         821 kernel/sched/cpufreq_schedutil.c 	count = gov_attr_set_put(&tunables->attr_set, &sg_policy->tunables_hook);
sg_policy         828 kernel/sched/cpufreq_schedutil.c 	sugov_kthread_stop(sg_policy);
sg_policy         829 kernel/sched/cpufreq_schedutil.c 	sugov_policy_free(sg_policy);
sg_policy         835 kernel/sched/cpufreq_schedutil.c 	struct sugov_policy *sg_policy = policy->governor_data;
sg_policy         838 kernel/sched/cpufreq_schedutil.c 	sg_policy->freq_update_delay_ns	= sg_policy->tunables->rate_limit_us * NSEC_PER_USEC;
sg_policy         839 kernel/sched/cpufreq_schedutil.c 	sg_policy->last_freq_update_time	= 0;
sg_policy         840 kernel/sched/cpufreq_schedutil.c 	sg_policy->next_freq			= 0;
sg_policy         841 kernel/sched/cpufreq_schedutil.c 	sg_policy->work_in_progress		= false;
sg_policy         842 kernel/sched/cpufreq_schedutil.c 	sg_policy->limits_changed		= false;
sg_policy         843 kernel/sched/cpufreq_schedutil.c 	sg_policy->need_freq_update		= false;
sg_policy         844 kernel/sched/cpufreq_schedutil.c 	sg_policy->cached_raw_freq		= 0;
sg_policy         851 kernel/sched/cpufreq_schedutil.c 		sg_cpu->sg_policy		= sg_policy;
sg_policy         867 kernel/sched/cpufreq_schedutil.c 	struct sugov_policy *sg_policy = policy->governor_data;
sg_policy         876 kernel/sched/cpufreq_schedutil.c 		irq_work_sync(&sg_policy->irq_work);
sg_policy         877 kernel/sched/cpufreq_schedutil.c 		kthread_cancel_work_sync(&sg_policy->work);
sg_policy         883 kernel/sched/cpufreq_schedutil.c 	struct sugov_policy *sg_policy = policy->governor_data;
sg_policy         886 kernel/sched/cpufreq_schedutil.c 		mutex_lock(&sg_policy->work_lock);
sg_policy         888 kernel/sched/cpufreq_schedutil.c 		mutex_unlock(&sg_policy->work_lock);
sg_policy         891 kernel/sched/cpufreq_schedutil.c 	sg_policy->limits_changed = true;