120 static void __clocksource_change_rating(struct clocksource *cs, int rating);
137 static void __clocksource_unstable(struct clocksource *cs)  in __clocksource_unstable()  argument
139 	cs->flags &= ~(CLOCK_SOURCE_VALID_FOR_HRES | CLOCK_SOURCE_WATCHDOG);  in __clocksource_unstable()
140 	cs->flags |= CLOCK_SOURCE_UNSTABLE;  in __clocksource_unstable()
154 void clocksource_mark_unstable(struct clocksource *cs)  in clocksource_mark_unstable()  argument
159 	if (!(cs->flags & CLOCK_SOURCE_UNSTABLE)) {  in clocksource_mark_unstable()
160 		if (list_empty(&cs->wd_list))  in clocksource_mark_unstable()
161 			list_add(&cs->wd_list, &watchdog_list);  in clocksource_mark_unstable()
162 		__clocksource_unstable(cs);  in clocksource_mark_unstable()
169 	struct clocksource *cs;  in clocksource_watchdog()  local
180 	list_for_each_entry(cs, &watchdog_list, wd_list) {  in clocksource_watchdog()
183 		if (cs->flags & CLOCK_SOURCE_UNSTABLE) {  in clocksource_watchdog()
190 		csnow = cs->read(cs);  in clocksource_watchdog()
195 		if (!(cs->flags & CLOCK_SOURCE_WATCHDOG) ||  in clocksource_watchdog()
197 			cs->flags |= CLOCK_SOURCE_WATCHDOG;  in clocksource_watchdog()
198 			cs->wd_last = wdnow;  in clocksource_watchdog()
199 			cs->cs_last = csnow;  in clocksource_watchdog()
203 		delta = clocksource_delta(wdnow, cs->wd_last, watchdog->mask);  in clocksource_watchdog()
207 		delta = clocksource_delta(csnow, cs->cs_last, cs->mask);  in clocksource_watchdog()
208 		cs_nsec = clocksource_cyc2ns(delta, cs->mult, cs->shift);  in clocksource_watchdog()
209 		wdlast = cs->wd_last; /* save these in case we print them */  in clocksource_watchdog()
210 		cslast = cs->cs_last;  in clocksource_watchdog()
211 		cs->cs_last = csnow;  in clocksource_watchdog()
212 		cs->wd_last = wdnow;  in clocksource_watchdog()
219 …eping watchdog: Marking clocksource '%s' as unstable, because the skew is too large:\n", cs->name);  in clocksource_watchdog()
223 				cs->name, csnow, cslast, cs->mask);  in clocksource_watchdog()
224 			__clocksource_unstable(cs);  in clocksource_watchdog()
228 		if (!(cs->flags & CLOCK_SOURCE_VALID_FOR_HRES) &&  in clocksource_watchdog()
229 		    (cs->flags & CLOCK_SOURCE_IS_CONTINUOUS) &&  in clocksource_watchdog()
232 			cs->flags |= CLOCK_SOURCE_VALID_FOR_HRES;  in clocksource_watchdog()
249 			if (cs != curr_clocksource) {  in clocksource_watchdog()
250 				cs->flags |= CLOCK_SOURCE_RESELECT;  in clocksource_watchdog()
299 	struct clocksource *cs;  in clocksource_reset_watchdog()  local
301 	list_for_each_entry(cs, &watchdog_list, wd_list)  in clocksource_reset_watchdog()
302 		cs->flags &= ~CLOCK_SOURCE_WATCHDOG;  in clocksource_reset_watchdog()
310 static void clocksource_enqueue_watchdog(struct clocksource *cs)  in clocksource_enqueue_watchdog()  argument
315 	if (cs->flags & CLOCK_SOURCE_MUST_VERIFY) {  in clocksource_enqueue_watchdog()
317 		list_add(&cs->wd_list, &watchdog_list);  in clocksource_enqueue_watchdog()
318 		cs->flags &= ~CLOCK_SOURCE_WATCHDOG;  in clocksource_enqueue_watchdog()
321 		if (cs->flags & CLOCK_SOURCE_IS_CONTINUOUS)  in clocksource_enqueue_watchdog()
322 			cs->flags |= CLOCK_SOURCE_VALID_FOR_HRES;  in clocksource_enqueue_watchdog()
324 		if (!watchdog || cs->rating > watchdog->rating) {  in clocksource_enqueue_watchdog()
325 			watchdog = cs;  in clocksource_enqueue_watchdog()
335 static void clocksource_dequeue_watchdog(struct clocksource *cs)  in clocksource_dequeue_watchdog()  argument
340 	if (cs != watchdog) {  in clocksource_dequeue_watchdog()
341 		if (cs->flags & CLOCK_SOURCE_MUST_VERIFY) {  in clocksource_dequeue_watchdog()
343 			list_del_init(&cs->wd_list);  in clocksource_dequeue_watchdog()
353 	struct clocksource *cs, *tmp;  in __clocksource_watchdog_kthread()  local
359 	list_for_each_entry_safe(cs, tmp, &watchdog_list, wd_list) {  in __clocksource_watchdog_kthread()
360 		if (cs->flags & CLOCK_SOURCE_UNSTABLE) {  in __clocksource_watchdog_kthread()
361 			list_del_init(&cs->wd_list);  in __clocksource_watchdog_kthread()
362 			list_add(&cs->wd_list, &unstable);  in __clocksource_watchdog_kthread()
365 		if (cs->flags & CLOCK_SOURCE_RESELECT) {  in __clocksource_watchdog_kthread()
366 			cs->flags &= ~CLOCK_SOURCE_RESELECT;  in __clocksource_watchdog_kthread()
375 	list_for_each_entry_safe(cs, tmp, &unstable, wd_list) {  in __clocksource_watchdog_kthread()
376 		list_del_init(&cs->wd_list);  in __clocksource_watchdog_kthread()
377 		__clocksource_change_rating(cs, 0);  in __clocksource_watchdog_kthread()
391 static bool clocksource_is_watchdog(struct clocksource *cs)  in clocksource_is_watchdog()  argument
393 	return cs == watchdog;  in clocksource_is_watchdog()
398 static void clocksource_enqueue_watchdog(struct clocksource *cs)  in clocksource_enqueue_watchdog()  argument
400 	if (cs->flags & CLOCK_SOURCE_IS_CONTINUOUS)  in clocksource_enqueue_watchdog()
401 		cs->flags |= CLOCK_SOURCE_VALID_FOR_HRES;  in clocksource_enqueue_watchdog()
404 static inline void clocksource_dequeue_watchdog(struct clocksource *cs) { }  in clocksource_dequeue_watchdog()  argument
407 static bool clocksource_is_watchdog(struct clocksource *cs) { return false; }  in clocksource_is_watchdog()  argument
408 void clocksource_mark_unstable(struct clocksource *cs) { }  in clocksource_mark_unstable()  argument
417 	struct clocksource *cs;  in clocksource_suspend()  local
419 	list_for_each_entry_reverse(cs, &clocksource_list, list)  in clocksource_suspend()
420 		if (cs->suspend)  in clocksource_suspend()
421 			cs->suspend(cs);  in clocksource_suspend()
429 	struct clocksource *cs;  in clocksource_resume()  local
431 	list_for_each_entry(cs, &clocksource_list, list)  in clocksource_resume()
432 		if (cs->resume)  in clocksource_resume()
433 			cs->resume(cs);  in clocksource_resume()
455 static u32 clocksource_max_adjustment(struct clocksource *cs)  in clocksource_max_adjustment()  argument
461 	ret = (u64)cs->mult * 11;  in clocksource_max_adjustment()
516 static inline void clocksource_update_max_deferment(struct clocksource *cs)  in clocksource_update_max_deferment()  argument
518 	cs->max_idle_ns = clocks_calc_max_nsecs(cs->mult, cs->shift,  in clocksource_update_max_deferment()
519 						cs->maxadj, cs->mask,  in clocksource_update_max_deferment()
520 						&cs->max_cycles);  in clocksource_update_max_deferment()
527 	struct clocksource *cs;  in clocksource_find_best()  local
537 	list_for_each_entry(cs, &clocksource_list, list) {  in clocksource_find_best()
538 		if (skipcur && cs == curr_clocksource)  in clocksource_find_best()
540 		if (oneshot && !(cs->flags & CLOCK_SOURCE_VALID_FOR_HRES))  in clocksource_find_best()
542 		return cs;  in clocksource_find_best()
550 	struct clocksource *best, *cs;  in __clocksource_select()  local
558 	list_for_each_entry(cs, &clocksource_list, list) {  in __clocksource_select()
559 		if (skipcur && cs == curr_clocksource)  in __clocksource_select()
561 		if (strcmp(cs->name, override_name) != 0)  in __clocksource_select()
568 		if (!(cs->flags & CLOCK_SOURCE_VALID_FOR_HRES) && oneshot) {  in __clocksource_select()
572 			       "HRT/NOHZ mode\n", cs->name);  in __clocksource_select()
576 			best = cs;  in __clocksource_select()
636 static void clocksource_enqueue(struct clocksource *cs)  in clocksource_enqueue()  argument
643 		if (tmp->rating >= cs->rating)  in clocksource_enqueue()
645 	list_add(&cs->list, entry);  in clocksource_enqueue()
660 void __clocksource_update_freq_scale(struct clocksource *cs, u32 scale, u32 freq)  in __clocksource_update_freq_scale()  argument
678 		sec = cs->mask;  in __clocksource_update_freq_scale()
683 		else if (sec > 600 && cs->mask > UINT_MAX)  in __clocksource_update_freq_scale()
686 		clocks_calc_mult_shift(&cs->mult, &cs->shift, freq,  in __clocksource_update_freq_scale()
693 	cs->maxadj = clocksource_max_adjustment(cs);  in __clocksource_update_freq_scale()
694 	while (freq && ((cs->mult + cs->maxadj < cs->mult)  in __clocksource_update_freq_scale()
695 		|| (cs->mult - cs->maxadj > cs->mult))) {  in __clocksource_update_freq_scale()
696 		cs->mult >>= 1;  in __clocksource_update_freq_scale()
697 		cs->shift--;  in __clocksource_update_freq_scale()
698 		cs->maxadj = clocksource_max_adjustment(cs);  in __clocksource_update_freq_scale()
705 	WARN_ONCE(cs->mult + cs->maxadj < cs->mult,  in __clocksource_update_freq_scale()
707 		cs->name);  in __clocksource_update_freq_scale()
709 	clocksource_update_max_deferment(cs);  in __clocksource_update_freq_scale()
712 			cs->name, cs->mask, cs->max_cycles, cs->max_idle_ns);  in __clocksource_update_freq_scale()
727 int __clocksource_register_scale(struct clocksource *cs, u32 scale, u32 freq)  in __clocksource_register_scale()  argument
731 	__clocksource_update_freq_scale(cs, scale, freq);  in __clocksource_register_scale()
735 	clocksource_enqueue(cs);  in __clocksource_register_scale()
736 	clocksource_enqueue_watchdog(cs);  in __clocksource_register_scale()
743 static void __clocksource_change_rating(struct clocksource *cs, int rating)  in __clocksource_change_rating()  argument
745 	list_del(&cs->list);  in __clocksource_change_rating()
746 	cs->rating = rating;  in __clocksource_change_rating()
747 	clocksource_enqueue(cs);  in __clocksource_change_rating()
755 void clocksource_change_rating(struct clocksource *cs, int rating)  in clocksource_change_rating()  argument
758 	__clocksource_change_rating(cs, rating);  in clocksource_change_rating()
767 static int clocksource_unbind(struct clocksource *cs)  in clocksource_unbind()  argument
773 	if (clocksource_is_watchdog(cs))  in clocksource_unbind()
776 	if (cs == curr_clocksource) {  in clocksource_unbind()
779 		if (curr_clocksource == cs)  in clocksource_unbind()
782 	clocksource_dequeue_watchdog(cs);  in clocksource_unbind()
783 	list_del_init(&cs->list);  in clocksource_unbind()
791 int clocksource_unregister(struct clocksource *cs)  in clocksource_unregister()  argument
796 	if (!list_empty(&cs->list))  in clocksource_unregister()
797 		ret = clocksource_unbind(cs);  in clocksource_unregister()
882 	struct clocksource *cs;  in sysfs_unbind_clocksource()  local
892 	list_for_each_entry(cs, &clocksource_list, list) {  in sysfs_unbind_clocksource()
893 		if (strcmp(cs->name, name))  in sysfs_unbind_clocksource()
895 		ret = clocksource_unbind(cs);  in sysfs_unbind_clocksource()