134 static void tick_sched_handle(struct tick_sched *ts, struct pt_regs *regs)  in tick_sched_handle()  argument
145 	if (ts->tick_stopped) {  in tick_sched_handle()
148 			ts->idle_jiffies++;  in tick_sched_handle()
440 update_ts_time_stats(int cpu, struct tick_sched *ts, ktime_t now, u64 *last_update_time)  in update_ts_time_stats()  argument
444 	if (ts->idle_active) {  in update_ts_time_stats()
445 		delta = ktime_sub(now, ts->idle_entrytime);  in update_ts_time_stats()
447 			ts->iowait_sleeptime = ktime_add(ts->iowait_sleeptime, delta);  in update_ts_time_stats()
449 			ts->idle_sleeptime = ktime_add(ts->idle_sleeptime, delta);  in update_ts_time_stats()
450 		ts->idle_entrytime = now;  in update_ts_time_stats()
458 static void tick_nohz_stop_idle(struct tick_sched *ts, ktime_t now)  in tick_nohz_stop_idle()  argument
460 	update_ts_time_stats(smp_processor_id(), ts, now, NULL);  in tick_nohz_stop_idle()
461 	ts->idle_active = 0;  in tick_nohz_stop_idle()
466 static ktime_t tick_nohz_start_idle(struct tick_sched *ts)  in tick_nohz_start_idle()  argument
470 	ts->idle_entrytime = now;  in tick_nohz_start_idle()
471 	ts->idle_active = 1;  in tick_nohz_start_idle()
492 	struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu);  in get_cpu_idle_time_us()  local
500 		update_ts_time_stats(cpu, ts, now, last_update_time);  in get_cpu_idle_time_us()
501 		idle = ts->idle_sleeptime;  in get_cpu_idle_time_us()
503 		if (ts->idle_active && !nr_iowait_cpu(cpu)) {  in get_cpu_idle_time_us()
504 			ktime_t delta = ktime_sub(now, ts->idle_entrytime);  in get_cpu_idle_time_us()
506 			idle = ktime_add(ts->idle_sleeptime, delta);  in get_cpu_idle_time_us()
508 			idle = ts->idle_sleeptime;  in get_cpu_idle_time_us()
533 	struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu);  in get_cpu_iowait_time_us()  local
541 		update_ts_time_stats(cpu, ts, now, last_update_time);  in get_cpu_iowait_time_us()
542 		iowait = ts->iowait_sleeptime;  in get_cpu_iowait_time_us()
544 		if (ts->idle_active && nr_iowait_cpu(cpu) > 0) {  in get_cpu_iowait_time_us()
545 			ktime_t delta = ktime_sub(now, ts->idle_entrytime);  in get_cpu_iowait_time_us()
547 			iowait = ktime_add(ts->iowait_sleeptime, delta);  in get_cpu_iowait_time_us()
549 			iowait = ts->iowait_sleeptime;  in get_cpu_iowait_time_us()
557 static void tick_nohz_restart(struct tick_sched *ts, ktime_t now)  in tick_nohz_restart()  argument
559 	hrtimer_cancel(&ts->sched_timer);  in tick_nohz_restart()
560 	hrtimer_set_expires(&ts->sched_timer, ts->last_tick);  in tick_nohz_restart()
563 	hrtimer_forward(&ts->sched_timer, now, tick_period);  in tick_nohz_restart()
565 	if (ts->nohz_mode == NOHZ_MODE_HIGHRES)  in tick_nohz_restart()
566 		hrtimer_start_expires(&ts->sched_timer, HRTIMER_MODE_ABS_PINNED);  in tick_nohz_restart()
568 		tick_program_event(hrtimer_get_expires(&ts->sched_timer), 1);  in tick_nohz_restart()
571 static ktime_t tick_nohz_stop_sched_tick(struct tick_sched *ts,  in tick_nohz_stop_sched_tick()  argument
585 	ts->last_jiffies = basejiff;  in tick_nohz_stop_sched_tick()
599 		ts->next_timer = next_tmr;  in tick_nohz_stop_sched_tick()
611 		if (!ts->tick_stopped)  in tick_nohz_stop_sched_tick()
615 			tick_nohz_restart(ts, now);  in tick_nohz_stop_sched_tick()
634 		ts->do_timer_last = 1;  in tick_nohz_stop_sched_tick()
637 		ts->do_timer_last = 0;  in tick_nohz_stop_sched_tick()
638 	} else if (!ts->do_timer_last) {  in tick_nohz_stop_sched_tick()
644 	if (!ts->inidle)  in tick_nohz_stop_sched_tick()
658 	if (ts->tick_stopped && (expires == dev->next_event.tv64))  in tick_nohz_stop_sched_tick()
668 	if (!ts->tick_stopped) {  in tick_nohz_stop_sched_tick()
672 		ts->last_tick = hrtimer_get_expires(&ts->sched_timer);  in tick_nohz_stop_sched_tick()
673 		ts->tick_stopped = 1;  in tick_nohz_stop_sched_tick()
682 		if (ts->nohz_mode == NOHZ_MODE_HIGHRES)  in tick_nohz_stop_sched_tick()
683 			hrtimer_cancel(&ts->sched_timer);  in tick_nohz_stop_sched_tick()
687 	if (ts->nohz_mode == NOHZ_MODE_HIGHRES)  in tick_nohz_stop_sched_tick()
688 		hrtimer_start(&ts->sched_timer, tick, HRTIMER_MODE_ABS_PINNED);  in tick_nohz_stop_sched_tick()
693 	ts->sleep_length = ktime_sub(dev->next_event, now);  in tick_nohz_stop_sched_tick()
697 static void tick_nohz_restart_sched_tick(struct tick_sched *ts, ktime_t now)  in tick_nohz_restart_sched_tick()  argument
708 	ts->tick_stopped  = 0;  in tick_nohz_restart_sched_tick()
709 	ts->idle_exittime = now;  in tick_nohz_restart_sched_tick()
711 	tick_nohz_restart(ts, now);  in tick_nohz_restart_sched_tick()
714 static void tick_nohz_full_update_tick(struct tick_sched *ts)  in tick_nohz_full_update_tick()  argument
722 	if (!ts->tick_stopped && ts->nohz_mode == NOHZ_MODE_INACTIVE)  in tick_nohz_full_update_tick()
726 		tick_nohz_stop_sched_tick(ts, ktime_get(), cpu);  in tick_nohz_full_update_tick()
727 	else if (ts->tick_stopped)  in tick_nohz_full_update_tick()
728 		tick_nohz_restart_sched_tick(ts, ktime_get());  in tick_nohz_full_update_tick()
732 static bool can_stop_idle_tick(int cpu, struct tick_sched *ts)  in can_stop_idle_tick()  argument
747 	if (unlikely(ts->nohz_mode == NOHZ_MODE_INACTIVE)) {  in can_stop_idle_tick()
748 		ts->sleep_length = (ktime_t) { .tv64 = NSEC_PER_SEC/HZ };  in can_stop_idle_tick()
785 static void __tick_nohz_idle_enter(struct tick_sched *ts)  in __tick_nohz_idle_enter()  argument
790 	now = tick_nohz_start_idle(ts);  in __tick_nohz_idle_enter()
792 	if (can_stop_idle_tick(cpu, ts)) {  in __tick_nohz_idle_enter()
793 		int was_stopped = ts->tick_stopped;  in __tick_nohz_idle_enter()
795 		ts->idle_calls++;  in __tick_nohz_idle_enter()
797 		expires = tick_nohz_stop_sched_tick(ts, now, cpu);  in __tick_nohz_idle_enter()
799 			ts->idle_sleeps++;  in __tick_nohz_idle_enter()
800 			ts->idle_expires = expires;  in __tick_nohz_idle_enter()
803 		if (!was_stopped && ts->tick_stopped)  in __tick_nohz_idle_enter()
804 			ts->idle_jiffies = ts->last_jiffies;  in __tick_nohz_idle_enter()
822 	struct tick_sched *ts;  in tick_nohz_idle_enter()  local
836 	ts = this_cpu_ptr(&tick_cpu_sched);  in tick_nohz_idle_enter()
837 	ts->inidle = 1;  in tick_nohz_idle_enter()
838 	__tick_nohz_idle_enter(ts);  in tick_nohz_idle_enter()
853 	struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);  in tick_nohz_irq_exit()  local
855 	if (ts->inidle)  in tick_nohz_irq_exit()
856 		__tick_nohz_idle_enter(ts);  in tick_nohz_irq_exit()
858 		tick_nohz_full_update_tick(ts);  in tick_nohz_irq_exit()
868 	struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);  in tick_nohz_get_sleep_length()  local
870 	return ts->sleep_length;  in tick_nohz_get_sleep_length()
873 static void tick_nohz_account_idle_ticks(struct tick_sched *ts)  in tick_nohz_account_idle_ticks()  argument
885 	ticks = jiffies - ts->idle_jiffies;  in tick_nohz_account_idle_ticks()
903 	struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);  in tick_nohz_idle_exit()  local
908 	WARN_ON_ONCE(!ts->inidle);  in tick_nohz_idle_exit()
910 	ts->inidle = 0;  in tick_nohz_idle_exit()
912 	if (ts->idle_active || ts->tick_stopped)  in tick_nohz_idle_exit()
915 	if (ts->idle_active)  in tick_nohz_idle_exit()
916 		tick_nohz_stop_idle(ts, now);  in tick_nohz_idle_exit()
918 	if (ts->tick_stopped) {  in tick_nohz_idle_exit()
919 		tick_nohz_restart_sched_tick(ts, now);  in tick_nohz_idle_exit()
920 		tick_nohz_account_idle_ticks(ts);  in tick_nohz_idle_exit()
931 	struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);  in tick_nohz_handler()  local
938 	tick_sched_handle(ts, regs);  in tick_nohz_handler()
941 	if (unlikely(ts->tick_stopped))  in tick_nohz_handler()
944 	hrtimer_forward(&ts->sched_timer, now, tick_period);  in tick_nohz_handler()
945 	tick_program_event(hrtimer_get_expires(&ts->sched_timer), 1);  in tick_nohz_handler()
948 static inline void tick_nohz_activate(struct tick_sched *ts, int mode)  in tick_nohz_activate()  argument
952 	ts->nohz_mode = mode;  in tick_nohz_activate()
963 	struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);  in tick_nohz_switch_to_nohz()  local
976 	hrtimer_init(&ts->sched_timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS);  in tick_nohz_switch_to_nohz()
980 	hrtimer_set_expires(&ts->sched_timer, next);  in tick_nohz_switch_to_nohz()
981 	hrtimer_forward_now(&ts->sched_timer, tick_period);  in tick_nohz_switch_to_nohz()
982 	tick_program_event(hrtimer_get_expires(&ts->sched_timer), 1);  in tick_nohz_switch_to_nohz()
983 	tick_nohz_activate(ts, NOHZ_MODE_LOWRES);  in tick_nohz_switch_to_nohz()
997 static void tick_nohz_kick_tick(struct tick_sched *ts, ktime_t now)  in tick_nohz_kick_tick()  argument
1007 	delta =	ktime_sub(hrtimer_get_expires(&ts->sched_timer), now);  in tick_nohz_kick_tick()
1011 	tick_nohz_restart(ts, now);  in tick_nohz_kick_tick()
1017 	struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);  in tick_nohz_irq_enter()  local
1020 	if (!ts->idle_active && !ts->tick_stopped)  in tick_nohz_irq_enter()
1023 	if (ts->idle_active)  in tick_nohz_irq_enter()
1024 		tick_nohz_stop_idle(ts, now);  in tick_nohz_irq_enter()
1025 	if (ts->tick_stopped) {  in tick_nohz_irq_enter()
1027 		tick_nohz_kick_tick(ts, now);  in tick_nohz_irq_enter()
1035 static inline void tick_nohz_activate(struct tick_sched *ts, int mode) { }  in tick_nohz_activate()  argument
1058 	struct tick_sched *ts =  in tick_sched_timer()  local
1070 		tick_sched_handle(ts, regs);  in tick_sched_timer()
1073 	if (unlikely(ts->tick_stopped))  in tick_sched_timer()
1096 	struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);  in tick_setup_sched_timer()  local
1102 	hrtimer_init(&ts->sched_timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS);  in tick_setup_sched_timer()
1103 	ts->sched_timer.function = tick_sched_timer;  in tick_setup_sched_timer()
1106 	hrtimer_set_expires(&ts->sched_timer, tick_init_jiffy_update());  in tick_setup_sched_timer()
1113 		hrtimer_add_expires_ns(&ts->sched_timer, offset);  in tick_setup_sched_timer()
1116 	hrtimer_forward(&ts->sched_timer, now, tick_period);  in tick_setup_sched_timer()
1117 	hrtimer_start_expires(&ts->sched_timer, HRTIMER_MODE_ABS_PINNED);  in tick_setup_sched_timer()
1118 	tick_nohz_activate(ts, NOHZ_MODE_HIGHRES);  in tick_setup_sched_timer()
1125 	struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu);  in tick_cancel_sched_timer()  local
1128 	if (ts->sched_timer.base)  in tick_cancel_sched_timer()
1129 		hrtimer_cancel(&ts->sched_timer);  in tick_cancel_sched_timer()
1132 	memset(ts, 0, sizeof(*ts));  in tick_cancel_sched_timer()
1152 	struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);  in tick_oneshot_notify()  local
1154 	set_bit(0, &ts->check_clocks);  in tick_oneshot_notify()
1167 	struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);  in tick_check_oneshot_change()  local
1169 	if (!test_and_clear_bit(0, &ts->check_clocks))  in tick_check_oneshot_change()
1172 	if (ts->nohz_mode != NOHZ_MODE_INACTIVE)  in tick_check_oneshot_change()