152 struct hrtimer_clock_base *lock_hrtimer_base(const struct hrtimer *timer,  in lock_hrtimer_base()  argument
158 		base = timer->base;  in lock_hrtimer_base()
161 			if (likely(base == timer->base))  in lock_hrtimer_base()
178 hrtimer_check_target(struct hrtimer *timer, struct hrtimer_clock_base *new_base)  in hrtimer_check_target()  argument
186 	expires = ktime_sub(hrtimer_get_expires(timer), new_base->offset);  in hrtimer_check_target()
197 switch_hrtimer_base(struct hrtimer *timer, struct hrtimer_clock_base *base,  in switch_hrtimer_base()  argument
220 		if (unlikely(hrtimer_callback_running(timer)))  in switch_hrtimer_base()
224 		timer->base = NULL;  in switch_hrtimer_base()
228 		if (cpu != this_cpu && hrtimer_check_target(timer, new_base)) {  in switch_hrtimer_base()
232 			timer->base = base;  in switch_hrtimer_base()
235 		timer->base = new_base;  in switch_hrtimer_base()
237 		if (cpu != this_cpu && hrtimer_check_target(timer, new_base)) {  in switch_hrtimer_base()
248 lock_hrtimer_base(const struct hrtimer *timer, unsigned long *flags)  in lock_hrtimer_base()  argument
250 	struct hrtimer_clock_base *base = timer->base;  in lock_hrtimer_base()
324 	struct hrtimer *timer = addr;  in hrtimer_fixup_init()  local
328 		hrtimer_cancel(timer);  in hrtimer_fixup_init()
329 		debug_object_init(timer, &hrtimer_debug_descr);  in hrtimer_fixup_init()
363 	struct hrtimer *timer = addr;  in hrtimer_fixup_free()  local
367 		hrtimer_cancel(timer);  in hrtimer_fixup_free()
368 		debug_object_free(timer, &hrtimer_debug_descr);  in hrtimer_fixup_free()
383 static inline void debug_hrtimer_init(struct hrtimer *timer)  in debug_hrtimer_init()  argument
385 	debug_object_init(timer, &hrtimer_debug_descr);  in debug_hrtimer_init()
388 static inline void debug_hrtimer_activate(struct hrtimer *timer)  in debug_hrtimer_activate()  argument
390 	debug_object_activate(timer, &hrtimer_debug_descr);  in debug_hrtimer_activate()
393 static inline void debug_hrtimer_deactivate(struct hrtimer *timer)  in debug_hrtimer_deactivate()  argument
395 	debug_object_deactivate(timer, &hrtimer_debug_descr);  in debug_hrtimer_deactivate()
398 static inline void debug_hrtimer_free(struct hrtimer *timer)  in debug_hrtimer_free()  argument
400 	debug_object_free(timer, &hrtimer_debug_descr);  in debug_hrtimer_free()
403 static void __hrtimer_init(struct hrtimer *timer, clockid_t clock_id,
406 void hrtimer_init_on_stack(struct hrtimer *timer, clockid_t clock_id,  in hrtimer_init_on_stack()  argument
409 	debug_object_init_on_stack(timer, &hrtimer_debug_descr);  in hrtimer_init_on_stack()
410 	__hrtimer_init(timer, clock_id, mode);  in hrtimer_init_on_stack()
414 void destroy_hrtimer_on_stack(struct hrtimer *timer)  in destroy_hrtimer_on_stack()  argument
416 	debug_object_free(timer, &hrtimer_debug_descr);  in destroy_hrtimer_on_stack()
420 static inline void debug_hrtimer_init(struct hrtimer *timer) { }  in debug_hrtimer_init()  argument
421 static inline void debug_hrtimer_activate(struct hrtimer *timer) { }  in debug_hrtimer_activate()  argument
422 static inline void debug_hrtimer_deactivate(struct hrtimer *timer) { }  in debug_hrtimer_deactivate()  argument
426 debug_init(struct hrtimer *timer, clockid_t clockid,  in debug_init()  argument
429 	debug_hrtimer_init(timer);  in debug_init()
430 	trace_hrtimer_init(timer, clockid, mode);  in debug_init()
433 static inline void debug_activate(struct hrtimer *timer)  in debug_activate()  argument
435 	debug_hrtimer_activate(timer);  in debug_activate()
436 	trace_hrtimer_start(timer);  in debug_activate()
439 static inline void debug_deactivate(struct hrtimer *timer)  in debug_deactivate()  argument
441 	debug_hrtimer_deactivate(timer);  in debug_deactivate()
442 	trace_hrtimer_cancel(timer);  in debug_deactivate()
454 		struct hrtimer *timer;  in __hrtimer_get_next_event()  local
460 		timer = container_of(next, struct hrtimer, node);  in __hrtimer_get_next_event()
461 		expires = ktime_sub(hrtimer_get_expires(timer), base->offset);  in __hrtimer_get_next_event()
566 static int hrtimer_reprogram(struct hrtimer *timer,  in hrtimer_reprogram()  argument
570 	ktime_t expires = ktime_sub(hrtimer_get_expires(timer), base->offset);  in hrtimer_reprogram()
573 	WARN_ON_ONCE(hrtimer_get_expires_tv64(timer) < 0);  in hrtimer_reprogram()
582 	if (hrtimer_callback_running(timer))  in hrtimer_reprogram()
714 static inline int hrtimer_reprogram(struct hrtimer *timer,  in hrtimer_reprogram()  argument
761 static inline void timer_stats_hrtimer_set_start_info(struct hrtimer *timer)  in timer_stats_hrtimer_set_start_info()  argument
764 	if (timer->start_site)  in timer_stats_hrtimer_set_start_info()
766 	timer->start_site = __builtin_return_address(0);  in timer_stats_hrtimer_set_start_info()
767 	memcpy(timer->start_comm, current->comm, TASK_COMM_LEN);  in timer_stats_hrtimer_set_start_info()
768 	timer->start_pid = current->pid;  in timer_stats_hrtimer_set_start_info()
772 static inline void timer_stats_hrtimer_clear_start_info(struct hrtimer *timer)  in timer_stats_hrtimer_clear_start_info()  argument
775 	timer->start_site = NULL;  in timer_stats_hrtimer_clear_start_info()
779 static inline void timer_stats_account_hrtimer(struct hrtimer *timer)  in timer_stats_account_hrtimer()  argument
784 	timer_stats_update_stats(timer, timer->start_pid, timer->start_site,  in timer_stats_account_hrtimer()
785 				 timer->function, timer->start_comm, 0);  in timer_stats_account_hrtimer()
793 void unlock_hrtimer_base(const struct hrtimer *timer, unsigned long *flags)  in unlock_hrtimer_base()  argument
795 	raw_spin_unlock_irqrestore(&timer->base->cpu_base->lock, *flags);  in unlock_hrtimer_base()
807 u64 hrtimer_forward(struct hrtimer *timer, ktime_t now, ktime_t interval)  in hrtimer_forward()  argument
812 	delta = ktime_sub(now, hrtimer_get_expires(timer));  in hrtimer_forward()
817 	if (interval.tv64 < timer->base->resolution.tv64)  in hrtimer_forward()
818 		interval.tv64 = timer->base->resolution.tv64;  in hrtimer_forward()
824 		hrtimer_add_expires_ns(timer, incr * orun);  in hrtimer_forward()
825 		if (hrtimer_get_expires_tv64(timer) > now.tv64)  in hrtimer_forward()
833 	hrtimer_add_expires(timer, interval);  in hrtimer_forward()
847 static int enqueue_hrtimer(struct hrtimer *timer,  in enqueue_hrtimer()  argument
850 	debug_activate(timer);  in enqueue_hrtimer()
852 	timerqueue_add(&base->active, &timer->node);  in enqueue_hrtimer()
859 	timer->state |= HRTIMER_STATE_ENQUEUED;  in enqueue_hrtimer()
861 	return (&timer->node == base->active.next);  in enqueue_hrtimer()
874 static void __remove_hrtimer(struct hrtimer *timer,  in __remove_hrtimer()  argument
879 	if (!(timer->state & HRTIMER_STATE_ENQUEUED))  in __remove_hrtimer()
883 	timerqueue_del(&base->active, &timer->node);  in __remove_hrtimer()
884 	if (&timer->node == next_timer) {  in __remove_hrtimer()
890 			expires = ktime_sub(hrtimer_get_expires(timer),  in __remove_hrtimer()
900 	timer->state = newstate;  in __remove_hrtimer()
907 remove_hrtimer(struct hrtimer *timer, struct hrtimer_clock_base *base)  in remove_hrtimer()  argument
909 	if (hrtimer_is_queued(timer)) {  in remove_hrtimer()
921 		debug_deactivate(timer);  in remove_hrtimer()
922 		timer_stats_hrtimer_clear_start_info(timer);  in remove_hrtimer()
929 		state = timer->state & HRTIMER_STATE_CALLBACK;  in remove_hrtimer()
930 		__remove_hrtimer(timer, base, state, reprogram);  in remove_hrtimer()
936 int __hrtimer_start_range_ns(struct hrtimer *timer, ktime_t tim,  in __hrtimer_start_range_ns()  argument
944 	base = lock_hrtimer_base(timer, &flags);  in __hrtimer_start_range_ns()
947 	ret = remove_hrtimer(timer, base);  in __hrtimer_start_range_ns()
963 	hrtimer_set_expires_range_ns(timer, tim, delta_ns);  in __hrtimer_start_range_ns()
966 	new_base = switch_hrtimer_base(timer, base, mode & HRTIMER_MODE_PINNED);  in __hrtimer_start_range_ns()
968 	timer_stats_hrtimer_set_start_info(timer);  in __hrtimer_start_range_ns()
970 	leftmost = enqueue_hrtimer(timer, new_base);  in __hrtimer_start_range_ns()
973 		unlock_hrtimer_base(timer, &flags);  in __hrtimer_start_range_ns()
977 	if (!hrtimer_is_hres_active(timer)) {  in __hrtimer_start_range_ns()
984 			hrtimer_reprogram(timer, new_base)) {  in __hrtimer_start_range_ns()
1005 	unlock_hrtimer_base(timer, &flags);  in __hrtimer_start_range_ns()
1023 int hrtimer_start_range_ns(struct hrtimer *timer, ktime_t tim,  in hrtimer_start_range_ns()  argument
1026 	return __hrtimer_start_range_ns(timer, tim, delta_ns, mode, 1);  in hrtimer_start_range_ns()
1042 hrtimer_start(struct hrtimer *timer, ktime_t tim, const enum hrtimer_mode mode)  in hrtimer_start()  argument
1044 	return __hrtimer_start_range_ns(timer, tim, 0, mode, 1);  in hrtimer_start()
1059 int hrtimer_try_to_cancel(struct hrtimer *timer)  in hrtimer_try_to_cancel()  argument
1065 	base = lock_hrtimer_base(timer, &flags);  in hrtimer_try_to_cancel()
1067 	if (!hrtimer_callback_running(timer))  in hrtimer_try_to_cancel()
1068 		ret = remove_hrtimer(timer, base);  in hrtimer_try_to_cancel()
1070 	unlock_hrtimer_base(timer, &flags);  in hrtimer_try_to_cancel()
1085 int hrtimer_cancel(struct hrtimer *timer)  in hrtimer_cancel()  argument
1088 		int ret = hrtimer_try_to_cancel(timer);  in hrtimer_cancel()
1101 ktime_t hrtimer_get_remaining(const struct hrtimer *timer)  in hrtimer_get_remaining()  argument
1106 	lock_hrtimer_base(timer, &flags);  in hrtimer_get_remaining()
1107 	rem = hrtimer_expires_remaining(timer);  in hrtimer_get_remaining()
1108 	unlock_hrtimer_base(timer, &flags);  in hrtimer_get_remaining()
1141 static void __hrtimer_init(struct hrtimer *timer, clockid_t clock_id,  in __hrtimer_init()  argument
1147 	memset(timer, 0, sizeof(struct hrtimer));  in __hrtimer_init()
1155 	timer->base = &cpu_base->clock_base[base];  in __hrtimer_init()
1156 	timerqueue_init(&timer->node);  in __hrtimer_init()
1159 	timer->start_site = NULL;  in __hrtimer_init()
1160 	timer->start_pid = -1;  in __hrtimer_init()
1161 	memset(timer->start_comm, 0, TASK_COMM_LEN);  in __hrtimer_init()
1171 void hrtimer_init(struct hrtimer *timer, clockid_t clock_id,  in hrtimer_init()  argument
1174 	debug_init(timer, clock_id, mode);  in hrtimer_init()
1175 	__hrtimer_init(timer, clock_id, mode);  in hrtimer_init()
1199 static void __run_hrtimer(struct hrtimer *timer, ktime_t *now)  in __run_hrtimer()  argument
1201 	struct hrtimer_clock_base *base = timer->base;  in __run_hrtimer()
1208 	debug_deactivate(timer);  in __run_hrtimer()
1209 	__remove_hrtimer(timer, base, HRTIMER_STATE_CALLBACK, 0);  in __run_hrtimer()
1210 	timer_stats_account_hrtimer(timer);  in __run_hrtimer()
1211 	fn = timer->function;  in __run_hrtimer()
1219 	trace_hrtimer_expire_entry(timer, now);  in __run_hrtimer()
1220 	restart = fn(timer);  in __run_hrtimer()
1221 	trace_hrtimer_expire_exit(timer);  in __run_hrtimer()
1230 		BUG_ON(timer->state != HRTIMER_STATE_CALLBACK);  in __run_hrtimer()
1231 		enqueue_hrtimer(timer, base);  in __run_hrtimer()
1234 	WARN_ON_ONCE(!(timer->state & HRTIMER_STATE_CALLBACK));  in __run_hrtimer()
1236 	timer->state &= ~HRTIMER_STATE_CALLBACK;  in __run_hrtimer()
1280 			struct hrtimer *timer;  in hrtimer_interrupt()  local
1282 			timer = container_of(node, struct hrtimer, node);  in hrtimer_interrupt()
1296 			if (basenow.tv64 < hrtimer_get_softexpires_tv64(timer))  in hrtimer_interrupt()
1299 			__run_hrtimer(timer, &basenow);  in hrtimer_interrupt()
1457 			struct hrtimer *timer;  in hrtimer_run_queues()  local
1459 			timer = container_of(node, struct hrtimer, node);  in hrtimer_run_queues()
1461 					hrtimer_get_expires_tv64(timer))  in hrtimer_run_queues()
1464 			__run_hrtimer(timer, &base->softirq_time);  in hrtimer_run_queues()
1473 static enum hrtimer_restart hrtimer_wakeup(struct hrtimer *timer)  in hrtimer_wakeup()  argument
1476 		container_of(timer, struct hrtimer_sleeper, timer);  in hrtimer_wakeup()
1488 	sl->timer.function = hrtimer_wakeup;  in hrtimer_init_sleeper()
1499 		hrtimer_start_expires(&t->timer, mode);  in do_nanosleep()
1500 		if (!hrtimer_active(&t->timer))  in do_nanosleep()
1506 		hrtimer_cancel(&t->timer);  in do_nanosleep()
1516 static int update_rmtp(struct hrtimer *timer, struct timespec __user *rmtp)  in update_rmtp()  argument
1521 	rem = hrtimer_expires_remaining(timer);  in update_rmtp()
1538 	hrtimer_init_on_stack(&t.timer, restart->nanosleep.clockid,  in hrtimer_nanosleep_restart()
1540 	hrtimer_set_expires_tv64(&t.timer, restart->nanosleep.expires);  in hrtimer_nanosleep_restart()
1547 		ret = update_rmtp(&t.timer, rmtp);  in hrtimer_nanosleep_restart()
1555 	destroy_hrtimer_on_stack(&t.timer);  in hrtimer_nanosleep_restart()
1571 	hrtimer_init_on_stack(&t.timer, clockid, mode);  in hrtimer_nanosleep()
1572 	hrtimer_set_expires_range_ns(&t.timer, timespec_to_ktime(*rqtp), slack);  in hrtimer_nanosleep()
1583 		ret = update_rmtp(&t.timer, rmtp);  in hrtimer_nanosleep()
1590 	restart->nanosleep.clockid = t.timer.base->clockid;  in hrtimer_nanosleep()
1592 	restart->nanosleep.expires = hrtimer_get_expires_tv64(&t.timer);  in hrtimer_nanosleep()
1596 	destroy_hrtimer_on_stack(&t.timer);  in hrtimer_nanosleep()
1636 	struct hrtimer *timer;  in migrate_hrtimer_list()  local
1640 		timer = container_of(node, struct hrtimer, node);  in migrate_hrtimer_list()
1641 		BUG_ON(hrtimer_callback_running(timer));  in migrate_hrtimer_list()
1642 		debug_deactivate(timer);  in migrate_hrtimer_list()
1649 		__remove_hrtimer(timer, old_base, HRTIMER_STATE_MIGRATE, 0);  in migrate_hrtimer_list()
1650 		timer->base = new_base;  in migrate_hrtimer_list()
1659 		enqueue_hrtimer(timer, new_base);  in migrate_hrtimer_list()
1662 		timer->state &= ~HRTIMER_STATE_MIGRATE;  in migrate_hrtimer_list()
1769 	hrtimer_init_on_stack(&t.timer, clock, mode);  in schedule_hrtimeout_range_clock()
1770 	hrtimer_set_expires_range_ns(&t.timer, *expires, delta);  in schedule_hrtimeout_range_clock()
1774 	hrtimer_start_expires(&t.timer, mode);  in schedule_hrtimeout_range_clock()
1775 	if (!hrtimer_active(&t.timer))  in schedule_hrtimeout_range_clock()
1781 	hrtimer_cancel(&t.timer);  in schedule_hrtimeout_range_clock()
1782 	destroy_hrtimer_on_stack(&t.timer);  in schedule_hrtimeout_range_clock()