139 struct hrtimer_clock_base *lock_hrtimer_base(const struct hrtimer *timer,  in lock_hrtimer_base()  argument
145 		base = timer->base;  in lock_hrtimer_base()
148 			if (likely(base == timer->base))  in lock_hrtimer_base()
165 hrtimer_check_target(struct hrtimer *timer, struct hrtimer_clock_base *new_base)  in hrtimer_check_target()  argument
173 	expires = ktime_sub(hrtimer_get_expires(timer), new_base->offset);  in hrtimer_check_target()
211 switch_hrtimer_base(struct hrtimer *timer, struct hrtimer_clock_base *base,  in switch_hrtimer_base()  argument
233 		if (unlikely(hrtimer_callback_running(timer)))  in switch_hrtimer_base()
237 		timer->base = &migration_base;  in switch_hrtimer_base()
242 		    hrtimer_check_target(timer, new_base)) {  in switch_hrtimer_base()
246 			timer->base = base;  in switch_hrtimer_base()
249 		timer->base = new_base;  in switch_hrtimer_base()
252 		    hrtimer_check_target(timer, new_base)) {  in switch_hrtimer_base()
263 lock_hrtimer_base(const struct hrtimer *timer, unsigned long *flags)  in lock_hrtimer_base()  argument
265 	struct hrtimer_clock_base *base = timer->base;  in lock_hrtimer_base()
339 	struct hrtimer *timer = addr;  in hrtimer_fixup_init()  local
343 		hrtimer_cancel(timer);  in hrtimer_fixup_init()
344 		debug_object_init(timer, &hrtimer_debug_descr);  in hrtimer_fixup_init()
378 	struct hrtimer *timer = addr;  in hrtimer_fixup_free()  local
382 		hrtimer_cancel(timer);  in hrtimer_fixup_free()
383 		debug_object_free(timer, &hrtimer_debug_descr);  in hrtimer_fixup_free()
398 static inline void debug_hrtimer_init(struct hrtimer *timer)  in debug_hrtimer_init()  argument
400 	debug_object_init(timer, &hrtimer_debug_descr);  in debug_hrtimer_init()
403 static inline void debug_hrtimer_activate(struct hrtimer *timer)  in debug_hrtimer_activate()  argument
405 	debug_object_activate(timer, &hrtimer_debug_descr);  in debug_hrtimer_activate()
408 static inline void debug_hrtimer_deactivate(struct hrtimer *timer)  in debug_hrtimer_deactivate()  argument
410 	debug_object_deactivate(timer, &hrtimer_debug_descr);  in debug_hrtimer_deactivate()
413 static inline void debug_hrtimer_free(struct hrtimer *timer)  in debug_hrtimer_free()  argument
415 	debug_object_free(timer, &hrtimer_debug_descr);  in debug_hrtimer_free()
418 static void __hrtimer_init(struct hrtimer *timer, clockid_t clock_id,
421 void hrtimer_init_on_stack(struct hrtimer *timer, clockid_t clock_id,  in hrtimer_init_on_stack()  argument
424 	debug_object_init_on_stack(timer, &hrtimer_debug_descr);  in hrtimer_init_on_stack()
425 	__hrtimer_init(timer, clock_id, mode);  in hrtimer_init_on_stack()
429 void destroy_hrtimer_on_stack(struct hrtimer *timer)  in destroy_hrtimer_on_stack()  argument
431 	debug_object_free(timer, &hrtimer_debug_descr);  in destroy_hrtimer_on_stack()
435 static inline void debug_hrtimer_init(struct hrtimer *timer) { }  in debug_hrtimer_init()  argument
436 static inline void debug_hrtimer_activate(struct hrtimer *timer) { }  in debug_hrtimer_activate()  argument
437 static inline void debug_hrtimer_deactivate(struct hrtimer *timer) { }  in debug_hrtimer_deactivate()  argument
441 debug_init(struct hrtimer *timer, clockid_t clockid,  in debug_init()  argument
444 	debug_hrtimer_init(timer);  in debug_init()
445 	trace_hrtimer_init(timer, clockid, mode);  in debug_init()
448 static inline void debug_activate(struct hrtimer *timer)  in debug_activate()  argument
450 	debug_hrtimer_activate(timer);  in debug_activate()
451 	trace_hrtimer_start(timer);  in debug_activate()
454 static inline void debug_deactivate(struct hrtimer *timer)  in debug_deactivate()  argument
456 	debug_hrtimer_deactivate(timer);  in debug_deactivate()
457 	trace_hrtimer_cancel(timer);  in debug_deactivate()
462 					     struct hrtimer *timer)  in hrtimer_update_next_timer()  argument
465 	cpu_base->next_timer = timer;  in hrtimer_update_next_timer()
478 		struct hrtimer *timer;  in __hrtimer_get_next_event()  local
484 		timer = container_of(next, struct hrtimer, node);  in __hrtimer_get_next_event()
485 		expires = ktime_sub(hrtimer_get_expires(timer), base->offset);  in __hrtimer_get_next_event()
488 			hrtimer_update_next_timer(cpu_base, timer);  in __hrtimer_get_next_event()
606 static void hrtimer_reprogram(struct hrtimer *timer,  in hrtimer_reprogram()  argument
610 	ktime_t expires = ktime_sub(hrtimer_get_expires(timer), base->offset);  in hrtimer_reprogram()
612 	WARN_ON_ONCE(hrtimer_get_expires_tv64(timer) < 0);  in hrtimer_reprogram()
642 	cpu_base->next_timer = timer;  in hrtimer_reprogram()
732 static inline int hrtimer_reprogram(struct hrtimer *timer,  in hrtimer_reprogram()  argument
779 static inline void timer_stats_hrtimer_set_start_info(struct hrtimer *timer)  in timer_stats_hrtimer_set_start_info()  argument
782 	if (timer->start_site)  in timer_stats_hrtimer_set_start_info()
784 	timer->start_site = __builtin_return_address(0);  in timer_stats_hrtimer_set_start_info()
785 	memcpy(timer->start_comm, current->comm, TASK_COMM_LEN);  in timer_stats_hrtimer_set_start_info()
786 	timer->start_pid = current->pid;  in timer_stats_hrtimer_set_start_info()
790 static inline void timer_stats_hrtimer_clear_start_info(struct hrtimer *timer)  in timer_stats_hrtimer_clear_start_info()  argument
793 	timer->start_site = NULL;  in timer_stats_hrtimer_clear_start_info()
797 static inline void timer_stats_account_hrtimer(struct hrtimer *timer)  in timer_stats_account_hrtimer()  argument
802 	timer_stats_update_stats(timer, timer->start_pid, timer->start_site,  in timer_stats_account_hrtimer()
803 				 timer->function, timer->start_comm, 0);  in timer_stats_account_hrtimer()
811 void unlock_hrtimer_base(const struct hrtimer *timer, unsigned long *flags)  in unlock_hrtimer_base()  argument
813 	raw_spin_unlock_irqrestore(&timer->base->cpu_base->lock, *flags);  in unlock_hrtimer_base()
833 u64 hrtimer_forward(struct hrtimer *timer, ktime_t now, ktime_t interval)  in hrtimer_forward()  argument
838 	delta = ktime_sub(now, hrtimer_get_expires(timer));  in hrtimer_forward()
843 	if (WARN_ON(timer->state & HRTIMER_STATE_ENQUEUED))  in hrtimer_forward()
853 		hrtimer_add_expires_ns(timer, incr * orun);  in hrtimer_forward()
854 		if (hrtimer_get_expires_tv64(timer) > now.tv64)  in hrtimer_forward()
862 	hrtimer_add_expires(timer, interval);  in hrtimer_forward()
876 static int enqueue_hrtimer(struct hrtimer *timer,  in enqueue_hrtimer()  argument
879 	debug_activate(timer);  in enqueue_hrtimer()
883 	timer->state = HRTIMER_STATE_ENQUEUED;  in enqueue_hrtimer()
885 	return timerqueue_add(&base->active, &timer->node);  in enqueue_hrtimer()
898 static void __remove_hrtimer(struct hrtimer *timer,  in __remove_hrtimer()  argument
903 	u8 state = timer->state;  in __remove_hrtimer()
905 	timer->state = newstate;  in __remove_hrtimer()
909 	if (!timerqueue_del(&base->active, &timer->node))  in __remove_hrtimer()
921 	if (reprogram && timer == cpu_base->next_timer)  in __remove_hrtimer()
930 remove_hrtimer(struct hrtimer *timer, struct hrtimer_clock_base *base, bool restart)  in remove_hrtimer()  argument
932 	if (hrtimer_is_queued(timer)) {  in remove_hrtimer()
933 		u8 state = timer->state;  in remove_hrtimer()
944 		debug_deactivate(timer);  in remove_hrtimer()
945 		timer_stats_hrtimer_clear_start_info(timer);  in remove_hrtimer()
951 		__remove_hrtimer(timer, base, state, reprogram);  in remove_hrtimer()
957 static inline ktime_t hrtimer_update_lowres(struct hrtimer *timer, ktime_t tim,  in hrtimer_update_lowres()  argument
966 	timer->is_rel = mode & HRTIMER_MODE_REL;  in hrtimer_update_lowres()
967 	if (timer->is_rel)  in hrtimer_update_lowres()
981 void hrtimer_start_range_ns(struct hrtimer *timer, ktime_t tim,  in hrtimer_start_range_ns()  argument
988 	base = lock_hrtimer_base(timer, &flags);  in hrtimer_start_range_ns()
991 	remove_hrtimer(timer, base, true);  in hrtimer_start_range_ns()
996 	tim = hrtimer_update_lowres(timer, tim, mode);  in hrtimer_start_range_ns()
998 	hrtimer_set_expires_range_ns(timer, tim, delta_ns);  in hrtimer_start_range_ns()
1001 	new_base = switch_hrtimer_base(timer, base, mode & HRTIMER_MODE_PINNED);  in hrtimer_start_range_ns()
1003 	timer_stats_hrtimer_set_start_info(timer);  in hrtimer_start_range_ns()
1005 	leftmost = enqueue_hrtimer(timer, new_base);  in hrtimer_start_range_ns()
1009 	if (!hrtimer_is_hres_active(timer)) {  in hrtimer_start_range_ns()
1017 		hrtimer_reprogram(timer, new_base);  in hrtimer_start_range_ns()
1020 	unlock_hrtimer_base(timer, &flags);  in hrtimer_start_range_ns()
1034 int hrtimer_try_to_cancel(struct hrtimer *timer)  in hrtimer_try_to_cancel()  argument
1046 	if (!hrtimer_active(timer))  in hrtimer_try_to_cancel()
1049 	base = lock_hrtimer_base(timer, &flags);  in hrtimer_try_to_cancel()
1051 	if (!hrtimer_callback_running(timer))  in hrtimer_try_to_cancel()
1052 		ret = remove_hrtimer(timer, base, false);  in hrtimer_try_to_cancel()
1054 	unlock_hrtimer_base(timer, &flags);  in hrtimer_try_to_cancel()
1069 int hrtimer_cancel(struct hrtimer *timer)  in hrtimer_cancel()  argument
1072 		int ret = hrtimer_try_to_cancel(timer);  in hrtimer_cancel()
1086 ktime_t __hrtimer_get_remaining(const struct hrtimer *timer, bool adjust)  in __hrtimer_get_remaining()  argument
1091 	lock_hrtimer_base(timer, &flags);  in __hrtimer_get_remaining()
1093 		rem = hrtimer_expires_remaining_adjusted(timer);  in __hrtimer_get_remaining()
1095 		rem = hrtimer_expires_remaining(timer);  in __hrtimer_get_remaining()
1096 	unlock_hrtimer_base(timer, &flags);  in __hrtimer_get_remaining()
1125 static void __hrtimer_init(struct hrtimer *timer, clockid_t clock_id,  in __hrtimer_init()  argument
1131 	memset(timer, 0, sizeof(struct hrtimer));  in __hrtimer_init()
1139 	timer->base = &cpu_base->clock_base[base];  in __hrtimer_init()
1140 	timerqueue_init(&timer->node);  in __hrtimer_init()
1143 	timer->start_site = NULL;  in __hrtimer_init()
1144 	timer->start_pid = -1;  in __hrtimer_init()
1145 	memset(timer->start_comm, 0, TASK_COMM_LEN);  in __hrtimer_init()
1155 void hrtimer_init(struct hrtimer *timer, clockid_t clock_id,  in hrtimer_init()  argument
1158 	debug_init(timer, clock_id, mode);  in hrtimer_init()
1159 	__hrtimer_init(timer, clock_id, mode);  in hrtimer_init()
1170 bool hrtimer_active(const struct hrtimer *timer)  in hrtimer_active()  argument
1176 		cpu_base = READ_ONCE(timer->base->cpu_base);  in hrtimer_active()
1179 		if (timer->state != HRTIMER_STATE_INACTIVE ||  in hrtimer_active()
1180 		    cpu_base->running == timer)  in hrtimer_active()
1184 		 cpu_base != READ_ONCE(timer->base->cpu_base));  in hrtimer_active()
1210 			  struct hrtimer *timer, ktime_t *now)  in __run_hrtimer()  argument
1217 	debug_deactivate(timer);  in __run_hrtimer()
1218 	cpu_base->running = timer;  in __run_hrtimer()
1229 	__remove_hrtimer(timer, base, HRTIMER_STATE_INACTIVE, 0);  in __run_hrtimer()
1230 	timer_stats_account_hrtimer(timer);  in __run_hrtimer()
1231 	fn = timer->function;  in __run_hrtimer()
1239 		timer->is_rel = false;  in __run_hrtimer()
1247 	trace_hrtimer_expire_entry(timer, now);  in __run_hrtimer()
1248 	restart = fn(timer);  in __run_hrtimer()
1249 	trace_hrtimer_expire_exit(timer);  in __run_hrtimer()
1262 	    !(timer->state & HRTIMER_STATE_ENQUEUED))  in __run_hrtimer()
1263 		enqueue_hrtimer(timer, base);  in __run_hrtimer()
1274 	WARN_ON_ONCE(cpu_base->running != timer);  in __run_hrtimer()
1293 			struct hrtimer *timer;  in __hrtimer_run_queues()  local
1295 			timer = container_of(node, struct hrtimer, node);  in __hrtimer_run_queues()
1309 			if (basenow.tv64 < hrtimer_get_softexpires_tv64(timer))  in __hrtimer_run_queues()
1312 			__run_hrtimer(cpu_base, base, timer, &basenow);  in __hrtimer_run_queues()
1461 static enum hrtimer_restart hrtimer_wakeup(struct hrtimer *timer)  in hrtimer_wakeup()  argument
1464 		container_of(timer, struct hrtimer_sleeper, timer);  in hrtimer_wakeup()
1476 	sl->timer.function = hrtimer_wakeup;  in hrtimer_init_sleeper()
1487 		hrtimer_start_expires(&t->timer, mode);  in do_nanosleep()
1492 		hrtimer_cancel(&t->timer);  in do_nanosleep()
1502 static int update_rmtp(struct hrtimer *timer, struct timespec __user *rmtp)  in update_rmtp()  argument
1507 	rem = hrtimer_expires_remaining(timer);  in update_rmtp()
1524 	hrtimer_init_on_stack(&t.timer, restart->nanosleep.clockid,  in hrtimer_nanosleep_restart()
1526 	hrtimer_set_expires_tv64(&t.timer, restart->nanosleep.expires);  in hrtimer_nanosleep_restart()
1533 		ret = update_rmtp(&t.timer, rmtp);  in hrtimer_nanosleep_restart()
1541 	destroy_hrtimer_on_stack(&t.timer);  in hrtimer_nanosleep_restart()
1557 	hrtimer_init_on_stack(&t.timer, clockid, mode);  in hrtimer_nanosleep()
1558 	hrtimer_set_expires_range_ns(&t.timer, timespec_to_ktime(*rqtp), slack);  in hrtimer_nanosleep()
1569 		ret = update_rmtp(&t.timer, rmtp);  in hrtimer_nanosleep()
1576 	restart->nanosleep.clockid = t.timer.base->clockid;  in hrtimer_nanosleep()
1578 	restart->nanosleep.expires = hrtimer_get_expires_tv64(&t.timer);  in hrtimer_nanosleep()
1582 	destroy_hrtimer_on_stack(&t.timer);  in hrtimer_nanosleep()
1622 	struct hrtimer *timer;  in migrate_hrtimer_list()  local
1626 		timer = container_of(node, struct hrtimer, node);  in migrate_hrtimer_list()
1627 		BUG_ON(hrtimer_callback_running(timer));  in migrate_hrtimer_list()
1628 		debug_deactivate(timer);  in migrate_hrtimer_list()
1635 		__remove_hrtimer(timer, old_base, HRTIMER_STATE_ENQUEUED, 0);  in migrate_hrtimer_list()
1636 		timer->base = new_base;  in migrate_hrtimer_list()
1645 		enqueue_hrtimer(timer, new_base);  in migrate_hrtimer_list()
1749 	hrtimer_init_on_stack(&t.timer, clock, mode);  in schedule_hrtimeout_range_clock()
1750 	hrtimer_set_expires_range_ns(&t.timer, *expires, delta);  in schedule_hrtimeout_range_clock()
1754 	hrtimer_start_expires(&t.timer, mode);  in schedule_hrtimeout_range_clock()
1759 	hrtimer_cancel(&t.timer);  in schedule_hrtimeout_range_clock()
1760 	destroy_hrtimer_on_stack(&t.timer);  in schedule_hrtimeout_range_clock()