367 void set_timer_slack(struct timer_list *timer, int slack_hz)  in set_timer_slack()  argument
369 	timer->slack = slack_hz;  in set_timer_slack()
374 __internal_add_timer(struct tvec_base *base, struct timer_list *timer)  in __internal_add_timer()  argument
376 	unsigned long expires = timer->expires;  in __internal_add_timer()
412 	hlist_add_head(&timer->entry, vec);  in __internal_add_timer()
415 static void internal_add_timer(struct tvec_base *base, struct timer_list *timer)  in internal_add_timer()  argument
421 	__internal_add_timer(base, timer);  in internal_add_timer()
425 	if (!(timer->flags & TIMER_DEFERRABLE)) {  in internal_add_timer()
427 		    time_before(timer->expires, base->next_timer))  in internal_add_timer()
428 			base->next_timer = timer->expires;  in internal_add_timer()
445 		if (!(timer->flags & TIMER_DEFERRABLE) ||  in internal_add_timer()
452 void __timer_stats_timer_set_start_info(struct timer_list *timer, void *addr)  in __timer_stats_timer_set_start_info()  argument
454 	if (timer->start_site)  in __timer_stats_timer_set_start_info()
457 	timer->start_site = addr;  in __timer_stats_timer_set_start_info()
458 	memcpy(timer->start_comm, current->comm, TASK_COMM_LEN);  in __timer_stats_timer_set_start_info()
459 	timer->start_pid = current->pid;  in __timer_stats_timer_set_start_info()
462 static void timer_stats_account_timer(struct timer_list *timer)  in timer_stats_account_timer()  argument
470 	site = READ_ONCE(timer->start_site);  in timer_stats_account_timer()
474 	timer_stats_update_stats(timer, timer->start_pid, site,  in timer_stats_account_timer()
475 				 timer->function, timer->start_comm,  in timer_stats_account_timer()
476 				 timer->flags);  in timer_stats_account_timer()
480 static void timer_stats_account_timer(struct timer_list *timer) {}  in timer_stats_account_timer()  argument
498 	struct timer_list *timer = addr;  in timer_fixup_init()  local
502 		del_timer_sync(timer);  in timer_fixup_init()
503 		debug_object_init(timer, &timer_debug_descr);  in timer_fixup_init()
523 	struct timer_list *timer = addr;  in timer_fixup_activate()  local
533 		if (timer->entry.pprev == NULL &&  in timer_fixup_activate()
534 		    timer->entry.next == TIMER_ENTRY_STATIC) {  in timer_fixup_activate()
535 			debug_object_init(timer, &timer_debug_descr);  in timer_fixup_activate()
536 			debug_object_activate(timer, &timer_debug_descr);  in timer_fixup_activate()
539 			setup_timer(timer, stub_timer, 0);  in timer_fixup_activate()
558 	struct timer_list *timer = addr;  in timer_fixup_free()  local
562 		del_timer_sync(timer);  in timer_fixup_free()
563 		debug_object_free(timer, &timer_debug_descr);  in timer_fixup_free()
576 	struct timer_list *timer = addr;  in timer_fixup_assert_init()  local
580 		if (timer->entry.next == TIMER_ENTRY_STATIC) {  in timer_fixup_assert_init()
586 			debug_object_init(timer, &timer_debug_descr);  in timer_fixup_assert_init()
589 			setup_timer(timer, stub_timer, 0);  in timer_fixup_assert_init()
606 static inline void debug_timer_init(struct timer_list *timer)  in debug_timer_init()  argument
608 	debug_object_init(timer, &timer_debug_descr);  in debug_timer_init()
611 static inline void debug_timer_activate(struct timer_list *timer)  in debug_timer_activate()  argument
613 	debug_object_activate(timer, &timer_debug_descr);  in debug_timer_activate()
616 static inline void debug_timer_deactivate(struct timer_list *timer)  in debug_timer_deactivate()  argument
618 	debug_object_deactivate(timer, &timer_debug_descr);  in debug_timer_deactivate()
621 static inline void debug_timer_free(struct timer_list *timer)  in debug_timer_free()  argument
623 	debug_object_free(timer, &timer_debug_descr);  in debug_timer_free()
626 static inline void debug_timer_assert_init(struct timer_list *timer)  in debug_timer_assert_init()  argument
628 	debug_object_assert_init(timer, &timer_debug_descr);  in debug_timer_assert_init()
631 static void do_init_timer(struct timer_list *timer, unsigned int flags,
634 void init_timer_on_stack_key(struct timer_list *timer, unsigned int flags,  in init_timer_on_stack_key()  argument
637 	debug_object_init_on_stack(timer, &timer_debug_descr);  in init_timer_on_stack_key()
638 	do_init_timer(timer, flags, name, key);  in init_timer_on_stack_key()
642 void destroy_timer_on_stack(struct timer_list *timer)  in destroy_timer_on_stack()  argument
644 	debug_object_free(timer, &timer_debug_descr);  in destroy_timer_on_stack()
649 static inline void debug_timer_init(struct timer_list *timer) { }  in debug_timer_init()  argument
650 static inline void debug_timer_activate(struct timer_list *timer) { }  in debug_timer_activate()  argument
651 static inline void debug_timer_deactivate(struct timer_list *timer) { }  in debug_timer_deactivate()  argument
652 static inline void debug_timer_assert_init(struct timer_list *timer) { }  in debug_timer_assert_init()  argument
655 static inline void debug_init(struct timer_list *timer)  in debug_init()  argument
657 	debug_timer_init(timer);  in debug_init()
658 	trace_timer_init(timer);  in debug_init()
662 debug_activate(struct timer_list *timer, unsigned long expires)  in debug_activate()  argument
664 	debug_timer_activate(timer);  in debug_activate()
665 	trace_timer_start(timer, expires, timer->flags);  in debug_activate()
668 static inline void debug_deactivate(struct timer_list *timer)  in debug_deactivate()  argument
670 	debug_timer_deactivate(timer);  in debug_deactivate()
671 	trace_timer_cancel(timer);  in debug_deactivate()
674 static inline void debug_assert_init(struct timer_list *timer)  in debug_assert_init()  argument
676 	debug_timer_assert_init(timer);  in debug_assert_init()
679 static void do_init_timer(struct timer_list *timer, unsigned int flags,  in do_init_timer()  argument
682 	timer->entry.pprev = NULL;  in do_init_timer()
683 	timer->flags = flags | raw_smp_processor_id();  in do_init_timer()
684 	timer->slack = -1;  in do_init_timer()
686 	timer->start_site = NULL;  in do_init_timer()
687 	timer->start_pid = -1;  in do_init_timer()
688 	memset(timer->start_comm, 0, TASK_COMM_LEN);  in do_init_timer()
690 	lockdep_init_map(&timer->lockdep_map, name, key, 0);  in do_init_timer()
704 void init_timer_key(struct timer_list *timer, unsigned int flags,  in init_timer_key()  argument
707 	debug_init(timer);  in init_timer_key()
708 	do_init_timer(timer, flags, name, key);  in init_timer_key()
712 static inline void detach_timer(struct timer_list *timer, bool clear_pending)  in detach_timer()  argument
714 	struct hlist_node *entry = &timer->entry;  in detach_timer()
716 	debug_deactivate(timer);  in detach_timer()
725 detach_expired_timer(struct timer_list *timer, struct tvec_base *base)  in detach_expired_timer()  argument
727 	detach_timer(timer, true);  in detach_expired_timer()
728 	if (!(timer->flags & TIMER_DEFERRABLE))  in detach_expired_timer()
733 static int detach_if_pending(struct timer_list *timer, struct tvec_base *base,  in detach_if_pending()  argument
736 	if (!timer_pending(timer))  in detach_if_pending()
739 	detach_timer(timer, clear_pending);  in detach_if_pending()
740 	if (!(timer->flags & TIMER_DEFERRABLE)) {  in detach_if_pending()
742 		if (timer->expires == base->next_timer)  in detach_if_pending()
762 static struct tvec_base *lock_timer_base(struct timer_list *timer,  in lock_timer_base()  argument
764 	__acquires(timer->base->lock)  in lock_timer_base()
767 		u32 tf = timer->flags;  in lock_timer_base()
773 			if (timer->flags == tf)  in lock_timer_base()
782 __mod_timer(struct timer_list *timer, unsigned long expires,  in __mod_timer()  argument
789 	timer_stats_timer_set_start_info(timer);  in __mod_timer()
790 	BUG_ON(!timer->function);  in __mod_timer()
792 	base = lock_timer_base(timer, &flags);  in __mod_timer()
794 	ret = detach_if_pending(timer, base, false);  in __mod_timer()
798 	debug_activate(timer, expires);  in __mod_timer()
810 		if (likely(base->running_timer != timer)) {  in __mod_timer()
812 			timer->flags |= TIMER_MIGRATING;  in __mod_timer()
817 			WRITE_ONCE(timer->flags,  in __mod_timer()
818 				   (timer->flags & ~TIMER_BASEMASK) | base->cpu);  in __mod_timer()
822 	timer->expires = expires;  in __mod_timer()
823 	internal_add_timer(base, timer);  in __mod_timer()
841 int mod_timer_pending(struct timer_list *timer, unsigned long expires)  in mod_timer_pending()  argument
843 	return __mod_timer(timer, expires, true, TIMER_NOT_PINNED);  in mod_timer_pending()
858 unsigned long apply_slack(struct timer_list *timer, unsigned long expires)  in apply_slack()  argument
863 	if (timer->slack >= 0) {  in apply_slack()
864 		expires_limit = expires + timer->slack;  in apply_slack()
906 int mod_timer(struct timer_list *timer, unsigned long expires)  in mod_timer()  argument
908 	expires = apply_slack(timer, expires);  in mod_timer()
915 	if (timer_pending(timer) && timer->expires == expires)  in mod_timer()
918 	return __mod_timer(timer, expires, false, TIMER_NOT_PINNED);  in mod_timer()
941 int mod_timer_pinned(struct timer_list *timer, unsigned long expires)  in mod_timer_pinned()  argument
943 	if (timer->expires == expires && timer_pending(timer))  in mod_timer_pinned()
946 	return __mod_timer(timer, expires, false, TIMER_PINNED);  in mod_timer_pinned()
964 void add_timer(struct timer_list *timer)  in add_timer()  argument
966 	BUG_ON(timer_pending(timer));  in add_timer()
967 	mod_timer(timer, timer->expires);  in add_timer()
978 void add_timer_on(struct timer_list *timer, int cpu)  in add_timer_on()  argument
984 	timer_stats_timer_set_start_info(timer);  in add_timer_on()
985 	BUG_ON(timer_pending(timer) || !timer->function);  in add_timer_on()
992 	base = lock_timer_base(timer, &flags);  in add_timer_on()
994 		timer->flags |= TIMER_MIGRATING;  in add_timer_on()
999 		WRITE_ONCE(timer->flags,  in add_timer_on()
1000 			   (timer->flags & ~TIMER_BASEMASK) | cpu);  in add_timer_on()
1003 	debug_activate(timer, timer->expires);  in add_timer_on()
1004 	internal_add_timer(base, timer);  in add_timer_on()
1020 int del_timer(struct timer_list *timer)  in del_timer()  argument
1026 	debug_assert_init(timer);  in del_timer()
1028 	timer_stats_timer_clear_start_info(timer);  in del_timer()
1029 	if (timer_pending(timer)) {  in del_timer()
1030 		base = lock_timer_base(timer, &flags);  in del_timer()
1031 		ret = detach_if_pending(timer, base, true);  in del_timer()
1046 int try_to_del_timer_sync(struct timer_list *timer)  in try_to_del_timer_sync()  argument
1052 	debug_assert_init(timer);  in try_to_del_timer_sync()
1054 	base = lock_timer_base(timer, &flags);  in try_to_del_timer_sync()
1056 	if (base->running_timer != timer) {  in try_to_del_timer_sync()
1057 		timer_stats_timer_clear_start_info(timer);  in try_to_del_timer_sync()
1058 		ret = detach_if_pending(timer, base, true);  in try_to_del_timer_sync()
1103 int del_timer_sync(struct timer_list *timer)  in del_timer_sync()  argument
1113 	lock_map_acquire(&timer->lockdep_map);  in del_timer_sync()
1114 	lock_map_release(&timer->lockdep_map);  in del_timer_sync()
1121 	WARN_ON(in_irq() && !(timer->flags & TIMER_IRQSAFE));  in del_timer_sync()
1123 		int ret = try_to_del_timer_sync(timer);  in del_timer_sync()
1135 	struct timer_list *timer;  in cascade()  local
1145 	hlist_for_each_entry_safe(timer, tmp, &tv_list, entry) {  in cascade()
1147 		__internal_add_timer(base, timer);  in cascade()
1153 static void call_timer_fn(struct timer_list *timer, void (*fn)(unsigned long),  in call_timer_fn()  argument
1168 	lockdep_copy_map(&lockdep_map, &timer->lockdep_map);  in call_timer_fn()
1177 	trace_timer_expire_entry(timer);  in call_timer_fn()
1179 	trace_timer_expire_exit(timer);  in call_timer_fn()
1207 	struct timer_list *timer;  in __run_timers()  local
1238 			timer = hlist_entry(head->first, struct timer_list, entry);  in __run_timers()
1239 			fn = timer->function;  in __run_timers()
1240 			data = timer->data;  in __run_timers()
1241 			irqsafe = timer->flags & TIMER_IRQSAFE;  in __run_timers()
1243 			timer_stats_account_timer(timer);  in __run_timers()
1245 			base->running_timer = timer;  in __run_timers()
1246 			detach_expired_timer(timer, base);  in __run_timers()
1250 				call_timer_fn(timer, fn, data);  in __run_timers()
1254 				call_timer_fn(timer, fn, data);  in __run_timers()
1495 	struct timer_list timer;  in schedule_timeout()  local
1529 	setup_timer_on_stack(&timer, process_timeout, (unsigned long)current);  in schedule_timeout()
1530 	__mod_timer(&timer, expire, false, TIMER_NOT_PINNED);  in schedule_timeout()
1532 	del_singleshot_timer_sync(&timer);  in schedule_timeout()
1535 	destroy_timer_on_stack(&timer);  in schedule_timeout()
1572 	struct timer_list *timer;  in migrate_timer_list()  local
1576 		timer = hlist_entry(head->first, struct timer_list, entry);  in migrate_timer_list()
1578 		detach_timer(timer, false);  in migrate_timer_list()
1579 		timer->flags = (timer->flags & ~TIMER_BASEMASK) | cpu;  in migrate_timer_list()
1580 		internal_add_timer(new_base, timer);  in migrate_timer_list()