Lines Matching refs:tk
67 static inline void tk_normalize_xtime(struct timekeeper *tk) in tk_normalize_xtime() argument
69 while (tk->tkr_mono.xtime_nsec >= ((u64)NSEC_PER_SEC << tk->tkr_mono.shift)) { in tk_normalize_xtime()
70 tk->tkr_mono.xtime_nsec -= (u64)NSEC_PER_SEC << tk->tkr_mono.shift; in tk_normalize_xtime()
71 tk->xtime_sec++; in tk_normalize_xtime()
75 static inline struct timespec64 tk_xtime(struct timekeeper *tk) in tk_xtime() argument
79 ts.tv_sec = tk->xtime_sec; in tk_xtime()
80 ts.tv_nsec = (long)(tk->tkr_mono.xtime_nsec >> tk->tkr_mono.shift); in tk_xtime()
84 static void tk_set_xtime(struct timekeeper *tk, const struct timespec64 *ts) in tk_set_xtime() argument
86 tk->xtime_sec = ts->tv_sec; in tk_set_xtime()
87 tk->tkr_mono.xtime_nsec = (u64)ts->tv_nsec << tk->tkr_mono.shift; in tk_set_xtime()
90 static void tk_xtime_add(struct timekeeper *tk, const struct timespec64 *ts) in tk_xtime_add() argument
92 tk->xtime_sec += ts->tv_sec; in tk_xtime_add()
93 tk->tkr_mono.xtime_nsec += (u64)ts->tv_nsec << tk->tkr_mono.shift; in tk_xtime_add()
94 tk_normalize_xtime(tk); in tk_xtime_add()
97 static void tk_set_wall_to_mono(struct timekeeper *tk, struct timespec64 wtm) in tk_set_wall_to_mono() argument
105 set_normalized_timespec64(&tmp, -tk->wall_to_monotonic.tv_sec, in tk_set_wall_to_mono()
106 -tk->wall_to_monotonic.tv_nsec); in tk_set_wall_to_mono()
107 WARN_ON_ONCE(tk->offs_real.tv64 != timespec64_to_ktime(tmp).tv64); in tk_set_wall_to_mono()
108 tk->wall_to_monotonic = wtm; in tk_set_wall_to_mono()
110 tk->offs_real = timespec64_to_ktime(tmp); in tk_set_wall_to_mono()
111 tk->offs_tai = ktime_add(tk->offs_real, ktime_set(tk->tai_offset, 0)); in tk_set_wall_to_mono()
114 static inline void tk_update_sleep_time(struct timekeeper *tk, ktime_t delta) in tk_update_sleep_time() argument
116 tk->offs_boot = ktime_add(tk->offs_boot, delta); in tk_update_sleep_time()
122 static void timekeeping_check_update(struct timekeeper *tk, cycle_t offset) in timekeeping_check_update() argument
125 cycle_t max_cycles = tk->tkr_mono.clock->max_cycles; in timekeeping_check_update()
126 const char *name = tk->tkr_mono.clock->name; in timekeeping_check_update()
140 if (tk->underflow_seen) { in timekeeping_check_update()
141 if (jiffies - tk->last_warning > WARNING_FREQ) { in timekeeping_check_update()
145 tk->last_warning = jiffies; in timekeeping_check_update()
147 tk->underflow_seen = 0; in timekeeping_check_update()
150 if (tk->overflow_seen) { in timekeeping_check_update()
151 if (jiffies - tk->last_warning > WARNING_FREQ) { in timekeeping_check_update()
155 tk->last_warning = jiffies; in timekeeping_check_update()
157 tk->overflow_seen = 0; in timekeeping_check_update()
163 struct timekeeper *tk = &tk_core.timekeeper; in timekeeping_get_delta() local
189 tk->underflow_seen = 1; in timekeeping_get_delta()
195 tk->overflow_seen = 1; in timekeeping_get_delta()
202 static inline void timekeeping_check_update(struct timekeeper *tk, cycle_t offset) in timekeeping_check_update() argument
230 static void tk_setup_internals(struct timekeeper *tk, struct clocksource *clock) in tk_setup_internals() argument
236 old_clock = tk->tkr_mono.clock; in tk_setup_internals()
237 tk->tkr_mono.clock = clock; in tk_setup_internals()
238 tk->tkr_mono.read = clock->read; in tk_setup_internals()
239 tk->tkr_mono.mask = clock->mask; in tk_setup_internals()
240 tk->tkr_mono.cycle_last = tk->tkr_mono.read(clock); in tk_setup_internals()
242 tk->tkr_raw.clock = clock; in tk_setup_internals()
243 tk->tkr_raw.read = clock->read; in tk_setup_internals()
244 tk->tkr_raw.mask = clock->mask; in tk_setup_internals()
245 tk->tkr_raw.cycle_last = tk->tkr_mono.cycle_last; in tk_setup_internals()
257 tk->cycle_interval = interval; in tk_setup_internals()
260 tk->xtime_interval = (u64) interval * clock->mult; in tk_setup_internals()
261 tk->xtime_remainder = ntpinterval - tk->xtime_interval; in tk_setup_internals()
262 tk->raw_interval = in tk_setup_internals()
269 tk->tkr_mono.xtime_nsec >>= -shift_change; in tk_setup_internals()
271 tk->tkr_mono.xtime_nsec <<= shift_change; in tk_setup_internals()
273 tk->tkr_raw.xtime_nsec = 0; in tk_setup_internals()
275 tk->tkr_mono.shift = clock->shift; in tk_setup_internals()
276 tk->tkr_raw.shift = clock->shift; in tk_setup_internals()
278 tk->ntp_error = 0; in tk_setup_internals()
279 tk->ntp_error_shift = NTP_SCALE_SHIFT - clock->shift; in tk_setup_internals()
280 tk->ntp_tick = ntpinterval << tk->ntp_error_shift; in tk_setup_internals()
287 tk->tkr_mono.mult = clock->mult; in tk_setup_internals()
288 tk->tkr_raw.mult = clock->mult; in tk_setup_internals()
289 tk->ntp_err_mult = 0; in tk_setup_internals()
422 static void halt_fast_timekeeper(struct timekeeper *tk) in halt_fast_timekeeper() argument
425 struct tk_read_base *tkr = &tk->tkr_mono; in halt_fast_timekeeper()
432 tkr = &tk->tkr_raw; in halt_fast_timekeeper()
440 static inline void update_vsyscall(struct timekeeper *tk) in update_vsyscall() argument
444 xt = timespec64_to_timespec(tk_xtime(tk)); in update_vsyscall()
445 wm = timespec64_to_timespec(tk->wall_to_monotonic); in update_vsyscall()
446 update_vsyscall_old(&xt, &wm, tk->tkr_mono.clock, tk->tkr_mono.mult, in update_vsyscall()
447 tk->tkr_mono.cycle_last); in update_vsyscall()
450 static inline void old_vsyscall_fixup(struct timekeeper *tk) in old_vsyscall_fixup() argument
464 remainder = tk->tkr_mono.xtime_nsec & ((1ULL << tk->tkr_mono.shift) - 1); in old_vsyscall_fixup()
465 tk->tkr_mono.xtime_nsec -= remainder; in old_vsyscall_fixup()
466 tk->tkr_mono.xtime_nsec += 1ULL << tk->tkr_mono.shift; in old_vsyscall_fixup()
467 tk->ntp_error += remainder << tk->ntp_error_shift; in old_vsyscall_fixup()
468 tk->ntp_error -= (1ULL << tk->tkr_mono.shift) << tk->ntp_error_shift; in old_vsyscall_fixup()
471 #define old_vsyscall_fixup(tk) argument
476 static void update_pvclock_gtod(struct timekeeper *tk, bool was_set) in update_pvclock_gtod() argument
478 raw_notifier_call_chain(&pvclock_gtod_chain, was_set, tk); in update_pvclock_gtod()
486 struct timekeeper *tk = &tk_core.timekeeper; in pvclock_gtod_register_notifier() local
492 update_pvclock_gtod(tk, true); in pvclock_gtod_register_notifier()
519 static inline void tk_update_leap_state(struct timekeeper *tk) in tk_update_leap_state() argument
521 tk->next_leap_ktime = ntp_get_next_leap(); in tk_update_leap_state()
522 if (tk->next_leap_ktime.tv64 != KTIME_MAX) in tk_update_leap_state()
524 tk->next_leap_ktime = ktime_sub(tk->next_leap_ktime, tk->offs_real); in tk_update_leap_state()
530 static inline void tk_update_ktime_data(struct timekeeper *tk) in tk_update_ktime_data() argument
542 seconds = (u64)(tk->xtime_sec + tk->wall_to_monotonic.tv_sec); in tk_update_ktime_data()
543 nsec = (u32) tk->wall_to_monotonic.tv_nsec; in tk_update_ktime_data()
544 tk->tkr_mono.base = ns_to_ktime(seconds * NSEC_PER_SEC + nsec); in tk_update_ktime_data()
547 tk->tkr_raw.base = timespec64_to_ktime(tk->raw_time); in tk_update_ktime_data()
554 nsec += (u32)(tk->tkr_mono.xtime_nsec >> tk->tkr_mono.shift); in tk_update_ktime_data()
557 tk->ktime_sec = seconds; in tk_update_ktime_data()
561 static void timekeeping_update(struct timekeeper *tk, unsigned int action) in timekeeping_update() argument
564 tk->ntp_error = 0; in timekeeping_update()
568 tk_update_leap_state(tk); in timekeeping_update()
569 tk_update_ktime_data(tk); in timekeeping_update()
571 update_vsyscall(tk); in timekeeping_update()
572 update_pvclock_gtod(tk, action & TK_CLOCK_WAS_SET); in timekeeping_update()
574 update_fast_timekeeper(&tk->tkr_mono, &tk_fast_mono); in timekeeping_update()
575 update_fast_timekeeper(&tk->tkr_raw, &tk_fast_raw); in timekeeping_update()
578 tk->clock_was_set_seq++; in timekeeping_update()
596 static void timekeeping_forward_now(struct timekeeper *tk) in timekeeping_forward_now() argument
598 struct clocksource *clock = tk->tkr_mono.clock; in timekeeping_forward_now()
602 cycle_now = tk->tkr_mono.read(clock); in timekeeping_forward_now()
603 delta = clocksource_delta(cycle_now, tk->tkr_mono.cycle_last, tk->tkr_mono.mask); in timekeeping_forward_now()
604 tk->tkr_mono.cycle_last = cycle_now; in timekeeping_forward_now()
605 tk->tkr_raw.cycle_last = cycle_now; in timekeeping_forward_now()
607 tk->tkr_mono.xtime_nsec += delta * tk->tkr_mono.mult; in timekeeping_forward_now()
610 tk->tkr_mono.xtime_nsec += (u64)arch_gettimeoffset() << tk->tkr_mono.shift; in timekeeping_forward_now()
612 tk_normalize_xtime(tk); in timekeeping_forward_now()
614 nsec = clocksource_cyc2ns(delta, tk->tkr_raw.mult, tk->tkr_raw.shift); in timekeeping_forward_now()
615 timespec64_add_ns(&tk->raw_time, nsec); in timekeeping_forward_now()
627 struct timekeeper *tk = &tk_core.timekeeper; in __getnstimeofday64() local
634 ts->tv_sec = tk->xtime_sec; in __getnstimeofday64()
635 nsecs = timekeeping_get_ns(&tk->tkr_mono); in __getnstimeofday64()
666 struct timekeeper *tk = &tk_core.timekeeper; in ktime_get() local
675 base = tk->tkr_mono.base; in ktime_get()
676 nsecs = timekeeping_get_ns(&tk->tkr_mono); in ktime_get()
686 struct timekeeper *tk = &tk_core.timekeeper; in ktime_get_resolution_ns() local
694 nsecs = tk->tkr_mono.mult >> tk->tkr_mono.shift; in ktime_get_resolution_ns()
709 struct timekeeper *tk = &tk_core.timekeeper; in ktime_get_with_offset() local
718 base = ktime_add(tk->tkr_mono.base, *offset); in ktime_get_with_offset()
719 nsecs = timekeeping_get_ns(&tk->tkr_mono); in ktime_get_with_offset()
753 struct timekeeper *tk = &tk_core.timekeeper; in ktime_get_raw() local
760 base = tk->tkr_raw.base; in ktime_get_raw()
761 nsecs = timekeeping_get_ns(&tk->tkr_raw); in ktime_get_raw()
779 struct timekeeper *tk = &tk_core.timekeeper; in ktime_get_ts64() local
788 ts->tv_sec = tk->xtime_sec; in ktime_get_ts64()
789 nsec = timekeeping_get_ns(&tk->tkr_mono); in ktime_get_ts64()
790 tomono = tk->wall_to_monotonic; in ktime_get_ts64()
811 struct timekeeper *tk = &tk_core.timekeeper; in ktime_get_seconds() local
814 return tk->ktime_sec; in ktime_get_seconds()
831 struct timekeeper *tk = &tk_core.timekeeper; in ktime_get_real_seconds() local
836 return tk->xtime_sec; in ktime_get_real_seconds()
840 seconds = tk->xtime_sec; in ktime_get_real_seconds()
861 struct timekeeper *tk = &tk_core.timekeeper; in ktime_get_raw_and_real_ts64() local
870 *ts_raw = tk->raw_time; in ktime_get_raw_and_real_ts64()
871 ts_real->tv_sec = tk->xtime_sec; in ktime_get_raw_and_real_ts64()
874 nsecs_raw = timekeeping_get_ns(&tk->tkr_raw); in ktime_get_raw_and_real_ts64()
875 nsecs_real = timekeeping_get_ns(&tk->tkr_mono); in ktime_get_raw_and_real_ts64()
910 struct timekeeper *tk = &tk_core.timekeeper; in do_settimeofday64() local
921 timekeeping_forward_now(tk); in do_settimeofday64()
923 xt = tk_xtime(tk); in do_settimeofday64()
927 if (timespec64_compare(&tk->wall_to_monotonic, &ts_delta) > 0) { in do_settimeofday64()
932 tk_set_wall_to_mono(tk, timespec64_sub(tk->wall_to_monotonic, ts_delta)); in do_settimeofday64()
934 tk_set_xtime(tk, ts); in do_settimeofday64()
936 timekeeping_update(tk, TK_CLEAR_NTP | TK_MIRROR | TK_CLOCK_WAS_SET); in do_settimeofday64()
956 struct timekeeper *tk = &tk_core.timekeeper; in timekeeping_inject_offset() local
969 timekeeping_forward_now(tk); in timekeeping_inject_offset()
972 tmp = timespec64_add(tk_xtime(tk), ts64); in timekeeping_inject_offset()
973 if (timespec64_compare(&tk->wall_to_monotonic, &ts64) > 0 || in timekeeping_inject_offset()
979 tk_xtime_add(tk, &ts64); in timekeeping_inject_offset()
980 tk_set_wall_to_mono(tk, timespec64_sub(tk->wall_to_monotonic, ts64)); in timekeeping_inject_offset()
983 timekeeping_update(tk, TK_CLEAR_NTP | TK_MIRROR | TK_CLOCK_WAS_SET); in timekeeping_inject_offset()
1002 struct timekeeper *tk = &tk_core.timekeeper; in timekeeping_get_tai_offset() local
1008 ret = tk->tai_offset; in timekeeping_get_tai_offset()
1018 static void __timekeeping_set_tai_offset(struct timekeeper *tk, s32 tai_offset) in __timekeeping_set_tai_offset() argument
1020 tk->tai_offset = tai_offset; in __timekeeping_set_tai_offset()
1021 tk->offs_tai = ktime_add(tk->offs_real, ktime_set(tai_offset, 0)); in __timekeeping_set_tai_offset()
1030 struct timekeeper *tk = &tk_core.timekeeper; in timekeeping_set_tai_offset() local
1035 __timekeeping_set_tai_offset(tk, tai_offset); in timekeeping_set_tai_offset()
1036 timekeeping_update(tk, TK_MIRROR | TK_CLOCK_WAS_SET); in timekeeping_set_tai_offset()
1049 struct timekeeper *tk = &tk_core.timekeeper; in change_clocksource() local
1058 timekeeping_forward_now(tk); in change_clocksource()
1065 old = tk->tkr_mono.clock; in change_clocksource()
1066 tk_setup_internals(tk, new); in change_clocksource()
1074 timekeeping_update(tk, TK_CLEAR_NTP | TK_MIRROR | TK_CLOCK_WAS_SET); in change_clocksource()
1091 struct timekeeper *tk = &tk_core.timekeeper; in timekeeping_notify() local
1093 if (tk->tkr_mono.clock == clock) in timekeeping_notify()
1097 return tk->tkr_mono.clock == clock ? 0 : -1; in timekeeping_notify()
1108 struct timekeeper *tk = &tk_core.timekeeper; in getrawmonotonic64() local
1115 nsecs = timekeeping_get_ns(&tk->tkr_raw); in getrawmonotonic64()
1116 ts64 = tk->raw_time; in getrawmonotonic64()
1131 struct timekeeper *tk = &tk_core.timekeeper; in timekeeping_valid_for_hres() local
1138 ret = tk->tkr_mono.clock->flags & CLOCK_SOURCE_VALID_FOR_HRES; in timekeeping_valid_for_hres()
1150 struct timekeeper *tk = &tk_core.timekeeper; in timekeeping_max_deferment() local
1157 ret = tk->tkr_mono.clock->max_idle_ns; in timekeeping_max_deferment()
1213 struct timekeeper *tk = &tk_core.timekeeper; in timekeeping_init() local
1242 tk_setup_internals(tk, clock); in timekeeping_init()
1244 tk_set_xtime(tk, &now); in timekeeping_init()
1245 tk->raw_time.tv_sec = 0; in timekeeping_init()
1246 tk->raw_time.tv_nsec = 0; in timekeeping_init()
1248 boot = tk_xtime(tk); in timekeeping_init()
1251 tk_set_wall_to_mono(tk, tmp); in timekeeping_init()
1253 timekeeping_update(tk, TK_MIRROR | TK_CLOCK_WAS_SET); in timekeeping_init()
1269 static void __timekeeping_inject_sleeptime(struct timekeeper *tk, in __timekeeping_inject_sleeptime() argument
1278 tk_xtime_add(tk, delta); in __timekeeping_inject_sleeptime()
1279 tk_set_wall_to_mono(tk, timespec64_sub(tk->wall_to_monotonic, *delta)); in __timekeeping_inject_sleeptime()
1280 tk_update_sleep_time(tk, timespec64_to_ktime(*delta)); in __timekeeping_inject_sleeptime()
1333 struct timekeeper *tk = &tk_core.timekeeper; in timekeeping_inject_sleeptime64() local
1339 timekeeping_forward_now(tk); in timekeeping_inject_sleeptime64()
1341 __timekeeping_inject_sleeptime(tk, delta); in timekeeping_inject_sleeptime64()
1343 timekeeping_update(tk, TK_CLEAR_NTP | TK_MIRROR | TK_CLOCK_WAS_SET); in timekeeping_inject_sleeptime64()
1358 struct timekeeper *tk = &tk_core.timekeeper; in timekeeping_resume() local
1359 struct clocksource *clock = tk->tkr_mono.clock; in timekeeping_resume()
1385 cycle_now = tk->tkr_mono.read(clock); in timekeeping_resume()
1387 cycle_now > tk->tkr_mono.cycle_last) { in timekeeping_resume()
1393 cycle_delta = clocksource_delta(cycle_now, tk->tkr_mono.cycle_last, in timekeeping_resume()
1394 tk->tkr_mono.mask); in timekeeping_resume()
1417 __timekeeping_inject_sleeptime(tk, &ts_delta); in timekeeping_resume()
1420 tk->tkr_mono.cycle_last = cycle_now; in timekeeping_resume()
1421 tk->tkr_raw.cycle_last = cycle_now; in timekeeping_resume()
1423 tk->ntp_error = 0; in timekeeping_resume()
1425 timekeeping_update(tk, TK_MIRROR | TK_CLOCK_WAS_SET); in timekeeping_resume()
1437 struct timekeeper *tk = &tk_core.timekeeper; in timekeeping_suspend() local
1454 timekeeping_forward_now(tk); in timekeeping_suspend()
1464 delta = timespec64_sub(tk_xtime(tk), timekeeping_suspend_time); in timekeeping_suspend()
1479 timekeeping_update(tk, TK_MIRROR); in timekeeping_suspend()
1480 halt_fast_timekeeper(tk); in timekeeping_suspend()
1507 static __always_inline void timekeeping_apply_adjustment(struct timekeeper *tk, in timekeeping_apply_adjustment() argument
1512 s64 interval = tk->cycle_interval; in timekeeping_apply_adjustment()
1573 if ((mult_adj > 0) && (tk->tkr_mono.mult + mult_adj < mult_adj)) { in timekeeping_apply_adjustment()
1579 tk->tkr_mono.mult += mult_adj; in timekeeping_apply_adjustment()
1580 tk->xtime_interval += interval; in timekeeping_apply_adjustment()
1581 tk->tkr_mono.xtime_nsec -= offset; in timekeeping_apply_adjustment()
1582 tk->ntp_error -= (interval - offset) << tk->ntp_error_shift; in timekeeping_apply_adjustment()
1589 static __always_inline void timekeeping_freqadjust(struct timekeeper *tk, in timekeeping_freqadjust() argument
1592 s64 interval = tk->cycle_interval; in timekeeping_freqadjust()
1593 s64 xinterval = tk->xtime_interval; in timekeeping_freqadjust()
1599 if (tk->ntp_err_mult) in timekeeping_freqadjust()
1600 xinterval -= tk->cycle_interval; in timekeeping_freqadjust()
1602 tk->ntp_tick = ntp_tick_length(); in timekeeping_freqadjust()
1605 tick_error = ntp_tick_length() >> tk->ntp_error_shift; in timekeeping_freqadjust()
1606 tick_error -= (xinterval + tk->xtime_remainder); in timekeeping_freqadjust()
1621 timekeeping_apply_adjustment(tk, offset, negative, adj); in timekeeping_freqadjust()
1628 static void timekeeping_adjust(struct timekeeper *tk, s64 offset) in timekeeping_adjust() argument
1631 timekeeping_freqadjust(tk, offset); in timekeeping_adjust()
1634 if (!tk->ntp_err_mult && (tk->ntp_error > 0)) { in timekeeping_adjust()
1635 tk->ntp_err_mult = 1; in timekeeping_adjust()
1636 timekeeping_apply_adjustment(tk, offset, 0, 0); in timekeeping_adjust()
1637 } else if (tk->ntp_err_mult && (tk->ntp_error <= 0)) { in timekeeping_adjust()
1639 timekeeping_apply_adjustment(tk, offset, 1, 0); in timekeeping_adjust()
1640 tk->ntp_err_mult = 0; in timekeeping_adjust()
1643 if (unlikely(tk->tkr_mono.clock->maxadj && in timekeeping_adjust()
1644 (abs(tk->tkr_mono.mult - tk->tkr_mono.clock->mult) in timekeeping_adjust()
1645 > tk->tkr_mono.clock->maxadj))) { in timekeeping_adjust()
1648 tk->tkr_mono.clock->name, (long)tk->tkr_mono.mult, in timekeeping_adjust()
1649 (long)tk->tkr_mono.clock->mult + tk->tkr_mono.clock->maxadj); in timekeeping_adjust()
1666 if (unlikely((s64)tk->tkr_mono.xtime_nsec < 0)) { in timekeeping_adjust()
1667 s64 neg = -(s64)tk->tkr_mono.xtime_nsec; in timekeeping_adjust()
1668 tk->tkr_mono.xtime_nsec = 0; in timekeeping_adjust()
1669 tk->ntp_error += neg << tk->ntp_error_shift; in timekeeping_adjust()
1681 static inline unsigned int accumulate_nsecs_to_secs(struct timekeeper *tk) in accumulate_nsecs_to_secs() argument
1683 u64 nsecps = (u64)NSEC_PER_SEC << tk->tkr_mono.shift; in accumulate_nsecs_to_secs()
1686 while (tk->tkr_mono.xtime_nsec >= nsecps) { in accumulate_nsecs_to_secs()
1689 tk->tkr_mono.xtime_nsec -= nsecps; in accumulate_nsecs_to_secs()
1690 tk->xtime_sec++; in accumulate_nsecs_to_secs()
1693 leap = second_overflow(tk->xtime_sec); in accumulate_nsecs_to_secs()
1697 tk->xtime_sec += leap; in accumulate_nsecs_to_secs()
1701 tk_set_wall_to_mono(tk, in accumulate_nsecs_to_secs()
1702 timespec64_sub(tk->wall_to_monotonic, ts)); in accumulate_nsecs_to_secs()
1704 __timekeeping_set_tai_offset(tk, tk->tai_offset - leap); in accumulate_nsecs_to_secs()
1721 static cycle_t logarithmic_accumulation(struct timekeeper *tk, cycle_t offset, in logarithmic_accumulation() argument
1725 cycle_t interval = tk->cycle_interval << shift; in logarithmic_accumulation()
1734 tk->tkr_mono.cycle_last += interval; in logarithmic_accumulation()
1735 tk->tkr_raw.cycle_last += interval; in logarithmic_accumulation()
1737 tk->tkr_mono.xtime_nsec += tk->xtime_interval << shift; in logarithmic_accumulation()
1738 *clock_set |= accumulate_nsecs_to_secs(tk); in logarithmic_accumulation()
1741 raw_nsecs = (u64)tk->raw_interval << shift; in logarithmic_accumulation()
1742 raw_nsecs += tk->raw_time.tv_nsec; in logarithmic_accumulation()
1746 tk->raw_time.tv_sec += raw_secs; in logarithmic_accumulation()
1748 tk->raw_time.tv_nsec = raw_nsecs; in logarithmic_accumulation()
1751 tk->ntp_error += tk->ntp_tick << shift; in logarithmic_accumulation()
1752 tk->ntp_error -= (tk->xtime_interval + tk->xtime_remainder) << in logarithmic_accumulation()
1753 (tk->ntp_error_shift + shift); in logarithmic_accumulation()
1765 struct timekeeper *tk = &shadow_timekeeper; in update_wall_time() local
1780 offset = clocksource_delta(tk->tkr_mono.read(tk->tkr_mono.clock), in update_wall_time()
1781 tk->tkr_mono.cycle_last, tk->tkr_mono.mask); in update_wall_time()
1799 shift = ilog2(offset) - ilog2(tk->cycle_interval); in update_wall_time()
1804 while (offset >= tk->cycle_interval) { in update_wall_time()
1805 offset = logarithmic_accumulation(tk, offset, shift, in update_wall_time()
1807 if (offset < tk->cycle_interval<<shift) in update_wall_time()
1812 timekeeping_adjust(tk, offset); in update_wall_time()
1818 old_vsyscall_fixup(tk); in update_wall_time()
1824 clock_set |= accumulate_nsecs_to_secs(tk); in update_wall_time()
1837 timekeeping_update(tk, clock_set); in update_wall_time()
1838 memcpy(real_tk, tk, sizeof(*tk)); in update_wall_time()
1861 struct timekeeper *tk = &tk_core.timekeeper; in getboottime64() local
1862 ktime_t t = ktime_sub(tk->offs_real, tk->offs_boot); in getboottime64()
1870 struct timekeeper *tk = &tk_core.timekeeper; in get_seconds() local
1872 return tk->xtime_sec; in get_seconds()
1878 struct timekeeper *tk = &tk_core.timekeeper; in __current_kernel_time() local
1880 return timespec64_to_timespec(tk_xtime(tk)); in __current_kernel_time()
1885 struct timekeeper *tk = &tk_core.timekeeper; in current_kernel_time64() local
1892 now = tk_xtime(tk); in current_kernel_time64()
1901 struct timekeeper *tk = &tk_core.timekeeper; in get_monotonic_coarse64() local
1908 now = tk_xtime(tk); in get_monotonic_coarse64()
1909 mono = tk->wall_to_monotonic; in get_monotonic_coarse64()
1943 struct timekeeper *tk = &tk_core.timekeeper; in ktime_get_update_offsets_now() local
1951 base = tk->tkr_mono.base; in ktime_get_update_offsets_now()
1952 nsecs = timekeeping_get_ns(&tk->tkr_mono); in ktime_get_update_offsets_now()
1955 if (*cwsseq != tk->clock_was_set_seq) { in ktime_get_update_offsets_now()
1956 *cwsseq = tk->clock_was_set_seq; in ktime_get_update_offsets_now()
1957 *offs_real = tk->offs_real; in ktime_get_update_offsets_now()
1958 *offs_boot = tk->offs_boot; in ktime_get_update_offsets_now()
1959 *offs_tai = tk->offs_tai; in ktime_get_update_offsets_now()
1963 if (unlikely(base.tv64 >= tk->next_leap_ktime.tv64)) in ktime_get_update_offsets_now()
1964 *offs_real = ktime_sub(tk->offs_real, ktime_set(1, 0)); in ktime_get_update_offsets_now()
1976 struct timekeeper *tk = &tk_core.timekeeper; in do_adjtimex() local
2003 orig_tai = tai = tk->tai_offset; in do_adjtimex()
2007 __timekeeping_set_tai_offset(tk, tai); in do_adjtimex()
2008 timekeeping_update(tk, TK_MIRROR | TK_CLOCK_WAS_SET); in do_adjtimex()
2010 tk_update_leap_state(tk); in do_adjtimex()