kernel/time/clocksource.c

/* [<][>][^][v][top][bottom][index][help] */
This source file includes following definitions.
clocks_calc_mult_shift
clocksource_watchdog_lock
clocksource_watchdog_unlock
clocksource_watchdog_work
__clocksource_unstable
clocksource_mark_unstable
clocksource_watchdog
clocksource_start_watchdog
clocksource_stop_watchdog
clocksource_reset_watchdog
clocksource_resume_watchdog
clocksource_enqueue_watchdog
clocksource_select_watchdog
clocksource_dequeue_watchdog
__clocksource_watchdog_kthread
clocksource_watchdog_kthread
clocksource_is_watchdog
clocksource_enqueue_watchdog
clocksource_select_watchdog
clocksource_dequeue_watchdog
clocksource_resume_watchdog
__clocksource_watchdog_kthread
clocksource_is_watchdog
clocksource_mark_unstable
clocksource_watchdog_lock
clocksource_watchdog_unlock
clocksource_is_suspend
__clocksource_suspend_select
clocksource_suspend_select
clocksource_start_suspend_timing
clocksource_stop_suspend_timing
clocksource_suspend
clocksource_resume
clocksource_touch_watchdog
clocksource_max_adjustment
clocks_calc_max_nsecs
clocksource_update_max_deferment
clocksource_find_best
__clocksource_select
clocksource_select
clocksource_select_fallback
clocksource_select
clocksource_select_fallback
clocksource_done_booting
clocksource_enqueue
__clocksource_update_freq_scale
__clocksource_register_scale
__clocksource_change_rating
clocksource_change_rating
clocksource_unbind
clocksource_unregister
current_clocksource_show
sysfs_get_uname
current_clocksource_store
unbind_clocksource_store
available_clocksource_show
init_clocksource_sysfs
boot_override_clocksource
boot_override_clock
   1 // SPDX-License-Identifier: GPL-2.0+
   2 /*
   3  * This file contains the functions which manage clocksource drivers.
   4  *
   5  * Copyright (C) 2004, 2005 IBM, John Stultz (johnstul@us.ibm.com)
   6  */
   7 
   8 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
   9 
  10 #include <linux/device.h>
  11 #include <linux/clocksource.h>
  12 #include <linux/init.h>
  13 #include <linux/module.h>
  14 #include <linux/sched.h> /* for spin_unlock_irq() using preempt_count() m68k */
  15 #include <linux/tick.h>
  16 #include <linux/kthread.h>
  17 
  18 #include "tick-internal.h"
  19 #include "timekeeping_internal.h"
  20 
  21 /**
  22  * clocks_calc_mult_shift - calculate mult/shift factors for scaled math of clocks
  23  * @mult:       pointer to mult variable
  24  * @shift:      pointer to shift variable
  25  * @from:       frequency to convert from
  26  * @to:         frequency to convert to
  27  * @maxsec:     guaranteed runtime conversion range in seconds
  28  *
  29  * The function evaluates the shift/mult pair for the scaled math
  30  * operations of clocksources and clockevents.
  31  *
  32  * @to and @from are frequency values in HZ. For clock sources @to is
  33  * NSEC_PER_SEC == 1GHz and @from is the counter frequency. For clock
  34  * event @to is the counter frequency and @from is NSEC_PER_SEC.
  35  *
  36  * The @maxsec conversion range argument controls the time frame in
  37  * seconds which must be covered by the runtime conversion with the
  38  * calculated mult and shift factors. This guarantees that no 64bit
  39  * overflow happens when the input value of the conversion is
  40  * multiplied with the calculated mult factor. Larger ranges may
  41  * reduce the conversion accuracy by chosing smaller mult and shift
  42  * factors.
  43  */
  44 void
  45 clocks_calc_mult_shift(u32 *mult, u32 *shift, u32 from, u32 to, u32 maxsec)
  46 {
  47         u64 tmp;
  48         u32 sft, sftacc= 32;
  49 
  50         /*
  51          * Calculate the shift factor which is limiting the conversion
  52          * range:
  53          */
  54         tmp = ((u64)maxsec * from) >> 32;
  55         while (tmp) {
  56                 tmp >>=1;
  57                 sftacc--;
  58         }
  59 
  60         /*
  61          * Find the conversion shift/mult pair which has the best
  62          * accuracy and fits the maxsec conversion range:
  63          */
  64         for (sft = 32; sft > 0; sft--) {
  65                 tmp = (u64) to << sft;
  66                 tmp += from / 2;
  67                 do_div(tmp, from);
  68                 if ((tmp >> sftacc) == 0)
  69                         break;
  70         }
  71         *mult = tmp;
  72         *shift = sft;
  73 }
  74 EXPORT_SYMBOL_GPL(clocks_calc_mult_shift);
  75 
  76 /*[Clocksource internal variables]---------
  77  * curr_clocksource:
  78  *      currently selected clocksource.
  79  * suspend_clocksource:
  80  *      used to calculate the suspend time.
  81  * clocksource_list:
  82  *      linked list with the registered clocksources
  83  * clocksource_mutex:
  84  *      protects manipulations to curr_clocksource and the clocksource_list
  85  * override_name:
  86  *      Name of the user-specified clocksource.
  87  */
  88 static struct clocksource *curr_clocksource;
  89 static struct clocksource *suspend_clocksource;
  90 static LIST_HEAD(clocksource_list);
  91 static DEFINE_MUTEX(clocksource_mutex);
  92 static char override_name[CS_NAME_LEN];
  93 static int finished_booting;
  94 static u64 suspend_start;
  95 
  96 #ifdef CONFIG_CLOCKSOURCE_WATCHDOG
  97 static void clocksource_watchdog_work(struct work_struct *work);
  98 static void clocksource_select(void);
  99 
 100 static LIST_HEAD(watchdog_list);
 101 static struct clocksource *watchdog;
 102 static struct timer_list watchdog_timer;
 103 static DECLARE_WORK(watchdog_work, clocksource_watchdog_work);
 104 static DEFINE_SPINLOCK(watchdog_lock);
 105 static int watchdog_running;
 106 static atomic_t watchdog_reset_pending;
 107 
 108 static inline void clocksource_watchdog_lock(unsigned long *flags)
 109 {
 110         spin_lock_irqsave(&watchdog_lock, *flags);
 111 }
 112 
 113 static inline void clocksource_watchdog_unlock(unsigned long *flags)
 114 {
 115         spin_unlock_irqrestore(&watchdog_lock, *flags);
 116 }
 117 
 118 static int clocksource_watchdog_kthread(void *data);
 119 static void __clocksource_change_rating(struct clocksource *cs, int rating);
 120 
 121 /*
 122  * Interval: 0.5sec Threshold: 0.0625s
 123  */
 124 #define WATCHDOG_INTERVAL (HZ >> 1)
 125 #define WATCHDOG_THRESHOLD (NSEC_PER_SEC >> 4)
 126 
 127 static void clocksource_watchdog_work(struct work_struct *work)
 128 {
 129         /*
 130          * We cannot directly run clocksource_watchdog_kthread() here, because
 131          * clocksource_select() calls timekeeping_notify() which uses
 132          * stop_machine(). One cannot use stop_machine() from a workqueue() due
 133          * lock inversions wrt CPU hotplug.
 134          *
 135          * Also, we only ever run this work once or twice during the lifetime
 136          * of the kernel, so there is no point in creating a more permanent
 137          * kthread for this.
 138          *
 139          * If kthread_run fails the next watchdog scan over the
 140          * watchdog_list will find the unstable clock again.
 141          */
 142         kthread_run(clocksource_watchdog_kthread, NULL, "kwatchdog");
 143 }
 144 
 145 static void __clocksource_unstable(struct clocksource *cs)
 146 {
 147         cs->flags &= ~(CLOCK_SOURCE_VALID_FOR_HRES | CLOCK_SOURCE_WATCHDOG);
 148         cs->flags |= CLOCK_SOURCE_UNSTABLE;
 149 
 150         /*
 151          * If the clocksource is registered clocksource_watchdog_kthread() will
 152          * re-rate and re-select.
 153          */
 154         if (list_empty(&cs->list)) {
 155                 cs->rating = 0;
 156                 return;
 157         }
 158 
 159         if (cs->mark_unstable)
 160                 cs->mark_unstable(cs);
 161 
 162         /* kick clocksource_watchdog_kthread() */
 163         if (finished_booting)
 164                 schedule_work(&watchdog_work);
 165 }
 166 
 167 /**
 168  * clocksource_mark_unstable - mark clocksource unstable via watchdog
 169  * @cs:         clocksource to be marked unstable
 170  *
 171  * This function is called by the x86 TSC code to mark clocksources as unstable;
 172  * it defers demotion and re-selection to a kthread.
 173  */
 174 void clocksource_mark_unstable(struct clocksource *cs)
 175 {
 176         unsigned long flags;
 177 
 178         spin_lock_irqsave(&watchdog_lock, flags);
 179         if (!(cs->flags & CLOCK_SOURCE_UNSTABLE)) {
 180                 if (!list_empty(&cs->list) && list_empty(&cs->wd_list))
 181                         list_add(&cs->wd_list, &watchdog_list);
 182                 __clocksource_unstable(cs);
 183         }
 184         spin_unlock_irqrestore(&watchdog_lock, flags);
 185 }
 186 
 187 static void clocksource_watchdog(struct timer_list *unused)
 188 {
 189         struct clocksource *cs;
 190         u64 csnow, wdnow, cslast, wdlast, delta;
 191         int64_t wd_nsec, cs_nsec;
 192         int next_cpu, reset_pending;
 193 
 194         spin_lock(&watchdog_lock);
 195         if (!watchdog_running)
 196                 goto out;
 197 
 198         reset_pending = atomic_read(&watchdog_reset_pending);
 199 
 200         list_for_each_entry(cs, &watchdog_list, wd_list) {
 201 
 202                 /* Clocksource already marked unstable? */
 203                 if (cs->flags & CLOCK_SOURCE_UNSTABLE) {
 204                         if (finished_booting)
 205                                 schedule_work(&watchdog_work);
 206                         continue;
 207                 }
 208 
 209                 local_irq_disable();
 210                 csnow = cs->read(cs);
 211                 wdnow = watchdog->read(watchdog);
 212                 local_irq_enable();
 213 
 214                 /* Clocksource initialized ? */
 215                 if (!(cs->flags & CLOCK_SOURCE_WATCHDOG) ||
 216                     atomic_read(&watchdog_reset_pending)) {
 217                         cs->flags |= CLOCK_SOURCE_WATCHDOG;
 218                         cs->wd_last = wdnow;
 219                         cs->cs_last = csnow;
 220                         continue;
 221                 }
 222 
 223                 delta = clocksource_delta(wdnow, cs->wd_last, watchdog->mask);
 224                 wd_nsec = clocksource_cyc2ns(delta, watchdog->mult,
 225                                              watchdog->shift);
 226 
 227                 delta = clocksource_delta(csnow, cs->cs_last, cs->mask);
 228                 cs_nsec = clocksource_cyc2ns(delta, cs->mult, cs->shift);
 229                 wdlast = cs->wd_last; /* save these in case we print them */
 230                 cslast = cs->cs_last;
 231                 cs->cs_last = csnow;
 232                 cs->wd_last = wdnow;
 233 
 234                 if (atomic_read(&watchdog_reset_pending))
 235                         continue;
 236 
 237                 /* Check the deviation from the watchdog clocksource. */
 238                 if (abs(cs_nsec - wd_nsec) > WATCHDOG_THRESHOLD) {
 239                         pr_warn("timekeeping watchdog on CPU%d: Marking clocksource '%s' as unstable because the skew is too large:\n",
 240                                 smp_processor_id(), cs->name);
 241                         pr_warn("                      '%s' wd_now: %llx wd_last: %llx mask: %llx\n",
 242                                 watchdog->name, wdnow, wdlast, watchdog->mask);
 243                         pr_warn("                      '%s' cs_now: %llx cs_last: %llx mask: %llx\n",
 244                                 cs->name, csnow, cslast, cs->mask);
 245                         __clocksource_unstable(cs);
 246                         continue;
 247                 }
 248 
 249                 if (cs == curr_clocksource && cs->tick_stable)
 250                         cs->tick_stable(cs);
 251 
 252                 if (!(cs->flags & CLOCK_SOURCE_VALID_FOR_HRES) &&
 253                     (cs->flags & CLOCK_SOURCE_IS_CONTINUOUS) &&
 254                     (watchdog->flags & CLOCK_SOURCE_IS_CONTINUOUS)) {
 255                         /* Mark it valid for high-res. */
 256                         cs->flags |= CLOCK_SOURCE_VALID_FOR_HRES;
 257 
 258                         /*
 259                          * clocksource_done_booting() will sort it if
 260                          * finished_booting is not set yet.
 261                          */
 262                         if (!finished_booting)
 263                                 continue;
 264 
 265                         /*
 266                          * If this is not the current clocksource let
 267                          * the watchdog thread reselect it. Due to the
 268                          * change to high res this clocksource might
 269                          * be preferred now. If it is the current
 270                          * clocksource let the tick code know about
 271                          * that change.
 272                          */
 273                         if (cs != curr_clocksource) {
 274                                 cs->flags |= CLOCK_SOURCE_RESELECT;
 275                                 schedule_work(&watchdog_work);
 276                         } else {
 277                                 tick_clock_notify();
 278                         }
 279                 }
 280         }
 281 
 282         /*
 283          * We only clear the watchdog_reset_pending, when we did a
 284          * full cycle through all clocksources.
 285          */
 286         if (reset_pending)
 287                 atomic_dec(&watchdog_reset_pending);
 288 
 289         /*
 290          * Cycle through CPUs to check if the CPUs stay synchronized
 291          * to each other.
 292          */
 293         next_cpu = cpumask_next(raw_smp_processor_id(), cpu_online_mask);
 294         if (next_cpu >= nr_cpu_ids)
 295                 next_cpu = cpumask_first(cpu_online_mask);
 296 
 297         /*
 298          * Arm timer if not already pending: could race with concurrent
 299          * pair clocksource_stop_watchdog() clocksource_start_watchdog().
 300          */
 301         if (!timer_pending(&watchdog_timer)) {
 302                 watchdog_timer.expires += WATCHDOG_INTERVAL;
 303                 add_timer_on(&watchdog_timer, next_cpu);
 304         }
 305 out:
 306         spin_unlock(&watchdog_lock);
 307 }
 308 
 309 static inline void clocksource_start_watchdog(void)
 310 {
 311         if (watchdog_running || !watchdog || list_empty(&watchdog_list))
 312                 return;
 313         timer_setup(&watchdog_timer, clocksource_watchdog, 0);
 314         watchdog_timer.expires = jiffies + WATCHDOG_INTERVAL;
 315         add_timer_on(&watchdog_timer, cpumask_first(cpu_online_mask));
 316         watchdog_running = 1;
 317 }
 318 
 319 static inline void clocksource_stop_watchdog(void)
 320 {
 321         if (!watchdog_running || (watchdog && !list_empty(&watchdog_list)))
 322                 return;
 323         del_timer(&watchdog_timer);
 324         watchdog_running = 0;
 325 }
 326 
 327 static inline void clocksource_reset_watchdog(void)
 328 {
 329         struct clocksource *cs;
 330 
 331         list_for_each_entry(cs, &watchdog_list, wd_list)
 332                 cs->flags &= ~CLOCK_SOURCE_WATCHDOG;
 333 }
 334 
 335 static void clocksource_resume_watchdog(void)
 336 {
 337         atomic_inc(&watchdog_reset_pending);
 338 }
 339 
 340 static void clocksource_enqueue_watchdog(struct clocksource *cs)
 341 {
 342         INIT_LIST_HEAD(&cs->wd_list);
 343 
 344         if (cs->flags & CLOCK_SOURCE_MUST_VERIFY) {
 345                 /* cs is a clocksource to be watched. */
 346                 list_add(&cs->wd_list, &watchdog_list);
 347                 cs->flags &= ~CLOCK_SOURCE_WATCHDOG;
 348         } else {
 349                 /* cs is a watchdog. */
 350                 if (cs->flags & CLOCK_SOURCE_IS_CONTINUOUS)
 351                         cs->flags |= CLOCK_SOURCE_VALID_FOR_HRES;
 352         }
 353 }
 354 
 355 static void clocksource_select_watchdog(bool fallback)
 356 {
 357         struct clocksource *cs, *old_wd;
 358         unsigned long flags;
 359 
 360         spin_lock_irqsave(&watchdog_lock, flags);
 361         /* save current watchdog */
 362         old_wd = watchdog;
 363         if (fallback)
 364                 watchdog = NULL;
 365 
 366         list_for_each_entry(cs, &clocksource_list, list) {
 367                 /* cs is a clocksource to be watched. */
 368                 if (cs->flags & CLOCK_SOURCE_MUST_VERIFY)
 369                         continue;
 370 
 371                 /* Skip current if we were requested for a fallback. */
 372                 if (fallback && cs == old_wd)
 373                         continue;
 374 
 375                 /* Pick the best watchdog. */
 376                 if (!watchdog || cs->rating > watchdog->rating)
 377                         watchdog = cs;
 378         }
 379         /* If we failed to find a fallback restore the old one. */
 380         if (!watchdog)
 381                 watchdog = old_wd;
 382 
 383         /* If we changed the watchdog we need to reset cycles. */
 384         if (watchdog != old_wd)
 385                 clocksource_reset_watchdog();
 386 
 387         /* Check if the watchdog timer needs to be started. */
 388         clocksource_start_watchdog();
 389         spin_unlock_irqrestore(&watchdog_lock, flags);
 390 }
 391 
 392 static void clocksource_dequeue_watchdog(struct clocksource *cs)
 393 {
 394         if (cs != watchdog) {
 395                 if (cs->flags & CLOCK_SOURCE_MUST_VERIFY) {
 396                         /* cs is a watched clocksource. */
 397                         list_del_init(&cs->wd_list);
 398                         /* Check if the watchdog timer needs to be stopped. */
 399                         clocksource_stop_watchdog();
 400                 }
 401         }
 402 }
 403 
 404 static int __clocksource_watchdog_kthread(void)
 405 {
 406         struct clocksource *cs, *tmp;
 407         unsigned long flags;
 408         int select = 0;
 409 
 410         spin_lock_irqsave(&watchdog_lock, flags);
 411         list_for_each_entry_safe(cs, tmp, &watchdog_list, wd_list) {
 412                 if (cs->flags & CLOCK_SOURCE_UNSTABLE) {
 413                         list_del_init(&cs->wd_list);
 414                         __clocksource_change_rating(cs, 0);
 415                         select = 1;
 416                 }
 417                 if (cs->flags & CLOCK_SOURCE_RESELECT) {
 418                         cs->flags &= ~CLOCK_SOURCE_RESELECT;
 419                         select = 1;
 420                 }
 421         }
 422         /* Check if the watchdog timer needs to be stopped. */
 423         clocksource_stop_watchdog();
 424         spin_unlock_irqrestore(&watchdog_lock, flags);
 425 
 426         return select;
 427 }
 428 
 429 static int clocksource_watchdog_kthread(void *data)
 430 {
 431         mutex_lock(&clocksource_mutex);
 432         if (__clocksource_watchdog_kthread())
 433                 clocksource_select();
 434         mutex_unlock(&clocksource_mutex);
 435         return 0;
 436 }
 437 
 438 static bool clocksource_is_watchdog(struct clocksource *cs)
 439 {
 440         return cs == watchdog;
 441 }
 442 
 443 #else /* CONFIG_CLOCKSOURCE_WATCHDOG */
 444 
 445 static void clocksource_enqueue_watchdog(struct clocksource *cs)
 446 {
 447         if (cs->flags & CLOCK_SOURCE_IS_CONTINUOUS)
 448                 cs->flags |= CLOCK_SOURCE_VALID_FOR_HRES;
 449 }
 450 
 451 static void clocksource_select_watchdog(bool fallback) { }
 452 static inline void clocksource_dequeue_watchdog(struct clocksource *cs) { }
 453 static inline void clocksource_resume_watchdog(void) { }
 454 static inline int __clocksource_watchdog_kthread(void) { return 0; }
 455 static bool clocksource_is_watchdog(struct clocksource *cs) { return false; }
 456 void clocksource_mark_unstable(struct clocksource *cs) { }
 457 
 458 static inline void clocksource_watchdog_lock(unsigned long *flags) { }
 459 static inline void clocksource_watchdog_unlock(unsigned long *flags) { }
 460 
 461 #endif /* CONFIG_CLOCKSOURCE_WATCHDOG */
 462 
 463 static bool clocksource_is_suspend(struct clocksource *cs)
 464 {
 465         return cs == suspend_clocksource;
 466 }
 467 
 468 static void __clocksource_suspend_select(struct clocksource *cs)
 469 {
 470         /*
 471          * Skip the clocksource which will be stopped in suspend state.
 472          */
 473         if (!(cs->flags & CLOCK_SOURCE_SUSPEND_NONSTOP))
 474                 return;
 475 
 476         /*
 477          * The nonstop clocksource can be selected as the suspend clocksource to
 478          * calculate the suspend time, so it should not supply suspend/resume
 479          * interfaces to suspend the nonstop clocksource when system suspends.
 480          */
 481         if (cs->suspend || cs->resume) {
 482                 pr_warn("Nonstop clocksource %s should not supply suspend/resume interfaces\n",
 483                         cs->name);
 484         }
 485 
 486         /* Pick the best rating. */
 487         if (!suspend_clocksource || cs->rating > suspend_clocksource->rating)
 488                 suspend_clocksource = cs;
 489 }
 490 
 491 /**
 492  * clocksource_suspend_select - Select the best clocksource for suspend timing
 493  * @fallback:   if select a fallback clocksource
 494  */
 495 static void clocksource_suspend_select(bool fallback)
 496 {
 497         struct clocksource *cs, *old_suspend;
 498 
 499         old_suspend = suspend_clocksource;
 500         if (fallback)
 501                 suspend_clocksource = NULL;
 502 
 503         list_for_each_entry(cs, &clocksource_list, list) {
 504                 /* Skip current if we were requested for a fallback. */
 505                 if (fallback && cs == old_suspend)
 506                         continue;
 507 
 508                 __clocksource_suspend_select(cs);
 509         }
 510 }
 511 
 512 /**
 513  * clocksource_start_suspend_timing - Start measuring the suspend timing
 514  * @cs:                 current clocksource from timekeeping
 515  * @start_cycles:       current cycles from timekeeping
 516  *
 517  * This function will save the start cycle values of suspend timer to calculate
 518  * the suspend time when resuming system.
 519  *
 520  * This function is called late in the suspend process from timekeeping_suspend(),
 521  * that means processes are freezed, non-boot cpus and interrupts are disabled
 522  * now. It is therefore possible to start the suspend timer without taking the
 523  * clocksource mutex.
 524  */
 525 void clocksource_start_suspend_timing(struct clocksource *cs, u64 start_cycles)
 526 {
 527         if (!suspend_clocksource)
 528                 return;
 529 
 530         /*
 531          * If current clocksource is the suspend timer, we should use the
 532          * tkr_mono.cycle_last value as suspend_start to avoid same reading
 533          * from suspend timer.
 534          */
 535         if (clocksource_is_suspend(cs)) {
 536                 suspend_start = start_cycles;
 537                 return;
 538         }
 539 
 540         if (suspend_clocksource->enable &&
 541             suspend_clocksource->enable(suspend_clocksource)) {
 542                 pr_warn_once("Failed to enable the non-suspend-able clocksource.\n");
 543                 return;
 544         }
 545 
 546         suspend_start = suspend_clocksource->read(suspend_clocksource);
 547 }
 548 
 549 /**
 550  * clocksource_stop_suspend_timing - Stop measuring the suspend timing
 551  * @cs:         current clocksource from timekeeping
 552  * @cycle_now:  current cycles from timekeeping
 553  *
 554  * This function will calculate the suspend time from suspend timer.
 555  *
 556  * Returns nanoseconds since suspend started, 0 if no usable suspend clocksource.
 557  *
 558  * This function is called early in the resume process from timekeeping_resume(),
 559  * that means there is only one cpu, no processes are running and the interrupts
 560  * are disabled. It is therefore possible to stop the suspend timer without
 561  * taking the clocksource mutex.
 562  */
 563 u64 clocksource_stop_suspend_timing(struct clocksource *cs, u64 cycle_now)
 564 {
 565         u64 now, delta, nsec = 0;
 566 
 567         if (!suspend_clocksource)
 568                 return 0;
 569 
 570         /*
 571          * If current clocksource is the suspend timer, we should use the
 572          * tkr_mono.cycle_last value from timekeeping as current cycle to
 573          * avoid same reading from suspend timer.
 574          */
 575         if (clocksource_is_suspend(cs))
 576                 now = cycle_now;
 577         else
 578                 now = suspend_clocksource->read(suspend_clocksource);
 579 
 580         if (now > suspend_start) {
 581                 delta = clocksource_delta(now, suspend_start,
 582                                           suspend_clocksource->mask);
 583                 nsec = mul_u64_u32_shr(delta, suspend_clocksource->mult,
 584                                        suspend_clocksource->shift);
 585         }
 586 
 587         /*
 588          * Disable the suspend timer to save power if current clocksource is
 589          * not the suspend timer.
 590          */
 591         if (!clocksource_is_suspend(cs) && suspend_clocksource->disable)
 592                 suspend_clocksource->disable(suspend_clocksource);
 593 
 594         return nsec;
 595 }
 596 
 597 /**
 598  * clocksource_suspend - suspend the clocksource(s)
 599  */
 600 void clocksource_suspend(void)
 601 {
 602         struct clocksource *cs;
 603 
 604         list_for_each_entry_reverse(cs, &clocksource_list, list)
 605                 if (cs->suspend)
 606                         cs->suspend(cs);
 607 }
 608 
 609 /**
 610  * clocksource_resume - resume the clocksource(s)
 611  */
 612 void clocksource_resume(void)
 613 {
 614         struct clocksource *cs;
 615 
 616         list_for_each_entry(cs, &clocksource_list, list)
 617                 if (cs->resume)
 618                         cs->resume(cs);
 619 
 620         clocksource_resume_watchdog();
 621 }
 622 
 623 /**
 624  * clocksource_touch_watchdog - Update watchdog
 625  *
 626  * Update the watchdog after exception contexts such as kgdb so as not
 627  * to incorrectly trip the watchdog. This might fail when the kernel
 628  * was stopped in code which holds watchdog_lock.
 629  */
 630 void clocksource_touch_watchdog(void)
 631 {
 632         clocksource_resume_watchdog();
 633 }
 634 
 635 /**
 636  * clocksource_max_adjustment- Returns max adjustment amount
 637  * @cs:         Pointer to clocksource
 638  *
 639  */
 640 static u32 clocksource_max_adjustment(struct clocksource *cs)
 641 {
 642         u64 ret;
 643         /*
 644          * We won't try to correct for more than 11% adjustments (110,000 ppm),
 645          */
 646         ret = (u64)cs->mult * 11;
 647         do_div(ret,100);
 648         return (u32)ret;
 649 }
 650 
 651 /**
 652  * clocks_calc_max_nsecs - Returns maximum nanoseconds that can be converted
 653  * @mult:       cycle to nanosecond multiplier
 654  * @shift:      cycle to nanosecond divisor (power of two)
 655  * @maxadj:     maximum adjustment value to mult (~11%)
 656  * @mask:       bitmask for two's complement subtraction of non 64 bit counters
 657  * @max_cyc:    maximum cycle value before potential overflow (does not include
 658  *              any safety margin)
 659  *
 660  * NOTE: This function includes a safety margin of 50%, in other words, we
 661  * return half the number of nanoseconds the hardware counter can technically
 662  * cover. This is done so that we can potentially detect problems caused by
 663  * delayed timers or bad hardware, which might result in time intervals that
 664  * are larger than what the math used can handle without overflows.
 665  */
 666 u64 clocks_calc_max_nsecs(u32 mult, u32 shift, u32 maxadj, u64 mask, u64 *max_cyc)
 667 {
 668         u64 max_nsecs, max_cycles;
 669 
 670         /*
 671          * Calculate the maximum number of cycles that we can pass to the
 672          * cyc2ns() function without overflowing a 64-bit result.
 673          */
 674         max_cycles = ULLONG_MAX;
 675         do_div(max_cycles, mult+maxadj);
 676 
 677         /*
 678          * The actual maximum number of cycles we can defer the clocksource is
 679          * determined by the minimum of max_cycles and mask.
 680          * Note: Here we subtract the maxadj to make sure we don't sleep for
 681          * too long if there's a large negative adjustment.
 682          */
 683         max_cycles = min(max_cycles, mask);
 684         max_nsecs = clocksource_cyc2ns(max_cycles, mult - maxadj, shift);
 685 
 686         /* return the max_cycles value as well if requested */
 687         if (max_cyc)
 688                 *max_cyc = max_cycles;
 689 
 690         /* Return 50% of the actual maximum, so we can detect bad values */
 691         max_nsecs >>= 1;
 692 
 693         return max_nsecs;
 694 }
 695 
 696 /**
 697  * clocksource_update_max_deferment - Updates the clocksource max_idle_ns & max_cycles
 698  * @cs:         Pointer to clocksource to be updated
 699  *
 700  */
 701 static inline void clocksource_update_max_deferment(struct clocksource *cs)
 702 {
 703         cs->max_idle_ns = clocks_calc_max_nsecs(cs->mult, cs->shift,
 704                                                 cs->maxadj, cs->mask,
 705                                                 &cs->max_cycles);
 706 }
 707 
 708 #ifndef CONFIG_ARCH_USES_GETTIMEOFFSET
 709 
 710 static struct clocksource *clocksource_find_best(bool oneshot, bool skipcur)
 711 {
 712         struct clocksource *cs;
 713 
 714         if (!finished_booting || list_empty(&clocksource_list))
 715                 return NULL;
 716 
 717         /*
 718          * We pick the clocksource with the highest rating. If oneshot
 719          * mode is active, we pick the highres valid clocksource with
 720          * the best rating.
 721          */
 722         list_for_each_entry(cs, &clocksource_list, list) {
 723                 if (skipcur && cs == curr_clocksource)
 724                         continue;
 725                 if (oneshot && !(cs->flags & CLOCK_SOURCE_VALID_FOR_HRES))
 726                         continue;
 727                 return cs;
 728         }
 729         return NULL;
 730 }
 731 
 732 static void __clocksource_select(bool skipcur)
 733 {
 734         bool oneshot = tick_oneshot_mode_active();
 735         struct clocksource *best, *cs;
 736 
 737         /* Find the best suitable clocksource */
 738         best = clocksource_find_best(oneshot, skipcur);
 739         if (!best)
 740                 return;
 741 
 742         if (!strlen(override_name))
 743                 goto found;
 744 
 745         /* Check for the override clocksource. */
 746         list_for_each_entry(cs, &clocksource_list, list) {
 747                 if (skipcur && cs == curr_clocksource)
 748                         continue;
 749                 if (strcmp(cs->name, override_name) != 0)
 750                         continue;
 751                 /*
 752                  * Check to make sure we don't switch to a non-highres
 753                  * capable clocksource if the tick code is in oneshot
 754                  * mode (highres or nohz)
 755                  */
 756                 if (!(cs->flags & CLOCK_SOURCE_VALID_FOR_HRES) && oneshot) {
 757                         /* Override clocksource cannot be used. */
 758                         if (cs->flags & CLOCK_SOURCE_UNSTABLE) {
 759                                 pr_warn("Override clocksource %s is unstable and not HRT compatible - cannot switch while in HRT/NOHZ mode\n",
 760                                         cs->name);
 761                                 override_name[0] = 0;
 762                         } else {
 763                                 /*
 764                                  * The override cannot be currently verified.
 765                                  * Deferring to let the watchdog check.
 766                                  */
 767                                 pr_info("Override clocksource %s is not currently HRT compatible - deferring\n",
 768                                         cs->name);
 769                         }
 770                 } else
 771                         /* Override clocksource can be used. */
 772                         best = cs;
 773                 break;
 774         }
 775 
 776 found:
 777         if (curr_clocksource != best && !timekeeping_notify(best)) {
 778                 pr_info("Switched to clocksource %s\n", best->name);
 779                 curr_clocksource = best;
 780         }
 781 }
 782 
 783 /**
 784  * clocksource_select - Select the best clocksource available
 785  *
 786  * Private function. Must hold clocksource_mutex when called.
 787  *
 788  * Select the clocksource with the best rating, or the clocksource,
 789  * which is selected by userspace override.
 790  */
 791 static void clocksource_select(void)
 792 {
 793         __clocksource_select(false);
 794 }
 795 
 796 static void clocksource_select_fallback(void)
 797 {
 798         __clocksource_select(true);
 799 }
 800 
 801 #else /* !CONFIG_ARCH_USES_GETTIMEOFFSET */
 802 static inline void clocksource_select(void) { }
 803 static inline void clocksource_select_fallback(void) { }
 804 
 805 #endif
 806 
 807 /*
 808  * clocksource_done_booting - Called near the end of core bootup
 809  *
 810  * Hack to avoid lots of clocksource churn at boot time.
 811  * We use fs_initcall because we want this to start before
 812  * device_initcall but after subsys_initcall.
 813  */
 814 static int __init clocksource_done_booting(void)
 815 {
 816         mutex_lock(&clocksource_mutex);
 817         curr_clocksource = clocksource_default_clock();
 818         finished_booting = 1;
 819         /*
 820          * Run the watchdog first to eliminate unstable clock sources
 821          */
 822         __clocksource_watchdog_kthread();
 823         clocksource_select();
 824         mutex_unlock(&clocksource_mutex);
 825         return 0;
 826 }
 827 fs_initcall(clocksource_done_booting);
 828 
 829 /*
 830  * Enqueue the clocksource sorted by rating
 831  */
 832 static void clocksource_enqueue(struct clocksource *cs)
 833 {
 834         struct list_head *entry = &clocksource_list;
 835         struct clocksource *tmp;
 836 
 837         list_for_each_entry(tmp, &clocksource_list, list) {
 838                 /* Keep track of the place, where to insert */
 839                 if (tmp->rating < cs->rating)
 840                         break;
 841                 entry = &tmp->list;
 842         }
 843         list_add(&cs->list, entry);
 844 }
 845 
 846 /**
 847  * __clocksource_update_freq_scale - Used update clocksource with new freq
 848  * @cs:         clocksource to be registered
 849  * @scale:      Scale factor multiplied against freq to get clocksource hz
 850  * @freq:       clocksource frequency (cycles per second) divided by scale
 851  *
 852  * This should only be called from the clocksource->enable() method.
 853  *
 854  * This *SHOULD NOT* be called directly! Please use the
 855  * __clocksource_update_freq_hz() or __clocksource_update_freq_khz() helper
 856  * functions.
 857  */
 858 void __clocksource_update_freq_scale(struct clocksource *cs, u32 scale, u32 freq)
 859 {
 860         u64 sec;
 861 
 862         /*
 863          * Default clocksources are *special* and self-define their mult/shift.
 864          * But, you're not special, so you should specify a freq value.
 865          */
 866         if (freq) {
 867                 /*
 868                  * Calc the maximum number of seconds which we can run before
 869                  * wrapping around. For clocksources which have a mask > 32-bit
 870                  * we need to limit the max sleep time to have a good
 871                  * conversion precision. 10 minutes is still a reasonable
 872                  * amount. That results in a shift value of 24 for a
 873                  * clocksource with mask >= 40-bit and f >= 4GHz. That maps to
 874                  * ~ 0.06ppm granularity for NTP.
 875                  */
 876                 sec = cs->mask;
 877                 do_div(sec, freq);
 878                 do_div(sec, scale);
 879                 if (!sec)
 880                         sec = 1;
 881                 else if (sec > 600 && cs->mask > UINT_MAX)
 882                         sec = 600;
 883 
 884                 clocks_calc_mult_shift(&cs->mult, &cs->shift, freq,
 885                                        NSEC_PER_SEC / scale, sec * scale);
 886         }
 887         /*
 888          * Ensure clocksources that have large 'mult' values don't overflow
 889          * when adjusted.
 890          */
 891         cs->maxadj = clocksource_max_adjustment(cs);
 892         while (freq && ((cs->mult + cs->maxadj < cs->mult)
 893                 || (cs->mult - cs->maxadj > cs->mult))) {
 894                 cs->mult >>= 1;
 895                 cs->shift--;
 896                 cs->maxadj = clocksource_max_adjustment(cs);
 897         }
 898 
 899         /*
 900          * Only warn for *special* clocksources that self-define
 901          * their mult/shift values and don't specify a freq.
 902          */
 903         WARN_ONCE(cs->mult + cs->maxadj < cs->mult,
 904                 "timekeeping: Clocksource %s might overflow on 11%% adjustment\n",
 905                 cs->name);
 906 
 907         clocksource_update_max_deferment(cs);
 908 
 909         pr_info("%s: mask: 0x%llx max_cycles: 0x%llx, max_idle_ns: %lld ns\n",
 910                 cs->name, cs->mask, cs->max_cycles, cs->max_idle_ns);
 911 }
 912 EXPORT_SYMBOL_GPL(__clocksource_update_freq_scale);
 913 
 914 /**
 915  * __clocksource_register_scale - Used to install new clocksources
 916  * @cs:         clocksource to be registered
 917  * @scale:      Scale factor multiplied against freq to get clocksource hz
 918  * @freq:       clocksource frequency (cycles per second) divided by scale
 919  *
 920  * Returns -EBUSY if registration fails, zero otherwise.
 921  *
 922  * This *SHOULD NOT* be called directly! Please use the
 923  * clocksource_register_hz() or clocksource_register_khz helper functions.
 924  */
 925 int __clocksource_register_scale(struct clocksource *cs, u32 scale, u32 freq)
 926 {
 927         unsigned long flags;
 928 
 929         clocksource_arch_init(cs);
 930 
 931         /* Initialize mult/shift and max_idle_ns */
 932         __clocksource_update_freq_scale(cs, scale, freq);
 933 
 934         /* Add clocksource to the clocksource list */
 935         mutex_lock(&clocksource_mutex);
 936 
 937         clocksource_watchdog_lock(&flags);
 938         clocksource_enqueue(cs);
 939         clocksource_enqueue_watchdog(cs);
 940         clocksource_watchdog_unlock(&flags);
 941 
 942         clocksource_select();
 943         clocksource_select_watchdog(false);
 944         __clocksource_suspend_select(cs);
 945         mutex_unlock(&clocksource_mutex);
 946         return 0;
 947 }
 948 EXPORT_SYMBOL_GPL(__clocksource_register_scale);
 949 
 950 static void __clocksource_change_rating(struct clocksource *cs, int rating)
 951 {
 952         list_del(&cs->list);
 953         cs->rating = rating;
 954         clocksource_enqueue(cs);
 955 }
 956 
 957 /**
 958  * clocksource_change_rating - Change the rating of a registered clocksource
 959  * @cs:         clocksource to be changed
 960  * @rating:     new rating
 961  */
 962 void clocksource_change_rating(struct clocksource *cs, int rating)
 963 {
 964         unsigned long flags;
 965 
 966         mutex_lock(&clocksource_mutex);
 967         clocksource_watchdog_lock(&flags);
 968         __clocksource_change_rating(cs, rating);
 969         clocksource_watchdog_unlock(&flags);
 970 
 971         clocksource_select();
 972         clocksource_select_watchdog(false);
 973         clocksource_suspend_select(false);
 974         mutex_unlock(&clocksource_mutex);
 975 }
 976 EXPORT_SYMBOL(clocksource_change_rating);
 977 
 978 /*
 979  * Unbind clocksource @cs. Called with clocksource_mutex held
 980  */
 981 static int clocksource_unbind(struct clocksource *cs)
 982 {
 983         unsigned long flags;
 984 
 985         if (clocksource_is_watchdog(cs)) {
 986                 /* Select and try to install a replacement watchdog. */
 987                 clocksource_select_watchdog(true);
 988                 if (clocksource_is_watchdog(cs))
 989                         return -EBUSY;
 990         }
 991 
 992         if (cs == curr_clocksource) {
 993                 /* Select and try to install a replacement clock source */
 994                 clocksource_select_fallback();
 995                 if (curr_clocksource == cs)
 996                         return -EBUSY;
 997         }
 998 
 999         if (clocksource_is_suspend(cs)) {
1000                 /*
1001                  * Select and try to install a replacement suspend clocksource.
1002                  * If no replacement suspend clocksource, we will just let the
1003                  * clocksource go and have no suspend clocksource.
1004                  */
1005                 clocksource_suspend_select(true);
1006         }
1007 
1008         clocksource_watchdog_lock(&flags);
1009         clocksource_dequeue_watchdog(cs);
1010         list_del_init(&cs->list);
1011         clocksource_watchdog_unlock(&flags);
1012 
1013         return 0;
1014 }
1015 
1016 /**
1017  * clocksource_unregister - remove a registered clocksource
1018  * @cs: clocksource to be unregistered
1019  */
1020 int clocksource_unregister(struct clocksource *cs)
1021 {
1022         int ret = 0;
1023 
1024         mutex_lock(&clocksource_mutex);
1025         if (!list_empty(&cs->list))
1026                 ret = clocksource_unbind(cs);
1027         mutex_unlock(&clocksource_mutex);
1028         return ret;
1029 }
1030 EXPORT_SYMBOL(clocksource_unregister);
1031 
1032 #ifdef CONFIG_SYSFS
1033 /**
1034  * current_clocksource_show - sysfs interface for current clocksource
1035  * @dev:        unused
1036  * @attr:       unused
1037  * @buf:        char buffer to be filled with clocksource list
1038  *
1039  * Provides sysfs interface for listing current clocksource.
1040  */
1041 static ssize_t current_clocksource_show(struct device *dev,
1042                                         struct device_attribute *attr,
1043                                         char *buf)
1044 {
1045         ssize_t count = 0;
1046 
1047         mutex_lock(&clocksource_mutex);
1048         count = snprintf(buf, PAGE_SIZE, "%s\n", curr_clocksource->name);
1049         mutex_unlock(&clocksource_mutex);
1050 
1051         return count;
1052 }
1053 
1054 ssize_t sysfs_get_uname(const char *buf, char *dst, size_t cnt)
1055 {
1056         size_t ret = cnt;
1057 
1058         /* strings from sysfs write are not 0 terminated! */
1059         if (!cnt || cnt >= CS_NAME_LEN)
1060                 return -EINVAL;
1061 
1062         /* strip of \n: */
1063         if (buf[cnt-1] == '\n')
1064                 cnt--;
1065         if (cnt > 0)
1066                 memcpy(dst, buf, cnt);
1067         dst[cnt] = 0;
1068         return ret;
1069 }
1070 
1071 /**
1072  * current_clocksource_store - interface for manually overriding clocksource
1073  * @dev:        unused
1074  * @attr:       unused
1075  * @buf:        name of override clocksource
1076  * @count:      length of buffer
1077  *
1078  * Takes input from sysfs interface for manually overriding the default
1079  * clocksource selection.
1080  */
1081 static ssize_t current_clocksource_store(struct device *dev,
1082                                          struct device_attribute *attr,
1083                                          const char *buf, size_t count)
1084 {
1085         ssize_t ret;
1086 
1087         mutex_lock(&clocksource_mutex);
1088 
1089         ret = sysfs_get_uname(buf, override_name, count);
1090         if (ret >= 0)
1091                 clocksource_select();
1092 
1093         mutex_unlock(&clocksource_mutex);
1094 
1095         return ret;
1096 }
1097 static DEVICE_ATTR_RW(current_clocksource);
1098 
1099 /**
1100  * unbind_clocksource_store - interface for manually unbinding clocksource
1101  * @dev:        unused
1102  * @attr:       unused
1103  * @buf:        unused
1104  * @count:      length of buffer
1105  *
1106  * Takes input from sysfs interface for manually unbinding a clocksource.
1107  */
1108 static ssize_t unbind_clocksource_store(struct device *dev,
1109                                         struct device_attribute *attr,
1110                                         const char *buf, size_t count)
1111 {
1112         struct clocksource *cs;
1113         char name[CS_NAME_LEN];
1114         ssize_t ret;
1115 
1116         ret = sysfs_get_uname(buf, name, count);
1117         if (ret < 0)
1118                 return ret;
1119 
1120         ret = -ENODEV;
1121         mutex_lock(&clocksource_mutex);
1122         list_for_each_entry(cs, &clocksource_list, list) {
1123                 if (strcmp(cs->name, name))
1124                         continue;
1125                 ret = clocksource_unbind(cs);
1126                 break;
1127         }
1128         mutex_unlock(&clocksource_mutex);
1129 
1130         return ret ? ret : count;
1131 }
1132 static DEVICE_ATTR_WO(unbind_clocksource);
1133 
1134 /**
1135  * available_clocksource_show - sysfs interface for listing clocksource
1136  * @dev:        unused
1137  * @attr:       unused
1138  * @buf:        char buffer to be filled with clocksource list
1139  *
1140  * Provides sysfs interface for listing registered clocksources
1141  */
1142 static ssize_t available_clocksource_show(struct device *dev,
1143                                           struct device_attribute *attr,
1144                                           char *buf)
1145 {
1146         struct clocksource *src;
1147         ssize_t count = 0;
1148 
1149         mutex_lock(&clocksource_mutex);
1150         list_for_each_entry(src, &clocksource_list, list) {
1151                 /*
1152                  * Don't show non-HRES clocksource if the tick code is
1153                  * in one shot mode (highres=on or nohz=on)
1154                  */
1155                 if (!tick_oneshot_mode_active() ||
1156                     (src->flags & CLOCK_SOURCE_VALID_FOR_HRES))
1157                         count += snprintf(buf + count,
1158                                   max((ssize_t)PAGE_SIZE - count, (ssize_t)0),
1159                                   "%s ", src->name);
1160         }
1161         mutex_unlock(&clocksource_mutex);
1162 
1163         count += snprintf(buf + count,
1164                           max((ssize_t)PAGE_SIZE - count, (ssize_t)0), "\n");
1165 
1166         return count;
1167 }
1168 static DEVICE_ATTR_RO(available_clocksource);
1169 
1170 static struct attribute *clocksource_attrs[] = {
1171         &dev_attr_current_clocksource.attr,
1172         &dev_attr_unbind_clocksource.attr,
1173         &dev_attr_available_clocksource.attr,
1174         NULL
1175 };
1176 ATTRIBUTE_GROUPS(clocksource);
1177 
1178 static struct bus_type clocksource_subsys = {
1179         .name = "clocksource",
1180         .dev_name = "clocksource",
1181 };
1182 
1183 static struct device device_clocksource = {
1184         .id     = 0,
1185         .bus    = &clocksource_subsys,
1186         .groups = clocksource_groups,
1187 };
1188 
1189 static int __init init_clocksource_sysfs(void)
1190 {
1191         int error = subsys_system_register(&clocksource_subsys, NULL);
1192 
1193         if (!error)
1194                 error = device_register(&device_clocksource);
1195 
1196         return error;
1197 }
1198 
1199 device_initcall(init_clocksource_sysfs);
1200 #endif /* CONFIG_SYSFS */
1201 
1202 /**
1203  * boot_override_clocksource - boot clock override
1204  * @str:        override name
1205  *
1206  * Takes a clocksource= boot argument and uses it
1207  * as the clocksource override name.
1208  */
1209 static int __init boot_override_clocksource(char* str)
1210 {
1211         mutex_lock(&clocksource_mutex);
1212         if (str)
1213                 strlcpy(override_name, str, sizeof(override_name));
1214         mutex_unlock(&clocksource_mutex);
1215         return 1;
1216 }
1217 
1218 __setup("clocksource=", boot_override_clocksource);
1219 
1220 /**
1221  * boot_override_clock - Compatibility layer for deprecated boot option
1222  * @str:        override name
1223  *
1224  * DEPRECATED! Takes a clock= boot argument and uses it
1225  * as the clocksource override name
1226  */
1227 static int __init boot_override_clock(char* str)
1228 {
1229         if (!strcmp(str, "pmtmr")) {
1230                 pr_warn("clock=pmtmr is deprecated - use clocksource=acpi_pm\n");
1231                 return boot_override_clocksource("acpi_pm");
1232         }
1233         pr_warn("clock= boot option is deprecated - use clocksource=xyz\n");
1234         return boot_override_clocksource(str);
1235 }
1236 
1237 __setup("clock=", boot_override_clock);
/* [<][>][^][v][top][bottom][index][help] */
root/kernel/time/clocksource.c

DEFINITIONS
