drivers/clocksource/arm_arch

/* [<][>][^][v][top][bottom][index][help] */
This source file includes following definitions.
early_evtstrm_cfg
arch_timer_reg_write
arch_timer_reg_read
arch_counter_get_cntpct_stable
arch_counter_get_cntpct
arch_counter_get_cntvct_stable
arch_counter_get_cntvct
arch_counter_read
arch_counter_read_cc
fsl_a008585_read_cntp_tval_el0
fsl_a008585_read_cntv_tval_el0
fsl_a008585_read_cntpct_el0
fsl_a008585_read_cntvct_el0
hisi_161010101_read_cntp_tval_el0
hisi_161010101_read_cntv_tval_el0
hisi_161010101_read_cntpct_el0
hisi_161010101_read_cntvct_el0
arm64_858921_read_cntpct_el0
arm64_858921_read_cntvct_el0
sun50i_a64_read_cntpct_el0
sun50i_a64_read_cntvct_el0
sun50i_a64_read_cntp_tval_el0
sun50i_a64_read_cntv_tval_el0
erratum_set_next_event_tval_generic
erratum_set_next_event_tval_virt
erratum_set_next_event_tval_phys
arch_timer_check_dt_erratum
arch_timer_check_local_cap_erratum
arch_timer_check_acpi_oem_erratum
arch_timer_iterate_errata
arch_timer_enable_workaround
arch_timer_check_ool_workaround
arch_timer_this_cpu_has_cntvct_wa
arch_timer_counter_has_wa
timer_handler
arch_timer_handler_virt
arch_timer_handler_phys
arch_timer_handler_phys_mem
arch_timer_handler_virt_mem
timer_shutdown
arch_timer_shutdown_virt
arch_timer_shutdown_phys
arch_timer_shutdown_virt_mem
arch_timer_shutdown_phys_mem
set_next_event
arch_timer_set_next_event_virt
arch_timer_set_next_event_phys
arch_timer_set_next_event_virt_mem
arch_timer_set_next_event_phys_mem
__arch_timer_setup
arch_timer_evtstrm_enable
arch_timer_configure_evtstream
arch_counter_set_user_access
arch_timer_has_nonsecure_ppi
check_ppi_trigger
arch_timer_starting_cpu
arch_timer_of_configure_rate
arch_timer_banner
arch_timer_get_rate
arch_timer_evtstrm_available
arch_counter_get_cntvct_mem
arch_timer_get_kvm_info
arch_counter_register
arch_timer_stop
arch_timer_dying_cpu
arch_timer_cpu_pm_notify
arch_timer_cpu_pm_init
arch_timer_cpu_pm_deinit
arch_timer_cpu_pm_init
arch_timer_cpu_pm_deinit
arch_timer_register
arch_timer_mem_register
arch_timer_needs_of_probing
arch_timer_common_init
arch_timer_select_ppi
arch_timer_populate_kvm_info
arch_timer_of_init
arch_timer_mem_frame_get_cntfrq
arch_timer_mem_find_best_frame
arch_timer_mem_frame_register
arch_timer_mem_of_init
arch_timer_mem_verify_cntfrq
arch_timer_mem_acpi_init
arch_timer_acpi_init
   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  *  linux/drivers/clocksource/arm_arch_timer.c
   4  *
   5  *  Copyright (C) 2011 ARM Ltd.
   6  *  All Rights Reserved
   7  */
   8 
   9 #define pr_fmt(fmt)     "arch_timer: " fmt
  10 
  11 #include <linux/init.h>
  12 #include <linux/kernel.h>
  13 #include <linux/device.h>
  14 #include <linux/smp.h>
  15 #include <linux/cpu.h>
  16 #include <linux/cpu_pm.h>
  17 #include <linux/clockchips.h>
  18 #include <linux/clocksource.h>
  19 #include <linux/interrupt.h>
  20 #include <linux/of_irq.h>
  21 #include <linux/of_address.h>
  22 #include <linux/io.h>
  23 #include <linux/slab.h>
  24 #include <linux/sched/clock.h>
  25 #include <linux/sched_clock.h>
  26 #include <linux/acpi.h>
  27 
  28 #include <asm/arch_timer.h>
  29 #include <asm/virt.h>
  30 
  31 #include <clocksource/arm_arch_timer.h>
  32 
  33 #define CNTTIDR         0x08
  34 #define CNTTIDR_VIRT(n) (BIT(1) << ((n) * 4))
  35 
  36 #define CNTACR(n)       (0x40 + ((n) * 4))
  37 #define CNTACR_RPCT     BIT(0)
  38 #define CNTACR_RVCT     BIT(1)
  39 #define CNTACR_RFRQ     BIT(2)
  40 #define CNTACR_RVOFF    BIT(3)
  41 #define CNTACR_RWVT     BIT(4)
  42 #define CNTACR_RWPT     BIT(5)
  43 
  44 #define CNTVCT_LO       0x08
  45 #define CNTVCT_HI       0x0c
  46 #define CNTFRQ          0x10
  47 #define CNTP_TVAL       0x28
  48 #define CNTP_CTL        0x2c
  49 #define CNTV_TVAL       0x38
  50 #define CNTV_CTL        0x3c
  51 
  52 static unsigned arch_timers_present __initdata;
  53 
  54 static void __iomem *arch_counter_base;
  55 
  56 struct arch_timer {
  57         void __iomem *base;
  58         struct clock_event_device evt;
  59 };
  60 
  61 #define to_arch_timer(e) container_of(e, struct arch_timer, evt)
  62 
  63 static u32 arch_timer_rate;
  64 static int arch_timer_ppi[ARCH_TIMER_MAX_TIMER_PPI];
  65 
  66 static struct clock_event_device __percpu *arch_timer_evt;
  67 
  68 static enum arch_timer_ppi_nr arch_timer_uses_ppi = ARCH_TIMER_VIRT_PPI;
  69 static bool arch_timer_c3stop;
  70 static bool arch_timer_mem_use_virtual;
  71 static bool arch_counter_suspend_stop;
  72 static bool vdso_default = true;
  73 
  74 static cpumask_t evtstrm_available = CPU_MASK_NONE;
  75 static bool evtstrm_enable = IS_ENABLED(CONFIG_ARM_ARCH_TIMER_EVTSTREAM);
  76 
  77 static int __init early_evtstrm_cfg(char *buf)
  78 {
  79         return strtobool(buf, &evtstrm_enable);
  80 }
  81 early_param("clocksource.arm_arch_timer.evtstrm", early_evtstrm_cfg);
  82 
  83 /*
  84  * Architected system timer support.
  85  */
  86 
  87 static __always_inline
  88 void arch_timer_reg_write(int access, enum arch_timer_reg reg, u32 val,
  89                           struct clock_event_device *clk)
  90 {
  91         if (access == ARCH_TIMER_MEM_PHYS_ACCESS) {
  92                 struct arch_timer *timer = to_arch_timer(clk);
  93                 switch (reg) {
  94                 case ARCH_TIMER_REG_CTRL:
  95                         writel_relaxed(val, timer->base + CNTP_CTL);
  96                         break;
  97                 case ARCH_TIMER_REG_TVAL:
  98                         writel_relaxed(val, timer->base + CNTP_TVAL);
  99                         break;
 100                 }
 101         } else if (access == ARCH_TIMER_MEM_VIRT_ACCESS) {
 102                 struct arch_timer *timer = to_arch_timer(clk);
 103                 switch (reg) {
 104                 case ARCH_TIMER_REG_CTRL:
 105                         writel_relaxed(val, timer->base + CNTV_CTL);
 106                         break;
 107                 case ARCH_TIMER_REG_TVAL:
 108                         writel_relaxed(val, timer->base + CNTV_TVAL);
 109                         break;
 110                 }
 111         } else {
 112                 arch_timer_reg_write_cp15(access, reg, val);
 113         }
 114 }
 115 
 116 static __always_inline
 117 u32 arch_timer_reg_read(int access, enum arch_timer_reg reg,
 118                         struct clock_event_device *clk)
 119 {
 120         u32 val;
 121 
 122         if (access == ARCH_TIMER_MEM_PHYS_ACCESS) {
 123                 struct arch_timer *timer = to_arch_timer(clk);
 124                 switch (reg) {
 125                 case ARCH_TIMER_REG_CTRL:
 126                         val = readl_relaxed(timer->base + CNTP_CTL);
 127                         break;
 128                 case ARCH_TIMER_REG_TVAL:
 129                         val = readl_relaxed(timer->base + CNTP_TVAL);
 130                         break;
 131                 }
 132         } else if (access == ARCH_TIMER_MEM_VIRT_ACCESS) {
 133                 struct arch_timer *timer = to_arch_timer(clk);
 134                 switch (reg) {
 135                 case ARCH_TIMER_REG_CTRL:
 136                         val = readl_relaxed(timer->base + CNTV_CTL);
 137                         break;
 138                 case ARCH_TIMER_REG_TVAL:
 139                         val = readl_relaxed(timer->base + CNTV_TVAL);
 140                         break;
 141                 }
 142         } else {
 143                 val = arch_timer_reg_read_cp15(access, reg);
 144         }
 145 
 146         return val;
 147 }
 148 
 149 static notrace u64 arch_counter_get_cntpct_stable(void)
 150 {
 151         return __arch_counter_get_cntpct_stable();
 152 }
 153 
 154 static notrace u64 arch_counter_get_cntpct(void)
 155 {
 156         return __arch_counter_get_cntpct();
 157 }
 158 
 159 static notrace u64 arch_counter_get_cntvct_stable(void)
 160 {
 161         return __arch_counter_get_cntvct_stable();
 162 }
 163 
 164 static notrace u64 arch_counter_get_cntvct(void)
 165 {
 166         return __arch_counter_get_cntvct();
 167 }
 168 
 169 /*
 170  * Default to cp15 based access because arm64 uses this function for
 171  * sched_clock() before DT is probed and the cp15 method is guaranteed
 172  * to exist on arm64. arm doesn't use this before DT is probed so even
 173  * if we don't have the cp15 accessors we won't have a problem.
 174  */
 175 u64 (*arch_timer_read_counter)(void) = arch_counter_get_cntvct;
 176 EXPORT_SYMBOL_GPL(arch_timer_read_counter);
 177 
 178 static u64 arch_counter_read(struct clocksource *cs)
 179 {
 180         return arch_timer_read_counter();
 181 }
 182 
 183 static u64 arch_counter_read_cc(const struct cyclecounter *cc)
 184 {
 185         return arch_timer_read_counter();
 186 }
 187 
 188 static struct clocksource clocksource_counter = {
 189         .name   = "arch_sys_counter",
 190         .rating = 400,
 191         .read   = arch_counter_read,
 192         .mask   = CLOCKSOURCE_MASK(56),
 193         .flags  = CLOCK_SOURCE_IS_CONTINUOUS,
 194 };
 195 
 196 static struct cyclecounter cyclecounter __ro_after_init = {
 197         .read   = arch_counter_read_cc,
 198         .mask   = CLOCKSOURCE_MASK(56),
 199 };
 200 
 201 struct ate_acpi_oem_info {
 202         char oem_id[ACPI_OEM_ID_SIZE + 1];
 203         char oem_table_id[ACPI_OEM_TABLE_ID_SIZE + 1];
 204         u32 oem_revision;
 205 };
 206 
 207 #ifdef CONFIG_FSL_ERRATUM_A008585
 208 /*
 209  * The number of retries is an arbitrary value well beyond the highest number
 210  * of iterations the loop has been observed to take.
 211  */
 212 #define __fsl_a008585_read_reg(reg) ({                  \
 213         u64 _old, _new;                                 \
 214         int _retries = 200;                             \
 215                                                         \
 216         do {                                            \
 217                 _old = read_sysreg(reg);                \
 218                 _new = read_sysreg(reg);                \
 219                 _retries--;                             \
 220         } while (unlikely(_old != _new) && _retries);   \
 221                                                         \
 222         WARN_ON_ONCE(!_retries);                        \
 223         _new;                                           \
 224 })
 225 
 226 static u32 notrace fsl_a008585_read_cntp_tval_el0(void)
 227 {
 228         return __fsl_a008585_read_reg(cntp_tval_el0);
 229 }
 230 
 231 static u32 notrace fsl_a008585_read_cntv_tval_el0(void)
 232 {
 233         return __fsl_a008585_read_reg(cntv_tval_el0);
 234 }
 235 
 236 static u64 notrace fsl_a008585_read_cntpct_el0(void)
 237 {
 238         return __fsl_a008585_read_reg(cntpct_el0);
 239 }
 240 
 241 static u64 notrace fsl_a008585_read_cntvct_el0(void)
 242 {
 243         return __fsl_a008585_read_reg(cntvct_el0);
 244 }
 245 #endif
 246 
 247 #ifdef CONFIG_HISILICON_ERRATUM_161010101
 248 /*
 249  * Verify whether the value of the second read is larger than the first by
 250  * less than 32 is the only way to confirm the value is correct, so clear the
 251  * lower 5 bits to check whether the difference is greater than 32 or not.
 252  * Theoretically the erratum should not occur more than twice in succession
 253  * when reading the system counter, but it is possible that some interrupts
 254  * may lead to more than twice read errors, triggering the warning, so setting
 255  * the number of retries far beyond the number of iterations the loop has been
 256  * observed to take.
 257  */
 258 #define __hisi_161010101_read_reg(reg) ({                               \
 259         u64 _old, _new;                                         \
 260         int _retries = 50;                                      \
 261                                                                 \
 262         do {                                                    \
 263                 _old = read_sysreg(reg);                        \
 264                 _new = read_sysreg(reg);                        \
 265                 _retries--;                                     \
 266         } while (unlikely((_new - _old) >> 5) && _retries);     \
 267                                                                 \
 268         WARN_ON_ONCE(!_retries);                                \
 269         _new;                                                   \
 270 })
 271 
 272 static u32 notrace hisi_161010101_read_cntp_tval_el0(void)
 273 {
 274         return __hisi_161010101_read_reg(cntp_tval_el0);
 275 }
 276 
 277 static u32 notrace hisi_161010101_read_cntv_tval_el0(void)
 278 {
 279         return __hisi_161010101_read_reg(cntv_tval_el0);
 280 }
 281 
 282 static u64 notrace hisi_161010101_read_cntpct_el0(void)
 283 {
 284         return __hisi_161010101_read_reg(cntpct_el0);
 285 }
 286 
 287 static u64 notrace hisi_161010101_read_cntvct_el0(void)
 288 {
 289         return __hisi_161010101_read_reg(cntvct_el0);
 290 }
 291 
 292 static struct ate_acpi_oem_info hisi_161010101_oem_info[] = {
 293         /*
 294          * Note that trailing spaces are required to properly match
 295          * the OEM table information.
 296          */
 297         {
 298                 .oem_id         = "HISI  ",
 299                 .oem_table_id   = "HIP05   ",
 300                 .oem_revision   = 0,
 301         },
 302         {
 303                 .oem_id         = "HISI  ",
 304                 .oem_table_id   = "HIP06   ",
 305                 .oem_revision   = 0,
 306         },
 307         {
 308                 .oem_id         = "HISI  ",
 309                 .oem_table_id   = "HIP07   ",
 310                 .oem_revision   = 0,
 311         },
 312         { /* Sentinel indicating the end of the OEM array */ },
 313 };
 314 #endif
 315 
 316 #ifdef CONFIG_ARM64_ERRATUM_858921
 317 static u64 notrace arm64_858921_read_cntpct_el0(void)
 318 {
 319         u64 old, new;
 320 
 321         old = read_sysreg(cntpct_el0);
 322         new = read_sysreg(cntpct_el0);
 323         return (((old ^ new) >> 32) & 1) ? old : new;
 324 }
 325 
 326 static u64 notrace arm64_858921_read_cntvct_el0(void)
 327 {
 328         u64 old, new;
 329 
 330         old = read_sysreg(cntvct_el0);
 331         new = read_sysreg(cntvct_el0);
 332         return (((old ^ new) >> 32) & 1) ? old : new;
 333 }
 334 #endif
 335 
 336 #ifdef CONFIG_SUN50I_ERRATUM_UNKNOWN1
 337 /*
 338  * The low bits of the counter registers are indeterminate while bit 10 or
 339  * greater is rolling over. Since the counter value can jump both backward
 340  * (7ff -> 000 -> 800) and forward (7ff -> fff -> 800), ignore register values
 341  * with all ones or all zeros in the low bits. Bound the loop by the maximum
 342  * number of CPU cycles in 3 consecutive 24 MHz counter periods.
 343  */
 344 #define __sun50i_a64_read_reg(reg) ({                                   \
 345         u64 _val;                                                       \
 346         int _retries = 150;                                             \
 347                                                                         \
 348         do {                                                            \
 349                 _val = read_sysreg(reg);                                \
 350                 _retries--;                                             \
 351         } while (((_val + 1) & GENMASK(9, 0)) <= 1 && _retries);        \
 352                                                                         \
 353         WARN_ON_ONCE(!_retries);                                        \
 354         _val;                                                           \
 355 })
 356 
 357 static u64 notrace sun50i_a64_read_cntpct_el0(void)
 358 {
 359         return __sun50i_a64_read_reg(cntpct_el0);
 360 }
 361 
 362 static u64 notrace sun50i_a64_read_cntvct_el0(void)
 363 {
 364         return __sun50i_a64_read_reg(cntvct_el0);
 365 }
 366 
 367 static u32 notrace sun50i_a64_read_cntp_tval_el0(void)
 368 {
 369         return read_sysreg(cntp_cval_el0) - sun50i_a64_read_cntpct_el0();
 370 }
 371 
 372 static u32 notrace sun50i_a64_read_cntv_tval_el0(void)
 373 {
 374         return read_sysreg(cntv_cval_el0) - sun50i_a64_read_cntvct_el0();
 375 }
 376 #endif
 377 
 378 #ifdef CONFIG_ARM_ARCH_TIMER_OOL_WORKAROUND
 379 DEFINE_PER_CPU(const struct arch_timer_erratum_workaround *, timer_unstable_counter_workaround);
 380 EXPORT_SYMBOL_GPL(timer_unstable_counter_workaround);
 381 
 382 static atomic_t timer_unstable_counter_workaround_in_use = ATOMIC_INIT(0);
 383 
 384 static void erratum_set_next_event_tval_generic(const int access, unsigned long evt,
 385                                                 struct clock_event_device *clk)
 386 {
 387         unsigned long ctrl;
 388         u64 cval;
 389 
 390         ctrl = arch_timer_reg_read(access, ARCH_TIMER_REG_CTRL, clk);
 391         ctrl |= ARCH_TIMER_CTRL_ENABLE;
 392         ctrl &= ~ARCH_TIMER_CTRL_IT_MASK;
 393 
 394         if (access == ARCH_TIMER_PHYS_ACCESS) {
 395                 cval = evt + arch_counter_get_cntpct();
 396                 write_sysreg(cval, cntp_cval_el0);
 397         } else {
 398                 cval = evt + arch_counter_get_cntvct();
 399                 write_sysreg(cval, cntv_cval_el0);
 400         }
 401 
 402         arch_timer_reg_write(access, ARCH_TIMER_REG_CTRL, ctrl, clk);
 403 }
 404 
 405 static __maybe_unused int erratum_set_next_event_tval_virt(unsigned long evt,
 406                                             struct clock_event_device *clk)
 407 {
 408         erratum_set_next_event_tval_generic(ARCH_TIMER_VIRT_ACCESS, evt, clk);
 409         return 0;
 410 }
 411 
 412 static __maybe_unused int erratum_set_next_event_tval_phys(unsigned long evt,
 413                                             struct clock_event_device *clk)
 414 {
 415         erratum_set_next_event_tval_generic(ARCH_TIMER_PHYS_ACCESS, evt, clk);
 416         return 0;
 417 }
 418 
 419 static const struct arch_timer_erratum_workaround ool_workarounds[] = {
 420 #ifdef CONFIG_FSL_ERRATUM_A008585
 421         {
 422                 .match_type = ate_match_dt,
 423                 .id = "fsl,erratum-a008585",
 424                 .desc = "Freescale erratum a005858",
 425                 .read_cntp_tval_el0 = fsl_a008585_read_cntp_tval_el0,
 426                 .read_cntv_tval_el0 = fsl_a008585_read_cntv_tval_el0,
 427                 .read_cntpct_el0 = fsl_a008585_read_cntpct_el0,
 428                 .read_cntvct_el0 = fsl_a008585_read_cntvct_el0,
 429                 .set_next_event_phys = erratum_set_next_event_tval_phys,
 430                 .set_next_event_virt = erratum_set_next_event_tval_virt,
 431         },
 432 #endif
 433 #ifdef CONFIG_HISILICON_ERRATUM_161010101
 434         {
 435                 .match_type = ate_match_dt,
 436                 .id = "hisilicon,erratum-161010101",
 437                 .desc = "HiSilicon erratum 161010101",
 438                 .read_cntp_tval_el0 = hisi_161010101_read_cntp_tval_el0,
 439                 .read_cntv_tval_el0 = hisi_161010101_read_cntv_tval_el0,
 440                 .read_cntpct_el0 = hisi_161010101_read_cntpct_el0,
 441                 .read_cntvct_el0 = hisi_161010101_read_cntvct_el0,
 442                 .set_next_event_phys = erratum_set_next_event_tval_phys,
 443                 .set_next_event_virt = erratum_set_next_event_tval_virt,
 444         },
 445         {
 446                 .match_type = ate_match_acpi_oem_info,
 447                 .id = hisi_161010101_oem_info,
 448                 .desc = "HiSilicon erratum 161010101",
 449                 .read_cntp_tval_el0 = hisi_161010101_read_cntp_tval_el0,
 450                 .read_cntv_tval_el0 = hisi_161010101_read_cntv_tval_el0,
 451                 .read_cntpct_el0 = hisi_161010101_read_cntpct_el0,
 452                 .read_cntvct_el0 = hisi_161010101_read_cntvct_el0,
 453                 .set_next_event_phys = erratum_set_next_event_tval_phys,
 454                 .set_next_event_virt = erratum_set_next_event_tval_virt,
 455         },
 456 #endif
 457 #ifdef CONFIG_ARM64_ERRATUM_858921
 458         {
 459                 .match_type = ate_match_local_cap_id,
 460                 .id = (void *)ARM64_WORKAROUND_858921,
 461                 .desc = "ARM erratum 858921",
 462                 .read_cntpct_el0 = arm64_858921_read_cntpct_el0,
 463                 .read_cntvct_el0 = arm64_858921_read_cntvct_el0,
 464         },
 465 #endif
 466 #ifdef CONFIG_SUN50I_ERRATUM_UNKNOWN1
 467         {
 468                 .match_type = ate_match_dt,
 469                 .id = "allwinner,erratum-unknown1",
 470                 .desc = "Allwinner erratum UNKNOWN1",
 471                 .read_cntp_tval_el0 = sun50i_a64_read_cntp_tval_el0,
 472                 .read_cntv_tval_el0 = sun50i_a64_read_cntv_tval_el0,
 473                 .read_cntpct_el0 = sun50i_a64_read_cntpct_el0,
 474                 .read_cntvct_el0 = sun50i_a64_read_cntvct_el0,
 475                 .set_next_event_phys = erratum_set_next_event_tval_phys,
 476                 .set_next_event_virt = erratum_set_next_event_tval_virt,
 477         },
 478 #endif
 479 };
 480 
 481 typedef bool (*ate_match_fn_t)(const struct arch_timer_erratum_workaround *,
 482                                const void *);
 483 
 484 static
 485 bool arch_timer_check_dt_erratum(const struct arch_timer_erratum_workaround *wa,
 486                                  const void *arg)
 487 {
 488         const struct device_node *np = arg;
 489 
 490         return of_property_read_bool(np, wa->id);
 491 }
 492 
 493 static
 494 bool arch_timer_check_local_cap_erratum(const struct arch_timer_erratum_workaround *wa,
 495                                         const void *arg)
 496 {
 497         return this_cpu_has_cap((uintptr_t)wa->id);
 498 }
 499 
 500 
 501 static
 502 bool arch_timer_check_acpi_oem_erratum(const struct arch_timer_erratum_workaround *wa,
 503                                        const void *arg)
 504 {
 505         static const struct ate_acpi_oem_info empty_oem_info = {};
 506         const struct ate_acpi_oem_info *info = wa->id;
 507         const struct acpi_table_header *table = arg;
 508 
 509         /* Iterate over the ACPI OEM info array, looking for a match */
 510         while (memcmp(info, &empty_oem_info, sizeof(*info))) {
 511                 if (!memcmp(info->oem_id, table->oem_id, ACPI_OEM_ID_SIZE) &&
 512                     !memcmp(info->oem_table_id, table->oem_table_id, ACPI_OEM_TABLE_ID_SIZE) &&
 513                     info->oem_revision == table->oem_revision)
 514                         return true;
 515 
 516                 info++;
 517         }
 518 
 519         return false;
 520 }
 521 
 522 static const struct arch_timer_erratum_workaround *
 523 arch_timer_iterate_errata(enum arch_timer_erratum_match_type type,
 524                           ate_match_fn_t match_fn,
 525                           void *arg)
 526 {
 527         int i;
 528 
 529         for (i = 0; i < ARRAY_SIZE(ool_workarounds); i++) {
 530                 if (ool_workarounds[i].match_type != type)
 531                         continue;
 532 
 533                 if (match_fn(&ool_workarounds[i], arg))
 534                         return &ool_workarounds[i];
 535         }
 536 
 537         return NULL;
 538 }
 539 
 540 static
 541 void arch_timer_enable_workaround(const struct arch_timer_erratum_workaround *wa,
 542                                   bool local)
 543 {
 544         int i;
 545 
 546         if (local) {
 547                 __this_cpu_write(timer_unstable_counter_workaround, wa);
 548         } else {
 549                 for_each_possible_cpu(i)
 550                         per_cpu(timer_unstable_counter_workaround, i) = wa;
 551         }
 552 
 553         if (wa->read_cntvct_el0 || wa->read_cntpct_el0)
 554                 atomic_set(&timer_unstable_counter_workaround_in_use, 1);
 555 
 556         /*
 557          * Don't use the vdso fastpath if errata require using the
 558          * out-of-line counter accessor. We may change our mind pretty
 559          * late in the game (with a per-CPU erratum, for example), so
 560          * change both the default value and the vdso itself.
 561          */
 562         if (wa->read_cntvct_el0) {
 563                 clocksource_counter.archdata.vdso_direct = false;
 564                 vdso_default = false;
 565         }
 566 }
 567 
 568 static void arch_timer_check_ool_workaround(enum arch_timer_erratum_match_type type,
 569                                             void *arg)
 570 {
 571         const struct arch_timer_erratum_workaround *wa, *__wa;
 572         ate_match_fn_t match_fn = NULL;
 573         bool local = false;
 574 
 575         switch (type) {
 576         case ate_match_dt:
 577                 match_fn = arch_timer_check_dt_erratum;
 578                 break;
 579         case ate_match_local_cap_id:
 580                 match_fn = arch_timer_check_local_cap_erratum;
 581                 local = true;
 582                 break;
 583         case ate_match_acpi_oem_info:
 584                 match_fn = arch_timer_check_acpi_oem_erratum;
 585                 break;
 586         default:
 587                 WARN_ON(1);
 588                 return;
 589         }
 590 
 591         wa = arch_timer_iterate_errata(type, match_fn, arg);
 592         if (!wa)
 593                 return;
 594 
 595         __wa = __this_cpu_read(timer_unstable_counter_workaround);
 596         if (__wa && wa != __wa)
 597                 pr_warn("Can't enable workaround for %s (clashes with %s\n)",
 598                         wa->desc, __wa->desc);
 599 
 600         if (__wa)
 601                 return;
 602 
 603         arch_timer_enable_workaround(wa, local);
 604         pr_info("Enabling %s workaround for %s\n",
 605                 local ? "local" : "global", wa->desc);
 606 }
 607 
 608 static bool arch_timer_this_cpu_has_cntvct_wa(void)
 609 {
 610         return has_erratum_handler(read_cntvct_el0);
 611 }
 612 
 613 static bool arch_timer_counter_has_wa(void)
 614 {
 615         return atomic_read(&timer_unstable_counter_workaround_in_use);
 616 }
 617 #else
 618 #define arch_timer_check_ool_workaround(t,a)            do { } while(0)
 619 #define arch_timer_this_cpu_has_cntvct_wa()             ({false;})
 620 #define arch_timer_counter_has_wa()                     ({false;})
 621 #endif /* CONFIG_ARM_ARCH_TIMER_OOL_WORKAROUND */
 622 
 623 static __always_inline irqreturn_t timer_handler(const int access,
 624                                         struct clock_event_device *evt)
 625 {
 626         unsigned long ctrl;
 627 
 628         ctrl = arch_timer_reg_read(access, ARCH_TIMER_REG_CTRL, evt);
 629         if (ctrl & ARCH_TIMER_CTRL_IT_STAT) {
 630                 ctrl |= ARCH_TIMER_CTRL_IT_MASK;
 631                 arch_timer_reg_write(access, ARCH_TIMER_REG_CTRL, ctrl, evt);
 632                 evt->event_handler(evt);
 633                 return IRQ_HANDLED;
 634         }
 635 
 636         return IRQ_NONE;
 637 }
 638 
 639 static irqreturn_t arch_timer_handler_virt(int irq, void *dev_id)
 640 {
 641         struct clock_event_device *evt = dev_id;
 642 
 643         return timer_handler(ARCH_TIMER_VIRT_ACCESS, evt);
 644 }
 645 
 646 static irqreturn_t arch_timer_handler_phys(int irq, void *dev_id)
 647 {
 648         struct clock_event_device *evt = dev_id;
 649 
 650         return timer_handler(ARCH_TIMER_PHYS_ACCESS, evt);
 651 }
 652 
 653 static irqreturn_t arch_timer_handler_phys_mem(int irq, void *dev_id)
 654 {
 655         struct clock_event_device *evt = dev_id;
 656 
 657         return timer_handler(ARCH_TIMER_MEM_PHYS_ACCESS, evt);
 658 }
 659 
 660 static irqreturn_t arch_timer_handler_virt_mem(int irq, void *dev_id)
 661 {
 662         struct clock_event_device *evt = dev_id;
 663 
 664         return timer_handler(ARCH_TIMER_MEM_VIRT_ACCESS, evt);
 665 }
 666 
 667 static __always_inline int timer_shutdown(const int access,
 668                                           struct clock_event_device *clk)
 669 {
 670         unsigned long ctrl;
 671 
 672         ctrl = arch_timer_reg_read(access, ARCH_TIMER_REG_CTRL, clk);
 673         ctrl &= ~ARCH_TIMER_CTRL_ENABLE;
 674         arch_timer_reg_write(access, ARCH_TIMER_REG_CTRL, ctrl, clk);
 675 
 676         return 0;
 677 }
 678 
 679 static int arch_timer_shutdown_virt(struct clock_event_device *clk)
 680 {
 681         return timer_shutdown(ARCH_TIMER_VIRT_ACCESS, clk);
 682 }
 683 
 684 static int arch_timer_shutdown_phys(struct clock_event_device *clk)
 685 {
 686         return timer_shutdown(ARCH_TIMER_PHYS_ACCESS, clk);
 687 }
 688 
 689 static int arch_timer_shutdown_virt_mem(struct clock_event_device *clk)
 690 {
 691         return timer_shutdown(ARCH_TIMER_MEM_VIRT_ACCESS, clk);
 692 }
 693 
 694 static int arch_timer_shutdown_phys_mem(struct clock_event_device *clk)
 695 {
 696         return timer_shutdown(ARCH_TIMER_MEM_PHYS_ACCESS, clk);
 697 }
 698 
 699 static __always_inline void set_next_event(const int access, unsigned long evt,
 700                                            struct clock_event_device *clk)
 701 {
 702         unsigned long ctrl;
 703         ctrl = arch_timer_reg_read(access, ARCH_TIMER_REG_CTRL, clk);
 704         ctrl |= ARCH_TIMER_CTRL_ENABLE;
 705         ctrl &= ~ARCH_TIMER_CTRL_IT_MASK;
 706         arch_timer_reg_write(access, ARCH_TIMER_REG_TVAL, evt, clk);
 707         arch_timer_reg_write(access, ARCH_TIMER_REG_CTRL, ctrl, clk);
 708 }
 709 
 710 static int arch_timer_set_next_event_virt(unsigned long evt,
 711                                           struct clock_event_device *clk)
 712 {
 713         set_next_event(ARCH_TIMER_VIRT_ACCESS, evt, clk);
 714         return 0;
 715 }
 716 
 717 static int arch_timer_set_next_event_phys(unsigned long evt,
 718                                           struct clock_event_device *clk)
 719 {
 720         set_next_event(ARCH_TIMER_PHYS_ACCESS, evt, clk);
 721         return 0;
 722 }
 723 
 724 static int arch_timer_set_next_event_virt_mem(unsigned long evt,
 725                                               struct clock_event_device *clk)
 726 {
 727         set_next_event(ARCH_TIMER_MEM_VIRT_ACCESS, evt, clk);
 728         return 0;
 729 }
 730 
 731 static int arch_timer_set_next_event_phys_mem(unsigned long evt,
 732                                               struct clock_event_device *clk)
 733 {
 734         set_next_event(ARCH_TIMER_MEM_PHYS_ACCESS, evt, clk);
 735         return 0;
 736 }
 737 
 738 static void __arch_timer_setup(unsigned type,
 739                                struct clock_event_device *clk)
 740 {
 741         clk->features = CLOCK_EVT_FEAT_ONESHOT;
 742 
 743         if (type == ARCH_TIMER_TYPE_CP15) {
 744                 typeof(clk->set_next_event) sne;
 745 
 746                 arch_timer_check_ool_workaround(ate_match_local_cap_id, NULL);
 747 
 748                 if (arch_timer_c3stop)
 749                         clk->features |= CLOCK_EVT_FEAT_C3STOP;
 750                 clk->name = "arch_sys_timer";
 751                 clk->rating = 450;
 752                 clk->cpumask = cpumask_of(smp_processor_id());
 753                 clk->irq = arch_timer_ppi[arch_timer_uses_ppi];
 754                 switch (arch_timer_uses_ppi) {
 755                 case ARCH_TIMER_VIRT_PPI:
 756                         clk->set_state_shutdown = arch_timer_shutdown_virt;
 757                         clk->set_state_oneshot_stopped = arch_timer_shutdown_virt;
 758                         sne = erratum_handler(set_next_event_virt);
 759                         break;
 760                 case ARCH_TIMER_PHYS_SECURE_PPI:
 761                 case ARCH_TIMER_PHYS_NONSECURE_PPI:
 762                 case ARCH_TIMER_HYP_PPI:
 763                         clk->set_state_shutdown = arch_timer_shutdown_phys;
 764                         clk->set_state_oneshot_stopped = arch_timer_shutdown_phys;
 765                         sne = erratum_handler(set_next_event_phys);
 766                         break;
 767                 default:
 768                         BUG();
 769                 }
 770 
 771                 clk->set_next_event = sne;
 772         } else {
 773                 clk->features |= CLOCK_EVT_FEAT_DYNIRQ;
 774                 clk->name = "arch_mem_timer";
 775                 clk->rating = 400;
 776                 clk->cpumask = cpu_possible_mask;
 777                 if (arch_timer_mem_use_virtual) {
 778                         clk->set_state_shutdown = arch_timer_shutdown_virt_mem;
 779                         clk->set_state_oneshot_stopped = arch_timer_shutdown_virt_mem;
 780                         clk->set_next_event =
 781                                 arch_timer_set_next_event_virt_mem;
 782                 } else {
 783                         clk->set_state_shutdown = arch_timer_shutdown_phys_mem;
 784                         clk->set_state_oneshot_stopped = arch_timer_shutdown_phys_mem;
 785                         clk->set_next_event =
 786                                 arch_timer_set_next_event_phys_mem;
 787                 }
 788         }
 789 
 790         clk->set_state_shutdown(clk);
 791 
 792         clockevents_config_and_register(clk, arch_timer_rate, 0xf, 0x7fffffff);
 793 }
 794 
 795 static void arch_timer_evtstrm_enable(int divider)
 796 {
 797         u32 cntkctl = arch_timer_get_cntkctl();
 798 
 799         cntkctl &= ~ARCH_TIMER_EVT_TRIGGER_MASK;
 800         /* Set the divider and enable virtual event stream */
 801         cntkctl |= (divider << ARCH_TIMER_EVT_TRIGGER_SHIFT)
 802                         | ARCH_TIMER_VIRT_EVT_EN;
 803         arch_timer_set_cntkctl(cntkctl);
 804         arch_timer_set_evtstrm_feature();
 805         cpumask_set_cpu(smp_processor_id(), &evtstrm_available);
 806 }
 807 
 808 static void arch_timer_configure_evtstream(void)
 809 {
 810         int evt_stream_div, pos;
 811 
 812         /* Find the closest power of two to the divisor */
 813         evt_stream_div = arch_timer_rate / ARCH_TIMER_EVT_STREAM_FREQ;
 814         pos = fls(evt_stream_div);
 815         if (pos > 1 && !(evt_stream_div & (1 << (pos - 2))))
 816                 pos--;
 817         /* enable event stream */
 818         arch_timer_evtstrm_enable(min(pos, 15));
 819 }
 820 
 821 static void arch_counter_set_user_access(void)
 822 {
 823         u32 cntkctl = arch_timer_get_cntkctl();
 824 
 825         /* Disable user access to the timers and both counters */
 826         /* Also disable virtual event stream */
 827         cntkctl &= ~(ARCH_TIMER_USR_PT_ACCESS_EN
 828                         | ARCH_TIMER_USR_VT_ACCESS_EN
 829                         | ARCH_TIMER_USR_VCT_ACCESS_EN
 830                         | ARCH_TIMER_VIRT_EVT_EN
 831                         | ARCH_TIMER_USR_PCT_ACCESS_EN);
 832 
 833         /*
 834          * Enable user access to the virtual counter if it doesn't
 835          * need to be workaround. The vdso may have been already
 836          * disabled though.
 837          */
 838         if (arch_timer_this_cpu_has_cntvct_wa())
 839                 pr_info("CPU%d: Trapping CNTVCT access\n", smp_processor_id());
 840         else
 841                 cntkctl |= ARCH_TIMER_USR_VCT_ACCESS_EN;
 842 
 843         arch_timer_set_cntkctl(cntkctl);
 844 }
 845 
 846 static bool arch_timer_has_nonsecure_ppi(void)
 847 {
 848         return (arch_timer_uses_ppi == ARCH_TIMER_PHYS_SECURE_PPI &&
 849                 arch_timer_ppi[ARCH_TIMER_PHYS_NONSECURE_PPI]);
 850 }
 851 
 852 static u32 check_ppi_trigger(int irq)
 853 {
 854         u32 flags = irq_get_trigger_type(irq);
 855 
 856         if (flags != IRQF_TRIGGER_HIGH && flags != IRQF_TRIGGER_LOW) {
 857                 pr_warn("WARNING: Invalid trigger for IRQ%d, assuming level low\n", irq);
 858                 pr_warn("WARNING: Please fix your firmware\n");
 859                 flags = IRQF_TRIGGER_LOW;
 860         }
 861 
 862         return flags;
 863 }
 864 
 865 static int arch_timer_starting_cpu(unsigned int cpu)
 866 {
 867         struct clock_event_device *clk = this_cpu_ptr(arch_timer_evt);
 868         u32 flags;
 869 
 870         __arch_timer_setup(ARCH_TIMER_TYPE_CP15, clk);
 871 
 872         flags = check_ppi_trigger(arch_timer_ppi[arch_timer_uses_ppi]);
 873         enable_percpu_irq(arch_timer_ppi[arch_timer_uses_ppi], flags);
 874 
 875         if (arch_timer_has_nonsecure_ppi()) {
 876                 flags = check_ppi_trigger(arch_timer_ppi[ARCH_TIMER_PHYS_NONSECURE_PPI]);
 877                 enable_percpu_irq(arch_timer_ppi[ARCH_TIMER_PHYS_NONSECURE_PPI],
 878                                   flags);
 879         }
 880 
 881         arch_counter_set_user_access();
 882         if (evtstrm_enable)
 883                 arch_timer_configure_evtstream();
 884 
 885         return 0;
 886 }
 887 
 888 /*
 889  * For historical reasons, when probing with DT we use whichever (non-zero)
 890  * rate was probed first, and don't verify that others match. If the first node
 891  * probed has a clock-frequency property, this overrides the HW register.
 892  */
 893 static void arch_timer_of_configure_rate(u32 rate, struct device_node *np)
 894 {
 895         /* Who has more than one independent system counter? */
 896         if (arch_timer_rate)
 897                 return;
 898 
 899         if (of_property_read_u32(np, "clock-frequency", &arch_timer_rate))
 900                 arch_timer_rate = rate;
 901 
 902         /* Check the timer frequency. */
 903         if (arch_timer_rate == 0)
 904                 pr_warn("frequency not available\n");
 905 }
 906 
 907 static void arch_timer_banner(unsigned type)
 908 {
 909         pr_info("%s%s%s timer(s) running at %lu.%02luMHz (%s%s%s).\n",
 910                 type & ARCH_TIMER_TYPE_CP15 ? "cp15" : "",
 911                 type == (ARCH_TIMER_TYPE_CP15 | ARCH_TIMER_TYPE_MEM) ?
 912                         " and " : "",
 913                 type & ARCH_TIMER_TYPE_MEM ? "mmio" : "",
 914                 (unsigned long)arch_timer_rate / 1000000,
 915                 (unsigned long)(arch_timer_rate / 10000) % 100,
 916                 type & ARCH_TIMER_TYPE_CP15 ?
 917                         (arch_timer_uses_ppi == ARCH_TIMER_VIRT_PPI) ? "virt" : "phys" :
 918                         "",
 919                 type == (ARCH_TIMER_TYPE_CP15 | ARCH_TIMER_TYPE_MEM) ? "/" : "",
 920                 type & ARCH_TIMER_TYPE_MEM ?
 921                         arch_timer_mem_use_virtual ? "virt" : "phys" :
 922                         "");
 923 }
 924 
 925 u32 arch_timer_get_rate(void)
 926 {
 927         return arch_timer_rate;
 928 }
 929 
 930 bool arch_timer_evtstrm_available(void)
 931 {
 932         /*
 933          * We might get called from a preemptible context. This is fine
 934          * because availability of the event stream should be always the same
 935          * for a preemptible context and context where we might resume a task.
 936          */
 937         return cpumask_test_cpu(raw_smp_processor_id(), &evtstrm_available);
 938 }
 939 
 940 static u64 arch_counter_get_cntvct_mem(void)
 941 {
 942         u32 vct_lo, vct_hi, tmp_hi;
 943 
 944         do {
 945                 vct_hi = readl_relaxed(arch_counter_base + CNTVCT_HI);
 946                 vct_lo = readl_relaxed(arch_counter_base + CNTVCT_LO);
 947                 tmp_hi = readl_relaxed(arch_counter_base + CNTVCT_HI);
 948         } while (vct_hi != tmp_hi);
 949 
 950         return ((u64) vct_hi << 32) | vct_lo;
 951 }
 952 
 953 static struct arch_timer_kvm_info arch_timer_kvm_info;
 954 
 955 struct arch_timer_kvm_info *arch_timer_get_kvm_info(void)
 956 {
 957         return &arch_timer_kvm_info;
 958 }
 959 
 960 static void __init arch_counter_register(unsigned type)
 961 {
 962         u64 start_count;
 963 
 964         /* Register the CP15 based counter if we have one */
 965         if (type & ARCH_TIMER_TYPE_CP15) {
 966                 u64 (*rd)(void);
 967 
 968                 if ((IS_ENABLED(CONFIG_ARM64) && !is_hyp_mode_available()) ||
 969                     arch_timer_uses_ppi == ARCH_TIMER_VIRT_PPI) {
 970                         if (arch_timer_counter_has_wa())
 971                                 rd = arch_counter_get_cntvct_stable;
 972                         else
 973                                 rd = arch_counter_get_cntvct;
 974                 } else {
 975                         if (arch_timer_counter_has_wa())
 976                                 rd = arch_counter_get_cntpct_stable;
 977                         else
 978                                 rd = arch_counter_get_cntpct;
 979                 }
 980 
 981                 arch_timer_read_counter = rd;
 982                 clocksource_counter.archdata.vdso_direct = vdso_default;
 983         } else {
 984                 arch_timer_read_counter = arch_counter_get_cntvct_mem;
 985         }
 986 
 987         if (!arch_counter_suspend_stop)
 988                 clocksource_counter.flags |= CLOCK_SOURCE_SUSPEND_NONSTOP;
 989         start_count = arch_timer_read_counter();
 990         clocksource_register_hz(&clocksource_counter, arch_timer_rate);
 991         cyclecounter.mult = clocksource_counter.mult;
 992         cyclecounter.shift = clocksource_counter.shift;
 993         timecounter_init(&arch_timer_kvm_info.timecounter,
 994                          &cyclecounter, start_count);
 995 
 996         /* 56 bits minimum, so we assume worst case rollover */
 997         sched_clock_register(arch_timer_read_counter, 56, arch_timer_rate);
 998 }
 999 
1000 static void arch_timer_stop(struct clock_event_device *clk)
1001 {
1002         pr_debug("disable IRQ%d cpu #%d\n", clk->irq, smp_processor_id());
1003 
1004         disable_percpu_irq(arch_timer_ppi[arch_timer_uses_ppi]);
1005         if (arch_timer_has_nonsecure_ppi())
1006                 disable_percpu_irq(arch_timer_ppi[ARCH_TIMER_PHYS_NONSECURE_PPI]);
1007 
1008         clk->set_state_shutdown(clk);
1009 }
1010 
1011 static int arch_timer_dying_cpu(unsigned int cpu)
1012 {
1013         struct clock_event_device *clk = this_cpu_ptr(arch_timer_evt);
1014 
1015         cpumask_clear_cpu(smp_processor_id(), &evtstrm_available);
1016 
1017         arch_timer_stop(clk);
1018         return 0;
1019 }
1020 
1021 #ifdef CONFIG_CPU_PM
1022 static DEFINE_PER_CPU(unsigned long, saved_cntkctl);
1023 static int arch_timer_cpu_pm_notify(struct notifier_block *self,
1024                                     unsigned long action, void *hcpu)
1025 {
1026         if (action == CPU_PM_ENTER) {
1027                 __this_cpu_write(saved_cntkctl, arch_timer_get_cntkctl());
1028 
1029                 cpumask_clear_cpu(smp_processor_id(), &evtstrm_available);
1030         } else if (action == CPU_PM_ENTER_FAILED || action == CPU_PM_EXIT) {
1031                 arch_timer_set_cntkctl(__this_cpu_read(saved_cntkctl));
1032 
1033                 if (arch_timer_have_evtstrm_feature())
1034                         cpumask_set_cpu(smp_processor_id(), &evtstrm_available);
1035         }
1036         return NOTIFY_OK;
1037 }
1038 
1039 static struct notifier_block arch_timer_cpu_pm_notifier = {
1040         .notifier_call = arch_timer_cpu_pm_notify,
1041 };
1042 
1043 static int __init arch_timer_cpu_pm_init(void)
1044 {
1045         return cpu_pm_register_notifier(&arch_timer_cpu_pm_notifier);
1046 }
1047 
1048 static void __init arch_timer_cpu_pm_deinit(void)
1049 {
1050         WARN_ON(cpu_pm_unregister_notifier(&arch_timer_cpu_pm_notifier));
1051 }
1052 
1053 #else
1054 static int __init arch_timer_cpu_pm_init(void)
1055 {
1056         return 0;
1057 }
1058 
1059 static void __init arch_timer_cpu_pm_deinit(void)
1060 {
1061 }
1062 #endif
1063 
1064 static int __init arch_timer_register(void)
1065 {
1066         int err;
1067         int ppi;
1068 
1069         arch_timer_evt = alloc_percpu(struct clock_event_device);
1070         if (!arch_timer_evt) {
1071                 err = -ENOMEM;
1072                 goto out;
1073         }
1074 
1075         ppi = arch_timer_ppi[arch_timer_uses_ppi];
1076         switch (arch_timer_uses_ppi) {
1077         case ARCH_TIMER_VIRT_PPI:
1078                 err = request_percpu_irq(ppi, arch_timer_handler_virt,
1079                                          "arch_timer", arch_timer_evt);
1080                 break;
1081         case ARCH_TIMER_PHYS_SECURE_PPI:
1082         case ARCH_TIMER_PHYS_NONSECURE_PPI:
1083                 err = request_percpu_irq(ppi, arch_timer_handler_phys,
1084                                          "arch_timer", arch_timer_evt);
1085                 if (!err && arch_timer_has_nonsecure_ppi()) {
1086                         ppi = arch_timer_ppi[ARCH_TIMER_PHYS_NONSECURE_PPI];
1087                         err = request_percpu_irq(ppi, arch_timer_handler_phys,
1088                                                  "arch_timer", arch_timer_evt);
1089                         if (err)
1090                                 free_percpu_irq(arch_timer_ppi[ARCH_TIMER_PHYS_SECURE_PPI],
1091                                                 arch_timer_evt);
1092                 }
1093                 break;
1094         case ARCH_TIMER_HYP_PPI:
1095                 err = request_percpu_irq(ppi, arch_timer_handler_phys,
1096                                          "arch_timer", arch_timer_evt);
1097                 break;
1098         default:
1099                 BUG();
1100         }
1101 
1102         if (err) {
1103                 pr_err("can't register interrupt %d (%d)\n", ppi, err);
1104                 goto out_free;
1105         }
1106 
1107         err = arch_timer_cpu_pm_init();
1108         if (err)
1109                 goto out_unreg_notify;
1110 
1111         /* Register and immediately configure the timer on the boot CPU */
1112         err = cpuhp_setup_state(CPUHP_AP_ARM_ARCH_TIMER_STARTING,
1113                                 "clockevents/arm/arch_timer:starting",
1114                                 arch_timer_starting_cpu, arch_timer_dying_cpu);
1115         if (err)
1116                 goto out_unreg_cpupm;
1117         return 0;
1118 
1119 out_unreg_cpupm:
1120         arch_timer_cpu_pm_deinit();
1121 
1122 out_unreg_notify:
1123         free_percpu_irq(arch_timer_ppi[arch_timer_uses_ppi], arch_timer_evt);
1124         if (arch_timer_has_nonsecure_ppi())
1125                 free_percpu_irq(arch_timer_ppi[ARCH_TIMER_PHYS_NONSECURE_PPI],
1126                                 arch_timer_evt);
1127 
1128 out_free:
1129         free_percpu(arch_timer_evt);
1130 out:
1131         return err;
1132 }
1133 
1134 static int __init arch_timer_mem_register(void __iomem *base, unsigned int irq)
1135 {
1136         int ret;
1137         irq_handler_t func;
1138         struct arch_timer *t;
1139 
1140         t = kzalloc(sizeof(*t), GFP_KERNEL);
1141         if (!t)
1142                 return -ENOMEM;
1143 
1144         t->base = base;
1145         t->evt.irq = irq;
1146         __arch_timer_setup(ARCH_TIMER_TYPE_MEM, &t->evt);
1147 
1148         if (arch_timer_mem_use_virtual)
1149                 func = arch_timer_handler_virt_mem;
1150         else
1151                 func = arch_timer_handler_phys_mem;
1152 
1153         ret = request_irq(irq, func, IRQF_TIMER, "arch_mem_timer", &t->evt);
1154         if (ret) {
1155                 pr_err("Failed to request mem timer irq\n");
1156                 kfree(t);
1157         }
1158 
1159         return ret;
1160 }
1161 
1162 static const struct of_device_id arch_timer_of_match[] __initconst = {
1163         { .compatible   = "arm,armv7-timer",    },
1164         { .compatible   = "arm,armv8-timer",    },
1165         {},
1166 };
1167 
1168 static const struct of_device_id arch_timer_mem_of_match[] __initconst = {
1169         { .compatible   = "arm,armv7-timer-mem", },
1170         {},
1171 };
1172 
1173 static bool __init arch_timer_needs_of_probing(void)
1174 {
1175         struct device_node *dn;
1176         bool needs_probing = false;
1177         unsigned int mask = ARCH_TIMER_TYPE_CP15 | ARCH_TIMER_TYPE_MEM;
1178 
1179         /* We have two timers, and both device-tree nodes are probed. */
1180         if ((arch_timers_present & mask) == mask)
1181                 return false;
1182 
1183         /*
1184          * Only one type of timer is probed,
1185          * check if we have another type of timer node in device-tree.
1186          */
1187         if (arch_timers_present & ARCH_TIMER_TYPE_CP15)
1188                 dn = of_find_matching_node(NULL, arch_timer_mem_of_match);
1189         else
1190                 dn = of_find_matching_node(NULL, arch_timer_of_match);
1191 
1192         if (dn && of_device_is_available(dn))
1193                 needs_probing = true;
1194 
1195         of_node_put(dn);
1196 
1197         return needs_probing;
1198 }
1199 
1200 static int __init arch_timer_common_init(void)
1201 {
1202         arch_timer_banner(arch_timers_present);
1203         arch_counter_register(arch_timers_present);
1204         return arch_timer_arch_init();
1205 }
1206 
1207 /**
1208  * arch_timer_select_ppi() - Select suitable PPI for the current system.
1209  *
1210  * If HYP mode is available, we know that the physical timer
1211  * has been configured to be accessible from PL1. Use it, so
1212  * that a guest can use the virtual timer instead.
1213  *
1214  * On ARMv8.1 with VH extensions, the kernel runs in HYP. VHE
1215  * accesses to CNTP_*_EL1 registers are silently redirected to
1216  * their CNTHP_*_EL2 counterparts, and use a different PPI
1217  * number.
1218  *
1219  * If no interrupt provided for virtual timer, we'll have to
1220  * stick to the physical timer. It'd better be accessible...
1221  * For arm64 we never use the secure interrupt.
1222  *
1223  * Return: a suitable PPI type for the current system.
1224  */
1225 static enum arch_timer_ppi_nr __init arch_timer_select_ppi(void)
1226 {
1227         if (is_kernel_in_hyp_mode())
1228                 return ARCH_TIMER_HYP_PPI;
1229 
1230         if (!is_hyp_mode_available() && arch_timer_ppi[ARCH_TIMER_VIRT_PPI])
1231                 return ARCH_TIMER_VIRT_PPI;
1232 
1233         if (IS_ENABLED(CONFIG_ARM64))
1234                 return ARCH_TIMER_PHYS_NONSECURE_PPI;
1235 
1236         return ARCH_TIMER_PHYS_SECURE_PPI;
1237 }
1238 
1239 static void __init arch_timer_populate_kvm_info(void)
1240 {
1241         arch_timer_kvm_info.virtual_irq = arch_timer_ppi[ARCH_TIMER_VIRT_PPI];
1242         if (is_kernel_in_hyp_mode())
1243                 arch_timer_kvm_info.physical_irq = arch_timer_ppi[ARCH_TIMER_PHYS_NONSECURE_PPI];
1244 }
1245 
1246 static int __init arch_timer_of_init(struct device_node *np)
1247 {
1248         int i, ret;
1249         u32 rate;
1250 
1251         if (arch_timers_present & ARCH_TIMER_TYPE_CP15) {
1252                 pr_warn("multiple nodes in dt, skipping\n");
1253                 return 0;
1254         }
1255 
1256         arch_timers_present |= ARCH_TIMER_TYPE_CP15;
1257         for (i = ARCH_TIMER_PHYS_SECURE_PPI; i < ARCH_TIMER_MAX_TIMER_PPI; i++)
1258                 arch_timer_ppi[i] = irq_of_parse_and_map(np, i);
1259 
1260         arch_timer_populate_kvm_info();
1261 
1262         rate = arch_timer_get_cntfrq();
1263         arch_timer_of_configure_rate(rate, np);
1264 
1265         arch_timer_c3stop = !of_property_read_bool(np, "always-on");
1266 
1267         /* Check for globally applicable workarounds */
1268         arch_timer_check_ool_workaround(ate_match_dt, np);
1269 
1270         /*
1271          * If we cannot rely on firmware initializing the timer registers then
1272          * we should use the physical timers instead.
1273          */
1274         if (IS_ENABLED(CONFIG_ARM) &&
1275             of_property_read_bool(np, "arm,cpu-registers-not-fw-configured"))
1276                 arch_timer_uses_ppi = ARCH_TIMER_PHYS_SECURE_PPI;
1277         else
1278                 arch_timer_uses_ppi = arch_timer_select_ppi();
1279 
1280         if (!arch_timer_ppi[arch_timer_uses_ppi]) {
1281                 pr_err("No interrupt available, giving up\n");
1282                 return -EINVAL;
1283         }
1284 
1285         /* On some systems, the counter stops ticking when in suspend. */
1286         arch_counter_suspend_stop = of_property_read_bool(np,
1287                                                          "arm,no-tick-in-suspend");
1288 
1289         ret = arch_timer_register();
1290         if (ret)
1291                 return ret;
1292 
1293         if (arch_timer_needs_of_probing())
1294                 return 0;
1295 
1296         return arch_timer_common_init();
1297 }
1298 TIMER_OF_DECLARE(armv7_arch_timer, "arm,armv7-timer", arch_timer_of_init);
1299 TIMER_OF_DECLARE(armv8_arch_timer, "arm,armv8-timer", arch_timer_of_init);
1300 
1301 static u32 __init
1302 arch_timer_mem_frame_get_cntfrq(struct arch_timer_mem_frame *frame)
1303 {
1304         void __iomem *base;
1305         u32 rate;
1306 
1307         base = ioremap(frame->cntbase, frame->size);
1308         if (!base) {
1309                 pr_err("Unable to map frame @ %pa\n", &frame->cntbase);
1310                 return 0;
1311         }
1312 
1313         rate = readl_relaxed(base + CNTFRQ);
1314 
1315         iounmap(base);
1316 
1317         return rate;
1318 }
1319 
1320 static struct arch_timer_mem_frame * __init
1321 arch_timer_mem_find_best_frame(struct arch_timer_mem *timer_mem)
1322 {
1323         struct arch_timer_mem_frame *frame, *best_frame = NULL;
1324         void __iomem *cntctlbase;
1325         u32 cnttidr;
1326         int i;
1327 
1328         cntctlbase = ioremap(timer_mem->cntctlbase, timer_mem->size);
1329         if (!cntctlbase) {
1330                 pr_err("Can't map CNTCTLBase @ %pa\n",
1331                         &timer_mem->cntctlbase);
1332                 return NULL;
1333         }
1334 
1335         cnttidr = readl_relaxed(cntctlbase + CNTTIDR);
1336 
1337         /*
1338          * Try to find a virtual capable frame. Otherwise fall back to a
1339          * physical capable frame.
1340          */
1341         for (i = 0; i < ARCH_TIMER_MEM_MAX_FRAMES; i++) {
1342                 u32 cntacr = CNTACR_RFRQ | CNTACR_RWPT | CNTACR_RPCT |
1343                              CNTACR_RWVT | CNTACR_RVOFF | CNTACR_RVCT;
1344 
1345                 frame = &timer_mem->frame[i];
1346                 if (!frame->valid)
1347                         continue;
1348 
1349                 /* Try enabling everything, and see what sticks */
1350                 writel_relaxed(cntacr, cntctlbase + CNTACR(i));
1351                 cntacr = readl_relaxed(cntctlbase + CNTACR(i));
1352 
1353                 if ((cnttidr & CNTTIDR_VIRT(i)) &&
1354                     !(~cntacr & (CNTACR_RWVT | CNTACR_RVCT))) {
1355                         best_frame = frame;
1356                         arch_timer_mem_use_virtual = true;
1357                         break;
1358                 }
1359 
1360                 if (~cntacr & (CNTACR_RWPT | CNTACR_RPCT))
1361                         continue;
1362 
1363                 best_frame = frame;
1364         }
1365 
1366         iounmap(cntctlbase);
1367 
1368         return best_frame;
1369 }
1370 
1371 static int __init
1372 arch_timer_mem_frame_register(struct arch_timer_mem_frame *frame)
1373 {
1374         void __iomem *base;
1375         int ret, irq = 0;
1376 
1377         if (arch_timer_mem_use_virtual)
1378                 irq = frame->virt_irq;
1379         else
1380                 irq = frame->phys_irq;
1381 
1382         if (!irq) {
1383                 pr_err("Frame missing %s irq.\n",
1384                        arch_timer_mem_use_virtual ? "virt" : "phys");
1385                 return -EINVAL;
1386         }
1387 
1388         if (!request_mem_region(frame->cntbase, frame->size,
1389                                 "arch_mem_timer"))
1390                 return -EBUSY;
1391 
1392         base = ioremap(frame->cntbase, frame->size);
1393         if (!base) {
1394                 pr_err("Can't map frame's registers\n");
1395                 return -ENXIO;
1396         }
1397 
1398         ret = arch_timer_mem_register(base, irq);
1399         if (ret) {
1400                 iounmap(base);
1401                 return ret;
1402         }
1403 
1404         arch_counter_base = base;
1405         arch_timers_present |= ARCH_TIMER_TYPE_MEM;
1406 
1407         return 0;
1408 }
1409 
1410 static int __init arch_timer_mem_of_init(struct device_node *np)
1411 {
1412         struct arch_timer_mem *timer_mem;
1413         struct arch_timer_mem_frame *frame;
1414         struct device_node *frame_node;
1415         struct resource res;
1416         int ret = -EINVAL;
1417         u32 rate;
1418 
1419         timer_mem = kzalloc(sizeof(*timer_mem), GFP_KERNEL);
1420         if (!timer_mem)
1421                 return -ENOMEM;
1422 
1423         if (of_address_to_resource(np, 0, &res))
1424                 goto out;
1425         timer_mem->cntctlbase = res.start;
1426         timer_mem->size = resource_size(&res);
1427 
1428         for_each_available_child_of_node(np, frame_node) {
1429                 u32 n;
1430                 struct arch_timer_mem_frame *frame;
1431 
1432                 if (of_property_read_u32(frame_node, "frame-number", &n)) {
1433                         pr_err(FW_BUG "Missing frame-number.\n");
1434                         of_node_put(frame_node);
1435                         goto out;
1436                 }
1437                 if (n >= ARCH_TIMER_MEM_MAX_FRAMES) {
1438                         pr_err(FW_BUG "Wrong frame-number, only 0-%u are permitted.\n",
1439                                ARCH_TIMER_MEM_MAX_FRAMES - 1);
1440                         of_node_put(frame_node);
1441                         goto out;
1442                 }
1443                 frame = &timer_mem->frame[n];
1444 
1445                 if (frame->valid) {
1446                         pr_err(FW_BUG "Duplicated frame-number.\n");
1447                         of_node_put(frame_node);
1448                         goto out;
1449                 }
1450 
1451                 if (of_address_to_resource(frame_node, 0, &res)) {
1452                         of_node_put(frame_node);
1453                         goto out;
1454                 }
1455                 frame->cntbase = res.start;
1456                 frame->size = resource_size(&res);
1457 
1458                 frame->virt_irq = irq_of_parse_and_map(frame_node,
1459                                                        ARCH_TIMER_VIRT_SPI);
1460                 frame->phys_irq = irq_of_parse_and_map(frame_node,
1461                                                        ARCH_TIMER_PHYS_SPI);
1462 
1463                 frame->valid = true;
1464         }
1465 
1466         frame = arch_timer_mem_find_best_frame(timer_mem);
1467         if (!frame) {
1468                 pr_err("Unable to find a suitable frame in timer @ %pa\n",
1469                         &timer_mem->cntctlbase);
1470                 ret = -EINVAL;
1471                 goto out;
1472         }
1473 
1474         rate = arch_timer_mem_frame_get_cntfrq(frame);
1475         arch_timer_of_configure_rate(rate, np);
1476 
1477         ret = arch_timer_mem_frame_register(frame);
1478         if (!ret && !arch_timer_needs_of_probing())
1479                 ret = arch_timer_common_init();
1480 out:
1481         kfree(timer_mem);
1482         return ret;
1483 }
1484 TIMER_OF_DECLARE(armv7_arch_timer_mem, "arm,armv7-timer-mem",
1485                        arch_timer_mem_of_init);
1486 
1487 #ifdef CONFIG_ACPI_GTDT
1488 static int __init
1489 arch_timer_mem_verify_cntfrq(struct arch_timer_mem *timer_mem)
1490 {
1491         struct arch_timer_mem_frame *frame;
1492         u32 rate;
1493         int i;
1494 
1495         for (i = 0; i < ARCH_TIMER_MEM_MAX_FRAMES; i++) {
1496                 frame = &timer_mem->frame[i];
1497 
1498                 if (!frame->valid)
1499                         continue;
1500 
1501                 rate = arch_timer_mem_frame_get_cntfrq(frame);
1502                 if (rate == arch_timer_rate)
1503                         continue;
1504 
1505                 pr_err(FW_BUG "CNTFRQ mismatch: frame @ %pa: (0x%08lx), CPU: (0x%08lx)\n",
1506                         &frame->cntbase,
1507                         (unsigned long)rate, (unsigned long)arch_timer_rate);
1508 
1509                 return -EINVAL;
1510         }
1511 
1512         return 0;
1513 }
1514 
1515 static int __init arch_timer_mem_acpi_init(int platform_timer_count)
1516 {
1517         struct arch_timer_mem *timers, *timer;
1518         struct arch_timer_mem_frame *frame, *best_frame = NULL;
1519         int timer_count, i, ret = 0;
1520 
1521         timers = kcalloc(platform_timer_count, sizeof(*timers),
1522                             GFP_KERNEL);
1523         if (!timers)
1524                 return -ENOMEM;
1525 
1526         ret = acpi_arch_timer_mem_init(timers, &timer_count);
1527         if (ret || !timer_count)
1528                 goto out;
1529 
1530         /*
1531          * While unlikely, it's theoretically possible that none of the frames
1532          * in a timer expose the combination of feature we want.
1533          */
1534         for (i = 0; i < timer_count; i++) {
1535                 timer = &timers[i];
1536 
1537                 frame = arch_timer_mem_find_best_frame(timer);
1538                 if (!best_frame)
1539                         best_frame = frame;
1540 
1541                 ret = arch_timer_mem_verify_cntfrq(timer);
1542                 if (ret) {
1543                         pr_err("Disabling MMIO timers due to CNTFRQ mismatch\n");
1544                         goto out;
1545                 }
1546 
1547                 if (!best_frame) /* implies !frame */
1548                         /*
1549                          * Only complain about missing suitable frames if we
1550                          * haven't already found one in a previous iteration.
1551                          */
1552                         pr_err("Unable to find a suitable frame in timer @ %pa\n",
1553                                 &timer->cntctlbase);
1554         }
1555 
1556         if (best_frame)
1557                 ret = arch_timer_mem_frame_register(best_frame);
1558 out:
1559         kfree(timers);
1560         return ret;
1561 }
1562 
1563 /* Initialize per-processor generic timer and memory-mapped timer(if present) */
1564 static int __init arch_timer_acpi_init(struct acpi_table_header *table)
1565 {
1566         int ret, platform_timer_count;
1567 
1568         if (arch_timers_present & ARCH_TIMER_TYPE_CP15) {
1569                 pr_warn("already initialized, skipping\n");
1570                 return -EINVAL;
1571         }
1572 
1573         arch_timers_present |= ARCH_TIMER_TYPE_CP15;
1574 
1575         ret = acpi_gtdt_init(table, &platform_timer_count);
1576         if (ret) {
1577                 pr_err("Failed to init GTDT table.\n");
1578                 return ret;
1579         }
1580 
1581         arch_timer_ppi[ARCH_TIMER_PHYS_NONSECURE_PPI] =
1582                 acpi_gtdt_map_ppi(ARCH_TIMER_PHYS_NONSECURE_PPI);
1583 
1584         arch_timer_ppi[ARCH_TIMER_VIRT_PPI] =
1585                 acpi_gtdt_map_ppi(ARCH_TIMER_VIRT_PPI);
1586 
1587         arch_timer_ppi[ARCH_TIMER_HYP_PPI] =
1588                 acpi_gtdt_map_ppi(ARCH_TIMER_HYP_PPI);
1589 
1590         arch_timer_populate_kvm_info();
1591 
1592         /*
1593          * When probing via ACPI, we have no mechanism to override the sysreg
1594          * CNTFRQ value. This *must* be correct.
1595          */
1596         arch_timer_rate = arch_timer_get_cntfrq();
1597         if (!arch_timer_rate) {
1598                 pr_err(FW_BUG "frequency not available.\n");
1599                 return -EINVAL;
1600         }
1601 
1602         arch_timer_uses_ppi = arch_timer_select_ppi();
1603         if (!arch_timer_ppi[arch_timer_uses_ppi]) {
1604                 pr_err("No interrupt available, giving up\n");
1605                 return -EINVAL;
1606         }
1607 
1608         /* Always-on capability */
1609         arch_timer_c3stop = acpi_gtdt_c3stop(arch_timer_uses_ppi);
1610 
1611         /* Check for globally applicable workarounds */
1612         arch_timer_check_ool_workaround(ate_match_acpi_oem_info, table);
1613 
1614         ret = arch_timer_register();
1615         if (ret)
1616                 return ret;
1617 
1618         if (platform_timer_count &&
1619             arch_timer_mem_acpi_init(platform_timer_count))
1620                 pr_err("Failed to initialize memory-mapped timer.\n");
1621 
1622         return arch_timer_common_init();
1623 }
1624 TIMER_ACPI_DECLARE(arch_timer, ACPI_SIG_GTDT, arch_timer_acpi_init);
1625 #endif
/* [<][>][^][v][top][bottom][index][help] */
root/drivers/clocksource/arm_arch_timer.c

DEFINITIONS
