1/* 2 * ARM big.LITTLE Platforms CPUFreq support 3 * 4 * Copyright (C) 2013 ARM Ltd. 5 * Sudeep KarkadaNagesha <sudeep.karkadanagesha@arm.com> 6 * 7 * Copyright (C) 2013 Linaro. 8 * Viresh Kumar <viresh.kumar@linaro.org> 9 * 10 * This program is free software; you can redistribute it and/or modify 11 * it under the terms of the GNU General Public License version 2 as 12 * published by the Free Software Foundation. 13 * 14 * This program is distributed "as is" WITHOUT ANY WARRANTY of any 15 * kind, whether express or implied; without even the implied warranty 16 * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 17 * GNU General Public License for more details. 18 */ 19 20#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 21 22#include <linux/clk.h> 23#include <linux/cpu.h> 24#include <linux/cpufreq.h> 25#include <linux/cpumask.h> 26#include <linux/export.h> 27#include <linux/module.h> 28#include <linux/mutex.h> 29#include <linux/of_platform.h> 30#include <linux/pm_opp.h> 31#include <linux/slab.h> 32#include <linux/topology.h> 33#include <linux/types.h> 34 35#include "arm_big_little.h" 36 37/* Currently we support only two clusters */ 38#define A15_CLUSTER 0 39#define A7_CLUSTER 1 40#define MAX_CLUSTERS 2 41 42#ifdef CONFIG_BL_SWITCHER 43#include <asm/bL_switcher.h> 44static bool bL_switching_enabled; 45#define is_bL_switching_enabled() bL_switching_enabled 46#define set_switching_enabled(x) (bL_switching_enabled = (x)) 47#else 48#define is_bL_switching_enabled() false 49#define set_switching_enabled(x) do { } while (0) 50#define bL_switch_request(...) do { } while (0) 51#define bL_switcher_put_enabled() do { } while (0) 52#define bL_switcher_get_enabled() do { } while (0) 53#endif 54 55#define ACTUAL_FREQ(cluster, freq) ((cluster == A7_CLUSTER) ? freq << 1 : freq) 56#define VIRT_FREQ(cluster, freq) ((cluster == A7_CLUSTER) ? freq >> 1 : freq) 57 58static struct cpufreq_arm_bL_ops *arm_bL_ops; 59static struct clk *clk[MAX_CLUSTERS]; 60static struct cpufreq_frequency_table *freq_table[MAX_CLUSTERS + 1]; 61static atomic_t cluster_usage[MAX_CLUSTERS + 1]; 62 63static unsigned int clk_big_min; /* (Big) clock frequencies */ 64static unsigned int clk_little_max; /* Maximum clock frequency (Little) */ 65 66static DEFINE_PER_CPU(unsigned int, physical_cluster); 67static DEFINE_PER_CPU(unsigned int, cpu_last_req_freq); 68 69static struct mutex cluster_lock[MAX_CLUSTERS]; 70 71static inline int raw_cpu_to_cluster(int cpu) 72{ 73 return topology_physical_package_id(cpu); 74} 75 76static inline int cpu_to_cluster(int cpu) 77{ 78 return is_bL_switching_enabled() ? 79 MAX_CLUSTERS : raw_cpu_to_cluster(cpu); 80} 81 82static unsigned int find_cluster_maxfreq(int cluster) 83{ 84 int j; 85 u32 max_freq = 0, cpu_freq; 86 87 for_each_online_cpu(j) { 88 cpu_freq = per_cpu(cpu_last_req_freq, j); 89 90 if ((cluster == per_cpu(physical_cluster, j)) && 91 (max_freq < cpu_freq)) 92 max_freq = cpu_freq; 93 } 94 95 pr_debug("%s: cluster: %d, max freq: %d\n", __func__, cluster, 96 max_freq); 97 98 return max_freq; 99} 100 101static unsigned int clk_get_cpu_rate(unsigned int cpu) 102{ 103 u32 cur_cluster = per_cpu(physical_cluster, cpu); 104 u32 rate = clk_get_rate(clk[cur_cluster]) / 1000; 105 106 /* For switcher we use virtual A7 clock rates */ 107 if (is_bL_switching_enabled()) 108 rate = VIRT_FREQ(cur_cluster, rate); 109 110 pr_debug("%s: cpu: %d, cluster: %d, freq: %u\n", __func__, cpu, 111 cur_cluster, rate); 112 113 return rate; 114} 115 116static unsigned int bL_cpufreq_get_rate(unsigned int cpu) 117{ 118 if (is_bL_switching_enabled()) { 119 pr_debug("%s: freq: %d\n", __func__, per_cpu(cpu_last_req_freq, 120 cpu)); 121 122 return per_cpu(cpu_last_req_freq, cpu); 123 } else { 124 return clk_get_cpu_rate(cpu); 125 } 126} 127 128static unsigned int 129bL_cpufreq_set_rate(u32 cpu, u32 old_cluster, u32 new_cluster, u32 rate) 130{ 131 u32 new_rate, prev_rate; 132 int ret; 133 bool bLs = is_bL_switching_enabled(); 134 135 mutex_lock(&cluster_lock[new_cluster]); 136 137 if (bLs) { 138 prev_rate = per_cpu(cpu_last_req_freq, cpu); 139 per_cpu(cpu_last_req_freq, cpu) = rate; 140 per_cpu(physical_cluster, cpu) = new_cluster; 141 142 new_rate = find_cluster_maxfreq(new_cluster); 143 new_rate = ACTUAL_FREQ(new_cluster, new_rate); 144 } else { 145 new_rate = rate; 146 } 147 148 pr_debug("%s: cpu: %d, old cluster: %d, new cluster: %d, freq: %d\n", 149 __func__, cpu, old_cluster, new_cluster, new_rate); 150 151 ret = clk_set_rate(clk[new_cluster], new_rate * 1000); 152 if (!ret) { 153 /* 154 * FIXME: clk_set_rate hasn't returned an error here however it 155 * may be that clk_change_rate failed due to hardware or 156 * firmware issues and wasn't able to report that due to the 157 * current design of the clk core layer. To work around this 158 * problem we will read back the clock rate and check it is 159 * correct. This needs to be removed once clk core is fixed. 160 */ 161 if (clk_get_rate(clk[new_cluster]) != new_rate * 1000) 162 ret = -EIO; 163 } 164 165 if (WARN_ON(ret)) { 166 pr_err("clk_set_rate failed: %d, new cluster: %d\n", ret, 167 new_cluster); 168 if (bLs) { 169 per_cpu(cpu_last_req_freq, cpu) = prev_rate; 170 per_cpu(physical_cluster, cpu) = old_cluster; 171 } 172 173 mutex_unlock(&cluster_lock[new_cluster]); 174 175 return ret; 176 } 177 178 mutex_unlock(&cluster_lock[new_cluster]); 179 180 /* Recalc freq for old cluster when switching clusters */ 181 if (old_cluster != new_cluster) { 182 pr_debug("%s: cpu: %d, old cluster: %d, new cluster: %d\n", 183 __func__, cpu, old_cluster, new_cluster); 184 185 /* Switch cluster */ 186 bL_switch_request(cpu, new_cluster); 187 188 mutex_lock(&cluster_lock[old_cluster]); 189 190 /* Set freq of old cluster if there are cpus left on it */ 191 new_rate = find_cluster_maxfreq(old_cluster); 192 new_rate = ACTUAL_FREQ(old_cluster, new_rate); 193 194 if (new_rate) { 195 pr_debug("%s: Updating rate of old cluster: %d, to freq: %d\n", 196 __func__, old_cluster, new_rate); 197 198 if (clk_set_rate(clk[old_cluster], new_rate * 1000)) 199 pr_err("%s: clk_set_rate failed: %d, old cluster: %d\n", 200 __func__, ret, old_cluster); 201 } 202 mutex_unlock(&cluster_lock[old_cluster]); 203 } 204 205 return 0; 206} 207 208/* Set clock frequency */ 209static int bL_cpufreq_set_target(struct cpufreq_policy *policy, 210 unsigned int index) 211{ 212 u32 cpu = policy->cpu, cur_cluster, new_cluster, actual_cluster; 213 unsigned int freqs_new; 214 215 cur_cluster = cpu_to_cluster(cpu); 216 new_cluster = actual_cluster = per_cpu(physical_cluster, cpu); 217 218 freqs_new = freq_table[cur_cluster][index].frequency; 219 220 if (is_bL_switching_enabled()) { 221 if ((actual_cluster == A15_CLUSTER) && 222 (freqs_new < clk_big_min)) { 223 new_cluster = A7_CLUSTER; 224 } else if ((actual_cluster == A7_CLUSTER) && 225 (freqs_new > clk_little_max)) { 226 new_cluster = A15_CLUSTER; 227 } 228 } 229 230 return bL_cpufreq_set_rate(cpu, actual_cluster, new_cluster, freqs_new); 231} 232 233static inline u32 get_table_count(struct cpufreq_frequency_table *table) 234{ 235 int count; 236 237 for (count = 0; table[count].frequency != CPUFREQ_TABLE_END; count++) 238 ; 239 240 return count; 241} 242 243/* get the minimum frequency in the cpufreq_frequency_table */ 244static inline u32 get_table_min(struct cpufreq_frequency_table *table) 245{ 246 struct cpufreq_frequency_table *pos; 247 uint32_t min_freq = ~0; 248 cpufreq_for_each_entry(pos, table) 249 if (pos->frequency < min_freq) 250 min_freq = pos->frequency; 251 return min_freq; 252} 253 254/* get the maximum frequency in the cpufreq_frequency_table */ 255static inline u32 get_table_max(struct cpufreq_frequency_table *table) 256{ 257 struct cpufreq_frequency_table *pos; 258 uint32_t max_freq = 0; 259 cpufreq_for_each_entry(pos, table) 260 if (pos->frequency > max_freq) 261 max_freq = pos->frequency; 262 return max_freq; 263} 264 265static int merge_cluster_tables(void) 266{ 267 int i, j, k = 0, count = 1; 268 struct cpufreq_frequency_table *table; 269 270 for (i = 0; i < MAX_CLUSTERS; i++) 271 count += get_table_count(freq_table[i]); 272 273 table = kzalloc(sizeof(*table) * count, GFP_KERNEL); 274 if (!table) 275 return -ENOMEM; 276 277 freq_table[MAX_CLUSTERS] = table; 278 279 /* Add in reverse order to get freqs in increasing order */ 280 for (i = MAX_CLUSTERS - 1; i >= 0; i--) { 281 for (j = 0; freq_table[i][j].frequency != CPUFREQ_TABLE_END; 282 j++) { 283 table[k].frequency = VIRT_FREQ(i, 284 freq_table[i][j].frequency); 285 pr_debug("%s: index: %d, freq: %d\n", __func__, k, 286 table[k].frequency); 287 k++; 288 } 289 } 290 291 table[k].driver_data = k; 292 table[k].frequency = CPUFREQ_TABLE_END; 293 294 pr_debug("%s: End, table: %p, count: %d\n", __func__, table, k); 295 296 return 0; 297} 298 299static void _put_cluster_clk_and_freq_table(struct device *cpu_dev) 300{ 301 u32 cluster = raw_cpu_to_cluster(cpu_dev->id); 302 303 if (!freq_table[cluster]) 304 return; 305 306 clk_put(clk[cluster]); 307 dev_pm_opp_free_cpufreq_table(cpu_dev, &freq_table[cluster]); 308 if (arm_bL_ops->free_opp_table) 309 arm_bL_ops->free_opp_table(cpu_dev); 310 dev_dbg(cpu_dev, "%s: cluster: %d\n", __func__, cluster); 311} 312 313static void put_cluster_clk_and_freq_table(struct device *cpu_dev) 314{ 315 u32 cluster = cpu_to_cluster(cpu_dev->id); 316 int i; 317 318 if (atomic_dec_return(&cluster_usage[cluster])) 319 return; 320 321 if (cluster < MAX_CLUSTERS) 322 return _put_cluster_clk_and_freq_table(cpu_dev); 323 324 for_each_present_cpu(i) { 325 struct device *cdev = get_cpu_device(i); 326 if (!cdev) { 327 pr_err("%s: failed to get cpu%d device\n", __func__, i); 328 return; 329 } 330 331 _put_cluster_clk_and_freq_table(cdev); 332 } 333 334 /* free virtual table */ 335 kfree(freq_table[cluster]); 336} 337 338static int _get_cluster_clk_and_freq_table(struct device *cpu_dev) 339{ 340 u32 cluster = raw_cpu_to_cluster(cpu_dev->id); 341 int ret; 342 343 if (freq_table[cluster]) 344 return 0; 345 346 ret = arm_bL_ops->init_opp_table(cpu_dev); 347 if (ret) { 348 dev_err(cpu_dev, "%s: init_opp_table failed, cpu: %d, err: %d\n", 349 __func__, cpu_dev->id, ret); 350 goto out; 351 } 352 353 ret = dev_pm_opp_init_cpufreq_table(cpu_dev, &freq_table[cluster]); 354 if (ret) { 355 dev_err(cpu_dev, "%s: failed to init cpufreq table, cpu: %d, err: %d\n", 356 __func__, cpu_dev->id, ret); 357 goto free_opp_table; 358 } 359 360 clk[cluster] = clk_get(cpu_dev, NULL); 361 if (!IS_ERR(clk[cluster])) { 362 dev_dbg(cpu_dev, "%s: clk: %p & freq table: %p, cluster: %d\n", 363 __func__, clk[cluster], freq_table[cluster], 364 cluster); 365 return 0; 366 } 367 368 dev_err(cpu_dev, "%s: Failed to get clk for cpu: %d, cluster: %d\n", 369 __func__, cpu_dev->id, cluster); 370 ret = PTR_ERR(clk[cluster]); 371 dev_pm_opp_free_cpufreq_table(cpu_dev, &freq_table[cluster]); 372 373free_opp_table: 374 if (arm_bL_ops->free_opp_table) 375 arm_bL_ops->free_opp_table(cpu_dev); 376out: 377 dev_err(cpu_dev, "%s: Failed to get data for cluster: %d\n", __func__, 378 cluster); 379 return ret; 380} 381 382static int get_cluster_clk_and_freq_table(struct device *cpu_dev) 383{ 384 u32 cluster = cpu_to_cluster(cpu_dev->id); 385 int i, ret; 386 387 if (atomic_inc_return(&cluster_usage[cluster]) != 1) 388 return 0; 389 390 if (cluster < MAX_CLUSTERS) { 391 ret = _get_cluster_clk_and_freq_table(cpu_dev); 392 if (ret) 393 atomic_dec(&cluster_usage[cluster]); 394 return ret; 395 } 396 397 /* 398 * Get data for all clusters and fill virtual cluster with a merge of 399 * both 400 */ 401 for_each_present_cpu(i) { 402 struct device *cdev = get_cpu_device(i); 403 if (!cdev) { 404 pr_err("%s: failed to get cpu%d device\n", __func__, i); 405 return -ENODEV; 406 } 407 408 ret = _get_cluster_clk_and_freq_table(cdev); 409 if (ret) 410 goto put_clusters; 411 } 412 413 ret = merge_cluster_tables(); 414 if (ret) 415 goto put_clusters; 416 417 /* Assuming 2 cluster, set clk_big_min and clk_little_max */ 418 clk_big_min = get_table_min(freq_table[0]); 419 clk_little_max = VIRT_FREQ(1, get_table_max(freq_table[1])); 420 421 pr_debug("%s: cluster: %d, clk_big_min: %d, clk_little_max: %d\n", 422 __func__, cluster, clk_big_min, clk_little_max); 423 424 return 0; 425 426put_clusters: 427 for_each_present_cpu(i) { 428 struct device *cdev = get_cpu_device(i); 429 if (!cdev) { 430 pr_err("%s: failed to get cpu%d device\n", __func__, i); 431 return -ENODEV; 432 } 433 434 _put_cluster_clk_and_freq_table(cdev); 435 } 436 437 atomic_dec(&cluster_usage[cluster]); 438 439 return ret; 440} 441 442/* Per-CPU initialization */ 443static int bL_cpufreq_init(struct cpufreq_policy *policy) 444{ 445 u32 cur_cluster = cpu_to_cluster(policy->cpu); 446 struct device *cpu_dev; 447 int ret; 448 449 cpu_dev = get_cpu_device(policy->cpu); 450 if (!cpu_dev) { 451 pr_err("%s: failed to get cpu%d device\n", __func__, 452 policy->cpu); 453 return -ENODEV; 454 } 455 456 ret = get_cluster_clk_and_freq_table(cpu_dev); 457 if (ret) 458 return ret; 459 460 ret = cpufreq_table_validate_and_show(policy, freq_table[cur_cluster]); 461 if (ret) { 462 dev_err(cpu_dev, "CPU %d, cluster: %d invalid freq table\n", 463 policy->cpu, cur_cluster); 464 put_cluster_clk_and_freq_table(cpu_dev); 465 return ret; 466 } 467 468 if (cur_cluster < MAX_CLUSTERS) { 469 int cpu; 470 471 cpumask_copy(policy->cpus, topology_core_cpumask(policy->cpu)); 472 473 for_each_cpu(cpu, policy->cpus) 474 per_cpu(physical_cluster, cpu) = cur_cluster; 475 } else { 476 /* Assumption: during init, we are always running on A15 */ 477 per_cpu(physical_cluster, policy->cpu) = A15_CLUSTER; 478 } 479 480 if (arm_bL_ops->get_transition_latency) 481 policy->cpuinfo.transition_latency = 482 arm_bL_ops->get_transition_latency(cpu_dev); 483 else 484 policy->cpuinfo.transition_latency = CPUFREQ_ETERNAL; 485 486 if (is_bL_switching_enabled()) 487 per_cpu(cpu_last_req_freq, policy->cpu) = clk_get_cpu_rate(policy->cpu); 488 489 dev_info(cpu_dev, "%s: CPU %d initialized\n", __func__, policy->cpu); 490 return 0; 491} 492 493static int bL_cpufreq_exit(struct cpufreq_policy *policy) 494{ 495 struct device *cpu_dev; 496 497 cpu_dev = get_cpu_device(policy->cpu); 498 if (!cpu_dev) { 499 pr_err("%s: failed to get cpu%d device\n", __func__, 500 policy->cpu); 501 return -ENODEV; 502 } 503 504 put_cluster_clk_and_freq_table(cpu_dev); 505 dev_dbg(cpu_dev, "%s: Exited, cpu: %d\n", __func__, policy->cpu); 506 507 return 0; 508} 509 510static struct cpufreq_driver bL_cpufreq_driver = { 511 .name = "arm-big-little", 512 .flags = CPUFREQ_STICKY | 513 CPUFREQ_HAVE_GOVERNOR_PER_POLICY | 514 CPUFREQ_NEED_INITIAL_FREQ_CHECK, 515 .verify = cpufreq_generic_frequency_table_verify, 516 .target_index = bL_cpufreq_set_target, 517 .get = bL_cpufreq_get_rate, 518 .init = bL_cpufreq_init, 519 .exit = bL_cpufreq_exit, 520 .attr = cpufreq_generic_attr, 521}; 522 523#ifdef CONFIG_BL_SWITCHER 524static int bL_cpufreq_switcher_notifier(struct notifier_block *nfb, 525 unsigned long action, void *_arg) 526{ 527 pr_debug("%s: action: %ld\n", __func__, action); 528 529 switch (action) { 530 case BL_NOTIFY_PRE_ENABLE: 531 case BL_NOTIFY_PRE_DISABLE: 532 cpufreq_unregister_driver(&bL_cpufreq_driver); 533 break; 534 535 case BL_NOTIFY_POST_ENABLE: 536 set_switching_enabled(true); 537 cpufreq_register_driver(&bL_cpufreq_driver); 538 break; 539 540 case BL_NOTIFY_POST_DISABLE: 541 set_switching_enabled(false); 542 cpufreq_register_driver(&bL_cpufreq_driver); 543 break; 544 545 default: 546 return NOTIFY_DONE; 547 } 548 549 return NOTIFY_OK; 550} 551 552static struct notifier_block bL_switcher_notifier = { 553 .notifier_call = bL_cpufreq_switcher_notifier, 554}; 555 556static int __bLs_register_notifier(void) 557{ 558 return bL_switcher_register_notifier(&bL_switcher_notifier); 559} 560 561static int __bLs_unregister_notifier(void) 562{ 563 return bL_switcher_unregister_notifier(&bL_switcher_notifier); 564} 565#else 566static int __bLs_register_notifier(void) { return 0; } 567static int __bLs_unregister_notifier(void) { return 0; } 568#endif 569 570int bL_cpufreq_register(struct cpufreq_arm_bL_ops *ops) 571{ 572 int ret, i; 573 574 if (arm_bL_ops) { 575 pr_debug("%s: Already registered: %s, exiting\n", __func__, 576 arm_bL_ops->name); 577 return -EBUSY; 578 } 579 580 if (!ops || !strlen(ops->name) || !ops->init_opp_table) { 581 pr_err("%s: Invalid arm_bL_ops, exiting\n", __func__); 582 return -ENODEV; 583 } 584 585 arm_bL_ops = ops; 586 587 set_switching_enabled(bL_switcher_get_enabled()); 588 589 for (i = 0; i < MAX_CLUSTERS; i++) 590 mutex_init(&cluster_lock[i]); 591 592 ret = cpufreq_register_driver(&bL_cpufreq_driver); 593 if (ret) { 594 pr_info("%s: Failed registering platform driver: %s, err: %d\n", 595 __func__, ops->name, ret); 596 arm_bL_ops = NULL; 597 } else { 598 ret = __bLs_register_notifier(); 599 if (ret) { 600 cpufreq_unregister_driver(&bL_cpufreq_driver); 601 arm_bL_ops = NULL; 602 } else { 603 pr_info("%s: Registered platform driver: %s\n", 604 __func__, ops->name); 605 } 606 } 607 608 bL_switcher_put_enabled(); 609 return ret; 610} 611EXPORT_SYMBOL_GPL(bL_cpufreq_register); 612 613void bL_cpufreq_unregister(struct cpufreq_arm_bL_ops *ops) 614{ 615 if (arm_bL_ops != ops) { 616 pr_err("%s: Registered with: %s, can't unregister, exiting\n", 617 __func__, arm_bL_ops->name); 618 return; 619 } 620 621 bL_switcher_get_enabled(); 622 __bLs_unregister_notifier(); 623 cpufreq_unregister_driver(&bL_cpufreq_driver); 624 bL_switcher_put_enabled(); 625 pr_info("%s: Un-registered platform driver: %s\n", __func__, 626 arm_bL_ops->name); 627 arm_bL_ops = NULL; 628} 629EXPORT_SYMBOL_GPL(bL_cpufreq_unregister); 630 631MODULE_AUTHOR("Viresh Kumar <viresh.kumar@linaro.org>"); 632MODULE_DESCRIPTION("Generic ARM big LITTLE cpufreq driver"); 633MODULE_LICENSE("GPL v2"); 634