1 /*
2 * Intel Running Average Power Limit (RAPL) Driver
3 * Copyright (c) 2013, Intel Corporation.
4 *
5 * This program is free software; you can redistribute it and/or modify it
6 * under the terms and conditions of the GNU General Public License,
7 * version 2, as published by the Free Software Foundation.
8 *
9 * This program is distributed in the hope it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
12 * more details.
13 *
14 * You should have received a copy of the GNU General Public License along with
15 * this program; if not, write to the Free Software Foundation, Inc.
16 *
17 */
18 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
19
20 #include <linux/kernel.h>
21 #include <linux/module.h>
22 #include <linux/list.h>
23 #include <linux/types.h>
24 #include <linux/device.h>
25 #include <linux/slab.h>
26 #include <linux/log2.h>
27 #include <linux/bitmap.h>
28 #include <linux/delay.h>
29 #include <linux/sysfs.h>
30 #include <linux/cpu.h>
31 #include <linux/powercap.h>
32 #include <asm/iosf_mbi.h>
33
34 #include <asm/processor.h>
35 #include <asm/cpu_device_id.h>
36
37 /* bitmasks for RAPL MSRs, used by primitive access functions */
38 #define ENERGY_STATUS_MASK 0xffffffff
39
40 #define POWER_LIMIT1_MASK 0x7FFF
41 #define POWER_LIMIT1_ENABLE BIT(15)
42 #define POWER_LIMIT1_CLAMP BIT(16)
43
44 #define POWER_LIMIT2_MASK (0x7FFFULL<<32)
45 #define POWER_LIMIT2_ENABLE BIT_ULL(47)
46 #define POWER_LIMIT2_CLAMP BIT_ULL(48)
47 #define POWER_PACKAGE_LOCK BIT_ULL(63)
48 #define POWER_PP_LOCK BIT(31)
49
50 #define TIME_WINDOW1_MASK (0x7FULL<<17)
51 #define TIME_WINDOW2_MASK (0x7FULL<<49)
52
53 #define POWER_UNIT_OFFSET 0
54 #define POWER_UNIT_MASK 0x0F
55
56 #define ENERGY_UNIT_OFFSET 0x08
57 #define ENERGY_UNIT_MASK 0x1F00
58
59 #define TIME_UNIT_OFFSET 0x10
60 #define TIME_UNIT_MASK 0xF0000
61
62 #define POWER_INFO_MAX_MASK (0x7fffULL<<32)
63 #define POWER_INFO_MIN_MASK (0x7fffULL<<16)
64 #define POWER_INFO_MAX_TIME_WIN_MASK (0x3fULL<<48)
65 #define POWER_INFO_THERMAL_SPEC_MASK 0x7fff
66
67 #define PERF_STATUS_THROTTLE_TIME_MASK 0xffffffff
68 #define PP_POLICY_MASK 0x1F
69
70 /* Non HW constants */
71 #define RAPL_PRIMITIVE_DERIVED BIT(1) /* not from raw data */
72 #define RAPL_PRIMITIVE_DUMMY BIT(2)
73
74 #define TIME_WINDOW_MAX_MSEC 40000
75 #define TIME_WINDOW_MIN_MSEC 250
76 #define ENERGY_UNIT_SCALE 1000 /* scale from driver unit to powercap unit */
77 enum unit_type {
78 ARBITRARY_UNIT, /* no translation */
79 POWER_UNIT,
80 ENERGY_UNIT,
81 TIME_UNIT,
82 };
83
84 enum rapl_domain_type {
85 RAPL_DOMAIN_PACKAGE, /* entire package/socket */
86 RAPL_DOMAIN_PP0, /* core power plane */
87 RAPL_DOMAIN_PP1, /* graphics uncore */
88 RAPL_DOMAIN_DRAM,/* DRAM control_type */
89 RAPL_DOMAIN_MAX,
90 };
91
92 enum rapl_domain_msr_id {
93 RAPL_DOMAIN_MSR_LIMIT,
94 RAPL_DOMAIN_MSR_STATUS,
95 RAPL_DOMAIN_MSR_PERF,
96 RAPL_DOMAIN_MSR_POLICY,
97 RAPL_DOMAIN_MSR_INFO,
98 RAPL_DOMAIN_MSR_MAX,
99 };
100
101 /* per domain data, some are optional */
102 enum rapl_primitives {
103 ENERGY_COUNTER,
104 POWER_LIMIT1,
105 POWER_LIMIT2,
106 FW_LOCK,
107
108 PL1_ENABLE, /* power limit 1, aka long term */
109 PL1_CLAMP, /* allow frequency to go below OS request */
110 PL2_ENABLE, /* power limit 2, aka short term, instantaneous */
111 PL2_CLAMP,
112
113 TIME_WINDOW1, /* long term */
114 TIME_WINDOW2, /* short term */
115 THERMAL_SPEC_POWER,
116 MAX_POWER,
117
118 MIN_POWER,
119 MAX_TIME_WINDOW,
120 THROTTLED_TIME,
121 PRIORITY_LEVEL,
122
123 /* below are not raw primitive data */
124 AVERAGE_POWER,
125 NR_RAPL_PRIMITIVES,
126 };
127
128 #define NR_RAW_PRIMITIVES (NR_RAPL_PRIMITIVES - 2)
129
130 /* Can be expanded to include events, etc.*/
131 struct rapl_domain_data {
132 u64 primitives[NR_RAPL_PRIMITIVES];
133 unsigned long timestamp;
134 };
135
136
137 #define DOMAIN_STATE_INACTIVE BIT(0)
138 #define DOMAIN_STATE_POWER_LIMIT_SET BIT(1)
139 #define DOMAIN_STATE_BIOS_LOCKED BIT(2)
140
141 #define NR_POWER_LIMITS (2)
142 struct rapl_power_limit {
143 struct powercap_zone_constraint *constraint;
144 int prim_id; /* primitive ID used to enable */
145 struct rapl_domain *domain;
146 const char *name;
147 };
148
149 static const char pl1_name[] = "long_term";
150 static const char pl2_name[] = "short_term";
151
152 struct rapl_domain {
153 const char *name;
154 enum rapl_domain_type id;
155 int msrs[RAPL_DOMAIN_MSR_MAX];
156 struct powercap_zone power_zone;
157 struct rapl_domain_data rdd;
158 struct rapl_power_limit rpl[NR_POWER_LIMITS];
159 u64 attr_map; /* track capabilities */
160 unsigned int state;
161 unsigned int domain_energy_unit;
162 int package_id;
163 };
164 #define power_zone_to_rapl_domain(_zone) \
165 container_of(_zone, struct rapl_domain, power_zone)
166
167
168 /* Each physical package contains multiple domains, these are the common
169 * data across RAPL domains within a package.
170 */
171 struct rapl_package {
172 unsigned int id; /* physical package/socket id */
173 unsigned int nr_domains;
174 unsigned long domain_map; /* bit map of active domains */
175 unsigned int power_unit;
176 unsigned int energy_unit;
177 unsigned int time_unit;
178 struct rapl_domain *domains; /* array of domains, sized at runtime */
179 struct powercap_zone *power_zone; /* keep track of parent zone */
180 int nr_cpus; /* active cpus on the package, topology info is lost during
181 * cpu hotplug. so we have to track ourselves.
182 */
183 unsigned long power_limit_irq; /* keep track of package power limit
184 * notify interrupt enable status.
185 */
186 struct list_head plist;
187 };
188
189 struct rapl_defaults {
190 int (*check_unit)(struct rapl_package *rp, int cpu);
191 void (*set_floor_freq)(struct rapl_domain *rd, bool mode);
192 u64 (*compute_time_window)(struct rapl_package *rp, u64 val,
193 bool to_raw);
194 unsigned int dram_domain_energy_unit;
195 };
196 static struct rapl_defaults *rapl_defaults;
197
198 /* Sideband MBI registers */
199 #define IOSF_CPU_POWER_BUDGET_CTL (0x2)
200
201 #define PACKAGE_PLN_INT_SAVED BIT(0)
202 #define MAX_PRIM_NAME (32)
203
204 /* per domain data. used to describe individual knobs such that access function
205 * can be consolidated into one instead of many inline functions.
206 */
207 struct rapl_primitive_info {
208 const char *name;
209 u64 mask;
210 int shift;
211 enum rapl_domain_msr_id id;
212 enum unit_type unit;
213 u32 flag;
214 };
215
216 #define PRIMITIVE_INFO_INIT(p, m, s, i, u, f) { \
217 .name = #p, \
218 .mask = m, \
219 .shift = s, \
220 .id = i, \
221 .unit = u, \
222 .flag = f \
223 }
224
225 static void rapl_init_domains(struct rapl_package *rp);
226 static int rapl_read_data_raw(struct rapl_domain *rd,
227 enum rapl_primitives prim,
228 bool xlate, u64 *data);
229 static int rapl_write_data_raw(struct rapl_domain *rd,
230 enum rapl_primitives prim,
231 unsigned long long value);
232 static u64 rapl_unit_xlate(struct rapl_domain *rd, int package,
233 enum unit_type type, u64 value,
234 int to_raw);
235 static void package_power_limit_irq_save(int package_id);
236
237 static LIST_HEAD(rapl_packages); /* guarded by CPU hotplug lock */
238
239 static const char * const rapl_domain_names[] = {
240 "package",
241 "core",
242 "uncore",
243 "dram",
244 };
245
246 static struct powercap_control_type *control_type; /* PowerCap Controller */
247
248 /* caller to ensure CPU hotplug lock is held */
find_package_by_id(int id)249 static struct rapl_package *find_package_by_id(int id)
250 {
251 struct rapl_package *rp;
252
253 list_for_each_entry(rp, &rapl_packages, plist) {
254 if (rp->id == id)
255 return rp;
256 }
257
258 return NULL;
259 }
260
261 /* caller to ensure CPU hotplug lock is held */
find_active_cpu_on_package(int package_id)262 static int find_active_cpu_on_package(int package_id)
263 {
264 int i;
265
266 for_each_online_cpu(i) {
267 if (topology_physical_package_id(i) == package_id)
268 return i;
269 }
270 /* all CPUs on this package are offline */
271
272 return -ENODEV;
273 }
274
275 /* caller must hold cpu hotplug lock */
rapl_cleanup_data(void)276 static void rapl_cleanup_data(void)
277 {
278 struct rapl_package *p, *tmp;
279
280 list_for_each_entry_safe(p, tmp, &rapl_packages, plist) {
281 kfree(p->domains);
282 list_del(&p->plist);
283 kfree(p);
284 }
285 }
286
get_energy_counter(struct powercap_zone * power_zone,u64 * energy_raw)287 static int get_energy_counter(struct powercap_zone *power_zone, u64 *energy_raw)
288 {
289 struct rapl_domain *rd;
290 u64 energy_now;
291
292 /* prevent CPU hotplug, make sure the RAPL domain does not go
293 * away while reading the counter.
294 */
295 get_online_cpus();
296 rd = power_zone_to_rapl_domain(power_zone);
297
298 if (!rapl_read_data_raw(rd, ENERGY_COUNTER, true, &energy_now)) {
299 *energy_raw = energy_now;
300 put_online_cpus();
301
302 return 0;
303 }
304 put_online_cpus();
305
306 return -EIO;
307 }
308
get_max_energy_counter(struct powercap_zone * pcd_dev,u64 * energy)309 static int get_max_energy_counter(struct powercap_zone *pcd_dev, u64 *energy)
310 {
311 struct rapl_domain *rd = power_zone_to_rapl_domain(pcd_dev);
312
313 *energy = rapl_unit_xlate(rd, 0, ENERGY_UNIT, ENERGY_STATUS_MASK, 0);
314 return 0;
315 }
316
release_zone(struct powercap_zone * power_zone)317 static int release_zone(struct powercap_zone *power_zone)
318 {
319 struct rapl_domain *rd = power_zone_to_rapl_domain(power_zone);
320 struct rapl_package *rp;
321
322 /* package zone is the last zone of a package, we can free
323 * memory here since all children has been unregistered.
324 */
325 if (rd->id == RAPL_DOMAIN_PACKAGE) {
326 rp = find_package_by_id(rd->package_id);
327 if (!rp) {
328 dev_warn(&power_zone->dev, "no package id %s\n",
329 rd->name);
330 return -ENODEV;
331 }
332 kfree(rd);
333 rp->domains = NULL;
334 }
335
336 return 0;
337
338 }
339
find_nr_power_limit(struct rapl_domain * rd)340 static int find_nr_power_limit(struct rapl_domain *rd)
341 {
342 int i;
343
344 for (i = 0; i < NR_POWER_LIMITS; i++) {
345 if (rd->rpl[i].name == NULL)
346 break;
347 }
348
349 return i;
350 }
351
set_domain_enable(struct powercap_zone * power_zone,bool mode)352 static int set_domain_enable(struct powercap_zone *power_zone, bool mode)
353 {
354 struct rapl_domain *rd = power_zone_to_rapl_domain(power_zone);
355
356 if (rd->state & DOMAIN_STATE_BIOS_LOCKED)
357 return -EACCES;
358
359 get_online_cpus();
360 rapl_write_data_raw(rd, PL1_ENABLE, mode);
361 rapl_defaults->set_floor_freq(rd, mode);
362 put_online_cpus();
363
364 return 0;
365 }
366
get_domain_enable(struct powercap_zone * power_zone,bool * mode)367 static int get_domain_enable(struct powercap_zone *power_zone, bool *mode)
368 {
369 struct rapl_domain *rd = power_zone_to_rapl_domain(power_zone);
370 u64 val;
371
372 if (rd->state & DOMAIN_STATE_BIOS_LOCKED) {
373 *mode = false;
374 return 0;
375 }
376 get_online_cpus();
377 if (rapl_read_data_raw(rd, PL1_ENABLE, true, &val)) {
378 put_online_cpus();
379 return -EIO;
380 }
381 *mode = val;
382 put_online_cpus();
383
384 return 0;
385 }
386
387 /* per RAPL domain ops, in the order of rapl_domain_type */
388 static struct powercap_zone_ops zone_ops[] = {
389 /* RAPL_DOMAIN_PACKAGE */
390 {
391 .get_energy_uj = get_energy_counter,
392 .get_max_energy_range_uj = get_max_energy_counter,
393 .release = release_zone,
394 .set_enable = set_domain_enable,
395 .get_enable = get_domain_enable,
396 },
397 /* RAPL_DOMAIN_PP0 */
398 {
399 .get_energy_uj = get_energy_counter,
400 .get_max_energy_range_uj = get_max_energy_counter,
401 .release = release_zone,
402 .set_enable = set_domain_enable,
403 .get_enable = get_domain_enable,
404 },
405 /* RAPL_DOMAIN_PP1 */
406 {
407 .get_energy_uj = get_energy_counter,
408 .get_max_energy_range_uj = get_max_energy_counter,
409 .release = release_zone,
410 .set_enable = set_domain_enable,
411 .get_enable = get_domain_enable,
412 },
413 /* RAPL_DOMAIN_DRAM */
414 {
415 .get_energy_uj = get_energy_counter,
416 .get_max_energy_range_uj = get_max_energy_counter,
417 .release = release_zone,
418 .set_enable = set_domain_enable,
419 .get_enable = get_domain_enable,
420 },
421 };
422
set_power_limit(struct powercap_zone * power_zone,int id,u64 power_limit)423 static int set_power_limit(struct powercap_zone *power_zone, int id,
424 u64 power_limit)
425 {
426 struct rapl_domain *rd;
427 struct rapl_package *rp;
428 int ret = 0;
429
430 get_online_cpus();
431 rd = power_zone_to_rapl_domain(power_zone);
432 rp = find_package_by_id(rd->package_id);
433 if (!rp) {
434 ret = -ENODEV;
435 goto set_exit;
436 }
437
438 if (rd->state & DOMAIN_STATE_BIOS_LOCKED) {
439 dev_warn(&power_zone->dev, "%s locked by BIOS, monitoring only\n",
440 rd->name);
441 ret = -EACCES;
442 goto set_exit;
443 }
444
445 switch (rd->rpl[id].prim_id) {
446 case PL1_ENABLE:
447 rapl_write_data_raw(rd, POWER_LIMIT1, power_limit);
448 break;
449 case PL2_ENABLE:
450 rapl_write_data_raw(rd, POWER_LIMIT2, power_limit);
451 break;
452 default:
453 ret = -EINVAL;
454 }
455 if (!ret)
456 package_power_limit_irq_save(rd->package_id);
457 set_exit:
458 put_online_cpus();
459 return ret;
460 }
461
get_current_power_limit(struct powercap_zone * power_zone,int id,u64 * data)462 static int get_current_power_limit(struct powercap_zone *power_zone, int id,
463 u64 *data)
464 {
465 struct rapl_domain *rd;
466 u64 val;
467 int prim;
468 int ret = 0;
469
470 get_online_cpus();
471 rd = power_zone_to_rapl_domain(power_zone);
472 switch (rd->rpl[id].prim_id) {
473 case PL1_ENABLE:
474 prim = POWER_LIMIT1;
475 break;
476 case PL2_ENABLE:
477 prim = POWER_LIMIT2;
478 break;
479 default:
480 put_online_cpus();
481 return -EINVAL;
482 }
483 if (rapl_read_data_raw(rd, prim, true, &val))
484 ret = -EIO;
485 else
486 *data = val;
487
488 put_online_cpus();
489
490 return ret;
491 }
492
set_time_window(struct powercap_zone * power_zone,int id,u64 window)493 static int set_time_window(struct powercap_zone *power_zone, int id,
494 u64 window)
495 {
496 struct rapl_domain *rd;
497 int ret = 0;
498
499 get_online_cpus();
500 rd = power_zone_to_rapl_domain(power_zone);
501 switch (rd->rpl[id].prim_id) {
502 case PL1_ENABLE:
503 rapl_write_data_raw(rd, TIME_WINDOW1, window);
504 break;
505 case PL2_ENABLE:
506 rapl_write_data_raw(rd, TIME_WINDOW2, window);
507 break;
508 default:
509 ret = -EINVAL;
510 }
511 put_online_cpus();
512 return ret;
513 }
514
get_time_window(struct powercap_zone * power_zone,int id,u64 * data)515 static int get_time_window(struct powercap_zone *power_zone, int id, u64 *data)
516 {
517 struct rapl_domain *rd;
518 u64 val;
519 int ret = 0;
520
521 get_online_cpus();
522 rd = power_zone_to_rapl_domain(power_zone);
523 switch (rd->rpl[id].prim_id) {
524 case PL1_ENABLE:
525 ret = rapl_read_data_raw(rd, TIME_WINDOW1, true, &val);
526 break;
527 case PL2_ENABLE:
528 ret = rapl_read_data_raw(rd, TIME_WINDOW2, true, &val);
529 break;
530 default:
531 put_online_cpus();
532 return -EINVAL;
533 }
534 if (!ret)
535 *data = val;
536 put_online_cpus();
537
538 return ret;
539 }
540
get_constraint_name(struct powercap_zone * power_zone,int id)541 static const char *get_constraint_name(struct powercap_zone *power_zone, int id)
542 {
543 struct rapl_power_limit *rpl;
544 struct rapl_domain *rd;
545
546 rd = power_zone_to_rapl_domain(power_zone);
547 rpl = (struct rapl_power_limit *) &rd->rpl[id];
548
549 return rpl->name;
550 }
551
552
get_max_power(struct powercap_zone * power_zone,int id,u64 * data)553 static int get_max_power(struct powercap_zone *power_zone, int id,
554 u64 *data)
555 {
556 struct rapl_domain *rd;
557 u64 val;
558 int prim;
559 int ret = 0;
560
561 get_online_cpus();
562 rd = power_zone_to_rapl_domain(power_zone);
563 switch (rd->rpl[id].prim_id) {
564 case PL1_ENABLE:
565 prim = THERMAL_SPEC_POWER;
566 break;
567 case PL2_ENABLE:
568 prim = MAX_POWER;
569 break;
570 default:
571 put_online_cpus();
572 return -EINVAL;
573 }
574 if (rapl_read_data_raw(rd, prim, true, &val))
575 ret = -EIO;
576 else
577 *data = val;
578
579 put_online_cpus();
580
581 return ret;
582 }
583
584 static struct powercap_zone_constraint_ops constraint_ops = {
585 .set_power_limit_uw = set_power_limit,
586 .get_power_limit_uw = get_current_power_limit,
587 .set_time_window_us = set_time_window,
588 .get_time_window_us = get_time_window,
589 .get_max_power_uw = get_max_power,
590 .get_name = get_constraint_name,
591 };
592
593 /* called after domain detection and package level data are set */
rapl_init_domains(struct rapl_package * rp)594 static void rapl_init_domains(struct rapl_package *rp)
595 {
596 int i;
597 struct rapl_domain *rd = rp->domains;
598
599 for (i = 0; i < RAPL_DOMAIN_MAX; i++) {
600 unsigned int mask = rp->domain_map & (1 << i);
601 switch (mask) {
602 case BIT(RAPL_DOMAIN_PACKAGE):
603 rd->name = rapl_domain_names[RAPL_DOMAIN_PACKAGE];
604 rd->id = RAPL_DOMAIN_PACKAGE;
605 rd->msrs[0] = MSR_PKG_POWER_LIMIT;
606 rd->msrs[1] = MSR_PKG_ENERGY_STATUS;
607 rd->msrs[2] = MSR_PKG_PERF_STATUS;
608 rd->msrs[3] = 0;
609 rd->msrs[4] = MSR_PKG_POWER_INFO;
610 rd->rpl[0].prim_id = PL1_ENABLE;
611 rd->rpl[0].name = pl1_name;
612 rd->rpl[1].prim_id = PL2_ENABLE;
613 rd->rpl[1].name = pl2_name;
614 break;
615 case BIT(RAPL_DOMAIN_PP0):
616 rd->name = rapl_domain_names[RAPL_DOMAIN_PP0];
617 rd->id = RAPL_DOMAIN_PP0;
618 rd->msrs[0] = MSR_PP0_POWER_LIMIT;
619 rd->msrs[1] = MSR_PP0_ENERGY_STATUS;
620 rd->msrs[2] = 0;
621 rd->msrs[3] = MSR_PP0_POLICY;
622 rd->msrs[4] = 0;
623 rd->rpl[0].prim_id = PL1_ENABLE;
624 rd->rpl[0].name = pl1_name;
625 break;
626 case BIT(RAPL_DOMAIN_PP1):
627 rd->name = rapl_domain_names[RAPL_DOMAIN_PP1];
628 rd->id = RAPL_DOMAIN_PP1;
629 rd->msrs[0] = MSR_PP1_POWER_LIMIT;
630 rd->msrs[1] = MSR_PP1_ENERGY_STATUS;
631 rd->msrs[2] = 0;
632 rd->msrs[3] = MSR_PP1_POLICY;
633 rd->msrs[4] = 0;
634 rd->rpl[0].prim_id = PL1_ENABLE;
635 rd->rpl[0].name = pl1_name;
636 break;
637 case BIT(RAPL_DOMAIN_DRAM):
638 rd->name = rapl_domain_names[RAPL_DOMAIN_DRAM];
639 rd->id = RAPL_DOMAIN_DRAM;
640 rd->msrs[0] = MSR_DRAM_POWER_LIMIT;
641 rd->msrs[1] = MSR_DRAM_ENERGY_STATUS;
642 rd->msrs[2] = MSR_DRAM_PERF_STATUS;
643 rd->msrs[3] = 0;
644 rd->msrs[4] = MSR_DRAM_POWER_INFO;
645 rd->rpl[0].prim_id = PL1_ENABLE;
646 rd->rpl[0].name = pl1_name;
647 rd->domain_energy_unit =
648 rapl_defaults->dram_domain_energy_unit;
649 if (rd->domain_energy_unit)
650 pr_info("DRAM domain energy unit %dpj\n",
651 rd->domain_energy_unit);
652 break;
653 }
654 if (mask) {
655 rd->package_id = rp->id;
656 rd++;
657 }
658 }
659 }
660
rapl_unit_xlate(struct rapl_domain * rd,int package,enum unit_type type,u64 value,int to_raw)661 static u64 rapl_unit_xlate(struct rapl_domain *rd, int package,
662 enum unit_type type, u64 value,
663 int to_raw)
664 {
665 u64 units = 1;
666 struct rapl_package *rp;
667 u64 scale = 1;
668
669 rp = find_package_by_id(package);
670 if (!rp)
671 return value;
672
673 switch (type) {
674 case POWER_UNIT:
675 units = rp->power_unit;
676 break;
677 case ENERGY_UNIT:
678 scale = ENERGY_UNIT_SCALE;
679 /* per domain unit takes precedence */
680 if (rd && rd->domain_energy_unit)
681 units = rd->domain_energy_unit;
682 else
683 units = rp->energy_unit;
684 break;
685 case TIME_UNIT:
686 return rapl_defaults->compute_time_window(rp, value, to_raw);
687 case ARBITRARY_UNIT:
688 default:
689 return value;
690 };
691
692 if (to_raw)
693 return div64_u64(value, units) * scale;
694
695 value *= units;
696
697 return div64_u64(value, scale);
698 }
699
700 /* in the order of enum rapl_primitives */
701 static struct rapl_primitive_info rpi[] = {
702 /* name, mask, shift, msr index, unit divisor */
703 PRIMITIVE_INFO_INIT(ENERGY_COUNTER, ENERGY_STATUS_MASK, 0,
704 RAPL_DOMAIN_MSR_STATUS, ENERGY_UNIT, 0),
705 PRIMITIVE_INFO_INIT(POWER_LIMIT1, POWER_LIMIT1_MASK, 0,
706 RAPL_DOMAIN_MSR_LIMIT, POWER_UNIT, 0),
707 PRIMITIVE_INFO_INIT(POWER_LIMIT2, POWER_LIMIT2_MASK, 32,
708 RAPL_DOMAIN_MSR_LIMIT, POWER_UNIT, 0),
709 PRIMITIVE_INFO_INIT(FW_LOCK, POWER_PP_LOCK, 31,
710 RAPL_DOMAIN_MSR_LIMIT, ARBITRARY_UNIT, 0),
711 PRIMITIVE_INFO_INIT(PL1_ENABLE, POWER_LIMIT1_ENABLE, 15,
712 RAPL_DOMAIN_MSR_LIMIT, ARBITRARY_UNIT, 0),
713 PRIMITIVE_INFO_INIT(PL1_CLAMP, POWER_LIMIT1_CLAMP, 16,
714 RAPL_DOMAIN_MSR_LIMIT, ARBITRARY_UNIT, 0),
715 PRIMITIVE_INFO_INIT(PL2_ENABLE, POWER_LIMIT2_ENABLE, 47,
716 RAPL_DOMAIN_MSR_LIMIT, ARBITRARY_UNIT, 0),
717 PRIMITIVE_INFO_INIT(PL2_CLAMP, POWER_LIMIT2_CLAMP, 48,
718 RAPL_DOMAIN_MSR_LIMIT, ARBITRARY_UNIT, 0),
719 PRIMITIVE_INFO_INIT(TIME_WINDOW1, TIME_WINDOW1_MASK, 17,
720 RAPL_DOMAIN_MSR_LIMIT, TIME_UNIT, 0),
721 PRIMITIVE_INFO_INIT(TIME_WINDOW2, TIME_WINDOW2_MASK, 49,
722 RAPL_DOMAIN_MSR_LIMIT, TIME_UNIT, 0),
723 PRIMITIVE_INFO_INIT(THERMAL_SPEC_POWER, POWER_INFO_THERMAL_SPEC_MASK,
724 0, RAPL_DOMAIN_MSR_INFO, POWER_UNIT, 0),
725 PRIMITIVE_INFO_INIT(MAX_POWER, POWER_INFO_MAX_MASK, 32,
726 RAPL_DOMAIN_MSR_INFO, POWER_UNIT, 0),
727 PRIMITIVE_INFO_INIT(MIN_POWER, POWER_INFO_MIN_MASK, 16,
728 RAPL_DOMAIN_MSR_INFO, POWER_UNIT, 0),
729 PRIMITIVE_INFO_INIT(MAX_TIME_WINDOW, POWER_INFO_MAX_TIME_WIN_MASK, 48,
730 RAPL_DOMAIN_MSR_INFO, TIME_UNIT, 0),
731 PRIMITIVE_INFO_INIT(THROTTLED_TIME, PERF_STATUS_THROTTLE_TIME_MASK, 0,
732 RAPL_DOMAIN_MSR_PERF, TIME_UNIT, 0),
733 PRIMITIVE_INFO_INIT(PRIORITY_LEVEL, PP_POLICY_MASK, 0,
734 RAPL_DOMAIN_MSR_POLICY, ARBITRARY_UNIT, 0),
735 /* non-hardware */
736 PRIMITIVE_INFO_INIT(AVERAGE_POWER, 0, 0, 0, POWER_UNIT,
737 RAPL_PRIMITIVE_DERIVED),
738 {NULL, 0, 0, 0},
739 };
740
741 /* Read primitive data based on its related struct rapl_primitive_info.
742 * if xlate flag is set, return translated data based on data units, i.e.
743 * time, energy, and power.
744 * RAPL MSRs are non-architectual and are laid out not consistently across
745 * domains. Here we use primitive info to allow writing consolidated access
746 * functions.
747 * For a given primitive, it is processed by MSR mask and shift. Unit conversion
748 * is pre-assigned based on RAPL unit MSRs read at init time.
749 * 63-------------------------- 31--------------------------- 0
750 * | xxxxx (mask) |
751 * | |<- shift ----------------|
752 * 63-------------------------- 31--------------------------- 0
753 */
rapl_read_data_raw(struct rapl_domain * rd,enum rapl_primitives prim,bool xlate,u64 * data)754 static int rapl_read_data_raw(struct rapl_domain *rd,
755 enum rapl_primitives prim,
756 bool xlate, u64 *data)
757 {
758 u64 value, final;
759 u32 msr;
760 struct rapl_primitive_info *rp = &rpi[prim];
761 int cpu;
762
763 if (!rp->name || rp->flag & RAPL_PRIMITIVE_DUMMY)
764 return -EINVAL;
765
766 msr = rd->msrs[rp->id];
767 if (!msr)
768 return -EINVAL;
769 /* use physical package id to look up active cpus */
770 cpu = find_active_cpu_on_package(rd->package_id);
771 if (cpu < 0)
772 return cpu;
773
774 /* special-case package domain, which uses a different bit*/
775 if (prim == FW_LOCK && rd->id == RAPL_DOMAIN_PACKAGE) {
776 rp->mask = POWER_PACKAGE_LOCK;
777 rp->shift = 63;
778 }
779 /* non-hardware data are collected by the polling thread */
780 if (rp->flag & RAPL_PRIMITIVE_DERIVED) {
781 *data = rd->rdd.primitives[prim];
782 return 0;
783 }
784
785 if (rdmsrl_safe_on_cpu(cpu, msr, &value)) {
786 pr_debug("failed to read msr 0x%x on cpu %d\n", msr, cpu);
787 return -EIO;
788 }
789
790 final = value & rp->mask;
791 final = final >> rp->shift;
792 if (xlate)
793 *data = rapl_unit_xlate(rd, rd->package_id, rp->unit, final, 0);
794 else
795 *data = final;
796
797 return 0;
798 }
799
800 /* Similar use of primitive info in the read counterpart */
rapl_write_data_raw(struct rapl_domain * rd,enum rapl_primitives prim,unsigned long long value)801 static int rapl_write_data_raw(struct rapl_domain *rd,
802 enum rapl_primitives prim,
803 unsigned long long value)
804 {
805 u64 msr_val;
806 u32 msr;
807 struct rapl_primitive_info *rp = &rpi[prim];
808 int cpu;
809
810 cpu = find_active_cpu_on_package(rd->package_id);
811 if (cpu < 0)
812 return cpu;
813 msr = rd->msrs[rp->id];
814 if (rdmsrl_safe_on_cpu(cpu, msr, &msr_val)) {
815 dev_dbg(&rd->power_zone.dev,
816 "failed to read msr 0x%x on cpu %d\n", msr, cpu);
817 return -EIO;
818 }
819 value = rapl_unit_xlate(rd, rd->package_id, rp->unit, value, 1);
820 msr_val &= ~rp->mask;
821 msr_val |= value << rp->shift;
822 if (wrmsrl_safe_on_cpu(cpu, msr, msr_val)) {
823 dev_dbg(&rd->power_zone.dev,
824 "failed to write msr 0x%x on cpu %d\n", msr, cpu);
825 return -EIO;
826 }
827
828 return 0;
829 }
830
831 /*
832 * Raw RAPL data stored in MSRs are in certain scales. We need to
833 * convert them into standard units based on the units reported in
834 * the RAPL unit MSRs. This is specific to CPUs as the method to
835 * calculate units differ on different CPUs.
836 * We convert the units to below format based on CPUs.
837 * i.e.
838 * energy unit: picoJoules : Represented in picoJoules by default
839 * power unit : microWatts : Represented in milliWatts by default
840 * time unit : microseconds: Represented in seconds by default
841 */
rapl_check_unit_core(struct rapl_package * rp,int cpu)842 static int rapl_check_unit_core(struct rapl_package *rp, int cpu)
843 {
844 u64 msr_val;
845 u32 value;
846
847 if (rdmsrl_safe_on_cpu(cpu, MSR_RAPL_POWER_UNIT, &msr_val)) {
848 pr_err("Failed to read power unit MSR 0x%x on CPU %d, exit.\n",
849 MSR_RAPL_POWER_UNIT, cpu);
850 return -ENODEV;
851 }
852
853 value = (msr_val & ENERGY_UNIT_MASK) >> ENERGY_UNIT_OFFSET;
854 rp->energy_unit = ENERGY_UNIT_SCALE * 1000000 / (1 << value);
855
856 value = (msr_val & POWER_UNIT_MASK) >> POWER_UNIT_OFFSET;
857 rp->power_unit = 1000000 / (1 << value);
858
859 value = (msr_val & TIME_UNIT_MASK) >> TIME_UNIT_OFFSET;
860 rp->time_unit = 1000000 / (1 << value);
861
862 pr_debug("Core CPU package %d energy=%dpJ, time=%dus, power=%duW\n",
863 rp->id, rp->energy_unit, rp->time_unit, rp->power_unit);
864
865 return 0;
866 }
867
rapl_check_unit_atom(struct rapl_package * rp,int cpu)868 static int rapl_check_unit_atom(struct rapl_package *rp, int cpu)
869 {
870 u64 msr_val;
871 u32 value;
872
873 if (rdmsrl_safe_on_cpu(cpu, MSR_RAPL_POWER_UNIT, &msr_val)) {
874 pr_err("Failed to read power unit MSR 0x%x on CPU %d, exit.\n",
875 MSR_RAPL_POWER_UNIT, cpu);
876 return -ENODEV;
877 }
878 value = (msr_val & ENERGY_UNIT_MASK) >> ENERGY_UNIT_OFFSET;
879 rp->energy_unit = ENERGY_UNIT_SCALE * 1 << value;
880
881 value = (msr_val & POWER_UNIT_MASK) >> POWER_UNIT_OFFSET;
882 rp->power_unit = (1 << value) * 1000;
883
884 value = (msr_val & TIME_UNIT_MASK) >> TIME_UNIT_OFFSET;
885 rp->time_unit = 1000000 / (1 << value);
886
887 pr_debug("Atom package %d energy=%dpJ, time=%dus, power=%duW\n",
888 rp->id, rp->energy_unit, rp->time_unit, rp->power_unit);
889
890 return 0;
891 }
892
893
894 /* REVISIT:
895 * When package power limit is set artificially low by RAPL, LVT
896 * thermal interrupt for package power limit should be ignored
897 * since we are not really exceeding the real limit. The intention
898 * is to avoid excessive interrupts while we are trying to save power.
899 * A useful feature might be routing the package_power_limit interrupt
900 * to userspace via eventfd. once we have a usecase, this is simple
901 * to do by adding an atomic notifier.
902 */
903
package_power_limit_irq_save(int package_id)904 static void package_power_limit_irq_save(int package_id)
905 {
906 u32 l, h = 0;
907 int cpu;
908 struct rapl_package *rp;
909
910 rp = find_package_by_id(package_id);
911 if (!rp)
912 return;
913
914 if (!boot_cpu_has(X86_FEATURE_PTS) || !boot_cpu_has(X86_FEATURE_PLN))
915 return;
916
917 cpu = find_active_cpu_on_package(package_id);
918 if (cpu < 0)
919 return;
920 /* save the state of PLN irq mask bit before disabling it */
921 rdmsr_safe_on_cpu(cpu, MSR_IA32_PACKAGE_THERM_INTERRUPT, &l, &h);
922 if (!(rp->power_limit_irq & PACKAGE_PLN_INT_SAVED)) {
923 rp->power_limit_irq = l & PACKAGE_THERM_INT_PLN_ENABLE;
924 rp->power_limit_irq |= PACKAGE_PLN_INT_SAVED;
925 }
926 l &= ~PACKAGE_THERM_INT_PLN_ENABLE;
927 wrmsr_on_cpu(cpu, MSR_IA32_PACKAGE_THERM_INTERRUPT, l, h);
928 }
929
930 /* restore per package power limit interrupt enable state */
package_power_limit_irq_restore(int package_id)931 static void package_power_limit_irq_restore(int package_id)
932 {
933 u32 l, h;
934 int cpu;
935 struct rapl_package *rp;
936
937 rp = find_package_by_id(package_id);
938 if (!rp)
939 return;
940
941 if (!boot_cpu_has(X86_FEATURE_PTS) || !boot_cpu_has(X86_FEATURE_PLN))
942 return;
943
944 cpu = find_active_cpu_on_package(package_id);
945 if (cpu < 0)
946 return;
947
948 /* irq enable state not saved, nothing to restore */
949 if (!(rp->power_limit_irq & PACKAGE_PLN_INT_SAVED))
950 return;
951 rdmsr_safe_on_cpu(cpu, MSR_IA32_PACKAGE_THERM_INTERRUPT, &l, &h);
952
953 if (rp->power_limit_irq & PACKAGE_THERM_INT_PLN_ENABLE)
954 l |= PACKAGE_THERM_INT_PLN_ENABLE;
955 else
956 l &= ~PACKAGE_THERM_INT_PLN_ENABLE;
957
958 wrmsr_on_cpu(cpu, MSR_IA32_PACKAGE_THERM_INTERRUPT, l, h);
959 }
960
set_floor_freq_default(struct rapl_domain * rd,bool mode)961 static void set_floor_freq_default(struct rapl_domain *rd, bool mode)
962 {
963 int nr_powerlimit = find_nr_power_limit(rd);
964
965 /* always enable clamp such that p-state can go below OS requested
966 * range. power capping priority over guranteed frequency.
967 */
968 rapl_write_data_raw(rd, PL1_CLAMP, mode);
969
970 /* some domains have pl2 */
971 if (nr_powerlimit > 1) {
972 rapl_write_data_raw(rd, PL2_ENABLE, mode);
973 rapl_write_data_raw(rd, PL2_CLAMP, mode);
974 }
975 }
976
set_floor_freq_atom(struct rapl_domain * rd,bool enable)977 static void set_floor_freq_atom(struct rapl_domain *rd, bool enable)
978 {
979 static u32 power_ctrl_orig_val;
980 u32 mdata;
981
982 if (!power_ctrl_orig_val)
983 iosf_mbi_read(BT_MBI_UNIT_PMC, BT_MBI_PMC_READ,
984 IOSF_CPU_POWER_BUDGET_CTL, &power_ctrl_orig_val);
985 mdata = power_ctrl_orig_val;
986 if (enable) {
987 mdata &= ~(0x7f << 8);
988 mdata |= 1 << 8;
989 }
990 iosf_mbi_write(BT_MBI_UNIT_PMC, BT_MBI_PMC_WRITE,
991 IOSF_CPU_POWER_BUDGET_CTL, mdata);
992 }
993
rapl_compute_time_window_core(struct rapl_package * rp,u64 value,bool to_raw)994 static u64 rapl_compute_time_window_core(struct rapl_package *rp, u64 value,
995 bool to_raw)
996 {
997 u64 f, y; /* fraction and exp. used for time unit */
998
999 /*
1000 * Special processing based on 2^Y*(1+F/4), refer
1001 * to Intel Software Developer's manual Vol.3B: CH 14.9.3.
1002 */
1003 if (!to_raw) {
1004 f = (value & 0x60) >> 5;
1005 y = value & 0x1f;
1006 value = (1 << y) * (4 + f) * rp->time_unit / 4;
1007 } else {
1008 do_div(value, rp->time_unit);
1009 y = ilog2(value);
1010 f = div64_u64(4 * (value - (1 << y)), 1 << y);
1011 value = (y & 0x1f) | ((f & 0x3) << 5);
1012 }
1013 return value;
1014 }
1015
rapl_compute_time_window_atom(struct rapl_package * rp,u64 value,bool to_raw)1016 static u64 rapl_compute_time_window_atom(struct rapl_package *rp, u64 value,
1017 bool to_raw)
1018 {
1019 /*
1020 * Atom time unit encoding is straight forward val * time_unit,
1021 * where time_unit is default to 1 sec. Never 0.
1022 */
1023 if (!to_raw)
1024 return (value) ? value *= rp->time_unit : rp->time_unit;
1025 else
1026 value = div64_u64(value, rp->time_unit);
1027
1028 return value;
1029 }
1030
1031 static const struct rapl_defaults rapl_defaults_core = {
1032 .check_unit = rapl_check_unit_core,
1033 .set_floor_freq = set_floor_freq_default,
1034 .compute_time_window = rapl_compute_time_window_core,
1035 };
1036
1037 static const struct rapl_defaults rapl_defaults_hsw_server = {
1038 .check_unit = rapl_check_unit_core,
1039 .set_floor_freq = set_floor_freq_default,
1040 .compute_time_window = rapl_compute_time_window_core,
1041 .dram_domain_energy_unit = 15300,
1042 };
1043
1044 static const struct rapl_defaults rapl_defaults_atom = {
1045 .check_unit = rapl_check_unit_atom,
1046 .set_floor_freq = set_floor_freq_atom,
1047 .compute_time_window = rapl_compute_time_window_atom,
1048 };
1049
1050 #define RAPL_CPU(_model, _ops) { \
1051 .vendor = X86_VENDOR_INTEL, \
1052 .family = 6, \
1053 .model = _model, \
1054 .driver_data = (kernel_ulong_t)&_ops, \
1055 }
1056
1057 static const struct x86_cpu_id rapl_ids[] __initconst = {
1058 RAPL_CPU(0x2a, rapl_defaults_core),/* Sandy Bridge */
1059 RAPL_CPU(0x2d, rapl_defaults_core),/* Sandy Bridge EP */
1060 RAPL_CPU(0x37, rapl_defaults_atom),/* Valleyview */
1061 RAPL_CPU(0x3a, rapl_defaults_core),/* Ivy Bridge */
1062 RAPL_CPU(0x3c, rapl_defaults_core),/* Haswell */
1063 RAPL_CPU(0x3d, rapl_defaults_core),/* Broadwell */
1064 RAPL_CPU(0x3f, rapl_defaults_hsw_server),/* Haswell servers */
1065 RAPL_CPU(0x4f, rapl_defaults_hsw_server),/* Broadwell servers */
1066 RAPL_CPU(0x45, rapl_defaults_core),/* Haswell ULT */
1067 RAPL_CPU(0x4E, rapl_defaults_core),/* Skylake */
1068 RAPL_CPU(0x4C, rapl_defaults_atom),/* Braswell */
1069 RAPL_CPU(0x4A, rapl_defaults_atom),/* Tangier */
1070 RAPL_CPU(0x56, rapl_defaults_core),/* Future Xeon */
1071 RAPL_CPU(0x5A, rapl_defaults_atom),/* Annidale */
1072 {}
1073 };
1074 MODULE_DEVICE_TABLE(x86cpu, rapl_ids);
1075
1076 /* read once for all raw primitive data for all packages, domains */
rapl_update_domain_data(void)1077 static void rapl_update_domain_data(void)
1078 {
1079 int dmn, prim;
1080 u64 val;
1081 struct rapl_package *rp;
1082
1083 list_for_each_entry(rp, &rapl_packages, plist) {
1084 for (dmn = 0; dmn < rp->nr_domains; dmn++) {
1085 pr_debug("update package %d domain %s data\n", rp->id,
1086 rp->domains[dmn].name);
1087 /* exclude non-raw primitives */
1088 for (prim = 0; prim < NR_RAW_PRIMITIVES; prim++)
1089 if (!rapl_read_data_raw(&rp->domains[dmn], prim,
1090 rpi[prim].unit,
1091 &val))
1092 rp->domains[dmn].rdd.primitives[prim] =
1093 val;
1094 }
1095 }
1096
1097 }
1098
rapl_unregister_powercap(void)1099 static int rapl_unregister_powercap(void)
1100 {
1101 struct rapl_package *rp;
1102 struct rapl_domain *rd, *rd_package = NULL;
1103
1104 /* unregister all active rapl packages from the powercap layer,
1105 * hotplug lock held
1106 */
1107 list_for_each_entry(rp, &rapl_packages, plist) {
1108 package_power_limit_irq_restore(rp->id);
1109
1110 for (rd = rp->domains; rd < rp->domains + rp->nr_domains;
1111 rd++) {
1112 pr_debug("remove package, undo power limit on %d: %s\n",
1113 rp->id, rd->name);
1114 rapl_write_data_raw(rd, PL1_ENABLE, 0);
1115 rapl_write_data_raw(rd, PL2_ENABLE, 0);
1116 rapl_write_data_raw(rd, PL1_CLAMP, 0);
1117 rapl_write_data_raw(rd, PL2_CLAMP, 0);
1118 if (rd->id == RAPL_DOMAIN_PACKAGE) {
1119 rd_package = rd;
1120 continue;
1121 }
1122 powercap_unregister_zone(control_type, &rd->power_zone);
1123 }
1124 /* do the package zone last */
1125 if (rd_package)
1126 powercap_unregister_zone(control_type,
1127 &rd_package->power_zone);
1128 }
1129 powercap_unregister_control_type(control_type);
1130
1131 return 0;
1132 }
1133
rapl_package_register_powercap(struct rapl_package * rp)1134 static int rapl_package_register_powercap(struct rapl_package *rp)
1135 {
1136 struct rapl_domain *rd;
1137 int ret = 0;
1138 char dev_name[17]; /* max domain name = 7 + 1 + 8 for int + 1 for null*/
1139 struct powercap_zone *power_zone = NULL;
1140 int nr_pl;
1141
1142 /* first we register package domain as the parent zone*/
1143 for (rd = rp->domains; rd < rp->domains + rp->nr_domains; rd++) {
1144 if (rd->id == RAPL_DOMAIN_PACKAGE) {
1145 nr_pl = find_nr_power_limit(rd);
1146 pr_debug("register socket %d package domain %s\n",
1147 rp->id, rd->name);
1148 memset(dev_name, 0, sizeof(dev_name));
1149 snprintf(dev_name, sizeof(dev_name), "%s-%d",
1150 rd->name, rp->id);
1151 power_zone = powercap_register_zone(&rd->power_zone,
1152 control_type,
1153 dev_name, NULL,
1154 &zone_ops[rd->id],
1155 nr_pl,
1156 &constraint_ops);
1157 if (IS_ERR(power_zone)) {
1158 pr_debug("failed to register package, %d\n",
1159 rp->id);
1160 ret = PTR_ERR(power_zone);
1161 goto exit_package;
1162 }
1163 /* track parent zone in per package/socket data */
1164 rp->power_zone = power_zone;
1165 /* done, only one package domain per socket */
1166 break;
1167 }
1168 }
1169 if (!power_zone) {
1170 pr_err("no package domain found, unknown topology!\n");
1171 ret = -ENODEV;
1172 goto exit_package;
1173 }
1174 /* now register domains as children of the socket/package*/
1175 for (rd = rp->domains; rd < rp->domains + rp->nr_domains; rd++) {
1176 if (rd->id == RAPL_DOMAIN_PACKAGE)
1177 continue;
1178 /* number of power limits per domain varies */
1179 nr_pl = find_nr_power_limit(rd);
1180 power_zone = powercap_register_zone(&rd->power_zone,
1181 control_type, rd->name,
1182 rp->power_zone,
1183 &zone_ops[rd->id], nr_pl,
1184 &constraint_ops);
1185
1186 if (IS_ERR(power_zone)) {
1187 pr_debug("failed to register power_zone, %d:%s:%s\n",
1188 rp->id, rd->name, dev_name);
1189 ret = PTR_ERR(power_zone);
1190 goto err_cleanup;
1191 }
1192 }
1193
1194 exit_package:
1195 return ret;
1196 err_cleanup:
1197 /* clean up previously initialized domains within the package if we
1198 * failed after the first domain setup.
1199 */
1200 while (--rd >= rp->domains) {
1201 pr_debug("unregister package %d domain %s\n", rp->id, rd->name);
1202 powercap_unregister_zone(control_type, &rd->power_zone);
1203 }
1204
1205 return ret;
1206 }
1207
rapl_register_powercap(void)1208 static int rapl_register_powercap(void)
1209 {
1210 struct rapl_domain *rd;
1211 struct rapl_package *rp;
1212 int ret = 0;
1213
1214 control_type = powercap_register_control_type(NULL, "intel-rapl", NULL);
1215 if (IS_ERR(control_type)) {
1216 pr_debug("failed to register powercap control_type.\n");
1217 return PTR_ERR(control_type);
1218 }
1219 /* read the initial data */
1220 rapl_update_domain_data();
1221 list_for_each_entry(rp, &rapl_packages, plist)
1222 if (rapl_package_register_powercap(rp))
1223 goto err_cleanup_package;
1224 return ret;
1225
1226 err_cleanup_package:
1227 /* clean up previously initialized packages */
1228 list_for_each_entry_continue_reverse(rp, &rapl_packages, plist) {
1229 for (rd = rp->domains; rd < rp->domains + rp->nr_domains;
1230 rd++) {
1231 pr_debug("unregister zone/package %d, %s domain\n",
1232 rp->id, rd->name);
1233 powercap_unregister_zone(control_type, &rd->power_zone);
1234 }
1235 }
1236
1237 return ret;
1238 }
1239
rapl_check_domain(int cpu,int domain)1240 static int rapl_check_domain(int cpu, int domain)
1241 {
1242 unsigned msr;
1243 u64 val = 0;
1244
1245 switch (domain) {
1246 case RAPL_DOMAIN_PACKAGE:
1247 msr = MSR_PKG_ENERGY_STATUS;
1248 break;
1249 case RAPL_DOMAIN_PP0:
1250 msr = MSR_PP0_ENERGY_STATUS;
1251 break;
1252 case RAPL_DOMAIN_PP1:
1253 msr = MSR_PP1_ENERGY_STATUS;
1254 break;
1255 case RAPL_DOMAIN_DRAM:
1256 msr = MSR_DRAM_ENERGY_STATUS;
1257 break;
1258 default:
1259 pr_err("invalid domain id %d\n", domain);
1260 return -EINVAL;
1261 }
1262 /* make sure domain counters are available and contains non-zero
1263 * values, otherwise skip it.
1264 */
1265 if (rdmsrl_safe_on_cpu(cpu, msr, &val) || !val)
1266 return -ENODEV;
1267
1268 return 0;
1269 }
1270
1271 /* Detect active and valid domains for the given CPU, caller must
1272 * ensure the CPU belongs to the targeted package and CPU hotlug is disabled.
1273 */
rapl_detect_domains(struct rapl_package * rp,int cpu)1274 static int rapl_detect_domains(struct rapl_package *rp, int cpu)
1275 {
1276 int i;
1277 int ret = 0;
1278 struct rapl_domain *rd;
1279 u64 locked;
1280
1281 for (i = 0; i < RAPL_DOMAIN_MAX; i++) {
1282 /* use physical package id to read counters */
1283 if (!rapl_check_domain(cpu, i)) {
1284 rp->domain_map |= 1 << i;
1285 pr_info("Found RAPL domain %s\n", rapl_domain_names[i]);
1286 }
1287 }
1288 rp->nr_domains = bitmap_weight(&rp->domain_map, RAPL_DOMAIN_MAX);
1289 if (!rp->nr_domains) {
1290 pr_err("no valid rapl domains found in package %d\n", rp->id);
1291 ret = -ENODEV;
1292 goto done;
1293 }
1294 pr_debug("found %d domains on package %d\n", rp->nr_domains, rp->id);
1295
1296 rp->domains = kcalloc(rp->nr_domains + 1, sizeof(struct rapl_domain),
1297 GFP_KERNEL);
1298 if (!rp->domains) {
1299 ret = -ENOMEM;
1300 goto done;
1301 }
1302 rapl_init_domains(rp);
1303
1304 for (rd = rp->domains; rd < rp->domains + rp->nr_domains; rd++) {
1305 /* check if the domain is locked by BIOS */
1306 if (rapl_read_data_raw(rd, FW_LOCK, false, &locked)) {
1307 pr_info("RAPL package %d domain %s locked by BIOS\n",
1308 rp->id, rd->name);
1309 rd->state |= DOMAIN_STATE_BIOS_LOCKED;
1310 }
1311 }
1312
1313
1314 done:
1315 return ret;
1316 }
1317
is_package_new(int package)1318 static bool is_package_new(int package)
1319 {
1320 struct rapl_package *rp;
1321
1322 /* caller prevents cpu hotplug, there will be no new packages added
1323 * or deleted while traversing the package list, no need for locking.
1324 */
1325 list_for_each_entry(rp, &rapl_packages, plist)
1326 if (package == rp->id)
1327 return false;
1328
1329 return true;
1330 }
1331
1332 /* RAPL interface can be made of a two-level hierarchy: package level and domain
1333 * level. We first detect the number of packages then domains of each package.
1334 * We have to consider the possiblity of CPU online/offline due to hotplug and
1335 * other scenarios.
1336 */
rapl_detect_topology(void)1337 static int rapl_detect_topology(void)
1338 {
1339 int i;
1340 int phy_package_id;
1341 struct rapl_package *new_package, *rp;
1342
1343 for_each_online_cpu(i) {
1344 phy_package_id = topology_physical_package_id(i);
1345 if (is_package_new(phy_package_id)) {
1346 new_package = kzalloc(sizeof(*rp), GFP_KERNEL);
1347 if (!new_package) {
1348 rapl_cleanup_data();
1349 return -ENOMEM;
1350 }
1351 /* add the new package to the list */
1352 new_package->id = phy_package_id;
1353 new_package->nr_cpus = 1;
1354
1355 /* check if the package contains valid domains */
1356 if (rapl_detect_domains(new_package, i) ||
1357 rapl_defaults->check_unit(new_package, i)) {
1358 kfree(new_package->domains);
1359 kfree(new_package);
1360 /* free up the packages already initialized */
1361 rapl_cleanup_data();
1362 return -ENODEV;
1363 }
1364 INIT_LIST_HEAD(&new_package->plist);
1365 list_add(&new_package->plist, &rapl_packages);
1366 } else {
1367 rp = find_package_by_id(phy_package_id);
1368 if (rp)
1369 ++rp->nr_cpus;
1370 }
1371 }
1372
1373 return 0;
1374 }
1375
1376 /* called from CPU hotplug notifier, hotplug lock held */
rapl_remove_package(struct rapl_package * rp)1377 static void rapl_remove_package(struct rapl_package *rp)
1378 {
1379 struct rapl_domain *rd, *rd_package = NULL;
1380
1381 for (rd = rp->domains; rd < rp->domains + rp->nr_domains; rd++) {
1382 if (rd->id == RAPL_DOMAIN_PACKAGE) {
1383 rd_package = rd;
1384 continue;
1385 }
1386 pr_debug("remove package %d, %s domain\n", rp->id, rd->name);
1387 powercap_unregister_zone(control_type, &rd->power_zone);
1388 }
1389 /* do parent zone last */
1390 powercap_unregister_zone(control_type, &rd_package->power_zone);
1391 list_del(&rp->plist);
1392 kfree(rp);
1393 }
1394
1395 /* called from CPU hotplug notifier, hotplug lock held */
rapl_add_package(int cpu)1396 static int rapl_add_package(int cpu)
1397 {
1398 int ret = 0;
1399 int phy_package_id;
1400 struct rapl_package *rp;
1401
1402 phy_package_id = topology_physical_package_id(cpu);
1403 rp = kzalloc(sizeof(struct rapl_package), GFP_KERNEL);
1404 if (!rp)
1405 return -ENOMEM;
1406
1407 /* add the new package to the list */
1408 rp->id = phy_package_id;
1409 rp->nr_cpus = 1;
1410 /* check if the package contains valid domains */
1411 if (rapl_detect_domains(rp, cpu) ||
1412 rapl_defaults->check_unit(rp, cpu)) {
1413 ret = -ENODEV;
1414 goto err_free_package;
1415 }
1416 if (!rapl_package_register_powercap(rp)) {
1417 INIT_LIST_HEAD(&rp->plist);
1418 list_add(&rp->plist, &rapl_packages);
1419 return ret;
1420 }
1421
1422 err_free_package:
1423 kfree(rp->domains);
1424 kfree(rp);
1425
1426 return ret;
1427 }
1428
1429 /* Handles CPU hotplug on multi-socket systems.
1430 * If a CPU goes online as the first CPU of the physical package
1431 * we add the RAPL package to the system. Similarly, when the last
1432 * CPU of the package is removed, we remove the RAPL package and its
1433 * associated domains. Cooling devices are handled accordingly at
1434 * per-domain level.
1435 */
rapl_cpu_callback(struct notifier_block * nfb,unsigned long action,void * hcpu)1436 static int rapl_cpu_callback(struct notifier_block *nfb,
1437 unsigned long action, void *hcpu)
1438 {
1439 unsigned long cpu = (unsigned long)hcpu;
1440 int phy_package_id;
1441 struct rapl_package *rp;
1442
1443 phy_package_id = topology_physical_package_id(cpu);
1444 switch (action) {
1445 case CPU_ONLINE:
1446 case CPU_ONLINE_FROZEN:
1447 case CPU_DOWN_FAILED:
1448 case CPU_DOWN_FAILED_FROZEN:
1449 rp = find_package_by_id(phy_package_id);
1450 if (rp)
1451 ++rp->nr_cpus;
1452 else
1453 rapl_add_package(cpu);
1454 break;
1455 case CPU_DOWN_PREPARE:
1456 case CPU_DOWN_PREPARE_FROZEN:
1457 rp = find_package_by_id(phy_package_id);
1458 if (!rp)
1459 break;
1460 if (--rp->nr_cpus == 0)
1461 rapl_remove_package(rp);
1462 }
1463
1464 return NOTIFY_OK;
1465 }
1466
1467 static struct notifier_block rapl_cpu_notifier = {
1468 .notifier_call = rapl_cpu_callback,
1469 };
1470
rapl_init(void)1471 static int __init rapl_init(void)
1472 {
1473 int ret = 0;
1474 const struct x86_cpu_id *id;
1475
1476 id = x86_match_cpu(rapl_ids);
1477 if (!id) {
1478 pr_err("driver does not support CPU family %d model %d\n",
1479 boot_cpu_data.x86, boot_cpu_data.x86_model);
1480
1481 return -ENODEV;
1482 }
1483
1484 rapl_defaults = (struct rapl_defaults *)id->driver_data;
1485
1486 cpu_notifier_register_begin();
1487
1488 /* prevent CPU hotplug during detection */
1489 get_online_cpus();
1490 ret = rapl_detect_topology();
1491 if (ret)
1492 goto done;
1493
1494 if (rapl_register_powercap()) {
1495 rapl_cleanup_data();
1496 ret = -ENODEV;
1497 goto done;
1498 }
1499 __register_hotcpu_notifier(&rapl_cpu_notifier);
1500 done:
1501 put_online_cpus();
1502 cpu_notifier_register_done();
1503
1504 return ret;
1505 }
1506
rapl_exit(void)1507 static void __exit rapl_exit(void)
1508 {
1509 cpu_notifier_register_begin();
1510 get_online_cpus();
1511 __unregister_hotcpu_notifier(&rapl_cpu_notifier);
1512 rapl_unregister_powercap();
1513 rapl_cleanup_data();
1514 put_online_cpus();
1515 cpu_notifier_register_done();
1516 }
1517
1518 module_init(rapl_init);
1519 module_exit(rapl_exit);
1520
1521 MODULE_DESCRIPTION("Driver for Intel RAPL (Running Average Power Limit)");
1522 MODULE_AUTHOR("Jacob Pan <jacob.jun.pan@intel.com>");
1523 MODULE_LICENSE("GPL v2");
1524