242 #define thm_readb(off) readb(ips->regmap + (off))
243 #define thm_readw(off) readw(ips->regmap + (off))
244 #define thm_readl(off) readl(ips->regmap + (off))
245 #define thm_readq(off) readq(ips->regmap + (off))
247 #define thm_writeb(off, val) writeb((val), ips->regmap + (off))
248 #define thm_writew(off, val) writew((val), ips->regmap + (off))
249 #define thm_writel(off, val) writel((val), ips->regmap + (off))
348 ips_gpu_turbo_enabled(struct ips_driver *ips);
359 static bool ips_cpu_busy(struct ips_driver *ips)  in ips_cpu_busy()  argument
377 static void ips_cpu_raise(struct ips_driver *ips)  in ips_cpu_raise()  argument
382 	if (!ips->cpu_turbo_enabled)  in ips_cpu_raise()
391 	if (((new_tdp_limit * 10) / 8) > ips->core_power_limit)  in ips_cpu_raise()
415 static void ips_cpu_lower(struct ips_driver *ips)  in ips_cpu_lower()  argument
426 	if (new_limit  < (ips->orig_turbo_limit & TURBO_TDP_MASK))  in ips_cpu_lower()
427 		new_limit = ips->orig_turbo_limit & TURBO_TDP_MASK;  in ips_cpu_lower()
466 static void ips_enable_cpu_turbo(struct ips_driver *ips)  in ips_enable_cpu_turbo()  argument
469 	if (ips->__cpu_turbo_on)  in ips_enable_cpu_turbo()
472 	if (ips->turbo_toggle_allowed)  in ips_enable_cpu_turbo()
473 		on_each_cpu(do_enable_cpu_turbo, ips, 1);  in ips_enable_cpu_turbo()
475 	ips->__cpu_turbo_on = true;  in ips_enable_cpu_turbo()
504 static void ips_disable_cpu_turbo(struct ips_driver *ips)  in ips_disable_cpu_turbo()  argument
507 	if (!ips->__cpu_turbo_on)  in ips_disable_cpu_turbo()
510 	if (ips->turbo_toggle_allowed)  in ips_disable_cpu_turbo()
511 		on_each_cpu(do_disable_cpu_turbo, ips, 1);  in ips_disable_cpu_turbo()
513 	ips->__cpu_turbo_on = false;  in ips_disable_cpu_turbo()
526 static bool ips_gpu_busy(struct ips_driver *ips)  in ips_gpu_busy()  argument
528 	if (!ips_gpu_turbo_enabled(ips))  in ips_gpu_busy()
531 	return ips->gpu_busy();  in ips_gpu_busy()
541 static void ips_gpu_raise(struct ips_driver *ips)  in ips_gpu_raise()  argument
543 	if (!ips_gpu_turbo_enabled(ips))  in ips_gpu_raise()
546 	if (!ips->gpu_raise())  in ips_gpu_raise()
547 		ips->gpu_turbo_enabled = false;  in ips_gpu_raise()
558 static void ips_gpu_lower(struct ips_driver *ips)  in ips_gpu_lower()  argument
560 	if (!ips_gpu_turbo_enabled(ips))  in ips_gpu_lower()
563 	if (!ips->gpu_lower())  in ips_gpu_lower()
564 		ips->gpu_turbo_enabled = false;  in ips_gpu_lower()
576 static void ips_enable_gpu_turbo(struct ips_driver *ips)  in ips_enable_gpu_turbo()  argument
578 	if (ips->__gpu_turbo_on)  in ips_enable_gpu_turbo()
580 	ips->__gpu_turbo_on = true;  in ips_enable_gpu_turbo()
589 static void ips_disable_gpu_turbo(struct ips_driver *ips)  in ips_disable_gpu_turbo()  argument
592 	if (!ips->__gpu_turbo_on)  in ips_disable_gpu_turbo()
595 	if (!ips->gpu_turbo_disable())  in ips_disable_gpu_turbo()
596 		dev_err(&ips->dev->dev, "failed to disable graphics turbo\n");  in ips_disable_gpu_turbo()
598 		ips->__gpu_turbo_on = false;  in ips_disable_gpu_turbo()
607 static bool mcp_exceeded(struct ips_driver *ips)  in mcp_exceeded()  argument
614 	spin_lock_irqsave(&ips->turbo_status_lock, flags);  in mcp_exceeded()
616 	temp_limit = ips->mcp_temp_limit * 100;  in mcp_exceeded()
617 	if (ips->mcp_avg_temp > temp_limit)  in mcp_exceeded()
620 	avg_power = ips->cpu_avg_power + ips->mch_avg_power;  in mcp_exceeded()
621 	if (avg_power > ips->mcp_power_limit)  in mcp_exceeded()
624 	spin_unlock_irqrestore(&ips->turbo_status_lock, flags);  in mcp_exceeded()
636 static bool cpu_exceeded(struct ips_driver *ips, int cpu)  in cpu_exceeded()  argument
642 	spin_lock_irqsave(&ips->turbo_status_lock, flags);  in cpu_exceeded()
643 	avg = cpu ? ips->ctv2_avg_temp : ips->ctv1_avg_temp;  in cpu_exceeded()
644 	if (avg > (ips->limits->core_temp_limit * 100))  in cpu_exceeded()
646 	if (ips->cpu_avg_power > ips->core_power_limit * 100)  in cpu_exceeded()
648 	spin_unlock_irqrestore(&ips->turbo_status_lock, flags);  in cpu_exceeded()
651 		dev_info(&ips->dev->dev,  in cpu_exceeded()
663 static bool mch_exceeded(struct ips_driver *ips)  in mch_exceeded()  argument
668 	spin_lock_irqsave(&ips->turbo_status_lock, flags);  in mch_exceeded()
669 	if (ips->mch_avg_temp > (ips->limits->mch_temp_limit * 100))  in mch_exceeded()
671 	if (ips->mch_avg_power > ips->mch_power_limit)  in mch_exceeded()
673 	spin_unlock_irqrestore(&ips->turbo_status_lock, flags);  in mch_exceeded()
686 static void verify_limits(struct ips_driver *ips)  in verify_limits()  argument
688 	if (ips->mcp_power_limit < ips->limits->mcp_power_limit ||  in verify_limits()
689 	    ips->mcp_power_limit > 35000)  in verify_limits()
690 		ips->mcp_power_limit = ips->limits->mcp_power_limit;  in verify_limits()
692 	if (ips->mcp_temp_limit < ips->limits->core_temp_limit ||  in verify_limits()
693 	    ips->mcp_temp_limit < ips->limits->mch_temp_limit ||  in verify_limits()
694 	    ips->mcp_temp_limit > 150)  in verify_limits()
695 		ips->mcp_temp_limit = min(ips->limits->core_temp_limit,  in verify_limits()
696 					  ips->limits->mch_temp_limit);  in verify_limits()
712 static void update_turbo_limits(struct ips_driver *ips)  in update_turbo_limits()  argument
716 	ips->cpu_turbo_enabled = !(hts & HTS_PCTD_DIS);  in update_turbo_limits()
721 	ips->cpu_turbo_enabled = false;  in update_turbo_limits()
723 	if (ips->gpu_busy)  in update_turbo_limits()
724 		ips->gpu_turbo_enabled = !(hts & HTS_GTD_DIS);  in update_turbo_limits()
726 	ips->core_power_limit = thm_readw(THM_MPCPC);  in update_turbo_limits()
727 	ips->mch_power_limit = thm_readw(THM_MMGPC);  in update_turbo_limits()
728 	ips->mcp_temp_limit = thm_readw(THM_PTL);  in update_turbo_limits()
729 	ips->mcp_power_limit = thm_readw(THM_MPPC);  in update_turbo_limits()
731 	verify_limits(ips);  in update_turbo_limits()
768 	struct ips_driver *ips = data;  in ips_adjust()  local
771 	dev_dbg(&ips->dev->dev, "starting ips-adjust thread\n");  in ips_adjust()
778 		bool cpu_busy = ips_cpu_busy(ips);  in ips_adjust()
779 		bool gpu_busy = ips_gpu_busy(ips);  in ips_adjust()
781 		spin_lock_irqsave(&ips->turbo_status_lock, flags);  in ips_adjust()
782 		if (ips->poll_turbo_status)  in ips_adjust()
783 			update_turbo_limits(ips);  in ips_adjust()
784 		spin_unlock_irqrestore(&ips->turbo_status_lock, flags);  in ips_adjust()
787 		if (ips->cpu_turbo_enabled)  in ips_adjust()
788 			ips_enable_cpu_turbo(ips);  in ips_adjust()
790 			ips_disable_cpu_turbo(ips);  in ips_adjust()
792 		if (ips->gpu_turbo_enabled)  in ips_adjust()
793 			ips_enable_gpu_turbo(ips);  in ips_adjust()
795 			ips_disable_gpu_turbo(ips);  in ips_adjust()
798 		if (mcp_exceeded(ips)) {  in ips_adjust()
799 			ips_cpu_lower(ips);  in ips_adjust()
800 			ips_gpu_lower(ips);  in ips_adjust()
804 		if (!cpu_exceeded(ips, 0) && cpu_busy)  in ips_adjust()
805 			ips_cpu_raise(ips);  in ips_adjust()
807 			ips_cpu_lower(ips);  in ips_adjust()
809 		if (!mch_exceeded(ips) && gpu_busy)  in ips_adjust()
810 			ips_gpu_raise(ips);  in ips_adjust()
812 			ips_gpu_lower(ips);  in ips_adjust()
818 	dev_dbg(&ips->dev->dev, "ips-adjust thread stopped\n");  in ips_adjust()
828 static u16 calc_avg_temp(struct ips_driver *ips, u16 *array)  in calc_avg_temp()  argument
844 static u16 read_mgtv(struct ips_driver *ips)  in read_mgtv()  argument
862 static u16 read_ptv(struct ips_driver *ips)  in read_ptv()  argument
866 	slope = (ips->pta_val & PTA_SLOPE_MASK) >> PTA_SLOPE_SHIFT;  in read_ptv()
867 	offset = ips->pta_val & PTA_OFFSET_MASK;  in read_ptv()
874 static u16 read_ctv(struct ips_driver *ips, int cpu)  in read_ctv()  argument
888 static u32 get_cpu_power(struct ips_driver *ips, u32 *last, int period)  in get_cpu_power()  argument
929 static u32 calc_avg_power(struct ips_driver *ips, u32 *array)  in calc_avg_power()  argument
962 	struct ips_driver *ips = data;  in ips_monitor()  local
978 		dev_err(&ips->dev->dev,  in ips_monitor()
1001 		mcp_samples[i] = read_ptv(ips);  in ips_monitor()
1003 		val = read_ctv(ips, 0);  in ips_monitor()
1006 		val = read_ctv(ips, 1);  in ips_monitor()
1009 		val = read_mgtv(ips);  in ips_monitor()
1012 		cpu_power = get_cpu_power(ips, &old_cpu_power,  in ips_monitor()
1016 		if (ips->read_mch_val) {  in ips_monitor()
1017 			mchp = ips->read_mch_val();  in ips_monitor()
1026 	ips->mcp_avg_temp = calc_avg_temp(ips, mcp_samples);  in ips_monitor()
1027 	ips->ctv1_avg_temp = calc_avg_temp(ips, ctv1_samples);  in ips_monitor()
1028 	ips->ctv2_avg_temp = calc_avg_temp(ips, ctv2_samples);  in ips_monitor()
1029 	ips->mch_avg_temp = calc_avg_temp(ips, mch_samples);  in ips_monitor()
1030 	ips->cpu_avg_power = calc_avg_power(ips, cpu_samples);  in ips_monitor()
1031 	ips->mch_avg_power = calc_avg_power(ips, mchp_samples);  in ips_monitor()
1040 	wake_up_process(ips->adjust);  in ips_monitor()
1058 		val = read_ptv(ips);  in ips_monitor()
1059 		ips->mcp_avg_temp = update_average_temp(ips->mcp_avg_temp, val);  in ips_monitor()
1062 		val = read_ctv(ips, 0);  in ips_monitor()
1063 		ips->ctv1_avg_temp =  in ips_monitor()
1064 			update_average_temp(ips->ctv1_avg_temp, val);  in ips_monitor()
1066 		cpu_val = get_cpu_power(ips, &old_cpu_power,  in ips_monitor()
1068 		ips->cpu_avg_power =  in ips_monitor()
1069 			update_average_power(ips->cpu_avg_power, cpu_val);  in ips_monitor()
1071 		if (ips->second_cpu) {  in ips_monitor()
1073 			val = read_ctv(ips, 1);  in ips_monitor()
1074 			ips->ctv2_avg_temp =  in ips_monitor()
1075 				update_average_temp(ips->ctv2_avg_temp, val);  in ips_monitor()
1079 		val = read_mgtv(ips);  in ips_monitor()
1080 		ips->mch_avg_temp = update_average_temp(ips->mch_avg_temp, val);  in ips_monitor()
1082 		if (ips->read_mch_val) {  in ips_monitor()
1083 			mch_val = ips->read_mch_val();  in ips_monitor()
1084 			ips->mch_avg_power =  in ips_monitor()
1085 				update_average_power(ips->mch_avg_power,  in ips_monitor()
1099 			dev_warn(&ips->dev->dev, "ME failed to update for more than 1s, likely hung\n");  in ips_monitor()
1121 	dev_dbg(&ips->dev->dev, "ips-monitor thread stopped\n");  in ips_monitor()
1130 	dev_dbg(&ips->dev->dev, #reg ": 0x%04x\n", val); \
1135 	dev_dbg(&ips->dev->dev, #reg ": 0x%08x\n", val); \
1140 	dev_dbg(&ips->dev->dev, #reg ": 0x%016x\n", val); \
1143 static void dump_thermal_info(struct ips_driver *ips)
1148 	dev_dbg(&ips->dev->dev, "Processor temp limit: %d\n", ptl);
1170 	struct ips_driver *ips = arg;  in ips_irq_handler()  local
1177 	dev_info(&ips->dev->dev, "TSES: 0x%02x\n", tses);  in ips_irq_handler()
1178 	dev_info(&ips->dev->dev, "TES: 0x%02x\n", tes);  in ips_irq_handler()
1188 			spin_lock(&ips->turbo_status_lock);  in ips_irq_handler()
1189 			ips->core_power_limit = (sts & STS_PCPL_MASK) >>  in ips_irq_handler()
1191 			ips->mch_power_limit = (sts & STS_GPL_MASK) >>  in ips_irq_handler()
1194 			ips->cpu_turbo_enabled = !(sts & STS_PCTD_DIS);  in ips_irq_handler()
1199 			ips->cpu_turbo_enabled = false;  in ips_irq_handler()
1200 			if (ips->gpu_busy)  in ips_irq_handler()
1201 				ips->gpu_turbo_enabled = !(sts & STS_GTD_DIS);  in ips_irq_handler()
1202 			ips->mcp_temp_limit = (sts & STS_PTL_MASK) >>  in ips_irq_handler()
1204 			ips->mcp_power_limit = (tc1 & STS_PPL_MASK) >>  in ips_irq_handler()
1206 			verify_limits(ips);  in ips_irq_handler()
1207 			spin_unlock(&ips->turbo_status_lock);  in ips_irq_handler()
1216 		dev_warn(&ips->dev->dev,  in ips_irq_handler()
1225 static void ips_debugfs_init(struct ips_driver *ips) { return; }  in ips_debugfs_init()  argument
1226 static void ips_debugfs_cleanup(struct ips_driver *ips) { return; }  in ips_debugfs_cleanup()  argument
1232 	struct ips_driver *ips;  member
1239 	struct ips_driver *ips = m->private;  in show_cpu_temp()  local
1241 	seq_printf(m, "%d.%02d\n", ips->ctv1_avg_temp / 100,  in show_cpu_temp()
1242 		   ips->ctv1_avg_temp % 100);  in show_cpu_temp()
1249 	struct ips_driver *ips = m->private;  in show_cpu_power()  local
1251 	seq_printf(m, "%dmW\n", ips->cpu_avg_power);  in show_cpu_power()
1279 	struct ips_driver *ips = m->private;  in show_mch_temp()  local
1281 	seq_printf(m, "%d.%02d\n", ips->mch_avg_temp / 100,  in show_mch_temp()
1282 		   ips->mch_avg_temp % 100);  in show_mch_temp()
1289 	struct ips_driver *ips = m->private;  in show_mch_power()  local
1291 	seq_printf(m, "%dmW\n", ips->mch_avg_power);  in show_mch_power()
1308 	return single_open(file, node->show, node->ips);  in ips_debugfs_open()
1319 static void ips_debugfs_cleanup(struct ips_driver *ips)  in ips_debugfs_cleanup()  argument
1321 	if (ips->debug_root)  in ips_debugfs_cleanup()
1322 		debugfs_remove_recursive(ips->debug_root);  in ips_debugfs_cleanup()
1326 static void ips_debugfs_init(struct ips_driver *ips)  in ips_debugfs_init()  argument
1330 	ips->debug_root = debugfs_create_dir("ips", NULL);  in ips_debugfs_init()
1331 	if (!ips->debug_root) {  in ips_debugfs_init()
1332 		dev_err(&ips->dev->dev,  in ips_debugfs_init()
1334 			PTR_ERR(ips->debug_root));  in ips_debugfs_init()
1342 		node->ips = ips;  in ips_debugfs_init()
1344 					  ips->debug_root, node,  in ips_debugfs_init()
1347 			dev_err(&ips->dev->dev,  in ips_debugfs_init()
1357 	ips_debugfs_cleanup(ips);  in ips_debugfs_init()
1368 static struct ips_mcp_limits *ips_detect_cpu(struct ips_driver *ips)  in ips_detect_cpu()  argument
1375 		dev_info(&ips->dev->dev, "Non-IPS CPU detected.\n");  in ips_detect_cpu()
1386 		ips->turbo_toggle_allowed = true;  in ips_detect_cpu()
1388 		ips->turbo_toggle_allowed = false;  in ips_detect_cpu()
1397 		dev_info(&ips->dev->dev, "No CPUID match found.\n");  in ips_detect_cpu()
1406 		dev_info(&ips->dev->dev, "CPU TDP doesn't match expected value (found %d, expected %d)\n",  in ips_detect_cpu()
1424 static bool ips_get_i915_syms(struct ips_driver *ips)  in ips_get_i915_syms()  argument
1426 	ips->read_mch_val = symbol_get(i915_read_mch_val);  in ips_get_i915_syms()
1427 	if (!ips->read_mch_val)  in ips_get_i915_syms()
1429 	ips->gpu_raise = symbol_get(i915_gpu_raise);  in ips_get_i915_syms()
1430 	if (!ips->gpu_raise)  in ips_get_i915_syms()
1432 	ips->gpu_lower = symbol_get(i915_gpu_lower);  in ips_get_i915_syms()
1433 	if (!ips->gpu_lower)  in ips_get_i915_syms()
1435 	ips->gpu_busy = symbol_get(i915_gpu_busy);  in ips_get_i915_syms()
1436 	if (!ips->gpu_busy)  in ips_get_i915_syms()
1438 	ips->gpu_turbo_disable = symbol_get(i915_gpu_turbo_disable);  in ips_get_i915_syms()
1439 	if (!ips->gpu_turbo_disable)  in ips_get_i915_syms()
1457 ips_gpu_turbo_enabled(struct ips_driver *ips)  in ips_gpu_turbo_enabled()  argument
1459 	if (!ips->gpu_busy && late_i915_load) {  in ips_gpu_turbo_enabled()
1460 		if (ips_get_i915_syms(ips)) {  in ips_gpu_turbo_enabled()
1461 			dev_info(&ips->dev->dev,  in ips_gpu_turbo_enabled()
1463 			ips->gpu_turbo_enabled = !(thm_readl(THM_HTS) & HTS_GTD_DIS);  in ips_gpu_turbo_enabled()
1467 	return ips->gpu_turbo_enabled;  in ips_gpu_turbo_enabled()
1510 	struct ips_driver *ips;  in ips_probe()  local
1519 	ips = kzalloc(sizeof(struct ips_driver), GFP_KERNEL);  in ips_probe()
1520 	if (!ips)  in ips_probe()
1523 	pci_set_drvdata(dev, ips);  in ips_probe()
1524 	ips->dev = dev;  in ips_probe()
1526 	ips->limits = ips_detect_cpu(ips);  in ips_probe()
1527 	if (!ips->limits) {  in ips_probe()
1533 	spin_lock_init(&ips->turbo_status_lock);  in ips_probe()
1554 	ips->regmap = ioremap(pci_resource_start(dev, 0),  in ips_probe()
1556 	if (!ips->regmap) {  in ips_probe()
1578 		ips->second_cpu = true;  in ips_probe()
1580 	update_turbo_limits(ips);  in ips_probe()
1582 		ips->mcp_power_limit / 10);  in ips_probe()
1584 		ips->core_power_limit / 10);  in ips_probe()
1587 		ips->poll_turbo_status = true;  in ips_probe()
1589 	if (!ips_get_i915_syms(ips)) {  in ips_probe()
1591 		ips->gpu_turbo_enabled = false;  in ips_probe()
1594 		ips->gpu_turbo_enabled = true;  in ips_probe()
1614 			  ips);  in ips_probe()
1626 	ips->cta_val = thm_readw(THM_CTA);  in ips_probe()
1627 	ips->pta_val = thm_readw(THM_PTA);  in ips_probe()
1628 	ips->mgta_val = thm_readw(THM_MGTA);  in ips_probe()
1631 	rdmsrl(TURBO_POWER_CURRENT_LIMIT, ips->orig_turbo_limit);  in ips_probe()
1633 	ips_disable_cpu_turbo(ips);  in ips_probe()
1634 	ips->cpu_turbo_enabled = false;  in ips_probe()
1637 	ips->adjust = kthread_create(ips_adjust, ips, "ips-adjust");  in ips_probe()
1638 	if (IS_ERR(ips->adjust)) {  in ips_probe()
1650 	ips->monitor = kthread_run(ips_monitor, ips, "ips-monitor");  in ips_probe()
1651 	if (IS_ERR(ips->monitor)) {  in ips_probe()
1658 	hts = (ips->core_power_limit << HTS_PCPL_SHIFT) |  in ips_probe()
1659 		(ips->mcp_temp_limit << HTS_PTL_SHIFT) | HTS_NVV;  in ips_probe()
1665 	ips_debugfs_init(ips);  in ips_probe()
1668 		 ips->mcp_temp_limit);  in ips_probe()
1672 	kthread_stop(ips->adjust);  in ips_probe()
1674 	free_irq(ips->dev->irq, ips);  in ips_probe()
1676 	iounmap(ips->regmap);  in ips_probe()
1680 	kfree(ips);  in ips_probe()
1686 	struct ips_driver *ips = pci_get_drvdata(dev);  in ips_remove()  local
1689 	if (!ips)  in ips_remove()
1692 	ips_debugfs_cleanup(ips);  in ips_remove()
1695 	if (ips->read_mch_val)  in ips_remove()
1697 	if (ips->gpu_raise)  in ips_remove()
1699 	if (ips->gpu_lower)  in ips_remove()
1701 	if (ips->gpu_busy)  in ips_remove()
1703 	if (ips->gpu_turbo_disable)  in ips_remove()
1709 	wrmsrl(TURBO_POWER_CURRENT_LIMIT, ips->orig_turbo_limit);  in ips_remove()
1711 	free_irq(ips->dev->irq, ips);  in ips_remove()
1712 	if (ips->adjust)  in ips_remove()
1713 		kthread_stop(ips->adjust);  in ips_remove()
1714 	if (ips->monitor)  in ips_remove()
1715 		kthread_stop(ips->monitor);  in ips_remove()
1716 	iounmap(ips->regmap);  in ips_remove()
1718 	kfree(ips);  in ips_remove()