1/*
2 * This program is free software; you can redistribute it and/or modify
3 * it under the terms of the GNU General Public License version 2 as
4 * published by the Free Software Foundation.
5 *
6 * This program is distributed in the hope that it will be useful,
7 * but WITHOUT ANY WARRANTY; without even the implied warranty of
8 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
9 * GNU General Public License for more details.
10 *
11 * You should have received a copy of the GNU General Public License
12 * along with this program; if not, write to the Free Software
13 * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
14 *
15 * Copyright (C) 2012 ARM Limited
16 *
17 * Author: Will Deacon <will.deacon@arm.com>
18 */
19#define pr_fmt(fmt) "CPU PMU: " fmt
20
21#include <linux/bitmap.h>
22#include <linux/export.h>
23#include <linux/kernel.h>
24#include <linux/of.h>
25#include <linux/platform_device.h>
26#include <linux/slab.h>
27#include <linux/spinlock.h>
28#include <linux/irq.h>
29#include <linux/irqdesc.h>
30
31#include <asm/cputype.h>
32#include <asm/irq_regs.h>
33#include <asm/pmu.h>
34
35/* Set at runtime when we know what CPU type we are. */
36static struct arm_pmu *cpu_pmu;
37
38/*
39 * Despite the names, these two functions are CPU-specific and are used
40 * by the OProfile/perf code.
41 */
42const char *perf_pmu_name(void)
43{
44	if (!cpu_pmu)
45		return NULL;
46
47	return cpu_pmu->name;
48}
49EXPORT_SYMBOL_GPL(perf_pmu_name);
50
51int perf_num_counters(void)
52{
53	int max_events = 0;
54
55	if (cpu_pmu != NULL)
56		max_events = cpu_pmu->num_events;
57
58	return max_events;
59}
60EXPORT_SYMBOL_GPL(perf_num_counters);
61
62/* Include the PMU-specific implementations. */
63#include "perf_event_xscale.c"
64#include "perf_event_v6.c"
65#include "perf_event_v7.c"
66
67static void cpu_pmu_enable_percpu_irq(void *data)
68{
69	int irq = *(int *)data;
70
71	enable_percpu_irq(irq, IRQ_TYPE_NONE);
72}
73
74static void cpu_pmu_disable_percpu_irq(void *data)
75{
76	int irq = *(int *)data;
77
78	disable_percpu_irq(irq);
79}
80
81static void cpu_pmu_free_irq(struct arm_pmu *cpu_pmu)
82{
83	int i, irq, irqs;
84	struct platform_device *pmu_device = cpu_pmu->plat_device;
85	struct pmu_hw_events __percpu *hw_events = cpu_pmu->hw_events;
86
87	irqs = min(pmu_device->num_resources, num_possible_cpus());
88
89	irq = platform_get_irq(pmu_device, 0);
90	if (irq >= 0 && irq_is_percpu(irq)) {
91		on_each_cpu(cpu_pmu_disable_percpu_irq, &irq, 1);
92		free_percpu_irq(irq, &hw_events->percpu_pmu);
93	} else {
94		for (i = 0; i < irqs; ++i) {
95			int cpu = i;
96
97			if (cpu_pmu->irq_affinity)
98				cpu = cpu_pmu->irq_affinity[i];
99
100			if (!cpumask_test_and_clear_cpu(cpu, &cpu_pmu->active_irqs))
101				continue;
102			irq = platform_get_irq(pmu_device, i);
103			if (irq >= 0)
104				free_irq(irq, per_cpu_ptr(&hw_events->percpu_pmu, cpu));
105		}
106	}
107}
108
109static int cpu_pmu_request_irq(struct arm_pmu *cpu_pmu, irq_handler_t handler)
110{
111	int i, err, irq, irqs;
112	struct platform_device *pmu_device = cpu_pmu->plat_device;
113	struct pmu_hw_events __percpu *hw_events = cpu_pmu->hw_events;
114
115	if (!pmu_device)
116		return -ENODEV;
117
118	irqs = min(pmu_device->num_resources, num_possible_cpus());
119	if (irqs < 1) {
120		pr_warn_once("perf/ARM: No irqs for PMU defined, sampling events not supported\n");
121		return 0;
122	}
123
124	irq = platform_get_irq(pmu_device, 0);
125	if (irq >= 0 && irq_is_percpu(irq)) {
126		err = request_percpu_irq(irq, handler, "arm-pmu",
127					 &hw_events->percpu_pmu);
128		if (err) {
129			pr_err("unable to request IRQ%d for ARM PMU counters\n",
130				irq);
131			return err;
132		}
133		on_each_cpu(cpu_pmu_enable_percpu_irq, &irq, 1);
134	} else {
135		for (i = 0; i < irqs; ++i) {
136			int cpu = i;
137
138			err = 0;
139			irq = platform_get_irq(pmu_device, i);
140			if (irq < 0)
141				continue;
142
143			if (cpu_pmu->irq_affinity)
144				cpu = cpu_pmu->irq_affinity[i];
145
146			/*
147			 * If we have a single PMU interrupt that we can't shift,
148			 * assume that we're running on a uniprocessor machine and
149			 * continue. Otherwise, continue without this interrupt.
150			 */
151			if (irq_set_affinity(irq, cpumask_of(cpu)) && irqs > 1) {
152				pr_warn("unable to set irq affinity (irq=%d, cpu=%u)\n",
153					irq, cpu);
154				continue;
155			}
156
157			err = request_irq(irq, handler,
158					  IRQF_NOBALANCING | IRQF_NO_THREAD, "arm-pmu",
159					  per_cpu_ptr(&hw_events->percpu_pmu, cpu));
160			if (err) {
161				pr_err("unable to request IRQ%d for ARM PMU counters\n",
162					irq);
163				return err;
164			}
165
166			cpumask_set_cpu(cpu, &cpu_pmu->active_irqs);
167		}
168	}
169
170	return 0;
171}
172
173/*
174 * PMU hardware loses all context when a CPU goes offline.
175 * When a CPU is hotplugged back in, since some hardware registers are
176 * UNKNOWN at reset, the PMU must be explicitly reset to avoid reading
177 * junk values out of them.
178 */
179static int cpu_pmu_notify(struct notifier_block *b, unsigned long action,
180			  void *hcpu)
181{
182	struct arm_pmu *pmu = container_of(b, struct arm_pmu, hotplug_nb);
183
184	if ((action & ~CPU_TASKS_FROZEN) != CPU_STARTING)
185		return NOTIFY_DONE;
186
187	if (pmu->reset)
188		pmu->reset(pmu);
189	else
190		return NOTIFY_DONE;
191
192	return NOTIFY_OK;
193}
194
195static int cpu_pmu_init(struct arm_pmu *cpu_pmu)
196{
197	int err;
198	int cpu;
199	struct pmu_hw_events __percpu *cpu_hw_events;
200
201	cpu_hw_events = alloc_percpu(struct pmu_hw_events);
202	if (!cpu_hw_events)
203		return -ENOMEM;
204
205	cpu_pmu->hotplug_nb.notifier_call = cpu_pmu_notify;
206	err = register_cpu_notifier(&cpu_pmu->hotplug_nb);
207	if (err)
208		goto out_hw_events;
209
210	for_each_possible_cpu(cpu) {
211		struct pmu_hw_events *events = per_cpu_ptr(cpu_hw_events, cpu);
212		raw_spin_lock_init(&events->pmu_lock);
213		events->percpu_pmu = cpu_pmu;
214	}
215
216	cpu_pmu->hw_events	= cpu_hw_events;
217	cpu_pmu->request_irq	= cpu_pmu_request_irq;
218	cpu_pmu->free_irq	= cpu_pmu_free_irq;
219
220	/* Ensure the PMU has sane values out of reset. */
221	if (cpu_pmu->reset)
222		on_each_cpu(cpu_pmu->reset, cpu_pmu, 1);
223
224	/* If no interrupts available, set the corresponding capability flag */
225	if (!platform_get_irq(cpu_pmu->plat_device, 0))
226		cpu_pmu->pmu.capabilities |= PERF_PMU_CAP_NO_INTERRUPT;
227
228	return 0;
229
230out_hw_events:
231	free_percpu(cpu_hw_events);
232	return err;
233}
234
235static void cpu_pmu_destroy(struct arm_pmu *cpu_pmu)
236{
237	unregister_cpu_notifier(&cpu_pmu->hotplug_nb);
238	free_percpu(cpu_pmu->hw_events);
239}
240
241/*
242 * PMU platform driver and devicetree bindings.
243 */
244static const struct of_device_id cpu_pmu_of_device_ids[] = {
245	{.compatible = "arm,cortex-a17-pmu",	.data = armv7_a17_pmu_init},
246	{.compatible = "arm,cortex-a15-pmu",	.data = armv7_a15_pmu_init},
247	{.compatible = "arm,cortex-a12-pmu",	.data = armv7_a12_pmu_init},
248	{.compatible = "arm,cortex-a9-pmu",	.data = armv7_a9_pmu_init},
249	{.compatible = "arm,cortex-a8-pmu",	.data = armv7_a8_pmu_init},
250	{.compatible = "arm,cortex-a7-pmu",	.data = armv7_a7_pmu_init},
251	{.compatible = "arm,cortex-a5-pmu",	.data = armv7_a5_pmu_init},
252	{.compatible = "arm,arm11mpcore-pmu",	.data = armv6mpcore_pmu_init},
253	{.compatible = "arm,arm1176-pmu",	.data = armv6_1176_pmu_init},
254	{.compatible = "arm,arm1136-pmu",	.data = armv6_1136_pmu_init},
255	{.compatible = "qcom,krait-pmu",	.data = krait_pmu_init},
256	{.compatible = "qcom,scorpion-pmu",	.data = scorpion_pmu_init},
257	{.compatible = "qcom,scorpion-mp-pmu",	.data = scorpion_mp_pmu_init},
258	{},
259};
260
261static struct platform_device_id cpu_pmu_plat_device_ids[] = {
262	{.name = "arm-pmu"},
263	{.name = "armv6-pmu"},
264	{.name = "armv7-pmu"},
265	{.name = "xscale-pmu"},
266	{},
267};
268
269static const struct pmu_probe_info pmu_probe_table[] = {
270	ARM_PMU_PROBE(ARM_CPU_PART_ARM1136, armv6_1136_pmu_init),
271	ARM_PMU_PROBE(ARM_CPU_PART_ARM1156, armv6_1156_pmu_init),
272	ARM_PMU_PROBE(ARM_CPU_PART_ARM1176, armv6_1176_pmu_init),
273	ARM_PMU_PROBE(ARM_CPU_PART_ARM11MPCORE, armv6mpcore_pmu_init),
274	ARM_PMU_PROBE(ARM_CPU_PART_CORTEX_A8, armv7_a8_pmu_init),
275	ARM_PMU_PROBE(ARM_CPU_PART_CORTEX_A9, armv7_a9_pmu_init),
276	XSCALE_PMU_PROBE(ARM_CPU_XSCALE_ARCH_V1, xscale1pmu_init),
277	XSCALE_PMU_PROBE(ARM_CPU_XSCALE_ARCH_V2, xscale2pmu_init),
278	{ /* sentinel value */ }
279};
280
281/*
282 * CPU PMU identification and probing.
283 */
284static int probe_current_pmu(struct arm_pmu *pmu)
285{
286	int cpu = get_cpu();
287	unsigned int cpuid = read_cpuid_id();
288	int ret = -ENODEV;
289	const struct pmu_probe_info *info;
290
291	pr_info("probing PMU on CPU %d\n", cpu);
292
293	for (info = pmu_probe_table; info->init != NULL; info++) {
294		if ((cpuid & info->mask) != info->cpuid)
295			continue;
296		ret = info->init(pmu);
297		break;
298	}
299
300	put_cpu();
301	return ret;
302}
303
304static int of_pmu_irq_cfg(struct platform_device *pdev)
305{
306	int i, irq;
307	int *irqs;
308
309	/* Don't bother with PPIs; they're already affine */
310	irq = platform_get_irq(pdev, 0);
311	if (irq >= 0 && irq_is_percpu(irq))
312		return 0;
313
314	irqs = kcalloc(pdev->num_resources, sizeof(*irqs), GFP_KERNEL);
315	if (!irqs)
316		return -ENOMEM;
317
318	for (i = 0; i < pdev->num_resources; ++i) {
319		struct device_node *dn;
320		int cpu;
321
322		dn = of_parse_phandle(pdev->dev.of_node, "interrupt-affinity",
323				      i);
324		if (!dn) {
325			pr_warn("Failed to parse %s/interrupt-affinity[%d]\n",
326				of_node_full_name(pdev->dev.of_node), i);
327			break;
328		}
329
330		for_each_possible_cpu(cpu)
331			if (arch_find_n_match_cpu_physical_id(dn, cpu, NULL))
332				break;
333
334		of_node_put(dn);
335		if (cpu >= nr_cpu_ids) {
336			pr_warn("Failed to find logical CPU for %s\n",
337				dn->name);
338			break;
339		}
340
341		irqs[i] = cpu;
342	}
343
344	if (i == pdev->num_resources)
345		cpu_pmu->irq_affinity = irqs;
346	else
347		kfree(irqs);
348
349	return 0;
350}
351
352static int cpu_pmu_device_probe(struct platform_device *pdev)
353{
354	const struct of_device_id *of_id;
355	const int (*init_fn)(struct arm_pmu *);
356	struct device_node *node = pdev->dev.of_node;
357	struct arm_pmu *pmu;
358	int ret = -ENODEV;
359
360	if (cpu_pmu) {
361		pr_info("attempt to register multiple PMU devices!\n");
362		return -ENOSPC;
363	}
364
365	pmu = kzalloc(sizeof(struct arm_pmu), GFP_KERNEL);
366	if (!pmu) {
367		pr_info("failed to allocate PMU device!\n");
368		return -ENOMEM;
369	}
370
371	cpu_pmu = pmu;
372	cpu_pmu->plat_device = pdev;
373
374	if (node && (of_id = of_match_node(cpu_pmu_of_device_ids, pdev->dev.of_node))) {
375		init_fn = of_id->data;
376
377		ret = of_pmu_irq_cfg(pdev);
378		if (!ret)
379			ret = init_fn(pmu);
380	} else {
381		ret = probe_current_pmu(pmu);
382	}
383
384	if (ret) {
385		pr_info("failed to probe PMU!\n");
386		goto out_free;
387	}
388
389	ret = cpu_pmu_init(cpu_pmu);
390	if (ret)
391		goto out_free;
392
393	ret = armpmu_register(cpu_pmu, -1);
394	if (ret)
395		goto out_destroy;
396
397	return 0;
398
399out_destroy:
400	cpu_pmu_destroy(cpu_pmu);
401out_free:
402	pr_info("failed to register PMU devices!\n");
403	kfree(pmu);
404	return ret;
405}
406
407static struct platform_driver cpu_pmu_driver = {
408	.driver		= {
409		.name	= "arm-pmu",
410		.pm	= &armpmu_dev_pm_ops,
411		.of_match_table = cpu_pmu_of_device_ids,
412	},
413	.probe		= cpu_pmu_device_probe,
414	.id_table	= cpu_pmu_plat_device_ids,
415};
416
417static int __init register_pmu_driver(void)
418{
419	return platform_driver_register(&cpu_pmu_driver);
420}
421device_initcall(register_pmu_driver);
422