1/*
2 * @file op_model_amd.c
3 * athlon / K7 / K8 / Family 10h model-specific MSR operations
4 *
5 * @remark Copyright 2002-2009 OProfile authors
6 * @remark Read the file COPYING
7 *
8 * @author John Levon
9 * @author Philippe Elie
10 * @author Graydon Hoare
11 * @author Robert Richter <robert.richter@amd.com>
12 * @author Barry Kasindorf <barry.kasindorf@amd.com>
13 * @author Jason Yeh <jason.yeh@amd.com>
14 * @author Suravee Suthikulpanit <suravee.suthikulpanit@amd.com>
15 */
16
17#include <linux/oprofile.h>
18#include <linux/device.h>
19#include <linux/pci.h>
20#include <linux/percpu.h>
21
22#include <asm/ptrace.h>
23#include <asm/msr.h>
24#include <asm/nmi.h>
25#include <asm/apic.h>
26#include <asm/processor.h>
27#include <asm/cpufeature.h>
28
29#include "op_x86_model.h"
30#include "op_counter.h"
31
32#ifdef CONFIG_OPROFILE_EVENT_MULTIPLEX
33#define NUM_VIRT_COUNTERS	32
34#else
35#define NUM_VIRT_COUNTERS	0
36#endif
37
38#define OP_EVENT_MASK			0x0FFF
39#define OP_CTR_OVERFLOW			(1ULL<<31)
40
41#define MSR_AMD_EVENTSEL_RESERVED	((0xFFFFFCF0ULL<<32)|(1ULL<<21))
42
43static int num_counters;
44static unsigned long reset_value[OP_MAX_COUNTER];
45
46#define IBS_FETCH_SIZE			6
47#define IBS_OP_SIZE			12
48
49static u32 ibs_caps;
50
51struct ibs_config {
52	unsigned long op_enabled;
53	unsigned long fetch_enabled;
54	unsigned long max_cnt_fetch;
55	unsigned long max_cnt_op;
56	unsigned long rand_en;
57	unsigned long dispatched_ops;
58	unsigned long branch_target;
59};
60
61struct ibs_state {
62	u64		ibs_op_ctl;
63	int		branch_target;
64	unsigned long	sample_size;
65};
66
67static struct ibs_config ibs_config;
68static struct ibs_state ibs_state;
69
70/*
71 * IBS randomization macros
72 */
73#define IBS_RANDOM_BITS			12
74#define IBS_RANDOM_MASK			((1ULL << IBS_RANDOM_BITS) - 1)
75#define IBS_RANDOM_MAXCNT_OFFSET	(1ULL << (IBS_RANDOM_BITS - 5))
76
77/*
78 * 16-bit Linear Feedback Shift Register (LFSR)
79 *
80 *                       16   14   13    11
81 * Feedback polynomial = X  + X  + X  +  X  + 1
82 */
83static unsigned int lfsr_random(void)
84{
85	static unsigned int lfsr_value = 0xF00D;
86	unsigned int bit;
87
88	/* Compute next bit to shift in */
89	bit = ((lfsr_value >> 0) ^
90	       (lfsr_value >> 2) ^
91	       (lfsr_value >> 3) ^
92	       (lfsr_value >> 5)) & 0x0001;
93
94	/* Advance to next register value */
95	lfsr_value = (lfsr_value >> 1) | (bit << 15);
96
97	return lfsr_value;
98}
99
100/*
101 * IBS software randomization
102 *
103 * The IBS periodic op counter is randomized in software. The lower 12
104 * bits of the 20 bit counter are randomized. IbsOpCurCnt is
105 * initialized with a 12 bit random value.
106 */
107static inline u64 op_amd_randomize_ibs_op(u64 val)
108{
109	unsigned int random = lfsr_random();
110
111	if (!(ibs_caps & IBS_CAPS_RDWROPCNT))
112		/*
113		 * Work around if the hw can not write to IbsOpCurCnt
114		 *
115		 * Randomize the lower 8 bits of the 16 bit
116		 * IbsOpMaxCnt [15:0] value in the range of -128 to
117		 * +127 by adding/subtracting an offset to the
118		 * maximum count (IbsOpMaxCnt).
119		 *
120		 * To avoid over or underflows and protect upper bits
121		 * starting at bit 16, the initial value for
122		 * IbsOpMaxCnt must fit in the range from 0x0081 to
123		 * 0xff80.
124		 */
125		val += (s8)(random >> 4);
126	else
127		val |= (u64)(random & IBS_RANDOM_MASK) << 32;
128
129	return val;
130}
131
132static inline void
133op_amd_handle_ibs(struct pt_regs * const regs,
134		  struct op_msrs const * const msrs)
135{
136	u64 val, ctl;
137	struct op_entry entry;
138
139	if (!ibs_caps)
140		return;
141
142	if (ibs_config.fetch_enabled) {
143		rdmsrl(MSR_AMD64_IBSFETCHCTL, ctl);
144		if (ctl & IBS_FETCH_VAL) {
145			rdmsrl(MSR_AMD64_IBSFETCHLINAD, val);
146			oprofile_write_reserve(&entry, regs, val,
147					       IBS_FETCH_CODE, IBS_FETCH_SIZE);
148			oprofile_add_data64(&entry, val);
149			oprofile_add_data64(&entry, ctl);
150			rdmsrl(MSR_AMD64_IBSFETCHPHYSAD, val);
151			oprofile_add_data64(&entry, val);
152			oprofile_write_commit(&entry);
153
154			/* reenable the IRQ */
155			ctl &= ~(IBS_FETCH_VAL | IBS_FETCH_CNT);
156			ctl |= IBS_FETCH_ENABLE;
157			wrmsrl(MSR_AMD64_IBSFETCHCTL, ctl);
158		}
159	}
160
161	if (ibs_config.op_enabled) {
162		rdmsrl(MSR_AMD64_IBSOPCTL, ctl);
163		if (ctl & IBS_OP_VAL) {
164			rdmsrl(MSR_AMD64_IBSOPRIP, val);
165			oprofile_write_reserve(&entry, regs, val, IBS_OP_CODE,
166					       ibs_state.sample_size);
167			oprofile_add_data64(&entry, val);
168			rdmsrl(MSR_AMD64_IBSOPDATA, val);
169			oprofile_add_data64(&entry, val);
170			rdmsrl(MSR_AMD64_IBSOPDATA2, val);
171			oprofile_add_data64(&entry, val);
172			rdmsrl(MSR_AMD64_IBSOPDATA3, val);
173			oprofile_add_data64(&entry, val);
174			rdmsrl(MSR_AMD64_IBSDCLINAD, val);
175			oprofile_add_data64(&entry, val);
176			rdmsrl(MSR_AMD64_IBSDCPHYSAD, val);
177			oprofile_add_data64(&entry, val);
178			if (ibs_state.branch_target) {
179				rdmsrl(MSR_AMD64_IBSBRTARGET, val);
180				oprofile_add_data(&entry, (unsigned long)val);
181			}
182			oprofile_write_commit(&entry);
183
184			/* reenable the IRQ */
185			ctl = op_amd_randomize_ibs_op(ibs_state.ibs_op_ctl);
186			wrmsrl(MSR_AMD64_IBSOPCTL, ctl);
187		}
188	}
189}
190
191static inline void op_amd_start_ibs(void)
192{
193	u64 val;
194
195	if (!ibs_caps)
196		return;
197
198	memset(&ibs_state, 0, sizeof(ibs_state));
199
200	/*
201	 * Note: Since the max count settings may out of range we
202	 * write back the actual used values so that userland can read
203	 * it.
204	 */
205
206	if (ibs_config.fetch_enabled) {
207		val = ibs_config.max_cnt_fetch >> 4;
208		val = min(val, IBS_FETCH_MAX_CNT);
209		ibs_config.max_cnt_fetch = val << 4;
210		val |= ibs_config.rand_en ? IBS_FETCH_RAND_EN : 0;
211		val |= IBS_FETCH_ENABLE;
212		wrmsrl(MSR_AMD64_IBSFETCHCTL, val);
213	}
214
215	if (ibs_config.op_enabled) {
216		val = ibs_config.max_cnt_op >> 4;
217		if (!(ibs_caps & IBS_CAPS_RDWROPCNT)) {
218			/*
219			 * IbsOpCurCnt not supported.  See
220			 * op_amd_randomize_ibs_op() for details.
221			 */
222			val = clamp(val, 0x0081ULL, 0xFF80ULL);
223			ibs_config.max_cnt_op = val << 4;
224		} else {
225			/*
226			 * The start value is randomized with a
227			 * positive offset, we need to compensate it
228			 * with the half of the randomized range. Also
229			 * avoid underflows.
230			 */
231			val += IBS_RANDOM_MAXCNT_OFFSET;
232			if (ibs_caps & IBS_CAPS_OPCNTEXT)
233				val = min(val, IBS_OP_MAX_CNT_EXT);
234			else
235				val = min(val, IBS_OP_MAX_CNT);
236			ibs_config.max_cnt_op =
237				(val - IBS_RANDOM_MAXCNT_OFFSET) << 4;
238		}
239		val = ((val & ~IBS_OP_MAX_CNT) << 4) | (val & IBS_OP_MAX_CNT);
240		val |= ibs_config.dispatched_ops ? IBS_OP_CNT_CTL : 0;
241		val |= IBS_OP_ENABLE;
242		ibs_state.ibs_op_ctl = val;
243		ibs_state.sample_size = IBS_OP_SIZE;
244		if (ibs_config.branch_target) {
245			ibs_state.branch_target = 1;
246			ibs_state.sample_size++;
247		}
248		val = op_amd_randomize_ibs_op(ibs_state.ibs_op_ctl);
249		wrmsrl(MSR_AMD64_IBSOPCTL, val);
250	}
251}
252
253static void op_amd_stop_ibs(void)
254{
255	if (!ibs_caps)
256		return;
257
258	if (ibs_config.fetch_enabled)
259		/* clear max count and enable */
260		wrmsrl(MSR_AMD64_IBSFETCHCTL, 0);
261
262	if (ibs_config.op_enabled)
263		/* clear max count and enable */
264		wrmsrl(MSR_AMD64_IBSOPCTL, 0);
265}
266
267#ifdef CONFIG_OPROFILE_EVENT_MULTIPLEX
268
269static void op_mux_switch_ctrl(struct op_x86_model_spec const *model,
270			       struct op_msrs const * const msrs)
271{
272	u64 val;
273	int i;
274
275	/* enable active counters */
276	for (i = 0; i < num_counters; ++i) {
277		int virt = op_x86_phys_to_virt(i);
278		if (!reset_value[virt])
279			continue;
280		rdmsrl(msrs->controls[i].addr, val);
281		val &= model->reserved;
282		val |= op_x86_get_ctrl(model, &counter_config[virt]);
283		wrmsrl(msrs->controls[i].addr, val);
284	}
285}
286
287#endif
288
289/* functions for op_amd_spec */
290
291static void op_amd_shutdown(struct op_msrs const * const msrs)
292{
293	int i;
294
295	for (i = 0; i < num_counters; ++i) {
296		if (!msrs->counters[i].addr)
297			continue;
298		release_perfctr_nmi(MSR_K7_PERFCTR0 + i);
299		release_evntsel_nmi(MSR_K7_EVNTSEL0 + i);
300	}
301}
302
303static int op_amd_fill_in_addresses(struct op_msrs * const msrs)
304{
305	int i;
306
307	for (i = 0; i < num_counters; i++) {
308		if (!reserve_perfctr_nmi(MSR_K7_PERFCTR0 + i))
309			goto fail;
310		if (!reserve_evntsel_nmi(MSR_K7_EVNTSEL0 + i)) {
311			release_perfctr_nmi(MSR_K7_PERFCTR0 + i);
312			goto fail;
313		}
314		/* both registers must be reserved */
315		if (num_counters == AMD64_NUM_COUNTERS_CORE) {
316			msrs->counters[i].addr = MSR_F15H_PERF_CTR + (i << 1);
317			msrs->controls[i].addr = MSR_F15H_PERF_CTL + (i << 1);
318		} else {
319			msrs->controls[i].addr = MSR_K7_EVNTSEL0 + i;
320			msrs->counters[i].addr = MSR_K7_PERFCTR0 + i;
321		}
322		continue;
323	fail:
324		if (!counter_config[i].enabled)
325			continue;
326		op_x86_warn_reserved(i);
327		op_amd_shutdown(msrs);
328		return -EBUSY;
329	}
330
331	return 0;
332}
333
334static void op_amd_setup_ctrs(struct op_x86_model_spec const *model,
335			      struct op_msrs const * const msrs)
336{
337	u64 val;
338	int i;
339
340	/* setup reset_value */
341	for (i = 0; i < OP_MAX_COUNTER; ++i) {
342		if (counter_config[i].enabled
343		    && msrs->counters[op_x86_virt_to_phys(i)].addr)
344			reset_value[i] = counter_config[i].count;
345		else
346			reset_value[i] = 0;
347	}
348
349	/* clear all counters */
350	for (i = 0; i < num_counters; ++i) {
351		if (!msrs->controls[i].addr)
352			continue;
353		rdmsrl(msrs->controls[i].addr, val);
354		if (val & ARCH_PERFMON_EVENTSEL_ENABLE)
355			op_x86_warn_in_use(i);
356		val &= model->reserved;
357		wrmsrl(msrs->controls[i].addr, val);
358		/*
359		 * avoid a false detection of ctr overflows in NMI
360		 * handler
361		 */
362		wrmsrl(msrs->counters[i].addr, -1LL);
363	}
364
365	/* enable active counters */
366	for (i = 0; i < num_counters; ++i) {
367		int virt = op_x86_phys_to_virt(i);
368		if (!reset_value[virt])
369			continue;
370
371		/* setup counter registers */
372		wrmsrl(msrs->counters[i].addr, -(u64)reset_value[virt]);
373
374		/* setup control registers */
375		rdmsrl(msrs->controls[i].addr, val);
376		val &= model->reserved;
377		val |= op_x86_get_ctrl(model, &counter_config[virt]);
378		wrmsrl(msrs->controls[i].addr, val);
379	}
380}
381
382static int op_amd_check_ctrs(struct pt_regs * const regs,
383			     struct op_msrs const * const msrs)
384{
385	u64 val;
386	int i;
387
388	for (i = 0; i < num_counters; ++i) {
389		int virt = op_x86_phys_to_virt(i);
390		if (!reset_value[virt])
391			continue;
392		rdmsrl(msrs->counters[i].addr, val);
393		/* bit is clear if overflowed: */
394		if (val & OP_CTR_OVERFLOW)
395			continue;
396		oprofile_add_sample(regs, virt);
397		wrmsrl(msrs->counters[i].addr, -(u64)reset_value[virt]);
398	}
399
400	op_amd_handle_ibs(regs, msrs);
401
402	/* See op_model_ppro.c */
403	return 1;
404}
405
406static void op_amd_start(struct op_msrs const * const msrs)
407{
408	u64 val;
409	int i;
410
411	for (i = 0; i < num_counters; ++i) {
412		if (!reset_value[op_x86_phys_to_virt(i)])
413			continue;
414		rdmsrl(msrs->controls[i].addr, val);
415		val |= ARCH_PERFMON_EVENTSEL_ENABLE;
416		wrmsrl(msrs->controls[i].addr, val);
417	}
418
419	op_amd_start_ibs();
420}
421
422static void op_amd_stop(struct op_msrs const * const msrs)
423{
424	u64 val;
425	int i;
426
427	/*
428	 * Subtle: stop on all counters to avoid race with setting our
429	 * pm callback
430	 */
431	for (i = 0; i < num_counters; ++i) {
432		if (!reset_value[op_x86_phys_to_virt(i)])
433			continue;
434		rdmsrl(msrs->controls[i].addr, val);
435		val &= ~ARCH_PERFMON_EVENTSEL_ENABLE;
436		wrmsrl(msrs->controls[i].addr, val);
437	}
438
439	op_amd_stop_ibs();
440}
441
442/*
443 * check and reserve APIC extended interrupt LVT offset for IBS if
444 * available
445 */
446
447static void init_ibs(void)
448{
449	ibs_caps = get_ibs_caps();
450
451	if (!ibs_caps)
452		return;
453
454	printk(KERN_INFO "oprofile: AMD IBS detected (0x%08x)\n", ibs_caps);
455}
456
457static int (*create_arch_files)(struct dentry *root);
458
459static int setup_ibs_files(struct dentry *root)
460{
461	struct dentry *dir;
462	int ret = 0;
463
464	/* architecture specific files */
465	if (create_arch_files)
466		ret = create_arch_files(root);
467
468	if (ret)
469		return ret;
470
471	if (!ibs_caps)
472		return ret;
473
474	/* model specific files */
475
476	/* setup some reasonable defaults */
477	memset(&ibs_config, 0, sizeof(ibs_config));
478	ibs_config.max_cnt_fetch = 250000;
479	ibs_config.max_cnt_op = 250000;
480
481	if (ibs_caps & IBS_CAPS_FETCHSAM) {
482		dir = oprofilefs_mkdir(root, "ibs_fetch");
483		oprofilefs_create_ulong(dir, "enable",
484					&ibs_config.fetch_enabled);
485		oprofilefs_create_ulong(dir, "max_count",
486					&ibs_config.max_cnt_fetch);
487		oprofilefs_create_ulong(dir, "rand_enable",
488					&ibs_config.rand_en);
489	}
490
491	if (ibs_caps & IBS_CAPS_OPSAM) {
492		dir = oprofilefs_mkdir(root, "ibs_op");
493		oprofilefs_create_ulong(dir, "enable",
494					&ibs_config.op_enabled);
495		oprofilefs_create_ulong(dir, "max_count",
496					&ibs_config.max_cnt_op);
497		if (ibs_caps & IBS_CAPS_OPCNT)
498			oprofilefs_create_ulong(dir, "dispatched_ops",
499						&ibs_config.dispatched_ops);
500		if (ibs_caps & IBS_CAPS_BRNTRGT)
501			oprofilefs_create_ulong(dir, "branch_target",
502						&ibs_config.branch_target);
503	}
504
505	return 0;
506}
507
508struct op_x86_model_spec op_amd_spec;
509
510static int op_amd_init(struct oprofile_operations *ops)
511{
512	init_ibs();
513	create_arch_files = ops->create_files;
514	ops->create_files = setup_ibs_files;
515
516	if (boot_cpu_data.x86 == 0x15) {
517		num_counters = AMD64_NUM_COUNTERS_CORE;
518	} else {
519		num_counters = AMD64_NUM_COUNTERS;
520	}
521
522	op_amd_spec.num_counters = num_counters;
523	op_amd_spec.num_controls = num_counters;
524	op_amd_spec.num_virt_counters = max(num_counters, NUM_VIRT_COUNTERS);
525
526	return 0;
527}
528
529struct op_x86_model_spec op_amd_spec = {
530	/* num_counters/num_controls filled in at runtime */
531	.reserved		= MSR_AMD_EVENTSEL_RESERVED,
532	.event_mask		= OP_EVENT_MASK,
533	.init			= op_amd_init,
534	.fill_in_addresses	= &op_amd_fill_in_addresses,
535	.setup_ctrs		= &op_amd_setup_ctrs,
536	.check_ctrs		= &op_amd_check_ctrs,
537	.start			= &op_amd_start,
538	.stop			= &op_amd_stop,
539	.shutdown		= &op_amd_shutdown,
540#ifdef CONFIG_OPROFILE_EVENT_MULTIPLEX
541	.switch_ctrl		= &op_mux_switch_ctrl,
542#endif
543};
544