1/*
2 * Copyright 2014, Michael Ellerman, IBM Corp.
3 * Licensed under GPLv2.
4 */
5
6#define _GNU_SOURCE	/* For CPU_ZERO etc. */
7
8#include <sched.h>
9#include <sys/wait.h>
10#include <setjmp.h>
11#include <signal.h>
12#include <stdio.h>
13#include <stdlib.h>
14#include <string.h>
15#include <sys/ioctl.h>
16#include <linux/auxvec.h>
17
18#include "trace.h"
19#include "reg.h"
20#include "ebb.h"
21
22
23void (*ebb_user_func)(void);
24
25void ebb_hook(void)
26{
27	if (ebb_user_func)
28		ebb_user_func();
29}
30
31struct ebb_state ebb_state;
32
33u64 sample_period = 0x40000000ull;
34
35void reset_ebb_with_clear_mask(unsigned long mmcr0_clear_mask)
36{
37	u64 val;
38
39	/* 2) clear MMCR0[PMAO] - docs say BESCR[PMEO] should do this */
40	/* 3) set MMCR0[PMAE]	- docs say BESCR[PME] should do this */
41	val = mfspr(SPRN_MMCR0);
42	mtspr(SPRN_MMCR0, (val & ~mmcr0_clear_mask) | MMCR0_PMAE);
43
44	/* 4) clear BESCR[PMEO] */
45	mtspr(SPRN_BESCRR, BESCR_PMEO);
46
47	/* 5) set BESCR[PME] */
48	mtspr(SPRN_BESCRS, BESCR_PME);
49
50	/* 6) rfebb 1 - done in our caller */
51}
52
53void reset_ebb(void)
54{
55	reset_ebb_with_clear_mask(MMCR0_PMAO | MMCR0_FC);
56}
57
58/* Called outside of the EBB handler to check MMCR0 is sane */
59int ebb_check_mmcr0(void)
60{
61	u64 val;
62
63	val = mfspr(SPRN_MMCR0);
64	if ((val & (MMCR0_FC | MMCR0_PMAO)) == MMCR0_FC) {
65		/* It's OK if we see FC & PMAO, but not FC by itself */
66		printf("Outside of loop, only FC set 0x%llx\n", val);
67		return 1;
68	}
69
70	return 0;
71}
72
73bool ebb_check_count(int pmc, u64 sample_period, int fudge)
74{
75	u64 count, upper, lower;
76
77	count = ebb_state.stats.pmc_count[PMC_INDEX(pmc)];
78
79	lower = ebb_state.stats.ebb_count * (sample_period - fudge);
80
81	if (count < lower) {
82		printf("PMC%d count (0x%llx) below lower limit 0x%llx (-0x%llx)\n",
83			pmc, count, lower, lower - count);
84		return false;
85	}
86
87	upper = ebb_state.stats.ebb_count * (sample_period + fudge);
88
89	if (count > upper) {
90		printf("PMC%d count (0x%llx) above upper limit 0x%llx (+0x%llx)\n",
91			pmc, count, upper, count - upper);
92		return false;
93	}
94
95	printf("PMC%d count (0x%llx) is between 0x%llx and 0x%llx delta +0x%llx/-0x%llx\n",
96		pmc, count, lower, upper, count - lower, upper - count);
97
98	return true;
99}
100
101void standard_ebb_callee(void)
102{
103	int found, i;
104	u64 val;
105
106	val = mfspr(SPRN_BESCR);
107	if (!(val & BESCR_PMEO)) {
108		ebb_state.stats.spurious++;
109		goto out;
110	}
111
112	ebb_state.stats.ebb_count++;
113	trace_log_counter(ebb_state.trace, ebb_state.stats.ebb_count);
114
115	val = mfspr(SPRN_MMCR0);
116	trace_log_reg(ebb_state.trace, SPRN_MMCR0, val);
117
118	found = 0;
119	for (i = 1; i <= 6; i++) {
120		if (ebb_state.pmc_enable[PMC_INDEX(i)])
121			found += count_pmc(i, sample_period);
122	}
123
124	if (!found)
125		ebb_state.stats.no_overflow++;
126
127out:
128	reset_ebb();
129}
130
131extern void ebb_handler(void);
132
133void setup_ebb_handler(void (*callee)(void))
134{
135	u64 entry;
136
137#if defined(_CALL_ELF) && _CALL_ELF == 2
138	entry = (u64)ebb_handler;
139#else
140	struct opd
141	{
142	    u64 entry;
143	    u64 toc;
144	} *opd;
145
146	opd = (struct opd *)ebb_handler;
147	entry = opd->entry;
148#endif
149	printf("EBB Handler is at %#llx\n", entry);
150
151	ebb_user_func = callee;
152
153	/* Ensure ebb_user_func is set before we set the handler */
154	mb();
155	mtspr(SPRN_EBBHR, entry);
156
157	/* Make sure the handler is set before we return */
158	mb();
159}
160
161void clear_ebb_stats(void)
162{
163	memset(&ebb_state.stats, 0, sizeof(ebb_state.stats));
164}
165
166void dump_summary_ebb_state(void)
167{
168	printf("ebb_state:\n"			\
169	       "  ebb_count    = %d\n"		\
170	       "  spurious     = %d\n"		\
171	       "  negative     = %d\n"		\
172	       "  no_overflow  = %d\n"		\
173	       "  pmc[1] count = 0x%llx\n"	\
174	       "  pmc[2] count = 0x%llx\n"	\
175	       "  pmc[3] count = 0x%llx\n"	\
176	       "  pmc[4] count = 0x%llx\n"	\
177	       "  pmc[5] count = 0x%llx\n"	\
178	       "  pmc[6] count = 0x%llx\n",
179		ebb_state.stats.ebb_count, ebb_state.stats.spurious,
180		ebb_state.stats.negative, ebb_state.stats.no_overflow,
181		ebb_state.stats.pmc_count[0], ebb_state.stats.pmc_count[1],
182		ebb_state.stats.pmc_count[2], ebb_state.stats.pmc_count[3],
183		ebb_state.stats.pmc_count[4], ebb_state.stats.pmc_count[5]);
184}
185
186static char *decode_mmcr0(u32 value)
187{
188	static char buf[16];
189
190	buf[0] = '\0';
191
192	if (value & (1 << 31))
193		strcat(buf, "FC ");
194	if (value & (1 << 26))
195		strcat(buf, "PMAE ");
196	if (value & (1 << 7))
197		strcat(buf, "PMAO ");
198
199	return buf;
200}
201
202static char *decode_bescr(u64 value)
203{
204	static char buf[16];
205
206	buf[0] = '\0';
207
208	if (value & (1ull << 63))
209		strcat(buf, "GE ");
210	if (value & (1ull << 32))
211		strcat(buf, "PMAE ");
212	if (value & 1)
213		strcat(buf, "PMAO ");
214
215	return buf;
216}
217
218void dump_ebb_hw_state(void)
219{
220	u64 bescr;
221	u32 mmcr0;
222
223	mmcr0 = mfspr(SPRN_MMCR0);
224	bescr = mfspr(SPRN_BESCR);
225
226	printf("HW state:\n"		\
227	       "MMCR0 0x%016x %s\n"	\
228	       "MMCR2 0x%016lx\n"	\
229	       "EBBHR 0x%016lx\n"	\
230	       "BESCR 0x%016llx %s\n"	\
231	       "PMC1  0x%016lx\n"	\
232	       "PMC2  0x%016lx\n"	\
233	       "PMC3  0x%016lx\n"	\
234	       "PMC4  0x%016lx\n"	\
235	       "PMC5  0x%016lx\n"	\
236	       "PMC6  0x%016lx\n"	\
237	       "SIAR  0x%016lx\n",
238	       mmcr0, decode_mmcr0(mmcr0), mfspr(SPRN_MMCR2),
239	       mfspr(SPRN_EBBHR), bescr, decode_bescr(bescr),
240	       mfspr(SPRN_PMC1), mfspr(SPRN_PMC2), mfspr(SPRN_PMC3),
241	       mfspr(SPRN_PMC4), mfspr(SPRN_PMC5), mfspr(SPRN_PMC6),
242	       mfspr(SPRN_SIAR));
243}
244
245void dump_ebb_state(void)
246{
247	dump_summary_ebb_state();
248
249	dump_ebb_hw_state();
250
251	trace_buffer_print(ebb_state.trace);
252}
253
254int count_pmc(int pmc, uint32_t sample_period)
255{
256	uint32_t start_value;
257	u64 val;
258
259	/* 0) Read PMC */
260	start_value = pmc_sample_period(sample_period);
261
262	val = read_pmc(pmc);
263	if (val < start_value)
264		ebb_state.stats.negative++;
265	else
266		ebb_state.stats.pmc_count[PMC_INDEX(pmc)] += val - start_value;
267
268	trace_log_reg(ebb_state.trace, SPRN_PMC1 + pmc - 1, val);
269
270	/* 1) Reset PMC */
271	write_pmc(pmc, start_value);
272
273	/* Report if we overflowed */
274	return val >= COUNTER_OVERFLOW;
275}
276
277int ebb_event_enable(struct event *e)
278{
279	int rc;
280
281	/* Ensure any SPR writes are ordered vs us */
282	mb();
283
284	rc = ioctl(e->fd, PERF_EVENT_IOC_ENABLE);
285	if (rc)
286		return rc;
287
288	rc = event_read(e);
289
290	/* Ditto */
291	mb();
292
293	return rc;
294}
295
296void ebb_freeze_pmcs(void)
297{
298	mtspr(SPRN_MMCR0, mfspr(SPRN_MMCR0) | MMCR0_FC);
299	mb();
300}
301
302void ebb_unfreeze_pmcs(void)
303{
304	/* Unfreeze counters */
305	mtspr(SPRN_MMCR0, mfspr(SPRN_MMCR0) & ~MMCR0_FC);
306	mb();
307}
308
309void ebb_global_enable(void)
310{
311	/* Enable EBBs globally and PMU EBBs */
312	mtspr(SPRN_BESCR, 0x8000000100000000ull);
313	mb();
314}
315
316void ebb_global_disable(void)
317{
318	/* Disable EBBs & freeze counters, events are still scheduled */
319	mtspr(SPRN_BESCRR, BESCR_PME);
320	mb();
321}
322
323bool ebb_is_supported(void)
324{
325#ifdef PPC_FEATURE2_EBB
326	/* EBB requires at least POWER8 */
327	return ((long)get_auxv_entry(AT_HWCAP2) & PPC_FEATURE2_EBB);
328#else
329	return false;
330#endif
331}
332
333void event_ebb_init(struct event *e)
334{
335	e->attr.config |= (1ull << 63);
336}
337
338void event_bhrb_init(struct event *e, unsigned ifm)
339{
340	e->attr.config |= (1ull << 62) | ((u64)ifm << 60);
341}
342
343void event_leader_ebb_init(struct event *e)
344{
345	event_ebb_init(e);
346
347	e->attr.exclusive = 1;
348	e->attr.pinned = 1;
349}
350
351int ebb_child(union pipe read_pipe, union pipe write_pipe)
352{
353	struct event event;
354	uint64_t val;
355
356	FAIL_IF(wait_for_parent(read_pipe));
357
358	event_init_named(&event, 0x1001e, "cycles");
359	event_leader_ebb_init(&event);
360
361	event.attr.exclude_kernel = 1;
362	event.attr.exclude_hv = 1;
363	event.attr.exclude_idle = 1;
364
365	FAIL_IF(event_open(&event));
366
367	ebb_enable_pmc_counting(1);
368	setup_ebb_handler(standard_ebb_callee);
369	ebb_global_enable();
370
371	FAIL_IF(event_enable(&event));
372
373	if (event_read(&event)) {
374		/*
375		 * Some tests expect to fail here, so don't report an error on
376		 * this line, and return a distinguisable error code. Tell the
377		 * parent an error happened.
378		 */
379		notify_parent_of_error(write_pipe);
380		return 2;
381	}
382
383	mtspr(SPRN_PMC1, pmc_sample_period(sample_period));
384
385	FAIL_IF(notify_parent(write_pipe));
386	FAIL_IF(wait_for_parent(read_pipe));
387	FAIL_IF(notify_parent(write_pipe));
388
389	while (ebb_state.stats.ebb_count < 20) {
390		FAIL_IF(core_busy_loop());
391
392		/* To try and hit SIGILL case */
393		val  = mfspr(SPRN_MMCRA);
394		val |= mfspr(SPRN_MMCR2);
395		val |= mfspr(SPRN_MMCR0);
396	}
397
398	ebb_global_disable();
399	ebb_freeze_pmcs();
400
401	count_pmc(1, sample_period);
402
403	dump_ebb_state();
404
405	event_close(&event);
406
407	FAIL_IF(ebb_state.stats.ebb_count == 0);
408
409	return 0;
410}
411
412static jmp_buf setjmp_env;
413
414static void sigill_handler(int signal)
415{
416	printf("Took sigill\n");
417	longjmp(setjmp_env, 1);
418}
419
420static struct sigaction sigill_action = {
421	.sa_handler = sigill_handler,
422};
423
424int catch_sigill(void (*func)(void))
425{
426	if (sigaction(SIGILL, &sigill_action, NULL)) {
427		perror("sigaction");
428		return 1;
429	}
430
431	if (setjmp(setjmp_env) == 0) {
432		func();
433		return 1;
434	}
435
436	return 0;
437}
438
439void write_pmc1(void)
440{
441	mtspr(SPRN_PMC1, 0);
442}
443
444void write_pmc(int pmc, u64 value)
445{
446	switch (pmc) {
447		case 1: mtspr(SPRN_PMC1, value); break;
448		case 2: mtspr(SPRN_PMC2, value); break;
449		case 3: mtspr(SPRN_PMC3, value); break;
450		case 4: mtspr(SPRN_PMC4, value); break;
451		case 5: mtspr(SPRN_PMC5, value); break;
452		case 6: mtspr(SPRN_PMC6, value); break;
453	}
454}
455
456u64 read_pmc(int pmc)
457{
458	switch (pmc) {
459		case 1: return mfspr(SPRN_PMC1);
460		case 2: return mfspr(SPRN_PMC2);
461		case 3: return mfspr(SPRN_PMC3);
462		case 4: return mfspr(SPRN_PMC4);
463		case 5: return mfspr(SPRN_PMC5);
464		case 6: return mfspr(SPRN_PMC6);
465	}
466
467	return 0;
468}
469
470static void term_handler(int signal)
471{
472	dump_summary_ebb_state();
473	dump_ebb_hw_state();
474	abort();
475}
476
477struct sigaction term_action = {
478	.sa_handler = term_handler,
479};
480
481static void __attribute__((constructor)) ebb_init(void)
482{
483	clear_ebb_stats();
484
485	if (sigaction(SIGTERM, &term_action, NULL))
486		perror("sigaction");
487
488	ebb_state.trace = trace_buffer_allocate(1 * 1024 * 1024);
489}
490