1/*
2 * Copyright 2014, Michael Ellerman, IBM Corp.
3 * Licensed under GPLv2.
4 */
5
6#define _GNU_SOURCE	/* For CPU_ZERO etc. */
7
8#include <sched.h>
9#include <sys/wait.h>
10#include <setjmp.h>
11#include <signal.h>
12#include <stdio.h>
13#include <stdlib.h>
14#include <string.h>
15#include <sys/ioctl.h>
16
17#include "trace.h"
18#include "reg.h"
19#include "ebb.h"
20
21
22void (*ebb_user_func)(void);
23
24void ebb_hook(void)
25{
26	if (ebb_user_func)
27		ebb_user_func();
28}
29
30struct ebb_state ebb_state;
31
32u64 sample_period = 0x40000000ull;
33
34void reset_ebb_with_clear_mask(unsigned long mmcr0_clear_mask)
35{
36	u64 val;
37
38	/* 2) clear MMCR0[PMAO] - docs say BESCR[PMEO] should do this */
39	/* 3) set MMCR0[PMAE]	- docs say BESCR[PME] should do this */
40	val = mfspr(SPRN_MMCR0);
41	mtspr(SPRN_MMCR0, (val & ~mmcr0_clear_mask) | MMCR0_PMAE);
42
43	/* 4) clear BESCR[PMEO] */
44	mtspr(SPRN_BESCRR, BESCR_PMEO);
45
46	/* 5) set BESCR[PME] */
47	mtspr(SPRN_BESCRS, BESCR_PME);
48
49	/* 6) rfebb 1 - done in our caller */
50}
51
52void reset_ebb(void)
53{
54	reset_ebb_with_clear_mask(MMCR0_PMAO | MMCR0_FC);
55}
56
57/* Called outside of the EBB handler to check MMCR0 is sane */
58int ebb_check_mmcr0(void)
59{
60	u64 val;
61
62	val = mfspr(SPRN_MMCR0);
63	if ((val & (MMCR0_FC | MMCR0_PMAO)) == MMCR0_FC) {
64		/* It's OK if we see FC & PMAO, but not FC by itself */
65		printf("Outside of loop, only FC set 0x%llx\n", val);
66		return 1;
67	}
68
69	return 0;
70}
71
72bool ebb_check_count(int pmc, u64 sample_period, int fudge)
73{
74	u64 count, upper, lower;
75
76	count = ebb_state.stats.pmc_count[PMC_INDEX(pmc)];
77
78	lower = ebb_state.stats.ebb_count * (sample_period - fudge);
79
80	if (count < lower) {
81		printf("PMC%d count (0x%llx) below lower limit 0x%llx (-0x%llx)\n",
82			pmc, count, lower, lower - count);
83		return false;
84	}
85
86	upper = ebb_state.stats.ebb_count * (sample_period + fudge);
87
88	if (count > upper) {
89		printf("PMC%d count (0x%llx) above upper limit 0x%llx (+0x%llx)\n",
90			pmc, count, upper, count - upper);
91		return false;
92	}
93
94	printf("PMC%d count (0x%llx) is between 0x%llx and 0x%llx delta +0x%llx/-0x%llx\n",
95		pmc, count, lower, upper, count - lower, upper - count);
96
97	return true;
98}
99
100void standard_ebb_callee(void)
101{
102	int found, i;
103	u64 val;
104
105	val = mfspr(SPRN_BESCR);
106	if (!(val & BESCR_PMEO)) {
107		ebb_state.stats.spurious++;
108		goto out;
109	}
110
111	ebb_state.stats.ebb_count++;
112	trace_log_counter(ebb_state.trace, ebb_state.stats.ebb_count);
113
114	val = mfspr(SPRN_MMCR0);
115	trace_log_reg(ebb_state.trace, SPRN_MMCR0, val);
116
117	found = 0;
118	for (i = 1; i <= 6; i++) {
119		if (ebb_state.pmc_enable[PMC_INDEX(i)])
120			found += count_pmc(i, sample_period);
121	}
122
123	if (!found)
124		ebb_state.stats.no_overflow++;
125
126out:
127	reset_ebb();
128}
129
130extern void ebb_handler(void);
131
132void setup_ebb_handler(void (*callee)(void))
133{
134	u64 entry;
135
136#if defined(_CALL_ELF) && _CALL_ELF == 2
137	entry = (u64)ebb_handler;
138#else
139	struct opd
140	{
141	    u64 entry;
142	    u64 toc;
143	} *opd;
144
145	opd = (struct opd *)ebb_handler;
146	entry = opd->entry;
147#endif
148	printf("EBB Handler is at %#llx\n", entry);
149
150	ebb_user_func = callee;
151
152	/* Ensure ebb_user_func is set before we set the handler */
153	mb();
154	mtspr(SPRN_EBBHR, entry);
155
156	/* Make sure the handler is set before we return */
157	mb();
158}
159
160void clear_ebb_stats(void)
161{
162	memset(&ebb_state.stats, 0, sizeof(ebb_state.stats));
163}
164
165void dump_summary_ebb_state(void)
166{
167	printf("ebb_state:\n"			\
168	       "  ebb_count    = %d\n"		\
169	       "  spurious     = %d\n"		\
170	       "  negative     = %d\n"		\
171	       "  no_overflow  = %d\n"		\
172	       "  pmc[1] count = 0x%llx\n"	\
173	       "  pmc[2] count = 0x%llx\n"	\
174	       "  pmc[3] count = 0x%llx\n"	\
175	       "  pmc[4] count = 0x%llx\n"	\
176	       "  pmc[5] count = 0x%llx\n"	\
177	       "  pmc[6] count = 0x%llx\n",
178		ebb_state.stats.ebb_count, ebb_state.stats.spurious,
179		ebb_state.stats.negative, ebb_state.stats.no_overflow,
180		ebb_state.stats.pmc_count[0], ebb_state.stats.pmc_count[1],
181		ebb_state.stats.pmc_count[2], ebb_state.stats.pmc_count[3],
182		ebb_state.stats.pmc_count[4], ebb_state.stats.pmc_count[5]);
183}
184
185static char *decode_mmcr0(u32 value)
186{
187	static char buf[16];
188
189	buf[0] = '\0';
190
191	if (value & (1 << 31))
192		strcat(buf, "FC ");
193	if (value & (1 << 26))
194		strcat(buf, "PMAE ");
195	if (value & (1 << 7))
196		strcat(buf, "PMAO ");
197
198	return buf;
199}
200
201static char *decode_bescr(u64 value)
202{
203	static char buf[16];
204
205	buf[0] = '\0';
206
207	if (value & (1ull << 63))
208		strcat(buf, "GE ");
209	if (value & (1ull << 32))
210		strcat(buf, "PMAE ");
211	if (value & 1)
212		strcat(buf, "PMAO ");
213
214	return buf;
215}
216
217void dump_ebb_hw_state(void)
218{
219	u64 bescr;
220	u32 mmcr0;
221
222	mmcr0 = mfspr(SPRN_MMCR0);
223	bescr = mfspr(SPRN_BESCR);
224
225	printf("HW state:\n"		\
226	       "MMCR0 0x%016x %s\n"	\
227	       "MMCR2 0x%016lx\n"	\
228	       "EBBHR 0x%016lx\n"	\
229	       "BESCR 0x%016llx %s\n"	\
230	       "PMC1  0x%016lx\n"	\
231	       "PMC2  0x%016lx\n"	\
232	       "PMC3  0x%016lx\n"	\
233	       "PMC4  0x%016lx\n"	\
234	       "PMC5  0x%016lx\n"	\
235	       "PMC6  0x%016lx\n"	\
236	       "SIAR  0x%016lx\n",
237	       mmcr0, decode_mmcr0(mmcr0), mfspr(SPRN_MMCR2),
238	       mfspr(SPRN_EBBHR), bescr, decode_bescr(bescr),
239	       mfspr(SPRN_PMC1), mfspr(SPRN_PMC2), mfspr(SPRN_PMC3),
240	       mfspr(SPRN_PMC4), mfspr(SPRN_PMC5), mfspr(SPRN_PMC6),
241	       mfspr(SPRN_SIAR));
242}
243
244void dump_ebb_state(void)
245{
246	dump_summary_ebb_state();
247
248	dump_ebb_hw_state();
249
250	trace_buffer_print(ebb_state.trace);
251}
252
253int count_pmc(int pmc, uint32_t sample_period)
254{
255	uint32_t start_value;
256	u64 val;
257
258	/* 0) Read PMC */
259	start_value = pmc_sample_period(sample_period);
260
261	val = read_pmc(pmc);
262	if (val < start_value)
263		ebb_state.stats.negative++;
264	else
265		ebb_state.stats.pmc_count[PMC_INDEX(pmc)] += val - start_value;
266
267	trace_log_reg(ebb_state.trace, SPRN_PMC1 + pmc - 1, val);
268
269	/* 1) Reset PMC */
270	write_pmc(pmc, start_value);
271
272	/* Report if we overflowed */
273	return val >= COUNTER_OVERFLOW;
274}
275
276int ebb_event_enable(struct event *e)
277{
278	int rc;
279
280	/* Ensure any SPR writes are ordered vs us */
281	mb();
282
283	rc = ioctl(e->fd, PERF_EVENT_IOC_ENABLE);
284	if (rc)
285		return rc;
286
287	rc = event_read(e);
288
289	/* Ditto */
290	mb();
291
292	return rc;
293}
294
295void ebb_freeze_pmcs(void)
296{
297	mtspr(SPRN_MMCR0, mfspr(SPRN_MMCR0) | MMCR0_FC);
298	mb();
299}
300
301void ebb_unfreeze_pmcs(void)
302{
303	/* Unfreeze counters */
304	mtspr(SPRN_MMCR0, mfspr(SPRN_MMCR0) & ~MMCR0_FC);
305	mb();
306}
307
308void ebb_global_enable(void)
309{
310	/* Enable EBBs globally and PMU EBBs */
311	mtspr(SPRN_BESCR, 0x8000000100000000ull);
312	mb();
313}
314
315void ebb_global_disable(void)
316{
317	/* Disable EBBs & freeze counters, events are still scheduled */
318	mtspr(SPRN_BESCRR, BESCR_PME);
319	mb();
320}
321
322void event_ebb_init(struct event *e)
323{
324	e->attr.config |= (1ull << 63);
325}
326
327void event_bhrb_init(struct event *e, unsigned ifm)
328{
329	e->attr.config |= (1ull << 62) | ((u64)ifm << 60);
330}
331
332void event_leader_ebb_init(struct event *e)
333{
334	event_ebb_init(e);
335
336	e->attr.exclusive = 1;
337	e->attr.pinned = 1;
338}
339
340int ebb_child(union pipe read_pipe, union pipe write_pipe)
341{
342	struct event event;
343	uint64_t val;
344
345	FAIL_IF(wait_for_parent(read_pipe));
346
347	event_init_named(&event, 0x1001e, "cycles");
348	event_leader_ebb_init(&event);
349
350	event.attr.exclude_kernel = 1;
351	event.attr.exclude_hv = 1;
352	event.attr.exclude_idle = 1;
353
354	FAIL_IF(event_open(&event));
355
356	ebb_enable_pmc_counting(1);
357	setup_ebb_handler(standard_ebb_callee);
358	ebb_global_enable();
359
360	FAIL_IF(event_enable(&event));
361
362	if (event_read(&event)) {
363		/*
364		 * Some tests expect to fail here, so don't report an error on
365		 * this line, and return a distinguisable error code. Tell the
366		 * parent an error happened.
367		 */
368		notify_parent_of_error(write_pipe);
369		return 2;
370	}
371
372	mtspr(SPRN_PMC1, pmc_sample_period(sample_period));
373
374	FAIL_IF(notify_parent(write_pipe));
375	FAIL_IF(wait_for_parent(read_pipe));
376	FAIL_IF(notify_parent(write_pipe));
377
378	while (ebb_state.stats.ebb_count < 20) {
379		FAIL_IF(core_busy_loop());
380
381		/* To try and hit SIGILL case */
382		val  = mfspr(SPRN_MMCRA);
383		val |= mfspr(SPRN_MMCR2);
384		val |= mfspr(SPRN_MMCR0);
385	}
386
387	ebb_global_disable();
388	ebb_freeze_pmcs();
389
390	count_pmc(1, sample_period);
391
392	dump_ebb_state();
393
394	event_close(&event);
395
396	FAIL_IF(ebb_state.stats.ebb_count == 0);
397
398	return 0;
399}
400
401static jmp_buf setjmp_env;
402
403static void sigill_handler(int signal)
404{
405	printf("Took sigill\n");
406	longjmp(setjmp_env, 1);
407}
408
409static struct sigaction sigill_action = {
410	.sa_handler = sigill_handler,
411};
412
413int catch_sigill(void (*func)(void))
414{
415	if (sigaction(SIGILL, &sigill_action, NULL)) {
416		perror("sigaction");
417		return 1;
418	}
419
420	if (setjmp(setjmp_env) == 0) {
421		func();
422		return 1;
423	}
424
425	return 0;
426}
427
428void write_pmc1(void)
429{
430	mtspr(SPRN_PMC1, 0);
431}
432
433void write_pmc(int pmc, u64 value)
434{
435	switch (pmc) {
436		case 1: mtspr(SPRN_PMC1, value); break;
437		case 2: mtspr(SPRN_PMC2, value); break;
438		case 3: mtspr(SPRN_PMC3, value); break;
439		case 4: mtspr(SPRN_PMC4, value); break;
440		case 5: mtspr(SPRN_PMC5, value); break;
441		case 6: mtspr(SPRN_PMC6, value); break;
442	}
443}
444
445u64 read_pmc(int pmc)
446{
447	switch (pmc) {
448		case 1: return mfspr(SPRN_PMC1);
449		case 2: return mfspr(SPRN_PMC2);
450		case 3: return mfspr(SPRN_PMC3);
451		case 4: return mfspr(SPRN_PMC4);
452		case 5: return mfspr(SPRN_PMC5);
453		case 6: return mfspr(SPRN_PMC6);
454	}
455
456	return 0;
457}
458
459static void term_handler(int signal)
460{
461	dump_summary_ebb_state();
462	dump_ebb_hw_state();
463	abort();
464}
465
466struct sigaction term_action = {
467	.sa_handler = term_handler,
468};
469
470static void __attribute__((constructor)) ebb_init(void)
471{
472	clear_ebb_stats();
473
474	if (sigaction(SIGTERM, &term_action, NULL))
475		perror("sigaction");
476
477	ebb_state.trace = trace_buffer_allocate(1 * 1024 * 1024);
478}
479