1/*
2 * bpf-loader.c
3 *
4 * Copyright (C) 2015 Wang Nan <wangnan0@huawei.com>
5 * Copyright (C) 2015 Huawei Inc.
6 */
7
8#include <bpf/libbpf.h>
9#include <linux/err.h>
10#include "perf.h"
11#include "debug.h"
12#include "bpf-loader.h"
13#include "probe-event.h"
14#include "probe-finder.h" // for MAX_PROBES
15#include "llvm-utils.h"
16
17#define DEFINE_PRINT_FN(name, level) \
18static int libbpf_##name(const char *fmt, ...)	\
19{						\
20	va_list args;				\
21	int ret;				\
22						\
23	va_start(args, fmt);			\
24	ret = veprintf(level, verbose, pr_fmt(fmt), args);\
25	va_end(args);				\
26	return ret;				\
27}
28
29DEFINE_PRINT_FN(warning, 1)
30DEFINE_PRINT_FN(info, 1)
31DEFINE_PRINT_FN(debug, 1)
32
33struct bpf_prog_priv {
34	struct perf_probe_event pev;
35};
36
37static bool libbpf_initialized;
38
39struct bpf_object *
40bpf__prepare_load_buffer(void *obj_buf, size_t obj_buf_sz, const char *name)
41{
42	struct bpf_object *obj;
43
44	if (!libbpf_initialized) {
45		libbpf_set_print(libbpf_warning,
46				 libbpf_info,
47				 libbpf_debug);
48		libbpf_initialized = true;
49	}
50
51	obj = bpf_object__open_buffer(obj_buf, obj_buf_sz, name);
52	if (IS_ERR(obj)) {
53		pr_debug("bpf: failed to load buffer\n");
54		return ERR_PTR(-EINVAL);
55	}
56
57	return obj;
58}
59
60struct bpf_object *bpf__prepare_load(const char *filename, bool source)
61{
62	struct bpf_object *obj;
63
64	if (!libbpf_initialized) {
65		libbpf_set_print(libbpf_warning,
66				 libbpf_info,
67				 libbpf_debug);
68		libbpf_initialized = true;
69	}
70
71	if (source) {
72		int err;
73		void *obj_buf;
74		size_t obj_buf_sz;
75
76		err = llvm__compile_bpf(filename, &obj_buf, &obj_buf_sz);
77		if (err)
78			return ERR_PTR(-BPF_LOADER_ERRNO__COMPILE);
79		obj = bpf_object__open_buffer(obj_buf, obj_buf_sz, filename);
80		free(obj_buf);
81	} else
82		obj = bpf_object__open(filename);
83
84	if (IS_ERR(obj)) {
85		pr_debug("bpf: failed to load %s\n", filename);
86		return obj;
87	}
88
89	return obj;
90}
91
92void bpf__clear(void)
93{
94	struct bpf_object *obj, *tmp;
95
96	bpf_object__for_each_safe(obj, tmp) {
97		bpf__unprobe(obj);
98		bpf_object__close(obj);
99	}
100}
101
102static void
103bpf_prog_priv__clear(struct bpf_program *prog __maybe_unused,
104		     void *_priv)
105{
106	struct bpf_prog_priv *priv = _priv;
107
108	cleanup_perf_probe_events(&priv->pev, 1);
109	free(priv);
110}
111
112static int
113config_bpf_program(struct bpf_program *prog)
114{
115	struct perf_probe_event *pev = NULL;
116	struct bpf_prog_priv *priv = NULL;
117	const char *config_str;
118	int err;
119
120	config_str = bpf_program__title(prog, false);
121	if (IS_ERR(config_str)) {
122		pr_debug("bpf: unable to get title for program\n");
123		return PTR_ERR(config_str);
124	}
125
126	priv = calloc(sizeof(*priv), 1);
127	if (!priv) {
128		pr_debug("bpf: failed to alloc priv\n");
129		return -ENOMEM;
130	}
131	pev = &priv->pev;
132
133	pr_debug("bpf: config program '%s'\n", config_str);
134	err = parse_perf_probe_command(config_str, pev);
135	if (err < 0) {
136		pr_debug("bpf: '%s' is not a valid config string\n",
137			 config_str);
138		err = -BPF_LOADER_ERRNO__CONFIG;
139		goto errout;
140	}
141
142	if (pev->group && strcmp(pev->group, PERF_BPF_PROBE_GROUP)) {
143		pr_debug("bpf: '%s': group for event is set and not '%s'.\n",
144			 config_str, PERF_BPF_PROBE_GROUP);
145		err = -BPF_LOADER_ERRNO__GROUP;
146		goto errout;
147	} else if (!pev->group)
148		pev->group = strdup(PERF_BPF_PROBE_GROUP);
149
150	if (!pev->group) {
151		pr_debug("bpf: strdup failed\n");
152		err = -ENOMEM;
153		goto errout;
154	}
155
156	if (!pev->event) {
157		pr_debug("bpf: '%s': event name is missing. Section name should be 'key=value'\n",
158			 config_str);
159		err = -BPF_LOADER_ERRNO__EVENTNAME;
160		goto errout;
161	}
162	pr_debug("bpf: config '%s' is ok\n", config_str);
163
164	err = bpf_program__set_private(prog, priv, bpf_prog_priv__clear);
165	if (err) {
166		pr_debug("Failed to set priv for program '%s'\n", config_str);
167		goto errout;
168	}
169
170	return 0;
171
172errout:
173	if (pev)
174		clear_perf_probe_event(pev);
175	free(priv);
176	return err;
177}
178
179static int bpf__prepare_probe(void)
180{
181	static int err = 0;
182	static bool initialized = false;
183
184	/*
185	 * Make err static, so if init failed the first, bpf__prepare_probe()
186	 * fails each time without calling init_probe_symbol_maps multiple
187	 * times.
188	 */
189	if (initialized)
190		return err;
191
192	initialized = true;
193	err = init_probe_symbol_maps(false);
194	if (err < 0)
195		pr_debug("Failed to init_probe_symbol_maps\n");
196	probe_conf.max_probes = MAX_PROBES;
197	return err;
198}
199
200int bpf__probe(struct bpf_object *obj)
201{
202	int err = 0;
203	struct bpf_program *prog;
204	struct bpf_prog_priv *priv;
205	struct perf_probe_event *pev;
206
207	err = bpf__prepare_probe();
208	if (err) {
209		pr_debug("bpf__prepare_probe failed\n");
210		return err;
211	}
212
213	bpf_object__for_each_program(prog, obj) {
214		err = config_bpf_program(prog);
215		if (err)
216			goto out;
217
218		err = bpf_program__get_private(prog, (void **)&priv);
219		if (err || !priv)
220			goto out;
221		pev = &priv->pev;
222
223		err = convert_perf_probe_events(pev, 1);
224		if (err < 0) {
225			pr_debug("bpf_probe: failed to convert perf probe events");
226			goto out;
227		}
228
229		err = apply_perf_probe_events(pev, 1);
230		if (err < 0) {
231			pr_debug("bpf_probe: failed to apply perf probe events");
232			goto out;
233		}
234	}
235out:
236	return err < 0 ? err : 0;
237}
238
239#define EVENTS_WRITE_BUFSIZE  4096
240int bpf__unprobe(struct bpf_object *obj)
241{
242	int err, ret = 0;
243	struct bpf_program *prog;
244	struct bpf_prog_priv *priv;
245
246	bpf_object__for_each_program(prog, obj) {
247		int i;
248
249		err = bpf_program__get_private(prog, (void **)&priv);
250		if (err || !priv)
251			continue;
252
253		for (i = 0; i < priv->pev.ntevs; i++) {
254			struct probe_trace_event *tev = &priv->pev.tevs[i];
255			char name_buf[EVENTS_WRITE_BUFSIZE];
256			struct strfilter *delfilter;
257
258			snprintf(name_buf, EVENTS_WRITE_BUFSIZE,
259				 "%s:%s", tev->group, tev->event);
260			name_buf[EVENTS_WRITE_BUFSIZE - 1] = '\0';
261
262			delfilter = strfilter__new(name_buf, NULL);
263			if (!delfilter) {
264				pr_debug("Failed to create filter for unprobing\n");
265				ret = -ENOMEM;
266				continue;
267			}
268
269			err = del_perf_probe_events(delfilter);
270			strfilter__delete(delfilter);
271			if (err) {
272				pr_debug("Failed to delete %s\n", name_buf);
273				ret = err;
274				continue;
275			}
276		}
277	}
278	return ret;
279}
280
281int bpf__load(struct bpf_object *obj)
282{
283	int err;
284
285	err = bpf_object__load(obj);
286	if (err) {
287		pr_debug("bpf: load objects failed\n");
288		return err;
289	}
290	return 0;
291}
292
293int bpf__foreach_tev(struct bpf_object *obj,
294		     bpf_prog_iter_callback_t func,
295		     void *arg)
296{
297	struct bpf_program *prog;
298	int err;
299
300	bpf_object__for_each_program(prog, obj) {
301		struct probe_trace_event *tev;
302		struct perf_probe_event *pev;
303		struct bpf_prog_priv *priv;
304		int i, fd;
305
306		err = bpf_program__get_private(prog,
307				(void **)&priv);
308		if (err || !priv) {
309			pr_debug("bpf: failed to get private field\n");
310			return -BPF_LOADER_ERRNO__INTERNAL;
311		}
312
313		pev = &priv->pev;
314		for (i = 0; i < pev->ntevs; i++) {
315			tev = &pev->tevs[i];
316
317			fd = bpf_program__fd(prog);
318			if (fd < 0) {
319				pr_debug("bpf: failed to get file descriptor\n");
320				return fd;
321			}
322
323			err = (*func)(tev, fd, arg);
324			if (err) {
325				pr_debug("bpf: call back failed, stop iterate\n");
326				return err;
327			}
328		}
329	}
330	return 0;
331}
332
333#define ERRNO_OFFSET(e)		((e) - __BPF_LOADER_ERRNO__START)
334#define ERRCODE_OFFSET(c)	ERRNO_OFFSET(BPF_LOADER_ERRNO__##c)
335#define NR_ERRNO	(__BPF_LOADER_ERRNO__END - __BPF_LOADER_ERRNO__START)
336
337static const char *bpf_loader_strerror_table[NR_ERRNO] = {
338	[ERRCODE_OFFSET(CONFIG)]	= "Invalid config string",
339	[ERRCODE_OFFSET(GROUP)]		= "Invalid group name",
340	[ERRCODE_OFFSET(EVENTNAME)]	= "No event name found in config string",
341	[ERRCODE_OFFSET(INTERNAL)]	= "BPF loader internal error",
342	[ERRCODE_OFFSET(COMPILE)]	= "Error when compiling BPF scriptlet",
343};
344
345static int
346bpf_loader_strerror(int err, char *buf, size_t size)
347{
348	char sbuf[STRERR_BUFSIZE];
349	const char *msg;
350
351	if (!buf || !size)
352		return -1;
353
354	err = err > 0 ? err : -err;
355
356	if (err >= __LIBBPF_ERRNO__START)
357		return libbpf_strerror(err, buf, size);
358
359	if (err >= __BPF_LOADER_ERRNO__START && err < __BPF_LOADER_ERRNO__END) {
360		msg = bpf_loader_strerror_table[ERRNO_OFFSET(err)];
361		snprintf(buf, size, "%s", msg);
362		buf[size - 1] = '\0';
363		return 0;
364	}
365
366	if (err >= __BPF_LOADER_ERRNO__END)
367		snprintf(buf, size, "Unknown bpf loader error %d", err);
368	else
369		snprintf(buf, size, "%s",
370			 strerror_r(err, sbuf, sizeof(sbuf)));
371
372	buf[size - 1] = '\0';
373	return -1;
374}
375
376#define bpf__strerror_head(err, buf, size) \
377	char sbuf[STRERR_BUFSIZE], *emsg;\
378	if (!size)\
379		return 0;\
380	if (err < 0)\
381		err = -err;\
382	bpf_loader_strerror(err, sbuf, sizeof(sbuf));\
383	emsg = sbuf;\
384	switch (err) {\
385	default:\
386		scnprintf(buf, size, "%s", emsg);\
387		break;
388
389#define bpf__strerror_entry(val, fmt...)\
390	case val: {\
391		scnprintf(buf, size, fmt);\
392		break;\
393	}
394
395#define bpf__strerror_end(buf, size)\
396	}\
397	buf[size - 1] = '\0';
398
399int bpf__strerror_prepare_load(const char *filename, bool source,
400			       int err, char *buf, size_t size)
401{
402	size_t n;
403	int ret;
404
405	n = snprintf(buf, size, "Failed to load %s%s: ",
406			 filename, source ? " from source" : "");
407	if (n >= size) {
408		buf[size - 1] = '\0';
409		return 0;
410	}
411	buf += n;
412	size -= n;
413
414	ret = bpf_loader_strerror(err, buf, size);
415	buf[size - 1] = '\0';
416	return ret;
417}
418
419int bpf__strerror_probe(struct bpf_object *obj __maybe_unused,
420			int err, char *buf, size_t size)
421{
422	bpf__strerror_head(err, buf, size);
423	bpf__strerror_entry(EEXIST, "Probe point exist. Try use 'perf probe -d \"*\"'");
424	bpf__strerror_entry(EACCES, "You need to be root");
425	bpf__strerror_entry(EPERM, "You need to be root, and /proc/sys/kernel/kptr_restrict should be 0");
426	bpf__strerror_entry(ENOENT, "You need to check probing points in BPF file");
427	bpf__strerror_end(buf, size);
428	return 0;
429}
430
431int bpf__strerror_load(struct bpf_object *obj,
432		       int err, char *buf, size_t size)
433{
434	bpf__strerror_head(err, buf, size);
435	case LIBBPF_ERRNO__KVER: {
436		unsigned int obj_kver = bpf_object__get_kversion(obj);
437		unsigned int real_kver;
438
439		if (fetch_kernel_version(&real_kver, NULL, 0)) {
440			scnprintf(buf, size, "Unable to fetch kernel version");
441			break;
442		}
443
444		if (obj_kver != real_kver) {
445			scnprintf(buf, size,
446				  "'version' ("KVER_FMT") doesn't match running kernel ("KVER_FMT")",
447				  KVER_PARAM(obj_kver),
448				  KVER_PARAM(real_kver));
449			break;
450		}
451
452		scnprintf(buf, size, "Failed to load program for unknown reason");
453		break;
454	}
455	bpf__strerror_end(buf, size);
456	return 0;
457}
458