1 #include <linux/module.h>
2 #include <linux/slab.h>
3 #include <linux/pci.h>
4 #include <linux/perf_event.h>
5 #include "perf_event.h"
6 
7 #define UNCORE_PMU_NAME_LEN		32
8 #define UNCORE_PMU_HRTIMER_INTERVAL	(60LL * NSEC_PER_SEC)
9 #define UNCORE_SNB_IMC_HRTIMER_INTERVAL (5ULL * NSEC_PER_SEC)
10 
11 #define UNCORE_FIXED_EVENT		0xff
12 #define UNCORE_PMC_IDX_MAX_GENERIC	8
13 #define UNCORE_PMC_IDX_FIXED		UNCORE_PMC_IDX_MAX_GENERIC
14 #define UNCORE_PMC_IDX_MAX		(UNCORE_PMC_IDX_FIXED + 1)
15 
16 #define UNCORE_PCI_DEV_DATA(type, idx)	((type << 8) | idx)
17 #define UNCORE_PCI_DEV_TYPE(data)	((data >> 8) & 0xff)
18 #define UNCORE_PCI_DEV_IDX(data)	(data & 0xff)
19 #define UNCORE_EXTRA_PCI_DEV		0xff
20 #define UNCORE_EXTRA_PCI_DEV_MAX	3
21 
22 /* support up to 8 sockets */
23 #define UNCORE_SOCKET_MAX		8
24 
25 #define UNCORE_EVENT_CONSTRAINT(c, n) EVENT_CONSTRAINT(c, n, 0xff)
26 
27 struct intel_uncore_ops;
28 struct intel_uncore_pmu;
29 struct intel_uncore_box;
30 struct uncore_event_desc;
31 
32 struct intel_uncore_type {
33 	const char *name;
34 	int num_counters;
35 	int num_boxes;
36 	int perf_ctr_bits;
37 	int fixed_ctr_bits;
38 	unsigned perf_ctr;
39 	unsigned event_ctl;
40 	unsigned event_mask;
41 	unsigned fixed_ctr;
42 	unsigned fixed_ctl;
43 	unsigned box_ctl;
44 	unsigned msr_offset;
45 	unsigned num_shared_regs:8;
46 	unsigned single_fixed:1;
47 	unsigned pair_ctr_ctl:1;
48 	unsigned *msr_offsets;
49 	struct event_constraint unconstrainted;
50 	struct event_constraint *constraints;
51 	struct intel_uncore_pmu *pmus;
52 	struct intel_uncore_ops *ops;
53 	struct uncore_event_desc *event_descs;
54 	const struct attribute_group *attr_groups[4];
55 	struct pmu *pmu; /* for custom pmu ops */
56 };
57 
58 #define pmu_group attr_groups[0]
59 #define format_group attr_groups[1]
60 #define events_group attr_groups[2]
61 
62 struct intel_uncore_ops {
63 	void (*init_box)(struct intel_uncore_box *);
64 	void (*disable_box)(struct intel_uncore_box *);
65 	void (*enable_box)(struct intel_uncore_box *);
66 	void (*disable_event)(struct intel_uncore_box *, struct perf_event *);
67 	void (*enable_event)(struct intel_uncore_box *, struct perf_event *);
68 	u64 (*read_counter)(struct intel_uncore_box *, struct perf_event *);
69 	int (*hw_config)(struct intel_uncore_box *, struct perf_event *);
70 	struct event_constraint *(*get_constraint)(struct intel_uncore_box *,
71 						   struct perf_event *);
72 	void (*put_constraint)(struct intel_uncore_box *, struct perf_event *);
73 };
74 
75 struct intel_uncore_pmu {
76 	struct pmu pmu;
77 	char name[UNCORE_PMU_NAME_LEN];
78 	int pmu_idx;
79 	int func_id;
80 	struct intel_uncore_type *type;
81 	struct intel_uncore_box ** __percpu box;
82 	struct list_head box_list;
83 };
84 
85 struct intel_uncore_extra_reg {
86 	raw_spinlock_t lock;
87 	u64 config, config1, config2;
88 	atomic_t ref;
89 };
90 
91 struct intel_uncore_box {
92 	int phys_id;
93 	int n_active;	/* number of active events */
94 	int n_events;
95 	int cpu;	/* cpu to collect events */
96 	unsigned long flags;
97 	atomic_t refcnt;
98 	struct perf_event *events[UNCORE_PMC_IDX_MAX];
99 	struct perf_event *event_list[UNCORE_PMC_IDX_MAX];
100 	struct event_constraint *event_constraint[UNCORE_PMC_IDX_MAX];
101 	unsigned long active_mask[BITS_TO_LONGS(UNCORE_PMC_IDX_MAX)];
102 	u64 tags[UNCORE_PMC_IDX_MAX];
103 	struct pci_dev *pci_dev;
104 	struct intel_uncore_pmu *pmu;
105 	u64 hrtimer_duration; /* hrtimer timeout for this box */
106 	struct hrtimer hrtimer;
107 	struct list_head list;
108 	struct list_head active_list;
109 	void *io_addr;
110 	struct intel_uncore_extra_reg shared_regs[0];
111 };
112 
113 #define UNCORE_BOX_FLAG_INITIATED	0
114 
115 struct uncore_event_desc {
116 	struct kobj_attribute attr;
117 	const char *config;
118 };
119 
120 ssize_t uncore_event_show(struct kobject *kobj,
121 			  struct kobj_attribute *attr, char *buf);
122 
123 #define INTEL_UNCORE_EVENT_DESC(_name, _config)			\
124 {								\
125 	.attr	= __ATTR(_name, 0444, uncore_event_show, NULL),	\
126 	.config	= _config,					\
127 }
128 
129 #define DEFINE_UNCORE_FORMAT_ATTR(_var, _name, _format)			\
130 static ssize_t __uncore_##_var##_show(struct kobject *kobj,		\
131 				struct kobj_attribute *attr,		\
132 				char *page)				\
133 {									\
134 	BUILD_BUG_ON(sizeof(_format) >= PAGE_SIZE);			\
135 	return sprintf(page, _format "\n");				\
136 }									\
137 static struct kobj_attribute format_attr_##_var =			\
138 	__ATTR(_name, 0444, __uncore_##_var##_show, NULL)
139 
uncore_pci_box_ctl(struct intel_uncore_box * box)140 static inline unsigned uncore_pci_box_ctl(struct intel_uncore_box *box)
141 {
142 	return box->pmu->type->box_ctl;
143 }
144 
uncore_pci_fixed_ctl(struct intel_uncore_box * box)145 static inline unsigned uncore_pci_fixed_ctl(struct intel_uncore_box *box)
146 {
147 	return box->pmu->type->fixed_ctl;
148 }
149 
uncore_pci_fixed_ctr(struct intel_uncore_box * box)150 static inline unsigned uncore_pci_fixed_ctr(struct intel_uncore_box *box)
151 {
152 	return box->pmu->type->fixed_ctr;
153 }
154 
155 static inline
uncore_pci_event_ctl(struct intel_uncore_box * box,int idx)156 unsigned uncore_pci_event_ctl(struct intel_uncore_box *box, int idx)
157 {
158 	return idx * 4 + box->pmu->type->event_ctl;
159 }
160 
161 static inline
uncore_pci_perf_ctr(struct intel_uncore_box * box,int idx)162 unsigned uncore_pci_perf_ctr(struct intel_uncore_box *box, int idx)
163 {
164 	return idx * 8 + box->pmu->type->perf_ctr;
165 }
166 
uncore_msr_box_offset(struct intel_uncore_box * box)167 static inline unsigned uncore_msr_box_offset(struct intel_uncore_box *box)
168 {
169 	struct intel_uncore_pmu *pmu = box->pmu;
170 	return pmu->type->msr_offsets ?
171 		pmu->type->msr_offsets[pmu->pmu_idx] :
172 		pmu->type->msr_offset * pmu->pmu_idx;
173 }
174 
uncore_msr_box_ctl(struct intel_uncore_box * box)175 static inline unsigned uncore_msr_box_ctl(struct intel_uncore_box *box)
176 {
177 	if (!box->pmu->type->box_ctl)
178 		return 0;
179 	return box->pmu->type->box_ctl + uncore_msr_box_offset(box);
180 }
181 
uncore_msr_fixed_ctl(struct intel_uncore_box * box)182 static inline unsigned uncore_msr_fixed_ctl(struct intel_uncore_box *box)
183 {
184 	if (!box->pmu->type->fixed_ctl)
185 		return 0;
186 	return box->pmu->type->fixed_ctl + uncore_msr_box_offset(box);
187 }
188 
uncore_msr_fixed_ctr(struct intel_uncore_box * box)189 static inline unsigned uncore_msr_fixed_ctr(struct intel_uncore_box *box)
190 {
191 	return box->pmu->type->fixed_ctr + uncore_msr_box_offset(box);
192 }
193 
194 static inline
uncore_msr_event_ctl(struct intel_uncore_box * box,int idx)195 unsigned uncore_msr_event_ctl(struct intel_uncore_box *box, int idx)
196 {
197 	return box->pmu->type->event_ctl +
198 		(box->pmu->type->pair_ctr_ctl ? 2 * idx : idx) +
199 		uncore_msr_box_offset(box);
200 }
201 
202 static inline
uncore_msr_perf_ctr(struct intel_uncore_box * box,int idx)203 unsigned uncore_msr_perf_ctr(struct intel_uncore_box *box, int idx)
204 {
205 	return box->pmu->type->perf_ctr +
206 		(box->pmu->type->pair_ctr_ctl ? 2 * idx : idx) +
207 		uncore_msr_box_offset(box);
208 }
209 
210 static inline
uncore_fixed_ctl(struct intel_uncore_box * box)211 unsigned uncore_fixed_ctl(struct intel_uncore_box *box)
212 {
213 	if (box->pci_dev)
214 		return uncore_pci_fixed_ctl(box);
215 	else
216 		return uncore_msr_fixed_ctl(box);
217 }
218 
219 static inline
uncore_fixed_ctr(struct intel_uncore_box * box)220 unsigned uncore_fixed_ctr(struct intel_uncore_box *box)
221 {
222 	if (box->pci_dev)
223 		return uncore_pci_fixed_ctr(box);
224 	else
225 		return uncore_msr_fixed_ctr(box);
226 }
227 
228 static inline
uncore_event_ctl(struct intel_uncore_box * box,int idx)229 unsigned uncore_event_ctl(struct intel_uncore_box *box, int idx)
230 {
231 	if (box->pci_dev)
232 		return uncore_pci_event_ctl(box, idx);
233 	else
234 		return uncore_msr_event_ctl(box, idx);
235 }
236 
237 static inline
uncore_perf_ctr(struct intel_uncore_box * box,int idx)238 unsigned uncore_perf_ctr(struct intel_uncore_box *box, int idx)
239 {
240 	if (box->pci_dev)
241 		return uncore_pci_perf_ctr(box, idx);
242 	else
243 		return uncore_msr_perf_ctr(box, idx);
244 }
245 
uncore_perf_ctr_bits(struct intel_uncore_box * box)246 static inline int uncore_perf_ctr_bits(struct intel_uncore_box *box)
247 {
248 	return box->pmu->type->perf_ctr_bits;
249 }
250 
uncore_fixed_ctr_bits(struct intel_uncore_box * box)251 static inline int uncore_fixed_ctr_bits(struct intel_uncore_box *box)
252 {
253 	return box->pmu->type->fixed_ctr_bits;
254 }
255 
uncore_num_counters(struct intel_uncore_box * box)256 static inline int uncore_num_counters(struct intel_uncore_box *box)
257 {
258 	return box->pmu->type->num_counters;
259 }
260 
uncore_disable_box(struct intel_uncore_box * box)261 static inline void uncore_disable_box(struct intel_uncore_box *box)
262 {
263 	if (box->pmu->type->ops->disable_box)
264 		box->pmu->type->ops->disable_box(box);
265 }
266 
uncore_enable_box(struct intel_uncore_box * box)267 static inline void uncore_enable_box(struct intel_uncore_box *box)
268 {
269 	if (box->pmu->type->ops->enable_box)
270 		box->pmu->type->ops->enable_box(box);
271 }
272 
uncore_disable_event(struct intel_uncore_box * box,struct perf_event * event)273 static inline void uncore_disable_event(struct intel_uncore_box *box,
274 				struct perf_event *event)
275 {
276 	box->pmu->type->ops->disable_event(box, event);
277 }
278 
uncore_enable_event(struct intel_uncore_box * box,struct perf_event * event)279 static inline void uncore_enable_event(struct intel_uncore_box *box,
280 				struct perf_event *event)
281 {
282 	box->pmu->type->ops->enable_event(box, event);
283 }
284 
uncore_read_counter(struct intel_uncore_box * box,struct perf_event * event)285 static inline u64 uncore_read_counter(struct intel_uncore_box *box,
286 				struct perf_event *event)
287 {
288 	return box->pmu->type->ops->read_counter(box, event);
289 }
290 
uncore_box_init(struct intel_uncore_box * box)291 static inline void uncore_box_init(struct intel_uncore_box *box)
292 {
293 	if (!test_and_set_bit(UNCORE_BOX_FLAG_INITIATED, &box->flags)) {
294 		if (box->pmu->type->ops->init_box)
295 			box->pmu->type->ops->init_box(box);
296 	}
297 }
298 
uncore_box_is_fake(struct intel_uncore_box * box)299 static inline bool uncore_box_is_fake(struct intel_uncore_box *box)
300 {
301 	return (box->phys_id < 0);
302 }
303 
304 struct intel_uncore_pmu *uncore_event_to_pmu(struct perf_event *event);
305 struct intel_uncore_box *uncore_pmu_to_box(struct intel_uncore_pmu *pmu, int cpu);
306 struct intel_uncore_box *uncore_event_to_box(struct perf_event *event);
307 u64 uncore_msr_read_counter(struct intel_uncore_box *box, struct perf_event *event);
308 void uncore_pmu_start_hrtimer(struct intel_uncore_box *box);
309 void uncore_pmu_cancel_hrtimer(struct intel_uncore_box *box);
310 void uncore_pmu_event_read(struct perf_event *event);
311 void uncore_perf_event_update(struct intel_uncore_box *box, struct perf_event *event);
312 struct event_constraint *
313 uncore_get_constraint(struct intel_uncore_box *box, struct perf_event *event);
314 void uncore_put_constraint(struct intel_uncore_box *box, struct perf_event *event);
315 u64 uncore_shared_reg_config(struct intel_uncore_box *box, int idx);
316 
317 extern struct intel_uncore_type **uncore_msr_uncores;
318 extern struct intel_uncore_type **uncore_pci_uncores;
319 extern struct pci_driver *uncore_pci_driver;
320 extern int uncore_pcibus_to_physid[256];
321 extern struct pci_dev *uncore_extra_pci_dev[UNCORE_SOCKET_MAX][UNCORE_EXTRA_PCI_DEV_MAX];
322 extern struct event_constraint uncore_constraint_empty;
323 
324 /* perf_event_intel_uncore_snb.c */
325 int snb_uncore_pci_init(void);
326 int ivb_uncore_pci_init(void);
327 int hsw_uncore_pci_init(void);
328 void snb_uncore_cpu_init(void);
329 void nhm_uncore_cpu_init(void);
330 
331 /* perf_event_intel_uncore_snbep.c */
332 int snbep_uncore_pci_init(void);
333 void snbep_uncore_cpu_init(void);
334 int ivbep_uncore_pci_init(void);
335 void ivbep_uncore_cpu_init(void);
336 int hswep_uncore_pci_init(void);
337 void hswep_uncore_cpu_init(void);
338 
339 /* perf_event_intel_uncore_nhmex.c */
340 void nhmex_uncore_cpu_init(void);
341