279 static inline struct perf_cgroup *event_to_cgroup(struct perf_event *event)  in event_to_cgroup()  argument
281 	if (event->attach_state & PERF_ATTACH_TASK)  in event_to_cgroup()
282 		return perf_cgroup_from_task(event->hw.target);  in event_to_cgroup()
284 	return event->cgrp;  in event_to_cgroup()
387 	struct perf_event *event;  in intel_cqm_xchg_rmid()  local
410 	list_for_each_entry(event, head, hw.cqm_group_entry)  in intel_cqm_xchg_rmid()
411 		event->hw.cqm_rmid = rmid;  in intel_cqm_xchg_rmid()
456 	struct perf_event *leader, *event;  in intel_cqm_sched_in_event()  local
462 	event = leader;  in intel_cqm_sched_in_event()
464 	list_for_each_entry_continue(event, &cache_groups,  in intel_cqm_sched_in_event()
466 		if (__rmid_valid(event->hw.cqm_rmid))  in intel_cqm_sched_in_event()
469 		if (__conflict_event(event, leader))  in intel_cqm_sched_in_event()
472 		intel_cqm_xchg_rmid(event, rmid);  in intel_cqm_sched_in_event()
626 static void intel_cqm_sched_out_conflicting_events(struct perf_event *event)  in intel_cqm_sched_out_conflicting_events()  argument
634 		if (group == event)  in intel_cqm_sched_out_conflicting_events()
648 		if (!__conflict_event(group, event))  in intel_cqm_sched_out_conflicting_events()
827 static void intel_cqm_setup_event(struct perf_event *event,  in intel_cqm_setup_event()  argument
837 		if (__match_event(iter, event)) {  in intel_cqm_setup_event()
839 			event->hw.cqm_rmid = rmid;  in intel_cqm_setup_event()
848 		if (__conflict_event(iter, event) && __rmid_valid(rmid))  in intel_cqm_setup_event()
857 	event->hw.cqm_rmid = rmid;  in intel_cqm_setup_event()
860 static void intel_cqm_event_read(struct perf_event *event)  in intel_cqm_event_read()  argument
869 	if (event->cpu == -1)  in intel_cqm_event_read()
873 	rmid = event->hw.cqm_rmid;  in intel_cqm_event_read()
886 	local64_set(&event->count, val);  in intel_cqm_event_read()
904 static inline bool cqm_group_leader(struct perf_event *event)  in cqm_group_leader()  argument
906 	return !list_empty(&event->hw.cqm_groups_entry);  in cqm_group_leader()
909 static u64 intel_cqm_event_count(struct perf_event *event)  in intel_cqm_event_count()  argument
921 	if (event->cpu != -1)  in intel_cqm_event_count()
922 		return __perf_event_count(event);  in intel_cqm_event_count()
933 	if (!cqm_group_leader(event))  in intel_cqm_event_count()
955 	rr.rmid = ACCESS_ONCE(event->hw.cqm_rmid);  in intel_cqm_event_count()
963 	if (event->hw.cqm_rmid == rr.rmid)  in intel_cqm_event_count()
964 		local64_set(&event->count, atomic64_read(&rr.value));  in intel_cqm_event_count()
967 	return __perf_event_count(event);  in intel_cqm_event_count()
970 static void intel_cqm_event_start(struct perf_event *event, int mode)  in intel_cqm_event_start()  argument
973 	unsigned int rmid = event->hw.cqm_rmid;  in intel_cqm_event_start()
976 	if (!(event->hw.cqm_state & PERF_HES_STOPPED))  in intel_cqm_event_start()
979 	event->hw.cqm_state &= ~PERF_HES_STOPPED;  in intel_cqm_event_start()
994 static void intel_cqm_event_stop(struct perf_event *event, int mode)  in intel_cqm_event_stop()  argument
999 	if (event->hw.cqm_state & PERF_HES_STOPPED)  in intel_cqm_event_stop()
1002 	event->hw.cqm_state |= PERF_HES_STOPPED;  in intel_cqm_event_stop()
1005 	intel_cqm_event_read(event);  in intel_cqm_event_stop()
1017 static int intel_cqm_event_add(struct perf_event *event, int mode)  in intel_cqm_event_add()  argument
1024 	event->hw.cqm_state = PERF_HES_STOPPED;  in intel_cqm_event_add()
1025 	rmid = event->hw.cqm_rmid;  in intel_cqm_event_add()
1028 		intel_cqm_event_start(event, mode);  in intel_cqm_event_add()
1035 static void intel_cqm_event_del(struct perf_event *event, int mode)  in intel_cqm_event_del()  argument
1037 	intel_cqm_event_stop(event, mode);  in intel_cqm_event_del()
1040 static void intel_cqm_event_destroy(struct perf_event *event)  in intel_cqm_event_destroy()  argument
1049 	if (!list_empty(&event->hw.cqm_group_entry)) {  in intel_cqm_event_destroy()
1050 		group_other = list_first_entry(&event->hw.cqm_group_entry,  in intel_cqm_event_destroy()
1053 		list_del(&event->hw.cqm_group_entry);  in intel_cqm_event_destroy()
1059 	if (cqm_group_leader(event)) {  in intel_cqm_event_destroy()
1065 			list_replace(&event->hw.cqm_groups_entry,  in intel_cqm_event_destroy()
1068 			unsigned int rmid = event->hw.cqm_rmid;  in intel_cqm_event_destroy()
1072 			list_del(&event->hw.cqm_groups_entry);  in intel_cqm_event_destroy()
1079 static int intel_cqm_event_init(struct perf_event *event)  in intel_cqm_event_init()  argument
1084 	if (event->attr.type != intel_cqm_pmu.type)  in intel_cqm_event_init()
1087 	if (event->attr.config & ~QOS_EVENT_MASK)  in intel_cqm_event_init()
1091 	if (event->attr.exclude_user   ||  in intel_cqm_event_init()
1092 	    event->attr.exclude_kernel ||  in intel_cqm_event_init()
1093 	    event->attr.exclude_hv     ||  in intel_cqm_event_init()
1094 	    event->attr.exclude_idle   ||  in intel_cqm_event_init()
1095 	    event->attr.exclude_host   ||  in intel_cqm_event_init()
1096 	    event->attr.exclude_guest  ||  in intel_cqm_event_init()
1097 	    event->attr.sample_period) /* no sampling */  in intel_cqm_event_init()
1100 	INIT_LIST_HEAD(&event->hw.cqm_group_entry);  in intel_cqm_event_init()
1101 	INIT_LIST_HEAD(&event->hw.cqm_groups_entry);  in intel_cqm_event_init()
1103 	event->destroy = intel_cqm_event_destroy;  in intel_cqm_event_init()
1108 	intel_cqm_setup_event(event, &group);  in intel_cqm_event_init()
1111 		list_add_tail(&event->hw.cqm_group_entry,  in intel_cqm_event_init()
1114 		list_add_tail(&event->hw.cqm_groups_entry,  in intel_cqm_event_init()
1124 		if (!__rmid_valid(event->hw.cqm_rmid))  in intel_cqm_event_init()
1156 PMU_FORMAT_ATTR(event, "config:0-7");