298 static inline struct perf_cgroup *event_to_cgroup(struct perf_event *event)  in event_to_cgroup()  argument
300 	if (event->attach_state & PERF_ATTACH_TASK)  in event_to_cgroup()
301 		return perf_cgroup_from_task(event->hw.target, event->ctx);  in event_to_cgroup()
303 	return event->cgrp;  in event_to_cgroup()
405 	struct perf_event *event;  in intel_cqm_xchg_rmid()  local
428 	list_for_each_entry(event, head, hw.cqm_group_entry)  in intel_cqm_xchg_rmid()
429 		event->hw.cqm_rmid = rmid;  in intel_cqm_xchg_rmid()
474 	struct perf_event *leader, *event;  in intel_cqm_sched_in_event()  local
480 	event = leader;  in intel_cqm_sched_in_event()
482 	list_for_each_entry_continue(event, &cache_groups,  in intel_cqm_sched_in_event()
484 		if (__rmid_valid(event->hw.cqm_rmid))  in intel_cqm_sched_in_event()
487 		if (__conflict_event(event, leader))  in intel_cqm_sched_in_event()
490 		intel_cqm_xchg_rmid(event, rmid);  in intel_cqm_sched_in_event()
644 static void intel_cqm_sched_out_conflicting_events(struct perf_event *event)  in intel_cqm_sched_out_conflicting_events()  argument
652 		if (group == event)  in intel_cqm_sched_out_conflicting_events()
666 		if (!__conflict_event(group, event))  in intel_cqm_sched_out_conflicting_events()
845 static void intel_cqm_setup_event(struct perf_event *event,  in intel_cqm_setup_event()  argument
855 		if (__match_event(iter, event)) {  in intel_cqm_setup_event()
857 			event->hw.cqm_rmid = rmid;  in intel_cqm_setup_event()
866 		if (__conflict_event(iter, event) && __rmid_valid(rmid))  in intel_cqm_setup_event()
875 	event->hw.cqm_rmid = rmid;  in intel_cqm_setup_event()
878 static void intel_cqm_event_read(struct perf_event *event)  in intel_cqm_event_read()  argument
887 	if (event->cpu == -1)  in intel_cqm_event_read()
891 	rmid = event->hw.cqm_rmid;  in intel_cqm_event_read()
904 	local64_set(&event->count, val);  in intel_cqm_event_read()
922 static inline bool cqm_group_leader(struct perf_event *event)  in cqm_group_leader()  argument
924 	return !list_empty(&event->hw.cqm_groups_entry);  in cqm_group_leader()
927 static u64 intel_cqm_event_count(struct perf_event *event)  in intel_cqm_event_count()  argument
939 	if (event->cpu != -1)  in intel_cqm_event_count()
940 		return __perf_event_count(event);  in intel_cqm_event_count()
951 	if (!cqm_group_leader(event))  in intel_cqm_event_count()
973 	rr.rmid = ACCESS_ONCE(event->hw.cqm_rmid);  in intel_cqm_event_count()
981 	if (event->hw.cqm_rmid == rr.rmid)  in intel_cqm_event_count()
982 		local64_set(&event->count, atomic64_read(&rr.value));  in intel_cqm_event_count()
985 	return __perf_event_count(event);  in intel_cqm_event_count()
988 static void intel_cqm_event_start(struct perf_event *event, int mode)  in intel_cqm_event_start()  argument
991 	u32 rmid = event->hw.cqm_rmid;  in intel_cqm_event_start()
993 	if (!(event->hw.cqm_state & PERF_HES_STOPPED))  in intel_cqm_event_start()
996 	event->hw.cqm_state &= ~PERF_HES_STOPPED;  in intel_cqm_event_start()
1009 static void intel_cqm_event_stop(struct perf_event *event, int mode)  in intel_cqm_event_stop()  argument
1013 	if (event->hw.cqm_state & PERF_HES_STOPPED)  in intel_cqm_event_stop()
1016 	event->hw.cqm_state |= PERF_HES_STOPPED;  in intel_cqm_event_stop()
1018 	intel_cqm_event_read(event);  in intel_cqm_event_stop()
1028 static int intel_cqm_event_add(struct perf_event *event, int mode)  in intel_cqm_event_add()  argument
1035 	event->hw.cqm_state = PERF_HES_STOPPED;  in intel_cqm_event_add()
1036 	rmid = event->hw.cqm_rmid;  in intel_cqm_event_add()
1039 		intel_cqm_event_start(event, mode);  in intel_cqm_event_add()
1046 static void intel_cqm_event_destroy(struct perf_event *event)  in intel_cqm_event_destroy()  argument
1055 	if (!list_empty(&event->hw.cqm_group_entry)) {  in intel_cqm_event_destroy()
1056 		group_other = list_first_entry(&event->hw.cqm_group_entry,  in intel_cqm_event_destroy()
1059 		list_del(&event->hw.cqm_group_entry);  in intel_cqm_event_destroy()
1065 	if (cqm_group_leader(event)) {  in intel_cqm_event_destroy()
1071 			list_replace(&event->hw.cqm_groups_entry,  in intel_cqm_event_destroy()
1074 			u32 rmid = event->hw.cqm_rmid;  in intel_cqm_event_destroy()
1078 			list_del(&event->hw.cqm_groups_entry);  in intel_cqm_event_destroy()
1085 static int intel_cqm_event_init(struct perf_event *event)  in intel_cqm_event_init()  argument
1090 	if (event->attr.type != intel_cqm_pmu.type)  in intel_cqm_event_init()
1093 	if (event->attr.config & ~QOS_EVENT_MASK)  in intel_cqm_event_init()
1097 	if (event->attr.exclude_user   ||  in intel_cqm_event_init()
1098 	    event->attr.exclude_kernel ||  in intel_cqm_event_init()
1099 	    event->attr.exclude_hv     ||  in intel_cqm_event_init()
1100 	    event->attr.exclude_idle   ||  in intel_cqm_event_init()
1101 	    event->attr.exclude_host   ||  in intel_cqm_event_init()
1102 	    event->attr.exclude_guest  ||  in intel_cqm_event_init()
1103 	    event->attr.sample_period) /* no sampling */  in intel_cqm_event_init()
1106 	INIT_LIST_HEAD(&event->hw.cqm_group_entry);  in intel_cqm_event_init()
1107 	INIT_LIST_HEAD(&event->hw.cqm_groups_entry);  in intel_cqm_event_init()
1109 	event->destroy = intel_cqm_event_destroy;  in intel_cqm_event_init()
1114 	intel_cqm_setup_event(event, &group);  in intel_cqm_event_init()
1117 		list_add_tail(&event->hw.cqm_group_entry,  in intel_cqm_event_init()
1120 		list_add_tail(&event->hw.cqm_groups_entry,  in intel_cqm_event_init()
1130 		if (!__rmid_valid(event->hw.cqm_rmid))  in intel_cqm_event_init()
1162 PMU_FORMAT_ATTR(event, "config:0-7");