1 #ifndef _LINUX_PSI_TYPES_H
2 #define _LINUX_PSI_TYPES_H
3
4 #include <linux/kthread.h>
5 #include <linux/seqlock.h>
6 #include <linux/types.h>
7 #include <linux/kref.h>
8 #include <linux/wait.h>
9
10 #ifdef CONFIG_PSI
11
12 /* Tracked task states */
13 enum psi_task_count {
14 NR_IOWAIT,
15 NR_MEMSTALL,
16 NR_RUNNING,
17 NR_PSI_TASK_COUNTS = 3,
18 };
19
20 /* Task state bitmasks */
21 #define TSK_IOWAIT (1 << NR_IOWAIT)
22 #define TSK_MEMSTALL (1 << NR_MEMSTALL)
23 #define TSK_RUNNING (1 << NR_RUNNING)
24
25 /* Resources that workloads could be stalled on */
26 enum psi_res {
27 PSI_IO,
28 PSI_MEM,
29 PSI_CPU,
30 NR_PSI_RESOURCES = 3,
31 };
32
33 /*
34 * Pressure states for each resource:
35 *
36 * SOME: Stalled tasks & working tasks
37 * FULL: Stalled tasks & no working tasks
38 */
39 enum psi_states {
40 PSI_IO_SOME,
41 PSI_IO_FULL,
42 PSI_MEM_SOME,
43 PSI_MEM_FULL,
44 PSI_CPU_SOME,
45 /* Only per-CPU, to weigh the CPU in the global average: */
46 PSI_NONIDLE,
47 NR_PSI_STATES = 6,
48 };
49
50 enum psi_aggregators {
51 PSI_AVGS = 0,
52 PSI_POLL,
53 NR_PSI_AGGREGATORS,
54 };
55
56 struct psi_group_cpu {
57 /* 1st cacheline updated by the scheduler */
58
59 /* Aggregator needs to know of concurrent changes */
60 seqcount_t seq ____cacheline_aligned_in_smp;
61
62 /* States of the tasks belonging to this group */
63 unsigned int tasks[NR_PSI_TASK_COUNTS];
64
65 /* Aggregate pressure state derived from the tasks */
66 u32 state_mask;
67
68 /* Period time sampling buckets for each state of interest (ns) */
69 u32 times[NR_PSI_STATES];
70
71 /* Time of last task change in this group (rq_clock) */
72 u64 state_start;
73
74 /* 2nd cacheline updated by the aggregator */
75
76 /* Delta detection against the sampling buckets */
77 u32 times_prev[NR_PSI_AGGREGATORS][NR_PSI_STATES]
78 ____cacheline_aligned_in_smp;
79 };
80
81 /* PSI growth tracking window */
82 struct psi_window {
83 /* Window size in ns */
84 u64 size;
85
86 /* Start time of the current window in ns */
87 u64 start_time;
88
89 /* Value at the start of the window */
90 u64 start_value;
91
92 /* Value growth in the previous window */
93 u64 prev_growth;
94 };
95
96 struct psi_trigger {
97 /* PSI state being monitored by the trigger */
98 enum psi_states state;
99
100 /* User-spacified threshold in ns */
101 u64 threshold;
102
103 /* List node inside triggers list */
104 struct list_head node;
105
106 /* Backpointer needed during trigger destruction */
107 struct psi_group *group;
108
109 /* Wait queue for polling */
110 wait_queue_head_t event_wait;
111
112 /* Pending event flag */
113 int event;
114
115 /* Tracking window */
116 struct psi_window win;
117
118 /*
119 * Time last event was generated. Used for rate-limiting
120 * events to one per window
121 */
122 u64 last_event_time;
123
124 /* Refcounting to prevent premature destruction */
125 struct kref refcount;
126 };
127
128 struct psi_group {
129 /* Protects data used by the aggregator */
130 struct mutex avgs_lock;
131
132 /* Per-cpu task state & time tracking */
133 struct psi_group_cpu __percpu *pcpu;
134
135 /* Running pressure averages */
136 u64 avg_total[NR_PSI_STATES - 1];
137 u64 avg_last_update;
138 u64 avg_next_update;
139
140 /* Aggregator work control */
141 struct delayed_work avgs_work;
142
143 /* Total stall times and sampled pressure averages */
144 u64 total[NR_PSI_AGGREGATORS][NR_PSI_STATES - 1];
145 unsigned long avg[NR_PSI_STATES - 1][3];
146
147 /* Monitor work control */
148 atomic_t poll_scheduled;
149 struct kthread_worker __rcu *poll_kworker;
150 struct kthread_delayed_work poll_work;
151
152 /* Protects data used by the monitor */
153 struct mutex trigger_lock;
154
155 /* Configured polling triggers */
156 struct list_head triggers;
157 u32 nr_triggers[NR_PSI_STATES - 1];
158 u32 poll_states;
159 u64 poll_min_period;
160
161 /* Total stall times at the start of monitor activation */
162 u64 polling_total[NR_PSI_STATES - 1];
163 u64 polling_next_update;
164 u64 polling_until;
165 };
166
167 #else /* CONFIG_PSI */
168
169 struct psi_group { };
170
171 #endif /* CONFIG_PSI */
172
173 #endif /* _LINUX_PSI_TYPES_H */