This source file includes following definitions.
- context_tracking_recursion_enter
- context_tracking_recursion_exit
- __context_tracking_enter
- context_tracking_enter
- context_tracking_user_enter
- __context_tracking_exit
- context_tracking_exit
- context_tracking_user_exit
- context_tracking_cpu_set
- context_tracking_init
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 #include <linux/context_tracking.h>
19 #include <linux/rcupdate.h>
20 #include <linux/sched.h>
21 #include <linux/hardirq.h>
22 #include <linux/export.h>
23 #include <linux/kprobes.h>
24
25 #define CREATE_TRACE_POINTS
26 #include <trace/events/context_tracking.h>
27
28 DEFINE_STATIC_KEY_FALSE(context_tracking_enabled);
29 EXPORT_SYMBOL_GPL(context_tracking_enabled);
30
31 DEFINE_PER_CPU(struct context_tracking, context_tracking);
32 EXPORT_SYMBOL_GPL(context_tracking);
33
34 static bool context_tracking_recursion_enter(void)
35 {
36 int recursion;
37
38 recursion = __this_cpu_inc_return(context_tracking.recursion);
39 if (recursion == 1)
40 return true;
41
42 WARN_ONCE((recursion < 1), "Invalid context tracking recursion value %d\n", recursion);
43 __this_cpu_dec(context_tracking.recursion);
44
45 return false;
46 }
47
48 static void context_tracking_recursion_exit(void)
49 {
50 __this_cpu_dec(context_tracking.recursion);
51 }
52
53
54
55
56
57
58
59
60
61
62 void __context_tracking_enter(enum ctx_state state)
63 {
64
65 WARN_ON_ONCE(!current->mm);
66
67 if (!context_tracking_recursion_enter())
68 return;
69
70 if ( __this_cpu_read(context_tracking.state) != state) {
71 if (__this_cpu_read(context_tracking.active)) {
72
73
74
75
76
77
78
79 if (state == CONTEXT_USER) {
80 trace_user_enter(0);
81 vtime_user_enter(current);
82 }
83 rcu_user_enter();
84 }
85
86
87
88
89
90
91
92
93
94
95
96
97
98 __this_cpu_write(context_tracking.state, state);
99 }
100 context_tracking_recursion_exit();
101 }
102 NOKPROBE_SYMBOL(__context_tracking_enter);
103 EXPORT_SYMBOL_GPL(__context_tracking_enter);
104
105 void context_tracking_enter(enum ctx_state state)
106 {
107 unsigned long flags;
108
109
110
111
112
113
114
115
116
117 if (in_interrupt())
118 return;
119
120 local_irq_save(flags);
121 __context_tracking_enter(state);
122 local_irq_restore(flags);
123 }
124 NOKPROBE_SYMBOL(context_tracking_enter);
125 EXPORT_SYMBOL_GPL(context_tracking_enter);
126
127 void context_tracking_user_enter(void)
128 {
129 user_enter();
130 }
131 NOKPROBE_SYMBOL(context_tracking_user_enter);
132
133
134
135
136
137
138
139
140
141
142
143
144
145 void __context_tracking_exit(enum ctx_state state)
146 {
147 if (!context_tracking_recursion_enter())
148 return;
149
150 if (__this_cpu_read(context_tracking.state) == state) {
151 if (__this_cpu_read(context_tracking.active)) {
152
153
154
155
156 rcu_user_exit();
157 if (state == CONTEXT_USER) {
158 vtime_user_exit(current);
159 trace_user_exit(0);
160 }
161 }
162 __this_cpu_write(context_tracking.state, CONTEXT_KERNEL);
163 }
164 context_tracking_recursion_exit();
165 }
166 NOKPROBE_SYMBOL(__context_tracking_exit);
167 EXPORT_SYMBOL_GPL(__context_tracking_exit);
168
169 void context_tracking_exit(enum ctx_state state)
170 {
171 unsigned long flags;
172
173 if (in_interrupt())
174 return;
175
176 local_irq_save(flags);
177 __context_tracking_exit(state);
178 local_irq_restore(flags);
179 }
180 NOKPROBE_SYMBOL(context_tracking_exit);
181 EXPORT_SYMBOL_GPL(context_tracking_exit);
182
183 void context_tracking_user_exit(void)
184 {
185 user_exit();
186 }
187 NOKPROBE_SYMBOL(context_tracking_user_exit);
188
189 void __init context_tracking_cpu_set(int cpu)
190 {
191 static __initdata bool initialized = false;
192
193 if (!per_cpu(context_tracking.active, cpu)) {
194 per_cpu(context_tracking.active, cpu) = true;
195 static_branch_inc(&context_tracking_enabled);
196 }
197
198 if (initialized)
199 return;
200
201
202
203
204
205 set_tsk_thread_flag(&init_task, TIF_NOHZ);
206 WARN_ON_ONCE(!tasklist_empty());
207
208 initialized = true;
209 }
210
211 #ifdef CONFIG_CONTEXT_TRACKING_FORCE
212 void __init context_tracking_init(void)
213 {
214 int cpu;
215
216 for_each_possible_cpu(cpu)
217 context_tracking_cpu_set(cpu);
218 }
219 #endif