1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*
3 * shadow.c - Shadow Variables
4 *
5 * Copyright (C) 2014 Josh Poimboeuf <jpoimboe@redhat.com>
6 * Copyright (C) 2014 Seth Jennings <sjenning@redhat.com>
7 * Copyright (C) 2017 Joe Lawrence <joe.lawrence@redhat.com>
8 */
9
10 /**
11 * DOC: Shadow variable API concurrency notes:
12 *
13 * The shadow variable API provides a simple relationship between an
14 * <obj, id> pair and a pointer value. It is the responsibility of the
15 * caller to provide any mutual exclusion required of the shadow data.
16 *
17 * Once a shadow variable is attached to its parent object via the
18 * klp_shadow_*alloc() API calls, it is considered live: any subsequent
19 * call to klp_shadow_get() may then return the shadow variable's data
20 * pointer. Callers of klp_shadow_*alloc() should prepare shadow data
21 * accordingly.
22 *
23 * The klp_shadow_*alloc() API calls may allocate memory for new shadow
24 * variable structures. Their implementation does not call kmalloc
25 * inside any spinlocks, but API callers should pass GFP flags according
26 * to their specific needs.
27 *
28 * The klp_shadow_hash is an RCU-enabled hashtable and is safe against
29 * concurrent klp_shadow_free() and klp_shadow_get() operations.
30 */
31
32 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
33
34 #include <linux/hashtable.h>
35 #include <linux/slab.h>
36 #include <linux/livepatch.h>
37
38 static DEFINE_HASHTABLE(klp_shadow_hash, 12);
39
40 /*
41 * klp_shadow_lock provides exclusive access to the klp_shadow_hash and
42 * the shadow variables it references.
43 */
44 static DEFINE_SPINLOCK(klp_shadow_lock);
45
46 /**
47 * struct klp_shadow - shadow variable structure
48 * @node: klp_shadow_hash hash table node
49 * @rcu_head: RCU is used to safely free this structure
50 * @obj: pointer to parent object
51 * @id: data identifier
52 * @data: data area
53 */
54 struct klp_shadow {
55 struct hlist_node node;
56 struct rcu_head rcu_head;
57 void *obj;
58 unsigned long id;
59 char data[];
60 };
61
62 /**
63 * klp_shadow_match() - verify a shadow variable matches given <obj, id>
64 * @shadow: shadow variable to match
65 * @obj: pointer to parent object
66 * @id: data identifier
67 *
68 * Return: true if the shadow variable matches.
69 */
70 static inline bool klp_shadow_match(struct klp_shadow *shadow, void *obj,
71 unsigned long id)
72 {
73 return shadow->obj == obj && shadow->id == id;
74 }
75
76 /**
77 * klp_shadow_get() - retrieve a shadow variable data pointer
78 * @obj: pointer to parent object
79 * @id: data identifier
80 *
81 * Return: the shadow variable data element, NULL on failure.
82 */
83 void *klp_shadow_get(void *obj, unsigned long id)
84 {
85 struct klp_shadow *shadow;
86
87 rcu_read_lock();
88
89 hash_for_each_possible_rcu(klp_shadow_hash, shadow, node,
90 (unsigned long)obj) {
91
92 if (klp_shadow_match(shadow, obj, id)) {
93 rcu_read_unlock();
94 return shadow->data;
95 }
96 }
97
98 rcu_read_unlock();
99
100 return NULL;
101 }
102 EXPORT_SYMBOL_GPL(klp_shadow_get);
103
104 static void *__klp_shadow_get_or_alloc(void *obj, unsigned long id,
105 size_t size, gfp_t gfp_flags,
106 klp_shadow_ctor_t ctor, void *ctor_data,
107 bool warn_on_exist)
108 {
109 struct klp_shadow *new_shadow;
110 void *shadow_data;
111 unsigned long flags;
112
113 /* Check if the shadow variable already exists */
114 shadow_data = klp_shadow_get(obj, id);
115 if (shadow_data)
116 goto exists;
117
118 /*
119 * Allocate a new shadow variable. Fill it with zeroes by default.
120 * More complex setting can be done by @ctor function. But it is
121 * called only when the buffer is really used (under klp_shadow_lock).
122 */
123 new_shadow = kzalloc(size + sizeof(*new_shadow), gfp_flags);
124 if (!new_shadow)
125 return NULL;
126
127 /* Look for <obj, id> again under the lock */
128 spin_lock_irqsave(&klp_shadow_lock, flags);
129 shadow_data = klp_shadow_get(obj, id);
130 if (unlikely(shadow_data)) {
131 /*
132 * Shadow variable was found, throw away speculative
133 * allocation.
134 */
135 spin_unlock_irqrestore(&klp_shadow_lock, flags);
136 kfree(new_shadow);
137 goto exists;
138 }
139
140 new_shadow->obj = obj;
141 new_shadow->id = id;
142
143 if (ctor) {
144 int err;
145
146 err = ctor(obj, new_shadow->data, ctor_data);
147 if (err) {
148 spin_unlock_irqrestore(&klp_shadow_lock, flags);
149 kfree(new_shadow);
150 pr_err("Failed to construct shadow variable <%p, %lx> (%d)\n",
151 obj, id, err);
152 return NULL;
153 }
154 }
155
156 /* No <obj, id> found, so attach the newly allocated one */
157 hash_add_rcu(klp_shadow_hash, &new_shadow->node,
158 (unsigned long)new_shadow->obj);
159 spin_unlock_irqrestore(&klp_shadow_lock, flags);
160
161 return new_shadow->data;
162
163 exists:
164 if (warn_on_exist) {
165 WARN(1, "Duplicate shadow variable <%p, %lx>\n", obj, id);
166 return NULL;
167 }
168
169 return shadow_data;
170 }
171
172 /**
173 * klp_shadow_alloc() - allocate and add a new shadow variable
174 * @obj: pointer to parent object
175 * @id: data identifier
176 * @size: size of attached data
177 * @gfp_flags: GFP mask for allocation
178 * @ctor: custom constructor to initialize the shadow data (optional)
179 * @ctor_data: pointer to any data needed by @ctor (optional)
180 *
181 * Allocates @size bytes for new shadow variable data using @gfp_flags.
182 * The data are zeroed by default. They are further initialized by @ctor
183 * function if it is not NULL. The new shadow variable is then added
184 * to the global hashtable.
185 *
186 * If an existing <obj, id> shadow variable can be found, this routine will
187 * issue a WARN, exit early and return NULL.
188 *
189 * This function guarantees that the constructor function is called only when
190 * the variable did not exist before. The cost is that @ctor is called
191 * in atomic context under a spin lock.
192 *
193 * Return: the shadow variable data element, NULL on duplicate or
194 * failure.
195 */
196 void *klp_shadow_alloc(void *obj, unsigned long id,
197 size_t size, gfp_t gfp_flags,
198 klp_shadow_ctor_t ctor, void *ctor_data)
199 {
200 return __klp_shadow_get_or_alloc(obj, id, size, gfp_flags,
201 ctor, ctor_data, true);
202 }
203 EXPORT_SYMBOL_GPL(klp_shadow_alloc);
204
205 /**
206 * klp_shadow_get_or_alloc() - get existing or allocate a new shadow variable
207 * @obj: pointer to parent object
208 * @id: data identifier
209 * @size: size of attached data
210 * @gfp_flags: GFP mask for allocation
211 * @ctor: custom constructor to initialize the shadow data (optional)
212 * @ctor_data: pointer to any data needed by @ctor (optional)
213 *
214 * Returns a pointer to existing shadow data if an <obj, id> shadow
215 * variable is already present. Otherwise, it creates a new shadow
216 * variable like klp_shadow_alloc().
217 *
218 * This function guarantees that only one shadow variable exists with the given
219 * @id for the given @obj. It also guarantees that the constructor function
220 * will be called only when the variable did not exist before. The cost is
221 * that @ctor is called in atomic context under a spin lock.
222 *
223 * Return: the shadow variable data element, NULL on failure.
224 */
225 void *klp_shadow_get_or_alloc(void *obj, unsigned long id,
226 size_t size, gfp_t gfp_flags,
227 klp_shadow_ctor_t ctor, void *ctor_data)
228 {
229 return __klp_shadow_get_or_alloc(obj, id, size, gfp_flags,
230 ctor, ctor_data, false);
231 }
232 EXPORT_SYMBOL_GPL(klp_shadow_get_or_alloc);
233
234 static void klp_shadow_free_struct(struct klp_shadow *shadow,
235 klp_shadow_dtor_t dtor)
236 {
237 hash_del_rcu(&shadow->node);
238 if (dtor)
239 dtor(shadow->obj, shadow->data);
240 kfree_rcu(shadow, rcu_head);
241 }
242
243 /**
244 * klp_shadow_free() - detach and free a <obj, id> shadow variable
245 * @obj: pointer to parent object
246 * @id: data identifier
247 * @dtor: custom callback that can be used to unregister the variable
248 * and/or free data that the shadow variable points to (optional)
249 *
250 * This function releases the memory for this <obj, id> shadow variable
251 * instance, callers should stop referencing it accordingly.
252 */
253 void klp_shadow_free(void *obj, unsigned long id, klp_shadow_dtor_t dtor)
254 {
255 struct klp_shadow *shadow;
256 unsigned long flags;
257
258 spin_lock_irqsave(&klp_shadow_lock, flags);
259
260 /* Delete <obj, id> from hash */
261 hash_for_each_possible(klp_shadow_hash, shadow, node,
262 (unsigned long)obj) {
263
264 if (klp_shadow_match(shadow, obj, id)) {
265 klp_shadow_free_struct(shadow, dtor);
266 break;
267 }
268 }
269
270 spin_unlock_irqrestore(&klp_shadow_lock, flags);
271 }
272 EXPORT_SYMBOL_GPL(klp_shadow_free);
273
274 /**
275 * klp_shadow_free_all() - detach and free all <*, id> shadow variables
276 * @id: data identifier
277 * @dtor: custom callback that can be used to unregister the variable
278 * and/or free data that the shadow variable points to (optional)
279 *
280 * This function releases the memory for all <*, id> shadow variable
281 * instances, callers should stop referencing them accordingly.
282 */
283 void klp_shadow_free_all(unsigned long id, klp_shadow_dtor_t dtor)
284 {
285 struct klp_shadow *shadow;
286 unsigned long flags;
287 int i;
288
289 spin_lock_irqsave(&klp_shadow_lock, flags);
290
291 /* Delete all <*, id> from hash */
292 hash_for_each(klp_shadow_hash, i, shadow, node) {
293 if (klp_shadow_match(shadow, shadow->obj, id))
294 klp_shadow_free_struct(shadow, dtor);
295 }
296
297 spin_unlock_irqrestore(&klp_shadow_lock, flags);
298 }
299 EXPORT_SYMBOL_GPL(klp_shadow_free_all);