1/*
2 * Copyright 2014 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 */
22
23#include <linux/mutex.h>
24#include <linux/log2.h>
25#include <linux/sched.h>
26#include <linux/slab.h>
27#include <linux/amd-iommu.h>
28#include <linux/notifier.h>
29#include <linux/compat.h>
30
31struct mm_struct;
32
33#include "kfd_priv.h"
34
35/*
36 * Initial size for the array of queues.
37 * The allocated size is doubled each time
38 * it is exceeded up to MAX_PROCESS_QUEUES.
39 */
40#define INITIAL_QUEUE_ARRAY_SIZE 16
41
42/*
43 * List of struct kfd_process (field kfd_process).
44 * Unique/indexed by mm_struct*
45 */
46#define KFD_PROCESS_TABLE_SIZE 5 /* bits: 32 entries */
47static DEFINE_HASHTABLE(kfd_processes_table, KFD_PROCESS_TABLE_SIZE);
48static DEFINE_MUTEX(kfd_processes_mutex);
49
50DEFINE_STATIC_SRCU(kfd_processes_srcu);
51
52static struct workqueue_struct *kfd_process_wq;
53
54struct kfd_process_release_work {
55	struct work_struct kfd_work;
56	struct kfd_process *p;
57};
58
59static struct kfd_process *find_process(const struct task_struct *thread);
60static struct kfd_process *create_process(const struct task_struct *thread);
61
62void kfd_process_create_wq(void)
63{
64	if (!kfd_process_wq)
65		kfd_process_wq = create_workqueue("kfd_process_wq");
66}
67
68void kfd_process_destroy_wq(void)
69{
70	if (kfd_process_wq) {
71		flush_workqueue(kfd_process_wq);
72		destroy_workqueue(kfd_process_wq);
73		kfd_process_wq = NULL;
74	}
75}
76
77struct kfd_process *kfd_create_process(const struct task_struct *thread)
78{
79	struct kfd_process *process;
80
81	BUG_ON(!kfd_process_wq);
82
83	if (thread->mm == NULL)
84		return ERR_PTR(-EINVAL);
85
86	/* Only the pthreads threading model is supported. */
87	if (thread->group_leader->mm != thread->mm)
88		return ERR_PTR(-EINVAL);
89
90	/* Take mmap_sem because we call __mmu_notifier_register inside */
91	down_write(&thread->mm->mmap_sem);
92
93	/*
94	 * take kfd processes mutex before starting of process creation
95	 * so there won't be a case where two threads of the same process
96	 * create two kfd_process structures
97	 */
98	mutex_lock(&kfd_processes_mutex);
99
100	/* A prior open of /dev/kfd could have already created the process. */
101	process = find_process(thread);
102	if (process)
103		pr_debug("kfd: process already found\n");
104
105	if (!process)
106		process = create_process(thread);
107
108	mutex_unlock(&kfd_processes_mutex);
109
110	up_write(&thread->mm->mmap_sem);
111
112	return process;
113}
114
115struct kfd_process *kfd_get_process(const struct task_struct *thread)
116{
117	struct kfd_process *process;
118
119	if (thread->mm == NULL)
120		return ERR_PTR(-EINVAL);
121
122	/* Only the pthreads threading model is supported. */
123	if (thread->group_leader->mm != thread->mm)
124		return ERR_PTR(-EINVAL);
125
126	process = find_process(thread);
127
128	return process;
129}
130
131static struct kfd_process *find_process_by_mm(const struct mm_struct *mm)
132{
133	struct kfd_process *process;
134
135	hash_for_each_possible_rcu(kfd_processes_table, process,
136					kfd_processes, (uintptr_t)mm)
137		if (process->mm == mm)
138			return process;
139
140	return NULL;
141}
142
143static struct kfd_process *find_process(const struct task_struct *thread)
144{
145	struct kfd_process *p;
146	int idx;
147
148	idx = srcu_read_lock(&kfd_processes_srcu);
149	p = find_process_by_mm(thread->mm);
150	srcu_read_unlock(&kfd_processes_srcu, idx);
151
152	return p;
153}
154
155static void kfd_process_wq_release(struct work_struct *work)
156{
157	struct kfd_process_release_work *my_work;
158	struct kfd_process_device *pdd, *temp;
159	struct kfd_process *p;
160
161	my_work = (struct kfd_process_release_work *) work;
162
163	p = my_work->p;
164
165	pr_debug("Releasing process (pasid %d) in workqueue\n",
166			p->pasid);
167
168	mutex_lock(&p->mutex);
169
170	list_for_each_entry_safe(pdd, temp, &p->per_device_data,
171							per_device_list) {
172		pr_debug("Releasing pdd (topology id %d) for process (pasid %d) in workqueue\n",
173				pdd->dev->id, p->pasid);
174
175		amd_iommu_unbind_pasid(pdd->dev->pdev, p->pasid);
176		list_del(&pdd->per_device_list);
177
178		kfree(pdd);
179	}
180
181	kfd_pasid_free(p->pasid);
182
183	mutex_unlock(&p->mutex);
184
185	mutex_destroy(&p->mutex);
186
187	kfree(p->queues);
188
189	kfree(p);
190
191	kfree((void *)work);
192}
193
194static void kfd_process_destroy_delayed(struct rcu_head *rcu)
195{
196	struct kfd_process_release_work *work;
197	struct kfd_process *p;
198
199	BUG_ON(!kfd_process_wq);
200
201	p = container_of(rcu, struct kfd_process, rcu);
202	BUG_ON(atomic_read(&p->mm->mm_count) <= 0);
203
204	mmdrop(p->mm);
205
206	work = (struct kfd_process_release_work *)
207		kmalloc(sizeof(struct kfd_process_release_work), GFP_ATOMIC);
208
209	if (work) {
210		INIT_WORK((struct work_struct *) work, kfd_process_wq_release);
211		work->p = p;
212		queue_work(kfd_process_wq, (struct work_struct *) work);
213	}
214}
215
216static void kfd_process_notifier_release(struct mmu_notifier *mn,
217					struct mm_struct *mm)
218{
219	struct kfd_process *p;
220
221	/*
222	 * The kfd_process structure can not be free because the
223	 * mmu_notifier srcu is read locked
224	 */
225	p = container_of(mn, struct kfd_process, mmu_notifier);
226	BUG_ON(p->mm != mm);
227
228	mutex_lock(&kfd_processes_mutex);
229	hash_del_rcu(&p->kfd_processes);
230	mutex_unlock(&kfd_processes_mutex);
231	synchronize_srcu(&kfd_processes_srcu);
232
233	mutex_lock(&p->mutex);
234
235	/* In case our notifier is called before IOMMU notifier */
236	pqm_uninit(&p->pqm);
237
238	mutex_unlock(&p->mutex);
239
240	/*
241	 * Because we drop mm_count inside kfd_process_destroy_delayed
242	 * and because the mmu_notifier_unregister function also drop
243	 * mm_count we need to take an extra count here.
244	 */
245	atomic_inc(&p->mm->mm_count);
246	mmu_notifier_unregister_no_release(&p->mmu_notifier, p->mm);
247	mmu_notifier_call_srcu(&p->rcu, &kfd_process_destroy_delayed);
248}
249
250static const struct mmu_notifier_ops kfd_process_mmu_notifier_ops = {
251	.release = kfd_process_notifier_release,
252};
253
254static struct kfd_process *create_process(const struct task_struct *thread)
255{
256	struct kfd_process *process;
257	int err = -ENOMEM;
258
259	process = kzalloc(sizeof(*process), GFP_KERNEL);
260
261	if (!process)
262		goto err_alloc_process;
263
264	process->queues = kmalloc_array(INITIAL_QUEUE_ARRAY_SIZE,
265					sizeof(process->queues[0]), GFP_KERNEL);
266	if (!process->queues)
267		goto err_alloc_queues;
268
269	process->pasid = kfd_pasid_alloc();
270	if (process->pasid == 0)
271		goto err_alloc_pasid;
272
273	mutex_init(&process->mutex);
274
275	process->mm = thread->mm;
276
277	/* register notifier */
278	process->mmu_notifier.ops = &kfd_process_mmu_notifier_ops;
279	err = __mmu_notifier_register(&process->mmu_notifier, process->mm);
280	if (err)
281		goto err_mmu_notifier;
282
283	hash_add_rcu(kfd_processes_table, &process->kfd_processes,
284			(uintptr_t)process->mm);
285
286	process->lead_thread = thread->group_leader;
287
288	process->queue_array_size = INITIAL_QUEUE_ARRAY_SIZE;
289
290	INIT_LIST_HEAD(&process->per_device_data);
291
292	err = pqm_init(&process->pqm, process);
293	if (err != 0)
294		goto err_process_pqm_init;
295
296	/* init process apertures*/
297	process->is_32bit_user_mode = is_compat_task();
298	if (kfd_init_apertures(process) != 0)
299		goto err_init_apretures;
300
301	return process;
302
303err_init_apretures:
304	pqm_uninit(&process->pqm);
305err_process_pqm_init:
306	hash_del_rcu(&process->kfd_processes);
307	synchronize_rcu();
308	mmu_notifier_unregister_no_release(&process->mmu_notifier, process->mm);
309err_mmu_notifier:
310	kfd_pasid_free(process->pasid);
311err_alloc_pasid:
312	kfree(process->queues);
313err_alloc_queues:
314	kfree(process);
315err_alloc_process:
316	return ERR_PTR(err);
317}
318
319struct kfd_process_device *kfd_get_process_device_data(struct kfd_dev *dev,
320							struct kfd_process *p)
321{
322	struct kfd_process_device *pdd = NULL;
323
324	list_for_each_entry(pdd, &p->per_device_data, per_device_list)
325		if (pdd->dev == dev)
326			break;
327
328	return pdd;
329}
330
331struct kfd_process_device *kfd_create_process_device_data(struct kfd_dev *dev,
332							struct kfd_process *p)
333{
334	struct kfd_process_device *pdd = NULL;
335
336	pdd = kzalloc(sizeof(*pdd), GFP_KERNEL);
337	if (pdd != NULL) {
338		pdd->dev = dev;
339		INIT_LIST_HEAD(&pdd->qpd.queues_list);
340		INIT_LIST_HEAD(&pdd->qpd.priv_queue_list);
341		pdd->qpd.dqm = dev->dqm;
342		list_add(&pdd->per_device_list, &p->per_device_data);
343	}
344
345	return pdd;
346}
347
348/*
349 * Direct the IOMMU to bind the process (specifically the pasid->mm)
350 * to the device.
351 * Unbinding occurs when the process dies or the device is removed.
352 *
353 * Assumes that the process lock is held.
354 */
355struct kfd_process_device *kfd_bind_process_to_device(struct kfd_dev *dev,
356							struct kfd_process *p)
357{
358	struct kfd_process_device *pdd;
359	int err;
360
361	pdd = kfd_get_process_device_data(dev, p);
362	if (!pdd) {
363		pr_err("Process device data doesn't exist\n");
364		return ERR_PTR(-ENOMEM);
365	}
366
367	if (pdd->bound)
368		return pdd;
369
370	err = amd_iommu_bind_pasid(dev->pdev, p->pasid, p->lead_thread);
371	if (err < 0)
372		return ERR_PTR(err);
373
374	pdd->bound = true;
375
376	return pdd;
377}
378
379void kfd_unbind_process_from_device(struct kfd_dev *dev, unsigned int pasid)
380{
381	struct kfd_process *p;
382	struct kfd_process_device *pdd;
383	int idx, i;
384
385	BUG_ON(dev == NULL);
386
387	idx = srcu_read_lock(&kfd_processes_srcu);
388
389	hash_for_each_rcu(kfd_processes_table, i, p, kfd_processes)
390		if (p->pasid == pasid)
391			break;
392
393	srcu_read_unlock(&kfd_processes_srcu, idx);
394
395	BUG_ON(p->pasid != pasid);
396
397	mutex_lock(&p->mutex);
398
399	pqm_uninit(&p->pqm);
400
401	pdd = kfd_get_process_device_data(dev, p);
402
403	/*
404	 * Just mark pdd as unbound, because we still need it to call
405	 * amd_iommu_unbind_pasid() in when the process exits.
406	 * We don't call amd_iommu_unbind_pasid() here
407	 * because the IOMMU called us.
408	 */
409	if (pdd)
410		pdd->bound = false;
411
412	mutex_unlock(&p->mutex);
413}
414
415struct kfd_process_device *kfd_get_first_process_device_data(struct kfd_process *p)
416{
417	return list_first_entry(&p->per_device_data,
418				struct kfd_process_device,
419				per_device_list);
420}
421
422struct kfd_process_device *kfd_get_next_process_device_data(struct kfd_process *p,
423						struct kfd_process_device *pdd)
424{
425	if (list_is_last(&pdd->per_device_list, &p->per_device_data))
426		return NULL;
427	return list_next_entry(pdd, per_device_list);
428}
429
430bool kfd_has_process_device_data(struct kfd_process *p)
431{
432	return !(list_empty(&p->per_device_data));
433}
434