This source file includes following definitions.
- get_amdgpu_device
- check_if_add_bo_to_vm
- amdgpu_amdkfd_gpuvm_init_mem_limits
- amdgpu_amdkfd_reserve_mem_limit
- unreserve_mem_limit
- amdgpu_amdkfd_unreserve_memory_limit
- amdgpu_amdkfd_remove_eviction_fence
- amdgpu_amdkfd_bo_validate
- amdgpu_amdkfd_validate
- vm_validate_pt_pd_bos
- vm_update_pds
- add_bo_to_vm
- remove_bo_from_vm
- add_kgd_mem_to_kfd_bo_list
- remove_kgd_mem_from_kfd_bo_list
- init_user_pages
- reserve_bo_and_vm
- reserve_bo_and_cond_vms
- unreserve_bo_and_vms
- unmap_bo_from_gpuvm
- update_gpuvm_pte
- map_bo_to_gpuvm
- create_doorbell_sg
- process_validate_vms
- process_sync_pds_resv
- process_update_pds
- init_kfd_vm
- amdgpu_amdkfd_gpuvm_create_process_vm
- amdgpu_amdkfd_gpuvm_acquire_process_vm
- amdgpu_amdkfd_gpuvm_destroy_cb
- amdgpu_amdkfd_gpuvm_destroy_process_vm
- amdgpu_amdkfd_gpuvm_release_process_vm
- amdgpu_amdkfd_gpuvm_get_process_page_dir
- amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu
- amdgpu_amdkfd_gpuvm_free_memory_of_gpu
- amdgpu_amdkfd_gpuvm_map_memory_to_gpu
- amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu
- amdgpu_amdkfd_gpuvm_sync_memory
- amdgpu_amdkfd_gpuvm_map_gtt_bo_to_kernel
- amdgpu_amdkfd_gpuvm_get_vm_fault_info
- amdgpu_amdkfd_gpuvm_import_dmabuf
- amdgpu_amdkfd_evict_userptr
- update_invalid_user_pages
- validate_invalid_user_pages
- amdgpu_amdkfd_restore_userptr_worker
- amdgpu_amdkfd_gpuvm_restore_process_bos
- amdgpu_amdkfd_add_gws_to_process
- amdgpu_amdkfd_remove_gws_from_process
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23 #define pr_fmt(fmt) "kfd2kgd: " fmt
24
25 #include <linux/dma-buf.h>
26 #include <linux/list.h>
27 #include <linux/pagemap.h>
28 #include <linux/sched/mm.h>
29 #include <linux/sched/task.h>
30
31 #include "amdgpu_object.h"
32 #include "amdgpu_vm.h"
33 #include "amdgpu_amdkfd.h"
34 #include "amdgpu_dma_buf.h"
35
36
37
38
39 #define VI_BO_SIZE_ALIGN (0x8000)
40
41
42 #define AMDGPU_AMDKFD_USERPTR_BO (1ULL << 63)
43
44
45
46
47 #define AMDGPU_USERPTR_RESTORE_DELAY_MS 1
48
49
50 static struct {
51 uint64_t max_system_mem_limit;
52 uint64_t max_ttm_mem_limit;
53 int64_t system_mem_used;
54 int64_t ttm_mem_used;
55 spinlock_t mem_limit_lock;
56 } kfd_mem_limit;
57
58
59 struct amdgpu_vm_parser {
60 uint32_t domain;
61 bool wait;
62 };
63
64 static const char * const domain_bit_to_string[] = {
65 "CPU",
66 "GTT",
67 "VRAM",
68 "GDS",
69 "GWS",
70 "OA"
71 };
72
73 #define domain_string(domain) domain_bit_to_string[ffs(domain)-1]
74
75 static void amdgpu_amdkfd_restore_userptr_worker(struct work_struct *work);
76
77
78 static inline struct amdgpu_device *get_amdgpu_device(struct kgd_dev *kgd)
79 {
80 return (struct amdgpu_device *)kgd;
81 }
82
83 static bool check_if_add_bo_to_vm(struct amdgpu_vm *avm,
84 struct kgd_mem *mem)
85 {
86 struct kfd_bo_va_list *entry;
87
88 list_for_each_entry(entry, &mem->bo_va_list, bo_list)
89 if (entry->bo_va->base.vm == avm)
90 return false;
91
92 return true;
93 }
94
95
96
97
98
99 void amdgpu_amdkfd_gpuvm_init_mem_limits(void)
100 {
101 struct sysinfo si;
102 uint64_t mem;
103
104 si_meminfo(&si);
105 mem = si.totalram - si.totalhigh;
106 mem *= si.mem_unit;
107
108 spin_lock_init(&kfd_mem_limit.mem_limit_lock);
109 kfd_mem_limit.max_system_mem_limit = (mem >> 1) + (mem >> 2);
110 kfd_mem_limit.max_ttm_mem_limit = (mem >> 1) - (mem >> 3);
111 pr_debug("Kernel memory limit %lluM, TTM limit %lluM\n",
112 (kfd_mem_limit.max_system_mem_limit >> 20),
113 (kfd_mem_limit.max_ttm_mem_limit >> 20));
114 }
115
116 static int amdgpu_amdkfd_reserve_mem_limit(struct amdgpu_device *adev,
117 uint64_t size, u32 domain, bool sg)
118 {
119 size_t acc_size, system_mem_needed, ttm_mem_needed, vram_needed;
120 uint64_t reserved_for_pt = amdgpu_amdkfd_total_mem_size >> 9;
121 int ret = 0;
122
123 acc_size = ttm_bo_dma_acc_size(&adev->mman.bdev, size,
124 sizeof(struct amdgpu_bo));
125
126 vram_needed = 0;
127 if (domain == AMDGPU_GEM_DOMAIN_GTT) {
128
129 system_mem_needed = acc_size + size;
130 ttm_mem_needed = acc_size + size;
131 } else if (domain == AMDGPU_GEM_DOMAIN_CPU && !sg) {
132
133 system_mem_needed = acc_size + size;
134 ttm_mem_needed = acc_size;
135 } else {
136
137 system_mem_needed = acc_size;
138 ttm_mem_needed = acc_size;
139 if (domain == AMDGPU_GEM_DOMAIN_VRAM)
140 vram_needed = size;
141 }
142
143 spin_lock(&kfd_mem_limit.mem_limit_lock);
144
145 if ((kfd_mem_limit.system_mem_used + system_mem_needed >
146 kfd_mem_limit.max_system_mem_limit) ||
147 (kfd_mem_limit.ttm_mem_used + ttm_mem_needed >
148 kfd_mem_limit.max_ttm_mem_limit) ||
149 (adev->kfd.vram_used + vram_needed >
150 adev->gmc.real_vram_size - reserved_for_pt)) {
151 ret = -ENOMEM;
152 } else {
153 kfd_mem_limit.system_mem_used += system_mem_needed;
154 kfd_mem_limit.ttm_mem_used += ttm_mem_needed;
155 adev->kfd.vram_used += vram_needed;
156 }
157
158 spin_unlock(&kfd_mem_limit.mem_limit_lock);
159 return ret;
160 }
161
162 static void unreserve_mem_limit(struct amdgpu_device *adev,
163 uint64_t size, u32 domain, bool sg)
164 {
165 size_t acc_size;
166
167 acc_size = ttm_bo_dma_acc_size(&adev->mman.bdev, size,
168 sizeof(struct amdgpu_bo));
169
170 spin_lock(&kfd_mem_limit.mem_limit_lock);
171 if (domain == AMDGPU_GEM_DOMAIN_GTT) {
172 kfd_mem_limit.system_mem_used -= (acc_size + size);
173 kfd_mem_limit.ttm_mem_used -= (acc_size + size);
174 } else if (domain == AMDGPU_GEM_DOMAIN_CPU && !sg) {
175 kfd_mem_limit.system_mem_used -= (acc_size + size);
176 kfd_mem_limit.ttm_mem_used -= acc_size;
177 } else {
178 kfd_mem_limit.system_mem_used -= acc_size;
179 kfd_mem_limit.ttm_mem_used -= acc_size;
180 if (domain == AMDGPU_GEM_DOMAIN_VRAM) {
181 adev->kfd.vram_used -= size;
182 WARN_ONCE(adev->kfd.vram_used < 0,
183 "kfd VRAM memory accounting unbalanced");
184 }
185 }
186 WARN_ONCE(kfd_mem_limit.system_mem_used < 0,
187 "kfd system memory accounting unbalanced");
188 WARN_ONCE(kfd_mem_limit.ttm_mem_used < 0,
189 "kfd TTM memory accounting unbalanced");
190
191 spin_unlock(&kfd_mem_limit.mem_limit_lock);
192 }
193
194 void amdgpu_amdkfd_unreserve_memory_limit(struct amdgpu_bo *bo)
195 {
196 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);
197 u32 domain = bo->preferred_domains;
198 bool sg = (bo->preferred_domains == AMDGPU_GEM_DOMAIN_CPU);
199
200 if (bo->flags & AMDGPU_AMDKFD_USERPTR_BO) {
201 domain = AMDGPU_GEM_DOMAIN_CPU;
202 sg = false;
203 }
204
205 unreserve_mem_limit(adev, amdgpu_bo_size(bo), domain, sg);
206 }
207
208
209
210
211
212
213
214
215
216
217
218 static int amdgpu_amdkfd_remove_eviction_fence(struct amdgpu_bo *bo,
219 struct amdgpu_amdkfd_fence *ef)
220 {
221 struct dma_resv *resv = bo->tbo.base.resv;
222 struct dma_resv_list *old, *new;
223 unsigned int i, j, k;
224
225 if (!ef)
226 return -EINVAL;
227
228 old = dma_resv_get_list(resv);
229 if (!old)
230 return 0;
231
232 new = kmalloc(offsetof(typeof(*new), shared[old->shared_max]),
233 GFP_KERNEL);
234 if (!new)
235 return -ENOMEM;
236
237
238
239
240 for (i = 0, j = old->shared_count, k = 0; i < old->shared_count; ++i) {
241 struct dma_fence *f;
242
243 f = rcu_dereference_protected(old->shared[i],
244 dma_resv_held(resv));
245
246 if (f->context == ef->base.context)
247 RCU_INIT_POINTER(new->shared[--j], f);
248 else
249 RCU_INIT_POINTER(new->shared[k++], f);
250 }
251 new->shared_max = old->shared_max;
252 new->shared_count = k;
253
254
255 preempt_disable();
256 write_seqcount_begin(&resv->seq);
257 RCU_INIT_POINTER(resv->fence, new);
258 write_seqcount_end(&resv->seq);
259 preempt_enable();
260
261
262 for (i = j, k = 0; i < old->shared_count; ++i) {
263 struct dma_fence *f;
264
265 f = rcu_dereference_protected(new->shared[i],
266 dma_resv_held(resv));
267 dma_fence_put(f);
268 }
269 kfree_rcu(old, rcu);
270
271 return 0;
272 }
273
274 static int amdgpu_amdkfd_bo_validate(struct amdgpu_bo *bo, uint32_t domain,
275 bool wait)
276 {
277 struct ttm_operation_ctx ctx = { false, false };
278 int ret;
279
280 if (WARN(amdgpu_ttm_tt_get_usermm(bo->tbo.ttm),
281 "Called with userptr BO"))
282 return -EINVAL;
283
284 amdgpu_bo_placement_from_domain(bo, domain);
285
286 ret = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
287 if (ret)
288 goto validate_fail;
289 if (wait)
290 amdgpu_bo_sync_wait(bo, AMDGPU_FENCE_OWNER_KFD, false);
291
292 validate_fail:
293 return ret;
294 }
295
296 static int amdgpu_amdkfd_validate(void *param, struct amdgpu_bo *bo)
297 {
298 struct amdgpu_vm_parser *p = param;
299
300 return amdgpu_amdkfd_bo_validate(bo, p->domain, p->wait);
301 }
302
303
304
305
306
307
308
309
310 static int vm_validate_pt_pd_bos(struct amdgpu_vm *vm)
311 {
312 struct amdgpu_bo *pd = vm->root.base.bo;
313 struct amdgpu_device *adev = amdgpu_ttm_adev(pd->tbo.bdev);
314 struct amdgpu_vm_parser param;
315 int ret;
316
317 param.domain = AMDGPU_GEM_DOMAIN_VRAM;
318 param.wait = false;
319
320 ret = amdgpu_vm_validate_pt_bos(adev, vm, amdgpu_amdkfd_validate,
321 ¶m);
322 if (ret) {
323 pr_err("amdgpu: failed to validate PT BOs\n");
324 return ret;
325 }
326
327 ret = amdgpu_amdkfd_validate(¶m, pd);
328 if (ret) {
329 pr_err("amdgpu: failed to validate PD\n");
330 return ret;
331 }
332
333 vm->pd_phys_addr = amdgpu_gmc_pd_addr(vm->root.base.bo);
334
335 if (vm->use_cpu_for_update) {
336 ret = amdgpu_bo_kmap(pd, NULL);
337 if (ret) {
338 pr_err("amdgpu: failed to kmap PD, ret=%d\n", ret);
339 return ret;
340 }
341 }
342
343 return 0;
344 }
345
346 static int vm_update_pds(struct amdgpu_vm *vm, struct amdgpu_sync *sync)
347 {
348 struct amdgpu_bo *pd = vm->root.base.bo;
349 struct amdgpu_device *adev = amdgpu_ttm_adev(pd->tbo.bdev);
350 int ret;
351
352 ret = amdgpu_vm_update_directories(adev, vm);
353 if (ret)
354 return ret;
355
356 return amdgpu_sync_fence(NULL, sync, vm->last_update, false);
357 }
358
359
360
361
362
363
364
365
366
367
368
369
370
371 static int add_bo_to_vm(struct amdgpu_device *adev, struct kgd_mem *mem,
372 struct amdgpu_vm *vm, bool is_aql,
373 struct kfd_bo_va_list **p_bo_va_entry)
374 {
375 int ret;
376 struct kfd_bo_va_list *bo_va_entry;
377 struct amdgpu_bo *bo = mem->bo;
378 uint64_t va = mem->va;
379 struct list_head *list_bo_va = &mem->bo_va_list;
380 unsigned long bo_size = bo->tbo.mem.size;
381
382 if (!va) {
383 pr_err("Invalid VA when adding BO to VM\n");
384 return -EINVAL;
385 }
386
387 if (is_aql)
388 va += bo_size;
389
390 bo_va_entry = kzalloc(sizeof(*bo_va_entry), GFP_KERNEL);
391 if (!bo_va_entry)
392 return -ENOMEM;
393
394 pr_debug("\t add VA 0x%llx - 0x%llx to vm %p\n", va,
395 va + bo_size, vm);
396
397
398 bo_va_entry->bo_va = amdgpu_vm_bo_add(adev, vm, bo);
399 if (!bo_va_entry->bo_va) {
400 ret = -EINVAL;
401 pr_err("Failed to add BO object to VM. ret == %d\n",
402 ret);
403 goto err_vmadd;
404 }
405
406 bo_va_entry->va = va;
407 bo_va_entry->pte_flags = amdgpu_gmc_get_pte_flags(adev,
408 mem->mapping_flags);
409 bo_va_entry->kgd_dev = (void *)adev;
410 list_add(&bo_va_entry->bo_list, list_bo_va);
411
412 if (p_bo_va_entry)
413 *p_bo_va_entry = bo_va_entry;
414
415
416 ret = vm_validate_pt_pd_bos(vm);
417 if (ret) {
418 pr_err("validate_pt_pd_bos() failed\n");
419 goto err_alloc_pts;
420 }
421
422 return 0;
423
424 err_alloc_pts:
425 amdgpu_vm_bo_rmv(adev, bo_va_entry->bo_va);
426 list_del(&bo_va_entry->bo_list);
427 err_vmadd:
428 kfree(bo_va_entry);
429 return ret;
430 }
431
432 static void remove_bo_from_vm(struct amdgpu_device *adev,
433 struct kfd_bo_va_list *entry, unsigned long size)
434 {
435 pr_debug("\t remove VA 0x%llx - 0x%llx in entry %p\n",
436 entry->va,
437 entry->va + size, entry);
438 amdgpu_vm_bo_rmv(adev, entry->bo_va);
439 list_del(&entry->bo_list);
440 kfree(entry);
441 }
442
443 static void add_kgd_mem_to_kfd_bo_list(struct kgd_mem *mem,
444 struct amdkfd_process_info *process_info,
445 bool userptr)
446 {
447 struct ttm_validate_buffer *entry = &mem->validate_list;
448 struct amdgpu_bo *bo = mem->bo;
449
450 INIT_LIST_HEAD(&entry->head);
451 entry->num_shared = 1;
452 entry->bo = &bo->tbo;
453 mutex_lock(&process_info->lock);
454 if (userptr)
455 list_add_tail(&entry->head, &process_info->userptr_valid_list);
456 else
457 list_add_tail(&entry->head, &process_info->kfd_bo_list);
458 mutex_unlock(&process_info->lock);
459 }
460
461 static void remove_kgd_mem_from_kfd_bo_list(struct kgd_mem *mem,
462 struct amdkfd_process_info *process_info)
463 {
464 struct ttm_validate_buffer *bo_list_entry;
465
466 bo_list_entry = &mem->validate_list;
467 mutex_lock(&process_info->lock);
468 list_del(&bo_list_entry->head);
469 mutex_unlock(&process_info->lock);
470 }
471
472
473
474
475
476
477
478
479
480
481
482
483
484 static int init_user_pages(struct kgd_mem *mem, struct mm_struct *mm,
485 uint64_t user_addr)
486 {
487 struct amdkfd_process_info *process_info = mem->process_info;
488 struct amdgpu_bo *bo = mem->bo;
489 struct ttm_operation_ctx ctx = { true, false };
490 int ret = 0;
491
492 mutex_lock(&process_info->lock);
493
494 ret = amdgpu_ttm_tt_set_userptr(bo->tbo.ttm, user_addr, 0);
495 if (ret) {
496 pr_err("%s: Failed to set userptr: %d\n", __func__, ret);
497 goto out;
498 }
499
500 ret = amdgpu_mn_register(bo, user_addr);
501 if (ret) {
502 pr_err("%s: Failed to register MMU notifier: %d\n",
503 __func__, ret);
504 goto out;
505 }
506
507 ret = amdgpu_ttm_tt_get_user_pages(bo, bo->tbo.ttm->pages);
508 if (ret) {
509 pr_err("%s: Failed to get user pages: %d\n", __func__, ret);
510 goto unregister_out;
511 }
512
513 ret = amdgpu_bo_reserve(bo, true);
514 if (ret) {
515 pr_err("%s: Failed to reserve BO\n", __func__);
516 goto release_out;
517 }
518 amdgpu_bo_placement_from_domain(bo, mem->domain);
519 ret = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
520 if (ret)
521 pr_err("%s: failed to validate BO\n", __func__);
522 amdgpu_bo_unreserve(bo);
523
524 release_out:
525 amdgpu_ttm_tt_get_user_pages_done(bo->tbo.ttm);
526 unregister_out:
527 if (ret)
528 amdgpu_mn_unregister(bo);
529 out:
530 mutex_unlock(&process_info->lock);
531 return ret;
532 }
533
534
535
536
537
538
539 struct bo_vm_reservation_context {
540 struct amdgpu_bo_list_entry kfd_bo;
541 unsigned int n_vms;
542 struct amdgpu_bo_list_entry *vm_pd;
543 struct ww_acquire_ctx ticket;
544 struct list_head list, duplicates;
545 struct amdgpu_sync *sync;
546 bool reserved;
547 };
548
549 enum bo_vm_match {
550 BO_VM_NOT_MAPPED = 0,
551 BO_VM_MAPPED,
552 BO_VM_ALL,
553 };
554
555
556
557
558
559
560
561 static int reserve_bo_and_vm(struct kgd_mem *mem,
562 struct amdgpu_vm *vm,
563 struct bo_vm_reservation_context *ctx)
564 {
565 struct amdgpu_bo *bo = mem->bo;
566 int ret;
567
568 WARN_ON(!vm);
569
570 ctx->reserved = false;
571 ctx->n_vms = 1;
572 ctx->sync = &mem->sync;
573
574 INIT_LIST_HEAD(&ctx->list);
575 INIT_LIST_HEAD(&ctx->duplicates);
576
577 ctx->vm_pd = kcalloc(ctx->n_vms, sizeof(*ctx->vm_pd), GFP_KERNEL);
578 if (!ctx->vm_pd)
579 return -ENOMEM;
580
581 ctx->kfd_bo.priority = 0;
582 ctx->kfd_bo.tv.bo = &bo->tbo;
583 ctx->kfd_bo.tv.num_shared = 1;
584 list_add(&ctx->kfd_bo.tv.head, &ctx->list);
585
586 amdgpu_vm_get_pd_bo(vm, &ctx->list, &ctx->vm_pd[0]);
587
588 ret = ttm_eu_reserve_buffers(&ctx->ticket, &ctx->list,
589 false, &ctx->duplicates, true);
590 if (!ret)
591 ctx->reserved = true;
592 else {
593 pr_err("Failed to reserve buffers in ttm\n");
594 kfree(ctx->vm_pd);
595 ctx->vm_pd = NULL;
596 }
597
598 return ret;
599 }
600
601
602
603
604
605
606
607
608
609
610
611 static int reserve_bo_and_cond_vms(struct kgd_mem *mem,
612 struct amdgpu_vm *vm, enum bo_vm_match map_type,
613 struct bo_vm_reservation_context *ctx)
614 {
615 struct amdgpu_bo *bo = mem->bo;
616 struct kfd_bo_va_list *entry;
617 unsigned int i;
618 int ret;
619
620 ctx->reserved = false;
621 ctx->n_vms = 0;
622 ctx->vm_pd = NULL;
623 ctx->sync = &mem->sync;
624
625 INIT_LIST_HEAD(&ctx->list);
626 INIT_LIST_HEAD(&ctx->duplicates);
627
628 list_for_each_entry(entry, &mem->bo_va_list, bo_list) {
629 if ((vm && vm != entry->bo_va->base.vm) ||
630 (entry->is_mapped != map_type
631 && map_type != BO_VM_ALL))
632 continue;
633
634 ctx->n_vms++;
635 }
636
637 if (ctx->n_vms != 0) {
638 ctx->vm_pd = kcalloc(ctx->n_vms, sizeof(*ctx->vm_pd),
639 GFP_KERNEL);
640 if (!ctx->vm_pd)
641 return -ENOMEM;
642 }
643
644 ctx->kfd_bo.priority = 0;
645 ctx->kfd_bo.tv.bo = &bo->tbo;
646 ctx->kfd_bo.tv.num_shared = 1;
647 list_add(&ctx->kfd_bo.tv.head, &ctx->list);
648
649 i = 0;
650 list_for_each_entry(entry, &mem->bo_va_list, bo_list) {
651 if ((vm && vm != entry->bo_va->base.vm) ||
652 (entry->is_mapped != map_type
653 && map_type != BO_VM_ALL))
654 continue;
655
656 amdgpu_vm_get_pd_bo(entry->bo_va->base.vm, &ctx->list,
657 &ctx->vm_pd[i]);
658 i++;
659 }
660
661 ret = ttm_eu_reserve_buffers(&ctx->ticket, &ctx->list,
662 false, &ctx->duplicates, true);
663 if (!ret)
664 ctx->reserved = true;
665 else
666 pr_err("Failed to reserve buffers in ttm.\n");
667
668 if (ret) {
669 kfree(ctx->vm_pd);
670 ctx->vm_pd = NULL;
671 }
672
673 return ret;
674 }
675
676
677
678
679
680
681
682
683
684
685
686 static int unreserve_bo_and_vms(struct bo_vm_reservation_context *ctx,
687 bool wait, bool intr)
688 {
689 int ret = 0;
690
691 if (wait)
692 ret = amdgpu_sync_wait(ctx->sync, intr);
693
694 if (ctx->reserved)
695 ttm_eu_backoff_reservation(&ctx->ticket, &ctx->list);
696 kfree(ctx->vm_pd);
697
698 ctx->sync = NULL;
699
700 ctx->reserved = false;
701 ctx->vm_pd = NULL;
702
703 return ret;
704 }
705
706 static int unmap_bo_from_gpuvm(struct amdgpu_device *adev,
707 struct kfd_bo_va_list *entry,
708 struct amdgpu_sync *sync)
709 {
710 struct amdgpu_bo_va *bo_va = entry->bo_va;
711 struct amdgpu_vm *vm = bo_va->base.vm;
712
713 amdgpu_vm_bo_unmap(adev, bo_va, entry->va);
714
715 amdgpu_vm_clear_freed(adev, vm, &bo_va->last_pt_update);
716
717 amdgpu_sync_fence(NULL, sync, bo_va->last_pt_update, false);
718
719 return 0;
720 }
721
722 static int update_gpuvm_pte(struct amdgpu_device *adev,
723 struct kfd_bo_va_list *entry,
724 struct amdgpu_sync *sync)
725 {
726 int ret;
727 struct amdgpu_bo_va *bo_va = entry->bo_va;
728
729
730 ret = amdgpu_vm_bo_update(adev, bo_va, false);
731 if (ret) {
732 pr_err("amdgpu_vm_bo_update failed\n");
733 return ret;
734 }
735
736 return amdgpu_sync_fence(NULL, sync, bo_va->last_pt_update, false);
737 }
738
739 static int map_bo_to_gpuvm(struct amdgpu_device *adev,
740 struct kfd_bo_va_list *entry, struct amdgpu_sync *sync,
741 bool no_update_pte)
742 {
743 int ret;
744
745
746 ret = amdgpu_vm_bo_map(adev, entry->bo_va, entry->va, 0,
747 amdgpu_bo_size(entry->bo_va->base.bo),
748 entry->pte_flags);
749 if (ret) {
750 pr_err("Failed to map VA 0x%llx in vm. ret %d\n",
751 entry->va, ret);
752 return ret;
753 }
754
755 if (no_update_pte)
756 return 0;
757
758 ret = update_gpuvm_pte(adev, entry, sync);
759 if (ret) {
760 pr_err("update_gpuvm_pte() failed\n");
761 goto update_gpuvm_pte_failed;
762 }
763
764 return 0;
765
766 update_gpuvm_pte_failed:
767 unmap_bo_from_gpuvm(adev, entry, sync);
768 return ret;
769 }
770
771 static struct sg_table *create_doorbell_sg(uint64_t addr, uint32_t size)
772 {
773 struct sg_table *sg = kmalloc(sizeof(*sg), GFP_KERNEL);
774
775 if (!sg)
776 return NULL;
777 if (sg_alloc_table(sg, 1, GFP_KERNEL)) {
778 kfree(sg);
779 return NULL;
780 }
781 sg->sgl->dma_address = addr;
782 sg->sgl->length = size;
783 #ifdef CONFIG_NEED_SG_DMA_LENGTH
784 sg->sgl->dma_length = size;
785 #endif
786 return sg;
787 }
788
789 static int process_validate_vms(struct amdkfd_process_info *process_info)
790 {
791 struct amdgpu_vm *peer_vm;
792 int ret;
793
794 list_for_each_entry(peer_vm, &process_info->vm_list_head,
795 vm_list_node) {
796 ret = vm_validate_pt_pd_bos(peer_vm);
797 if (ret)
798 return ret;
799 }
800
801 return 0;
802 }
803
804 static int process_sync_pds_resv(struct amdkfd_process_info *process_info,
805 struct amdgpu_sync *sync)
806 {
807 struct amdgpu_vm *peer_vm;
808 int ret;
809
810 list_for_each_entry(peer_vm, &process_info->vm_list_head,
811 vm_list_node) {
812 struct amdgpu_bo *pd = peer_vm->root.base.bo;
813
814 ret = amdgpu_sync_resv(NULL,
815 sync, pd->tbo.base.resv,
816 AMDGPU_FENCE_OWNER_KFD, false);
817 if (ret)
818 return ret;
819 }
820
821 return 0;
822 }
823
824 static int process_update_pds(struct amdkfd_process_info *process_info,
825 struct amdgpu_sync *sync)
826 {
827 struct amdgpu_vm *peer_vm;
828 int ret;
829
830 list_for_each_entry(peer_vm, &process_info->vm_list_head,
831 vm_list_node) {
832 ret = vm_update_pds(peer_vm, sync);
833 if (ret)
834 return ret;
835 }
836
837 return 0;
838 }
839
840 static int init_kfd_vm(struct amdgpu_vm *vm, void **process_info,
841 struct dma_fence **ef)
842 {
843 struct amdkfd_process_info *info = NULL;
844 int ret;
845
846 if (!*process_info) {
847 info = kzalloc(sizeof(*info), GFP_KERNEL);
848 if (!info)
849 return -ENOMEM;
850
851 mutex_init(&info->lock);
852 INIT_LIST_HEAD(&info->vm_list_head);
853 INIT_LIST_HEAD(&info->kfd_bo_list);
854 INIT_LIST_HEAD(&info->userptr_valid_list);
855 INIT_LIST_HEAD(&info->userptr_inval_list);
856
857 info->eviction_fence =
858 amdgpu_amdkfd_fence_create(dma_fence_context_alloc(1),
859 current->mm);
860 if (!info->eviction_fence) {
861 pr_err("Failed to create eviction fence\n");
862 ret = -ENOMEM;
863 goto create_evict_fence_fail;
864 }
865
866 info->pid = get_task_pid(current->group_leader, PIDTYPE_PID);
867 atomic_set(&info->evicted_bos, 0);
868 INIT_DELAYED_WORK(&info->restore_userptr_work,
869 amdgpu_amdkfd_restore_userptr_worker);
870
871 *process_info = info;
872 *ef = dma_fence_get(&info->eviction_fence->base);
873 }
874
875 vm->process_info = *process_info;
876
877
878 ret = amdgpu_bo_reserve(vm->root.base.bo, true);
879 if (ret)
880 goto reserve_pd_fail;
881 ret = vm_validate_pt_pd_bos(vm);
882 if (ret) {
883 pr_err("validate_pt_pd_bos() failed\n");
884 goto validate_pd_fail;
885 }
886 ret = amdgpu_bo_sync_wait(vm->root.base.bo,
887 AMDGPU_FENCE_OWNER_KFD, false);
888 if (ret)
889 goto wait_pd_fail;
890 ret = dma_resv_reserve_shared(vm->root.base.bo->tbo.base.resv, 1);
891 if (ret)
892 goto reserve_shared_fail;
893 amdgpu_bo_fence(vm->root.base.bo,
894 &vm->process_info->eviction_fence->base, true);
895 amdgpu_bo_unreserve(vm->root.base.bo);
896
897
898 mutex_lock(&vm->process_info->lock);
899 list_add_tail(&vm->vm_list_node,
900 &(vm->process_info->vm_list_head));
901 vm->process_info->n_vms++;
902 mutex_unlock(&vm->process_info->lock);
903
904 return 0;
905
906 reserve_shared_fail:
907 wait_pd_fail:
908 validate_pd_fail:
909 amdgpu_bo_unreserve(vm->root.base.bo);
910 reserve_pd_fail:
911 vm->process_info = NULL;
912 if (info) {
913
914 dma_fence_put(&info->eviction_fence->base);
915 dma_fence_put(*ef);
916 *ef = NULL;
917 *process_info = NULL;
918 put_pid(info->pid);
919 create_evict_fence_fail:
920 mutex_destroy(&info->lock);
921 kfree(info);
922 }
923 return ret;
924 }
925
926 int amdgpu_amdkfd_gpuvm_create_process_vm(struct kgd_dev *kgd, unsigned int pasid,
927 void **vm, void **process_info,
928 struct dma_fence **ef)
929 {
930 struct amdgpu_device *adev = get_amdgpu_device(kgd);
931 struct amdgpu_vm *new_vm;
932 int ret;
933
934 new_vm = kzalloc(sizeof(*new_vm), GFP_KERNEL);
935 if (!new_vm)
936 return -ENOMEM;
937
938
939 ret = amdgpu_vm_init(adev, new_vm, AMDGPU_VM_CONTEXT_COMPUTE, pasid);
940 if (ret) {
941 pr_err("Failed init vm ret %d\n", ret);
942 goto amdgpu_vm_init_fail;
943 }
944
945
946 ret = init_kfd_vm(new_vm, process_info, ef);
947 if (ret)
948 goto init_kfd_vm_fail;
949
950 *vm = (void *) new_vm;
951
952 return 0;
953
954 init_kfd_vm_fail:
955 amdgpu_vm_fini(adev, new_vm);
956 amdgpu_vm_init_fail:
957 kfree(new_vm);
958 return ret;
959 }
960
961 int amdgpu_amdkfd_gpuvm_acquire_process_vm(struct kgd_dev *kgd,
962 struct file *filp, unsigned int pasid,
963 void **vm, void **process_info,
964 struct dma_fence **ef)
965 {
966 struct amdgpu_device *adev = get_amdgpu_device(kgd);
967 struct drm_file *drm_priv = filp->private_data;
968 struct amdgpu_fpriv *drv_priv = drm_priv->driver_priv;
969 struct amdgpu_vm *avm = &drv_priv->vm;
970 int ret;
971
972
973 if (avm->process_info)
974 return -EINVAL;
975
976
977 ret = amdgpu_vm_make_compute(adev, avm, pasid);
978 if (ret)
979 return ret;
980
981
982 ret = init_kfd_vm(avm, process_info, ef);
983 if (ret)
984 return ret;
985
986 *vm = (void *)avm;
987
988 return 0;
989 }
990
991 void amdgpu_amdkfd_gpuvm_destroy_cb(struct amdgpu_device *adev,
992 struct amdgpu_vm *vm)
993 {
994 struct amdkfd_process_info *process_info = vm->process_info;
995 struct amdgpu_bo *pd = vm->root.base.bo;
996
997 if (!process_info)
998 return;
999
1000
1001 amdgpu_bo_reserve(pd, false);
1002 amdgpu_bo_fence(pd, NULL, false);
1003 amdgpu_bo_unreserve(pd);
1004
1005
1006 mutex_lock(&process_info->lock);
1007 process_info->n_vms--;
1008 list_del(&vm->vm_list_node);
1009 mutex_unlock(&process_info->lock);
1010
1011
1012 if (!process_info->n_vms) {
1013 WARN_ON(!list_empty(&process_info->kfd_bo_list));
1014 WARN_ON(!list_empty(&process_info->userptr_valid_list));
1015 WARN_ON(!list_empty(&process_info->userptr_inval_list));
1016
1017 dma_fence_put(&process_info->eviction_fence->base);
1018 cancel_delayed_work_sync(&process_info->restore_userptr_work);
1019 put_pid(process_info->pid);
1020 mutex_destroy(&process_info->lock);
1021 kfree(process_info);
1022 }
1023 }
1024
1025 void amdgpu_amdkfd_gpuvm_destroy_process_vm(struct kgd_dev *kgd, void *vm)
1026 {
1027 struct amdgpu_device *adev = get_amdgpu_device(kgd);
1028 struct amdgpu_vm *avm = (struct amdgpu_vm *)vm;
1029
1030 if (WARN_ON(!kgd || !vm))
1031 return;
1032
1033 pr_debug("Destroying process vm %p\n", vm);
1034
1035
1036 amdgpu_vm_fini(adev, avm);
1037 kfree(vm);
1038 }
1039
1040 void amdgpu_amdkfd_gpuvm_release_process_vm(struct kgd_dev *kgd, void *vm)
1041 {
1042 struct amdgpu_device *adev = get_amdgpu_device(kgd);
1043 struct amdgpu_vm *avm = (struct amdgpu_vm *)vm;
1044
1045 if (WARN_ON(!kgd || !vm))
1046 return;
1047
1048 pr_debug("Releasing process vm %p\n", vm);
1049
1050
1051
1052
1053
1054
1055
1056 amdgpu_vm_release_compute(adev, avm);
1057 }
1058
1059 uint64_t amdgpu_amdkfd_gpuvm_get_process_page_dir(void *vm)
1060 {
1061 struct amdgpu_vm *avm = (struct amdgpu_vm *)vm;
1062 struct amdgpu_bo *pd = avm->root.base.bo;
1063 struct amdgpu_device *adev = amdgpu_ttm_adev(pd->tbo.bdev);
1064
1065 if (adev->asic_type < CHIP_VEGA10)
1066 return avm->pd_phys_addr >> AMDGPU_GPU_PAGE_SHIFT;
1067 return avm->pd_phys_addr;
1068 }
1069
1070 int amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu(
1071 struct kgd_dev *kgd, uint64_t va, uint64_t size,
1072 void *vm, struct kgd_mem **mem,
1073 uint64_t *offset, uint32_t flags)
1074 {
1075 struct amdgpu_device *adev = get_amdgpu_device(kgd);
1076 struct amdgpu_vm *avm = (struct amdgpu_vm *)vm;
1077 enum ttm_bo_type bo_type = ttm_bo_type_device;
1078 struct sg_table *sg = NULL;
1079 uint64_t user_addr = 0;
1080 struct amdgpu_bo *bo;
1081 struct amdgpu_bo_param bp;
1082 int byte_align;
1083 u32 domain, alloc_domain;
1084 u64 alloc_flags;
1085 uint32_t mapping_flags;
1086 int ret;
1087
1088
1089
1090
1091 if (flags & ALLOC_MEM_FLAGS_VRAM) {
1092 domain = alloc_domain = AMDGPU_GEM_DOMAIN_VRAM;
1093 alloc_flags = AMDGPU_GEM_CREATE_VRAM_WIPE_ON_RELEASE;
1094 alloc_flags |= (flags & ALLOC_MEM_FLAGS_PUBLIC) ?
1095 AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED :
1096 AMDGPU_GEM_CREATE_NO_CPU_ACCESS;
1097 } else if (flags & ALLOC_MEM_FLAGS_GTT) {
1098 domain = alloc_domain = AMDGPU_GEM_DOMAIN_GTT;
1099 alloc_flags = 0;
1100 } else if (flags & ALLOC_MEM_FLAGS_USERPTR) {
1101 domain = AMDGPU_GEM_DOMAIN_GTT;
1102 alloc_domain = AMDGPU_GEM_DOMAIN_CPU;
1103 alloc_flags = 0;
1104 if (!offset || !*offset)
1105 return -EINVAL;
1106 user_addr = untagged_addr(*offset);
1107 } else if (flags & (ALLOC_MEM_FLAGS_DOORBELL |
1108 ALLOC_MEM_FLAGS_MMIO_REMAP)) {
1109 domain = AMDGPU_GEM_DOMAIN_GTT;
1110 alloc_domain = AMDGPU_GEM_DOMAIN_CPU;
1111 bo_type = ttm_bo_type_sg;
1112 alloc_flags = 0;
1113 if (size > UINT_MAX)
1114 return -EINVAL;
1115 sg = create_doorbell_sg(*offset, size);
1116 if (!sg)
1117 return -ENOMEM;
1118 } else {
1119 return -EINVAL;
1120 }
1121
1122 *mem = kzalloc(sizeof(struct kgd_mem), GFP_KERNEL);
1123 if (!*mem) {
1124 ret = -ENOMEM;
1125 goto err;
1126 }
1127 INIT_LIST_HEAD(&(*mem)->bo_va_list);
1128 mutex_init(&(*mem)->lock);
1129 (*mem)->aql_queue = !!(flags & ALLOC_MEM_FLAGS_AQL_QUEUE_MEM);
1130
1131
1132
1133
1134
1135 if ((*mem)->aql_queue)
1136 size = size >> 1;
1137
1138
1139 byte_align = (adev->family == AMDGPU_FAMILY_VI &&
1140 adev->asic_type != CHIP_FIJI &&
1141 adev->asic_type != CHIP_POLARIS10 &&
1142 adev->asic_type != CHIP_POLARIS11 &&
1143 adev->asic_type != CHIP_POLARIS12 &&
1144 adev->asic_type != CHIP_VEGAM) ?
1145 VI_BO_SIZE_ALIGN : 1;
1146
1147 mapping_flags = AMDGPU_VM_PAGE_READABLE;
1148 if (flags & ALLOC_MEM_FLAGS_WRITABLE)
1149 mapping_flags |= AMDGPU_VM_PAGE_WRITEABLE;
1150 if (flags & ALLOC_MEM_FLAGS_EXECUTABLE)
1151 mapping_flags |= AMDGPU_VM_PAGE_EXECUTABLE;
1152 if (flags & ALLOC_MEM_FLAGS_COHERENT)
1153 mapping_flags |= AMDGPU_VM_MTYPE_UC;
1154 else
1155 mapping_flags |= AMDGPU_VM_MTYPE_NC;
1156 (*mem)->mapping_flags = mapping_flags;
1157
1158 amdgpu_sync_create(&(*mem)->sync);
1159
1160 ret = amdgpu_amdkfd_reserve_mem_limit(adev, size, alloc_domain, !!sg);
1161 if (ret) {
1162 pr_debug("Insufficient system memory\n");
1163 goto err_reserve_limit;
1164 }
1165
1166 pr_debug("\tcreate BO VA 0x%llx size 0x%llx domain %s\n",
1167 va, size, domain_string(alloc_domain));
1168
1169 memset(&bp, 0, sizeof(bp));
1170 bp.size = size;
1171 bp.byte_align = byte_align;
1172 bp.domain = alloc_domain;
1173 bp.flags = alloc_flags;
1174 bp.type = bo_type;
1175 bp.resv = NULL;
1176 ret = amdgpu_bo_create(adev, &bp, &bo);
1177 if (ret) {
1178 pr_debug("Failed to create BO on domain %s. ret %d\n",
1179 domain_string(alloc_domain), ret);
1180 goto err_bo_create;
1181 }
1182 if (bo_type == ttm_bo_type_sg) {
1183 bo->tbo.sg = sg;
1184 bo->tbo.ttm->sg = sg;
1185 }
1186 bo->kfd_bo = *mem;
1187 (*mem)->bo = bo;
1188 if (user_addr)
1189 bo->flags |= AMDGPU_AMDKFD_USERPTR_BO;
1190
1191 (*mem)->va = va;
1192 (*mem)->domain = domain;
1193 (*mem)->mapped_to_gpu_memory = 0;
1194 (*mem)->process_info = avm->process_info;
1195 add_kgd_mem_to_kfd_bo_list(*mem, avm->process_info, user_addr);
1196
1197 if (user_addr) {
1198 ret = init_user_pages(*mem, current->mm, user_addr);
1199 if (ret)
1200 goto allocate_init_user_pages_failed;
1201 }
1202
1203 if (offset)
1204 *offset = amdgpu_bo_mmap_offset(bo);
1205
1206 return 0;
1207
1208 allocate_init_user_pages_failed:
1209 remove_kgd_mem_from_kfd_bo_list(*mem, avm->process_info);
1210 amdgpu_bo_unref(&bo);
1211
1212 goto err_reserve_limit;
1213 err_bo_create:
1214 unreserve_mem_limit(adev, size, alloc_domain, !!sg);
1215 err_reserve_limit:
1216 mutex_destroy(&(*mem)->lock);
1217 kfree(*mem);
1218 err:
1219 if (sg) {
1220 sg_free_table(sg);
1221 kfree(sg);
1222 }
1223 return ret;
1224 }
1225
1226 int amdgpu_amdkfd_gpuvm_free_memory_of_gpu(
1227 struct kgd_dev *kgd, struct kgd_mem *mem)
1228 {
1229 struct amdkfd_process_info *process_info = mem->process_info;
1230 unsigned long bo_size = mem->bo->tbo.mem.size;
1231 struct kfd_bo_va_list *entry, *tmp;
1232 struct bo_vm_reservation_context ctx;
1233 struct ttm_validate_buffer *bo_list_entry;
1234 int ret;
1235
1236 mutex_lock(&mem->lock);
1237
1238 if (mem->mapped_to_gpu_memory > 0) {
1239 pr_debug("BO VA 0x%llx size 0x%lx is still mapped.\n",
1240 mem->va, bo_size);
1241 mutex_unlock(&mem->lock);
1242 return -EBUSY;
1243 }
1244
1245 mutex_unlock(&mem->lock);
1246
1247
1248
1249
1250
1251 amdgpu_mn_unregister(mem->bo);
1252
1253
1254 bo_list_entry = &mem->validate_list;
1255 mutex_lock(&process_info->lock);
1256 list_del(&bo_list_entry->head);
1257 mutex_unlock(&process_info->lock);
1258
1259 ret = reserve_bo_and_cond_vms(mem, NULL, BO_VM_ALL, &ctx);
1260 if (unlikely(ret))
1261 return ret;
1262
1263
1264
1265
1266
1267 amdgpu_amdkfd_remove_eviction_fence(mem->bo,
1268 process_info->eviction_fence);
1269 pr_debug("Release VA 0x%llx - 0x%llx\n", mem->va,
1270 mem->va + bo_size * (1 + mem->aql_queue));
1271
1272
1273 list_for_each_entry_safe(entry, tmp, &mem->bo_va_list, bo_list)
1274 remove_bo_from_vm((struct amdgpu_device *)entry->kgd_dev,
1275 entry, bo_size);
1276
1277 ret = unreserve_bo_and_vms(&ctx, false, false);
1278
1279
1280 amdgpu_sync_free(&mem->sync);
1281
1282
1283
1284
1285 if (mem->bo->tbo.sg) {
1286 sg_free_table(mem->bo->tbo.sg);
1287 kfree(mem->bo->tbo.sg);
1288 }
1289
1290
1291 drm_gem_object_put_unlocked(&mem->bo->tbo.base);
1292 mutex_destroy(&mem->lock);
1293 kfree(mem);
1294
1295 return ret;
1296 }
1297
1298 int amdgpu_amdkfd_gpuvm_map_memory_to_gpu(
1299 struct kgd_dev *kgd, struct kgd_mem *mem, void *vm)
1300 {
1301 struct amdgpu_device *adev = get_amdgpu_device(kgd);
1302 struct amdgpu_vm *avm = (struct amdgpu_vm *)vm;
1303 int ret;
1304 struct amdgpu_bo *bo;
1305 uint32_t domain;
1306 struct kfd_bo_va_list *entry;
1307 struct bo_vm_reservation_context ctx;
1308 struct kfd_bo_va_list *bo_va_entry = NULL;
1309 struct kfd_bo_va_list *bo_va_entry_aql = NULL;
1310 unsigned long bo_size;
1311 bool is_invalid_userptr = false;
1312
1313 bo = mem->bo;
1314 if (!bo) {
1315 pr_err("Invalid BO when mapping memory to GPU\n");
1316 return -EINVAL;
1317 }
1318
1319
1320
1321
1322
1323 mutex_lock(&mem->process_info->lock);
1324
1325
1326
1327
1328
1329 if (amdgpu_ttm_tt_get_usermm(bo->tbo.ttm)) {
1330 down_write(¤t->mm->mmap_sem);
1331 is_invalid_userptr = atomic_read(&mem->invalid);
1332 up_write(¤t->mm->mmap_sem);
1333 }
1334
1335 mutex_lock(&mem->lock);
1336
1337 domain = mem->domain;
1338 bo_size = bo->tbo.mem.size;
1339
1340 pr_debug("Map VA 0x%llx - 0x%llx to vm %p domain %s\n",
1341 mem->va,
1342 mem->va + bo_size * (1 + mem->aql_queue),
1343 vm, domain_string(domain));
1344
1345 ret = reserve_bo_and_vm(mem, vm, &ctx);
1346 if (unlikely(ret))
1347 goto out;
1348
1349
1350
1351
1352
1353
1354 if (amdgpu_ttm_tt_get_usermm(bo->tbo.ttm) &&
1355 bo->tbo.mem.mem_type == TTM_PL_SYSTEM)
1356 is_invalid_userptr = true;
1357
1358 if (check_if_add_bo_to_vm(avm, mem)) {
1359 ret = add_bo_to_vm(adev, mem, avm, false,
1360 &bo_va_entry);
1361 if (ret)
1362 goto add_bo_to_vm_failed;
1363 if (mem->aql_queue) {
1364 ret = add_bo_to_vm(adev, mem, avm,
1365 true, &bo_va_entry_aql);
1366 if (ret)
1367 goto add_bo_to_vm_failed_aql;
1368 }
1369 } else {
1370 ret = vm_validate_pt_pd_bos(avm);
1371 if (unlikely(ret))
1372 goto add_bo_to_vm_failed;
1373 }
1374
1375 if (mem->mapped_to_gpu_memory == 0 &&
1376 !amdgpu_ttm_tt_get_usermm(bo->tbo.ttm)) {
1377
1378
1379
1380
1381 ret = amdgpu_amdkfd_bo_validate(bo, domain, true);
1382 if (ret) {
1383 pr_debug("Validate failed\n");
1384 goto map_bo_to_gpuvm_failed;
1385 }
1386 }
1387
1388 list_for_each_entry(entry, &mem->bo_va_list, bo_list) {
1389 if (entry->bo_va->base.vm == vm && !entry->is_mapped) {
1390 pr_debug("\t map VA 0x%llx - 0x%llx in entry %p\n",
1391 entry->va, entry->va + bo_size,
1392 entry);
1393
1394 ret = map_bo_to_gpuvm(adev, entry, ctx.sync,
1395 is_invalid_userptr);
1396 if (ret) {
1397 pr_err("Failed to map bo to gpuvm\n");
1398 goto map_bo_to_gpuvm_failed;
1399 }
1400
1401 ret = vm_update_pds(vm, ctx.sync);
1402 if (ret) {
1403 pr_err("Failed to update page directories\n");
1404 goto map_bo_to_gpuvm_failed;
1405 }
1406
1407 entry->is_mapped = true;
1408 mem->mapped_to_gpu_memory++;
1409 pr_debug("\t INC mapping count %d\n",
1410 mem->mapped_to_gpu_memory);
1411 }
1412 }
1413
1414 if (!amdgpu_ttm_tt_get_usermm(bo->tbo.ttm) && !bo->pin_count)
1415 amdgpu_bo_fence(bo,
1416 &avm->process_info->eviction_fence->base,
1417 true);
1418 ret = unreserve_bo_and_vms(&ctx, false, false);
1419
1420 goto out;
1421
1422 map_bo_to_gpuvm_failed:
1423 if (bo_va_entry_aql)
1424 remove_bo_from_vm(adev, bo_va_entry_aql, bo_size);
1425 add_bo_to_vm_failed_aql:
1426 if (bo_va_entry)
1427 remove_bo_from_vm(adev, bo_va_entry, bo_size);
1428 add_bo_to_vm_failed:
1429 unreserve_bo_and_vms(&ctx, false, false);
1430 out:
1431 mutex_unlock(&mem->process_info->lock);
1432 mutex_unlock(&mem->lock);
1433 return ret;
1434 }
1435
1436 int amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu(
1437 struct kgd_dev *kgd, struct kgd_mem *mem, void *vm)
1438 {
1439 struct amdgpu_device *adev = get_amdgpu_device(kgd);
1440 struct amdkfd_process_info *process_info =
1441 ((struct amdgpu_vm *)vm)->process_info;
1442 unsigned long bo_size = mem->bo->tbo.mem.size;
1443 struct kfd_bo_va_list *entry;
1444 struct bo_vm_reservation_context ctx;
1445 int ret;
1446
1447 mutex_lock(&mem->lock);
1448
1449 ret = reserve_bo_and_cond_vms(mem, vm, BO_VM_MAPPED, &ctx);
1450 if (unlikely(ret))
1451 goto out;
1452
1453 if (ctx.n_vms == 0) {
1454 ret = -EINVAL;
1455 goto unreserve_out;
1456 }
1457
1458 ret = vm_validate_pt_pd_bos((struct amdgpu_vm *)vm);
1459 if (unlikely(ret))
1460 goto unreserve_out;
1461
1462 pr_debug("Unmap VA 0x%llx - 0x%llx from vm %p\n",
1463 mem->va,
1464 mem->va + bo_size * (1 + mem->aql_queue),
1465 vm);
1466
1467 list_for_each_entry(entry, &mem->bo_va_list, bo_list) {
1468 if (entry->bo_va->base.vm == vm && entry->is_mapped) {
1469 pr_debug("\t unmap VA 0x%llx - 0x%llx from entry %p\n",
1470 entry->va,
1471 entry->va + bo_size,
1472 entry);
1473
1474 ret = unmap_bo_from_gpuvm(adev, entry, ctx.sync);
1475 if (ret == 0) {
1476 entry->is_mapped = false;
1477 } else {
1478 pr_err("failed to unmap VA 0x%llx\n",
1479 mem->va);
1480 goto unreserve_out;
1481 }
1482
1483 mem->mapped_to_gpu_memory--;
1484 pr_debug("\t DEC mapping count %d\n",
1485 mem->mapped_to_gpu_memory);
1486 }
1487 }
1488
1489
1490
1491
1492 if (mem->mapped_to_gpu_memory == 0 &&
1493 !amdgpu_ttm_tt_get_usermm(mem->bo->tbo.ttm) && !mem->bo->pin_count)
1494 amdgpu_amdkfd_remove_eviction_fence(mem->bo,
1495 process_info->eviction_fence);
1496
1497 unreserve_out:
1498 unreserve_bo_and_vms(&ctx, false, false);
1499 out:
1500 mutex_unlock(&mem->lock);
1501 return ret;
1502 }
1503
1504 int amdgpu_amdkfd_gpuvm_sync_memory(
1505 struct kgd_dev *kgd, struct kgd_mem *mem, bool intr)
1506 {
1507 struct amdgpu_sync sync;
1508 int ret;
1509
1510 amdgpu_sync_create(&sync);
1511
1512 mutex_lock(&mem->lock);
1513 amdgpu_sync_clone(&mem->sync, &sync);
1514 mutex_unlock(&mem->lock);
1515
1516 ret = amdgpu_sync_wait(&sync, intr);
1517 amdgpu_sync_free(&sync);
1518 return ret;
1519 }
1520
1521 int amdgpu_amdkfd_gpuvm_map_gtt_bo_to_kernel(struct kgd_dev *kgd,
1522 struct kgd_mem *mem, void **kptr, uint64_t *size)
1523 {
1524 int ret;
1525 struct amdgpu_bo *bo = mem->bo;
1526
1527 if (amdgpu_ttm_tt_get_usermm(bo->tbo.ttm)) {
1528 pr_err("userptr can't be mapped to kernel\n");
1529 return -EINVAL;
1530 }
1531
1532
1533
1534
1535 mutex_lock(&mem->process_info->lock);
1536
1537 ret = amdgpu_bo_reserve(bo, true);
1538 if (ret) {
1539 pr_err("Failed to reserve bo. ret %d\n", ret);
1540 goto bo_reserve_failed;
1541 }
1542
1543 ret = amdgpu_bo_pin(bo, AMDGPU_GEM_DOMAIN_GTT);
1544 if (ret) {
1545 pr_err("Failed to pin bo. ret %d\n", ret);
1546 goto pin_failed;
1547 }
1548
1549 ret = amdgpu_bo_kmap(bo, kptr);
1550 if (ret) {
1551 pr_err("Failed to map bo to kernel. ret %d\n", ret);
1552 goto kmap_failed;
1553 }
1554
1555 amdgpu_amdkfd_remove_eviction_fence(
1556 bo, mem->process_info->eviction_fence);
1557 list_del_init(&mem->validate_list.head);
1558
1559 if (size)
1560 *size = amdgpu_bo_size(bo);
1561
1562 amdgpu_bo_unreserve(bo);
1563
1564 mutex_unlock(&mem->process_info->lock);
1565 return 0;
1566
1567 kmap_failed:
1568 amdgpu_bo_unpin(bo);
1569 pin_failed:
1570 amdgpu_bo_unreserve(bo);
1571 bo_reserve_failed:
1572 mutex_unlock(&mem->process_info->lock);
1573
1574 return ret;
1575 }
1576
1577 int amdgpu_amdkfd_gpuvm_get_vm_fault_info(struct kgd_dev *kgd,
1578 struct kfd_vm_fault_info *mem)
1579 {
1580 struct amdgpu_device *adev;
1581
1582 adev = (struct amdgpu_device *)kgd;
1583 if (atomic_read(&adev->gmc.vm_fault_info_updated) == 1) {
1584 *mem = *adev->gmc.vm_fault_info;
1585 mb();
1586 atomic_set(&adev->gmc.vm_fault_info_updated, 0);
1587 }
1588 return 0;
1589 }
1590
1591 int amdgpu_amdkfd_gpuvm_import_dmabuf(struct kgd_dev *kgd,
1592 struct dma_buf *dma_buf,
1593 uint64_t va, void *vm,
1594 struct kgd_mem **mem, uint64_t *size,
1595 uint64_t *mmap_offset)
1596 {
1597 struct amdgpu_device *adev = (struct amdgpu_device *)kgd;
1598 struct drm_gem_object *obj;
1599 struct amdgpu_bo *bo;
1600 struct amdgpu_vm *avm = (struct amdgpu_vm *)vm;
1601
1602 if (dma_buf->ops != &amdgpu_dmabuf_ops)
1603
1604 return -EINVAL;
1605
1606 obj = dma_buf->priv;
1607 if (obj->dev->dev_private != adev)
1608
1609 return -EINVAL;
1610
1611 bo = gem_to_amdgpu_bo(obj);
1612 if (!(bo->preferred_domains & (AMDGPU_GEM_DOMAIN_VRAM |
1613 AMDGPU_GEM_DOMAIN_GTT)))
1614
1615 return -EINVAL;
1616
1617 *mem = kzalloc(sizeof(struct kgd_mem), GFP_KERNEL);
1618 if (!*mem)
1619 return -ENOMEM;
1620
1621 if (size)
1622 *size = amdgpu_bo_size(bo);
1623
1624 if (mmap_offset)
1625 *mmap_offset = amdgpu_bo_mmap_offset(bo);
1626
1627 INIT_LIST_HEAD(&(*mem)->bo_va_list);
1628 mutex_init(&(*mem)->lock);
1629 (*mem)->mapping_flags =
1630 AMDGPU_VM_PAGE_READABLE | AMDGPU_VM_PAGE_WRITEABLE |
1631 AMDGPU_VM_PAGE_EXECUTABLE | AMDGPU_VM_MTYPE_NC;
1632
1633 drm_gem_object_get(&bo->tbo.base);
1634 (*mem)->bo = bo;
1635 (*mem)->va = va;
1636 (*mem)->domain = (bo->preferred_domains & AMDGPU_GEM_DOMAIN_VRAM) ?
1637 AMDGPU_GEM_DOMAIN_VRAM : AMDGPU_GEM_DOMAIN_GTT;
1638 (*mem)->mapped_to_gpu_memory = 0;
1639 (*mem)->process_info = avm->process_info;
1640 add_kgd_mem_to_kfd_bo_list(*mem, avm->process_info, false);
1641 amdgpu_sync_create(&(*mem)->sync);
1642
1643 return 0;
1644 }
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657 int amdgpu_amdkfd_evict_userptr(struct kgd_mem *mem,
1658 struct mm_struct *mm)
1659 {
1660 struct amdkfd_process_info *process_info = mem->process_info;
1661 int invalid, evicted_bos;
1662 int r = 0;
1663
1664 invalid = atomic_inc_return(&mem->invalid);
1665 evicted_bos = atomic_inc_return(&process_info->evicted_bos);
1666 if (evicted_bos == 1) {
1667
1668 r = kgd2kfd_quiesce_mm(mm);
1669 if (r)
1670 pr_err("Failed to quiesce KFD\n");
1671 schedule_delayed_work(&process_info->restore_userptr_work,
1672 msecs_to_jiffies(AMDGPU_USERPTR_RESTORE_DELAY_MS));
1673 }
1674
1675 return r;
1676 }
1677
1678
1679
1680
1681
1682
1683
1684 static int update_invalid_user_pages(struct amdkfd_process_info *process_info,
1685 struct mm_struct *mm)
1686 {
1687 struct kgd_mem *mem, *tmp_mem;
1688 struct amdgpu_bo *bo;
1689 struct ttm_operation_ctx ctx = { false, false };
1690 int invalid, ret;
1691
1692
1693
1694
1695 list_for_each_entry_safe(mem, tmp_mem,
1696 &process_info->userptr_valid_list,
1697 validate_list.head) {
1698 if (!atomic_read(&mem->invalid))
1699 continue;
1700
1701 bo = mem->bo;
1702
1703 if (amdgpu_bo_reserve(bo, true))
1704 return -EAGAIN;
1705 amdgpu_bo_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_CPU);
1706 ret = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
1707 amdgpu_bo_unreserve(bo);
1708 if (ret) {
1709 pr_err("%s: Failed to invalidate userptr BO\n",
1710 __func__);
1711 return -EAGAIN;
1712 }
1713
1714 list_move_tail(&mem->validate_list.head,
1715 &process_info->userptr_inval_list);
1716 }
1717
1718 if (list_empty(&process_info->userptr_inval_list))
1719 return 0;
1720
1721
1722 list_for_each_entry(mem, &process_info->userptr_inval_list,
1723 validate_list.head) {
1724 invalid = atomic_read(&mem->invalid);
1725 if (!invalid)
1726
1727
1728
1729 continue;
1730
1731 bo = mem->bo;
1732
1733
1734 ret = amdgpu_ttm_tt_get_user_pages(bo, bo->tbo.ttm->pages);
1735 if (ret) {
1736 pr_debug("%s: Failed to get user pages: %d\n",
1737 __func__, ret);
1738
1739
1740 return ret;
1741 }
1742
1743 amdgpu_ttm_tt_get_user_pages_done(bo->tbo.ttm);
1744
1745
1746
1747
1748 if (atomic_cmpxchg(&mem->invalid, invalid, 0) != invalid)
1749 return -EAGAIN;
1750 }
1751
1752 return 0;
1753 }
1754
1755
1756
1757
1758
1759
1760
1761 static int validate_invalid_user_pages(struct amdkfd_process_info *process_info)
1762 {
1763 struct amdgpu_bo_list_entry *pd_bo_list_entries;
1764 struct list_head resv_list, duplicates;
1765 struct ww_acquire_ctx ticket;
1766 struct amdgpu_sync sync;
1767
1768 struct amdgpu_vm *peer_vm;
1769 struct kgd_mem *mem, *tmp_mem;
1770 struct amdgpu_bo *bo;
1771 struct ttm_operation_ctx ctx = { false, false };
1772 int i, ret;
1773
1774 pd_bo_list_entries = kcalloc(process_info->n_vms,
1775 sizeof(struct amdgpu_bo_list_entry),
1776 GFP_KERNEL);
1777 if (!pd_bo_list_entries) {
1778 pr_err("%s: Failed to allocate PD BO list entries\n", __func__);
1779 ret = -ENOMEM;
1780 goto out_no_mem;
1781 }
1782
1783 INIT_LIST_HEAD(&resv_list);
1784 INIT_LIST_HEAD(&duplicates);
1785
1786
1787 i = 0;
1788 list_for_each_entry(peer_vm, &process_info->vm_list_head,
1789 vm_list_node)
1790 amdgpu_vm_get_pd_bo(peer_vm, &resv_list,
1791 &pd_bo_list_entries[i++]);
1792
1793 list_for_each_entry(mem, &process_info->userptr_inval_list,
1794 validate_list.head) {
1795 list_add_tail(&mem->resv_list.head, &resv_list);
1796 mem->resv_list.bo = mem->validate_list.bo;
1797 mem->resv_list.num_shared = mem->validate_list.num_shared;
1798 }
1799
1800
1801 ret = ttm_eu_reserve_buffers(&ticket, &resv_list, false, &duplicates,
1802 true);
1803 WARN(!list_empty(&duplicates), "Duplicates should be empty");
1804 if (ret)
1805 goto out_free;
1806
1807 amdgpu_sync_create(&sync);
1808
1809 ret = process_validate_vms(process_info);
1810 if (ret)
1811 goto unreserve_out;
1812
1813
1814 list_for_each_entry_safe(mem, tmp_mem,
1815 &process_info->userptr_inval_list,
1816 validate_list.head) {
1817 struct kfd_bo_va_list *bo_va_entry;
1818
1819 bo = mem->bo;
1820
1821
1822 if (bo->tbo.ttm->pages[0]) {
1823 amdgpu_bo_placement_from_domain(bo, mem->domain);
1824 ret = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
1825 if (ret) {
1826 pr_err("%s: failed to validate BO\n", __func__);
1827 goto unreserve_out;
1828 }
1829 }
1830
1831 list_move_tail(&mem->validate_list.head,
1832 &process_info->userptr_valid_list);
1833
1834
1835
1836
1837
1838
1839
1840 list_for_each_entry(bo_va_entry, &mem->bo_va_list, bo_list) {
1841 if (!bo_va_entry->is_mapped)
1842 continue;
1843
1844 ret = update_gpuvm_pte((struct amdgpu_device *)
1845 bo_va_entry->kgd_dev,
1846 bo_va_entry, &sync);
1847 if (ret) {
1848 pr_err("%s: update PTE failed\n", __func__);
1849
1850 atomic_inc(&mem->invalid);
1851 goto unreserve_out;
1852 }
1853 }
1854 }
1855
1856
1857 ret = process_update_pds(process_info, &sync);
1858
1859 unreserve_out:
1860 ttm_eu_backoff_reservation(&ticket, &resv_list);
1861 amdgpu_sync_wait(&sync, false);
1862 amdgpu_sync_free(&sync);
1863 out_free:
1864 kfree(pd_bo_list_entries);
1865 out_no_mem:
1866
1867 return ret;
1868 }
1869
1870
1871
1872
1873
1874
1875
1876 static void amdgpu_amdkfd_restore_userptr_worker(struct work_struct *work)
1877 {
1878 struct delayed_work *dwork = to_delayed_work(work);
1879 struct amdkfd_process_info *process_info =
1880 container_of(dwork, struct amdkfd_process_info,
1881 restore_userptr_work);
1882 struct task_struct *usertask;
1883 struct mm_struct *mm;
1884 int evicted_bos;
1885
1886 evicted_bos = atomic_read(&process_info->evicted_bos);
1887 if (!evicted_bos)
1888 return;
1889
1890
1891 usertask = get_pid_task(process_info->pid, PIDTYPE_PID);
1892 if (!usertask)
1893 return;
1894 mm = get_task_mm(usertask);
1895 if (!mm) {
1896 put_task_struct(usertask);
1897 return;
1898 }
1899
1900 mutex_lock(&process_info->lock);
1901
1902 if (update_invalid_user_pages(process_info, mm))
1903 goto unlock_out;
1904
1905
1906
1907
1908 if (!list_empty(&process_info->userptr_inval_list)) {
1909 if (atomic_read(&process_info->evicted_bos) != evicted_bos)
1910 goto unlock_out;
1911
1912 if (validate_invalid_user_pages(process_info))
1913 goto unlock_out;
1914 }
1915
1916
1917
1918
1919
1920 if (atomic_cmpxchg(&process_info->evicted_bos, evicted_bos, 0) !=
1921 evicted_bos)
1922 goto unlock_out;
1923 evicted_bos = 0;
1924 if (kgd2kfd_resume_mm(mm)) {
1925 pr_err("%s: Failed to resume KFD\n", __func__);
1926
1927
1928
1929 }
1930
1931 unlock_out:
1932 mutex_unlock(&process_info->lock);
1933 mmput(mm);
1934 put_task_struct(usertask);
1935
1936
1937 if (evicted_bos)
1938 schedule_delayed_work(&process_info->restore_userptr_work,
1939 msecs_to_jiffies(AMDGPU_USERPTR_RESTORE_DELAY_MS));
1940 }
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960 int amdgpu_amdkfd_gpuvm_restore_process_bos(void *info, struct dma_fence **ef)
1961 {
1962 struct amdgpu_bo_list_entry *pd_bo_list;
1963 struct amdkfd_process_info *process_info = info;
1964 struct amdgpu_vm *peer_vm;
1965 struct kgd_mem *mem;
1966 struct bo_vm_reservation_context ctx;
1967 struct amdgpu_amdkfd_fence *new_fence;
1968 int ret = 0, i;
1969 struct list_head duplicate_save;
1970 struct amdgpu_sync sync_obj;
1971
1972 INIT_LIST_HEAD(&duplicate_save);
1973 INIT_LIST_HEAD(&ctx.list);
1974 INIT_LIST_HEAD(&ctx.duplicates);
1975
1976 pd_bo_list = kcalloc(process_info->n_vms,
1977 sizeof(struct amdgpu_bo_list_entry),
1978 GFP_KERNEL);
1979 if (!pd_bo_list)
1980 return -ENOMEM;
1981
1982 i = 0;
1983 mutex_lock(&process_info->lock);
1984 list_for_each_entry(peer_vm, &process_info->vm_list_head,
1985 vm_list_node)
1986 amdgpu_vm_get_pd_bo(peer_vm, &ctx.list, &pd_bo_list[i++]);
1987
1988
1989
1990
1991 list_for_each_entry(mem, &process_info->kfd_bo_list,
1992 validate_list.head) {
1993
1994 list_add_tail(&mem->resv_list.head, &ctx.list);
1995 mem->resv_list.bo = mem->validate_list.bo;
1996 mem->resv_list.num_shared = mem->validate_list.num_shared;
1997 }
1998
1999 ret = ttm_eu_reserve_buffers(&ctx.ticket, &ctx.list,
2000 false, &duplicate_save, true);
2001 if (ret) {
2002 pr_debug("Memory eviction: TTM Reserve Failed. Try again\n");
2003 goto ttm_reserve_fail;
2004 }
2005
2006 amdgpu_sync_create(&sync_obj);
2007
2008
2009 ret = process_validate_vms(process_info);
2010 if (ret)
2011 goto validate_map_fail;
2012
2013 ret = process_sync_pds_resv(process_info, &sync_obj);
2014 if (ret) {
2015 pr_debug("Memory eviction: Failed to sync to PD BO moving fence. Try again\n");
2016 goto validate_map_fail;
2017 }
2018
2019
2020 list_for_each_entry(mem, &process_info->kfd_bo_list,
2021 validate_list.head) {
2022
2023 struct amdgpu_bo *bo = mem->bo;
2024 uint32_t domain = mem->domain;
2025 struct kfd_bo_va_list *bo_va_entry;
2026
2027 ret = amdgpu_amdkfd_bo_validate(bo, domain, false);
2028 if (ret) {
2029 pr_debug("Memory eviction: Validate BOs failed. Try again\n");
2030 goto validate_map_fail;
2031 }
2032 ret = amdgpu_sync_fence(NULL, &sync_obj, bo->tbo.moving, false);
2033 if (ret) {
2034 pr_debug("Memory eviction: Sync BO fence failed. Try again\n");
2035 goto validate_map_fail;
2036 }
2037 list_for_each_entry(bo_va_entry, &mem->bo_va_list,
2038 bo_list) {
2039 ret = update_gpuvm_pte((struct amdgpu_device *)
2040 bo_va_entry->kgd_dev,
2041 bo_va_entry,
2042 &sync_obj);
2043 if (ret) {
2044 pr_debug("Memory eviction: update PTE failed. Try again\n");
2045 goto validate_map_fail;
2046 }
2047 }
2048 }
2049
2050
2051 ret = process_update_pds(process_info, &sync_obj);
2052 if (ret) {
2053 pr_debug("Memory eviction: update PDs failed. Try again\n");
2054 goto validate_map_fail;
2055 }
2056
2057
2058 amdgpu_sync_wait(&sync_obj, false);
2059
2060
2061
2062
2063
2064 new_fence = amdgpu_amdkfd_fence_create(
2065 process_info->eviction_fence->base.context,
2066 process_info->eviction_fence->mm);
2067 if (!new_fence) {
2068 pr_err("Failed to create eviction fence\n");
2069 ret = -ENOMEM;
2070 goto validate_map_fail;
2071 }
2072 dma_fence_put(&process_info->eviction_fence->base);
2073 process_info->eviction_fence = new_fence;
2074 *ef = dma_fence_get(&new_fence->base);
2075
2076
2077 list_for_each_entry(mem, &process_info->kfd_bo_list,
2078 validate_list.head)
2079 amdgpu_bo_fence(mem->bo,
2080 &process_info->eviction_fence->base, true);
2081
2082
2083 list_for_each_entry(peer_vm, &process_info->vm_list_head,
2084 vm_list_node) {
2085 struct amdgpu_bo *bo = peer_vm->root.base.bo;
2086
2087 amdgpu_bo_fence(bo, &process_info->eviction_fence->base, true);
2088 }
2089
2090 validate_map_fail:
2091 ttm_eu_backoff_reservation(&ctx.ticket, &ctx.list);
2092 amdgpu_sync_free(&sync_obj);
2093 ttm_reserve_fail:
2094 mutex_unlock(&process_info->lock);
2095 kfree(pd_bo_list);
2096 return ret;
2097 }
2098
2099 int amdgpu_amdkfd_add_gws_to_process(void *info, void *gws, struct kgd_mem **mem)
2100 {
2101 struct amdkfd_process_info *process_info = (struct amdkfd_process_info *)info;
2102 struct amdgpu_bo *gws_bo = (struct amdgpu_bo *)gws;
2103 int ret;
2104
2105 if (!info || !gws)
2106 return -EINVAL;
2107
2108 *mem = kzalloc(sizeof(struct kgd_mem), GFP_KERNEL);
2109 if (!*mem)
2110 return -ENOMEM;
2111
2112 mutex_init(&(*mem)->lock);
2113 (*mem)->bo = amdgpu_bo_ref(gws_bo);
2114 (*mem)->domain = AMDGPU_GEM_DOMAIN_GWS;
2115 (*mem)->process_info = process_info;
2116 add_kgd_mem_to_kfd_bo_list(*mem, process_info, false);
2117 amdgpu_sync_create(&(*mem)->sync);
2118
2119
2120
2121 mutex_lock(&(*mem)->process_info->lock);
2122 ret = amdgpu_bo_reserve(gws_bo, false);
2123 if (unlikely(ret)) {
2124 pr_err("Reserve gws bo failed %d\n", ret);
2125 goto bo_reservation_failure;
2126 }
2127
2128 ret = amdgpu_amdkfd_bo_validate(gws_bo, AMDGPU_GEM_DOMAIN_GWS, true);
2129 if (ret) {
2130 pr_err("GWS BO validate failed %d\n", ret);
2131 goto bo_validation_failure;
2132 }
2133
2134
2135
2136
2137 ret = dma_resv_reserve_shared(gws_bo->tbo.base.resv, 1);
2138 if (ret)
2139 goto reserve_shared_fail;
2140 amdgpu_bo_fence(gws_bo, &process_info->eviction_fence->base, true);
2141 amdgpu_bo_unreserve(gws_bo);
2142 mutex_unlock(&(*mem)->process_info->lock);
2143
2144 return ret;
2145
2146 reserve_shared_fail:
2147 bo_validation_failure:
2148 amdgpu_bo_unreserve(gws_bo);
2149 bo_reservation_failure:
2150 mutex_unlock(&(*mem)->process_info->lock);
2151 amdgpu_sync_free(&(*mem)->sync);
2152 remove_kgd_mem_from_kfd_bo_list(*mem, process_info);
2153 amdgpu_bo_unref(&gws_bo);
2154 mutex_destroy(&(*mem)->lock);
2155 kfree(*mem);
2156 *mem = NULL;
2157 return ret;
2158 }
2159
2160 int amdgpu_amdkfd_remove_gws_from_process(void *info, void *mem)
2161 {
2162 int ret;
2163 struct amdkfd_process_info *process_info = (struct amdkfd_process_info *)info;
2164 struct kgd_mem *kgd_mem = (struct kgd_mem *)mem;
2165 struct amdgpu_bo *gws_bo = kgd_mem->bo;
2166
2167
2168
2169
2170 remove_kgd_mem_from_kfd_bo_list(kgd_mem, process_info);
2171
2172 ret = amdgpu_bo_reserve(gws_bo, false);
2173 if (unlikely(ret)) {
2174 pr_err("Reserve gws bo failed %d\n", ret);
2175
2176 return ret;
2177 }
2178 amdgpu_amdkfd_remove_eviction_fence(gws_bo,
2179 process_info->eviction_fence);
2180 amdgpu_bo_unreserve(gws_bo);
2181 amdgpu_sync_free(&kgd_mem->sync);
2182 amdgpu_bo_unref(&gws_bo);
2183 mutex_destroy(&kgd_mem->lock);
2184 kfree(mem);
2185 return 0;
2186 }