root/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c

DEFINITIONS

1. INTERVAL_TREE_DEFINE
2. amdgpu_vm_level_shift
3. amdgpu_vm_num_entries
4. amdgpu_vm_num_ats_entries
5. amdgpu_vm_entries_mask
6. amdgpu_vm_bo_size
7. amdgpu_vm_bo_evicted
8. amdgpu_vm_bo_relocated
9. amdgpu_vm_bo_moved
10. amdgpu_vm_bo_idle
11. amdgpu_vm_bo_invalidated
12. amdgpu_vm_bo_done
13. amdgpu_vm_bo_base_init
14. amdgpu_vm_pt_parent
15. amdgpu_vm_pt_start
16. amdgpu_vm_pt_descendant
17. amdgpu_vm_pt_sibling
18. amdgpu_vm_pt_ancestor
19. amdgpu_vm_pt_next
20. amdgpu_vm_pt_first_dfs
21. amdgpu_vm_pt_continue_dfs
22. amdgpu_vm_pt_next_dfs
23. amdgpu_vm_get_pd_bo
24. amdgpu_vm_del_from_lru_notify
25. amdgpu_vm_move_to_lru_tail
26. amdgpu_vm_validate_pt_bos
27. amdgpu_vm_ready
28. amdgpu_vm_clear_bo
29. amdgpu_vm_bo_param
30. amdgpu_vm_alloc_pts
31. amdgpu_vm_free_table
32. amdgpu_vm_free_pts
33. amdgpu_vm_check_compute_bug
34. amdgpu_vm_need_pipeline_sync
35. amdgpu_vm_flush
36. amdgpu_vm_bo_find
37. amdgpu_vm_map_gart
38. amdgpu_vm_update_pde
39. amdgpu_vm_invalidate_pds
40. amdgpu_vm_update_directories
41. amdgpu_vm_update_flags
42. amdgpu_vm_fragment
43. amdgpu_vm_update_ptes
44. amdgpu_vm_bo_update_mapping
45. amdgpu_vm_bo_split_mapping
46. amdgpu_vm_bo_update
47. amdgpu_vm_update_prt_state
48. amdgpu_vm_prt_get
49. amdgpu_vm_prt_put
50. amdgpu_vm_prt_cb
51. amdgpu_vm_add_prt_cb
52. amdgpu_vm_free_mapping
53. amdgpu_vm_prt_fini
54. amdgpu_vm_clear_freed
55. amdgpu_vm_handle_moved
56. amdgpu_vm_bo_add
57. amdgpu_vm_bo_insert_map
58. amdgpu_vm_bo_map
59. amdgpu_vm_bo_replace_map
60. amdgpu_vm_bo_unmap
61. amdgpu_vm_bo_clear_mappings
62. amdgpu_vm_bo_lookup_mapping
63. amdgpu_vm_bo_trace_cs
64. amdgpu_vm_bo_rmv
65. amdgpu_vm_bo_invalidate
66. amdgpu_vm_get_block_size
67. amdgpu_vm_adjust_size
68. amdgpu_vm_wait_idle
69. amdgpu_vm_init
70. amdgpu_vm_check_clean_reserved
71. amdgpu_vm_make_compute
72. amdgpu_vm_release_compute
73. amdgpu_vm_fini
74. amdgpu_vm_manager_init
75. amdgpu_vm_manager_fini
76. amdgpu_vm_ioctl
77. amdgpu_vm_get_task_info
78. amdgpu_vm_set_task_info

   1 /*
   2  * Copyright 2008 Advanced Micro Devices, Inc.
   3  * Copyright 2008 Red Hat Inc.
   4  * Copyright 2009 Jerome Glisse.
   5  *
   6  * Permission is hereby granted, free of charge, to any person obtaining a
   7  * copy of this software and associated documentation files (the "Software"),
   8  * to deal in the Software without restriction, including without limitation
   9  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  10  * and/or sell copies of the Software, and to permit persons to whom the
  11  * Software is furnished to do so, subject to the following conditions:
  12  *
  13  * The above copyright notice and this permission notice shall be included in
  14  * all copies or substantial portions of the Software.
  15  *
  16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  19  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  20  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  21  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  22  * OTHER DEALINGS IN THE SOFTWARE.
  23  *
  24  * Authors: Dave Airlie
  25  *          Alex Deucher
  26  *          Jerome Glisse
  27  */
  28 #include <linux/dma-fence-array.h>
  29 #include <linux/interval_tree_generic.h>
  30 #include <linux/idr.h>
  31 
  32 #include <drm/amdgpu_drm.h>
  33 #include "amdgpu.h"
  34 #include "amdgpu_trace.h"
  35 #include "amdgpu_amdkfd.h"
  36 #include "amdgpu_gmc.h"
  37 #include "amdgpu_xgmi.h"
  38 
  39 /**
  40  * DOC: GPUVM
  41  *
  42  * GPUVM is similar to the legacy gart on older asics, however
  43  * rather than there being a single global gart table
  44  * for the entire GPU, there are multiple VM page tables active
  45  * at any given time.  The VM page tables can contain a mix
  46  * vram pages and system memory pages and system memory pages
  47  * can be mapped as snooped (cached system pages) or unsnooped
  48  * (uncached system pages).
  49  * Each VM has an ID associated with it and there is a page table
  50  * associated with each VMID.  When execting a command buffer,
  51  * the kernel tells the the ring what VMID to use for that command
  52  * buffer.  VMIDs are allocated dynamically as commands are submitted.
  53  * The userspace drivers maintain their own address space and the kernel
  54  * sets up their pages tables accordingly when they submit their
  55  * command buffers and a VMID is assigned.
  56  * Cayman/Trinity support up to 8 active VMs at any given time;
  57  * SI supports 16.
  58  */
  59 
  60 #define START(node) ((node)->start)
  61 #define LAST(node) ((node)->last)
  62 
  63 INTERVAL_TREE_DEFINE(struct amdgpu_bo_va_mapping, rb, uint64_t, __subtree_last,
  64                      START, LAST, static, amdgpu_vm_it)
  65 
  66 #undef START
  67 #undef LAST
  68 
  69 /**
  70  * struct amdgpu_prt_cb - Helper to disable partial resident texture feature from a fence callback
  71  */
  72 struct amdgpu_prt_cb {
  73 
  74         /**
  75          * @adev: amdgpu device
  76          */
  77         struct amdgpu_device *adev;
  78 
  79         /**
  80          * @cb: callback
  81          */
  82         struct dma_fence_cb cb;
  83 };
  84 
  85 /**
  86  * amdgpu_vm_level_shift - return the addr shift for each level
  87  *
  88  * @adev: amdgpu_device pointer
  89  * @level: VMPT level
  90  *
  91  * Returns:
  92  * The number of bits the pfn needs to be right shifted for a level.
  93  */
  94 static unsigned amdgpu_vm_level_shift(struct amdgpu_device *adev,
  95                                       unsigned level)
  96 {
  97         unsigned shift = 0xff;
  98 
  99         switch (level) {
 100         case AMDGPU_VM_PDB2:
 101         case AMDGPU_VM_PDB1:
 102         case AMDGPU_VM_PDB0:
 103                 shift = 9 * (AMDGPU_VM_PDB0 - level) +
 104                         adev->vm_manager.block_size;
 105                 break;
 106         case AMDGPU_VM_PTB:
 107                 shift = 0;
 108                 break;
 109         default:
 110                 dev_err(adev->dev, "the level%d isn't supported.\n", level);
 111         }
 112 
 113         return shift;
 114 }
 115 
 116 /**
 117  * amdgpu_vm_num_entries - return the number of entries in a PD/PT
 118  *
 119  * @adev: amdgpu_device pointer
 120  * @level: VMPT level
 121  *
 122  * Returns:
 123  * The number of entries in a page directory or page table.
 124  */
 125 static unsigned amdgpu_vm_num_entries(struct amdgpu_device *adev,
 126                                       unsigned level)
 127 {
 128         unsigned shift = amdgpu_vm_level_shift(adev,
 129                                                adev->vm_manager.root_level);
 130 
 131         if (level == adev->vm_manager.root_level)
 132                 /* For the root directory */
 133                 return round_up(adev->vm_manager.max_pfn, 1ULL << shift) >> shift;
 134         else if (level != AMDGPU_VM_PTB)
 135                 /* Everything in between */
 136                 return 512;
 137         else
 138                 /* For the page tables on the leaves */
 139                 return AMDGPU_VM_PTE_COUNT(adev);
 140 }
 141 
 142 /**
 143  * amdgpu_vm_num_ats_entries - return the number of ATS entries in the root PD
 144  *
 145  * @adev: amdgpu_device pointer
 146  *
 147  * Returns:
 148  * The number of entries in the root page directory which needs the ATS setting.
 149  */
 150 static unsigned amdgpu_vm_num_ats_entries(struct amdgpu_device *adev)
 151 {
 152         unsigned shift;
 153 
 154         shift = amdgpu_vm_level_shift(adev, adev->vm_manager.root_level);
 155         return AMDGPU_GMC_HOLE_START >> (shift + AMDGPU_GPU_PAGE_SHIFT);
 156 }
 157 
 158 /**
 159  * amdgpu_vm_entries_mask - the mask to get the entry number of a PD/PT
 160  *
 161  * @adev: amdgpu_device pointer
 162  * @level: VMPT level
 163  *
 164  * Returns:
 165  * The mask to extract the entry number of a PD/PT from an address.
 166  */
 167 static uint32_t amdgpu_vm_entries_mask(struct amdgpu_device *adev,
 168                                        unsigned int level)
 169 {
 170         if (level <= adev->vm_manager.root_level)
 171                 return 0xffffffff;
 172         else if (level != AMDGPU_VM_PTB)
 173                 return 0x1ff;
 174         else
 175                 return AMDGPU_VM_PTE_COUNT(adev) - 1;
 176 }
 177 
 178 /**
 179  * amdgpu_vm_bo_size - returns the size of the BOs in bytes
 180  *
 181  * @adev: amdgpu_device pointer
 182  * @level: VMPT level
 183  *
 184  * Returns:
 185  * The size of the BO for a page directory or page table in bytes.
 186  */
 187 static unsigned amdgpu_vm_bo_size(struct amdgpu_device *adev, unsigned level)
 188 {
 189         return AMDGPU_GPU_PAGE_ALIGN(amdgpu_vm_num_entries(adev, level) * 8);
 190 }
 191 
 192 /**
 193  * amdgpu_vm_bo_evicted - vm_bo is evicted
 194  *
 195  * @vm_bo: vm_bo which is evicted
 196  *
 197  * State for PDs/PTs and per VM BOs which are not at the location they should
 198  * be.
 199  */
 200 static void amdgpu_vm_bo_evicted(struct amdgpu_vm_bo_base *vm_bo)
 201 {
 202         struct amdgpu_vm *vm = vm_bo->vm;
 203         struct amdgpu_bo *bo = vm_bo->bo;
 204 
 205         vm_bo->moved = true;
 206         if (bo->tbo.type == ttm_bo_type_kernel)
 207                 list_move(&vm_bo->vm_status, &vm->evicted);
 208         else
 209                 list_move_tail(&vm_bo->vm_status, &vm->evicted);
 210 }
 211 
 212 /**
 213  * amdgpu_vm_bo_relocated - vm_bo is reloacted
 214  *
 215  * @vm_bo: vm_bo which is relocated
 216  *
 217  * State for PDs/PTs which needs to update their parent PD.
 218  */
 219 static void amdgpu_vm_bo_relocated(struct amdgpu_vm_bo_base *vm_bo)
 220 {
 221         list_move(&vm_bo->vm_status, &vm_bo->vm->relocated);
 222 }
 223 
 224 /**
 225  * amdgpu_vm_bo_moved - vm_bo is moved
 226  *
 227  * @vm_bo: vm_bo which is moved
 228  *
 229  * State for per VM BOs which are moved, but that change is not yet reflected
 230  * in the page tables.
 231  */
 232 static void amdgpu_vm_bo_moved(struct amdgpu_vm_bo_base *vm_bo)
 233 {
 234         list_move(&vm_bo->vm_status, &vm_bo->vm->moved);
 235 }
 236 
 237 /**
 238  * amdgpu_vm_bo_idle - vm_bo is idle
 239  *
 240  * @vm_bo: vm_bo which is now idle
 241  *
 242  * State for PDs/PTs and per VM BOs which have gone through the state machine
 243  * and are now idle.
 244  */
 245 static void amdgpu_vm_bo_idle(struct amdgpu_vm_bo_base *vm_bo)
 246 {
 247         list_move(&vm_bo->vm_status, &vm_bo->vm->idle);
 248         vm_bo->moved = false;
 249 }
 250 
 251 /**
 252  * amdgpu_vm_bo_invalidated - vm_bo is invalidated
 253  *
 254  * @vm_bo: vm_bo which is now invalidated
 255  *
 256  * State for normal BOs which are invalidated and that change not yet reflected
 257  * in the PTs.
 258  */
 259 static void amdgpu_vm_bo_invalidated(struct amdgpu_vm_bo_base *vm_bo)
 260 {
 261         spin_lock(&vm_bo->vm->invalidated_lock);
 262         list_move(&vm_bo->vm_status, &vm_bo->vm->invalidated);
 263         spin_unlock(&vm_bo->vm->invalidated_lock);
 264 }
 265 
 266 /**
 267  * amdgpu_vm_bo_done - vm_bo is done
 268  *
 269  * @vm_bo: vm_bo which is now done
 270  *
 271  * State for normal BOs which are invalidated and that change has been updated
 272  * in the PTs.
 273  */
 274 static void amdgpu_vm_bo_done(struct amdgpu_vm_bo_base *vm_bo)
 275 {
 276         spin_lock(&vm_bo->vm->invalidated_lock);
 277         list_del_init(&vm_bo->vm_status);
 278         spin_unlock(&vm_bo->vm->invalidated_lock);
 279 }
 280 
 281 /**
 282  * amdgpu_vm_bo_base_init - Adds bo to the list of bos associated with the vm
 283  *
 284  * @base: base structure for tracking BO usage in a VM
 285  * @vm: vm to which bo is to be added
 286  * @bo: amdgpu buffer object
 287  *
 288  * Initialize a bo_va_base structure and add it to the appropriate lists
 289  *
 290  */
 291 static void amdgpu_vm_bo_base_init(struct amdgpu_vm_bo_base *base,
 292                                    struct amdgpu_vm *vm,
 293                                    struct amdgpu_bo *bo)
 294 {
 295         base->vm = vm;
 296         base->bo = bo;
 297         base->next = NULL;
 298         INIT_LIST_HEAD(&base->vm_status);
 299 
 300         if (!bo)
 301                 return;
 302         base->next = bo->vm_bo;
 303         bo->vm_bo = base;
 304 
 305         if (bo->tbo.base.resv != vm->root.base.bo->tbo.base.resv)
 306                 return;
 307 
 308         vm->bulk_moveable = false;
 309         if (bo->tbo.type == ttm_bo_type_kernel && bo->parent)
 310                 amdgpu_vm_bo_relocated(base);
 311         else
 312                 amdgpu_vm_bo_idle(base);
 313 
 314         if (bo->preferred_domains &
 315             amdgpu_mem_type_to_domain(bo->tbo.mem.mem_type))
 316                 return;
 317 
 318         /*
 319          * we checked all the prerequisites, but it looks like this per vm bo
 320          * is currently evicted. add the bo to the evicted list to make sure it
 321          * is validated on next vm use to avoid fault.
 322          * */
 323         amdgpu_vm_bo_evicted(base);
 324 }
 325 
 326 /**
 327  * amdgpu_vm_pt_parent - get the parent page directory
 328  *
 329  * @pt: child page table
 330  *
 331  * Helper to get the parent entry for the child page table. NULL if we are at
 332  * the root page directory.
 333  */
 334 static struct amdgpu_vm_pt *amdgpu_vm_pt_parent(struct amdgpu_vm_pt *pt)
 335 {
 336         struct amdgpu_bo *parent = pt->base.bo->parent;
 337 
 338         if (!parent)
 339                 return NULL;
 340 
 341         return container_of(parent->vm_bo, struct amdgpu_vm_pt, base);
 342 }
 343 
 344 /**
 345  * amdgpu_vm_pt_cursor - state for for_each_amdgpu_vm_pt
 346  */
 347 struct amdgpu_vm_pt_cursor {
 348         uint64_t pfn;
 349         struct amdgpu_vm_pt *parent;
 350         struct amdgpu_vm_pt *entry;
 351         unsigned level;
 352 };
 353 
 354 /**
 355  * amdgpu_vm_pt_start - start PD/PT walk
 356  *
 357  * @adev: amdgpu_device pointer
 358  * @vm: amdgpu_vm structure
 359  * @start: start address of the walk
 360  * @cursor: state to initialize
 361  *
 362  * Initialize a amdgpu_vm_pt_cursor to start a walk.
 363  */
 364 static void amdgpu_vm_pt_start(struct amdgpu_device *adev,
 365                                struct amdgpu_vm *vm, uint64_t start,
 366                                struct amdgpu_vm_pt_cursor *cursor)
 367 {
 368         cursor->pfn = start;
 369         cursor->parent = NULL;
 370         cursor->entry = &vm->root;
 371         cursor->level = adev->vm_manager.root_level;
 372 }
 373 
 374 /**
 375  * amdgpu_vm_pt_descendant - go to child node
 376  *
 377  * @adev: amdgpu_device pointer
 378  * @cursor: current state
 379  *
 380  * Walk to the child node of the current node.
 381  * Returns:
 382  * True if the walk was possible, false otherwise.
 383  */
 384 static bool amdgpu_vm_pt_descendant(struct amdgpu_device *adev,
 385                                     struct amdgpu_vm_pt_cursor *cursor)
 386 {
 387         unsigned mask, shift, idx;
 388 
 389         if (!cursor->entry->entries)
 390                 return false;
 391 
 392         BUG_ON(!cursor->entry->base.bo);
 393         mask = amdgpu_vm_entries_mask(adev, cursor->level);
 394         shift = amdgpu_vm_level_shift(adev, cursor->level);
 395 
 396         ++cursor->level;
 397         idx = (cursor->pfn >> shift) & mask;
 398         cursor->parent = cursor->entry;
 399         cursor->entry = &cursor->entry->entries[idx];
 400         return true;
 401 }
 402 
 403 /**
 404  * amdgpu_vm_pt_sibling - go to sibling node
 405  *
 406  * @adev: amdgpu_device pointer
 407  * @cursor: current state
 408  *
 409  * Walk to the sibling node of the current node.
 410  * Returns:
 411  * True if the walk was possible, false otherwise.
 412  */
 413 static bool amdgpu_vm_pt_sibling(struct amdgpu_device *adev,
 414                                  struct amdgpu_vm_pt_cursor *cursor)
 415 {
 416         unsigned shift, num_entries;
 417 
 418         /* Root doesn't have a sibling */
 419         if (!cursor->parent)
 420                 return false;
 421 
 422         /* Go to our parents and see if we got a sibling */
 423         shift = amdgpu_vm_level_shift(adev, cursor->level - 1);
 424         num_entries = amdgpu_vm_num_entries(adev, cursor->level - 1);
 425 
 426         if (cursor->entry == &cursor->parent->entries[num_entries - 1])
 427                 return false;
 428 
 429         cursor->pfn += 1ULL << shift;
 430         cursor->pfn &= ~((1ULL << shift) - 1);
 431         ++cursor->entry;
 432         return true;
 433 }
 434 
 435 /**
 436  * amdgpu_vm_pt_ancestor - go to parent node
 437  *
 438  * @cursor: current state
 439  *
 440  * Walk to the parent node of the current node.
 441  * Returns:
 442  * True if the walk was possible, false otherwise.
 443  */
 444 static bool amdgpu_vm_pt_ancestor(struct amdgpu_vm_pt_cursor *cursor)
 445 {
 446         if (!cursor->parent)
 447                 return false;
 448 
 449         --cursor->level;
 450         cursor->entry = cursor->parent;
 451         cursor->parent = amdgpu_vm_pt_parent(cursor->parent);
 452         return true;
 453 }
 454 
 455 /**
 456  * amdgpu_vm_pt_next - get next PD/PT in hieratchy
 457  *
 458  * @adev: amdgpu_device pointer
 459  * @cursor: current state
 460  *
 461  * Walk the PD/PT tree to the next node.
 462  */
 463 static void amdgpu_vm_pt_next(struct amdgpu_device *adev,
 464                               struct amdgpu_vm_pt_cursor *cursor)
 465 {
 466         /* First try a newborn child */
 467         if (amdgpu_vm_pt_descendant(adev, cursor))
 468                 return;
 469 
 470         /* If that didn't worked try to find a sibling */
 471         while (!amdgpu_vm_pt_sibling(adev, cursor)) {
 472                 /* No sibling, go to our parents and grandparents */
 473                 if (!amdgpu_vm_pt_ancestor(cursor)) {
 474                         cursor->pfn = ~0ll;
 475                         return;
 476                 }
 477         }
 478 }
 479 
 480 /**
 481  * amdgpu_vm_pt_first_dfs - start a deep first search
 482  *
 483  * @adev: amdgpu_device structure
 484  * @vm: amdgpu_vm structure
 485  * @cursor: state to initialize
 486  *
 487  * Starts a deep first traversal of the PD/PT tree.
 488  */
 489 static void amdgpu_vm_pt_first_dfs(struct amdgpu_device *adev,
 490                                    struct amdgpu_vm *vm,
 491                                    struct amdgpu_vm_pt_cursor *start,
 492                                    struct amdgpu_vm_pt_cursor *cursor)
 493 {
 494         if (start)
 495                 *cursor = *start;
 496         else
 497                 amdgpu_vm_pt_start(adev, vm, 0, cursor);
 498         while (amdgpu_vm_pt_descendant(adev, cursor));
 499 }
 500 
 501 /**
 502  * amdgpu_vm_pt_continue_dfs - check if the deep first search should continue
 503  *
 504  * @start: starting point for the search
 505  * @entry: current entry
 506  *
 507  * Returns:
 508  * True when the search should continue, false otherwise.
 509  */
 510 static bool amdgpu_vm_pt_continue_dfs(struct amdgpu_vm_pt_cursor *start,
 511                                       struct amdgpu_vm_pt *entry)
 512 {
 513         return entry && (!start || entry != start->entry);
 514 }
 515 
 516 /**
 517  * amdgpu_vm_pt_next_dfs - get the next node for a deep first search
 518  *
 519  * @adev: amdgpu_device structure
 520  * @cursor: current state
 521  *
 522  * Move the cursor to the next node in a deep first search.
 523  */
 524 static void amdgpu_vm_pt_next_dfs(struct amdgpu_device *adev,
 525                                   struct amdgpu_vm_pt_cursor *cursor)
 526 {
 527         if (!cursor->entry)
 528                 return;
 529 
 530         if (!cursor->parent)
 531                 cursor->entry = NULL;
 532         else if (amdgpu_vm_pt_sibling(adev, cursor))
 533                 while (amdgpu_vm_pt_descendant(adev, cursor));
 534         else
 535                 amdgpu_vm_pt_ancestor(cursor);
 536 }
 537 
 538 /**
 539  * for_each_amdgpu_vm_pt_dfs_safe - safe deep first search of all PDs/PTs
 540  */
 541 #define for_each_amdgpu_vm_pt_dfs_safe(adev, vm, start, cursor, entry)          \
 542         for (amdgpu_vm_pt_first_dfs((adev), (vm), (start), &(cursor)),          \
 543              (entry) = (cursor).entry, amdgpu_vm_pt_next_dfs((adev), &(cursor));\
 544              amdgpu_vm_pt_continue_dfs((start), (entry));                       \
 545              (entry) = (cursor).entry, amdgpu_vm_pt_next_dfs((adev), &(cursor)))
 546 
 547 /**
 548  * amdgpu_vm_get_pd_bo - add the VM PD to a validation list
 549  *
 550  * @vm: vm providing the BOs
 551  * @validated: head of validation list
 552  * @entry: entry to add
 553  *
 554  * Add the page directory to the list of BOs to
 555  * validate for command submission.
 556  */
 557 void amdgpu_vm_get_pd_bo(struct amdgpu_vm *vm,
 558                          struct list_head *validated,
 559                          struct amdgpu_bo_list_entry *entry)
 560 {
 561         entry->priority = 0;
 562         entry->tv.bo = &vm->root.base.bo->tbo;
 563         /* One for the VM updates, one for TTM and one for the CS job */
 564         entry->tv.num_shared = 3;
 565         entry->user_pages = NULL;
 566         list_add(&entry->tv.head, validated);
 567 }
 568 
 569 void amdgpu_vm_del_from_lru_notify(struct ttm_buffer_object *bo)
 570 {
 571         struct amdgpu_bo *abo;
 572         struct amdgpu_vm_bo_base *bo_base;
 573 
 574         if (!amdgpu_bo_is_amdgpu_bo(bo))
 575                 return;
 576 
 577         if (bo->mem.placement & TTM_PL_FLAG_NO_EVICT)
 578                 return;
 579 
 580         abo = ttm_to_amdgpu_bo(bo);
 581         if (!abo->parent)
 582                 return;
 583         for (bo_base = abo->vm_bo; bo_base; bo_base = bo_base->next) {
 584                 struct amdgpu_vm *vm = bo_base->vm;
 585 
 586                 if (abo->tbo.base.resv == vm->root.base.bo->tbo.base.resv)
 587                         vm->bulk_moveable = false;
 588         }
 589 
 590 }
 591 /**
 592  * amdgpu_vm_move_to_lru_tail - move all BOs to the end of LRU
 593  *
 594  * @adev: amdgpu device pointer
 595  * @vm: vm providing the BOs
 596  *
 597  * Move all BOs to the end of LRU and remember their positions to put them
 598  * together.
 599  */
 600 void amdgpu_vm_move_to_lru_tail(struct amdgpu_device *adev,
 601                                 struct amdgpu_vm *vm)
 602 {
 603         struct ttm_bo_global *glob = adev->mman.bdev.glob;
 604         struct amdgpu_vm_bo_base *bo_base;
 605 
 606         if (vm->bulk_moveable) {
 607                 spin_lock(&glob->lru_lock);
 608                 ttm_bo_bulk_move_lru_tail(&vm->lru_bulk_move);
 609                 spin_unlock(&glob->lru_lock);
 610                 return;
 611         }
 612 
 613         memset(&vm->lru_bulk_move, 0, sizeof(vm->lru_bulk_move));
 614 
 615         spin_lock(&glob->lru_lock);
 616         list_for_each_entry(bo_base, &vm->idle, vm_status) {
 617                 struct amdgpu_bo *bo = bo_base->bo;
 618 
 619                 if (!bo->parent)
 620                         continue;
 621 
 622                 ttm_bo_move_to_lru_tail(&bo->tbo, &vm->lru_bulk_move);
 623                 if (bo->shadow)
 624                         ttm_bo_move_to_lru_tail(&bo->shadow->tbo,
 625                                                 &vm->lru_bulk_move);
 626         }
 627         spin_unlock(&glob->lru_lock);
 628 
 629         vm->bulk_moveable = true;
 630 }
 631 
 632 /**
 633  * amdgpu_vm_validate_pt_bos - validate the page table BOs
 634  *
 635  * @adev: amdgpu device pointer
 636  * @vm: vm providing the BOs
 637  * @validate: callback to do the validation
 638  * @param: parameter for the validation callback
 639  *
 640  * Validate the page table BOs on command submission if neccessary.
 641  *
 642  * Returns:
 643  * Validation result.
 644  */
 645 int amdgpu_vm_validate_pt_bos(struct amdgpu_device *adev, struct amdgpu_vm *vm,
 646                               int (*validate)(void *p, struct amdgpu_bo *bo),
 647                               void *param)
 648 {
 649         struct amdgpu_vm_bo_base *bo_base, *tmp;
 650         int r = 0;
 651 
 652         vm->bulk_moveable &= list_empty(&vm->evicted);
 653 
 654         list_for_each_entry_safe(bo_base, tmp, &vm->evicted, vm_status) {
 655                 struct amdgpu_bo *bo = bo_base->bo;
 656 
 657                 r = validate(param, bo);
 658                 if (r)
 659                         break;
 660 
 661                 if (bo->tbo.type != ttm_bo_type_kernel) {
 662                         amdgpu_vm_bo_moved(bo_base);
 663                 } else {
 664                         vm->update_funcs->map_table(bo);
 665                         if (bo->parent)
 666                                 amdgpu_vm_bo_relocated(bo_base);
 667                         else
 668                                 amdgpu_vm_bo_idle(bo_base);
 669                 }
 670         }
 671 
 672         return r;
 673 }
 674 
 675 /**
 676  * amdgpu_vm_ready - check VM is ready for updates
 677  *
 678  * @vm: VM to check
 679  *
 680  * Check if all VM PDs/PTs are ready for updates
 681  *
 682  * Returns:
 683  * True if eviction list is empty.
 684  */
 685 bool amdgpu_vm_ready(struct amdgpu_vm *vm)
 686 {
 687         return list_empty(&vm->evicted);
 688 }
 689 
 690 /**
 691  * amdgpu_vm_clear_bo - initially clear the PDs/PTs
 692  *
 693  * @adev: amdgpu_device pointer
 694  * @vm: VM to clear BO from
 695  * @bo: BO to clear
 696  *
 697  * Root PD needs to be reserved when calling this.
 698  *
 699  * Returns:
 700  * 0 on success, errno otherwise.
 701  */
 702 static int amdgpu_vm_clear_bo(struct amdgpu_device *adev,
 703                               struct amdgpu_vm *vm,
 704                               struct amdgpu_bo *bo)
 705 {
 706         struct ttm_operation_ctx ctx = { true, false };
 707         unsigned level = adev->vm_manager.root_level;
 708         struct amdgpu_vm_update_params params;
 709         struct amdgpu_bo *ancestor = bo;
 710         unsigned entries, ats_entries;
 711         uint64_t addr;
 712         int r;
 713 
 714         /* Figure out our place in the hierarchy */
 715         if (ancestor->parent) {
 716                 ++level;
 717                 while (ancestor->parent->parent) {
 718                         ++level;
 719                         ancestor = ancestor->parent;
 720                 }
 721         }
 722 
 723         entries = amdgpu_bo_size(bo) / 8;
 724         if (!vm->pte_support_ats) {
 725                 ats_entries = 0;
 726 
 727         } else if (!bo->parent) {
 728                 ats_entries = amdgpu_vm_num_ats_entries(adev);
 729                 ats_entries = min(ats_entries, entries);
 730                 entries -= ats_entries;
 731 
 732         } else {
 733                 struct amdgpu_vm_pt *pt;
 734 
 735                 pt = container_of(ancestor->vm_bo, struct amdgpu_vm_pt, base);
 736                 ats_entries = amdgpu_vm_num_ats_entries(adev);
 737                 if ((pt - vm->root.entries) >= ats_entries) {
 738                         ats_entries = 0;
 739                 } else {
 740                         ats_entries = entries;
 741                         entries = 0;
 742                 }
 743         }
 744 
 745         r = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
 746         if (r)
 747                 return r;
 748 
 749         if (bo->shadow) {
 750                 r = ttm_bo_validate(&bo->shadow->tbo, &bo->shadow->placement,
 751                                     &ctx);
 752                 if (r)
 753                         return r;
 754         }
 755 
 756         r = vm->update_funcs->map_table(bo);
 757         if (r)
 758                 return r;
 759 
 760         memset(&params, 0, sizeof(params));
 761         params.adev = adev;
 762         params.vm = vm;
 763 
 764         r = vm->update_funcs->prepare(&params, AMDGPU_FENCE_OWNER_KFD, NULL);
 765         if (r)
 766                 return r;
 767 
 768         addr = 0;
 769         if (ats_entries) {
 770                 uint64_t value = 0, flags;
 771 
 772                 flags = AMDGPU_PTE_DEFAULT_ATC;
 773                 if (level != AMDGPU_VM_PTB) {
 774                         /* Handle leaf PDEs as PTEs */
 775                         flags |= AMDGPU_PDE_PTE;
 776                         amdgpu_gmc_get_vm_pde(adev, level, &value, &flags);
 777                 }
 778 
 779                 r = vm->update_funcs->update(&params, bo, addr, 0, ats_entries,
 780                                              value, flags);
 781                 if (r)
 782                         return r;
 783 
 784                 addr += ats_entries * 8;
 785         }
 786 
 787         if (entries) {
 788                 uint64_t value = 0, flags = 0;
 789 
 790                 if (adev->asic_type >= CHIP_VEGA10) {
 791                         if (level != AMDGPU_VM_PTB) {
 792                                 /* Handle leaf PDEs as PTEs */
 793                                 flags |= AMDGPU_PDE_PTE;
 794                                 amdgpu_gmc_get_vm_pde(adev, level,
 795                                                       &value, &flags);
 796                         } else {
 797                                 /* Workaround for fault priority problem on GMC9 */
 798                                 flags = AMDGPU_PTE_EXECUTABLE;
 799                         }
 800                 }
 801 
 802                 r = vm->update_funcs->update(&params, bo, addr, 0, entries,
 803                                              value, flags);
 804                 if (r)
 805                         return r;
 806         }
 807 
 808         return vm->update_funcs->commit(&params, NULL);
 809 }
 810 
 811 /**
 812  * amdgpu_vm_bo_param - fill in parameters for PD/PT allocation
 813  *
 814  * @adev: amdgpu_device pointer
 815  * @vm: requesting vm
 816  * @bp: resulting BO allocation parameters
 817  */
 818 static void amdgpu_vm_bo_param(struct amdgpu_device *adev, struct amdgpu_vm *vm,
 819                                int level, struct amdgpu_bo_param *bp)
 820 {
 821         memset(bp, 0, sizeof(*bp));
 822 
 823         bp->size = amdgpu_vm_bo_size(adev, level);
 824         bp->byte_align = AMDGPU_GPU_PAGE_SIZE;
 825         bp->domain = AMDGPU_GEM_DOMAIN_VRAM;
 826         bp->domain = amdgpu_bo_get_preferred_pin_domain(adev, bp->domain);
 827         bp->flags = AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS |
 828                 AMDGPU_GEM_CREATE_CPU_GTT_USWC;
 829         if (vm->use_cpu_for_update)
 830                 bp->flags |= AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED;
 831         else if (!vm->root.base.bo || vm->root.base.bo->shadow)
 832                 bp->flags |= AMDGPU_GEM_CREATE_SHADOW;
 833         bp->type = ttm_bo_type_kernel;
 834         if (vm->root.base.bo)
 835                 bp->resv = vm->root.base.bo->tbo.base.resv;
 836 }
 837 
 838 /**
 839  * amdgpu_vm_alloc_pts - Allocate a specific page table
 840  *
 841  * @adev: amdgpu_device pointer
 842  * @vm: VM to allocate page tables for
 843  * @cursor: Which page table to allocate
 844  *
 845  * Make sure a specific page table or directory is allocated.
 846  *
 847  * Returns:
 848  * 1 if page table needed to be allocated, 0 if page table was already
 849  * allocated, negative errno if an error occurred.
 850  */
 851 static int amdgpu_vm_alloc_pts(struct amdgpu_device *adev,
 852                                struct amdgpu_vm *vm,
 853                                struct amdgpu_vm_pt_cursor *cursor)
 854 {
 855         struct amdgpu_vm_pt *entry = cursor->entry;
 856         struct amdgpu_bo_param bp;
 857         struct amdgpu_bo *pt;
 858         int r;
 859 
 860         if (cursor->level < AMDGPU_VM_PTB && !entry->entries) {
 861                 unsigned num_entries;
 862 
 863                 num_entries = amdgpu_vm_num_entries(adev, cursor->level);
 864                 entry->entries = kvmalloc_array(num_entries,
 865                                                 sizeof(*entry->entries),
 866                                                 GFP_KERNEL | __GFP_ZERO);
 867                 if (!entry->entries)
 868                         return -ENOMEM;
 869         }
 870 
 871         if (entry->base.bo)
 872                 return 0;
 873 
 874         amdgpu_vm_bo_param(adev, vm, cursor->level, &bp);
 875 
 876         r = amdgpu_bo_create(adev, &bp, &pt);
 877         if (r)
 878                 return r;
 879 
 880         /* Keep a reference to the root directory to avoid
 881          * freeing them up in the wrong order.
 882          */
 883         pt->parent = amdgpu_bo_ref(cursor->parent->base.bo);
 884         amdgpu_vm_bo_base_init(&entry->base, vm, pt);
 885 
 886         r = amdgpu_vm_clear_bo(adev, vm, pt);
 887         if (r)
 888                 goto error_free_pt;
 889 
 890         return 0;
 891 
 892 error_free_pt:
 893         amdgpu_bo_unref(&pt->shadow);
 894         amdgpu_bo_unref(&pt);
 895         return r;
 896 }
 897 
 898 /**
 899  * amdgpu_vm_free_table - fre one PD/PT
 900  *
 901  * @entry: PDE to free
 902  */
 903 static void amdgpu_vm_free_table(struct amdgpu_vm_pt *entry)
 904 {
 905         if (entry->base.bo) {
 906                 entry->base.bo->vm_bo = NULL;
 907                 list_del(&entry->base.vm_status);
 908                 amdgpu_bo_unref(&entry->base.bo->shadow);
 909                 amdgpu_bo_unref(&entry->base.bo);
 910         }
 911         kvfree(entry->entries);
 912         entry->entries = NULL;
 913 }
 914 
 915 /**
 916  * amdgpu_vm_free_pts - free PD/PT levels
 917  *
 918  * @adev: amdgpu device structure
 919  * @vm: amdgpu vm structure
 920  * @start: optional cursor where to start freeing PDs/PTs
 921  *
 922  * Free the page directory or page table level and all sub levels.
 923  */
 924 static void amdgpu_vm_free_pts(struct amdgpu_device *adev,
 925                                struct amdgpu_vm *vm,
 926                                struct amdgpu_vm_pt_cursor *start)
 927 {
 928         struct amdgpu_vm_pt_cursor cursor;
 929         struct amdgpu_vm_pt *entry;
 930 
 931         vm->bulk_moveable = false;
 932 
 933         for_each_amdgpu_vm_pt_dfs_safe(adev, vm, start, cursor, entry)
 934                 amdgpu_vm_free_table(entry);
 935 
 936         if (start)
 937                 amdgpu_vm_free_table(start->entry);
 938 }
 939 
 940 /**
 941  * amdgpu_vm_check_compute_bug - check whether asic has compute vm bug
 942  *
 943  * @adev: amdgpu_device pointer
 944  */
 945 void amdgpu_vm_check_compute_bug(struct amdgpu_device *adev)
 946 {
 947         const struct amdgpu_ip_block *ip_block;
 948         bool has_compute_vm_bug;
 949         struct amdgpu_ring *ring;
 950         int i;
 951 
 952         has_compute_vm_bug = false;
 953 
 954         ip_block = amdgpu_device_ip_get_ip_block(adev, AMD_IP_BLOCK_TYPE_GFX);
 955         if (ip_block) {
 956                 /* Compute has a VM bug for GFX version < 7.
 957                    Compute has a VM bug for GFX 8 MEC firmware version < 673.*/
 958                 if (ip_block->version->major <= 7)
 959                         has_compute_vm_bug = true;
 960                 else if (ip_block->version->major == 8)
 961                         if (adev->gfx.mec_fw_version < 673)
 962                                 has_compute_vm_bug = true;
 963         }
 964 
 965         for (i = 0; i < adev->num_rings; i++) {
 966                 ring = adev->rings[i];
 967                 if (ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE)
 968                         /* only compute rings */
 969                         ring->has_compute_vm_bug = has_compute_vm_bug;
 970                 else
 971                         ring->has_compute_vm_bug = false;
 972         }
 973 }
 974 
 975 /**
 976  * amdgpu_vm_need_pipeline_sync - Check if pipe sync is needed for job.
 977  *
 978  * @ring: ring on which the job will be submitted
 979  * @job: job to submit
 980  *
 981  * Returns:
 982  * True if sync is needed.
 983  */
 984 bool amdgpu_vm_need_pipeline_sync(struct amdgpu_ring *ring,
 985                                   struct amdgpu_job *job)
 986 {
 987         struct amdgpu_device *adev = ring->adev;
 988         unsigned vmhub = ring->funcs->vmhub;
 989         struct amdgpu_vmid_mgr *id_mgr = &adev->vm_manager.id_mgr[vmhub];
 990         struct amdgpu_vmid *id;
 991         bool gds_switch_needed;
 992         bool vm_flush_needed = job->vm_needs_flush || ring->has_compute_vm_bug;
 993 
 994         if (job->vmid == 0)
 995                 return false;
 996         id = &id_mgr->ids[job->vmid];
 997         gds_switch_needed = ring->funcs->emit_gds_switch && (
 998                 id->gds_base != job->gds_base ||
 999                 id->gds_size != job->gds_size ||
1000                 id->gws_base != job->gws_base ||
1001                 id->gws_size != job->gws_size ||
1002                 id->oa_base != job->oa_base ||
1003                 id->oa_size != job->oa_size);
1004 
1005         if (amdgpu_vmid_had_gpu_reset(adev, id))
1006                 return true;
1007 
1008         return vm_flush_needed || gds_switch_needed;
1009 }
1010 
1011 /**
1012  * amdgpu_vm_flush - hardware flush the vm
1013  *
1014  * @ring: ring to use for flush
1015  * @job:  related job
1016  * @need_pipe_sync: is pipe sync needed
1017  *
1018  * Emit a VM flush when it is necessary.
1019  *
1020  * Returns:
1021  * 0 on success, errno otherwise.
1022  */
1023 int amdgpu_vm_flush(struct amdgpu_ring *ring, struct amdgpu_job *job, bool need_pipe_sync)
1024 {
1025         struct amdgpu_device *adev = ring->adev;
1026         unsigned vmhub = ring->funcs->vmhub;
1027         struct amdgpu_vmid_mgr *id_mgr = &adev->vm_manager.id_mgr[vmhub];
1028         struct amdgpu_vmid *id = &id_mgr->ids[job->vmid];
1029         bool gds_switch_needed = ring->funcs->emit_gds_switch && (
1030                 id->gds_base != job->gds_base ||
1031                 id->gds_size != job->gds_size ||
1032                 id->gws_base != job->gws_base ||
1033                 id->gws_size != job->gws_size ||
1034                 id->oa_base != job->oa_base ||
1035                 id->oa_size != job->oa_size);
1036         bool vm_flush_needed = job->vm_needs_flush;
1037         struct dma_fence *fence = NULL;
1038         bool pasid_mapping_needed = false;
1039         unsigned patch_offset = 0;
1040         int r;
1041 
1042         if (amdgpu_vmid_had_gpu_reset(adev, id)) {
1043                 gds_switch_needed = true;
1044                 vm_flush_needed = true;
1045                 pasid_mapping_needed = true;
1046         }
1047 
1048         mutex_lock(&id_mgr->lock);
1049         if (id->pasid != job->pasid || !id->pasid_mapping ||
1050             !dma_fence_is_signaled(id->pasid_mapping))
1051                 pasid_mapping_needed = true;
1052         mutex_unlock(&id_mgr->lock);
1053 
1054         gds_switch_needed &= !!ring->funcs->emit_gds_switch;
1055         vm_flush_needed &= !!ring->funcs->emit_vm_flush  &&
1056                         job->vm_pd_addr != AMDGPU_BO_INVALID_OFFSET;
1057         pasid_mapping_needed &= adev->gmc.gmc_funcs->emit_pasid_mapping &&
1058                 ring->funcs->emit_wreg;
1059 
1060         if (!vm_flush_needed && !gds_switch_needed && !need_pipe_sync)
1061                 return 0;
1062 
1063         if (ring->funcs->init_cond_exec)
1064                 patch_offset = amdgpu_ring_init_cond_exec(ring);
1065 
1066         if (need_pipe_sync)
1067                 amdgpu_ring_emit_pipeline_sync(ring);
1068 
1069         if (vm_flush_needed) {
1070                 trace_amdgpu_vm_flush(ring, job->vmid, job->vm_pd_addr);
1071                 amdgpu_ring_emit_vm_flush(ring, job->vmid, job->vm_pd_addr);
1072         }
1073 
1074         if (pasid_mapping_needed)
1075                 amdgpu_gmc_emit_pasid_mapping(ring, job->vmid, job->pasid);
1076 
1077         if (vm_flush_needed || pasid_mapping_needed) {
1078                 r = amdgpu_fence_emit(ring, &fence, 0);
1079                 if (r)
1080                         return r;
1081         }
1082 
1083         if (vm_flush_needed) {
1084                 mutex_lock(&id_mgr->lock);
1085                 dma_fence_put(id->last_flush);
1086                 id->last_flush = dma_fence_get(fence);
1087                 id->current_gpu_reset_count =
1088                         atomic_read(&adev->gpu_reset_counter);
1089                 mutex_unlock(&id_mgr->lock);
1090         }
1091 
1092         if (pasid_mapping_needed) {
1093                 mutex_lock(&id_mgr->lock);
1094                 id->pasid = job->pasid;
1095                 dma_fence_put(id->pasid_mapping);
1096                 id->pasid_mapping = dma_fence_get(fence);
1097                 mutex_unlock(&id_mgr->lock);
1098         }
1099         dma_fence_put(fence);
1100 
1101         if (ring->funcs->emit_gds_switch && gds_switch_needed) {
1102                 id->gds_base = job->gds_base;
1103                 id->gds_size = job->gds_size;
1104                 id->gws_base = job->gws_base;
1105                 id->gws_size = job->gws_size;
1106                 id->oa_base = job->oa_base;
1107                 id->oa_size = job->oa_size;
1108                 amdgpu_ring_emit_gds_switch(ring, job->vmid, job->gds_base,
1109                                             job->gds_size, job->gws_base,
1110                                             job->gws_size, job->oa_base,
1111                                             job->oa_size);
1112         }
1113 
1114         if (ring->funcs->patch_cond_exec)
1115                 amdgpu_ring_patch_cond_exec(ring, patch_offset);
1116 
1117         /* the double SWITCH_BUFFER here *cannot* be skipped by COND_EXEC */
1118         if (ring->funcs->emit_switch_buffer) {
1119                 amdgpu_ring_emit_switch_buffer(ring);
1120                 amdgpu_ring_emit_switch_buffer(ring);
1121         }
1122         return 0;
1123 }
1124 
1125 /**
1126  * amdgpu_vm_bo_find - find the bo_va for a specific vm & bo
1127  *
1128  * @vm: requested vm
1129  * @bo: requested buffer object
1130  *
1131  * Find @bo inside the requested vm.
1132  * Search inside the @bos vm list for the requested vm
1133  * Returns the found bo_va or NULL if none is found
1134  *
1135  * Object has to be reserved!
1136  *
1137  * Returns:
1138  * Found bo_va or NULL.
1139  */
1140 struct amdgpu_bo_va *amdgpu_vm_bo_find(struct amdgpu_vm *vm,
1141                                        struct amdgpu_bo *bo)
1142 {
1143         struct amdgpu_vm_bo_base *base;
1144 
1145         for (base = bo->vm_bo; base; base = base->next) {
1146                 if (base->vm != vm)
1147                         continue;
1148 
1149                 return container_of(base, struct amdgpu_bo_va, base);
1150         }
1151         return NULL;
1152 }
1153 
1154 /**
1155  * amdgpu_vm_map_gart - Resolve gart mapping of addr
1156  *
1157  * @pages_addr: optional DMA address to use for lookup
1158  * @addr: the unmapped addr
1159  *
1160  * Look up the physical address of the page that the pte resolves
1161  * to.
1162  *
1163  * Returns:
1164  * The pointer for the page table entry.
1165  */
1166 uint64_t amdgpu_vm_map_gart(const dma_addr_t *pages_addr, uint64_t addr)
1167 {
1168         uint64_t result;
1169 
1170         /* page table offset */
1171         result = pages_addr[addr >> PAGE_SHIFT];
1172 
1173         /* in case cpu page size != gpu page size*/
1174         result |= addr & (~PAGE_MASK);
1175 
1176         result &= 0xFFFFFFFFFFFFF000ULL;
1177 
1178         return result;
1179 }
1180 
1181 /*
1182  * amdgpu_vm_update_pde - update a single level in the hierarchy
1183  *
1184  * @param: parameters for the update
1185  * @vm: requested vm
1186  * @entry: entry to update
1187  *
1188  * Makes sure the requested entry in parent is up to date.
1189  */
1190 static int amdgpu_vm_update_pde(struct amdgpu_vm_update_params *params,
1191                                 struct amdgpu_vm *vm,
1192                                 struct amdgpu_vm_pt *entry)
1193 {
1194         struct amdgpu_vm_pt *parent = amdgpu_vm_pt_parent(entry);
1195         struct amdgpu_bo *bo = parent->base.bo, *pbo;
1196         uint64_t pde, pt, flags;
1197         unsigned level;
1198 
1199         for (level = 0, pbo = bo->parent; pbo; ++level)
1200                 pbo = pbo->parent;
1201 
1202         level += params->adev->vm_manager.root_level;
1203         amdgpu_gmc_get_pde_for_bo(entry->base.bo, level, &pt, &flags);
1204         pde = (entry - parent->entries) * 8;
1205         return vm->update_funcs->update(params, bo, pde, pt, 1, 0, flags);
1206 }
1207 
1208 /*
1209  * amdgpu_vm_invalidate_pds - mark all PDs as invalid
1210  *
1211  * @adev: amdgpu_device pointer
1212  * @vm: related vm
1213  *
1214  * Mark all PD level as invalid after an error.
1215  */
1216 static void amdgpu_vm_invalidate_pds(struct amdgpu_device *adev,
1217                                      struct amdgpu_vm *vm)
1218 {
1219         struct amdgpu_vm_pt_cursor cursor;
1220         struct amdgpu_vm_pt *entry;
1221 
1222         for_each_amdgpu_vm_pt_dfs_safe(adev, vm, NULL, cursor, entry)
1223                 if (entry->base.bo && !entry->base.moved)
1224                         amdgpu_vm_bo_relocated(&entry->base);
1225 }
1226 
1227 /*
1228  * amdgpu_vm_update_directories - make sure that all directories are valid
1229  *
1230  * @adev: amdgpu_device pointer
1231  * @vm: requested vm
1232  *
1233  * Makes sure all directories are up to date.
1234  *
1235  * Returns:
1236  * 0 for success, error for failure.
1237  */
1238 int amdgpu_vm_update_directories(struct amdgpu_device *adev,
1239                                  struct amdgpu_vm *vm)
1240 {
1241         struct amdgpu_vm_update_params params;
1242         int r;
1243 
1244         if (list_empty(&vm->relocated))
1245                 return 0;
1246 
1247         memset(&params, 0, sizeof(params));
1248         params.adev = adev;
1249         params.vm = vm;
1250 
1251         r = vm->update_funcs->prepare(&params, AMDGPU_FENCE_OWNER_VM, NULL);
1252         if (r)
1253                 return r;
1254 
1255         while (!list_empty(&vm->relocated)) {
1256                 struct amdgpu_vm_pt *entry;
1257 
1258                 entry = list_first_entry(&vm->relocated, struct amdgpu_vm_pt,
1259                                          base.vm_status);
1260                 amdgpu_vm_bo_idle(&entry->base);
1261 
1262                 r = amdgpu_vm_update_pde(&params, vm, entry);
1263                 if (r)
1264                         goto error;
1265         }
1266 
1267         r = vm->update_funcs->commit(&params, &vm->last_update);
1268         if (r)
1269                 goto error;
1270         return 0;
1271 
1272 error:
1273         amdgpu_vm_invalidate_pds(adev, vm);
1274         return r;
1275 }
1276 
1277 /**
1278  * amdgpu_vm_update_flags - figure out flags for PTE updates
1279  *
1280  * Make sure to set the right flags for the PTEs at the desired level.
1281  */
1282 static void amdgpu_vm_update_flags(struct amdgpu_vm_update_params *params,
1283                                    struct amdgpu_bo *bo, unsigned level,
1284                                    uint64_t pe, uint64_t addr,
1285                                    unsigned count, uint32_t incr,
1286                                    uint64_t flags)
1287 
1288 {
1289         if (level != AMDGPU_VM_PTB) {
1290                 flags |= AMDGPU_PDE_PTE;
1291                 amdgpu_gmc_get_vm_pde(params->adev, level, &addr, &flags);
1292 
1293         } else if (params->adev->asic_type >= CHIP_VEGA10 &&
1294                    !(flags & AMDGPU_PTE_VALID) &&
1295                    !(flags & AMDGPU_PTE_PRT)) {
1296 
1297                 /* Workaround for fault priority problem on GMC9 */
1298                 flags |= AMDGPU_PTE_EXECUTABLE;
1299         }
1300 
1301         params->vm->update_funcs->update(params, bo, pe, addr, count, incr,
1302                                          flags);
1303 }
1304 
1305 /**
1306  * amdgpu_vm_fragment - get fragment for PTEs
1307  *
1308  * @params: see amdgpu_vm_update_params definition
1309  * @start: first PTE to handle
1310  * @end: last PTE to handle
1311  * @flags: hw mapping flags
1312  * @frag: resulting fragment size
1313  * @frag_end: end of this fragment
1314  *
1315  * Returns the first possible fragment for the start and end address.
1316  */
1317 static void amdgpu_vm_fragment(struct amdgpu_vm_update_params *params,
1318                                uint64_t start, uint64_t end, uint64_t flags,
1319                                unsigned int *frag, uint64_t *frag_end)
1320 {
1321         /**
1322          * The MC L1 TLB supports variable sized pages, based on a fragment
1323          * field in the PTE. When this field is set to a non-zero value, page
1324          * granularity is increased from 4KB to (1 << (12 + frag)). The PTE
1325          * flags are considered valid for all PTEs within the fragment range
1326          * and corresponding mappings are assumed to be physically contiguous.
1327          *
1328          * The L1 TLB can store a single PTE for the whole fragment,
1329          * significantly increasing the space available for translation
1330          * caching. This leads to large improvements in throughput when the
1331          * TLB is under pressure.
1332          *
1333          * The L2 TLB distributes small and large fragments into two
1334          * asymmetric partitions. The large fragment cache is significantly
1335          * larger. Thus, we try to use large fragments wherever possible.
1336          * Userspace can support this by aligning virtual base address and
1337          * allocation size to the fragment size.
1338          *
1339          * Starting with Vega10 the fragment size only controls the L1. The L2
1340          * is now directly feed with small/huge/giant pages from the walker.
1341          */
1342         unsigned max_frag;
1343 
1344         if (params->adev->asic_type < CHIP_VEGA10)
1345                 max_frag = params->adev->vm_manager.fragment_size;
1346         else
1347                 max_frag = 31;
1348 
1349         /* system pages are non continuously */
1350         if (params->pages_addr) {
1351                 *frag = 0;
1352                 *frag_end = end;
1353                 return;
1354         }
1355 
1356         /* This intentionally wraps around if no bit is set */
1357         *frag = min((unsigned)ffs(start) - 1, (unsigned)fls64(end - start) - 1);
1358         if (*frag >= max_frag) {
1359                 *frag = max_frag;
1360                 *frag_end = end & ~((1ULL << max_frag) - 1);
1361         } else {
1362                 *frag_end = start + (1 << *frag);
1363         }
1364 }
1365 
1366 /**
1367  * amdgpu_vm_update_ptes - make sure that page tables are valid
1368  *
1369  * @params: see amdgpu_vm_update_params definition
1370  * @start: start of GPU address range
1371  * @end: end of GPU address range
1372  * @dst: destination address to map to, the next dst inside the function
1373  * @flags: mapping flags
1374  *
1375  * Update the page tables in the range @start - @end.
1376  *
1377  * Returns:
1378  * 0 for success, -EINVAL for failure.
1379  */
1380 static int amdgpu_vm_update_ptes(struct amdgpu_vm_update_params *params,
1381                                  uint64_t start, uint64_t end,
1382                                  uint64_t dst, uint64_t flags)
1383 {
1384         struct amdgpu_device *adev = params->adev;
1385         struct amdgpu_vm_pt_cursor cursor;
1386         uint64_t frag_start = start, frag_end;
1387         unsigned int frag;
1388         int r;
1389 
1390         /* figure out the initial fragment */
1391         amdgpu_vm_fragment(params, frag_start, end, flags, &frag, &frag_end);
1392 
1393         /* walk over the address space and update the PTs */
1394         amdgpu_vm_pt_start(adev, params->vm, start, &cursor);
1395         while (cursor.pfn < end) {
1396                 unsigned shift, parent_shift, mask;
1397                 uint64_t incr, entry_end, pe_start;
1398                 struct amdgpu_bo *pt;
1399 
1400                 r = amdgpu_vm_alloc_pts(params->adev, params->vm, &cursor);
1401                 if (r)
1402                         return r;
1403 
1404                 pt = cursor.entry->base.bo;
1405 
1406                 /* The root level can't be a huge page */
1407                 if (cursor.level == adev->vm_manager.root_level) {
1408                         if (!amdgpu_vm_pt_descendant(adev, &cursor))
1409                                 return -ENOENT;
1410                         continue;
1411                 }
1412 
1413                 shift = amdgpu_vm_level_shift(adev, cursor.level);
1414                 parent_shift = amdgpu_vm_level_shift(adev, cursor.level - 1);
1415                 if (adev->asic_type < CHIP_VEGA10 &&
1416                     (flags & AMDGPU_PTE_VALID)) {
1417                         /* No huge page support before GMC v9 */
1418                         if (cursor.level != AMDGPU_VM_PTB) {
1419                                 if (!amdgpu_vm_pt_descendant(adev, &cursor))
1420                                         return -ENOENT;
1421                                 continue;
1422                         }
1423                 } else if (frag < shift) {
1424                         /* We can't use this level when the fragment size is
1425                          * smaller than the address shift. Go to the next
1426                          * child entry and try again.
1427                          */
1428                         if (!amdgpu_vm_pt_descendant(adev, &cursor))
1429                                 return -ENOENT;
1430                         continue;
1431                 } else if (frag >= parent_shift &&
1432                            cursor.level - 1 != adev->vm_manager.root_level) {
1433                         /* If the fragment size is even larger than the parent
1434                          * shift we should go up one level and check it again
1435                          * unless one level up is the root level.
1436                          */
1437                         if (!amdgpu_vm_pt_ancestor(&cursor))
1438                                 return -ENOENT;
1439                         continue;
1440                 }
1441 
1442                 /* Looks good so far, calculate parameters for the update */
1443                 incr = (uint64_t)AMDGPU_GPU_PAGE_SIZE << shift;
1444                 mask = amdgpu_vm_entries_mask(adev, cursor.level);
1445                 pe_start = ((cursor.pfn >> shift) & mask) * 8;
1446                 entry_end = (uint64_t)(mask + 1) << shift;
1447                 entry_end += cursor.pfn & ~(entry_end - 1);
1448                 entry_end = min(entry_end, end);
1449 
1450                 do {
1451                         uint64_t upd_end = min(entry_end, frag_end);
1452                         unsigned nptes = (upd_end - frag_start) >> shift;
1453 
1454                         amdgpu_vm_update_flags(params, pt, cursor.level,
1455                                                pe_start, dst, nptes, incr,
1456                                                flags | AMDGPU_PTE_FRAG(frag));
1457 
1458                         pe_start += nptes * 8;
1459                         dst += (uint64_t)nptes * AMDGPU_GPU_PAGE_SIZE << shift;
1460 
1461                         frag_start = upd_end;
1462                         if (frag_start >= frag_end) {
1463                                 /* figure out the next fragment */
1464                                 amdgpu_vm_fragment(params, frag_start, end,
1465                                                    flags, &frag, &frag_end);
1466                                 if (frag < shift)
1467                                         break;
1468                         }
1469                 } while (frag_start < entry_end);
1470 
1471                 if (amdgpu_vm_pt_descendant(adev, &cursor)) {
1472                         /* Free all child entries */
1473                         while (cursor.pfn < frag_start) {
1474                                 amdgpu_vm_free_pts(adev, params->vm, &cursor);
1475                                 amdgpu_vm_pt_next(adev, &cursor);
1476                         }
1477 
1478                 } else if (frag >= shift) {
1479                         /* or just move on to the next on the same level. */
1480                         amdgpu_vm_pt_next(adev, &cursor);
1481                 }
1482         }
1483 
1484         return 0;
1485 }
1486 
1487 /**
1488  * amdgpu_vm_bo_update_mapping - update a mapping in the vm page table
1489  *
1490  * @adev: amdgpu_device pointer
1491  * @exclusive: fence we need to sync to
1492  * @pages_addr: DMA addresses to use for mapping
1493  * @vm: requested vm
1494  * @start: start of mapped range
1495  * @last: last mapped entry
1496  * @flags: flags for the entries
1497  * @addr: addr to set the area to
1498  * @fence: optional resulting fence
1499  *
1500  * Fill in the page table entries between @start and @last.
1501  *
1502  * Returns:
1503  * 0 for success, -EINVAL for failure.
1504  */
1505 static int amdgpu_vm_bo_update_mapping(struct amdgpu_device *adev,
1506                                        struct dma_fence *exclusive,
1507                                        dma_addr_t *pages_addr,
1508                                        struct amdgpu_vm *vm,
1509                                        uint64_t start, uint64_t last,
1510                                        uint64_t flags, uint64_t addr,
1511                                        struct dma_fence **fence)
1512 {
1513         struct amdgpu_vm_update_params params;
1514         void *owner = AMDGPU_FENCE_OWNER_VM;
1515         int r;
1516 
1517         memset(&params, 0, sizeof(params));
1518         params.adev = adev;
1519         params.vm = vm;
1520         params.pages_addr = pages_addr;
1521 
1522         /* sync to everything except eviction fences on unmapping */
1523         if (!(flags & AMDGPU_PTE_VALID))
1524                 owner = AMDGPU_FENCE_OWNER_KFD;
1525 
1526         r = vm->update_funcs->prepare(&params, owner, exclusive);
1527         if (r)
1528                 return r;
1529 
1530         r = amdgpu_vm_update_ptes(&params, start, last + 1, addr, flags);
1531         if (r)
1532                 return r;
1533 
1534         return vm->update_funcs->commit(&params, fence);
1535 }
1536 
1537 /**
1538  * amdgpu_vm_bo_split_mapping - split a mapping into smaller chunks
1539  *
1540  * @adev: amdgpu_device pointer
1541  * @exclusive: fence we need to sync to
1542  * @pages_addr: DMA addresses to use for mapping
1543  * @vm: requested vm
1544  * @mapping: mapped range and flags to use for the update
1545  * @flags: HW flags for the mapping
1546  * @bo_adev: amdgpu_device pointer that bo actually been allocated
1547  * @nodes: array of drm_mm_nodes with the MC addresses
1548  * @fence: optional resulting fence
1549  *
1550  * Split the mapping into smaller chunks so that each update fits
1551  * into a SDMA IB.
1552  *
1553  * Returns:
1554  * 0 for success, -EINVAL for failure.
1555  */
1556 static int amdgpu_vm_bo_split_mapping(struct amdgpu_device *adev,
1557                                       struct dma_fence *exclusive,
1558                                       dma_addr_t *pages_addr,
1559                                       struct amdgpu_vm *vm,
1560                                       struct amdgpu_bo_va_mapping *mapping,
1561                                       uint64_t flags,
1562                                       struct amdgpu_device *bo_adev,
1563                                       struct drm_mm_node *nodes,
1564                                       struct dma_fence **fence)
1565 {
1566         unsigned min_linear_pages = 1 << adev->vm_manager.fragment_size;
1567         uint64_t pfn, start = mapping->start;
1568         int r;
1569 
1570         /* normally,bo_va->flags only contians READABLE and WIRTEABLE bit go here
1571          * but in case of something, we filter the flags in first place
1572          */
1573         if (!(mapping->flags & AMDGPU_PTE_READABLE))
1574                 flags &= ~AMDGPU_PTE_READABLE;
1575         if (!(mapping->flags & AMDGPU_PTE_WRITEABLE))
1576                 flags &= ~AMDGPU_PTE_WRITEABLE;
1577 
1578         flags &= ~AMDGPU_PTE_EXECUTABLE;
1579         flags |= mapping->flags & AMDGPU_PTE_EXECUTABLE;
1580 
1581         if (adev->asic_type >= CHIP_NAVI10) {
1582                 flags &= ~AMDGPU_PTE_MTYPE_NV10_MASK;
1583                 flags |= (mapping->flags & AMDGPU_PTE_MTYPE_NV10_MASK);
1584         } else {
1585                 flags &= ~AMDGPU_PTE_MTYPE_VG10_MASK;
1586                 flags |= (mapping->flags & AMDGPU_PTE_MTYPE_VG10_MASK);
1587         }
1588 
1589         if ((mapping->flags & AMDGPU_PTE_PRT) &&
1590             (adev->asic_type >= CHIP_VEGA10)) {
1591                 flags |= AMDGPU_PTE_PRT;
1592                 if (adev->asic_type >= CHIP_NAVI10) {
1593                         flags |= AMDGPU_PTE_SNOOPED;
1594                         flags |= AMDGPU_PTE_LOG;
1595                         flags |= AMDGPU_PTE_SYSTEM;
1596                 }
1597                 flags &= ~AMDGPU_PTE_VALID;
1598         }
1599 
1600         trace_amdgpu_vm_bo_update(mapping);
1601 
1602         pfn = mapping->offset >> PAGE_SHIFT;
1603         if (nodes) {
1604                 while (pfn >= nodes->size) {
1605                         pfn -= nodes->size;
1606                         ++nodes;
1607                 }
1608         }
1609 
1610         do {
1611                 dma_addr_t *dma_addr = NULL;
1612                 uint64_t max_entries;
1613                 uint64_t addr, last;
1614 
1615                 if (nodes) {
1616                         addr = nodes->start << PAGE_SHIFT;
1617                         max_entries = (nodes->size - pfn) *
1618                                 AMDGPU_GPU_PAGES_IN_CPU_PAGE;
1619                 } else {
1620                         addr = 0;
1621                         max_entries = S64_MAX;
1622                 }
1623 
1624                 if (pages_addr) {
1625                         uint64_t count;
1626 
1627                         for (count = 1;
1628                              count < max_entries / AMDGPU_GPU_PAGES_IN_CPU_PAGE;
1629                              ++count) {
1630                                 uint64_t idx = pfn + count;
1631 
1632                                 if (pages_addr[idx] !=
1633                                     (pages_addr[idx - 1] + PAGE_SIZE))
1634                                         break;
1635                         }
1636 
1637                         if (count < min_linear_pages) {
1638                                 addr = pfn << PAGE_SHIFT;
1639                                 dma_addr = pages_addr;
1640                         } else {
1641                                 addr = pages_addr[pfn];
1642                                 max_entries = count * AMDGPU_GPU_PAGES_IN_CPU_PAGE;
1643                         }
1644 
1645                 } else if (flags & AMDGPU_PTE_VALID) {
1646                         addr += bo_adev->vm_manager.vram_base_offset;
1647                         addr += pfn << PAGE_SHIFT;
1648                 }
1649 
1650                 last = min((uint64_t)mapping->last, start + max_entries - 1);
1651                 r = amdgpu_vm_bo_update_mapping(adev, exclusive, dma_addr, vm,
1652                                                 start, last, flags, addr,
1653                                                 fence);
1654                 if (r)
1655                         return r;
1656 
1657                 pfn += (last - start + 1) / AMDGPU_GPU_PAGES_IN_CPU_PAGE;
1658                 if (nodes && nodes->size == pfn) {
1659                         pfn = 0;
1660                         ++nodes;
1661                 }
1662                 start = last + 1;
1663 
1664         } while (unlikely(start != mapping->last + 1));
1665 
1666         return 0;
1667 }
1668 
1669 /**
1670  * amdgpu_vm_bo_update - update all BO mappings in the vm page table
1671  *
1672  * @adev: amdgpu_device pointer
1673  * @bo_va: requested BO and VM object
1674  * @clear: if true clear the entries
1675  *
1676  * Fill in the page table entries for @bo_va.
1677  *
1678  * Returns:
1679  * 0 for success, -EINVAL for failure.
1680  */
1681 int amdgpu_vm_bo_update(struct amdgpu_device *adev,
1682                         struct amdgpu_bo_va *bo_va,
1683                         bool clear)
1684 {
1685         struct amdgpu_bo *bo = bo_va->base.bo;
1686         struct amdgpu_vm *vm = bo_va->base.vm;
1687         struct amdgpu_bo_va_mapping *mapping;
1688         dma_addr_t *pages_addr = NULL;
1689         struct ttm_mem_reg *mem;
1690         struct drm_mm_node *nodes;
1691         struct dma_fence *exclusive, **last_update;
1692         uint64_t flags;
1693         struct amdgpu_device *bo_adev = adev;
1694         int r;
1695 
1696         if (clear || !bo) {
1697                 mem = NULL;
1698                 nodes = NULL;
1699                 exclusive = NULL;
1700         } else {
1701                 struct ttm_dma_tt *ttm;
1702 
1703                 mem = &bo->tbo.mem;
1704                 nodes = mem->mm_node;
1705                 if (mem->mem_type == TTM_PL_TT) {
1706                         ttm = container_of(bo->tbo.ttm, struct ttm_dma_tt, ttm);
1707                         pages_addr = ttm->dma_address;
1708                 }
1709                 exclusive = dma_resv_get_excl(bo->tbo.base.resv);
1710         }
1711 
1712         if (bo) {
1713                 flags = amdgpu_ttm_tt_pte_flags(adev, bo->tbo.ttm, mem);
1714                 bo_adev = amdgpu_ttm_adev(bo->tbo.bdev);
1715         } else {
1716                 flags = 0x0;
1717         }
1718 
1719         if (clear || (bo && bo->tbo.base.resv == vm->root.base.bo->tbo.base.resv))
1720                 last_update = &vm->last_update;
1721         else
1722                 last_update = &bo_va->last_pt_update;
1723 
1724         if (!clear && bo_va->base.moved) {
1725                 bo_va->base.moved = false;
1726                 list_splice_init(&bo_va->valids, &bo_va->invalids);
1727 
1728         } else if (bo_va->cleared != clear) {
1729                 list_splice_init(&bo_va->valids, &bo_va->invalids);
1730         }
1731 
1732         list_for_each_entry(mapping, &bo_va->invalids, list) {
1733                 r = amdgpu_vm_bo_split_mapping(adev, exclusive, pages_addr, vm,
1734                                                mapping, flags, bo_adev, nodes,
1735                                                last_update);
1736                 if (r)
1737                         return r;
1738         }
1739 
1740         if (vm->use_cpu_for_update) {
1741                 /* Flush HDP */
1742                 mb();
1743                 amdgpu_asic_flush_hdp(adev, NULL);
1744         }
1745 
1746         /* If the BO is not in its preferred location add it back to
1747          * the evicted list so that it gets validated again on the
1748          * next command submission.
1749          */
1750         if (bo && bo->tbo.base.resv == vm->root.base.bo->tbo.base.resv) {
1751                 uint32_t mem_type = bo->tbo.mem.mem_type;
1752 
1753                 if (!(bo->preferred_domains & amdgpu_mem_type_to_domain(mem_type)))
1754                         amdgpu_vm_bo_evicted(&bo_va->base);
1755                 else
1756                         amdgpu_vm_bo_idle(&bo_va->base);
1757         } else {
1758                 amdgpu_vm_bo_done(&bo_va->base);
1759         }
1760 
1761         list_splice_init(&bo_va->invalids, &bo_va->valids);
1762         bo_va->cleared = clear;
1763 
1764         if (trace_amdgpu_vm_bo_mapping_enabled()) {
1765                 list_for_each_entry(mapping, &bo_va->valids, list)
1766                         trace_amdgpu_vm_bo_mapping(mapping);
1767         }
1768 
1769         return 0;
1770 }
1771 
1772 /**
1773  * amdgpu_vm_update_prt_state - update the global PRT state
1774  *
1775  * @adev: amdgpu_device pointer
1776  */
1777 static void amdgpu_vm_update_prt_state(struct amdgpu_device *adev)
1778 {
1779         unsigned long flags;
1780         bool enable;
1781 
1782         spin_lock_irqsave(&adev->vm_manager.prt_lock, flags);
1783         enable = !!atomic_read(&adev->vm_manager.num_prt_users);
1784         adev->gmc.gmc_funcs->set_prt(adev, enable);
1785         spin_unlock_irqrestore(&adev->vm_manager.prt_lock, flags);
1786 }
1787 
1788 /**
1789  * amdgpu_vm_prt_get - add a PRT user
1790  *
1791  * @adev: amdgpu_device pointer
1792  */
1793 static void amdgpu_vm_prt_get(struct amdgpu_device *adev)
1794 {
1795         if (!adev->gmc.gmc_funcs->set_prt)
1796                 return;
1797 
1798         if (atomic_inc_return(&adev->vm_manager.num_prt_users) == 1)
1799                 amdgpu_vm_update_prt_state(adev);
1800 }
1801 
1802 /**
1803  * amdgpu_vm_prt_put - drop a PRT user
1804  *
1805  * @adev: amdgpu_device pointer
1806  */
1807 static void amdgpu_vm_prt_put(struct amdgpu_device *adev)
1808 {
1809         if (atomic_dec_return(&adev->vm_manager.num_prt_users) == 0)
1810                 amdgpu_vm_update_prt_state(adev);
1811 }
1812 
1813 /**
1814  * amdgpu_vm_prt_cb - callback for updating the PRT status
1815  *
1816  * @fence: fence for the callback
1817  * @_cb: the callback function
1818  */
1819 static void amdgpu_vm_prt_cb(struct dma_fence *fence, struct dma_fence_cb *_cb)
1820 {
1821         struct amdgpu_prt_cb *cb = container_of(_cb, struct amdgpu_prt_cb, cb);
1822 
1823         amdgpu_vm_prt_put(cb->adev);
1824         kfree(cb);
1825 }
1826 
1827 /**
1828  * amdgpu_vm_add_prt_cb - add callback for updating the PRT status
1829  *
1830  * @adev: amdgpu_device pointer
1831  * @fence: fence for the callback
1832  */
1833 static void amdgpu_vm_add_prt_cb(struct amdgpu_device *adev,
1834                                  struct dma_fence *fence)
1835 {
1836         struct amdgpu_prt_cb *cb;
1837 
1838         if (!adev->gmc.gmc_funcs->set_prt)
1839                 return;
1840 
1841         cb = kmalloc(sizeof(struct amdgpu_prt_cb), GFP_KERNEL);
1842         if (!cb) {
1843                 /* Last resort when we are OOM */
1844                 if (fence)
1845                         dma_fence_wait(fence, false);
1846 
1847                 amdgpu_vm_prt_put(adev);
1848         } else {
1849                 cb->adev = adev;
1850                 if (!fence || dma_fence_add_callback(fence, &cb->cb,
1851                                                      amdgpu_vm_prt_cb))
1852                         amdgpu_vm_prt_cb(fence, &cb->cb);
1853         }
1854 }
1855 
1856 /**
1857  * amdgpu_vm_free_mapping - free a mapping
1858  *
1859  * @adev: amdgpu_device pointer
1860  * @vm: requested vm
1861  * @mapping: mapping to be freed
1862  * @fence: fence of the unmap operation
1863  *
1864  * Free a mapping and make sure we decrease the PRT usage count if applicable.
1865  */
1866 static void amdgpu_vm_free_mapping(struct amdgpu_device *adev,
1867                                    struct amdgpu_vm *vm,
1868                                    struct amdgpu_bo_va_mapping *mapping,
1869                                    struct dma_fence *fence)
1870 {
1871         if (mapping->flags & AMDGPU_PTE_PRT)
1872                 amdgpu_vm_add_prt_cb(adev, fence);
1873         kfree(mapping);
1874 }
1875 
1876 /**
1877  * amdgpu_vm_prt_fini - finish all prt mappings
1878  *
1879  * @adev: amdgpu_device pointer
1880  * @vm: requested vm
1881  *
1882  * Register a cleanup callback to disable PRT support after VM dies.
1883  */
1884 static void amdgpu_vm_prt_fini(struct amdgpu_device *adev, struct amdgpu_vm *vm)
1885 {
1886         struct dma_resv *resv = vm->root.base.bo->tbo.base.resv;
1887         struct dma_fence *excl, **shared;
1888         unsigned i, shared_count;
1889         int r;
1890 
1891         r = dma_resv_get_fences_rcu(resv, &excl,
1892                                               &shared_count, &shared);
1893         if (r) {
1894                 /* Not enough memory to grab the fence list, as last resort
1895                  * block for all the fences to complete.
1896                  */
1897                 dma_resv_wait_timeout_rcu(resv, true, false,
1898                                                     MAX_SCHEDULE_TIMEOUT);
1899                 return;
1900         }
1901 
1902         /* Add a callback for each fence in the reservation object */
1903         amdgpu_vm_prt_get(adev);
1904         amdgpu_vm_add_prt_cb(adev, excl);
1905 
1906         for (i = 0; i < shared_count; ++i) {
1907                 amdgpu_vm_prt_get(adev);
1908                 amdgpu_vm_add_prt_cb(adev, shared[i]);
1909         }
1910 
1911         kfree(shared);
1912 }
1913 
1914 /**
1915  * amdgpu_vm_clear_freed - clear freed BOs in the PT
1916  *
1917  * @adev: amdgpu_device pointer
1918  * @vm: requested vm
1919  * @fence: optional resulting fence (unchanged if no work needed to be done
1920  * or if an error occurred)
1921  *
1922  * Make sure all freed BOs are cleared in the PT.
1923  * PTs have to be reserved and mutex must be locked!
1924  *
1925  * Returns:
1926  * 0 for success.
1927  *
1928  */
1929 int amdgpu_vm_clear_freed(struct amdgpu_device *adev,
1930                           struct amdgpu_vm *vm,
1931                           struct dma_fence **fence)
1932 {
1933         struct amdgpu_bo_va_mapping *mapping;
1934         uint64_t init_pte_value = 0;
1935         struct dma_fence *f = NULL;
1936         int r;
1937 
1938         while (!list_empty(&vm->freed)) {
1939                 mapping = list_first_entry(&vm->freed,
1940                         struct amdgpu_bo_va_mapping, list);
1941                 list_del(&mapping->list);
1942 
1943                 if (vm->pte_support_ats &&
1944                     mapping->start < AMDGPU_GMC_HOLE_START)
1945                         init_pte_value = AMDGPU_PTE_DEFAULT_ATC;
1946 
1947                 r = amdgpu_vm_bo_update_mapping(adev, NULL, NULL, vm,
1948                                                 mapping->start, mapping->last,
1949                                                 init_pte_value, 0, &f);
1950                 amdgpu_vm_free_mapping(adev, vm, mapping, f);
1951                 if (r) {
1952                         dma_fence_put(f);
1953                         return r;
1954                 }
1955         }
1956 
1957         if (fence && f) {
1958                 dma_fence_put(*fence);
1959                 *fence = f;
1960         } else {
1961                 dma_fence_put(f);
1962         }
1963 
1964         return 0;
1965 
1966 }
1967 
1968 /**
1969  * amdgpu_vm_handle_moved - handle moved BOs in the PT
1970  *
1971  * @adev: amdgpu_device pointer
1972  * @vm: requested vm
1973  *
1974  * Make sure all BOs which are moved are updated in the PTs.
1975  *
1976  * Returns:
1977  * 0 for success.
1978  *
1979  * PTs have to be reserved!
1980  */
1981 int amdgpu_vm_handle_moved(struct amdgpu_device *adev,
1982                            struct amdgpu_vm *vm)
1983 {
1984         struct amdgpu_bo_va *bo_va, *tmp;
1985         struct dma_resv *resv;
1986         bool clear;
1987         int r;
1988 
1989         list_for_each_entry_safe(bo_va, tmp, &vm->moved, base.vm_status) {
1990                 /* Per VM BOs never need to bo cleared in the page tables */
1991                 r = amdgpu_vm_bo_update(adev, bo_va, false);
1992                 if (r)
1993                         return r;
1994         }
1995 
1996         spin_lock(&vm->invalidated_lock);
1997         while (!list_empty(&vm->invalidated)) {
1998                 bo_va = list_first_entry(&vm->invalidated, struct amdgpu_bo_va,
1999                                          base.vm_status);
2000                 resv = bo_va->base.bo->tbo.base.resv;
2001                 spin_unlock(&vm->invalidated_lock);
2002 
2003                 /* Try to reserve the BO to avoid clearing its ptes */
2004                 if (!amdgpu_vm_debug && dma_resv_trylock(resv))
2005                         clear = false;
2006                 /* Somebody else is using the BO right now */
2007                 else
2008                         clear = true;
2009 
2010                 r = amdgpu_vm_bo_update(adev, bo_va, clear);
2011                 if (r)
2012                         return r;
2013 
2014                 if (!clear)
2015                         dma_resv_unlock(resv);
2016                 spin_lock(&vm->invalidated_lock);
2017         }
2018         spin_unlock(&vm->invalidated_lock);
2019 
2020         return 0;
2021 }
2022 
2023 /**
2024  * amdgpu_vm_bo_add - add a bo to a specific vm
2025  *
2026  * @adev: amdgpu_device pointer
2027  * @vm: requested vm
2028  * @bo: amdgpu buffer object
2029  *
2030  * Add @bo into the requested vm.
2031  * Add @bo to the list of bos associated with the vm
2032  *
2033  * Returns:
2034  * Newly added bo_va or NULL for failure
2035  *
2036  * Object has to be reserved!
2037  */
2038 struct amdgpu_bo_va *amdgpu_vm_bo_add(struct amdgpu_device *adev,
2039                                       struct amdgpu_vm *vm,
2040                                       struct amdgpu_bo *bo)
2041 {
2042         struct amdgpu_bo_va *bo_va;
2043 
2044         bo_va = kzalloc(sizeof(struct amdgpu_bo_va), GFP_KERNEL);
2045         if (bo_va == NULL) {
2046                 return NULL;
2047         }
2048         amdgpu_vm_bo_base_init(&bo_va->base, vm, bo);
2049 
2050         bo_va->ref_count = 1;
2051         INIT_LIST_HEAD(&bo_va->valids);
2052         INIT_LIST_HEAD(&bo_va->invalids);
2053 
2054         if (bo && amdgpu_xgmi_same_hive(adev, amdgpu_ttm_adev(bo->tbo.bdev)) &&
2055             (bo->preferred_domains & AMDGPU_GEM_DOMAIN_VRAM)) {
2056                 bo_va->is_xgmi = true;
2057                 mutex_lock(&adev->vm_manager.lock_pstate);
2058                 /* Power up XGMI if it can be potentially used */
2059                 if (++adev->vm_manager.xgmi_map_counter == 1)
2060                         amdgpu_xgmi_set_pstate(adev, 1);
2061                 mutex_unlock(&adev->vm_manager.lock_pstate);
2062         }
2063 
2064         return bo_va;
2065 }
2066 
2067 
2068 /**
2069  * amdgpu_vm_bo_insert_mapping - insert a new mapping
2070  *
2071  * @adev: amdgpu_device pointer
2072  * @bo_va: bo_va to store the address
2073  * @mapping: the mapping to insert
2074  *
2075  * Insert a new mapping into all structures.
2076  */
2077 static void amdgpu_vm_bo_insert_map(struct amdgpu_device *adev,
2078                                     struct amdgpu_bo_va *bo_va,
2079                                     struct amdgpu_bo_va_mapping *mapping)
2080 {
2081         struct amdgpu_vm *vm = bo_va->base.vm;
2082         struct amdgpu_bo *bo = bo_va->base.bo;
2083 
2084         mapping->bo_va = bo_va;
2085         list_add(&mapping->list, &bo_va->invalids);
2086         amdgpu_vm_it_insert(mapping, &vm->va);
2087 
2088         if (mapping->flags & AMDGPU_PTE_PRT)
2089                 amdgpu_vm_prt_get(adev);
2090 
2091         if (bo && bo->tbo.base.resv == vm->root.base.bo->tbo.base.resv &&
2092             !bo_va->base.moved) {
2093                 list_move(&bo_va->base.vm_status, &vm->moved);
2094         }
2095         trace_amdgpu_vm_bo_map(bo_va, mapping);
2096 }
2097 
2098 /**
2099  * amdgpu_vm_bo_map - map bo inside a vm
2100  *
2101  * @adev: amdgpu_device pointer
2102  * @bo_va: bo_va to store the address
2103  * @saddr: where to map the BO
2104  * @offset: requested offset in the BO
2105  * @size: BO size in bytes
2106  * @flags: attributes of pages (read/write/valid/etc.)
2107  *
2108  * Add a mapping of the BO at the specefied addr into the VM.
2109  *
2110  * Returns:
2111  * 0 for success, error for failure.
2112  *
2113  * Object has to be reserved and unreserved outside!
2114  */
2115 int amdgpu_vm_bo_map(struct amdgpu_device *adev,
2116                      struct amdgpu_bo_va *bo_va,
2117                      uint64_t saddr, uint64_t offset,
2118                      uint64_t size, uint64_t flags)
2119 {
2120         struct amdgpu_bo_va_mapping *mapping, *tmp;
2121         struct amdgpu_bo *bo = bo_va->base.bo;
2122         struct amdgpu_vm *vm = bo_va->base.vm;
2123         uint64_t eaddr;
2124 
2125         /* validate the parameters */
2126         if (saddr & AMDGPU_GPU_PAGE_MASK || offset & AMDGPU_GPU_PAGE_MASK ||
2127             size == 0 || size & AMDGPU_GPU_PAGE_MASK)
2128                 return -EINVAL;
2129 
2130         /* make sure object fit at this offset */
2131         eaddr = saddr + size - 1;
2132         if (saddr >= eaddr ||
2133             (bo && offset + size > amdgpu_bo_size(bo)))
2134                 return -EINVAL;
2135 
2136         saddr /= AMDGPU_GPU_PAGE_SIZE;
2137         eaddr /= AMDGPU_GPU_PAGE_SIZE;
2138 
2139         tmp = amdgpu_vm_it_iter_first(&vm->va, saddr, eaddr);
2140         if (tmp) {
2141                 /* bo and tmp overlap, invalid addr */
2142                 dev_err(adev->dev, "bo %p va 0x%010Lx-0x%010Lx conflict with "
2143                         "0x%010Lx-0x%010Lx\n", bo, saddr, eaddr,
2144                         tmp->start, tmp->last + 1);
2145                 return -EINVAL;
2146         }
2147 
2148         mapping = kmalloc(sizeof(*mapping), GFP_KERNEL);
2149         if (!mapping)
2150                 return -ENOMEM;
2151 
2152         mapping->start = saddr;
2153         mapping->last = eaddr;
2154         mapping->offset = offset;
2155         mapping->flags = flags;
2156 
2157         amdgpu_vm_bo_insert_map(adev, bo_va, mapping);
2158 
2159         return 0;
2160 }
2161 
2162 /**
2163  * amdgpu_vm_bo_replace_map - map bo inside a vm, replacing existing mappings
2164  *
2165  * @adev: amdgpu_device pointer
2166  * @bo_va: bo_va to store the address
2167  * @saddr: where to map the BO
2168  * @offset: requested offset in the BO
2169  * @size: BO size in bytes
2170  * @flags: attributes of pages (read/write/valid/etc.)
2171  *
2172  * Add a mapping of the BO at the specefied addr into the VM. Replace existing
2173  * mappings as we do so.
2174  *
2175  * Returns:
2176  * 0 for success, error for failure.
2177  *
2178  * Object has to be reserved and unreserved outside!
2179  */
2180 int amdgpu_vm_bo_replace_map(struct amdgpu_device *adev,
2181                              struct amdgpu_bo_va *bo_va,
2182                              uint64_t saddr, uint64_t offset,
2183                              uint64_t size, uint64_t flags)
2184 {
2185         struct amdgpu_bo_va_mapping *mapping;
2186         struct amdgpu_bo *bo = bo_va->base.bo;
2187         uint64_t eaddr;
2188         int r;
2189 
2190         /* validate the parameters */
2191         if (saddr & AMDGPU_GPU_PAGE_MASK || offset & AMDGPU_GPU_PAGE_MASK ||
2192             size == 0 || size & AMDGPU_GPU_PAGE_MASK)
2193                 return -EINVAL;
2194 
2195         /* make sure object fit at this offset */
2196         eaddr = saddr + size - 1;
2197         if (saddr >= eaddr ||
2198             (bo && offset + size > amdgpu_bo_size(bo)))
2199                 return -EINVAL;
2200 
2201         /* Allocate all the needed memory */
2202         mapping = kmalloc(sizeof(*mapping), GFP_KERNEL);
2203         if (!mapping)
2204                 return -ENOMEM;
2205 
2206         r = amdgpu_vm_bo_clear_mappings(adev, bo_va->base.vm, saddr, size);
2207         if (r) {
2208                 kfree(mapping);
2209                 return r;
2210         }
2211 
2212         saddr /= AMDGPU_GPU_PAGE_SIZE;
2213         eaddr /= AMDGPU_GPU_PAGE_SIZE;
2214 
2215         mapping->start = saddr;
2216         mapping->last = eaddr;
2217         mapping->offset = offset;
2218         mapping->flags = flags;
2219 
2220         amdgpu_vm_bo_insert_map(adev, bo_va, mapping);
2221 
2222         return 0;
2223 }
2224 
2225 /**
2226  * amdgpu_vm_bo_unmap - remove bo mapping from vm
2227  *
2228  * @adev: amdgpu_device pointer
2229  * @bo_va: bo_va to remove the address from
2230  * @saddr: where to the BO is mapped
2231  *
2232  * Remove a mapping of the BO at the specefied addr from the VM.
2233  *
2234  * Returns:
2235  * 0 for success, error for failure.
2236  *
2237  * Object has to be reserved and unreserved outside!
2238  */
2239 int amdgpu_vm_bo_unmap(struct amdgpu_device *adev,
2240                        struct amdgpu_bo_va *bo_va,
2241                        uint64_t saddr)
2242 {
2243         struct amdgpu_bo_va_mapping *mapping;
2244         struct amdgpu_vm *vm = bo_va->base.vm;
2245         bool valid = true;
2246 
2247         saddr /= AMDGPU_GPU_PAGE_SIZE;
2248 
2249         list_for_each_entry(mapping, &bo_va->valids, list) {
2250                 if (mapping->start == saddr)
2251                         break;
2252         }
2253 
2254         if (&mapping->list == &bo_va->valids) {
2255                 valid = false;
2256 
2257                 list_for_each_entry(mapping, &bo_va->invalids, list) {
2258                         if (mapping->start == saddr)
2259                                 break;
2260                 }
2261 
2262                 if (&mapping->list == &bo_va->invalids)
2263                         return -ENOENT;
2264         }
2265 
2266         list_del(&mapping->list);
2267         amdgpu_vm_it_remove(mapping, &vm->va);
2268         mapping->bo_va = NULL;
2269         trace_amdgpu_vm_bo_unmap(bo_va, mapping);
2270 
2271         if (valid)
2272                 list_add(&mapping->list, &vm->freed);
2273         else
2274                 amdgpu_vm_free_mapping(adev, vm, mapping,
2275                                        bo_va->last_pt_update);
2276 
2277         return 0;
2278 }
2279 
2280 /**
2281  * amdgpu_vm_bo_clear_mappings - remove all mappings in a specific range
2282  *
2283  * @adev: amdgpu_device pointer
2284  * @vm: VM structure to use
2285  * @saddr: start of the range
2286  * @size: size of the range
2287  *
2288  * Remove all mappings in a range, split them as appropriate.
2289  *
2290  * Returns:
2291  * 0 for success, error for failure.
2292  */
2293 int amdgpu_vm_bo_clear_mappings(struct amdgpu_device *adev,
2294                                 struct amdgpu_vm *vm,
2295                                 uint64_t saddr, uint64_t size)
2296 {
2297         struct amdgpu_bo_va_mapping *before, *after, *tmp, *next;
2298         LIST_HEAD(removed);
2299         uint64_t eaddr;
2300 
2301         eaddr = saddr + size - 1;
2302         saddr /= AMDGPU_GPU_PAGE_SIZE;
2303         eaddr /= AMDGPU_GPU_PAGE_SIZE;
2304 
2305         /* Allocate all the needed memory */
2306         before = kzalloc(sizeof(*before), GFP_KERNEL);
2307         if (!before)
2308                 return -ENOMEM;
2309         INIT_LIST_HEAD(&before->list);
2310 
2311         after = kzalloc(sizeof(*after), GFP_KERNEL);
2312         if (!after) {
2313                 kfree(before);
2314                 return -ENOMEM;
2315         }
2316         INIT_LIST_HEAD(&after->list);
2317 
2318         /* Now gather all removed mappings */
2319         tmp = amdgpu_vm_it_iter_first(&vm->va, saddr, eaddr);
2320         while (tmp) {
2321                 /* Remember mapping split at the start */
2322                 if (tmp->start < saddr) {
2323                         before->start = tmp->start;
2324                         before->last = saddr - 1;
2325                         before->offset = tmp->offset;
2326                         before->flags = tmp->flags;
2327                         before->bo_va = tmp->bo_va;
2328                         list_add(&before->list, &tmp->bo_va->invalids);
2329                 }
2330 
2331                 /* Remember mapping split at the end */
2332                 if (tmp->last > eaddr) {
2333                         after->start = eaddr + 1;
2334                         after->last = tmp->last;
2335                         after->offset = tmp->offset;
2336                         after->offset += after->start - tmp->start;
2337                         after->flags = tmp->flags;
2338                         after->bo_va = tmp->bo_va;
2339                         list_add(&after->list, &tmp->bo_va->invalids);
2340                 }
2341 
2342                 list_del(&tmp->list);
2343                 list_add(&tmp->list, &removed);
2344 
2345                 tmp = amdgpu_vm_it_iter_next(tmp, saddr, eaddr);
2346         }
2347 
2348         /* And free them up */
2349         list_for_each_entry_safe(tmp, next, &removed, list) {
2350                 amdgpu_vm_it_remove(tmp, &vm->va);
2351                 list_del(&tmp->list);
2352 
2353                 if (tmp->start < saddr)
2354                     tmp->start = saddr;
2355                 if (tmp->last > eaddr)
2356                     tmp->last = eaddr;
2357 
2358                 tmp->bo_va = NULL;
2359                 list_add(&tmp->list, &vm->freed);
2360                 trace_amdgpu_vm_bo_unmap(NULL, tmp);
2361         }
2362 
2363         /* Insert partial mapping before the range */
2364         if (!list_empty(&before->list)) {
2365                 amdgpu_vm_it_insert(before, &vm->va);
2366                 if (before->flags & AMDGPU_PTE_PRT)
2367                         amdgpu_vm_prt_get(adev);
2368         } else {
2369                 kfree(before);
2370         }
2371 
2372         /* Insert partial mapping after the range */
2373         if (!list_empty(&after->list)) {
2374                 amdgpu_vm_it_insert(after, &vm->va);
2375                 if (after->flags & AMDGPU_PTE_PRT)
2376                         amdgpu_vm_prt_get(adev);
2377         } else {
2378                 kfree(after);
2379         }
2380 
2381         return 0;
2382 }
2383 
2384 /**
2385  * amdgpu_vm_bo_lookup_mapping - find mapping by address
2386  *
2387  * @vm: the requested VM
2388  * @addr: the address
2389  *
2390  * Find a mapping by it's address.
2391  *
2392  * Returns:
2393  * The amdgpu_bo_va_mapping matching for addr or NULL
2394  *
2395  */
2396 struct amdgpu_bo_va_mapping *amdgpu_vm_bo_lookup_mapping(struct amdgpu_vm *vm,
2397                                                          uint64_t addr)
2398 {
2399         return amdgpu_vm_it_iter_first(&vm->va, addr, addr);
2400 }
2401 
2402 /**
2403  * amdgpu_vm_bo_trace_cs - trace all reserved mappings
2404  *
2405  * @vm: the requested vm
2406  * @ticket: CS ticket
2407  *
2408  * Trace all mappings of BOs reserved during a command submission.
2409  */
2410 void amdgpu_vm_bo_trace_cs(struct amdgpu_vm *vm, struct ww_acquire_ctx *ticket)
2411 {
2412         struct amdgpu_bo_va_mapping *mapping;
2413 
2414         if (!trace_amdgpu_vm_bo_cs_enabled())
2415                 return;
2416 
2417         for (mapping = amdgpu_vm_it_iter_first(&vm->va, 0, U64_MAX); mapping;
2418              mapping = amdgpu_vm_it_iter_next(mapping, 0, U64_MAX)) {
2419                 if (mapping->bo_va && mapping->bo_va->base.bo) {
2420                         struct amdgpu_bo *bo;
2421 
2422                         bo = mapping->bo_va->base.bo;
2423                         if (dma_resv_locking_ctx(bo->tbo.base.resv) !=
2424                             ticket)
2425                                 continue;
2426                 }
2427 
2428                 trace_amdgpu_vm_bo_cs(mapping);
2429         }
2430 }
2431 
2432 /**
2433  * amdgpu_vm_bo_rmv - remove a bo to a specific vm
2434  *
2435  * @adev: amdgpu_device pointer
2436  * @bo_va: requested bo_va
2437  *
2438  * Remove @bo_va->bo from the requested vm.
2439  *
2440  * Object have to be reserved!
2441  */
2442 void amdgpu_vm_bo_rmv(struct amdgpu_device *adev,
2443                       struct amdgpu_bo_va *bo_va)
2444 {
2445         struct amdgpu_bo_va_mapping *mapping, *next;
2446         struct amdgpu_bo *bo = bo_va->base.bo;
2447         struct amdgpu_vm *vm = bo_va->base.vm;
2448         struct amdgpu_vm_bo_base **base;
2449 
2450         if (bo) {
2451                 if (bo->tbo.base.resv == vm->root.base.bo->tbo.base.resv)
2452                         vm->bulk_moveable = false;
2453 
2454                 for (base = &bo_va->base.bo->vm_bo; *base;
2455                      base = &(*base)->next) {
2456                         if (*base != &bo_va->base)
2457                                 continue;
2458 
2459                         *base = bo_va->base.next;
2460                         break;
2461                 }
2462         }
2463 
2464         spin_lock(&vm->invalidated_lock);
2465         list_del(&bo_va->base.vm_status);
2466         spin_unlock(&vm->invalidated_lock);
2467 
2468         list_for_each_entry_safe(mapping, next, &bo_va->valids, list) {
2469                 list_del(&mapping->list);
2470                 amdgpu_vm_it_remove(mapping, &vm->va);
2471                 mapping->bo_va = NULL;
2472                 trace_amdgpu_vm_bo_unmap(bo_va, mapping);
2473                 list_add(&mapping->list, &vm->freed);
2474         }
2475         list_for_each_entry_safe(mapping, next, &bo_va->invalids, list) {
2476                 list_del(&mapping->list);
2477                 amdgpu_vm_it_remove(mapping, &vm->va);
2478                 amdgpu_vm_free_mapping(adev, vm, mapping,
2479                                        bo_va->last_pt_update);
2480         }
2481 
2482         dma_fence_put(bo_va->last_pt_update);
2483 
2484         if (bo && bo_va->is_xgmi) {
2485                 mutex_lock(&adev->vm_manager.lock_pstate);
2486                 if (--adev->vm_manager.xgmi_map_counter == 0)
2487                         amdgpu_xgmi_set_pstate(adev, 0);
2488                 mutex_unlock(&adev->vm_manager.lock_pstate);
2489         }
2490 
2491         kfree(bo_va);
2492 }
2493 
2494 /**
2495  * amdgpu_vm_bo_invalidate - mark the bo as invalid
2496  *
2497  * @adev: amdgpu_device pointer
2498  * @bo: amdgpu buffer object
2499  * @evicted: is the BO evicted
2500  *
2501  * Mark @bo as invalid.
2502  */
2503 void amdgpu_vm_bo_invalidate(struct amdgpu_device *adev,
2504                              struct amdgpu_bo *bo, bool evicted)
2505 {
2506         struct amdgpu_vm_bo_base *bo_base;
2507 
2508         /* shadow bo doesn't have bo base, its validation needs its parent */
2509         if (bo->parent && bo->parent->shadow == bo)
2510                 bo = bo->parent;
2511 
2512         for (bo_base = bo->vm_bo; bo_base; bo_base = bo_base->next) {
2513                 struct amdgpu_vm *vm = bo_base->vm;
2514 
2515                 if (evicted && bo->tbo.base.resv == vm->root.base.bo->tbo.base.resv) {
2516                         amdgpu_vm_bo_evicted(bo_base);
2517                         continue;
2518                 }
2519 
2520                 if (bo_base->moved)
2521                         continue;
2522                 bo_base->moved = true;
2523 
2524                 if (bo->tbo.type == ttm_bo_type_kernel)
2525                         amdgpu_vm_bo_relocated(bo_base);
2526                 else if (bo->tbo.base.resv == vm->root.base.bo->tbo.base.resv)
2527                         amdgpu_vm_bo_moved(bo_base);
2528                 else
2529                         amdgpu_vm_bo_invalidated(bo_base);
2530         }
2531 }
2532 
2533 /**
2534  * amdgpu_vm_get_block_size - calculate VM page table size as power of two
2535  *
2536  * @vm_size: VM size
2537  *
2538  * Returns:
2539  * VM page table as power of two
2540  */
2541 static uint32_t amdgpu_vm_get_block_size(uint64_t vm_size)
2542 {
2543         /* Total bits covered by PD + PTs */
2544         unsigned bits = ilog2(vm_size) + 18;
2545 
2546         /* Make sure the PD is 4K in size up to 8GB address space.
2547            Above that split equal between PD and PTs */
2548         if (vm_size <= 8)
2549                 return (bits - 9);
2550         else
2551                 return ((bits + 3) / 2);
2552 }
2553 
2554 /**
2555  * amdgpu_vm_adjust_size - adjust vm size, block size and fragment size
2556  *
2557  * @adev: amdgpu_device pointer
2558  * @min_vm_size: the minimum vm size in GB if it's set auto
2559  * @fragment_size_default: Default PTE fragment size
2560  * @max_level: max VMPT level
2561  * @max_bits: max address space size in bits
2562  *
2563  */
2564 void amdgpu_vm_adjust_size(struct amdgpu_device *adev, uint32_t min_vm_size,
2565                            uint32_t fragment_size_default, unsigned max_level,
2566                            unsigned max_bits)
2567 {
2568         unsigned int max_size = 1 << (max_bits - 30);
2569         unsigned int vm_size;
2570         uint64_t tmp;
2571 
2572         /* adjust vm size first */
2573         if (amdgpu_vm_size != -1) {
2574                 vm_size = amdgpu_vm_size;
2575                 if (vm_size > max_size) {
2576                         dev_warn(adev->dev, "VM size (%d) too large, max is %u GB\n",
2577                                  amdgpu_vm_size, max_size);
2578                         vm_size = max_size;
2579                 }
2580         } else {
2581                 struct sysinfo si;
2582                 unsigned int phys_ram_gb;
2583 
2584                 /* Optimal VM size depends on the amount of physical
2585                  * RAM available. Underlying requirements and
2586                  * assumptions:
2587                  *
2588                  *  - Need to map system memory and VRAM from all GPUs
2589                  *     - VRAM from other GPUs not known here
2590                  *     - Assume VRAM <= system memory
2591                  *  - On GFX8 and older, VM space can be segmented for
2592                  *    different MTYPEs
2593                  *  - Need to allow room for fragmentation, guard pages etc.
2594                  *
2595                  * This adds up to a rough guess of system memory x3.
2596                  * Round up to power of two to maximize the available
2597                  * VM size with the given page table size.
2598                  */
2599                 si_meminfo(&si);
2600                 phys_ram_gb = ((uint64_t)si.totalram * si.mem_unit +
2601                                (1 << 30) - 1) >> 30;
2602                 vm_size = roundup_pow_of_two(
2603                         min(max(phys_ram_gb * 3, min_vm_size), max_size));
2604         }
2605 
2606         adev->vm_manager.max_pfn = (uint64_t)vm_size << 18;
2607 
2608         tmp = roundup_pow_of_two(adev->vm_manager.max_pfn);
2609         if (amdgpu_vm_block_size != -1)
2610                 tmp >>= amdgpu_vm_block_size - 9;
2611         tmp = DIV_ROUND_UP(fls64(tmp) - 1, 9) - 1;
2612         adev->vm_manager.num_level = min(max_level, (unsigned)tmp);
2613         switch (adev->vm_manager.num_level) {
2614         case 3:
2615                 adev->vm_manager.root_level = AMDGPU_VM_PDB2;
2616                 break;
2617         case 2:
2618                 adev->vm_manager.root_level = AMDGPU_VM_PDB1;
2619                 break;
2620         case 1:
2621                 adev->vm_manager.root_level = AMDGPU_VM_PDB0;
2622                 break;
2623         default:
2624                 dev_err(adev->dev, "VMPT only supports 2~4+1 levels\n");
2625         }
2626         /* block size depends on vm size and hw setup*/
2627         if (amdgpu_vm_block_size != -1)
2628                 adev->vm_manager.block_size =
2629                         min((unsigned)amdgpu_vm_block_size, max_bits
2630                             - AMDGPU_GPU_PAGE_SHIFT
2631                             - 9 * adev->vm_manager.num_level);
2632         else if (adev->vm_manager.num_level > 1)
2633                 adev->vm_manager.block_size = 9;
2634         else
2635                 adev->vm_manager.block_size = amdgpu_vm_get_block_size(tmp);
2636 
2637         if (amdgpu_vm_fragment_size == -1)
2638                 adev->vm_manager.fragment_size = fragment_size_default;
2639         else
2640                 adev->vm_manager.fragment_size = amdgpu_vm_fragment_size;
2641 
2642         DRM_INFO("vm size is %u GB, %u levels, block size is %u-bit, fragment size is %u-bit\n",
2643                  vm_size, adev->vm_manager.num_level + 1,
2644                  adev->vm_manager.block_size,
2645                  adev->vm_manager.fragment_size);
2646 }
2647 
2648 /**
2649  * amdgpu_vm_wait_idle - wait for the VM to become idle
2650  *
2651  * @vm: VM object to wait for
2652  * @timeout: timeout to wait for VM to become idle
2653  */
2654 long amdgpu_vm_wait_idle(struct amdgpu_vm *vm, long timeout)
2655 {
2656         return dma_resv_wait_timeout_rcu(vm->root.base.bo->tbo.base.resv,
2657                                                    true, true, timeout);
2658 }
2659 
2660 /**
2661  * amdgpu_vm_init - initialize a vm instance
2662  *
2663  * @adev: amdgpu_device pointer
2664  * @vm: requested vm
2665  * @vm_context: Indicates if it GFX or Compute context
2666  * @pasid: Process address space identifier
2667  *
2668  * Init @vm fields.
2669  *
2670  * Returns:
2671  * 0 for success, error for failure.
2672  */
2673 int amdgpu_vm_init(struct amdgpu_device *adev, struct amdgpu_vm *vm,
2674                    int vm_context, unsigned int pasid)
2675 {
2676         struct amdgpu_bo_param bp;
2677         struct amdgpu_bo *root;
2678         int r, i;
2679 
2680         vm->va = RB_ROOT_CACHED;
2681         for (i = 0; i < AMDGPU_MAX_VMHUBS; i++)
2682                 vm->reserved_vmid[i] = NULL;
2683         INIT_LIST_HEAD(&vm->evicted);
2684         INIT_LIST_HEAD(&vm->relocated);
2685         INIT_LIST_HEAD(&vm->moved);
2686         INIT_LIST_HEAD(&vm->idle);
2687         INIT_LIST_HEAD(&vm->invalidated);
2688         spin_lock_init(&vm->invalidated_lock);
2689         INIT_LIST_HEAD(&vm->freed);
2690 
2691         /* create scheduler entity for page table updates */
2692         r = drm_sched_entity_init(&vm->entity, adev->vm_manager.vm_pte_rqs,
2693                                   adev->vm_manager.vm_pte_num_rqs, NULL);
2694         if (r)
2695                 return r;
2696 
2697         vm->pte_support_ats = false;
2698 
2699         if (vm_context == AMDGPU_VM_CONTEXT_COMPUTE) {
2700                 vm->use_cpu_for_update = !!(adev->vm_manager.vm_update_mode &
2701                                                 AMDGPU_VM_USE_CPU_FOR_COMPUTE);
2702 
2703                 if (adev->asic_type == CHIP_RAVEN)
2704                         vm->pte_support_ats = true;
2705         } else {
2706                 vm->use_cpu_for_update = !!(adev->vm_manager.vm_update_mode &
2707                                                 AMDGPU_VM_USE_CPU_FOR_GFX);
2708         }
2709         DRM_DEBUG_DRIVER("VM update mode is %s\n",
2710                          vm->use_cpu_for_update ? "CPU" : "SDMA");
2711         WARN_ONCE((vm->use_cpu_for_update && !amdgpu_gmc_vram_full_visible(&adev->gmc)),
2712                   "CPU update of VM recommended only for large BAR system\n");
2713 
2714         if (vm->use_cpu_for_update)
2715                 vm->update_funcs = &amdgpu_vm_cpu_funcs;
2716         else
2717                 vm->update_funcs = &amdgpu_vm_sdma_funcs;
2718         vm->last_update = NULL;
2719 
2720         amdgpu_vm_bo_param(adev, vm, adev->vm_manager.root_level, &bp);
2721         if (vm_context == AMDGPU_VM_CONTEXT_COMPUTE)
2722                 bp.flags &= ~AMDGPU_GEM_CREATE_SHADOW;
2723         r = amdgpu_bo_create(adev, &bp, &root);
2724         if (r)
2725                 goto error_free_sched_entity;
2726 
2727         r = amdgpu_bo_reserve(root, true);
2728         if (r)
2729                 goto error_free_root;
2730 
2731         r = dma_resv_reserve_shared(root->tbo.base.resv, 1);
2732         if (r)
2733                 goto error_unreserve;
2734 
2735         amdgpu_vm_bo_base_init(&vm->root.base, vm, root);
2736 
2737         r = amdgpu_vm_clear_bo(adev, vm, root);
2738         if (r)
2739                 goto error_unreserve;
2740 
2741         amdgpu_bo_unreserve(vm->root.base.bo);
2742 
2743         if (pasid) {
2744                 unsigned long flags;
2745 
2746                 spin_lock_irqsave(&adev->vm_manager.pasid_lock, flags);
2747                 r = idr_alloc(&adev->vm_manager.pasid_idr, vm, pasid, pasid + 1,
2748                               GFP_ATOMIC);
2749                 spin_unlock_irqrestore(&adev->vm_manager.pasid_lock, flags);
2750                 if (r < 0)
2751                         goto error_free_root;
2752 
2753                 vm->pasid = pasid;
2754         }
2755 
2756         INIT_KFIFO(vm->faults);
2757 
2758         return 0;
2759 
2760 error_unreserve:
2761         amdgpu_bo_unreserve(vm->root.base.bo);
2762 
2763 error_free_root:
2764         amdgpu_bo_unref(&vm->root.base.bo->shadow);
2765         amdgpu_bo_unref(&vm->root.base.bo);
2766         vm->root.base.bo = NULL;
2767 
2768 error_free_sched_entity:
2769         drm_sched_entity_destroy(&vm->entity);
2770 
2771         return r;
2772 }
2773 
2774 /**
2775  * amdgpu_vm_check_clean_reserved - check if a VM is clean
2776  *
2777  * @adev: amdgpu_device pointer
2778  * @vm: the VM to check
2779  *
2780  * check all entries of the root PD, if any subsequent PDs are allocated,
2781  * it means there are page table creating and filling, and is no a clean
2782  * VM
2783  *
2784  * Returns:
2785  *      0 if this VM is clean
2786  */
2787 static int amdgpu_vm_check_clean_reserved(struct amdgpu_device *adev,
2788         struct amdgpu_vm *vm)
2789 {
2790         enum amdgpu_vm_level root = adev->vm_manager.root_level;
2791         unsigned int entries = amdgpu_vm_num_entries(adev, root);
2792         unsigned int i = 0;
2793 
2794         if (!(vm->root.entries))
2795                 return 0;
2796 
2797         for (i = 0; i < entries; i++) {
2798                 if (vm->root.entries[i].base.bo)
2799                         return -EINVAL;
2800         }
2801 
2802         return 0;
2803 }
2804 
2805 /**
2806  * amdgpu_vm_make_compute - Turn a GFX VM into a compute VM
2807  *
2808  * @adev: amdgpu_device pointer
2809  * @vm: requested vm
2810  *
2811  * This only works on GFX VMs that don't have any BOs added and no
2812  * page tables allocated yet.
2813  *
2814  * Changes the following VM parameters:
2815  * - use_cpu_for_update
2816  * - pte_supports_ats
2817  * - pasid (old PASID is released, because compute manages its own PASIDs)
2818  *
2819  * Reinitializes the page directory to reflect the changed ATS
2820  * setting.
2821  *
2822  * Returns:
2823  * 0 for success, -errno for errors.
2824  */
2825 int amdgpu_vm_make_compute(struct amdgpu_device *adev, struct amdgpu_vm *vm, unsigned int pasid)
2826 {
2827         bool pte_support_ats = (adev->asic_type == CHIP_RAVEN);
2828         int r;
2829 
2830         r = amdgpu_bo_reserve(vm->root.base.bo, true);
2831         if (r)
2832                 return r;
2833 
2834         /* Sanity checks */
2835         r = amdgpu_vm_check_clean_reserved(adev, vm);
2836         if (r)
2837                 goto unreserve_bo;
2838 
2839         if (pasid) {
2840                 unsigned long flags;
2841 
2842                 spin_lock_irqsave(&adev->vm_manager.pasid_lock, flags);
2843                 r = idr_alloc(&adev->vm_manager.pasid_idr, vm, pasid, pasid + 1,
2844                               GFP_ATOMIC);
2845                 spin_unlock_irqrestore(&adev->vm_manager.pasid_lock, flags);
2846 
2847                 if (r == -ENOSPC)
2848                         goto unreserve_bo;
2849                 r = 0;
2850         }
2851 
2852         /* Check if PD needs to be reinitialized and do it before
2853          * changing any other state, in case it fails.
2854          */
2855         if (pte_support_ats != vm->pte_support_ats) {
2856                 vm->pte_support_ats = pte_support_ats;
2857                 r = amdgpu_vm_clear_bo(adev, vm, vm->root.base.bo);
2858                 if (r)
2859                         goto free_idr;
2860         }
2861 
2862         /* Update VM state */
2863         vm->use_cpu_for_update = !!(adev->vm_manager.vm_update_mode &
2864                                     AMDGPU_VM_USE_CPU_FOR_COMPUTE);
2865         DRM_DEBUG_DRIVER("VM update mode is %s\n",
2866                          vm->use_cpu_for_update ? "CPU" : "SDMA");
2867         WARN_ONCE((vm->use_cpu_for_update && !amdgpu_gmc_vram_full_visible(&adev->gmc)),
2868                   "CPU update of VM recommended only for large BAR system\n");
2869 
2870         if (vm->use_cpu_for_update)
2871                 vm->update_funcs = &amdgpu_vm_cpu_funcs;
2872         else
2873                 vm->update_funcs = &amdgpu_vm_sdma_funcs;
2874         dma_fence_put(vm->last_update);
2875         vm->last_update = NULL;
2876 
2877         if (vm->pasid) {
2878                 unsigned long flags;
2879 
2880                 spin_lock_irqsave(&adev->vm_manager.pasid_lock, flags);
2881                 idr_remove(&adev->vm_manager.pasid_idr, vm->pasid);
2882                 spin_unlock_irqrestore(&adev->vm_manager.pasid_lock, flags);
2883 
2884                 /* Free the original amdgpu allocated pasid
2885                  * Will be replaced with kfd allocated pasid
2886                  */
2887                 amdgpu_pasid_free(vm->pasid);
2888                 vm->pasid = 0;
2889         }
2890 
2891         /* Free the shadow bo for compute VM */
2892         amdgpu_bo_unref(&vm->root.base.bo->shadow);
2893 
2894         if (pasid)
2895                 vm->pasid = pasid;
2896 
2897         goto unreserve_bo;
2898 
2899 free_idr:
2900         if (pasid) {
2901                 unsigned long flags;
2902 
2903                 spin_lock_irqsave(&adev->vm_manager.pasid_lock, flags);
2904                 idr_remove(&adev->vm_manager.pasid_idr, pasid);
2905                 spin_unlock_irqrestore(&adev->vm_manager.pasid_lock, flags);
2906         }
2907 unreserve_bo:
2908         amdgpu_bo_unreserve(vm->root.base.bo);
2909         return r;
2910 }
2911 
2912 /**
2913  * amdgpu_vm_release_compute - release a compute vm
2914  * @adev: amdgpu_device pointer
2915  * @vm: a vm turned into compute vm by calling amdgpu_vm_make_compute
2916  *
2917  * This is a correspondant of amdgpu_vm_make_compute. It decouples compute
2918  * pasid from vm. Compute should stop use of vm after this call.
2919  */
2920 void amdgpu_vm_release_compute(struct amdgpu_device *adev, struct amdgpu_vm *vm)
2921 {
2922         if (vm->pasid) {
2923                 unsigned long flags;
2924 
2925                 spin_lock_irqsave(&adev->vm_manager.pasid_lock, flags);
2926                 idr_remove(&adev->vm_manager.pasid_idr, vm->pasid);
2927                 spin_unlock_irqrestore(&adev->vm_manager.pasid_lock, flags);
2928         }
2929         vm->pasid = 0;
2930 }
2931 
2932 /**
2933  * amdgpu_vm_fini - tear down a vm instance
2934  *
2935  * @adev: amdgpu_device pointer
2936  * @vm: requested vm
2937  *
2938  * Tear down @vm.
2939  * Unbind the VM and remove all bos from the vm bo list
2940  */
2941 void amdgpu_vm_fini(struct amdgpu_device *adev, struct amdgpu_vm *vm)
2942 {
2943         struct amdgpu_bo_va_mapping *mapping, *tmp;
2944         bool prt_fini_needed = !!adev->gmc.gmc_funcs->set_prt;
2945         struct amdgpu_bo *root;
2946         int i, r;
2947 
2948         amdgpu_amdkfd_gpuvm_destroy_cb(adev, vm);
2949 
2950         if (vm->pasid) {
2951                 unsigned long flags;
2952 
2953                 spin_lock_irqsave(&adev->vm_manager.pasid_lock, flags);
2954                 idr_remove(&adev->vm_manager.pasid_idr, vm->pasid);
2955                 spin_unlock_irqrestore(&adev->vm_manager.pasid_lock, flags);
2956         }
2957 
2958         drm_sched_entity_destroy(&vm->entity);
2959 
2960         if (!RB_EMPTY_ROOT(&vm->va.rb_root)) {
2961                 dev_err(adev->dev, "still active bo inside vm\n");
2962         }
2963         rbtree_postorder_for_each_entry_safe(mapping, tmp,
2964                                              &vm->va.rb_root, rb) {
2965                 /* Don't remove the mapping here, we don't want to trigger a
2966                  * rebalance and the tree is about to be destroyed anyway.
2967                  */
2968                 list_del(&mapping->list);
2969                 kfree(mapping);
2970         }
2971         list_for_each_entry_safe(mapping, tmp, &vm->freed, list) {
2972                 if (mapping->flags & AMDGPU_PTE_PRT && prt_fini_needed) {
2973                         amdgpu_vm_prt_fini(adev, vm);
2974                         prt_fini_needed = false;
2975                 }
2976 
2977                 list_del(&mapping->list);
2978                 amdgpu_vm_free_mapping(adev, vm, mapping, NULL);
2979         }
2980 
2981         root = amdgpu_bo_ref(vm->root.base.bo);
2982         r = amdgpu_bo_reserve(root, true);
2983         if (r) {
2984                 dev_err(adev->dev, "Leaking page tables because BO reservation failed\n");
2985         } else {
2986                 amdgpu_vm_free_pts(adev, vm, NULL);
2987                 amdgpu_bo_unreserve(root);
2988         }
2989         amdgpu_bo_unref(&root);
2990         WARN_ON(vm->root.base.bo);
2991         dma_fence_put(vm->last_update);
2992         for (i = 0; i < AMDGPU_MAX_VMHUBS; i++)
2993                 amdgpu_vmid_free_reserved(adev, vm, i);
2994 }
2995 
2996 /**
2997  * amdgpu_vm_manager_init - init the VM manager
2998  *
2999  * @adev: amdgpu_device pointer
3000  *
3001  * Initialize the VM manager structures
3002  */
3003 void amdgpu_vm_manager_init(struct amdgpu_device *adev)
3004 {
3005         unsigned i;
3006 
3007         amdgpu_vmid_mgr_init(adev);
3008 
3009         adev->vm_manager.fence_context =
3010                 dma_fence_context_alloc(AMDGPU_MAX_RINGS);
3011         for (i = 0; i < AMDGPU_MAX_RINGS; ++i)
3012                 adev->vm_manager.seqno[i] = 0;
3013 
3014         spin_lock_init(&adev->vm_manager.prt_lock);
3015         atomic_set(&adev->vm_manager.num_prt_users, 0);
3016 
3017         /* If not overridden by the user, by default, only in large BAR systems
3018          * Compute VM tables will be updated by CPU
3019          */
3020 #ifdef CONFIG_X86_64
3021         if (amdgpu_vm_update_mode == -1) {
3022                 if (amdgpu_gmc_vram_full_visible(&adev->gmc))
3023                         adev->vm_manager.vm_update_mode =
3024                                 AMDGPU_VM_USE_CPU_FOR_COMPUTE;
3025                 else
3026                         adev->vm_manager.vm_update_mode = 0;
3027         } else
3028                 adev->vm_manager.vm_update_mode = amdgpu_vm_update_mode;
3029 #else
3030         adev->vm_manager.vm_update_mode = 0;
3031 #endif
3032 
3033         idr_init(&adev->vm_manager.pasid_idr);
3034         spin_lock_init(&adev->vm_manager.pasid_lock);
3035 
3036         adev->vm_manager.xgmi_map_counter = 0;
3037         mutex_init(&adev->vm_manager.lock_pstate);
3038 }
3039 
3040 /**
3041  * amdgpu_vm_manager_fini - cleanup VM manager
3042  *
3043  * @adev: amdgpu_device pointer
3044  *
3045  * Cleanup the VM manager and free resources.
3046  */
3047 void amdgpu_vm_manager_fini(struct amdgpu_device *adev)
3048 {
3049         WARN_ON(!idr_is_empty(&adev->vm_manager.pasid_idr));
3050         idr_destroy(&adev->vm_manager.pasid_idr);
3051 
3052         amdgpu_vmid_mgr_fini(adev);
3053 }
3054 
3055 /**
3056  * amdgpu_vm_ioctl - Manages VMID reservation for vm hubs.
3057  *
3058  * @dev: drm device pointer
3059  * @data: drm_amdgpu_vm
3060  * @filp: drm file pointer
3061  *
3062  * Returns:
3063  * 0 for success, -errno for errors.
3064  */
3065 int amdgpu_vm_ioctl(struct drm_device *dev, void *data, struct drm_file *filp)
3066 {
3067         union drm_amdgpu_vm *args = data;
3068         struct amdgpu_device *adev = dev->dev_private;
3069         struct amdgpu_fpriv *fpriv = filp->driver_priv;
3070         int r;
3071 
3072         switch (args->in.op) {
3073         case AMDGPU_VM_OP_RESERVE_VMID:
3074                 /* current, we only have requirement to reserve vmid from gfxhub */
3075                 r = amdgpu_vmid_alloc_reserved(adev, &fpriv->vm, AMDGPU_GFXHUB_0);
3076                 if (r)
3077                         return r;
3078                 break;
3079         case AMDGPU_VM_OP_UNRESERVE_VMID:
3080                 amdgpu_vmid_free_reserved(adev, &fpriv->vm, AMDGPU_GFXHUB_0);
3081                 break;
3082         default:
3083                 return -EINVAL;
3084         }
3085 
3086         return 0;
3087 }
3088 
3089 /**
3090  * amdgpu_vm_get_task_info - Extracts task info for a PASID.
3091  *
3092  * @adev: drm device pointer
3093  * @pasid: PASID identifier for VM
3094  * @task_info: task_info to fill.
3095  */
3096 void amdgpu_vm_get_task_info(struct amdgpu_device *adev, unsigned int pasid,
3097                          struct amdgpu_task_info *task_info)
3098 {
3099         struct amdgpu_vm *vm;
3100         unsigned long flags;
3101 
3102         spin_lock_irqsave(&adev->vm_manager.pasid_lock, flags);
3103 
3104         vm = idr_find(&adev->vm_manager.pasid_idr, pasid);
3105         if (vm)
3106                 *task_info = vm->task_info;
3107 
3108         spin_unlock_irqrestore(&adev->vm_manager.pasid_lock, flags);
3109 }
3110 
3111 /**
3112  * amdgpu_vm_set_task_info - Sets VMs task info.
3113  *
3114  * @vm: vm for which to set the info
3115  */
3116 void amdgpu_vm_set_task_info(struct amdgpu_vm *vm)
3117 {
3118         if (!vm->task_info.pid) {
3119                 vm->task_info.pid = current->pid;
3120                 get_task_comm(vm->task_info.task_name, current);
3121 
3122                 if (current->group_leader->mm == current->mm) {
3123                         vm->task_info.tgid = current->group_leader->pid;
3124                         get_task_comm(vm->task_info.process_name, current->group_leader);
3125                 }
3126         }
3127 }