33 	struct nvkm_mmu *mmu = vm->mmu;  in nvkm_vm_map_at()  local
35 	int big = vma->node->type != mmu->spg_shift;  in nvkm_vm_map_at()
38 	u32 pde  = (offset >> mmu->pgt_bits) - vm->fpde;  in nvkm_vm_map_at()
39 	u32 pte  = (offset & ((1 << mmu->pgt_bits) - 1)) >> bits;  in nvkm_vm_map_at()
40 	u32 max  = 1 << (mmu->pgt_bits - bits);  in nvkm_vm_map_at()
56 			mmu->map(vma, pgt, node, pte, len, phys, delta);  in nvkm_vm_map_at()
70 	mmu->flush(vm);  in nvkm_vm_map_at()
78 	struct nvkm_mmu *mmu = vm->mmu;  in nvkm_vm_map_sg_table()  local
79 	int big = vma->node->type != mmu->spg_shift;  in nvkm_vm_map_sg_table()
83 	u32 pde  = (offset >> mmu->pgt_bits) - vm->fpde;  in nvkm_vm_map_sg_table()
84 	u32 pte  = (offset & ((1 << mmu->pgt_bits) - 1)) >> bits;  in nvkm_vm_map_sg_table()
85 	u32 max  = 1 << (mmu->pgt_bits - bits);  in nvkm_vm_map_sg_table()
103 			mmu->map_sg(vma, pgt, mem, pte, 1, &addr);  in nvkm_vm_map_sg_table()
118 				mmu->map_sg(vma, pgt, mem, pte, 1, &addr);  in nvkm_vm_map_sg_table()
128 	mmu->flush(vm);  in nvkm_vm_map_sg_table()
136 	struct nvkm_mmu *mmu = vm->mmu;  in nvkm_vm_map_sg()  local
138 	int big = vma->node->type != mmu->spg_shift;  in nvkm_vm_map_sg()
142 	u32 pde  = (offset >> mmu->pgt_bits) - vm->fpde;  in nvkm_vm_map_sg()
143 	u32 pte  = (offset & ((1 << mmu->pgt_bits) - 1)) >> bits;  in nvkm_vm_map_sg()
144 	u32 max  = 1 << (mmu->pgt_bits - bits);  in nvkm_vm_map_sg()
155 		mmu->map_sg(vma, pgt, mem, pte, len, list);  in nvkm_vm_map_sg()
166 	mmu->flush(vm);  in nvkm_vm_map_sg()
185 	struct nvkm_mmu *mmu = vm->mmu;  in nvkm_vm_unmap_at()  local
186 	int big = vma->node->type != mmu->spg_shift;  in nvkm_vm_unmap_at()
190 	u32 pde  = (offset >> mmu->pgt_bits) - vm->fpde;  in nvkm_vm_unmap_at()
191 	u32 pte  = (offset & ((1 << mmu->pgt_bits) - 1)) >> bits;  in nvkm_vm_unmap_at()
192 	u32 max  = 1 << (mmu->pgt_bits - bits);  in nvkm_vm_unmap_at()
203 		mmu->unmap(pgt, pte, len);  in nvkm_vm_unmap_at()
213 	mmu->flush(vm);  in nvkm_vm_unmap_at()
225 	struct nvkm_mmu *mmu = vm->mmu;  in nvkm_vm_unmap_pgt()  local
240 			mmu->map_pgt(vpgd->obj, pde, vpgt->obj);  in nvkm_vm_unmap_pgt()
243 		mutex_unlock(&nv_subdev(mmu)->mutex);  in nvkm_vm_unmap_pgt()
245 		mutex_lock(&nv_subdev(mmu)->mutex);  in nvkm_vm_unmap_pgt()
252 	struct nvkm_mmu *mmu = vm->mmu;  in nvkm_vm_map_pgt()  local
256 	int big = (type != mmu->spg_shift);  in nvkm_vm_map_pgt()
260 	pgt_size  = (1 << (mmu->pgt_bits + 12)) >> type;  in nvkm_vm_map_pgt()
263 	mutex_unlock(&nv_subdev(mmu)->mutex);  in nvkm_vm_map_pgt()
264 	ret = nvkm_gpuobj_new(nv_object(vm->mmu), NULL, pgt_size, 0x1000,  in nvkm_vm_map_pgt()
266 	mutex_lock(&nv_subdev(mmu)->mutex);  in nvkm_vm_map_pgt()
272 		mutex_unlock(&nv_subdev(mmu)->mutex);  in nvkm_vm_map_pgt()
274 		mutex_lock(&nv_subdev(mmu)->mutex);  in nvkm_vm_map_pgt()
280 		mmu->map_pgt(vpgd->obj, pde, vpgt->obj);  in nvkm_vm_map_pgt()
290 	struct nvkm_mmu *mmu = vm->mmu;  in nvkm_vm_get()  local
296 	mutex_lock(&nv_subdev(mmu)->mutex);  in nvkm_vm_get()
300 		mutex_unlock(&nv_subdev(mmu)->mutex);  in nvkm_vm_get()
304 	fpde = (vma->node->offset >> mmu->pgt_bits);  in nvkm_vm_get()
305 	lpde = (vma->node->offset + vma->node->length - 1) >> mmu->pgt_bits;  in nvkm_vm_get()
309 		int big = (vma->node->type != mmu->spg_shift);  in nvkm_vm_get()
321 			mutex_unlock(&nv_subdev(mmu)->mutex);  in nvkm_vm_get()
325 	mutex_unlock(&nv_subdev(mmu)->mutex);  in nvkm_vm_get()
338 	struct nvkm_mmu *mmu = vm->mmu;  in nvkm_vm_put()  local
343 	fpde = (vma->node->offset >> mmu->pgt_bits);  in nvkm_vm_put()
344 	lpde = (vma->node->offset + vma->node->length - 1) >> mmu->pgt_bits;  in nvkm_vm_put()
346 	mutex_lock(&nv_subdev(mmu)->mutex);  in nvkm_vm_put()
347 	nvkm_vm_unmap_pgt(vm, vma->node->type != mmu->spg_shift, fpde, lpde);  in nvkm_vm_put()
349 	mutex_unlock(&nv_subdev(mmu)->mutex);  in nvkm_vm_put()
355 nvkm_vm_create(struct nvkm_mmu *mmu, u64 offset, u64 length, u64 mm_offset,  in nvkm_vm_create()  argument
367 	vm->mmu = mmu;  in nvkm_vm_create()
369 	vm->fpde = offset >> (mmu->pgt_bits + 12);  in nvkm_vm_create()
370 	vm->lpde = (offset + length - 1) >> (mmu->pgt_bits + 12);  in nvkm_vm_create()
395 	struct nvkm_mmu *mmu = nvkm_mmu(device);  in nvkm_vm_new()  local
396 	return mmu->create(mmu, offset, length, mm_offset, pvm);  in nvkm_vm_new()
402 	struct nvkm_mmu *mmu = vm->mmu;  in nvkm_vm_link()  local
415 	mutex_lock(&nv_subdev(mmu)->mutex);  in nvkm_vm_link()
417 		mmu->map_pgt(pgd, i, vm->pgt[i - vm->fpde].obj);  in nvkm_vm_link()
419 	mutex_unlock(&nv_subdev(mmu)->mutex);  in nvkm_vm_link()
426 	struct nvkm_mmu *mmu = vm->mmu;  in nvkm_vm_unlink()  local
433 	mutex_lock(&nv_subdev(mmu)->mutex);  in nvkm_vm_unlink()
442 	mutex_unlock(&nv_subdev(mmu)->mutex);  in nvkm_vm_unlink()