149 static void intel_flush_svm_range_dev (struct intel_svm *svm, struct intel_svm_dev *sdev,  in intel_flush_svm_range_dev()  argument
159 			desc.low = QI_EIOTLB_PASID(svm->pasid) | QI_EIOTLB_DID(sdev->did) |  in intel_flush_svm_range_dev()
162 			desc.low = QI_EIOTLB_PASID(svm->pasid) | QI_EIOTLB_DID(sdev->did) |  in intel_flush_svm_range_dev()
168 		desc.low = QI_EIOTLB_PASID(svm->pasid) | QI_EIOTLB_DID(sdev->did) |  in intel_flush_svm_range_dev()
175 	if (sdev->dev_iotlb) {  in intel_flush_svm_range_dev()
176 		desc.low = QI_DEV_EIOTLB_PASID(svm->pasid) | QI_DEV_EIOTLB_SID(sdev->sid) |  in intel_flush_svm_range_dev()
177 			QI_DEV_EIOTLB_QDEP(sdev->qdep) | QI_DEIOTLB_TYPE;  in intel_flush_svm_range_dev()
198 	struct intel_svm_dev *sdev;  in intel_flush_svm_range()  local
206 	list_for_each_entry_rcu(sdev, &svm->devs, list)  in intel_flush_svm_range()
207 		intel_flush_svm_range_dev(svm, sdev, address, pages, ih, gl);  in intel_flush_svm_range()
239 static void intel_flush_pasid_dev(struct intel_svm *svm, struct intel_svm_dev *sdev, int pasid)  in intel_flush_pasid_dev()  argument
244 	desc.low = QI_PC_TYPE | QI_PC_DID(sdev->did) | QI_PC_PASID_SEL | QI_PC_PASID(pasid);  in intel_flush_pasid_dev()
252 	struct intel_svm_dev *sdev;  in intel_mm_release()  local
270 	list_for_each_entry_rcu(sdev, &svm->devs, list) {  in intel_mm_release()
271 		intel_flush_pasid_dev(svm, sdev, svm->pasid);  in intel_mm_release()
272 		intel_flush_svm_range_dev(svm, sdev, 0, -1, 0, !svm->mm);  in intel_mm_release()
290 	struct intel_svm_dev *sdev;  in intel_svm_bind_mm()  local
331 			list_for_each_entry(sdev, &svm->devs, list) {  in intel_svm_bind_mm()
332 				if (dev == sdev->dev) {  in intel_svm_bind_mm()
333 					if (sdev->ops != ops) {  in intel_svm_bind_mm()
337 					sdev->users++;  in intel_svm_bind_mm()
346 	sdev = kzalloc(sizeof(*sdev), GFP_KERNEL);  in intel_svm_bind_mm()
347 	if (!sdev) {  in intel_svm_bind_mm()
351 	sdev->dev = dev;  in intel_svm_bind_mm()
353 	ret = intel_iommu_enable_pasid(iommu, sdev);  in intel_svm_bind_mm()
357 		kfree(sdev);  in intel_svm_bind_mm()
361 	sdev->users = 1;  in intel_svm_bind_mm()
362 	sdev->ops = ops;  in intel_svm_bind_mm()
363 	init_rcu_head(&sdev->rcu);  in intel_svm_bind_mm()
369 			kfree(sdev);  in intel_svm_bind_mm()
396 				kfree(sdev);  in intel_svm_bind_mm()
413 			intel_flush_pasid_dev(svm, sdev, 0);  in intel_svm_bind_mm()
415 	list_add_rcu(&sdev->list, &svm->devs);  in intel_svm_bind_mm()
430 	struct intel_svm_dev *sdev;  in intel_svm_unbind_mm()  local
444 	list_for_each_entry(sdev, &svm->devs, list) {  in intel_svm_unbind_mm()
445 		if (dev == sdev->dev) {  in intel_svm_unbind_mm()
447 			sdev->users--;  in intel_svm_unbind_mm()
448 			if (!sdev->users) {  in intel_svm_unbind_mm()
449 				list_del_rcu(&sdev->list);  in intel_svm_unbind_mm()
457 				intel_flush_pasid_dev(svm, sdev, svm->pasid);  in intel_svm_unbind_mm()
458 				intel_flush_svm_range_dev(svm, sdev, 0, -1, 0, !svm->mm);  in intel_svm_unbind_mm()
459 				kfree_rcu(sdev, rcu);  in intel_svm_unbind_mm()
534 		struct intel_svm_dev *sdev;  in prq_event_thread()  local
598 		list_for_each_entry_rcu(sdev, &svm->devs, list) {  in prq_event_thread()
599 			if (sdev->sid == PCI_DEVID(req->bus, req->devfn))  in prq_event_thread()
607 		if (WARN_ON(&sdev->list == &svm->devs))  in prq_event_thread()
608 			sdev = NULL;  in prq_event_thread()
610 		if (sdev && sdev->ops && sdev->ops->fault_cb) {  in prq_event_thread()
613 			sdev->ops->fault_cb(sdev->dev, req->pasid, req->addr, req->private, rwxp, result);  in prq_event_thread()
617 		sdev = NULL;  in prq_event_thread()