137 	struct iommu_dev_data *dev_data;  in alloc_dev_data()  local
140 	dev_data = kzalloc(sizeof(*dev_data), GFP_KERNEL);  in alloc_dev_data()
141 	if (!dev_data)  in alloc_dev_data()
144 	INIT_LIST_HEAD(&dev_data->alias_list);  in alloc_dev_data()
146 	dev_data->devid = devid;  in alloc_dev_data()
149 	list_add_tail(&dev_data->dev_data_list, &dev_data_list);  in alloc_dev_data()
152 	return dev_data;  in alloc_dev_data()
155 static void free_dev_data(struct iommu_dev_data *dev_data)  in free_dev_data()  argument
160 	list_del(&dev_data->dev_data_list);  in free_dev_data()
163 	kfree(dev_data);  in free_dev_data()
168 	struct iommu_dev_data *dev_data;  in search_dev_data()  local
172 	list_for_each_entry(dev_data, &dev_data_list, dev_data_list) {  in search_dev_data()
173 		if (dev_data->devid == devid)  in search_dev_data()
177 	dev_data = NULL;  in search_dev_data()
182 	return dev_data;  in search_dev_data()
187 	struct iommu_dev_data *dev_data;  in find_dev_data()  local
189 	dev_data = search_dev_data(devid);  in find_dev_data()
191 	if (dev_data == NULL)  in find_dev_data()
192 		dev_data = alloc_dev_data(devid);  in find_dev_data()
194 	return dev_data;  in find_dev_data()
229 	struct iommu_dev_data *dev_data;  in pdev_pri_erratum()  local
231 	dev_data = get_dev_data(&pdev->dev);  in pdev_pri_erratum()
233 	return dev_data->errata & (1 << erratum) ? true : false;  in pdev_pri_erratum()
365 	struct iommu_dev_data *dev_data;  in iommu_init_device()  local
371 	dev_data = find_dev_data(get_device_id(dev));  in iommu_init_device()
372 	if (!dev_data)  in iommu_init_device()
377 	if (alias != dev_data->devid) {  in iommu_init_device()
384 			free_dev_data(dev_data);  in iommu_init_device()
387 		dev_data->alias_data = alias_data;  in iommu_init_device()
390 		list_add(&dev_data->alias_list, &alias_data->alias_list);  in iommu_init_device()
396 		iommu              = amd_iommu_rlookup_table[dev_data->devid];  in iommu_init_device()
397 		dev_data->iommu_v2 = iommu->is_iommu_v2;  in iommu_init_device()
400 	dev->archdata.iommu = dev_data;  in iommu_init_device()
402 	iommu_device_link(amd_iommu_rlookup_table[dev_data->devid]->iommu_dev,  in iommu_init_device()
424 	struct iommu_dev_data *dev_data = search_dev_data(get_device_id(dev));  in iommu_uninit_device()  local
426 	if (!dev_data)  in iommu_uninit_device()
429 	iommu_device_unlink(amd_iommu_rlookup_table[dev_data->devid]->iommu_dev,  in iommu_uninit_device()
435 	dev_data->alias_data = NULL;  in iommu_uninit_device()
445 	struct iommu_dev_data *dev_data, *n;  in amd_iommu_uninit_devices()  local
457 	list_for_each_entry_safe(dev_data, n, &dev_data_list, dev_data_list)  in amd_iommu_uninit_devices()
458 		free_dev_data(dev_data);  in amd_iommu_uninit_devices()
1131 static int device_flush_iotlb(struct iommu_dev_data *dev_data,  in device_flush_iotlb()  argument
1138 	qdep     = dev_data->ats.qdep;  in device_flush_iotlb()
1139 	iommu    = amd_iommu_rlookup_table[dev_data->devid];  in device_flush_iotlb()
1141 	build_inv_iotlb_pages(&cmd, dev_data->devid, qdep, address, size);  in device_flush_iotlb()
1149 static int device_flush_dte(struct iommu_dev_data *dev_data)  in device_flush_dte()  argument
1154 	iommu = amd_iommu_rlookup_table[dev_data->devid];  in device_flush_dte()
1156 	ret = iommu_flush_dte(iommu, dev_data->devid);  in device_flush_dte()
1160 	if (dev_data->ats.enabled)  in device_flush_dte()
1161 		ret = device_flush_iotlb(dev_data, 0, ~0UL);  in device_flush_dte()
1174 	struct iommu_dev_data *dev_data;  in __domain_flush_pages()  local
1191 	list_for_each_entry(dev_data, &domain->dev_list, list) {  in __domain_flush_pages()
1193 		if (!dev_data->ats.enabled)  in __domain_flush_pages()
1196 		ret |= device_flush_iotlb(dev_data, address, size);  in __domain_flush_pages()
1242 	struct iommu_dev_data *dev_data;  in domain_flush_devices()  local
1244 	list_for_each_entry(dev_data, &domain->dev_list, list)  in domain_flush_devices()
1245 		device_flush_dte(dev_data);  in domain_flush_devices()
2108 static void do_attach(struct iommu_dev_data *dev_data,  in do_attach()  argument
2114 	iommu = amd_iommu_rlookup_table[dev_data->devid];  in do_attach()
2115 	ats   = dev_data->ats.enabled;  in do_attach()
2118 	dev_data->domain = domain;  in do_attach()
2119 	list_add(&dev_data->list, &domain->dev_list);  in do_attach()
2120 	set_dte_entry(dev_data->devid, domain, ats);  in do_attach()
2127 	device_flush_dte(dev_data);  in do_attach()
2130 static void do_detach(struct iommu_dev_data *dev_data)  in do_detach()  argument
2134 	iommu = amd_iommu_rlookup_table[dev_data->devid];  in do_detach()
2137 	dev_data->domain->dev_iommu[iommu->index] -= 1;  in do_detach()
2138 	dev_data->domain->dev_cnt                 -= 1;  in do_detach()
2141 	dev_data->domain = NULL;  in do_detach()
2142 	list_del(&dev_data->list);  in do_detach()
2143 	clear_dte_entry(dev_data->devid);  in do_detach()
2146 	device_flush_dte(dev_data);  in do_detach()
2153 static int __attach_device(struct iommu_dev_data *dev_data,  in __attach_device()  argument
2162 	head = dev_data;  in __attach_device()
2285 	struct iommu_dev_data *dev_data;  in attach_device()  local
2289 	dev_data = get_dev_data(dev);  in attach_device()
2292 		if (!dev_data->iommu_v2 || !dev_data->passthrough)  in attach_device()
2298 		dev_data->ats.enabled = true;  in attach_device()
2299 		dev_data->ats.qdep    = pci_ats_queue_depth(pdev);  in attach_device()
2300 		dev_data->pri_tlp     = pci_pri_tlp_required(pdev);  in attach_device()
2303 		dev_data->ats.enabled = true;  in attach_device()
2304 		dev_data->ats.qdep    = pci_ats_queue_depth(pdev);  in attach_device()
2308 	ret = __attach_device(dev_data, domain);  in attach_device()
2324 static void __detach_device(struct iommu_dev_data *dev_data)  in __detach_device()  argument
2330 	BUG_ON(!dev_data->domain);  in __detach_device()
2332 	domain = dev_data->domain;  in __detach_device()
2336 	head = dev_data;  in __detach_device()
2352 	if (dev_data->passthrough &&  in __detach_device()
2353 	    (dev_data->domain == NULL && domain != pt_domain))  in __detach_device()
2354 		__attach_device(dev_data, pt_domain);  in __detach_device()
2363 	struct iommu_dev_data *dev_data;  in detach_device()  local
2366 	dev_data = get_dev_data(dev);  in detach_device()
2367 	domain   = dev_data->domain;  in detach_device()
2371 	__detach_device(dev_data);  in detach_device()
2376 	else if (dev_data->ats.enabled)  in detach_device()
2379 	dev_data->ats.enabled = false;  in detach_device()
2388 	struct iommu_dev_data *dev_data;  in domain_for_device()  local
2392 	dev_data   = get_dev_data(dev);  in domain_for_device()
2394 	if (dev_data->domain)  in domain_for_device()
2395 		return dev_data->domain;  in domain_for_device()
2397 	if (dev_data->alias_data != NULL) {  in domain_for_device()
2398 		struct iommu_dev_data *alias_data = dev_data->alias_data;  in domain_for_device()
2402 			__attach_device(dev_data, alias_data->domain);  in domain_for_device()
2416 	struct iommu_dev_data *dev_data;  in device_change_notifier()  local
2427 	dev_data = get_dev_data(dev);  in device_change_notifier()
2439 		dev_data = get_dev_data(dev);  in device_change_notifier()
2441 		if (iommu_pass_through || dev_data->iommu_v2) {  in device_change_notifier()
2442 			dev_data->passthrough = true;  in device_change_notifier()
2530 	struct iommu_dev_data *dev_data;  in update_device_table()  local
2532 	list_for_each_entry(dev_data, &domain->dev_list, list)  in update_device_table()
2533 		set_dte_entry(dev_data->devid, domain, dev_data->ats.enabled);  in update_device_table()
3030 	struct iommu_dev_data *dev_data;  in prealloc_protection_domains()  local
3041 		dev_data = get_dev_data(&dev->dev);  in prealloc_protection_domains()
3042 		if (!amd_iommu_force_isolation && dev_data->iommu_v2) {  in prealloc_protection_domains()
3045 			dev_data->passthrough = true;  in prealloc_protection_domains()
3081 	struct iommu_dev_data *dev_data;  in device_dma_ops_init()  local
3094 		dev_data = get_dev_data(&pdev->dev);  in device_dma_ops_init()
3096 		if (!dev_data->passthrough)  in device_dma_ops_init()
3301 	struct iommu_dev_data *dev_data = dev->archdata.iommu;  in amd_iommu_detach_device()  local
3310 	if (dev_data->domain != NULL)  in amd_iommu_detach_device()
3324 	struct iommu_dev_data *dev_data;  in amd_iommu_attach_device()  local
3331 	dev_data = dev->archdata.iommu;  in amd_iommu_attach_device()
3333 	iommu = amd_iommu_rlookup_table[dev_data->devid];  in amd_iommu_attach_device()
3337 	if (dev_data->domain)  in amd_iommu_attach_device()
3447 	struct iommu_dev_data *dev_data;  in amd_iommu_init_passthrough()  local
3459 		dev_data = get_dev_data(&dev->dev);  in amd_iommu_init_passthrough()
3460 		dev_data->passthrough = true;  in amd_iommu_init_passthrough()
3556 	struct iommu_dev_data *dev_data;  in __flush_pasid()  local
3582 	list_for_each_entry(dev_data, &domain->dev_list, list) {  in __flush_pasid()
3586 		BUG_ON(!dev_data->ats.enabled);  in __flush_pasid()
3588 		qdep  = dev_data->ats.qdep;  in __flush_pasid()
3589 		iommu = amd_iommu_rlookup_table[dev_data->devid];  in __flush_pasid()
3591 		build_inv_iotlb_pasid(&cmd, dev_data->devid, pasid,  in __flush_pasid()
3751 	struct iommu_dev_data *dev_data;  in amd_iommu_complete_ppr()  local
3757 	dev_data = get_dev_data(&pdev->dev);  in amd_iommu_complete_ppr()
3758 	iommu    = amd_iommu_rlookup_table[dev_data->devid];  in amd_iommu_complete_ppr()
3760 	build_complete_ppr(&cmd, dev_data->devid, pasid, status,  in amd_iommu_complete_ppr()
3761 			   tag, dev_data->pri_tlp);  in amd_iommu_complete_ppr()
3785 	struct iommu_dev_data *dev_data;  in amd_iommu_enable_device_erratum()  local
3790 	dev_data = get_dev_data(&pdev->dev);  in amd_iommu_enable_device_erratum()
3791 	dev_data->errata |= (1 << erratum);  in amd_iommu_enable_device_erratum()