365 static struct dmar_domain *si_domain;
381 struct dmar_domain {  struct
431 	struct dmar_domain *domain; /* pointer to domain */  argument
465 	struct dmar_domain *domain[HIGH_WATER_MARK];
480 static void domain_exit(struct dmar_domain *domain);
481 static void domain_remove_dev_info(struct dmar_domain *domain);
482 static void dmar_remove_one_dev_info(struct dmar_domain *domain,
487 static int domain_detach_iommu(struct dmar_domain *domain,
565 static struct dmar_domain *to_dmar_domain(struct iommu_domain *dom)  in to_dmar_domain()
567 	return container_of(dom, struct dmar_domain, domain);  in to_dmar_domain()
615 static struct dmar_domain* get_iommu_domain(struct intel_iommu *iommu, u16 did)  in get_iommu_domain()
617 	struct dmar_domain **domains;  in get_iommu_domain()
628 			     struct dmar_domain *domain)  in set_iommu_domain()
630 	struct dmar_domain **domains;  in set_iommu_domain()
634 		size_t size = 256 * sizeof(struct dmar_domain *);  in set_iommu_domain()
681 static inline int domain_type_is_vm(struct dmar_domain *domain)  in domain_type_is_vm()
686 static inline int domain_type_is_si(struct dmar_domain *domain)  in domain_type_is_si()
691 static inline int domain_type_is_vm_or_si(struct dmar_domain *domain)  in domain_type_is_vm_or_si()
697 static inline int domain_pfn_supported(struct dmar_domain *domain,  in domain_pfn_supported()
739 static struct intel_iommu *domain_get_iommu(struct dmar_domain *domain)  in domain_get_iommu()
754 static void domain_update_iommu_coherency(struct dmar_domain *domain)  in domain_update_iommu_coherency()
829 static void domain_update_iommu_cap(struct dmar_domain *domain)  in domain_update_iommu_cap()
930 static void domain_flush_cache(struct dmar_domain *domain,  in domain_flush_cache()
994 static struct dma_pte *pfn_to_dma_pte(struct dmar_domain *domain,  in pfn_to_dma_pte()
1050 static struct dma_pte *dma_pfn_level_pte(struct dmar_domain *domain,  in dma_pfn_level_pte()
1082 static void dma_pte_clear_range(struct dmar_domain *domain,  in dma_pte_clear_range()
1113 static void dma_pte_free_level(struct dmar_domain *domain, int level,  in dma_pte_free_level()
1147 static void dma_pte_free_pagetable(struct dmar_domain *domain,  in dma_pte_free_pagetable()
1174 static struct page *dma_pte_list_pagetables(struct dmar_domain *domain,  in dma_pte_list_pagetables()
1198 static struct page *dma_pte_clear_level(struct dmar_domain *domain, int level,  in dma_pte_clear_level()
1250 static struct page *domain_unmap(struct dmar_domain *domain,  in domain_unmap()
1440 iommu_support_dev_iotlb (struct dmar_domain *domain, struct intel_iommu *iommu,  in iommu_support_dev_iotlb()
1513 static void iommu_flush_dev_iotlb(struct dmar_domain *domain,  in iommu_flush_dev_iotlb()
1533 				  struct dmar_domain *domain,  in iommu_flush_iotlb_psi()
1636 	size = ((ndomains >> 8) + 1) * sizeof(struct dmar_domain **);  in iommu_init_domains()
1640 		size = 256 * sizeof(struct dmar_domain *);  in iommu_init_domains()
1677 		struct dmar_domain *domain;  in disable_dmar_iommu()
1726 static struct dmar_domain *alloc_domain(int flags)  in alloc_domain()
1728 	struct dmar_domain *domain;  in alloc_domain()
1743 static int domain_attach_iommu(struct dmar_domain *domain,  in domain_attach_iommu()
1777 static int domain_detach_iommu(struct dmar_domain *domain,  in domain_detach_iommu()
1842 static void domain_reserve_special_ranges(struct dmar_domain *domain)  in domain_reserve_special_ranges()
1861 static int domain_init(struct dmar_domain *domain, struct intel_iommu *iommu,  in domain_init()
1912 static void domain_exit(struct dmar_domain *domain)  in domain_exit()
1939 static int domain_context_mapping_one(struct dmar_domain *domain,  in domain_context_mapping_one()
2039 	struct dmar_domain *domain;
2053 domain_context_mapping(struct dmar_domain *domain, struct device *dev)  in domain_context_mapping()
2106 static inline int hardware_largepage_caps(struct dmar_domain *domain,  in hardware_largepage_caps()
2133 static int __domain_mapping(struct dmar_domain *domain, unsigned long iov_pfn,  in __domain_mapping()
2243 static inline int domain_sg_mapping(struct dmar_domain *domain, unsigned long iov_pfn,  in domain_sg_mapping()
2250 static inline int domain_pfn_mapping(struct dmar_domain *domain, unsigned long iov_pfn,  in domain_pfn_mapping()
2277 static void domain_remove_dev_info(struct dmar_domain *domain)  in domain_remove_dev_info()
2292 static struct dmar_domain *find_domain(struct device *dev)  in find_domain()
2316 static struct dmar_domain *dmar_insert_one_dev_info(struct intel_iommu *iommu,  in dmar_insert_one_dev_info()
2319 						    struct dmar_domain *domain)  in dmar_insert_one_dev_info()
2321 	struct dmar_domain *found = NULL;  in dmar_insert_one_dev_info()
2412 static struct dmar_domain *get_domain_for_dev(struct device *dev, int gaw)  in get_domain_for_dev()
2415 	struct dmar_domain *domain, *tmp;  in get_domain_for_dev()
2485 static int iommu_domain_identity_map(struct dmar_domain *domain,  in iommu_domain_identity_map()
2511 				       struct dmar_domain *domain,  in domain_prepare_identity_map()
2554 	struct dmar_domain *domain;  in iommu_prepare_identity_map()
2602 static int md_domain_init(struct dmar_domain *domain, int guest_width);
2651 static int domain_add_dev_info(struct dmar_domain *domain, struct device *dev)  in domain_add_dev_info()
2653 	struct dmar_domain *ndomain;  in domain_add_dev_info()
3303 				     struct dmar_domain *domain,  in intel_alloc_iova()
3334 static struct dmar_domain *__get_valid_domain_for_dev(struct device *dev)  in __get_valid_domain_for_dev()
3337 	struct dmar_domain *domain;  in __get_valid_domain_for_dev()
3368 static inline struct dmar_domain *get_valid_domain_for_dev(struct device *dev)  in get_valid_domain_for_dev()
3427 	struct dmar_domain *domain;  in __intel_map_single()
3522 			struct dmar_domain *domain = deferred_flush[i].domain[j];  in flush_unmaps()
3553 static void add_unmap(struct dmar_domain *dom, struct iova *iova, struct page *freelist)  in add_unmap()
3582 	struct dmar_domain *domain;  in intel_unmap()
3717 	struct dmar_domain *domain;  in intel_map_sg()
3795 					 sizeof(struct dmar_domain),  in iommu_domain_cache_init()
4410 	struct dmar_domain *domain;  in device_notifier()
4713 static void dmar_remove_one_dev_info(struct dmar_domain *domain,  in dmar_remove_one_dev_info()
4725 static int md_domain_init(struct dmar_domain *domain, int guest_width)  in md_domain_init()
4753 	struct dmar_domain *dmar_domain;  in intel_iommu_domain_alloc()  local
4759 	dmar_domain = alloc_domain(DOMAIN_FLAG_VIRTUAL_MACHINE);  in intel_iommu_domain_alloc()
4760 	if (!dmar_domain) {  in intel_iommu_domain_alloc()
4764 	if (md_domain_init(dmar_domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) {  in intel_iommu_domain_alloc()
4766 		domain_exit(dmar_domain);  in intel_iommu_domain_alloc()
4769 	domain_update_iommu_cap(dmar_domain);  in intel_iommu_domain_alloc()
4771 	domain = &dmar_domain->domain;  in intel_iommu_domain_alloc()
4773 	domain->geometry.aperture_end   = __DOMAIN_MAX_ADDR(dmar_domain->gaw);  in intel_iommu_domain_alloc()
4787 	struct dmar_domain *dmar_domain = to_dmar_domain(domain);  in intel_iommu_attach_device()  local
4799 		struct dmar_domain *old_domain;  in intel_iommu_attach_device()
4822 	if (dmar_domain->max_addr > (1LL << addr_width)) {  in intel_iommu_attach_device()
4825 		       __func__, addr_width, dmar_domain->max_addr);  in intel_iommu_attach_device()
4828 	dmar_domain->gaw = addr_width;  in intel_iommu_attach_device()
4833 	while (iommu->agaw < dmar_domain->agaw) {  in intel_iommu_attach_device()
4836 		pte = dmar_domain->pgd;  in intel_iommu_attach_device()
4838 			dmar_domain->pgd = (struct dma_pte *)  in intel_iommu_attach_device()
4842 		dmar_domain->agaw--;  in intel_iommu_attach_device()
4845 	return domain_add_dev_info(dmar_domain, dev);  in intel_iommu_attach_device()
4858 	struct dmar_domain *dmar_domain = to_dmar_domain(domain);  in intel_iommu_map()  local
4867 	if ((iommu_prot & IOMMU_CACHE) && dmar_domain->iommu_snooping)  in intel_iommu_map()
4871 	if (dmar_domain->max_addr < max_addr) {  in intel_iommu_map()
4875 		end = __DOMAIN_MAX_ADDR(dmar_domain->gaw) + 1;  in intel_iommu_map()
4879 			       __func__, dmar_domain->gaw, max_addr);  in intel_iommu_map()
4882 		dmar_domain->max_addr = max_addr;  in intel_iommu_map()
4887 	ret = domain_pfn_mapping(dmar_domain, iova >> VTD_PAGE_SHIFT,  in intel_iommu_map()
4895 	struct dmar_domain *dmar_domain = to_dmar_domain(domain);  in intel_iommu_unmap()  local
4904 	BUG_ON(!pfn_to_dma_pte(dmar_domain, iova >> VTD_PAGE_SHIFT, &level));  in intel_iommu_unmap()
4912 	freelist = domain_unmap(dmar_domain, start_pfn, last_pfn);  in intel_iommu_unmap()
4916 	for_each_domain_iommu(iommu_id, dmar_domain) {  in intel_iommu_unmap()
4919 		iommu_flush_iotlb_psi(g_iommus[iommu_id], dmar_domain,  in intel_iommu_unmap()
4925 	if (dmar_domain->max_addr == iova + size)  in intel_iommu_unmap()
4926 		dmar_domain->max_addr = iova;  in intel_iommu_unmap()
4934 	struct dmar_domain *dmar_domain = to_dmar_domain(domain);  in intel_iommu_iova_to_phys()  local
4939 	pte = pfn_to_dma_pte(dmar_domain, iova >> VTD_PAGE_SHIFT, &level);  in intel_iommu_iova_to_phys()
4996 	struct dmar_domain *domain;  in intel_iommu_enable_pasid()