57 static void pnv_pci_ioda2_table_free_pages(struct iommu_table *tbl);
1267 	struct iommu_table    *tbl;  in pnv_pci_ioda2_release_dma_pe()  local
1270 	tbl = pe->table_group.tables[0];  in pnv_pci_ioda2_release_dma_pe()
1280 	pnv_pci_ioda2_table_free_pages(tbl);  in pnv_pci_ioda2_release_dma_pe()
1281 	iommu_free_table(tbl, of_node_full_name(dev->dev.of_node));  in pnv_pci_ioda2_release_dma_pe()
1640 static void pnv_pci_ioda1_tce_invalidate(struct iommu_table *tbl,  in pnv_pci_ioda1_tce_invalidate()  argument
1644 			&tbl->it_group_list, struct iommu_table_group_link,  in pnv_pci_ioda1_tce_invalidate()
1652 	const unsigned shift = tbl->it_page_shift;  in pnv_pci_ioda1_tce_invalidate()
1654 	start = __pa(((__be64 *)tbl->it_base) + index - tbl->it_offset);  in pnv_pci_ioda1_tce_invalidate()
1655 	end = __pa(((__be64 *)tbl->it_base) + index - tbl->it_offset +  in pnv_pci_ioda1_tce_invalidate()
1659 	if (tbl->it_busno) {  in pnv_pci_ioda1_tce_invalidate()
1663 		start |= tbl->it_busno;  in pnv_pci_ioda1_tce_invalidate()
1664 		end |= tbl->it_busno;  in pnv_pci_ioda1_tce_invalidate()
1665 	} else if (tbl->it_type & TCE_PCI_SWINV_PAIR) {  in pnv_pci_ioda1_tce_invalidate()
1692 static int pnv_ioda1_tce_build(struct iommu_table *tbl, long index,  in pnv_ioda1_tce_build()  argument
1697 	int ret = pnv_tce_build(tbl, index, npages, uaddr, direction,  in pnv_ioda1_tce_build()
1700 	if (!ret && (tbl->it_type & TCE_PCI_SWINV_CREATE))  in pnv_ioda1_tce_build()
1701 		pnv_pci_ioda1_tce_invalidate(tbl, index, npages, false);  in pnv_ioda1_tce_build()
1707 static int pnv_ioda1_tce_xchg(struct iommu_table *tbl, long index,  in pnv_ioda1_tce_xchg()  argument
1710 	long ret = pnv_tce_xchg(tbl, index, hpa, direction);  in pnv_ioda1_tce_xchg()
1712 	if (!ret && (tbl->it_type &  in pnv_ioda1_tce_xchg()
1714 		pnv_pci_ioda1_tce_invalidate(tbl, index, 1, false);  in pnv_ioda1_tce_xchg()
1720 static void pnv_ioda1_tce_free(struct iommu_table *tbl, long index,  in pnv_ioda1_tce_free()  argument
1723 	pnv_tce_free(tbl, index, npages);  in pnv_ioda1_tce_free()
1725 	if (tbl->it_type & TCE_PCI_SWINV_FREE)  in pnv_ioda1_tce_free()
1726 		pnv_pci_ioda1_tce_invalidate(tbl, index, npages, false);  in pnv_ioda1_tce_free()
1777 static void pnv_pci_ioda2_tce_invalidate(struct iommu_table *tbl,  in pnv_pci_ioda2_tce_invalidate()  argument
1782 	list_for_each_entry_rcu(tgl, &tbl->it_group_list, next) {  in pnv_pci_ioda2_tce_invalidate()
1790 			invalidate, tbl->it_page_shift,  in pnv_pci_ioda2_tce_invalidate()
1795 static int pnv_ioda2_tce_build(struct iommu_table *tbl, long index,  in pnv_ioda2_tce_build()  argument
1800 	int ret = pnv_tce_build(tbl, index, npages, uaddr, direction,  in pnv_ioda2_tce_build()
1803 	if (!ret && (tbl->it_type & TCE_PCI_SWINV_CREATE))  in pnv_ioda2_tce_build()
1804 		pnv_pci_ioda2_tce_invalidate(tbl, index, npages, false);  in pnv_ioda2_tce_build()
1810 static int pnv_ioda2_tce_xchg(struct iommu_table *tbl, long index,  in pnv_ioda2_tce_xchg()  argument
1813 	long ret = pnv_tce_xchg(tbl, index, hpa, direction);  in pnv_ioda2_tce_xchg()
1815 	if (!ret && (tbl->it_type &  in pnv_ioda2_tce_xchg()
1817 		pnv_pci_ioda2_tce_invalidate(tbl, index, 1, false);  in pnv_ioda2_tce_xchg()
1823 static void pnv_ioda2_tce_free(struct iommu_table *tbl, long index,  in pnv_ioda2_tce_free()  argument
1826 	pnv_tce_free(tbl, index, npages);  in pnv_ioda2_tce_free()
1828 	if (tbl->it_type & TCE_PCI_SWINV_FREE)  in pnv_ioda2_tce_free()
1829 		pnv_pci_ioda2_tce_invalidate(tbl, index, npages, false);  in pnv_ioda2_tce_free()
1832 static void pnv_ioda2_table_free(struct iommu_table *tbl)  in pnv_ioda2_table_free()  argument
1834 	pnv_pci_ioda2_table_free_pages(tbl);  in pnv_ioda2_table_free()
1835 	iommu_free_table(tbl, "pnv");  in pnv_ioda2_table_free()
1854 	struct iommu_table *tbl;  in pnv_pci_ioda_setup_dma_pe()  local
1867 	tbl = pnv_pci_table_alloc(phb->hose->node);  in pnv_pci_ioda_setup_dma_pe()
1870 	pnv_pci_link_table_and_group(phb->hose->node, 0, tbl, &pe->table_group);  in pnv_pci_ioda_setup_dma_pe()
1906 	pnv_pci_setup_iommu_table(tbl, addr, TCE32_TABLE_SIZE * segs,  in pnv_pci_ioda_setup_dma_pe()
1911 		tbl->it_type |= (TCE_PCI_SWINV_CREATE |  in pnv_pci_ioda_setup_dma_pe()
1915 	tbl->it_ops = &pnv_ioda1_iommu_ops;  in pnv_pci_ioda_setup_dma_pe()
1916 	pe->table_group.tce32_start = tbl->it_offset << tbl->it_page_shift;  in pnv_pci_ioda_setup_dma_pe()
1917 	pe->table_group.tce32_size = tbl->it_size << tbl->it_page_shift;  in pnv_pci_ioda_setup_dma_pe()
1918 	iommu_init_table(tbl, phb->hose->node);  in pnv_pci_ioda_setup_dma_pe()
1926 		set_iommu_table_base(&pe->pdev->dev, tbl);  in pnv_pci_ioda_setup_dma_pe()
1938 	if (tbl) {  in pnv_pci_ioda_setup_dma_pe()
1939 		pnv_pci_unlink_table_and_group(tbl, &pe->table_group);  in pnv_pci_ioda_setup_dma_pe()
1940 		iommu_free_table(tbl, "pnv");  in pnv_pci_ioda_setup_dma_pe()
1945 		int num, struct iommu_table *tbl)  in pnv_pci_ioda2_set_window()  argument
1951 	const unsigned long size = tbl->it_indirect_levels ?  in pnv_pci_ioda2_set_window()
1952 			tbl->it_level_size : tbl->it_size;  in pnv_pci_ioda2_set_window()
1953 	const __u64 start_addr = tbl->it_offset << tbl->it_page_shift;  in pnv_pci_ioda2_set_window()
1954 	const __u64 win_size = tbl->it_size << tbl->it_page_shift;  in pnv_pci_ioda2_set_window()
1958 			IOMMU_PAGE_SIZE(tbl));  in pnv_pci_ioda2_set_window()
1967 			tbl->it_indirect_levels + 1,  in pnv_pci_ioda2_set_window()
1968 			__pa(tbl->it_base),  in pnv_pci_ioda2_set_window()
1970 			IOMMU_PAGE_SIZE(tbl));  in pnv_pci_ioda2_set_window()
1977 			tbl, &pe->table_group);  in pnv_pci_ioda2_set_window()
2013 		struct iommu_table *tbl);
2024 	struct iommu_table *tbl;  in pnv_pci_ioda2_create_table()  local
2026 	tbl = pnv_pci_table_alloc(nid);  in pnv_pci_ioda2_create_table()
2027 	if (!tbl)  in pnv_pci_ioda2_create_table()
2032 			levels, tbl);  in pnv_pci_ioda2_create_table()
2034 		iommu_free_table(tbl, "pnv");  in pnv_pci_ioda2_create_table()
2038 	tbl->it_ops = &pnv_ioda2_iommu_ops;  in pnv_pci_ioda2_create_table()
2040 		tbl->it_type |= (TCE_PCI_SWINV_CREATE | TCE_PCI_SWINV_FREE);  in pnv_pci_ioda2_create_table()
2042 	*ptbl = tbl;  in pnv_pci_ioda2_create_table()
2049 	struct iommu_table *tbl = NULL;  in pnv_pci_ioda2_setup_default_config()  local
2069 			POWERNV_IOMMU_DEFAULT_LEVELS, &tbl);  in pnv_pci_ioda2_setup_default_config()
2076 	iommu_init_table(tbl, pe->phb->hose->node);  in pnv_pci_ioda2_setup_default_config()
2078 	rc = pnv_pci_ioda2_set_window(&pe->table_group, 0, tbl);  in pnv_pci_ioda2_setup_default_config()
2082 		pnv_ioda2_table_free(tbl);  in pnv_pci_ioda2_setup_default_config()
2091 		tbl->it_type |= (TCE_PCI_SWINV_CREATE | TCE_PCI_SWINV_FREE);  in pnv_pci_ioda2_setup_default_config()
2099 		set_iommu_table_base(&pe->pdev->dev, tbl);  in pnv_pci_ioda2_setup_default_config()
2168 	struct iommu_table *tbl = pe->table_group.tables[0];  in pnv_ioda2_take_ownership()  local
2172 	pnv_ioda2_table_free(tbl);  in pnv_ioda2_take_ownership()
2253 		struct iommu_table *tbl)  in pnv_pci_ioda2_table_alloc_pages()  argument
2293 	pnv_pci_setup_iommu_table(tbl, addr, tce_table_size, bus_offset,  in pnv_pci_ioda2_table_alloc_pages()
2295 	tbl->it_level_size = 1ULL << (level_shift - 3);  in pnv_pci_ioda2_table_alloc_pages()
2296 	tbl->it_indirect_levels = levels - 1;  in pnv_pci_ioda2_table_alloc_pages()
2297 	tbl->it_allocated_size = total_allocated;  in pnv_pci_ioda2_table_alloc_pages()
2329 static void pnv_pci_ioda2_table_free_pages(struct iommu_table *tbl)  in pnv_pci_ioda2_table_free_pages()  argument
2331 	const unsigned long size = tbl->it_indirect_levels ?  in pnv_pci_ioda2_table_free_pages()
2332 			tbl->it_level_size : tbl->it_size;  in pnv_pci_ioda2_table_free_pages()
2334 	if (!tbl->it_size)  in pnv_pci_ioda2_table_free_pages()
2337 	pnv_pci_ioda2_table_do_free_pages((__be64 *)tbl->it_base, size,  in pnv_pci_ioda2_table_free_pages()
2338 			tbl->it_indirect_levels);  in pnv_pci_ioda2_table_free_pages()