576 static __be64 *pnv_tce(struct iommu_table *tbl, long idx)  in pnv_tce()  argument
578 	__be64 *tmp = ((__be64 *)tbl->it_base);  in pnv_tce()
579 	int  level = tbl->it_indirect_levels;  in pnv_tce()
580 	const long shift = ilog2(tbl->it_level_size);  in pnv_tce()
581 	unsigned long mask = (tbl->it_level_size - 1) << (level * shift);  in pnv_tce()
596 int pnv_tce_build(struct iommu_table *tbl, long index, long npages,  in pnv_tce_build()  argument
601 	u64 rpn = __pa(uaddr) >> tbl->it_page_shift;  in pnv_tce_build()
609 			((rpn + i) << tbl->it_page_shift);  in pnv_tce_build()
610 		unsigned long idx = index - tbl->it_offset + i;  in pnv_tce_build()
612 		*(pnv_tce(tbl, idx)) = cpu_to_be64(newtce);  in pnv_tce_build()
619 int pnv_tce_xchg(struct iommu_table *tbl, long index,  in pnv_tce_xchg()  argument
624 	unsigned long idx = index - tbl->it_offset;  in pnv_tce_xchg()
626 	BUG_ON(*hpa & ~IOMMU_PAGE_MASK(tbl));  in pnv_tce_xchg()
631 	oldtce = xchg(pnv_tce(tbl, idx), cpu_to_be64(newtce));  in pnv_tce_xchg()
639 void pnv_tce_free(struct iommu_table *tbl, long index, long npages)  in pnv_tce_free()  argument
644 		unsigned long idx = index - tbl->it_offset + i;  in pnv_tce_free()
646 		*(pnv_tce(tbl, idx)) = cpu_to_be64(0);  in pnv_tce_free()
650 unsigned long pnv_tce_get(struct iommu_table *tbl, long index)  in pnv_tce_get()  argument
652 	return *(pnv_tce(tbl, index - tbl->it_offset));  in pnv_tce_get()
657 	struct iommu_table *tbl;  in pnv_pci_table_alloc()  local
659 	tbl = kzalloc_node(sizeof(struct iommu_table), GFP_KERNEL, nid);  in pnv_pci_table_alloc()
660 	INIT_LIST_HEAD_RCU(&tbl->it_group_list);  in pnv_pci_table_alloc()
662 	return tbl;  in pnv_pci_table_alloc()
666 		struct iommu_table *tbl,  in pnv_pci_link_table_and_group()  argument
671 	if (WARN_ON(!tbl || !table_group))  in pnv_pci_link_table_and_group()
680 	list_add_rcu(&tgl->next, &tbl->it_group_list);  in pnv_pci_link_table_and_group()
682 	table_group->tables[num] = tbl;  in pnv_pci_link_table_and_group()
695 void pnv_pci_unlink_table_and_group(struct iommu_table *tbl,  in pnv_pci_unlink_table_and_group()  argument
702 	if (!tbl || !table_group)  in pnv_pci_unlink_table_and_group()
707 	list_for_each_entry_rcu(tgl, &tbl->it_group_list, next) {  in pnv_pci_unlink_table_and_group()
721 		if (table_group->tables[i] == tbl) {  in pnv_pci_unlink_table_and_group()
730 void pnv_pci_setup_iommu_table(struct iommu_table *tbl,  in pnv_pci_setup_iommu_table()  argument
734 	tbl->it_blocksize = 16;  in pnv_pci_setup_iommu_table()
735 	tbl->it_base = (unsigned long)tce_mem;  in pnv_pci_setup_iommu_table()
736 	tbl->it_page_shift = page_shift;  in pnv_pci_setup_iommu_table()
737 	tbl->it_offset = dma_offset >> tbl->it_page_shift;  in pnv_pci_setup_iommu_table()
738 	tbl->it_index = 0;  in pnv_pci_setup_iommu_table()
739 	tbl->it_size = tce_size >> 3;  in pnv_pci_setup_iommu_table()
740 	tbl->it_busno = 0;  in pnv_pci_setup_iommu_table()
741 	tbl->it_type = TCE_PCI;  in pnv_pci_setup_iommu_table()