59 	struct iommu_table *tbl = NULL;  in iommu_pseries_alloc_group()  local
67 	tbl = kzalloc_node(sizeof(struct iommu_table), GFP_KERNEL, node);  in iommu_pseries_alloc_group()
68 	if (!tbl)  in iommu_pseries_alloc_group()
76 	INIT_LIST_HEAD_RCU(&tbl->it_group_list);  in iommu_pseries_alloc_group()
78 	list_add_rcu(&tgl->next, &tbl->it_group_list);  in iommu_pseries_alloc_group()
80 	table_group->tables[0] = tbl;  in iommu_pseries_alloc_group()
87 	kfree(tbl);  in iommu_pseries_alloc_group()
95 	struct iommu_table *tbl;  in iommu_pseries_free_group()  local
103 	tbl = table_group->tables[0];  in iommu_pseries_free_group()
105 	tgl = list_first_entry_or_null(&tbl->it_group_list,  in iommu_pseries_free_group()
118 	iommu_free_table(tbl, node_name);  in iommu_pseries_free_group()
123 static void tce_invalidate_pSeries_sw(struct iommu_table *tbl,  in tce_invalidate_pSeries_sw()  argument
126 	u64 __iomem *invalidate = (u64 __iomem *)tbl->it_index;  in tce_invalidate_pSeries_sw()
135 	if (tbl->it_busno) {  in tce_invalidate_pSeries_sw()
139 		start |= tbl->it_busno;  in tce_invalidate_pSeries_sw()
140 		end |= tbl->it_busno;  in tce_invalidate_pSeries_sw()
152 static int tce_build_pSeries(struct iommu_table *tbl, long index,  in tce_build_pSeries()  argument
166 	tces = tcep = ((__be64 *)tbl->it_base) + index;  in tce_build_pSeries()
177 	if (tbl->it_type & TCE_PCI_SWINV_CREATE)  in tce_build_pSeries()
178 		tce_invalidate_pSeries_sw(tbl, tces, tcep - 1);  in tce_build_pSeries()
183 static void tce_free_pSeries(struct iommu_table *tbl, long index, long npages)  in tce_free_pSeries()  argument
187 	tces = tcep = ((__be64 *)tbl->it_base) + index;  in tce_free_pSeries()
192 	if (tbl->it_type & TCE_PCI_SWINV_FREE)  in tce_free_pSeries()
193 		tce_invalidate_pSeries_sw(tbl, tces, tcep - 1);  in tce_free_pSeries()
196 static unsigned long tce_get_pseries(struct iommu_table *tbl, long index)  in tce_get_pseries()  argument
200 	tcep = ((__be64 *)tbl->it_base) + index;  in tce_get_pseries()
208 static int tce_build_pSeriesLP(struct iommu_table *tbl, long tcenum,  in tce_build_pSeriesLP()  argument
226 		rc = plpar_tce_put((u64)tbl->it_index, (u64)tcenum << 12, tce);  in tce_build_pSeriesLP()
230 			tce_free_pSeriesLP(tbl, tcenum_start,  in tce_build_pSeriesLP()
237 			printk("\tindex   = 0x%llx\n", (u64)tbl->it_index);  in tce_build_pSeriesLP()
251 static int tce_buildmulti_pSeriesLP(struct iommu_table *tbl, long tcenum,  in tce_buildmulti_pSeriesLP()  argument
266 		return tce_build_pSeriesLP(tbl, tcenum, npages, uaddr,  in tce_buildmulti_pSeriesLP()
282 			return tce_build_pSeriesLP(tbl, tcenum, npages, uaddr,  in tce_buildmulti_pSeriesLP()
306 		rc = plpar_tce_put_indirect((u64)tbl->it_index,  in tce_buildmulti_pSeriesLP()
319 		tce_freemulti_pSeriesLP(tbl, tcenum_start,  in tce_buildmulti_pSeriesLP()
326 		printk("\tindex   = 0x%llx\n", (u64)tbl->it_index);  in tce_buildmulti_pSeriesLP()
334 static void tce_free_pSeriesLP(struct iommu_table *tbl, long tcenum, long npages)  in tce_free_pSeriesLP()  argument
339 		rc = plpar_tce_put((u64)tbl->it_index, (u64)tcenum << 12, 0);  in tce_free_pSeriesLP()
343 			printk("\tindex   = 0x%llx\n", (u64)tbl->it_index);  in tce_free_pSeriesLP()
353 static void tce_freemulti_pSeriesLP(struct iommu_table *tbl, long tcenum, long npages)  in tce_freemulti_pSeriesLP()  argument
358 		return tce_free_pSeriesLP(tbl, tcenum, npages);  in tce_freemulti_pSeriesLP()
360 	rc = plpar_tce_stuff((u64)tbl->it_index, (u64)tcenum << 12, 0, npages);  in tce_freemulti_pSeriesLP()
365 		printk("\tindex   = 0x%llx\n", (u64)tbl->it_index);  in tce_freemulti_pSeriesLP()
371 static unsigned long tce_get_pSeriesLP(struct iommu_table *tbl, long tcenum)  in tce_get_pSeriesLP()  argument
376 	rc = plpar_tce_get((u64)tbl->it_index, (u64)tcenum << 12, &tce_ret);  in tce_get_pSeriesLP()
380 		printk("\tindex   = 0x%llx\n", (u64)tbl->it_index);  in tce_get_pSeriesLP()
537 				 struct iommu_table *tbl)  in iommu_table_setparms()  argument
553 	tbl->it_base = (unsigned long)__va(*basep);  in iommu_table_setparms()
556 		memset((void *)tbl->it_base, 0, *sizep);  in iommu_table_setparms()
558 	tbl->it_busno = phb->bus->number;  in iommu_table_setparms()
559 	tbl->it_page_shift = IOMMU_PAGE_SHIFT_4K;  in iommu_table_setparms()
562 	tbl->it_offset = phb->dma_window_base_cur >> tbl->it_page_shift;  in iommu_table_setparms()
573 	tbl->it_size = phb->dma_window_size >> tbl->it_page_shift;  in iommu_table_setparms()
575 	tbl->it_index = 0;  in iommu_table_setparms()
576 	tbl->it_blocksize = 16;  in iommu_table_setparms()
577 	tbl->it_type = TCE_PCI;  in iommu_table_setparms()
590 		tbl->it_index = (unsigned long) ioremap(sw_inval[0], 8);  in iommu_table_setparms()
591 		tbl->it_busno = sw_inval[1]; /* overload this with magic */  in iommu_table_setparms()
592 		tbl->it_type = TCE_PCI_SWINV_CREATE | TCE_PCI_SWINV_FREE;  in iommu_table_setparms()
603 				      struct iommu_table *tbl,  in iommu_table_setparms_lpar()  argument
608 	of_parse_dma_window(dn, dma_window, &tbl->it_index, &offset, &size);  in iommu_table_setparms_lpar()
610 	tbl->it_busno = phb->bus->number;  in iommu_table_setparms_lpar()
611 	tbl->it_page_shift = IOMMU_PAGE_SHIFT_4K;  in iommu_table_setparms_lpar()
612 	tbl->it_base   = 0;  in iommu_table_setparms_lpar()
613 	tbl->it_blocksize  = 16;  in iommu_table_setparms_lpar()
614 	tbl->it_type = TCE_PCI;  in iommu_table_setparms_lpar()
615 	tbl->it_offset = offset >> tbl->it_page_shift;  in iommu_table_setparms_lpar()
616 	tbl->it_size = size >> tbl->it_page_shift;  in iommu_table_setparms_lpar()
628 	struct iommu_table *tbl;  in pci_dma_bus_setup_pSeries()  local
690 	tbl = pci->table_group->tables[0];  in pci_dma_bus_setup_pSeries()
692 	iommu_table_setparms(pci->phb, dn, tbl);  in pci_dma_bus_setup_pSeries()
693 	tbl->it_ops = &iommu_table_pseries_ops;  in pci_dma_bus_setup_pSeries()
694 	iommu_init_table(tbl, pci->phb->node);  in pci_dma_bus_setup_pSeries()
713 	struct iommu_table *tbl;  in pci_dma_bus_setup_pSeriesLP()  local
742 		tbl = ppci->table_group->tables[0];  in pci_dma_bus_setup_pSeriesLP()
743 		iommu_table_setparms_lpar(ppci->phb, pdn, tbl, dma_window);  in pci_dma_bus_setup_pSeriesLP()
744 		tbl->it_ops = &iommu_table_lpar_multi_ops;  in pci_dma_bus_setup_pSeriesLP()
745 		iommu_init_table(tbl, ppci->phb->node);  in pci_dma_bus_setup_pSeriesLP()
756 	struct iommu_table *tbl;  in pci_dma_dev_setup_pSeries()  local
771 		tbl = PCI_DN(dn)->table_group->tables[0];  in pci_dma_dev_setup_pSeries()
772 		iommu_table_setparms(phb, dn, tbl);  in pci_dma_dev_setup_pSeries()
773 		tbl->it_ops = &iommu_table_pseries_ops;  in pci_dma_dev_setup_pSeries()
774 		iommu_init_table(tbl, phb->node);  in pci_dma_dev_setup_pSeries()
777 		set_iommu_table_base(&dev->dev, tbl);  in pci_dma_dev_setup_pSeries()
1163 	struct iommu_table *tbl;  in pci_dma_dev_setup_pSeriesLP()  local
1196 		tbl = pci->table_group->tables[0];  in pci_dma_dev_setup_pSeriesLP()
1197 		iommu_table_setparms_lpar(pci->phb, pdn, tbl, dma_window);  in pci_dma_dev_setup_pSeriesLP()
1198 		tbl->it_ops = &iommu_table_lpar_multi_ops;  in pci_dma_dev_setup_pSeriesLP()
1199 		iommu_init_table(tbl, pci->phb->node);  in pci_dma_dev_setup_pSeriesLP()