174 static void calgary_handle_quirks(struct iommu_table *tbl, struct pci_dev *dev);
175 static void calgary_tce_cache_blast(struct iommu_table *tbl);
176 static void calgary_dump_error_regs(struct iommu_table *tbl);
177 static void calioc2_handle_quirks(struct iommu_table *tbl, struct pci_dev *dev);
178 static void calioc2_tce_cache_blast(struct iommu_table *tbl);
179 static void calioc2_dump_error_regs(struct iommu_table *tbl);
180 static void calgary_init_bitmap_from_tce_table(struct iommu_table *tbl);
197 static inline int translation_enabled(struct iommu_table *tbl)  in translation_enabled()  argument
200 	return (tbl != NULL);  in translation_enabled()
203 static void iommu_range_reserve(struct iommu_table *tbl,  in iommu_range_reserve()  argument
213 	if (index >= tbl->it_size)  in iommu_range_reserve()
217 	if (end > tbl->it_size) /* don't go off the table */  in iommu_range_reserve()
218 		end = tbl->it_size;  in iommu_range_reserve()
220 	spin_lock_irqsave(&tbl->it_lock, flags);  in iommu_range_reserve()
222 	bitmap_set(tbl->it_map, index, npages);  in iommu_range_reserve()
224 	spin_unlock_irqrestore(&tbl->it_lock, flags);  in iommu_range_reserve()
228 				       struct iommu_table *tbl,  in iommu_range_alloc()  argument
240 	spin_lock_irqsave(&tbl->it_lock, flags);  in iommu_range_alloc()
242 	offset = iommu_area_alloc(tbl->it_map, tbl->it_size, tbl->it_hint,  in iommu_range_alloc()
245 		tbl->chip_ops->tce_cache_blast(tbl);  in iommu_range_alloc()
247 		offset = iommu_area_alloc(tbl->it_map, tbl->it_size, 0,  in iommu_range_alloc()
251 			spin_unlock_irqrestore(&tbl->it_lock, flags);  in iommu_range_alloc()
259 	tbl->it_hint = offset + npages;  in iommu_range_alloc()
260 	BUG_ON(tbl->it_hint > tbl->it_size);  in iommu_range_alloc()
262 	spin_unlock_irqrestore(&tbl->it_lock, flags);  in iommu_range_alloc()
267 static dma_addr_t iommu_alloc(struct device *dev, struct iommu_table *tbl,  in iommu_alloc()  argument
273 	entry = iommu_range_alloc(dev, tbl, npages);  in iommu_alloc()
277 			npages, tbl);  in iommu_alloc()
285 	tce_build(tbl, entry, npages, (unsigned long)vaddr & PAGE_MASK,  in iommu_alloc()
290 static void iommu_free(struct iommu_table *tbl, dma_addr_t dma_addr,  in iommu_free()  argument
307 	BUG_ON(entry + npages > tbl->it_size);  in iommu_free()
309 	tce_free(tbl, entry, npages);  in iommu_free()
311 	spin_lock_irqsave(&tbl->it_lock, flags);  in iommu_free()
313 	bitmap_clear(tbl->it_map, entry, npages);  in iommu_free()
315 	spin_unlock_irqrestore(&tbl->it_lock, flags);  in iommu_free()
322 	struct iommu_table *tbl;  in find_iommu_table()  local
329 		tbl = pci_iommu(pbus);  in find_iommu_table()
330 		if (tbl && tbl->it_busno == pbus->number)  in find_iommu_table()
332 		tbl = NULL;  in find_iommu_table()
336 	BUG_ON(tbl && (tbl->it_busno != pbus->number));  in find_iommu_table()
338 	return tbl;  in find_iommu_table()
345 	struct iommu_table *tbl = find_iommu_table(dev);  in calgary_unmap_sg()  local
349 	if (!translation_enabled(tbl))  in calgary_unmap_sg()
361 		iommu_free(tbl, dma, npages);  in calgary_unmap_sg()
369 	struct iommu_table *tbl = find_iommu_table(dev);  in calgary_map_sg()  local
382 		entry = iommu_range_alloc(dev, tbl, npages);  in calgary_map_sg()
392 		tce_build(tbl, entry, npages, vaddr & PAGE_MASK, dir);  in calgary_map_sg()
415 	struct iommu_table *tbl = find_iommu_table(dev);  in calgary_map_page()  local
420 	return iommu_alloc(dev, tbl, vaddr, npages, dir);  in calgary_map_page()
427 	struct iommu_table *tbl = find_iommu_table(dev);  in calgary_unmap_page()  local
431 	iommu_free(tbl, dma_addr, npages);  in calgary_unmap_page()
440 	struct iommu_table *tbl = find_iommu_table(dev);  in calgary_alloc_coherent()  local
455 	mapping = iommu_alloc(dev, tbl, ret, npages, DMA_BIDIRECTIONAL);  in calgary_alloc_coherent()
472 	struct iommu_table *tbl = find_iommu_table(dev);  in calgary_free_coherent()  local
477 	iommu_free(tbl, dma_handle, npages);  in calgary_free_coherent()
542 static void calgary_tce_cache_blast(struct iommu_table *tbl)  in calgary_tce_cache_blast()  argument
547 	void __iomem *bbar = tbl->bbar;  in calgary_tce_cache_blast()
551 	target = calgary_reg(bbar, phb_offset(tbl->it_busno) | PHB_AER_OFFSET);  in calgary_tce_cache_blast()
556 	target = calgary_reg(bbar, phb_offset(tbl->it_busno) | PHB_PLSSR_OFFSET);  in calgary_tce_cache_blast()
560 	target = calgary_reg(bbar, split_queue_offset(tbl->it_busno));  in calgary_tce_cache_blast()
569 	target = calgary_reg(bbar, tar_offset(tbl->it_busno));  in calgary_tce_cache_blast()
570 	writeq(tbl->tar_val, target);  in calgary_tce_cache_blast()
573 	target = calgary_reg(bbar, phb_offset(tbl->it_busno) | PHB_AER_OFFSET);  in calgary_tce_cache_blast()
578 static void calioc2_tce_cache_blast(struct iommu_table *tbl)  in calioc2_tce_cache_blast()  argument
580 	void __iomem *bbar = tbl->bbar;  in calioc2_tce_cache_blast()
586 	unsigned char bus = tbl->it_busno;  in calioc2_tce_cache_blast()
637 	writeq(tbl->tar_val, target);  in calioc2_tce_cache_blast()
672 	struct iommu_table *tbl = pci_iommu(dev->bus);  in calgary_reserve_peripheral_mem_1()  local
674 	void __iomem *bbar = tbl->bbar;  in calgary_reserve_peripheral_mem_1()
696 	struct iommu_table *tbl = pci_iommu(dev->bus);  in calgary_reserve_peripheral_mem_2()  local
698 	void __iomem *bbar = tbl->bbar;  in calgary_reserve_peripheral_mem_2()
732 	struct iommu_table *tbl = pci_iommu(dev->bus);  in calgary_reserve_regions()  local
735 	iommu_range_reserve(tbl, DMA_ERROR_CODE, EMERGENCY_PAGES);  in calgary_reserve_regions()
746 	iommu_range_reserve(tbl, start, npages);  in calgary_reserve_regions()
759 	struct iommu_table *tbl;  in calgary_setup_tar()  local
766 	tbl = pci_iommu(dev->bus);  in calgary_setup_tar()
767 	tbl->it_base = (unsigned long)bus_info[dev->bus->number].tce_space;  in calgary_setup_tar()
770 		calgary_init_bitmap_from_tce_table(tbl);  in calgary_setup_tar()
772 		tce_free(tbl, 0, tbl->it_size);  in calgary_setup_tar()
775 		tbl->chip_ops = &calgary_chip_ops;  in calgary_setup_tar()
777 		tbl->chip_ops = &calioc2_chip_ops;  in calgary_setup_tar()
789 	table_phys = (u64)__pa(tbl->it_base);  in calgary_setup_tar()
796 	tbl->tar_val = cpu_to_be64(val64);  in calgary_setup_tar()
798 	writeq(tbl->tar_val, target);  in calgary_setup_tar()
807 	struct iommu_table *tbl = pci_iommu(dev->bus);  in calgary_free_bus()  local
811 	target = calgary_reg(tbl->bbar, tar_offset(dev->bus->number));  in calgary_free_bus()
817 	bitmapsz = tbl->it_size / BITS_PER_BYTE;  in calgary_free_bus()
818 	free_pages((unsigned long)tbl->it_map, get_order(bitmapsz));  in calgary_free_bus()
819 	tbl->it_map = NULL;  in calgary_free_bus()
821 	kfree(tbl);  in calgary_free_bus()
829 static void calgary_dump_error_regs(struct iommu_table *tbl)  in calgary_dump_error_regs()  argument
831 	void __iomem *bbar = tbl->bbar;  in calgary_dump_error_regs()
835 	target = calgary_reg(bbar, phb_offset(tbl->it_busno) | PHB_CSR_OFFSET);  in calgary_dump_error_regs()
838 	target = calgary_reg(bbar, phb_offset(tbl->it_busno) | PHB_PLSSR_OFFSET);  in calgary_dump_error_regs()
843 		 tbl->it_busno, csr, plssr);  in calgary_dump_error_regs()
846 static void calioc2_dump_error_regs(struct iommu_table *tbl)  in calioc2_dump_error_regs()  argument
848 	void __iomem *bbar = tbl->bbar;  in calioc2_dump_error_regs()
851 	unsigned long phboff = phb_offset(tbl->it_busno);  in calioc2_dump_error_regs()
869 	pr_emerg("DMA error on CalIOC2 PHB 0x%x\n", tbl->it_busno);  in calioc2_dump_error_regs()
895 	struct iommu_table *tbl = pci_iommu(dev->bus);  in calgary_watchdog()  local
896 	void __iomem *bbar = tbl->bbar;  in calgary_watchdog()
900 	target = calgary_reg(bbar, phb_offset(tbl->it_busno) | PHB_CSR_OFFSET);  in calgary_watchdog()
905 		tbl->chip_ops->dump_error_regs(tbl);  in calgary_watchdog()
911 		target = calgary_reg(bbar, phb_offset(tbl->it_busno) |  in calgary_watchdog()
919 		mod_timer(&tbl->watchdog_timer, jiffies + 2 * HZ);  in calgary_watchdog()
955 static void __init calioc2_handle_quirks(struct iommu_table *tbl, struct pci_dev *dev)  in calioc2_handle_quirks()  argument
958 	void __iomem *bbar = tbl->bbar;  in calioc2_handle_quirks()
971 static void __init calgary_handle_quirks(struct iommu_table *tbl, struct pci_dev *dev)  in calgary_handle_quirks()  argument
980 		calgary_set_split_completion_timeout(tbl->bbar, busnum,  in calgary_handle_quirks()
990 	struct iommu_table *tbl;  in calgary_enable_translation()  local
993 	tbl = pci_iommu(dev->bus);  in calgary_enable_translation()
994 	bbar = tbl->bbar;  in calgary_enable_translation()
1010 	init_timer(&tbl->watchdog_timer);  in calgary_enable_translation()
1011 	tbl->watchdog_timer.function = &calgary_watchdog;  in calgary_enable_translation()
1012 	tbl->watchdog_timer.data = (unsigned long)dev;  in calgary_enable_translation()
1013 	mod_timer(&tbl->watchdog_timer, jiffies);  in calgary_enable_translation()
1022 	struct iommu_table *tbl;  in calgary_disable_translation()  local
1025 	tbl = pci_iommu(dev->bus);  in calgary_disable_translation()
1026 	bbar = tbl->bbar;  in calgary_disable_translation()
1037 	del_timer_sync(&tbl->watchdog_timer);  in calgary_disable_translation()
1055 	struct iommu_table *tbl;  in calgary_init_one()  local
1073 	tbl = pci_iommu(dev->bus);  in calgary_init_one()
1074 	tbl->chip_ops->handle_quirks(tbl, dev);  in calgary_init_one()
1175 		struct iommu_table *tbl;  in calgary_init()  local
1177 		tbl = find_iommu_table(&dev->dev);  in calgary_init()
1179 		if (translation_enabled(tbl))  in calgary_init()
1307 static void calgary_init_bitmap_from_tce_table(struct iommu_table *tbl)  in calgary_init_bitmap_from_tce_table()  argument
1311 	tp = ((u64 *)tbl->it_base);  in calgary_init_bitmap_from_tce_table()
1312 	for (index = 0 ; index < tbl->it_size; index++) {  in calgary_init_bitmap_from_tce_table()
1314 			set_bit(index, tbl->it_map);  in calgary_init_bitmap_from_tce_table()
1377 	void *tbl;  in detect_calgary()  local
1452 				tbl = alloc_tce_table();  in detect_calgary()
1453 				if (!tbl)  in detect_calgary()
1455 				info->tce_space = tbl;  in detect_calgary()
1545 	struct iommu_table *tbl;  in calgary_fixup_one_tce_space()  local
1549 	tbl = pci_iommu(dev->bus);  in calgary_fixup_one_tce_space()
1566 		iommu_range_reserve(tbl, r->start, npages);  in calgary_fixup_one_tce_space()