89 	struct omap_iommu *obj = dev_to_omap_iommu(dev);  in omap_iommu_save_ctx()  local
90 	u32 *p = obj->ctx;  in omap_iommu_save_ctx()
94 		p[i] = iommu_read_reg(obj, i * sizeof(u32));  in omap_iommu_save_ctx()
95 		dev_dbg(obj->dev, "%s\t[%02d] %08x\n", __func__, i, p[i]);  in omap_iommu_save_ctx()
106 	struct omap_iommu *obj = dev_to_omap_iommu(dev);  in omap_iommu_restore_ctx()  local
107 	u32 *p = obj->ctx;  in omap_iommu_restore_ctx()
111 		iommu_write_reg(obj, p[i], i * sizeof(u32));  in omap_iommu_restore_ctx()
112 		dev_dbg(obj->dev, "%s\t[%02d] %08x\n", __func__, i, p[i]);  in omap_iommu_restore_ctx()
117 static void dra7_cfg_dspsys_mmu(struct omap_iommu *obj, bool enable)  in dra7_cfg_dspsys_mmu()  argument
121 	if (!obj->syscfg)  in dra7_cfg_dspsys_mmu()
124 	mask = (1 << (obj->id * DSP_SYS_MMU_CONFIG_EN_SHIFT));  in dra7_cfg_dspsys_mmu()
126 	regmap_update_bits(obj->syscfg, DSP_SYS_MMU_CONFIG, mask, val);  in dra7_cfg_dspsys_mmu()
129 static void __iommu_set_twl(struct omap_iommu *obj, bool on)  in __iommu_set_twl()  argument
131 	u32 l = iommu_read_reg(obj, MMU_CNTL);  in __iommu_set_twl()
134 		iommu_write_reg(obj, MMU_IRQ_TWL_MASK, MMU_IRQENABLE);  in __iommu_set_twl()
136 		iommu_write_reg(obj, MMU_IRQ_TLB_MISS_MASK, MMU_IRQENABLE);  in __iommu_set_twl()
144 	iommu_write_reg(obj, l, MMU_CNTL);  in __iommu_set_twl()
147 static int omap2_iommu_enable(struct omap_iommu *obj)  in omap2_iommu_enable()  argument
151 	if (!obj->iopgd || !IS_ALIGNED((u32)obj->iopgd,  SZ_16K))  in omap2_iommu_enable()
154 	pa = virt_to_phys(obj->iopgd);  in omap2_iommu_enable()
158 	l = iommu_read_reg(obj, MMU_REVISION);  in omap2_iommu_enable()
159 	dev_info(obj->dev, "%s: version %d.%d\n", obj->name,  in omap2_iommu_enable()
162 	iommu_write_reg(obj, pa, MMU_TTB);  in omap2_iommu_enable()
164 	dra7_cfg_dspsys_mmu(obj, true);  in omap2_iommu_enable()
166 	if (obj->has_bus_err_back)  in omap2_iommu_enable()
167 		iommu_write_reg(obj, MMU_GP_REG_BUS_ERR_BACK_EN, MMU_GP_REG);  in omap2_iommu_enable()
169 	__iommu_set_twl(obj, true);  in omap2_iommu_enable()
174 static void omap2_iommu_disable(struct omap_iommu *obj)  in omap2_iommu_disable()  argument
176 	u32 l = iommu_read_reg(obj, MMU_CNTL);  in omap2_iommu_disable()
179 	iommu_write_reg(obj, l, MMU_CNTL);  in omap2_iommu_disable()
180 	dra7_cfg_dspsys_mmu(obj, false);  in omap2_iommu_disable()
182 	dev_dbg(obj->dev, "%s is shutting down\n", obj->name);  in omap2_iommu_disable()
185 static int iommu_enable(struct omap_iommu *obj)  in iommu_enable()  argument
188 	struct platform_device *pdev = to_platform_device(obj->dev);  in iommu_enable()
194 			dev_err(obj->dev, "deassert_reset failed: %d\n", err);  in iommu_enable()
199 	pm_runtime_get_sync(obj->dev);  in iommu_enable()
201 	err = omap2_iommu_enable(obj);  in iommu_enable()
206 static void iommu_disable(struct omap_iommu *obj)  in iommu_disable()  argument
208 	struct platform_device *pdev = to_platform_device(obj->dev);  in iommu_disable()
211 	omap2_iommu_disable(obj);  in iommu_disable()
213 	pm_runtime_put_sync(obj->dev);  in iommu_disable()
242 static u32 iommu_report_fault(struct omap_iommu *obj, u32 *da)  in iommu_report_fault()  argument
246 	status = iommu_read_reg(obj, MMU_IRQSTATUS);  in iommu_report_fault()
253 	fault_addr = iommu_read_reg(obj, MMU_FAULT_AD);  in iommu_report_fault()
256 	iommu_write_reg(obj, status, MMU_IRQSTATUS);  in iommu_report_fault()
261 void iotlb_lock_get(struct omap_iommu *obj, struct iotlb_lock *l)  in iotlb_lock_get()  argument
265 	val = iommu_read_reg(obj, MMU_LOCK);  in iotlb_lock_get()
271 void iotlb_lock_set(struct omap_iommu *obj, struct iotlb_lock *l)  in iotlb_lock_set()  argument
278 	iommu_write_reg(obj, val, MMU_LOCK);  in iotlb_lock_set()
281 static void iotlb_read_cr(struct omap_iommu *obj, struct cr_regs *cr)  in iotlb_read_cr()  argument
283 	cr->cam = iommu_read_reg(obj, MMU_READ_CAM);  in iotlb_read_cr()
284 	cr->ram = iommu_read_reg(obj, MMU_READ_RAM);  in iotlb_read_cr()
287 static void iotlb_load_cr(struct omap_iommu *obj, struct cr_regs *cr)  in iotlb_load_cr()  argument
289 	iommu_write_reg(obj, cr->cam | MMU_CAM_V, MMU_CAM);  in iotlb_load_cr()
290 	iommu_write_reg(obj, cr->ram, MMU_RAM);  in iotlb_load_cr()
292 	iommu_write_reg(obj, 1, MMU_FLUSH_ENTRY);  in iotlb_load_cr()
293 	iommu_write_reg(obj, 1, MMU_LD_TLB);  in iotlb_load_cr()
297 struct cr_regs __iotlb_read_cr(struct omap_iommu *obj, int n)  in __iotlb_read_cr()  argument
302 	iotlb_lock_get(obj, &l);  in __iotlb_read_cr()
304 	iotlb_lock_set(obj, &l);  in __iotlb_read_cr()
305 	iotlb_read_cr(obj, &cr);  in __iotlb_read_cr()
311 static struct cr_regs *iotlb_alloc_cr(struct omap_iommu *obj,  in iotlb_alloc_cr()  argument
320 		dev_err(obj->dev, "%s:\twrong alignment: %08x\n", __func__,  in iotlb_alloc_cr()
340 static int load_iotlb_entry(struct omap_iommu *obj, struct iotlb_entry *e)  in load_iotlb_entry()  argument
346 	if (!obj || !obj->nr_tlb_entries || !e)  in load_iotlb_entry()
349 	pm_runtime_get_sync(obj->dev);  in load_iotlb_entry()
351 	iotlb_lock_get(obj, &l);  in load_iotlb_entry()
352 	if (l.base == obj->nr_tlb_entries) {  in load_iotlb_entry()
353 		dev_warn(obj->dev, "%s: preserve entries full\n", __func__);  in load_iotlb_entry()
361 		for_each_iotlb_cr(obj, obj->nr_tlb_entries, i, tmp)  in load_iotlb_entry()
365 		if (i == obj->nr_tlb_entries) {  in load_iotlb_entry()
366 			dev_dbg(obj->dev, "%s: full: no entry\n", __func__);  in load_iotlb_entry()
371 		iotlb_lock_get(obj, &l);  in load_iotlb_entry()
374 		iotlb_lock_set(obj, &l);  in load_iotlb_entry()
377 	cr = iotlb_alloc_cr(obj, e);  in load_iotlb_entry()
379 		pm_runtime_put_sync(obj->dev);  in load_iotlb_entry()
383 	iotlb_load_cr(obj, cr);  in load_iotlb_entry()
389 	if (++l.vict == obj->nr_tlb_entries)  in load_iotlb_entry()
391 	iotlb_lock_set(obj, &l);  in load_iotlb_entry()
393 	pm_runtime_put_sync(obj->dev);  in load_iotlb_entry()
399 static int load_iotlb_entry(struct omap_iommu *obj, struct iotlb_entry *e)  in load_iotlb_entry()  argument
406 static int prefetch_iotlb_entry(struct omap_iommu *obj, struct iotlb_entry *e)  in prefetch_iotlb_entry()  argument
408 	return load_iotlb_entry(obj, e);  in prefetch_iotlb_entry()
418 static void flush_iotlb_page(struct omap_iommu *obj, u32 da)  in flush_iotlb_page()  argument
423 	pm_runtime_get_sync(obj->dev);  in flush_iotlb_page()
425 	for_each_iotlb_cr(obj, obj->nr_tlb_entries, i, cr) {  in flush_iotlb_page()
436 			dev_dbg(obj->dev, "%s: %08x<=%08x(%x)\n",  in flush_iotlb_page()
438 			iotlb_load_cr(obj, &cr);  in flush_iotlb_page()
439 			iommu_write_reg(obj, 1, MMU_FLUSH_ENTRY);  in flush_iotlb_page()
443 	pm_runtime_put_sync(obj->dev);  in flush_iotlb_page()
445 	if (i == obj->nr_tlb_entries)  in flush_iotlb_page()
446 		dev_dbg(obj->dev, "%s: no page for %08x\n", __func__, da);  in flush_iotlb_page()
453 static void flush_iotlb_all(struct omap_iommu *obj)  in flush_iotlb_all()  argument
457 	pm_runtime_get_sync(obj->dev);  in flush_iotlb_all()
461 	iotlb_lock_set(obj, &l);  in flush_iotlb_all()
463 	iommu_write_reg(obj, 1, MMU_GFLUSH);  in flush_iotlb_all()
465 	pm_runtime_put_sync(obj->dev);  in flush_iotlb_all()
498 static u32 *iopte_alloc(struct omap_iommu *obj, u32 *iopgd, u32 da)  in iopte_alloc()  argument
509 	spin_unlock(&obj->page_table_lock);  in iopte_alloc()
511 	spin_lock(&obj->page_table_lock);  in iopte_alloc()
520 		dev_vdbg(obj->dev, "%s: a new pte:%p\n", __func__, iopte);  in iopte_alloc()
529 	dev_vdbg(obj->dev,  in iopte_alloc()
536 static int iopgd_alloc_section(struct omap_iommu *obj, u32 da, u32 pa, u32 prot)  in iopgd_alloc_section()  argument
538 	u32 *iopgd = iopgd_offset(obj, da);  in iopgd_alloc_section()
541 		dev_err(obj->dev, "%s: %08x:%08x should aligned on %08lx\n",  in iopgd_alloc_section()
551 static int iopgd_alloc_super(struct omap_iommu *obj, u32 da, u32 pa, u32 prot)  in iopgd_alloc_super()  argument
553 	u32 *iopgd = iopgd_offset(obj, da);  in iopgd_alloc_super()
557 		dev_err(obj->dev, "%s: %08x:%08x should aligned on %08lx\n",  in iopgd_alloc_super()
568 static int iopte_alloc_page(struct omap_iommu *obj, u32 da, u32 pa, u32 prot)  in iopte_alloc_page()  argument
570 	u32 *iopgd = iopgd_offset(obj, da);  in iopte_alloc_page()
571 	u32 *iopte = iopte_alloc(obj, iopgd, da);  in iopte_alloc_page()
579 	dev_vdbg(obj->dev, "%s: da:%08x pa:%08x pte:%p *pte:%08x\n",  in iopte_alloc_page()
585 static int iopte_alloc_large(struct omap_iommu *obj, u32 da, u32 pa, u32 prot)  in iopte_alloc_large()  argument
587 	u32 *iopgd = iopgd_offset(obj, da);  in iopte_alloc_large()
588 	u32 *iopte = iopte_alloc(obj, iopgd, da);  in iopte_alloc_large()
592 		dev_err(obj->dev, "%s: %08x:%08x should aligned on %08lx\n",  in iopte_alloc_large()
607 iopgtable_store_entry_core(struct omap_iommu *obj, struct iotlb_entry *e)  in iopgtable_store_entry_core()  argument
613 	if (!obj || !e)  in iopgtable_store_entry_core()
637 	spin_lock(&obj->page_table_lock);  in iopgtable_store_entry_core()
638 	err = fn(obj, e->da, e->pa, prot);  in iopgtable_store_entry_core()
639 	spin_unlock(&obj->page_table_lock);  in iopgtable_store_entry_core()
650 omap_iopgtable_store_entry(struct omap_iommu *obj, struct iotlb_entry *e)  in omap_iopgtable_store_entry()  argument
654 	flush_iotlb_page(obj, e->da);  in omap_iopgtable_store_entry()
655 	err = iopgtable_store_entry_core(obj, e);  in omap_iopgtable_store_entry()
657 		prefetch_iotlb_entry(obj, e);  in omap_iopgtable_store_entry()
669 iopgtable_lookup_entry(struct omap_iommu *obj, u32 da, u32 **ppgd, u32 **ppte)  in iopgtable_lookup_entry()  argument
673 	iopgd = iopgd_offset(obj, da);  in iopgtable_lookup_entry()
684 static size_t iopgtable_clear_entry_core(struct omap_iommu *obj, u32 da)  in iopgtable_clear_entry_core()  argument
687 	u32 *iopgd = iopgd_offset(obj, da);  in iopgtable_clear_entry_core()
722 			iopgd = iopgd_offset(obj, (da & IOSUPER_MASK));  in iopgtable_clear_entry_core()
737 static size_t iopgtable_clear_entry(struct omap_iommu *obj, u32 da)  in iopgtable_clear_entry()  argument
741 	spin_lock(&obj->page_table_lock);  in iopgtable_clear_entry()
743 	bytes = iopgtable_clear_entry_core(obj, da);  in iopgtable_clear_entry()
744 	flush_iotlb_page(obj, da);  in iopgtable_clear_entry()
746 	spin_unlock(&obj->page_table_lock);  in iopgtable_clear_entry()
751 static void iopgtable_clear_entry_all(struct omap_iommu *obj)  in iopgtable_clear_entry_all()  argument
755 	spin_lock(&obj->page_table_lock);  in iopgtable_clear_entry_all()
762 		iopgd = iopgd_offset(obj, da);  in iopgtable_clear_entry_all()
774 	flush_iotlb_all(obj);  in iopgtable_clear_entry_all()
776 	spin_unlock(&obj->page_table_lock);  in iopgtable_clear_entry_all()
786 	struct omap_iommu *obj = data;  in iommu_fault_handler()  local
787 	struct iommu_domain *domain = obj->domain;  in iommu_fault_handler()
793 	errs = iommu_report_fault(obj, &da);  in iommu_fault_handler()
798 	if (!report_iommu_fault(domain, obj->dev, da, 0))  in iommu_fault_handler()
801 	iommu_disable(obj);  in iommu_fault_handler()
803 	iopgd = iopgd_offset(obj, da);  in iommu_fault_handler()
806 		dev_err(obj->dev, "%s: errs:0x%08x da:0x%08x pgd:0x%p *pgd:px%08x\n",  in iommu_fault_handler()
807 			obj->name, errs, da, iopgd, *iopgd);  in iommu_fault_handler()
813 	dev_err(obj->dev, "%s: errs:0x%08x da:0x%08x pgd:0x%p *pgd:0x%08x pte:0x%p *pte:0x%08x\n",  in iommu_fault_handler()
814 		obj->name, errs, da, iopgd, *iopgd, iopte, *iopte);  in iommu_fault_handler()
821 	struct omap_iommu *obj = to_iommu(dev);  in device_match_by_alias()  local
824 	pr_debug("%s: %s %s\n", __func__, obj->name, name);  in device_match_by_alias()
826 	return strcmp(obj->name, name) == 0;  in device_match_by_alias()
838 	struct omap_iommu *obj;  in omap_iommu_attach()  local
845 	obj = to_iommu(dev);  in omap_iommu_attach()
847 	spin_lock(&obj->iommu_lock);  in omap_iommu_attach()
849 	obj->iopgd = iopgd;  in omap_iommu_attach()
850 	err = iommu_enable(obj);  in omap_iommu_attach()
853 	flush_iotlb_all(obj);  in omap_iommu_attach()
855 	spin_unlock(&obj->iommu_lock);  in omap_iommu_attach()
857 	dev_dbg(obj->dev, "%s: %s\n", __func__, obj->name);  in omap_iommu_attach()
858 	return obj;  in omap_iommu_attach()
861 	spin_unlock(&obj->iommu_lock);  in omap_iommu_attach()
869 static void omap_iommu_detach(struct omap_iommu *obj)  in omap_iommu_detach()  argument
871 	if (!obj || IS_ERR(obj))  in omap_iommu_detach()
874 	spin_lock(&obj->iommu_lock);  in omap_iommu_detach()
876 	iommu_disable(obj);  in omap_iommu_detach()
877 	obj->iopgd = NULL;  in omap_iommu_detach()
879 	spin_unlock(&obj->iommu_lock);  in omap_iommu_detach()
881 	dev_dbg(obj->dev, "%s: %s\n", __func__, obj->name);  in omap_iommu_detach()
885 					      struct omap_iommu *obj)  in omap_iommu_dra7_get_dsp_system_cfg()  argument
898 	obj->syscfg =  in omap_iommu_dra7_get_dsp_system_cfg()
900 	if (IS_ERR(obj->syscfg)) {  in omap_iommu_dra7_get_dsp_system_cfg()
902 		ret = PTR_ERR(obj->syscfg);  in omap_iommu_dra7_get_dsp_system_cfg()
907 				       &obj->id)) {  in omap_iommu_dra7_get_dsp_system_cfg()
912 	if (obj->id != 0 && obj->id != 1) {  in omap_iommu_dra7_get_dsp_system_cfg()
927 	struct omap_iommu *obj;  in omap_iommu_probe()  local
932 	obj = devm_kzalloc(&pdev->dev, sizeof(*obj) + MMU_REG_SIZE, GFP_KERNEL);  in omap_iommu_probe()
933 	if (!obj)  in omap_iommu_probe()
937 		obj->name = dev_name(&pdev->dev);  in omap_iommu_probe()
938 		obj->nr_tlb_entries = 32;  in omap_iommu_probe()
940 					   &obj->nr_tlb_entries);  in omap_iommu_probe()
943 		if (obj->nr_tlb_entries != 32 && obj->nr_tlb_entries != 8)  in omap_iommu_probe()
946 			obj->has_bus_err_back = MMU_GP_REG_BUS_ERR_BACK_EN;  in omap_iommu_probe()
948 		obj->nr_tlb_entries = pdata->nr_tlb_entries;  in omap_iommu_probe()
949 		obj->name = pdata->name;  in omap_iommu_probe()
952 	obj->dev = &pdev->dev;  in omap_iommu_probe()
953 	obj->ctx = (void *)obj + sizeof(*obj);  in omap_iommu_probe()
955 	spin_lock_init(&obj->iommu_lock);  in omap_iommu_probe()
956 	spin_lock_init(&obj->page_table_lock);  in omap_iommu_probe()
959 	obj->regbase = devm_ioremap_resource(obj->dev, res);  in omap_iommu_probe()
960 	if (IS_ERR(obj->regbase))  in omap_iommu_probe()
961 		return PTR_ERR(obj->regbase);  in omap_iommu_probe()
963 	err = omap_iommu_dra7_get_dsp_system_cfg(pdev, obj);  in omap_iommu_probe()
971 	err = devm_request_irq(obj->dev, irq, iommu_fault_handler, IRQF_SHARED,  in omap_iommu_probe()
972 			       dev_name(obj->dev), obj);  in omap_iommu_probe()
975 	platform_set_drvdata(pdev, obj);  in omap_iommu_probe()
977 	pm_runtime_irq_safe(obj->dev);  in omap_iommu_probe()
978 	pm_runtime_enable(obj->dev);  in omap_iommu_probe()
980 	omap_iommu_debugfs_add(obj);  in omap_iommu_probe()
982 	dev_info(&pdev->dev, "%s registered\n", obj->name);  in omap_iommu_probe()
988 	struct omap_iommu *obj = platform_get_drvdata(pdev);  in omap_iommu_remove()  local
990 	iopgtable_clear_entry_all(obj);  in omap_iommu_remove()
991 	omap_iommu_debugfs_remove(obj);  in omap_iommu_remove()
993 	pm_runtime_disable(obj->dev);  in omap_iommu_remove()
995 	dev_info(&pdev->dev, "%s removed\n", obj->name);  in omap_iommu_remove()