41 #define for_each_iotlb_cr(obj, n, __i, cr)				\  argument
43 	     (__i < (n)) && (cr = __iotlb_read_cr((obj), __i), true);	\
98 	struct omap_iommu *obj = dev_to_omap_iommu(dev);  in omap_iommu_save_ctx()  local
99 	u32 *p = obj->ctx;  in omap_iommu_save_ctx()
103 		p[i] = iommu_read_reg(obj, i * sizeof(u32));  in omap_iommu_save_ctx()
104 		dev_dbg(obj->dev, "%s\t[%02d] %08x\n", __func__, i, p[i]);  in omap_iommu_save_ctx()
115 	struct omap_iommu *obj = dev_to_omap_iommu(dev);  in omap_iommu_restore_ctx()  local
116 	u32 *p = obj->ctx;  in omap_iommu_restore_ctx()
120 		iommu_write_reg(obj, p[i], i * sizeof(u32));  in omap_iommu_restore_ctx()
121 		dev_dbg(obj->dev, "%s\t[%02d] %08x\n", __func__, i, p[i]);  in omap_iommu_restore_ctx()
126 static void __iommu_set_twl(struct omap_iommu *obj, bool on)  in __iommu_set_twl()  argument
128 	u32 l = iommu_read_reg(obj, MMU_CNTL);  in __iommu_set_twl()
131 		iommu_write_reg(obj, MMU_IRQ_TWL_MASK, MMU_IRQENABLE);  in __iommu_set_twl()
133 		iommu_write_reg(obj, MMU_IRQ_TLB_MISS_MASK, MMU_IRQENABLE);  in __iommu_set_twl()
141 	iommu_write_reg(obj, l, MMU_CNTL);  in __iommu_set_twl()
144 static int omap2_iommu_enable(struct omap_iommu *obj)  in omap2_iommu_enable()  argument
148 	if (!obj->iopgd || !IS_ALIGNED((u32)obj->iopgd,  SZ_16K))  in omap2_iommu_enable()
151 	pa = virt_to_phys(obj->iopgd);  in omap2_iommu_enable()
155 	l = iommu_read_reg(obj, MMU_REVISION);  in omap2_iommu_enable()
156 	dev_info(obj->dev, "%s: version %d.%d\n", obj->name,  in omap2_iommu_enable()
159 	iommu_write_reg(obj, pa, MMU_TTB);  in omap2_iommu_enable()
161 	if (obj->has_bus_err_back)  in omap2_iommu_enable()
162 		iommu_write_reg(obj, MMU_GP_REG_BUS_ERR_BACK_EN, MMU_GP_REG);  in omap2_iommu_enable()
164 	__iommu_set_twl(obj, true);  in omap2_iommu_enable()
169 static void omap2_iommu_disable(struct omap_iommu *obj)  in omap2_iommu_disable()  argument
171 	u32 l = iommu_read_reg(obj, MMU_CNTL);  in omap2_iommu_disable()
174 	iommu_write_reg(obj, l, MMU_CNTL);  in omap2_iommu_disable()
176 	dev_dbg(obj->dev, "%s is shutting down\n", obj->name);  in omap2_iommu_disable()
179 static int iommu_enable(struct omap_iommu *obj)  in iommu_enable()  argument
182 	struct platform_device *pdev = to_platform_device(obj->dev);  in iommu_enable()
188 			dev_err(obj->dev, "deassert_reset failed: %d\n", err);  in iommu_enable()
193 	pm_runtime_get_sync(obj->dev);  in iommu_enable()
195 	err = omap2_iommu_enable(obj);  in iommu_enable()
200 static void iommu_disable(struct omap_iommu *obj)  in iommu_disable()  argument
202 	struct platform_device *pdev = to_platform_device(obj->dev);  in iommu_disable()
205 	omap2_iommu_disable(obj);  in iommu_disable()
207 	pm_runtime_put_sync(obj->dev);  in iommu_disable()
244 static u32 iommu_report_fault(struct omap_iommu *obj, u32 *da)  in iommu_report_fault()  argument
248 	status = iommu_read_reg(obj, MMU_IRQSTATUS);  in iommu_report_fault()
255 	fault_addr = iommu_read_reg(obj, MMU_FAULT_AD);  in iommu_report_fault()
258 	iommu_write_reg(obj, status, MMU_IRQSTATUS);  in iommu_report_fault()
263 static void iotlb_lock_get(struct omap_iommu *obj, struct iotlb_lock *l)  in iotlb_lock_get()  argument
267 	val = iommu_read_reg(obj, MMU_LOCK);  in iotlb_lock_get()
274 static void iotlb_lock_set(struct omap_iommu *obj, struct iotlb_lock *l)  in iotlb_lock_set()  argument
281 	iommu_write_reg(obj, val, MMU_LOCK);  in iotlb_lock_set()
284 static void iotlb_read_cr(struct omap_iommu *obj, struct cr_regs *cr)  in iotlb_read_cr()  argument
286 	cr->cam = iommu_read_reg(obj, MMU_READ_CAM);  in iotlb_read_cr()
287 	cr->ram = iommu_read_reg(obj, MMU_READ_RAM);  in iotlb_read_cr()
290 static void iotlb_load_cr(struct omap_iommu *obj, struct cr_regs *cr)  in iotlb_load_cr()  argument
292 	iommu_write_reg(obj, cr->cam | MMU_CAM_V, MMU_CAM);  in iotlb_load_cr()
293 	iommu_write_reg(obj, cr->ram, MMU_RAM);  in iotlb_load_cr()
295 	iommu_write_reg(obj, 1, MMU_FLUSH_ENTRY);  in iotlb_load_cr()
296 	iommu_write_reg(obj, 1, MMU_LD_TLB);  in iotlb_load_cr()
300 static struct cr_regs __iotlb_read_cr(struct omap_iommu *obj, int n)  in __iotlb_read_cr()  argument
305 	iotlb_lock_get(obj, &l);  in __iotlb_read_cr()
307 	iotlb_lock_set(obj, &l);  in __iotlb_read_cr()
308 	iotlb_read_cr(obj, &cr);  in __iotlb_read_cr()
314 static struct cr_regs *iotlb_alloc_cr(struct omap_iommu *obj,  in iotlb_alloc_cr()  argument
323 		dev_err(obj->dev, "%s:\twrong alignment: %08x\n", __func__,  in iotlb_alloc_cr()
343 static int load_iotlb_entry(struct omap_iommu *obj, struct iotlb_entry *e)  in load_iotlb_entry()  argument
349 	if (!obj || !obj->nr_tlb_entries || !e)  in load_iotlb_entry()
352 	pm_runtime_get_sync(obj->dev);  in load_iotlb_entry()
354 	iotlb_lock_get(obj, &l);  in load_iotlb_entry()
355 	if (l.base == obj->nr_tlb_entries) {  in load_iotlb_entry()
356 		dev_warn(obj->dev, "%s: preserve entries full\n", __func__);  in load_iotlb_entry()
364 		for_each_iotlb_cr(obj, obj->nr_tlb_entries, i, tmp)  in load_iotlb_entry()
368 		if (i == obj->nr_tlb_entries) {  in load_iotlb_entry()
369 			dev_dbg(obj->dev, "%s: full: no entry\n", __func__);  in load_iotlb_entry()
374 		iotlb_lock_get(obj, &l);  in load_iotlb_entry()
377 		iotlb_lock_set(obj, &l);  in load_iotlb_entry()
380 	cr = iotlb_alloc_cr(obj, e);  in load_iotlb_entry()
382 		pm_runtime_put_sync(obj->dev);  in load_iotlb_entry()
386 	iotlb_load_cr(obj, cr);  in load_iotlb_entry()
392 	if (++l.vict == obj->nr_tlb_entries)  in load_iotlb_entry()
394 	iotlb_lock_set(obj, &l);  in load_iotlb_entry()
396 	pm_runtime_put_sync(obj->dev);  in load_iotlb_entry()
402 static int load_iotlb_entry(struct omap_iommu *obj, struct iotlb_entry *e)  in load_iotlb_entry()  argument
409 static int prefetch_iotlb_entry(struct omap_iommu *obj, struct iotlb_entry *e)  in prefetch_iotlb_entry()  argument
411 	return load_iotlb_entry(obj, e);  in prefetch_iotlb_entry()
421 static void flush_iotlb_page(struct omap_iommu *obj, u32 da)  in flush_iotlb_page()  argument
426 	pm_runtime_get_sync(obj->dev);  in flush_iotlb_page()
428 	for_each_iotlb_cr(obj, obj->nr_tlb_entries, i, cr) {  in flush_iotlb_page()
439 			dev_dbg(obj->dev, "%s: %08x<=%08x(%x)\n",  in flush_iotlb_page()
441 			iotlb_load_cr(obj, &cr);  in flush_iotlb_page()
442 			iommu_write_reg(obj, 1, MMU_FLUSH_ENTRY);  in flush_iotlb_page()
446 	pm_runtime_put_sync(obj->dev);  in flush_iotlb_page()
448 	if (i == obj->nr_tlb_entries)  in flush_iotlb_page()
449 		dev_dbg(obj->dev, "%s: no page for %08x\n", __func__, da);  in flush_iotlb_page()
456 static void flush_iotlb_all(struct omap_iommu *obj)  in flush_iotlb_all()  argument
460 	pm_runtime_get_sync(obj->dev);  in flush_iotlb_all()
464 	iotlb_lock_set(obj, &l);  in flush_iotlb_all()
466 	iommu_write_reg(obj, 1, MMU_GFLUSH);  in flush_iotlb_all()
468 	pm_runtime_put_sync(obj->dev);  in flush_iotlb_all()
479 				 iommu_read_reg(obj, MMU_##name));	\
487 omap2_iommu_dump_ctx(struct omap_iommu *obj, char *buf, ssize_t len)  in omap2_iommu_dump_ctx()  argument
511 ssize_t omap_iommu_dump_ctx(struct omap_iommu *obj, char *buf, ssize_t bytes)  in omap_iommu_dump_ctx()  argument
513 	if (!obj || !buf)  in omap_iommu_dump_ctx()
516 	pm_runtime_get_sync(obj->dev);  in omap_iommu_dump_ctx()
518 	bytes = omap2_iommu_dump_ctx(obj, buf, bytes);  in omap_iommu_dump_ctx()
520 	pm_runtime_put_sync(obj->dev);  in omap_iommu_dump_ctx()
526 __dump_tlb_entries(struct omap_iommu *obj, struct cr_regs *crs, int num)  in __dump_tlb_entries()  argument
533 	pm_runtime_get_sync(obj->dev);  in __dump_tlb_entries()
534 	iotlb_lock_get(obj, &saved);  in __dump_tlb_entries()
536 	for_each_iotlb_cr(obj, num, i, tmp) {  in __dump_tlb_entries()
542 	iotlb_lock_set(obj, &saved);  in __dump_tlb_entries()
543 	pm_runtime_put_sync(obj->dev);  in __dump_tlb_entries()
554 static ssize_t iotlb_dump_cr(struct omap_iommu *obj, struct cr_regs *cr,  in iotlb_dump_cr()  argument
571 size_t omap_dump_tlb_entries(struct omap_iommu *obj, char *buf, ssize_t bytes)  in omap_dump_tlb_entries()  argument
578 	num = min(obj->nr_tlb_entries, num);  in omap_dump_tlb_entries()
584 	num = __dump_tlb_entries(obj, cr, num);  in omap_dump_tlb_entries()
586 		p += iotlb_dump_cr(obj, cr + i, p);  in omap_dump_tlb_entries()
624 static u32 *iopte_alloc(struct omap_iommu *obj, u32 *iopgd, u32 da)  in iopte_alloc()  argument
635 	spin_unlock(&obj->page_table_lock);  in iopte_alloc()
637 	spin_lock(&obj->page_table_lock);  in iopte_alloc()
646 		dev_vdbg(obj->dev, "%s: a new pte:%p\n", __func__, iopte);  in iopte_alloc()
655 	dev_vdbg(obj->dev,  in iopte_alloc()
662 static int iopgd_alloc_section(struct omap_iommu *obj, u32 da, u32 pa, u32 prot)  in iopgd_alloc_section()  argument
664 	u32 *iopgd = iopgd_offset(obj, da);  in iopgd_alloc_section()
667 		dev_err(obj->dev, "%s: %08x:%08x should aligned on %08lx\n",  in iopgd_alloc_section()
677 static int iopgd_alloc_super(struct omap_iommu *obj, u32 da, u32 pa, u32 prot)  in iopgd_alloc_super()  argument
679 	u32 *iopgd = iopgd_offset(obj, da);  in iopgd_alloc_super()
683 		dev_err(obj->dev, "%s: %08x:%08x should aligned on %08lx\n",  in iopgd_alloc_super()
694 static int iopte_alloc_page(struct omap_iommu *obj, u32 da, u32 pa, u32 prot)  in iopte_alloc_page()  argument
696 	u32 *iopgd = iopgd_offset(obj, da);  in iopte_alloc_page()
697 	u32 *iopte = iopte_alloc(obj, iopgd, da);  in iopte_alloc_page()
705 	dev_vdbg(obj->dev, "%s: da:%08x pa:%08x pte:%p *pte:%08x\n",  in iopte_alloc_page()
711 static int iopte_alloc_large(struct omap_iommu *obj, u32 da, u32 pa, u32 prot)  in iopte_alloc_large()  argument
713 	u32 *iopgd = iopgd_offset(obj, da);  in iopte_alloc_large()
714 	u32 *iopte = iopte_alloc(obj, iopgd, da);  in iopte_alloc_large()
718 		dev_err(obj->dev, "%s: %08x:%08x should aligned on %08lx\n",  in iopte_alloc_large()
733 iopgtable_store_entry_core(struct omap_iommu *obj, struct iotlb_entry *e)  in iopgtable_store_entry_core()  argument
739 	if (!obj || !e)  in iopgtable_store_entry_core()
763 	spin_lock(&obj->page_table_lock);  in iopgtable_store_entry_core()
764 	err = fn(obj, e->da, e->pa, prot);  in iopgtable_store_entry_core()
765 	spin_unlock(&obj->page_table_lock);  in iopgtable_store_entry_core()
776 omap_iopgtable_store_entry(struct omap_iommu *obj, struct iotlb_entry *e)  in omap_iopgtable_store_entry()  argument
780 	flush_iotlb_page(obj, e->da);  in omap_iopgtable_store_entry()
781 	err = iopgtable_store_entry_core(obj, e);  in omap_iopgtable_store_entry()
783 		prefetch_iotlb_entry(obj, e);  in omap_iopgtable_store_entry()
795 iopgtable_lookup_entry(struct omap_iommu *obj, u32 da, u32 **ppgd, u32 **ppte)  in iopgtable_lookup_entry()  argument
799 	iopgd = iopgd_offset(obj, da);  in iopgtable_lookup_entry()
810 static size_t iopgtable_clear_entry_core(struct omap_iommu *obj, u32 da)  in iopgtable_clear_entry_core()  argument
813 	u32 *iopgd = iopgd_offset(obj, da);  in iopgtable_clear_entry_core()
848 			iopgd = iopgd_offset(obj, (da & IOSUPER_MASK));  in iopgtable_clear_entry_core()
863 static size_t iopgtable_clear_entry(struct omap_iommu *obj, u32 da)  in iopgtable_clear_entry()  argument
867 	spin_lock(&obj->page_table_lock);  in iopgtable_clear_entry()
869 	bytes = iopgtable_clear_entry_core(obj, da);  in iopgtable_clear_entry()
870 	flush_iotlb_page(obj, da);  in iopgtable_clear_entry()
872 	spin_unlock(&obj->page_table_lock);  in iopgtable_clear_entry()
877 static void iopgtable_clear_entry_all(struct omap_iommu *obj)  in iopgtable_clear_entry_all()  argument
881 	spin_lock(&obj->page_table_lock);  in iopgtable_clear_entry_all()
888 		iopgd = iopgd_offset(obj, da);  in iopgtable_clear_entry_all()
900 	flush_iotlb_all(obj);  in iopgtable_clear_entry_all()
902 	spin_unlock(&obj->page_table_lock);  in iopgtable_clear_entry_all()
912 	struct omap_iommu *obj = data;  in iommu_fault_handler()  local
913 	struct iommu_domain *domain = obj->domain;  in iommu_fault_handler()
919 	errs = iommu_report_fault(obj, &da);  in iommu_fault_handler()
924 	if (!report_iommu_fault(domain, obj->dev, da, 0))  in iommu_fault_handler()
927 	iommu_disable(obj);  in iommu_fault_handler()
929 	iopgd = iopgd_offset(obj, da);  in iommu_fault_handler()
932 		dev_err(obj->dev, "%s: errs:0x%08x da:0x%08x pgd:0x%p *pgd:px%08x\n",  in iommu_fault_handler()
933 				obj->name, errs, da, iopgd, *iopgd);  in iommu_fault_handler()
939 	dev_err(obj->dev, "%s: errs:0x%08x da:0x%08x pgd:0x%p *pgd:0x%08x pte:0x%p *pte:0x%08x\n",  in iommu_fault_handler()
940 			obj->name, errs, da, iopgd, *iopgd, iopte, *iopte);  in iommu_fault_handler()
947 	struct omap_iommu *obj = to_iommu(dev);  in device_match_by_alias()  local
950 	pr_debug("%s: %s %s\n", __func__, obj->name, name);  in device_match_by_alias()
952 	return strcmp(obj->name, name) == 0;  in device_match_by_alias()
964 	struct omap_iommu *obj;  in omap_iommu_attach()  local
972 	obj = to_iommu(dev);  in omap_iommu_attach()
974 	spin_lock(&obj->iommu_lock);  in omap_iommu_attach()
976 	obj->iopgd = iopgd;  in omap_iommu_attach()
977 	err = iommu_enable(obj);  in omap_iommu_attach()
980 	flush_iotlb_all(obj);  in omap_iommu_attach()
982 	spin_unlock(&obj->iommu_lock);  in omap_iommu_attach()
984 	dev_dbg(obj->dev, "%s: %s\n", __func__, obj->name);  in omap_iommu_attach()
985 	return obj;  in omap_iommu_attach()
988 	spin_unlock(&obj->iommu_lock);  in omap_iommu_attach()
996 static void omap_iommu_detach(struct omap_iommu *obj)  in omap_iommu_detach()  argument
998 	if (!obj || IS_ERR(obj))  in omap_iommu_detach()
1001 	spin_lock(&obj->iommu_lock);  in omap_iommu_detach()
1003 	iommu_disable(obj);  in omap_iommu_detach()
1004 	obj->iopgd = NULL;  in omap_iommu_detach()
1006 	spin_unlock(&obj->iommu_lock);  in omap_iommu_detach()
1008 	dev_dbg(obj->dev, "%s: %s\n", __func__, obj->name);  in omap_iommu_detach()
1018 	struct omap_iommu *obj;  in omap_iommu_probe()  local
1023 	obj = devm_kzalloc(&pdev->dev, sizeof(*obj) + MMU_REG_SIZE, GFP_KERNEL);  in omap_iommu_probe()
1024 	if (!obj)  in omap_iommu_probe()
1028 		obj->name = dev_name(&pdev->dev);  in omap_iommu_probe()
1029 		obj->nr_tlb_entries = 32;  in omap_iommu_probe()
1031 					   &obj->nr_tlb_entries);  in omap_iommu_probe()
1034 		if (obj->nr_tlb_entries != 32 && obj->nr_tlb_entries != 8)  in omap_iommu_probe()
1037 			obj->has_bus_err_back = MMU_GP_REG_BUS_ERR_BACK_EN;  in omap_iommu_probe()
1039 		obj->nr_tlb_entries = pdata->nr_tlb_entries;  in omap_iommu_probe()
1040 		obj->name = pdata->name;  in omap_iommu_probe()
1043 	obj->dev = &pdev->dev;  in omap_iommu_probe()
1044 	obj->ctx = (void *)obj + sizeof(*obj);  in omap_iommu_probe()
1046 	spin_lock_init(&obj->iommu_lock);  in omap_iommu_probe()
1047 	spin_lock_init(&obj->page_table_lock);  in omap_iommu_probe()
1050 	obj->regbase = devm_ioremap_resource(obj->dev, res);  in omap_iommu_probe()
1051 	if (IS_ERR(obj->regbase))  in omap_iommu_probe()
1052 		return PTR_ERR(obj->regbase);  in omap_iommu_probe()
1058 	err = devm_request_irq(obj->dev, irq, iommu_fault_handler, IRQF_SHARED,  in omap_iommu_probe()
1059 			       dev_name(obj->dev), obj);  in omap_iommu_probe()
1062 	platform_set_drvdata(pdev, obj);  in omap_iommu_probe()
1064 	pm_runtime_irq_safe(obj->dev);  in omap_iommu_probe()
1065 	pm_runtime_enable(obj->dev);  in omap_iommu_probe()
1067 	omap_iommu_debugfs_add(obj);  in omap_iommu_probe()
1069 	dev_info(&pdev->dev, "%s registered\n", obj->name);  in omap_iommu_probe()
1075 	struct omap_iommu *obj = platform_get_drvdata(pdev);  in omap_iommu_remove()  local
1077 	iopgtable_clear_entry_all(obj);  in omap_iommu_remove()
1078 	omap_iommu_debugfs_remove(obj);  in omap_iommu_remove()
1080 	pm_runtime_disable(obj->dev);  in omap_iommu_remove()
1082 	dev_info(&pdev->dev, "%s removed\n", obj->name);  in omap_iommu_remove()