239 static bool set_sysmmu_active(struct sysmmu_drvdata *data)  in set_sysmmu_active()  argument
243 	return ++data->activations == 1;  in set_sysmmu_active()
246 static bool set_sysmmu_inactive(struct sysmmu_drvdata *data)  in set_sysmmu_inactive()  argument
249 	BUG_ON(data->activations < 1);  in set_sysmmu_inactive()
250 	return --data->activations == 0;  in set_sysmmu_inactive()
253 static bool is_sysmmu_active(struct sysmmu_drvdata *data)  in is_sysmmu_active()  argument
255 	return data->activations > 0;  in is_sysmmu_active()
328 	struct sysmmu_drvdata *data = dev_id;  in exynos_sysmmu_irq()  local
333 	WARN_ON(!is_sysmmu_active(data));  in exynos_sysmmu_irq()
335 	spin_lock(&data->lock);  in exynos_sysmmu_irq()
337 	if (!IS_ERR(data->clk_master))  in exynos_sysmmu_irq()
338 		clk_enable(data->clk_master);  in exynos_sysmmu_irq()
341 		__ffs(__raw_readl(data->sfrbase + REG_INT_STATUS));  in exynos_sysmmu_irq()
345 		addr = __raw_readl(data->sfrbase + fault_reg_offset[itype]);  in exynos_sysmmu_irq()
349 			__func__, dev_name(data->sysmmu));  in exynos_sysmmu_irq()
355 				__raw_readl(data->sfrbase + REG_PT_BASE_ADDR);  in exynos_sysmmu_irq()
356 		show_fault_information(dev_name(data->sysmmu),  in exynos_sysmmu_irq()
358 		if (data->domain)  in exynos_sysmmu_irq()
359 			ret = report_iommu_fault(&data->domain->domain,  in exynos_sysmmu_irq()
360 					data->master, addr, itype);  in exynos_sysmmu_irq()
366 	__raw_writel(1 << itype, data->sfrbase + REG_INT_CLEAR);  in exynos_sysmmu_irq()
368 	sysmmu_unblock(data->sfrbase);  in exynos_sysmmu_irq()
370 	if (!IS_ERR(data->clk_master))  in exynos_sysmmu_irq()
371 		clk_disable(data->clk_master);  in exynos_sysmmu_irq()
373 	spin_unlock(&data->lock);  in exynos_sysmmu_irq()
378 static void __sysmmu_disable_nocount(struct sysmmu_drvdata *data)  in __sysmmu_disable_nocount()  argument
380 	if (!IS_ERR(data->clk_master))  in __sysmmu_disable_nocount()
381 		clk_enable(data->clk_master);  in __sysmmu_disable_nocount()
383 	__raw_writel(CTRL_DISABLE, data->sfrbase + REG_MMU_CTRL);  in __sysmmu_disable_nocount()
384 	__raw_writel(0, data->sfrbase + REG_MMU_CFG);  in __sysmmu_disable_nocount()
386 	clk_disable(data->clk);  in __sysmmu_disable_nocount()
387 	if (!IS_ERR(data->clk_master))  in __sysmmu_disable_nocount()
388 		clk_disable(data->clk_master);  in __sysmmu_disable_nocount()
391 static bool __sysmmu_disable(struct sysmmu_drvdata *data)  in __sysmmu_disable()  argument
396 	spin_lock_irqsave(&data->lock, flags);  in __sysmmu_disable()
398 	disabled = set_sysmmu_inactive(data);  in __sysmmu_disable()
401 		data->pgtable = 0;  in __sysmmu_disable()
402 		data->domain = NULL;  in __sysmmu_disable()
404 		__sysmmu_disable_nocount(data);  in __sysmmu_disable()
406 		dev_dbg(data->sysmmu, "Disabled\n");  in __sysmmu_disable()
408 		dev_dbg(data->sysmmu, "%d times left to disable\n",  in __sysmmu_disable()
409 					data->activations);  in __sysmmu_disable()
412 	spin_unlock_irqrestore(&data->lock, flags);  in __sysmmu_disable()
417 static void __sysmmu_init_config(struct sysmmu_drvdata *data)  in __sysmmu_init_config()  argument
422 	ver = MMU_RAW_VER(__raw_readl(data->sfrbase + REG_MMU_VERSION));  in __sysmmu_init_config()
435 	__raw_writel(cfg, data->sfrbase + REG_MMU_CFG);  in __sysmmu_init_config()
436 	data->version = ver;  in __sysmmu_init_config()
439 static void __sysmmu_enable_nocount(struct sysmmu_drvdata *data)  in __sysmmu_enable_nocount()  argument
441 	if (!IS_ERR(data->clk_master))  in __sysmmu_enable_nocount()
442 		clk_enable(data->clk_master);  in __sysmmu_enable_nocount()
443 	clk_enable(data->clk);  in __sysmmu_enable_nocount()
445 	__raw_writel(CTRL_BLOCK, data->sfrbase + REG_MMU_CTRL);  in __sysmmu_enable_nocount()
447 	__sysmmu_init_config(data);  in __sysmmu_enable_nocount()
449 	__sysmmu_set_ptbase(data->sfrbase, data->pgtable);  in __sysmmu_enable_nocount()
451 	__raw_writel(CTRL_ENABLE, data->sfrbase + REG_MMU_CTRL);  in __sysmmu_enable_nocount()
453 	if (!IS_ERR(data->clk_master))  in __sysmmu_enable_nocount()
454 		clk_disable(data->clk_master);  in __sysmmu_enable_nocount()
457 static int __sysmmu_enable(struct sysmmu_drvdata *data, phys_addr_t pgtable,  in __sysmmu_enable()  argument
463 	spin_lock_irqsave(&data->lock, flags);  in __sysmmu_enable()
464 	if (set_sysmmu_active(data)) {  in __sysmmu_enable()
465 		data->pgtable = pgtable;  in __sysmmu_enable()
466 		data->domain = domain;  in __sysmmu_enable()
468 		__sysmmu_enable_nocount(data);  in __sysmmu_enable()
470 		dev_dbg(data->sysmmu, "Enabled\n");  in __sysmmu_enable()
472 		ret = (pgtable == data->pgtable) ? 1 : -EBUSY;  in __sysmmu_enable()
474 		dev_dbg(data->sysmmu, "already enabled\n");  in __sysmmu_enable()
478 		set_sysmmu_inactive(data); /* decrement count */  in __sysmmu_enable()
480 	spin_unlock_irqrestore(&data->lock, flags);  in __sysmmu_enable()
485 static void __sysmmu_tlb_invalidate_flpdcache(struct sysmmu_drvdata *data,  in __sysmmu_tlb_invalidate_flpdcache()  argument
488 	if (data->version == MAKE_MMU_VER(3, 3))  in __sysmmu_tlb_invalidate_flpdcache()
489 		__raw_writel(iova | 0x1, data->sfrbase + REG_MMU_FLUSH_ENTRY);  in __sysmmu_tlb_invalidate_flpdcache()
492 static void sysmmu_tlb_invalidate_flpdcache(struct sysmmu_drvdata *data,  in sysmmu_tlb_invalidate_flpdcache()  argument
497 	if (!IS_ERR(data->clk_master))  in sysmmu_tlb_invalidate_flpdcache()
498 		clk_enable(data->clk_master);  in sysmmu_tlb_invalidate_flpdcache()
500 	spin_lock_irqsave(&data->lock, flags);  in sysmmu_tlb_invalidate_flpdcache()
501 	if (is_sysmmu_active(data))  in sysmmu_tlb_invalidate_flpdcache()
502 		__sysmmu_tlb_invalidate_flpdcache(data, iova);  in sysmmu_tlb_invalidate_flpdcache()
503 	spin_unlock_irqrestore(&data->lock, flags);  in sysmmu_tlb_invalidate_flpdcache()
505 	if (!IS_ERR(data->clk_master))  in sysmmu_tlb_invalidate_flpdcache()
506 		clk_disable(data->clk_master);  in sysmmu_tlb_invalidate_flpdcache()
509 static void sysmmu_tlb_invalidate_entry(struct sysmmu_drvdata *data,  in sysmmu_tlb_invalidate_entry()  argument
514 	spin_lock_irqsave(&data->lock, flags);  in sysmmu_tlb_invalidate_entry()
515 	if (is_sysmmu_active(data)) {  in sysmmu_tlb_invalidate_entry()
518 		if (!IS_ERR(data->clk_master))  in sysmmu_tlb_invalidate_entry()
519 			clk_enable(data->clk_master);  in sysmmu_tlb_invalidate_entry()
531 		if (MMU_MAJ_VER(data->version) == 2)  in sysmmu_tlb_invalidate_entry()
534 		if (sysmmu_block(data->sfrbase)) {  in sysmmu_tlb_invalidate_entry()
536 				data->sfrbase, iova, num_inv);  in sysmmu_tlb_invalidate_entry()
537 			sysmmu_unblock(data->sfrbase);  in sysmmu_tlb_invalidate_entry()
539 		if (!IS_ERR(data->clk_master))  in sysmmu_tlb_invalidate_entry()
540 			clk_disable(data->clk_master);  in sysmmu_tlb_invalidate_entry()
542 		dev_dbg(data->master,  in sysmmu_tlb_invalidate_entry()
545 	spin_unlock_irqrestore(&data->lock, flags);  in sysmmu_tlb_invalidate_entry()
552 	struct sysmmu_drvdata *data;  in exynos_sysmmu_probe()  local
555 	data = devm_kzalloc(dev, sizeof(*data), GFP_KERNEL);  in exynos_sysmmu_probe()
556 	if (!data)  in exynos_sysmmu_probe()
560 	data->sfrbase = devm_ioremap_resource(dev, res);  in exynos_sysmmu_probe()
561 	if (IS_ERR(data->sfrbase))  in exynos_sysmmu_probe()
562 		return PTR_ERR(data->sfrbase);  in exynos_sysmmu_probe()
571 				dev_name(dev), data);  in exynos_sysmmu_probe()
577 	data->clk = devm_clk_get(dev, "sysmmu");  in exynos_sysmmu_probe()
578 	if (IS_ERR(data->clk)) {  in exynos_sysmmu_probe()
580 		return PTR_ERR(data->clk);  in exynos_sysmmu_probe()
582 		ret = clk_prepare(data->clk);  in exynos_sysmmu_probe()
589 	data->clk_master = devm_clk_get(dev, "master");  in exynos_sysmmu_probe()
590 	if (!IS_ERR(data->clk_master)) {  in exynos_sysmmu_probe()
591 		ret = clk_prepare(data->clk_master);  in exynos_sysmmu_probe()
593 			clk_unprepare(data->clk);  in exynos_sysmmu_probe()
599 	data->sysmmu = dev;  in exynos_sysmmu_probe()
600 	spin_lock_init(&data->lock);  in exynos_sysmmu_probe()
602 	platform_set_drvdata(pdev, data);  in exynos_sysmmu_probe()
612 	struct sysmmu_drvdata *data = dev_get_drvdata(dev);  in exynos_sysmmu_suspend()  local
615 	if (is_sysmmu_active(data)) {  in exynos_sysmmu_suspend()
616 		__sysmmu_disable_nocount(data);  in exynos_sysmmu_suspend()
624 	struct sysmmu_drvdata *data = dev_get_drvdata(dev);  in exynos_sysmmu_resume()  local
627 	if (is_sysmmu_active(data)) {  in exynos_sysmmu_resume()
629 		__sysmmu_enable_nocount(data);  in exynos_sysmmu_resume()
714 	struct sysmmu_drvdata *data, *next;  in exynos_iommu_domain_free()  local
722 	list_for_each_entry_safe(data, next, &domain->clients, domain_node) {  in exynos_iommu_domain_free()
723 		if (__sysmmu_disable(data))  in exynos_iommu_domain_free()
724 			data->master = NULL;  in exynos_iommu_domain_free()
725 		list_del_init(&data->domain_node);  in exynos_iommu_domain_free()
745 	struct sysmmu_drvdata *data;  in exynos_iommu_attach_device()  local
753 	list_for_each_entry(data, &owner->controllers, owner_node) {  in exynos_iommu_attach_device()
754 		pm_runtime_get_sync(data->sysmmu);  in exynos_iommu_attach_device()
755 		ret = __sysmmu_enable(data, pagetable, domain);  in exynos_iommu_attach_device()
757 			data->master = dev;  in exynos_iommu_attach_device()
760 			list_add_tail(&data->domain_node, &domain->clients);  in exynos_iommu_attach_device()
782 	struct sysmmu_drvdata *data, *next;  in exynos_iommu_detach_device()  local
790 	list_for_each_entry_safe(data, next, &domain->clients, domain_node) {  in exynos_iommu_detach_device()
791 		if (data->master == dev) {  in exynos_iommu_detach_device()
792 			if (__sysmmu_disable(data)) {  in exynos_iommu_detach_device()
793 				data->master = NULL;  in exynos_iommu_detach_device()
794 				list_del_init(&data->domain_node);  in exynos_iommu_detach_device()
796 			pm_runtime_put(data->sysmmu);  in exynos_iommu_detach_device()
850 			struct sysmmu_drvdata *data;  in alloc_lv2entry()  local
853 			list_for_each_entry(data, &domain->clients, domain_node)  in alloc_lv2entry()
854 				sysmmu_tlb_invalidate_flpdcache(data, iova);  in alloc_lv2entry()
889 		struct sysmmu_drvdata *data;  in lv1set_section()  local
894 		list_for_each_entry(data, &domain->clients, domain_node)  in lv1set_section()
895 			sysmmu_tlb_invalidate_flpdcache(data, iova);  in lv1set_section()
1001 	struct sysmmu_drvdata *data;  in exynos_iommu_tlb_invalidate_entry()  local
1006 	list_for_each_entry(data, &domain->clients, domain_node)  in exynos_iommu_tlb_invalidate_entry()
1007 		sysmmu_tlb_invalidate_entry(data, iova, size);  in exynos_iommu_tlb_invalidate_entry()
1154 	struct sysmmu_drvdata *data;  in exynos_iommu_of_xlate()  local
1159 	data = platform_get_drvdata(sysmmu);  in exynos_iommu_of_xlate()
1160 	if (!data)  in exynos_iommu_of_xlate()
1172 	list_add_tail(&data->owner_node, &owner->controllers);  in exynos_iommu_of_xlate()