59 #define ARM_SMMU_GR0(smmu)		((smmu)->base)  argument
60 #define ARM_SMMU_GR1(smmu)		((smmu)->base + (1 << (smmu)->pgshift))  argument
67 #define ARM_SMMU_GR0_NS(smmu)						\  argument
68 	((smmu)->base +							\
69 		((smmu->options & ARM_SMMU_OPT_SECURE_CFG_ACCESS)	\
181 #define ARM_SMMU_CB_BASE(smmu)		((smmu)->base + ((smmu)->size >> 1))  argument
182 #define ARM_SMMU_CB(smmu, n)		((n) * (1 << (smmu)->pgshift))  argument
333 	struct arm_smmu_device		*smmu;  member
362 static void parse_driver_options(struct arm_smmu_device *smmu)  in parse_driver_options()  argument
367 		if (of_property_read_bool(smmu->dev->of_node,  in parse_driver_options()
369 			smmu->options |= arm_smmu_options[i].opt;  in parse_driver_options()
370 			dev_notice(smmu->dev, "option %s\n",  in parse_driver_options()
389 static struct arm_smmu_master *find_smmu_master(struct arm_smmu_device *smmu,  in find_smmu_master()  argument
392 	struct rb_node *node = smmu->masters.rb_node;  in find_smmu_master()
424 static int insert_smmu_master(struct arm_smmu_device *smmu,  in insert_smmu_master()  argument
429 	new = &smmu->masters.rb_node;  in insert_smmu_master()
445 	rb_insert_color(&master->node, &smmu->masters);  in insert_smmu_master()
449 static int register_smmu_master(struct arm_smmu_device *smmu,  in register_smmu_master()  argument
456 	master = find_smmu_master(smmu, masterspec->np);  in register_smmu_master()
481 		if (!(smmu->features & ARM_SMMU_FEAT_STREAM_MATCH) &&  in register_smmu_master()
482 		     (streamid >= smmu->num_mapping_groups)) {  in register_smmu_master()
485 				masterspec->np->name, smmu->num_mapping_groups);  in register_smmu_master()
490 	return insert_smmu_master(smmu, master);  in register_smmu_master()
495 	struct arm_smmu_device *smmu;  in find_smmu_for_device()  local
500 	list_for_each_entry(smmu, &arm_smmu_devices, list) {  in find_smmu_for_device()
501 		master = find_smmu_master(smmu, dev_node);  in find_smmu_for_device()
507 	return master ? smmu : NULL;  in find_smmu_for_device()
529 static void __arm_smmu_tlb_sync(struct arm_smmu_device *smmu)  in __arm_smmu_tlb_sync()  argument
532 	void __iomem *gr0_base = ARM_SMMU_GR0(smmu);  in __arm_smmu_tlb_sync()
539 			dev_err_ratelimited(smmu->dev,  in __arm_smmu_tlb_sync()
550 	__arm_smmu_tlb_sync(smmu_domain->smmu);  in arm_smmu_tlb_sync()
557 	struct arm_smmu_device *smmu = smmu_domain->smmu;  in arm_smmu_tlb_inv_context()  local
562 		base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx);  in arm_smmu_tlb_inv_context()
566 		base = ARM_SMMU_GR0(smmu);  in arm_smmu_tlb_inv_context()
571 	__arm_smmu_tlb_sync(smmu);  in arm_smmu_tlb_inv_context()
579 	struct arm_smmu_device *smmu = smmu_domain->smmu;  in arm_smmu_tlb_inv_range_nosync()  local
584 		reg = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx);  in arm_smmu_tlb_inv_range_nosync()
587 		if (!IS_ENABLED(CONFIG_64BIT) || smmu->version == ARM_SMMU_V1) {  in arm_smmu_tlb_inv_range_nosync()
599 	} else if (smmu->version == ARM_SMMU_V2) {  in arm_smmu_tlb_inv_range_nosync()
600 		reg = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx);  in arm_smmu_tlb_inv_range_nosync()
606 		reg = ARM_SMMU_GR0(smmu) + ARM_SMMU_GR0_TLBIVMID;  in arm_smmu_tlb_inv_range_nosync()
614 	struct arm_smmu_device *smmu = smmu_domain->smmu;  in arm_smmu_flush_pgtable()  local
619 	if (smmu->features & ARM_SMMU_FEAT_COHERENT_WALK) {  in arm_smmu_flush_pgtable()
629 		dma_map_page(smmu->dev, virt_to_page(addr), offset, size,  in arm_smmu_flush_pgtable()
649 	struct arm_smmu_device *smmu = smmu_domain->smmu;  in arm_smmu_context_fault()  local
652 	cb_base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx);  in arm_smmu_context_fault()
659 		dev_err_ratelimited(smmu->dev,  in arm_smmu_context_fault()
673 	if (!report_iommu_fault(domain, smmu->dev, iova, flags)) {  in arm_smmu_context_fault()
677 		dev_err_ratelimited(smmu->dev,  in arm_smmu_context_fault()
697 	struct arm_smmu_device *smmu = dev;  in arm_smmu_global_fault()  local
698 	void __iomem *gr0_base = ARM_SMMU_GR0_NS(smmu);  in arm_smmu_global_fault()
708 	dev_err_ratelimited(smmu->dev,  in arm_smmu_global_fault()
710 	dev_err_ratelimited(smmu->dev,  in arm_smmu_global_fault()
724 	struct arm_smmu_device *smmu = smmu_domain->smmu;  in arm_smmu_init_context_bank()  local
727 	gr0_base = ARM_SMMU_GR0(smmu);  in arm_smmu_init_context_bank()
728 	gr1_base = ARM_SMMU_GR1(smmu);  in arm_smmu_init_context_bank()
730 	cb_base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx);  in arm_smmu_init_context_bank()
732 	if (smmu->version > ARM_SMMU_V1) {  in arm_smmu_init_context_bank()
748 	if (smmu->version == ARM_SMMU_V1)  in arm_smmu_init_context_bank()
787 		if (smmu->version > ARM_SMMU_V1) {  in arm_smmu_init_context_bank()
816 					struct arm_smmu_device *smmu)  in arm_smmu_init_domain_context()  argument
827 	if (smmu_domain->smmu)  in arm_smmu_init_domain_context()
848 	if (!(smmu->features & ARM_SMMU_FEAT_TRANS_S1))  in arm_smmu_init_domain_context()
850 	if (!(smmu->features & ARM_SMMU_FEAT_TRANS_S2))  in arm_smmu_init_domain_context()
856 		start = smmu->num_s2_context_banks;  in arm_smmu_init_domain_context()
857 		ias = smmu->va_size;  in arm_smmu_init_domain_context()
858 		oas = smmu->ipa_size;  in arm_smmu_init_domain_context()
872 		ias = smmu->ipa_size;  in arm_smmu_init_domain_context()
873 		oas = smmu->pa_size;  in arm_smmu_init_domain_context()
884 	ret = __arm_smmu_alloc_bitmap(smmu->context_map, start,  in arm_smmu_init_domain_context()
885 				      smmu->num_context_banks);  in arm_smmu_init_domain_context()
890 	if (smmu->version == ARM_SMMU_V1) {  in arm_smmu_init_domain_context()
891 		cfg->irptndx = atomic_inc_return(&smmu->irptndx);  in arm_smmu_init_domain_context()
892 		cfg->irptndx %= smmu->num_context_irqs;  in arm_smmu_init_domain_context()
904 	smmu_domain->smmu = smmu;  in arm_smmu_init_domain_context()
921 	irq = smmu->irqs[smmu->num_global_irqs + cfg->irptndx];  in arm_smmu_init_domain_context()
925 		dev_err(smmu->dev, "failed to request context IRQ %d (%u)\n",  in arm_smmu_init_domain_context()
937 	smmu_domain->smmu = NULL;  in arm_smmu_init_domain_context()
946 	struct arm_smmu_device *smmu = smmu_domain->smmu;  in arm_smmu_destroy_domain_context()  local
951 	if (!smmu)  in arm_smmu_destroy_domain_context()
958 	cb_base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx);  in arm_smmu_destroy_domain_context()
962 		irq = smmu->irqs[smmu->num_global_irqs + cfg->irptndx];  in arm_smmu_destroy_domain_context()
969 	__arm_smmu_free_bitmap(smmu->context_map, cfg->cbndx);  in arm_smmu_destroy_domain_context()
1005 static int arm_smmu_master_configure_smrs(struct arm_smmu_device *smmu,  in arm_smmu_master_configure_smrs()  argument
1010 	void __iomem *gr0_base = ARM_SMMU_GR0(smmu);  in arm_smmu_master_configure_smrs()
1012 	if (!(smmu->features & ARM_SMMU_FEAT_STREAM_MATCH))  in arm_smmu_master_configure_smrs()
1020 		dev_err(smmu->dev, "failed to allocate %d SMRs\n",  in arm_smmu_master_configure_smrs()
1027 		int idx = __arm_smmu_alloc_bitmap(smmu->smr_map, 0,  in arm_smmu_master_configure_smrs()
1028 						  smmu->num_mapping_groups);  in arm_smmu_master_configure_smrs()
1030 			dev_err(smmu->dev, "failed to allocate free SMR\n");  in arm_smmu_master_configure_smrs()
1053 		__arm_smmu_free_bitmap(smmu->smr_map, smrs[i].idx);  in arm_smmu_master_configure_smrs()
1058 static void arm_smmu_master_free_smrs(struct arm_smmu_device *smmu,  in arm_smmu_master_free_smrs()  argument
1062 	void __iomem *gr0_base = ARM_SMMU_GR0(smmu);  in arm_smmu_master_free_smrs()
1073 		__arm_smmu_free_bitmap(smmu->smr_map, idx);  in arm_smmu_master_free_smrs()
1084 	struct arm_smmu_device *smmu = smmu_domain->smmu;  in arm_smmu_domain_add_master()  local
1085 	void __iomem *gr0_base = ARM_SMMU_GR0(smmu);  in arm_smmu_domain_add_master()
1088 	ret = arm_smmu_master_configure_smrs(smmu, cfg);  in arm_smmu_domain_add_master()
1108 	struct arm_smmu_device *smmu = smmu_domain->smmu;  in arm_smmu_domain_remove_master()  local
1109 	void __iomem *gr0_base = ARM_SMMU_GR0(smmu);  in arm_smmu_domain_remove_master()
1112 	if ((smmu->features & ARM_SMMU_FEAT_STREAM_MATCH) && !cfg->smrs)  in arm_smmu_domain_remove_master()
1126 	arm_smmu_master_free_smrs(smmu, cfg);  in arm_smmu_domain_remove_master()
1133 	struct arm_smmu_device *smmu;  in arm_smmu_attach_dev()  local
1136 	smmu = find_smmu_for_device(dev);  in arm_smmu_attach_dev()
1137 	if (!smmu) {  in arm_smmu_attach_dev()
1148 	ret = arm_smmu_init_domain_context(domain, smmu);  in arm_smmu_attach_dev()
1156 	if (smmu_domain->smmu != smmu) {  in arm_smmu_attach_dev()
1159 			dev_name(smmu_domain->smmu->dev), dev_name(smmu->dev));  in arm_smmu_attach_dev()
1225 	struct arm_smmu_device *smmu = smmu_domain->smmu;  in arm_smmu_iova_to_phys_hard()  local
1228 	struct device *dev = smmu->dev;  in arm_smmu_iova_to_phys_hard()
1233 	cb_base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx);  in arm_smmu_iova_to_phys_hard()
1235 	if (smmu->version == 1) {  in arm_smmu_iova_to_phys_hard()
1277 	if (smmu_domain->smmu->features & ARM_SMMU_FEAT_TRANS_OPS &&  in arm_smmu_iova_to_phys()
1369 	struct arm_smmu_device *smmu = find_smmu_for_device(dev);  in arm_smmu_add_platform_device()  local
1371 	if (!smmu)  in arm_smmu_add_platform_device()
1374 	master = find_smmu_master(smmu, dev->of_node);  in arm_smmu_add_platform_device()
1424 		if (smmu_domain->smmu) {  in arm_smmu_domain_set_attr()
1461 static void arm_smmu_device_reset(struct arm_smmu_device *smmu)  in arm_smmu_device_reset()  argument
1463 	void __iomem *gr0_base = ARM_SMMU_GR0(smmu);  in arm_smmu_device_reset()
1469 	reg = readl_relaxed(ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sGFSR);  in arm_smmu_device_reset()
1470 	writel(reg, ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sGFSR);  in arm_smmu_device_reset()
1473 	for (i = 0; i < smmu->num_mapping_groups; ++i) {  in arm_smmu_device_reset()
1480 	for (i = 0; i < smmu->num_context_banks; ++i) {  in arm_smmu_device_reset()
1481 		cb_base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, i);  in arm_smmu_device_reset()
1490 	reg = readl_relaxed(ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sCR0);  in arm_smmu_device_reset()
1508 	__arm_smmu_tlb_sync(smmu);  in arm_smmu_device_reset()
1509 	writel(reg, ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sCR0);  in arm_smmu_device_reset()
1531 static int arm_smmu_device_cfg_probe(struct arm_smmu_device *smmu)  in arm_smmu_device_cfg_probe()  argument
1534 	void __iomem *gr0_base = ARM_SMMU_GR0(smmu);  in arm_smmu_device_cfg_probe()
1537 	dev_notice(smmu->dev, "probing hardware configuration...\n");  in arm_smmu_device_cfg_probe()
1538 	dev_notice(smmu->dev, "SMMUv%d with:\n", smmu->version);  in arm_smmu_device_cfg_probe()
1550 		smmu->features |= ARM_SMMU_FEAT_TRANS_S1;  in arm_smmu_device_cfg_probe()
1551 		dev_notice(smmu->dev, "\tstage 1 translation\n");  in arm_smmu_device_cfg_probe()
1555 		smmu->features |= ARM_SMMU_FEAT_TRANS_S2;  in arm_smmu_device_cfg_probe()
1556 		dev_notice(smmu->dev, "\tstage 2 translation\n");  in arm_smmu_device_cfg_probe()
1560 		smmu->features |= ARM_SMMU_FEAT_TRANS_NESTED;  in arm_smmu_device_cfg_probe()
1561 		dev_notice(smmu->dev, "\tnested translation\n");  in arm_smmu_device_cfg_probe()
1564 	if (!(smmu->features &  in arm_smmu_device_cfg_probe()
1566 		dev_err(smmu->dev, "\tno translation support!\n");  in arm_smmu_device_cfg_probe()
1570 	if ((id & ID0_S1TS) && ((smmu->version == 1) || !(id & ID0_ATOSNS))) {  in arm_smmu_device_cfg_probe()
1571 		smmu->features |= ARM_SMMU_FEAT_TRANS_OPS;  in arm_smmu_device_cfg_probe()
1572 		dev_notice(smmu->dev, "\taddress translation ops\n");  in arm_smmu_device_cfg_probe()
1576 		smmu->features |= ARM_SMMU_FEAT_COHERENT_WALK;  in arm_smmu_device_cfg_probe()
1577 		dev_notice(smmu->dev, "\tcoherent table walk\n");  in arm_smmu_device_cfg_probe()
1583 		smmu->features |= ARM_SMMU_FEAT_STREAM_MATCH;  in arm_smmu_device_cfg_probe()
1584 		smmu->num_mapping_groups = (id >> ID0_NUMSMRG_SHIFT) &  in arm_smmu_device_cfg_probe()
1586 		if (smmu->num_mapping_groups == 0) {  in arm_smmu_device_cfg_probe()
1587 			dev_err(smmu->dev,  in arm_smmu_device_cfg_probe()
1600 			dev_err(smmu->dev,  in arm_smmu_device_cfg_probe()
1606 		dev_notice(smmu->dev,  in arm_smmu_device_cfg_probe()
1608 			   smmu->num_mapping_groups, mask);  in arm_smmu_device_cfg_probe()
1610 		smmu->num_mapping_groups = (id >> ID0_NUMSIDB_SHIFT) &  in arm_smmu_device_cfg_probe()
1616 	smmu->pgshift = (id & ID1_PAGESIZE) ? 16 : 12;  in arm_smmu_device_cfg_probe()
1620 	size *= 2 << smmu->pgshift;  in arm_smmu_device_cfg_probe()
1621 	if (smmu->size != size)  in arm_smmu_device_cfg_probe()
1622 		dev_warn(smmu->dev,  in arm_smmu_device_cfg_probe()
1624 			size, smmu->size);  in arm_smmu_device_cfg_probe()
1626 	smmu->num_s2_context_banks = (id >> ID1_NUMS2CB_SHIFT) & ID1_NUMS2CB_MASK;  in arm_smmu_device_cfg_probe()
1627 	smmu->num_context_banks = (id >> ID1_NUMCB_SHIFT) & ID1_NUMCB_MASK;  in arm_smmu_device_cfg_probe()
1628 	if (smmu->num_s2_context_banks > smmu->num_context_banks) {  in arm_smmu_device_cfg_probe()
1629 		dev_err(smmu->dev, "impossible number of S2 context banks!\n");  in arm_smmu_device_cfg_probe()
1632 	dev_notice(smmu->dev, "\t%u context banks (%u stage-2 only)\n",  in arm_smmu_device_cfg_probe()
1633 		   smmu->num_context_banks, smmu->num_s2_context_banks);  in arm_smmu_device_cfg_probe()
1638 	smmu->ipa_size = size;  in arm_smmu_device_cfg_probe()
1642 	smmu->pa_size = size;  in arm_smmu_device_cfg_probe()
1649 	if (dma_set_mask_and_coherent(smmu->dev, DMA_BIT_MASK(size)))  in arm_smmu_device_cfg_probe()
1650 		dev_warn(smmu->dev,  in arm_smmu_device_cfg_probe()
1653 	if (smmu->version == ARM_SMMU_V1) {  in arm_smmu_device_cfg_probe()
1654 		smmu->va_size = smmu->ipa_size;  in arm_smmu_device_cfg_probe()
1658 		smmu->va_size = arm_smmu_id_size_to_bits(size);  in arm_smmu_device_cfg_probe()
1660 		smmu->va_size = min(32UL, smmu->va_size);  in arm_smmu_device_cfg_probe()
1672 	dev_notice(smmu->dev, "\tSupported page sizes: 0x%08lx\n", size);  in arm_smmu_device_cfg_probe()
1674 	if (smmu->features & ARM_SMMU_FEAT_TRANS_S1)  in arm_smmu_device_cfg_probe()
1675 		dev_notice(smmu->dev, "\tStage-1: %lu-bit VA -> %lu-bit IPA\n",  in arm_smmu_device_cfg_probe()
1676 			   smmu->va_size, smmu->ipa_size);  in arm_smmu_device_cfg_probe()
1678 	if (smmu->features & ARM_SMMU_FEAT_TRANS_S2)  in arm_smmu_device_cfg_probe()
1679 		dev_notice(smmu->dev, "\tStage-2: %lu-bit IPA -> %lu-bit PA\n",  in arm_smmu_device_cfg_probe()
1680 			   smmu->ipa_size, smmu->pa_size);  in arm_smmu_device_cfg_probe()
1699 	struct arm_smmu_device *smmu;  in arm_smmu_device_dt_probe()  local
1705 	smmu = devm_kzalloc(dev, sizeof(*smmu), GFP_KERNEL);  in arm_smmu_device_dt_probe()
1706 	if (!smmu) {  in arm_smmu_device_dt_probe()
1710 	smmu->dev = dev;  in arm_smmu_device_dt_probe()
1713 	smmu->version = (enum arm_smmu_arch_version)of_id->data;  in arm_smmu_device_dt_probe()
1716 	smmu->base = devm_ioremap_resource(dev, res);  in arm_smmu_device_dt_probe()
1717 	if (IS_ERR(smmu->base))  in arm_smmu_device_dt_probe()
1718 		return PTR_ERR(smmu->base);  in arm_smmu_device_dt_probe()
1719 	smmu->size = resource_size(res);  in arm_smmu_device_dt_probe()
1722 				 &smmu->num_global_irqs)) {  in arm_smmu_device_dt_probe()
1730 		if (num_irqs > smmu->num_global_irqs)  in arm_smmu_device_dt_probe()
1731 			smmu->num_context_irqs++;  in arm_smmu_device_dt_probe()
1734 	if (!smmu->num_context_irqs) {  in arm_smmu_device_dt_probe()
1736 			num_irqs, smmu->num_global_irqs + 1);  in arm_smmu_device_dt_probe()
1740 	smmu->irqs = devm_kzalloc(dev, sizeof(*smmu->irqs) * num_irqs,  in arm_smmu_device_dt_probe()
1742 	if (!smmu->irqs) {  in arm_smmu_device_dt_probe()
1754 		smmu->irqs[i] = irq;  in arm_smmu_device_dt_probe()
1757 	err = arm_smmu_device_cfg_probe(smmu);  in arm_smmu_device_dt_probe()
1762 	smmu->masters = RB_ROOT;  in arm_smmu_device_dt_probe()
1766 		err = register_smmu_master(smmu, dev, &masterspec);  in arm_smmu_device_dt_probe()
1777 	parse_driver_options(smmu);  in arm_smmu_device_dt_probe()
1779 	if (smmu->version > ARM_SMMU_V1 &&  in arm_smmu_device_dt_probe()
1780 	    smmu->num_context_banks != smmu->num_context_irqs) {  in arm_smmu_device_dt_probe()
1783 			smmu->num_context_irqs, smmu->num_context_banks);  in arm_smmu_device_dt_probe()
1788 	for (i = 0; i < smmu->num_global_irqs; ++i) {  in arm_smmu_device_dt_probe()
1789 		err = request_irq(smmu->irqs[i],  in arm_smmu_device_dt_probe()
1793 				  smmu);  in arm_smmu_device_dt_probe()
1796 				i, smmu->irqs[i]);  in arm_smmu_device_dt_probe()
1801 	INIT_LIST_HEAD(&smmu->list);  in arm_smmu_device_dt_probe()
1803 	list_add(&smmu->list, &arm_smmu_devices);  in arm_smmu_device_dt_probe()
1806 	arm_smmu_device_reset(smmu);  in arm_smmu_device_dt_probe()
1811 		free_irq(smmu->irqs[i], smmu);  in arm_smmu_device_dt_probe()
1814 	for (node = rb_first(&smmu->masters); node; node = rb_next(node)) {  in arm_smmu_device_dt_probe()
1827 	struct arm_smmu_device *curr, *smmu = NULL;  in arm_smmu_device_remove()  local
1833 			smmu = curr;  in arm_smmu_device_remove()
1834 			list_del(&smmu->list);  in arm_smmu_device_remove()
1840 	if (!smmu)  in arm_smmu_device_remove()
1843 	for (node = rb_first(&smmu->masters); node; node = rb_next(node)) {  in arm_smmu_device_remove()
1849 	if (!bitmap_empty(smmu->context_map, ARM_SMMU_MAX_CBS))  in arm_smmu_device_remove()
1852 	for (i = 0; i < smmu->num_global_irqs; ++i)  in arm_smmu_device_remove()
1853 		free_irq(smmu->irqs[i], smmu);  in arm_smmu_device_remove()
1856 	writel(sCR0_CLIENTPD, ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sCR0);  in arm_smmu_device_remove()