60 #define ARM_SMMU_GR0(smmu)		((smmu)->base)  argument
61 #define ARM_SMMU_GR1(smmu)		((smmu)->base + (1 << (smmu)->pgshift))  argument
68 #define ARM_SMMU_GR0_NS(smmu)						\  argument
69 	((smmu)->base +							\
70 		((smmu->options & ARM_SMMU_OPT_SECURE_CFG_ACCESS)	\
194 #define ARM_SMMU_CB_BASE(smmu)		((smmu)->base + ((smmu)->size >> 1))  argument
195 #define ARM_SMMU_CB(smmu, n)		((n) * (1 << (smmu)->pgshift))  argument
343 	struct arm_smmu_device		*smmu;  member
372 static void parse_driver_options(struct arm_smmu_device *smmu)  in parse_driver_options()  argument
377 		if (of_property_read_bool(smmu->dev->of_node,  in parse_driver_options()
379 			smmu->options |= arm_smmu_options[i].opt;  in parse_driver_options()
380 			dev_notice(smmu->dev, "option %s\n",  in parse_driver_options()
399 static struct arm_smmu_master *find_smmu_master(struct arm_smmu_device *smmu,  in find_smmu_master()  argument
402 	struct rb_node *node = smmu->masters.rb_node;  in find_smmu_master()
434 static int insert_smmu_master(struct arm_smmu_device *smmu,  in insert_smmu_master()  argument
439 	new = &smmu->masters.rb_node;  in insert_smmu_master()
455 	rb_insert_color(&master->node, &smmu->masters);  in insert_smmu_master()
459 static int register_smmu_master(struct arm_smmu_device *smmu,  in register_smmu_master()  argument
466 	master = find_smmu_master(smmu, masterspec->np);  in register_smmu_master()
491 		if (!(smmu->features & ARM_SMMU_FEAT_STREAM_MATCH) &&  in register_smmu_master()
492 		     (streamid >= smmu->num_mapping_groups)) {  in register_smmu_master()
495 				masterspec->np->name, smmu->num_mapping_groups);  in register_smmu_master()
500 	return insert_smmu_master(smmu, master);  in register_smmu_master()
505 	struct arm_smmu_device *smmu;  in find_smmu_for_device()  local
510 	list_for_each_entry(smmu, &arm_smmu_devices, list) {  in find_smmu_for_device()
511 		master = find_smmu_master(smmu, dev_node);  in find_smmu_for_device()
517 	return master ? smmu : NULL;  in find_smmu_for_device()
539 static void __arm_smmu_tlb_sync(struct arm_smmu_device *smmu)  in __arm_smmu_tlb_sync()  argument
542 	void __iomem *gr0_base = ARM_SMMU_GR0(smmu);  in __arm_smmu_tlb_sync()
549 			dev_err_ratelimited(smmu->dev,  in __arm_smmu_tlb_sync()
560 	__arm_smmu_tlb_sync(smmu_domain->smmu);  in arm_smmu_tlb_sync()
567 	struct arm_smmu_device *smmu = smmu_domain->smmu;  in arm_smmu_tlb_inv_context()  local
572 		base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx);  in arm_smmu_tlb_inv_context()
576 		base = ARM_SMMU_GR0(smmu);  in arm_smmu_tlb_inv_context()
581 	__arm_smmu_tlb_sync(smmu);  in arm_smmu_tlb_inv_context()
589 	struct arm_smmu_device *smmu = smmu_domain->smmu;  in arm_smmu_tlb_inv_range_nosync()  local
594 		reg = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx);  in arm_smmu_tlb_inv_range_nosync()
597 		if (!IS_ENABLED(CONFIG_64BIT) || smmu->version == ARM_SMMU_V1) {  in arm_smmu_tlb_inv_range_nosync()
609 	} else if (smmu->version == ARM_SMMU_V2) {  in arm_smmu_tlb_inv_range_nosync()
610 		reg = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx);  in arm_smmu_tlb_inv_range_nosync()
616 		reg = ARM_SMMU_GR0(smmu) + ARM_SMMU_GR0_TLBIVMID;  in arm_smmu_tlb_inv_range_nosync()
635 	struct arm_smmu_device *smmu = smmu_domain->smmu;  in arm_smmu_context_fault()  local
638 	cb_base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx);  in arm_smmu_context_fault()
645 		dev_err_ratelimited(smmu->dev,  in arm_smmu_context_fault()
659 	if (!report_iommu_fault(domain, smmu->dev, iova, flags)) {  in arm_smmu_context_fault()
663 		dev_err_ratelimited(smmu->dev,  in arm_smmu_context_fault()
683 	struct arm_smmu_device *smmu = dev;  in arm_smmu_global_fault()  local
684 	void __iomem *gr0_base = ARM_SMMU_GR0_NS(smmu);  in arm_smmu_global_fault()
694 	dev_err_ratelimited(smmu->dev,  in arm_smmu_global_fault()
696 	dev_err_ratelimited(smmu->dev,  in arm_smmu_global_fault()
711 	struct arm_smmu_device *smmu = smmu_domain->smmu;  in arm_smmu_init_context_bank()  local
714 	gr1_base = ARM_SMMU_GR1(smmu);  in arm_smmu_init_context_bank()
716 	cb_base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx);  in arm_smmu_init_context_bank()
718 	if (smmu->version > ARM_SMMU_V1) {  in arm_smmu_init_context_bank()
734 	if (smmu->version == ARM_SMMU_V1)  in arm_smmu_init_context_bank()
768 		if (smmu->version > ARM_SMMU_V1) {  in arm_smmu_init_context_bank()
797 					struct arm_smmu_device *smmu)  in arm_smmu_init_domain_context()  argument
808 	if (smmu_domain->smmu)  in arm_smmu_init_domain_context()
829 	if (!(smmu->features & ARM_SMMU_FEAT_TRANS_S1))  in arm_smmu_init_domain_context()
831 	if (!(smmu->features & ARM_SMMU_FEAT_TRANS_S2))  in arm_smmu_init_domain_context()
837 		start = smmu->num_s2_context_banks;  in arm_smmu_init_domain_context()
838 		ias = smmu->va_size;  in arm_smmu_init_domain_context()
839 		oas = smmu->ipa_size;  in arm_smmu_init_domain_context()
853 		ias = smmu->ipa_size;  in arm_smmu_init_domain_context()
854 		oas = smmu->pa_size;  in arm_smmu_init_domain_context()
865 	ret = __arm_smmu_alloc_bitmap(smmu->context_map, start,  in arm_smmu_init_domain_context()
866 				      smmu->num_context_banks);  in arm_smmu_init_domain_context()
871 	if (smmu->version == ARM_SMMU_V1) {  in arm_smmu_init_domain_context()
872 		cfg->irptndx = atomic_inc_return(&smmu->irptndx);  in arm_smmu_init_domain_context()
873 		cfg->irptndx %= smmu->num_context_irqs;  in arm_smmu_init_domain_context()
883 		.iommu_dev	= smmu->dev,  in arm_smmu_init_domain_context()
886 	smmu_domain->smmu = smmu;  in arm_smmu_init_domain_context()
903 	irq = smmu->irqs[smmu->num_global_irqs + cfg->irptndx];  in arm_smmu_init_domain_context()
907 		dev_err(smmu->dev, "failed to request context IRQ %d (%u)\n",  in arm_smmu_init_domain_context()
919 	smmu_domain->smmu = NULL;  in arm_smmu_init_domain_context()
928 	struct arm_smmu_device *smmu = smmu_domain->smmu;  in arm_smmu_destroy_domain_context()  local
933 	if (!smmu)  in arm_smmu_destroy_domain_context()
940 	cb_base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx);  in arm_smmu_destroy_domain_context()
944 		irq = smmu->irqs[smmu->num_global_irqs + cfg->irptndx];  in arm_smmu_destroy_domain_context()
951 	__arm_smmu_free_bitmap(smmu->context_map, cfg->cbndx);  in arm_smmu_destroy_domain_context()
987 static int arm_smmu_master_configure_smrs(struct arm_smmu_device *smmu,  in arm_smmu_master_configure_smrs()  argument
992 	void __iomem *gr0_base = ARM_SMMU_GR0(smmu);  in arm_smmu_master_configure_smrs()
994 	if (!(smmu->features & ARM_SMMU_FEAT_STREAM_MATCH))  in arm_smmu_master_configure_smrs()
1002 		dev_err(smmu->dev, "failed to allocate %d SMRs\n",  in arm_smmu_master_configure_smrs()
1009 		int idx = __arm_smmu_alloc_bitmap(smmu->smr_map, 0,  in arm_smmu_master_configure_smrs()
1010 						  smmu->num_mapping_groups);  in arm_smmu_master_configure_smrs()
1012 			dev_err(smmu->dev, "failed to allocate free SMR\n");  in arm_smmu_master_configure_smrs()
1035 		__arm_smmu_free_bitmap(smmu->smr_map, smrs[i].idx);  in arm_smmu_master_configure_smrs()
1040 static void arm_smmu_master_free_smrs(struct arm_smmu_device *smmu,  in arm_smmu_master_free_smrs()  argument
1044 	void __iomem *gr0_base = ARM_SMMU_GR0(smmu);  in arm_smmu_master_free_smrs()
1055 		__arm_smmu_free_bitmap(smmu->smr_map, idx);  in arm_smmu_master_free_smrs()
1066 	struct arm_smmu_device *smmu = smmu_domain->smmu;  in arm_smmu_domain_add_master()  local
1067 	void __iomem *gr0_base = ARM_SMMU_GR0(smmu);  in arm_smmu_domain_add_master()
1070 	ret = arm_smmu_master_configure_smrs(smmu, cfg);  in arm_smmu_domain_add_master()
1090 	struct arm_smmu_device *smmu = smmu_domain->smmu;  in arm_smmu_domain_remove_master()  local
1091 	void __iomem *gr0_base = ARM_SMMU_GR0(smmu);  in arm_smmu_domain_remove_master()
1094 	if ((smmu->features & ARM_SMMU_FEAT_STREAM_MATCH) && !cfg->smrs)  in arm_smmu_domain_remove_master()
1108 	arm_smmu_master_free_smrs(smmu, cfg);  in arm_smmu_domain_remove_master()
1115 	struct arm_smmu_device *smmu;  in arm_smmu_attach_dev()  local
1118 	smmu = find_smmu_for_device(dev);  in arm_smmu_attach_dev()
1119 	if (!smmu) {  in arm_smmu_attach_dev()
1130 	ret = arm_smmu_init_domain_context(domain, smmu);  in arm_smmu_attach_dev()
1138 	if (smmu_domain->smmu != smmu) {  in arm_smmu_attach_dev()
1141 			dev_name(smmu_domain->smmu->dev), dev_name(smmu->dev));  in arm_smmu_attach_dev()
1207 	struct arm_smmu_device *smmu = smmu_domain->smmu;  in arm_smmu_iova_to_phys_hard()  local
1210 	struct device *dev = smmu->dev;  in arm_smmu_iova_to_phys_hard()
1216 	cb_base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx);  in arm_smmu_iova_to_phys_hard()
1220 	if (smmu->version == ARM_SMMU_V2)  in arm_smmu_iova_to_phys_hard()
1257 	if (smmu_domain->smmu->features & ARM_SMMU_FEAT_TRANS_OPS &&  in arm_smmu_iova_to_phys()
1337 	struct arm_smmu_device *smmu = find_smmu_for_device(dev);  in arm_smmu_init_platform_device()  local
1340 	if (!smmu)  in arm_smmu_init_platform_device()
1343 	master = find_smmu_master(smmu, dev->of_node);  in arm_smmu_init_platform_device()
1418 		if (smmu_domain->smmu) {  in arm_smmu_domain_set_attr()
1456 static void arm_smmu_device_reset(struct arm_smmu_device *smmu)  in arm_smmu_device_reset()  argument
1458 	void __iomem *gr0_base = ARM_SMMU_GR0(smmu);  in arm_smmu_device_reset()
1464 	reg = readl_relaxed(ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sGFSR);  in arm_smmu_device_reset()
1465 	writel(reg, ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sGFSR);  in arm_smmu_device_reset()
1468 	for (i = 0; i < smmu->num_mapping_groups; ++i) {  in arm_smmu_device_reset()
1475 	for (i = 0; i < smmu->num_context_banks; ++i) {  in arm_smmu_device_reset()
1476 		cb_base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, i);  in arm_smmu_device_reset()
1485 	reg = readl_relaxed(ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sCR0);  in arm_smmu_device_reset()
1503 	__arm_smmu_tlb_sync(smmu);  in arm_smmu_device_reset()
1504 	writel(reg, ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sCR0);  in arm_smmu_device_reset()
1526 static int arm_smmu_device_cfg_probe(struct arm_smmu_device *smmu)  in arm_smmu_device_cfg_probe()  argument
1529 	void __iomem *gr0_base = ARM_SMMU_GR0(smmu);  in arm_smmu_device_cfg_probe()
1533 	dev_notice(smmu->dev, "probing hardware configuration...\n");  in arm_smmu_device_cfg_probe()
1534 	dev_notice(smmu->dev, "SMMUv%d with:\n", smmu->version);  in arm_smmu_device_cfg_probe()
1546 		smmu->features |= ARM_SMMU_FEAT_TRANS_S1;  in arm_smmu_device_cfg_probe()
1547 		dev_notice(smmu->dev, "\tstage 1 translation\n");  in arm_smmu_device_cfg_probe()
1551 		smmu->features |= ARM_SMMU_FEAT_TRANS_S2;  in arm_smmu_device_cfg_probe()
1552 		dev_notice(smmu->dev, "\tstage 2 translation\n");  in arm_smmu_device_cfg_probe()
1556 		smmu->features |= ARM_SMMU_FEAT_TRANS_NESTED;  in arm_smmu_device_cfg_probe()
1557 		dev_notice(smmu->dev, "\tnested translation\n");  in arm_smmu_device_cfg_probe()
1560 	if (!(smmu->features &  in arm_smmu_device_cfg_probe()
1562 		dev_err(smmu->dev, "\tno translation support!\n");  in arm_smmu_device_cfg_probe()
1566 	if ((id & ID0_S1TS) && ((smmu->version == 1) || !(id & ID0_ATOSNS))) {  in arm_smmu_device_cfg_probe()
1567 		smmu->features |= ARM_SMMU_FEAT_TRANS_OPS;  in arm_smmu_device_cfg_probe()
1568 		dev_notice(smmu->dev, "\taddress translation ops\n");  in arm_smmu_device_cfg_probe()
1577 	cttw_dt = of_dma_is_coherent(smmu->dev->of_node);  in arm_smmu_device_cfg_probe()
1580 		smmu->features |= ARM_SMMU_FEAT_COHERENT_WALK;  in arm_smmu_device_cfg_probe()
1582 		dev_notice(smmu->dev, "\t%scoherent table walk\n",  in arm_smmu_device_cfg_probe()
1585 		dev_notice(smmu->dev,  in arm_smmu_device_cfg_probe()
1591 		smmu->features |= ARM_SMMU_FEAT_STREAM_MATCH;  in arm_smmu_device_cfg_probe()
1592 		smmu->num_mapping_groups = (id >> ID0_NUMSMRG_SHIFT) &  in arm_smmu_device_cfg_probe()
1594 		if (smmu->num_mapping_groups == 0) {  in arm_smmu_device_cfg_probe()
1595 			dev_err(smmu->dev,  in arm_smmu_device_cfg_probe()
1608 			dev_err(smmu->dev,  in arm_smmu_device_cfg_probe()
1614 		dev_notice(smmu->dev,  in arm_smmu_device_cfg_probe()
1616 			   smmu->num_mapping_groups, mask);  in arm_smmu_device_cfg_probe()
1618 		smmu->num_mapping_groups = (id >> ID0_NUMSIDB_SHIFT) &  in arm_smmu_device_cfg_probe()
1624 	smmu->pgshift = (id & ID1_PAGESIZE) ? 16 : 12;  in arm_smmu_device_cfg_probe()
1628 	size *= 2 << smmu->pgshift;  in arm_smmu_device_cfg_probe()
1629 	if (smmu->size != size)  in arm_smmu_device_cfg_probe()
1630 		dev_warn(smmu->dev,  in arm_smmu_device_cfg_probe()
1632 			size, smmu->size);  in arm_smmu_device_cfg_probe()
1634 	smmu->num_s2_context_banks = (id >> ID1_NUMS2CB_SHIFT) & ID1_NUMS2CB_MASK;  in arm_smmu_device_cfg_probe()
1635 	smmu->num_context_banks = (id >> ID1_NUMCB_SHIFT) & ID1_NUMCB_MASK;  in arm_smmu_device_cfg_probe()
1636 	if (smmu->num_s2_context_banks > smmu->num_context_banks) {  in arm_smmu_device_cfg_probe()
1637 		dev_err(smmu->dev, "impossible number of S2 context banks!\n");  in arm_smmu_device_cfg_probe()
1640 	dev_notice(smmu->dev, "\t%u context banks (%u stage-2 only)\n",  in arm_smmu_device_cfg_probe()
1641 		   smmu->num_context_banks, smmu->num_s2_context_banks);  in arm_smmu_device_cfg_probe()
1646 	smmu->ipa_size = size;  in arm_smmu_device_cfg_probe()
1650 	smmu->pa_size = size;  in arm_smmu_device_cfg_probe()
1657 	if (dma_set_mask_and_coherent(smmu->dev, DMA_BIT_MASK(size)))  in arm_smmu_device_cfg_probe()
1658 		dev_warn(smmu->dev,  in arm_smmu_device_cfg_probe()
1661 	if (smmu->version == ARM_SMMU_V1) {  in arm_smmu_device_cfg_probe()
1662 		smmu->va_size = smmu->ipa_size;  in arm_smmu_device_cfg_probe()
1666 		smmu->va_size = arm_smmu_id_size_to_bits(size);  in arm_smmu_device_cfg_probe()
1668 		smmu->va_size = min(32UL, smmu->va_size);  in arm_smmu_device_cfg_probe()
1680 	dev_notice(smmu->dev, "\tSupported page sizes: 0x%08lx\n", size);  in arm_smmu_device_cfg_probe()
1682 	if (smmu->features & ARM_SMMU_FEAT_TRANS_S1)  in arm_smmu_device_cfg_probe()
1683 		dev_notice(smmu->dev, "\tStage-1: %lu-bit VA -> %lu-bit IPA\n",  in arm_smmu_device_cfg_probe()
1684 			   smmu->va_size, smmu->ipa_size);  in arm_smmu_device_cfg_probe()
1686 	if (smmu->features & ARM_SMMU_FEAT_TRANS_S2)  in arm_smmu_device_cfg_probe()
1687 		dev_notice(smmu->dev, "\tStage-2: %lu-bit IPA -> %lu-bit PA\n",  in arm_smmu_device_cfg_probe()
1688 			   smmu->ipa_size, smmu->pa_size);  in arm_smmu_device_cfg_probe()
1707 	struct arm_smmu_device *smmu;  in arm_smmu_device_dt_probe()  local
1713 	smmu = devm_kzalloc(dev, sizeof(*smmu), GFP_KERNEL);  in arm_smmu_device_dt_probe()
1714 	if (!smmu) {  in arm_smmu_device_dt_probe()
1718 	smmu->dev = dev;  in arm_smmu_device_dt_probe()
1721 	smmu->version = (enum arm_smmu_arch_version)of_id->data;  in arm_smmu_device_dt_probe()
1724 	smmu->base = devm_ioremap_resource(dev, res);  in arm_smmu_device_dt_probe()
1725 	if (IS_ERR(smmu->base))  in arm_smmu_device_dt_probe()
1726 		return PTR_ERR(smmu->base);  in arm_smmu_device_dt_probe()
1727 	smmu->size = resource_size(res);  in arm_smmu_device_dt_probe()
1730 				 &smmu->num_global_irqs)) {  in arm_smmu_device_dt_probe()
1738 		if (num_irqs > smmu->num_global_irqs)  in arm_smmu_device_dt_probe()
1739 			smmu->num_context_irqs++;  in arm_smmu_device_dt_probe()
1742 	if (!smmu->num_context_irqs) {  in arm_smmu_device_dt_probe()
1744 			num_irqs, smmu->num_global_irqs + 1);  in arm_smmu_device_dt_probe()
1748 	smmu->irqs = devm_kzalloc(dev, sizeof(*smmu->irqs) * num_irqs,  in arm_smmu_device_dt_probe()
1750 	if (!smmu->irqs) {  in arm_smmu_device_dt_probe()
1762 		smmu->irqs[i] = irq;  in arm_smmu_device_dt_probe()
1765 	err = arm_smmu_device_cfg_probe(smmu);  in arm_smmu_device_dt_probe()
1770 	smmu->masters = RB_ROOT;  in arm_smmu_device_dt_probe()
1774 		err = register_smmu_master(smmu, dev, &masterspec);  in arm_smmu_device_dt_probe()
1785 	parse_driver_options(smmu);  in arm_smmu_device_dt_probe()
1787 	if (smmu->version > ARM_SMMU_V1 &&  in arm_smmu_device_dt_probe()
1788 	    smmu->num_context_banks != smmu->num_context_irqs) {  in arm_smmu_device_dt_probe()
1791 			smmu->num_context_irqs, smmu->num_context_banks);  in arm_smmu_device_dt_probe()
1796 	for (i = 0; i < smmu->num_global_irqs; ++i) {  in arm_smmu_device_dt_probe()
1797 		err = request_irq(smmu->irqs[i],  in arm_smmu_device_dt_probe()
1801 				  smmu);  in arm_smmu_device_dt_probe()
1804 				i, smmu->irqs[i]);  in arm_smmu_device_dt_probe()
1809 	INIT_LIST_HEAD(&smmu->list);  in arm_smmu_device_dt_probe()
1811 	list_add(&smmu->list, &arm_smmu_devices);  in arm_smmu_device_dt_probe()
1814 	arm_smmu_device_reset(smmu);  in arm_smmu_device_dt_probe()
1819 		free_irq(smmu->irqs[i], smmu);  in arm_smmu_device_dt_probe()
1822 	for (node = rb_first(&smmu->masters); node; node = rb_next(node)) {  in arm_smmu_device_dt_probe()
1835 	struct arm_smmu_device *curr, *smmu = NULL;  in arm_smmu_device_remove()  local
1841 			smmu = curr;  in arm_smmu_device_remove()
1842 			list_del(&smmu->list);  in arm_smmu_device_remove()
1848 	if (!smmu)  in arm_smmu_device_remove()
1851 	for (node = rb_first(&smmu->masters); node; node = rb_next(node)) {  in arm_smmu_device_remove()
1857 	if (!bitmap_empty(smmu->context_map, ARM_SMMU_MAX_CBS))  in arm_smmu_device_remove()
1860 	for (i = 0; i < smmu->num_global_irqs; ++i)  in arm_smmu_device_remove()
1861 		free_irq(smmu->irqs[i], smmu);  in arm_smmu_device_remove()
1864 	writel(sCR0_CLIENTPD, ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sCR0);  in arm_smmu_device_remove()