42 	struct tegra_smmu *smmu;  member
57 static inline void smmu_writel(struct tegra_smmu *smmu, u32 value,  in smmu_writel()  argument
60 	writel(value, smmu->regs + offset);  in smmu_writel()
63 static inline u32 smmu_readl(struct tegra_smmu *smmu, unsigned long offset)  in smmu_readl()  argument
65 	return readl(smmu->regs + offset);  in smmu_readl()
74 #define  SMMU_TLB_CONFIG_ACTIVE_LINES(smmu) \  argument
75 	((smmu)->soc->num_tlb_lines & (smmu)->tlb_mask)
151 static bool smmu_dma_addr_valid(struct tegra_smmu *smmu, dma_addr_t addr)  in smmu_dma_addr_valid()  argument
154 	return (addr & smmu->pfn_mask) == addr;  in smmu_dma_addr_valid()
162 static void smmu_flush_ptc_all(struct tegra_smmu *smmu)  in smmu_flush_ptc_all()  argument
164 	smmu_writel(smmu, SMMU_PTC_FLUSH_TYPE_ALL, SMMU_PTC_FLUSH);  in smmu_flush_ptc_all()
167 static inline void smmu_flush_ptc(struct tegra_smmu *smmu, dma_addr_t dma,  in smmu_flush_ptc()  argument
172 	offset &= ~(smmu->mc->soc->atom_size - 1);  in smmu_flush_ptc()
174 	if (smmu->mc->soc->num_address_bits > 32) {  in smmu_flush_ptc()
180 		smmu_writel(smmu, value, SMMU_PTC_FLUSH_HI);  in smmu_flush_ptc()
184 	smmu_writel(smmu, value, SMMU_PTC_FLUSH);  in smmu_flush_ptc()
187 static inline void smmu_flush_tlb(struct tegra_smmu *smmu)  in smmu_flush_tlb()  argument
189 	smmu_writel(smmu, SMMU_TLB_FLUSH_VA_MATCH_ALL, SMMU_TLB_FLUSH);  in smmu_flush_tlb()
192 static inline void smmu_flush_tlb_asid(struct tegra_smmu *smmu,  in smmu_flush_tlb_asid()  argument
199 	smmu_writel(smmu, value, SMMU_TLB_FLUSH);  in smmu_flush_tlb_asid()
202 static inline void smmu_flush_tlb_section(struct tegra_smmu *smmu,  in smmu_flush_tlb_section()  argument
210 	smmu_writel(smmu, value, SMMU_TLB_FLUSH);  in smmu_flush_tlb_section()
213 static inline void smmu_flush_tlb_group(struct tegra_smmu *smmu,  in smmu_flush_tlb_group()  argument
221 	smmu_writel(smmu, value, SMMU_TLB_FLUSH);  in smmu_flush_tlb_group()
224 static inline void smmu_flush(struct tegra_smmu *smmu)  in smmu_flush()  argument
226 	smmu_readl(smmu, SMMU_CONFIG);  in smmu_flush()
229 static int tegra_smmu_alloc_asid(struct tegra_smmu *smmu, unsigned int *idp)  in tegra_smmu_alloc_asid()  argument
233 	mutex_lock(&smmu->lock);  in tegra_smmu_alloc_asid()
235 	id = find_first_zero_bit(smmu->asids, smmu->soc->num_asids);  in tegra_smmu_alloc_asid()
236 	if (id >= smmu->soc->num_asids) {  in tegra_smmu_alloc_asid()
237 		mutex_unlock(&smmu->lock);  in tegra_smmu_alloc_asid()
241 	set_bit(id, smmu->asids);  in tegra_smmu_alloc_asid()
244 	mutex_unlock(&smmu->lock);  in tegra_smmu_alloc_asid()
248 static void tegra_smmu_free_asid(struct tegra_smmu *smmu, unsigned int id)  in tegra_smmu_free_asid()  argument
250 	mutex_lock(&smmu->lock);  in tegra_smmu_free_asid()
251 	clear_bit(id, smmu->asids);  in tegra_smmu_free_asid()
252 	mutex_unlock(&smmu->lock);  in tegra_smmu_free_asid()
312 tegra_smmu_find_swgroup(struct tegra_smmu *smmu, unsigned int swgroup)  in tegra_smmu_find_swgroup()  argument
317 	for (i = 0; i < smmu->soc->num_swgroups; i++) {  in tegra_smmu_find_swgroup()
318 		if (smmu->soc->swgroups[i].swgroup == swgroup) {  in tegra_smmu_find_swgroup()
319 			group = &smmu->soc->swgroups[i];  in tegra_smmu_find_swgroup()
327 static void tegra_smmu_enable(struct tegra_smmu *smmu, unsigned int swgroup,  in tegra_smmu_enable()  argument
334 	for (i = 0; i < smmu->soc->num_clients; i++) {  in tegra_smmu_enable()
335 		const struct tegra_mc_client *client = &smmu->soc->clients[i];  in tegra_smmu_enable()
340 		value = smmu_readl(smmu, client->smmu.reg);  in tegra_smmu_enable()
341 		value |= BIT(client->smmu.bit);  in tegra_smmu_enable()
342 		smmu_writel(smmu, value, client->smmu.reg);  in tegra_smmu_enable()
345 	group = tegra_smmu_find_swgroup(smmu, swgroup);  in tegra_smmu_enable()
347 		value = smmu_readl(smmu, group->reg);  in tegra_smmu_enable()
351 		smmu_writel(smmu, value, group->reg);  in tegra_smmu_enable()
355 static void tegra_smmu_disable(struct tegra_smmu *smmu, unsigned int swgroup,  in tegra_smmu_disable()  argument
362 	group = tegra_smmu_find_swgroup(smmu, swgroup);  in tegra_smmu_disable()
364 		value = smmu_readl(smmu, group->reg);  in tegra_smmu_disable()
368 		smmu_writel(smmu, value, group->reg);  in tegra_smmu_disable()
371 	for (i = 0; i < smmu->soc->num_clients; i++) {  in tegra_smmu_disable()
372 		const struct tegra_mc_client *client = &smmu->soc->clients[i];  in tegra_smmu_disable()
377 		value = smmu_readl(smmu, client->smmu.reg);  in tegra_smmu_disable()
378 		value &= ~BIT(client->smmu.bit);  in tegra_smmu_disable()
379 		smmu_writel(smmu, value, client->smmu.reg);  in tegra_smmu_disable()
383 static int tegra_smmu_as_prepare(struct tegra_smmu *smmu,  in tegra_smmu_as_prepare()  argument
394 	as->pd_dma = dma_map_page(smmu->dev, as->pd, 0, SMMU_SIZE_PD,  in tegra_smmu_as_prepare()
396 	if (dma_mapping_error(smmu->dev, as->pd_dma))  in tegra_smmu_as_prepare()
400 	if (!smmu_dma_addr_valid(smmu, as->pd_dma)) {  in tegra_smmu_as_prepare()
405 	err = tegra_smmu_alloc_asid(smmu, &as->id);  in tegra_smmu_as_prepare()
409 	smmu_flush_ptc(smmu, as->pd_dma, 0);  in tegra_smmu_as_prepare()
410 	smmu_flush_tlb_asid(smmu, as->id);  in tegra_smmu_as_prepare()
412 	smmu_writel(smmu, as->id & 0x7f, SMMU_PTB_ASID);  in tegra_smmu_as_prepare()
414 	smmu_writel(smmu, value, SMMU_PTB_DATA);  in tegra_smmu_as_prepare()
415 	smmu_flush(smmu);  in tegra_smmu_as_prepare()
417 	as->smmu = smmu;  in tegra_smmu_as_prepare()
423 	dma_unmap_page(smmu->dev, as->pd_dma, SMMU_SIZE_PD, DMA_TO_DEVICE);  in tegra_smmu_as_prepare()
427 static void tegra_smmu_as_unprepare(struct tegra_smmu *smmu,  in tegra_smmu_as_unprepare()  argument
433 	tegra_smmu_free_asid(smmu, as->id);  in tegra_smmu_as_unprepare()
435 	dma_unmap_page(smmu->dev, as->pd_dma, SMMU_SIZE_PD, DMA_TO_DEVICE);  in tegra_smmu_as_unprepare()
437 	as->smmu = NULL;  in tegra_smmu_as_unprepare()
443 	struct tegra_smmu *smmu = dev->archdata.iommu;  in tegra_smmu_attach_dev()  local
454 		if (args.np != smmu->dev->of_node) {  in tegra_smmu_attach_dev()
461 		err = tegra_smmu_as_prepare(smmu, as);  in tegra_smmu_attach_dev()
465 		tegra_smmu_enable(smmu, swgroup, as->id);  in tegra_smmu_attach_dev()
479 	struct tegra_smmu *smmu = as->smmu;  in tegra_smmu_detach_dev()  local
487 		if (args.np != smmu->dev->of_node) {  in tegra_smmu_detach_dev()
494 		tegra_smmu_disable(smmu, swgroup, as->id);  in tegra_smmu_detach_dev()
495 		tegra_smmu_as_unprepare(smmu, as);  in tegra_smmu_detach_dev()
504 	struct tegra_smmu *smmu = as->smmu;  in tegra_smmu_set_pde()  local
512 	dma_sync_single_range_for_device(smmu->dev, as->pd_dma, offset,  in tegra_smmu_set_pde()
516 	smmu_flush_ptc(smmu, as->pd_dma, offset);  in tegra_smmu_set_pde()
517 	smmu_flush_tlb_section(smmu, as->id, iova);  in tegra_smmu_set_pde()
518 	smmu_flush(smmu);  in tegra_smmu_set_pde()
549 	struct tegra_smmu *smmu = as->smmu;  in as_get_pte()  local
559 		dma = dma_map_page(smmu->dev, page, 0, SMMU_SIZE_PT,  in as_get_pte()
561 		if (dma_mapping_error(smmu->dev, dma)) {  in as_get_pte()
566 		if (!smmu_dma_addr_valid(smmu, dma)) {  in as_get_pte()
567 			dma_unmap_page(smmu->dev, dma, SMMU_SIZE_PT,  in as_get_pte()
605 		struct tegra_smmu *smmu = as->smmu;  in tegra_smmu_pte_put_use()  local
611 		dma_unmap_page(smmu->dev, pte_dma, SMMU_SIZE_PT, DMA_TO_DEVICE);  in tegra_smmu_pte_put_use()
620 	struct tegra_smmu *smmu = as->smmu;  in tegra_smmu_set_pte()  local
625 	dma_sync_single_range_for_device(smmu->dev, pte_dma, offset,  in tegra_smmu_set_pte()
627 	smmu_flush_ptc(smmu, pte_dma, offset);  in tegra_smmu_set_pte()
628 	smmu_flush_tlb_group(smmu, as->id, iova);  in tegra_smmu_set_pte()
629 	smmu_flush(smmu);  in tegra_smmu_set_pte()
682 	pfn = *pte & as->smmu->pfn_mask;  in tegra_smmu_iova_to_phys()
700 	return mc->smmu;  in tegra_smmu_find()
711 		struct tegra_smmu *smmu;  in tegra_smmu_add_device()  local
713 		smmu = tegra_smmu_find(args.np);  in tegra_smmu_add_device()
714 		if (smmu) {  in tegra_smmu_add_device()
720 			dev->archdata.iommu = smmu;  in tegra_smmu_add_device()
768 	struct tegra_smmu *smmu = s->private;  in tegra_smmu_swgroups_show()  local
775 	for (i = 0; i < smmu->soc->num_swgroups; i++) {  in tegra_smmu_swgroups_show()
776 		const struct tegra_smmu_swgroup *group = &smmu->soc->swgroups[i];  in tegra_smmu_swgroups_show()
780 		value = smmu_readl(smmu, group->reg);  in tegra_smmu_swgroups_show()
810 	struct tegra_smmu *smmu = s->private;  in tegra_smmu_clients_show()  local
817 	for (i = 0; i < smmu->soc->num_clients; i++) {  in tegra_smmu_clients_show()
818 		const struct tegra_mc_client *client = &smmu->soc->clients[i];  in tegra_smmu_clients_show()
821 		value = smmu_readl(smmu, client->smmu.reg);  in tegra_smmu_clients_show()
823 		if (value & BIT(client->smmu.bit))  in tegra_smmu_clients_show()
846 static void tegra_smmu_debugfs_init(struct tegra_smmu *smmu)  in tegra_smmu_debugfs_init()  argument
848 	smmu->debugfs = debugfs_create_dir("smmu", NULL);  in tegra_smmu_debugfs_init()
849 	if (!smmu->debugfs)  in tegra_smmu_debugfs_init()
852 	debugfs_create_file("swgroups", S_IRUGO, smmu->debugfs, smmu,  in tegra_smmu_debugfs_init()
854 	debugfs_create_file("clients", S_IRUGO, smmu->debugfs, smmu,  in tegra_smmu_debugfs_init()
858 static void tegra_smmu_debugfs_exit(struct tegra_smmu *smmu)  in tegra_smmu_debugfs_exit()  argument
860 	debugfs_remove_recursive(smmu->debugfs);  in tegra_smmu_debugfs_exit()
867 	struct tegra_smmu *smmu;  in tegra_smmu_probe()  local
876 	smmu = devm_kzalloc(dev, sizeof(*smmu), GFP_KERNEL);  in tegra_smmu_probe()
877 	if (!smmu)  in tegra_smmu_probe()
888 	mc->smmu = smmu;  in tegra_smmu_probe()
892 	smmu->asids = devm_kzalloc(dev, size, GFP_KERNEL);  in tegra_smmu_probe()
893 	if (!smmu->asids)  in tegra_smmu_probe()
896 	mutex_init(&smmu->lock);  in tegra_smmu_probe()
898 	smmu->regs = mc->regs;  in tegra_smmu_probe()
899 	smmu->soc = soc;  in tegra_smmu_probe()
900 	smmu->dev = dev;  in tegra_smmu_probe()
901 	smmu->mc = mc;  in tegra_smmu_probe()
903 	smmu->pfn_mask = BIT_MASK(mc->soc->num_address_bits - PAGE_SHIFT) - 1;  in tegra_smmu_probe()
905 		mc->soc->num_address_bits, smmu->pfn_mask);  in tegra_smmu_probe()
906 	smmu->tlb_mask = (smmu->soc->num_tlb_lines << 1) - 1;  in tegra_smmu_probe()
907 	dev_dbg(dev, "TLB lines: %u, mask: %#lx\n", smmu->soc->num_tlb_lines,  in tegra_smmu_probe()
908 		smmu->tlb_mask);  in tegra_smmu_probe()
915 	smmu_writel(smmu, value, SMMU_PTC_CONFIG);  in tegra_smmu_probe()
918 		SMMU_TLB_CONFIG_ACTIVE_LINES(smmu);  in tegra_smmu_probe()
923 	smmu_writel(smmu, value, SMMU_TLB_CONFIG);  in tegra_smmu_probe()
925 	smmu_flush_ptc_all(smmu);  in tegra_smmu_probe()
926 	smmu_flush_tlb(smmu);  in tegra_smmu_probe()
927 	smmu_writel(smmu, SMMU_CONFIG_ENABLE, SMMU_CONFIG);  in tegra_smmu_probe()
928 	smmu_flush(smmu);  in tegra_smmu_probe()
937 		tegra_smmu_debugfs_init(smmu);  in tegra_smmu_probe()
939 	return smmu;  in tegra_smmu_probe()
942 void tegra_smmu_remove(struct tegra_smmu *smmu)  in tegra_smmu_remove()  argument
945 		tegra_smmu_debugfs_exit(smmu);  in tegra_smmu_remove()