39 	struct tegra_smmu *smmu;  member
52 static inline void smmu_writel(struct tegra_smmu *smmu, u32 value,  in smmu_writel()  argument
55 	writel(value, smmu->regs + offset);  in smmu_writel()
58 static inline u32 smmu_readl(struct tegra_smmu *smmu, unsigned long offset)  in smmu_readl()  argument
60 	return readl(smmu->regs + offset);  in smmu_readl()
69 #define  SMMU_TLB_CONFIG_ACTIVE_LINES(smmu) \  argument
70 	((smmu)->soc->num_tlb_lines & (smmu)->tlb_mask)
136 static inline void smmu_flush_ptc(struct tegra_smmu *smmu, struct page *page,  in smmu_flush_ptc()  argument
143 		offset &= ~(smmu->mc->soc->atom_size - 1);  in smmu_flush_ptc()
145 		if (smmu->mc->soc->num_address_bits > 32) {  in smmu_flush_ptc()
151 			smmu_writel(smmu, value, SMMU_PTC_FLUSH_HI);  in smmu_flush_ptc()
159 	smmu_writel(smmu, value, SMMU_PTC_FLUSH);  in smmu_flush_ptc()
162 static inline void smmu_flush_tlb(struct tegra_smmu *smmu)  in smmu_flush_tlb()  argument
164 	smmu_writel(smmu, SMMU_TLB_FLUSH_VA_MATCH_ALL, SMMU_TLB_FLUSH);  in smmu_flush_tlb()
167 static inline void smmu_flush_tlb_asid(struct tegra_smmu *smmu,  in smmu_flush_tlb_asid()  argument
174 	smmu_writel(smmu, value, SMMU_TLB_FLUSH);  in smmu_flush_tlb_asid()
177 static inline void smmu_flush_tlb_section(struct tegra_smmu *smmu,  in smmu_flush_tlb_section()  argument
185 	smmu_writel(smmu, value, SMMU_TLB_FLUSH);  in smmu_flush_tlb_section()
188 static inline void smmu_flush_tlb_group(struct tegra_smmu *smmu,  in smmu_flush_tlb_group()  argument
196 	smmu_writel(smmu, value, SMMU_TLB_FLUSH);  in smmu_flush_tlb_group()
199 static inline void smmu_flush(struct tegra_smmu *smmu)  in smmu_flush()  argument
201 	smmu_readl(smmu, SMMU_CONFIG);  in smmu_flush()
204 static int tegra_smmu_alloc_asid(struct tegra_smmu *smmu, unsigned int *idp)  in tegra_smmu_alloc_asid()  argument
208 	mutex_lock(&smmu->lock);  in tegra_smmu_alloc_asid()
210 	id = find_first_zero_bit(smmu->asids, smmu->soc->num_asids);  in tegra_smmu_alloc_asid()
211 	if (id >= smmu->soc->num_asids) {  in tegra_smmu_alloc_asid()
212 		mutex_unlock(&smmu->lock);  in tegra_smmu_alloc_asid()
216 	set_bit(id, smmu->asids);  in tegra_smmu_alloc_asid()
219 	mutex_unlock(&smmu->lock);  in tegra_smmu_alloc_asid()
223 static void tegra_smmu_free_asid(struct tegra_smmu *smmu, unsigned int id)  in tegra_smmu_free_asid()  argument
225 	mutex_lock(&smmu->lock);  in tegra_smmu_free_asid()
226 	clear_bit(id, smmu->asids);  in tegra_smmu_free_asid()
227 	mutex_unlock(&smmu->lock);  in tegra_smmu_free_asid()
296 tegra_smmu_find_swgroup(struct tegra_smmu *smmu, unsigned int swgroup)  in tegra_smmu_find_swgroup()  argument
301 	for (i = 0; i < smmu->soc->num_swgroups; i++) {  in tegra_smmu_find_swgroup()
302 		if (smmu->soc->swgroups[i].swgroup == swgroup) {  in tegra_smmu_find_swgroup()
303 			group = &smmu->soc->swgroups[i];  in tegra_smmu_find_swgroup()
311 static void tegra_smmu_enable(struct tegra_smmu *smmu, unsigned int swgroup,  in tegra_smmu_enable()  argument
318 	for (i = 0; i < smmu->soc->num_clients; i++) {  in tegra_smmu_enable()
319 		const struct tegra_mc_client *client = &smmu->soc->clients[i];  in tegra_smmu_enable()
324 		value = smmu_readl(smmu, client->smmu.reg);  in tegra_smmu_enable()
325 		value |= BIT(client->smmu.bit);  in tegra_smmu_enable()
326 		smmu_writel(smmu, value, client->smmu.reg);  in tegra_smmu_enable()
329 	group = tegra_smmu_find_swgroup(smmu, swgroup);  in tegra_smmu_enable()
331 		value = smmu_readl(smmu, group->reg);  in tegra_smmu_enable()
335 		smmu_writel(smmu, value, group->reg);  in tegra_smmu_enable()
339 static void tegra_smmu_disable(struct tegra_smmu *smmu, unsigned int swgroup,  in tegra_smmu_disable()  argument
346 	group = tegra_smmu_find_swgroup(smmu, swgroup);  in tegra_smmu_disable()
348 		value = smmu_readl(smmu, group->reg);  in tegra_smmu_disable()
352 		smmu_writel(smmu, value, group->reg);  in tegra_smmu_disable()
355 	for (i = 0; i < smmu->soc->num_clients; i++) {  in tegra_smmu_disable()
356 		const struct tegra_mc_client *client = &smmu->soc->clients[i];  in tegra_smmu_disable()
361 		value = smmu_readl(smmu, client->smmu.reg);  in tegra_smmu_disable()
362 		value &= ~BIT(client->smmu.bit);  in tegra_smmu_disable()
363 		smmu_writel(smmu, value, client->smmu.reg);  in tegra_smmu_disable()
367 static int tegra_smmu_as_prepare(struct tegra_smmu *smmu,  in tegra_smmu_as_prepare()  argument
378 	err = tegra_smmu_alloc_asid(smmu, &as->id);  in tegra_smmu_as_prepare()
382 	smmu->soc->ops->flush_dcache(as->pd, 0, SMMU_SIZE_PD);  in tegra_smmu_as_prepare()
383 	smmu_flush_ptc(smmu, as->pd, 0);  in tegra_smmu_as_prepare()
384 	smmu_flush_tlb_asid(smmu, as->id);  in tegra_smmu_as_prepare()
386 	smmu_writel(smmu, as->id & 0x7f, SMMU_PTB_ASID);  in tegra_smmu_as_prepare()
388 	smmu_writel(smmu, value, SMMU_PTB_DATA);  in tegra_smmu_as_prepare()
389 	smmu_flush(smmu);  in tegra_smmu_as_prepare()
391 	as->smmu = smmu;  in tegra_smmu_as_prepare()
397 static void tegra_smmu_as_unprepare(struct tegra_smmu *smmu,  in tegra_smmu_as_unprepare()  argument
403 	tegra_smmu_free_asid(smmu, as->id);  in tegra_smmu_as_unprepare()
404 	as->smmu = NULL;  in tegra_smmu_as_unprepare()
410 	struct tegra_smmu *smmu = dev->archdata.iommu;  in tegra_smmu_attach_dev()  local
421 		if (args.np != smmu->dev->of_node) {  in tegra_smmu_attach_dev()
428 		err = tegra_smmu_as_prepare(smmu, as);  in tegra_smmu_attach_dev()
432 		tegra_smmu_enable(smmu, swgroup, as->id);  in tegra_smmu_attach_dev()
446 	struct tegra_smmu *smmu = as->smmu;  in tegra_smmu_detach_dev()  local
454 		if (args.np != smmu->dev->of_node) {  in tegra_smmu_detach_dev()
461 		tegra_smmu_disable(smmu, swgroup, as->id);  in tegra_smmu_detach_dev()
462 		tegra_smmu_as_unprepare(smmu, as);  in tegra_smmu_detach_dev()
473 	struct tegra_smmu *smmu = as->smmu;  in as_get_pte()  local
488 		smmu->soc->ops->flush_dcache(page, 0, SMMU_SIZE_PT);  in as_get_pte()
492 		smmu->soc->ops->flush_dcache(as->pd, pde << 2, 4);  in as_get_pte()
493 		smmu_flush_ptc(smmu, as->pd, pde << 2);  in as_get_pte()
494 		smmu_flush_tlb_section(smmu, as->id, iova);  in as_get_pte()
495 		smmu_flush(smmu);  in as_get_pte()
497 		page = pfn_to_page(pd[pde] & smmu->pfn_mask);  in as_get_pte()
519 	page = pfn_to_page(pd[pde] & as->smmu->pfn_mask);  in as_put_pte()
541 	struct tegra_smmu *smmu = as->smmu;  in tegra_smmu_map()  local
553 	smmu->soc->ops->flush_dcache(page, offset, 4);  in tegra_smmu_map()
554 	smmu_flush_ptc(smmu, page, offset);  in tegra_smmu_map()
555 	smmu_flush_tlb_group(smmu, as->id, iova);  in tegra_smmu_map()
556 	smmu_flush(smmu);  in tegra_smmu_map()
565 	struct tegra_smmu *smmu = as->smmu;  in tegra_smmu_unmap()  local
577 	smmu->soc->ops->flush_dcache(page, offset, 4);  in tegra_smmu_unmap()
578 	smmu_flush_ptc(smmu, page, offset);  in tegra_smmu_unmap()
579 	smmu_flush_tlb_group(smmu, as->id, iova);  in tegra_smmu_unmap()
580 	smmu_flush(smmu);  in tegra_smmu_unmap()
594 	pfn = *pte & as->smmu->pfn_mask;  in tegra_smmu_iova_to_phys()
612 	return mc->smmu;  in tegra_smmu_find()
623 		struct tegra_smmu *smmu;  in tegra_smmu_add_device()  local
625 		smmu = tegra_smmu_find(args.np);  in tegra_smmu_add_device()
626 		if (smmu) {  in tegra_smmu_add_device()
632 			dev->archdata.iommu = smmu;  in tegra_smmu_add_device()
682 	struct tegra_smmu *smmu;  in tegra_smmu_probe()  local
691 	smmu = devm_kzalloc(dev, sizeof(*smmu), GFP_KERNEL);  in tegra_smmu_probe()
692 	if (!smmu)  in tegra_smmu_probe()
703 	mc->smmu = smmu;  in tegra_smmu_probe()
707 	smmu->asids = devm_kzalloc(dev, size, GFP_KERNEL);  in tegra_smmu_probe()
708 	if (!smmu->asids)  in tegra_smmu_probe()
711 	mutex_init(&smmu->lock);  in tegra_smmu_probe()
713 	smmu->regs = mc->regs;  in tegra_smmu_probe()
714 	smmu->soc = soc;  in tegra_smmu_probe()
715 	smmu->dev = dev;  in tegra_smmu_probe()
716 	smmu->mc = mc;  in tegra_smmu_probe()
718 	smmu->pfn_mask = BIT_MASK(mc->soc->num_address_bits - PAGE_SHIFT) - 1;  in tegra_smmu_probe()
720 		mc->soc->num_address_bits, smmu->pfn_mask);  in tegra_smmu_probe()
721 	smmu->tlb_mask = (smmu->soc->num_tlb_lines << 1) - 1;  in tegra_smmu_probe()
722 	dev_dbg(dev, "TLB lines: %u, mask: %#lx\n", smmu->soc->num_tlb_lines,  in tegra_smmu_probe()
723 		smmu->tlb_mask);  in tegra_smmu_probe()
730 	smmu_writel(smmu, value, SMMU_PTC_CONFIG);  in tegra_smmu_probe()
733 		SMMU_TLB_CONFIG_ACTIVE_LINES(smmu);  in tegra_smmu_probe()
738 	smmu_writel(smmu, value, SMMU_TLB_CONFIG);  in tegra_smmu_probe()
740 	smmu_flush_ptc(smmu, NULL, 0);  in tegra_smmu_probe()
741 	smmu_flush_tlb(smmu);  in tegra_smmu_probe()
742 	smmu_writel(smmu, SMMU_CONFIG_ENABLE, SMMU_CONFIG);  in tegra_smmu_probe()
743 	smmu_flush(smmu);  in tegra_smmu_probe()
751 	return smmu;  in tegra_smmu_probe()