68 	struct gart_device *gart;		/* link to gart device   */  member
86 #define FLUSH_GART_REGS(gart)	((void)readl((gart)->regs + GART_CONFIG))  argument
88 #define for_each_gart_pte(gart, iova)					\  argument
89 	for (iova = gart->iovmm_base;					\
90 	     iova < gart->iovmm_base + GART_PAGE_SIZE * gart->page_count; \
93 static inline void gart_set_pte(struct gart_device *gart,  in gart_set_pte()  argument
96 	writel(offs, gart->regs + GART_ENTRY_ADDR);  in gart_set_pte()
97 	writel(pte, gart->regs + GART_ENTRY_DATA);  in gart_set_pte()
99 	dev_dbg(gart->dev, "%s %08lx:%08x\n",  in gart_set_pte()
103 static inline unsigned long gart_read_pte(struct gart_device *gart,  in gart_read_pte()  argument
108 	writel(offs, gart->regs + GART_ENTRY_ADDR);  in gart_read_pte()
109 	pte = readl(gart->regs + GART_ENTRY_DATA);  in gart_read_pte()
114 static void do_gart_setup(struct gart_device *gart, const u32 *data)  in do_gart_setup()  argument
118 	for_each_gart_pte(gart, iova)  in do_gart_setup()
119 		gart_set_pte(gart, iova, data ? *(data++) : 0);  in do_gart_setup()
121 	writel(1, gart->regs + GART_CONFIG);  in do_gart_setup()
122 	FLUSH_GART_REGS(gart);  in do_gart_setup()
126 static void gart_dump_table(struct gart_device *gart)  in gart_dump_table()  argument
131 	spin_lock_irqsave(&gart->pte_lock, flags);  in gart_dump_table()
132 	for_each_gart_pte(gart, iova) {  in gart_dump_table()
135 		pte = gart_read_pte(gart, iova);  in gart_dump_table()
137 		dev_dbg(gart->dev, "%s %08lx:%08lx\n",  in gart_dump_table()
141 	spin_unlock_irqrestore(&gart->pte_lock, flags);  in gart_dump_table()
144 static inline void gart_dump_table(struct gart_device *gart)  in gart_dump_table()  argument
149 static inline bool gart_iova_range_valid(struct gart_device *gart,  in gart_iova_range_valid()  argument
156 	gart_start = gart->iovmm_base;  in gart_iova_range_valid()
157 	gart_end = gart_start + gart->page_count * GART_PAGE_SIZE - 1;  in gart_iova_range_valid()
170 	struct gart_device *gart = gart_domain->gart;  in gart_iommu_attach_dev()  local
174 	client = devm_kzalloc(gart->dev, sizeof(*c), GFP_KERNEL);  in gart_iommu_attach_dev()
179 	spin_lock(&gart->client_lock);  in gart_iommu_attach_dev()
180 	list_for_each_entry(c, &gart->client, list) {  in gart_iommu_attach_dev()
182 			dev_err(gart->dev,  in gart_iommu_attach_dev()
188 	list_add(&client->list, &gart->client);  in gart_iommu_attach_dev()
189 	spin_unlock(&gart->client_lock);  in gart_iommu_attach_dev()
190 	dev_dbg(gart->dev, "Attached %s\n", dev_name(dev));  in gart_iommu_attach_dev()
194 	devm_kfree(gart->dev, client);  in gart_iommu_attach_dev()
195 	spin_unlock(&gart->client_lock);  in gart_iommu_attach_dev()
203 	struct gart_device *gart = gart_domain->gart;  in gart_iommu_detach_dev()  local
206 	spin_lock(&gart->client_lock);  in gart_iommu_detach_dev()
208 	list_for_each_entry(c, &gart->client, list) {  in gart_iommu_detach_dev()
211 			devm_kfree(gart->dev, c);  in gart_iommu_detach_dev()
212 			dev_dbg(gart->dev, "Detached %s\n", dev_name(dev));  in gart_iommu_detach_dev()
216 	dev_err(gart->dev, "Couldn't find\n");  in gart_iommu_detach_dev()
218 	spin_unlock(&gart->client_lock);  in gart_iommu_detach_dev()
224 	struct gart_device *gart;  in gart_iommu_domain_alloc()  local
229 	gart = gart_handle;  in gart_iommu_domain_alloc()
230 	if (!gart)  in gart_iommu_domain_alloc()
237 	gart_domain->gart = gart;  in gart_iommu_domain_alloc()
238 	gart_domain->domain.geometry.aperture_start = gart->iovmm_base;  in gart_iommu_domain_alloc()
239 	gart_domain->domain.geometry.aperture_end = gart->iovmm_base +  in gart_iommu_domain_alloc()
240 					gart->page_count * GART_PAGE_SIZE - 1;  in gart_iommu_domain_alloc()
249 	struct gart_device *gart = gart_domain->gart;  in gart_iommu_domain_free()  local
251 	if (gart) {  in gart_iommu_domain_free()
252 		spin_lock(&gart->client_lock);  in gart_iommu_domain_free()
253 		if (!list_empty(&gart->client)) {  in gart_iommu_domain_free()
256 			list_for_each_entry(c, &gart->client, list)  in gart_iommu_domain_free()
259 		spin_unlock(&gart->client_lock);  in gart_iommu_domain_free()
269 	struct gart_device *gart = gart_domain->gart;  in gart_iommu_map()  local
273 	if (!gart_iova_range_valid(gart, iova, bytes))  in gart_iommu_map()
276 	spin_lock_irqsave(&gart->pte_lock, flags);  in gart_iommu_map()
279 		dev_err(gart->dev, "Invalid page: %pa\n", &pa);  in gart_iommu_map()
280 		spin_unlock_irqrestore(&gart->pte_lock, flags);  in gart_iommu_map()
283 	gart_set_pte(gart, iova, GART_PTE(pfn));  in gart_iommu_map()
284 	FLUSH_GART_REGS(gart);  in gart_iommu_map()
285 	spin_unlock_irqrestore(&gart->pte_lock, flags);  in gart_iommu_map()
293 	struct gart_device *gart = gart_domain->gart;  in gart_iommu_unmap()  local
296 	if (!gart_iova_range_valid(gart, iova, bytes))  in gart_iommu_unmap()
299 	spin_lock_irqsave(&gart->pte_lock, flags);  in gart_iommu_unmap()
300 	gart_set_pte(gart, iova, 0);  in gart_iommu_unmap()
301 	FLUSH_GART_REGS(gart);  in gart_iommu_unmap()
302 	spin_unlock_irqrestore(&gart->pte_lock, flags);  in gart_iommu_unmap()
310 	struct gart_device *gart = gart_domain->gart;  in gart_iommu_iova_to_phys()  local
315 	if (!gart_iova_range_valid(gart, iova, 0))  in gart_iommu_iova_to_phys()
318 	spin_lock_irqsave(&gart->pte_lock, flags);  in gart_iommu_iova_to_phys()
319 	pte = gart_read_pte(gart, iova);  in gart_iommu_iova_to_phys()
320 	spin_unlock_irqrestore(&gart->pte_lock, flags);  in gart_iommu_iova_to_phys()
324 		dev_err(gart->dev, "No entry for %08llx:%pa\n",  in gart_iommu_iova_to_phys()
326 		gart_dump_table(gart);  in gart_iommu_iova_to_phys()
352 	struct gart_device *gart = dev_get_drvdata(dev);  in tegra_gart_suspend()  local
354 	u32 *data = gart->savedata;  in tegra_gart_suspend()
357 	spin_lock_irqsave(&gart->pte_lock, flags);  in tegra_gart_suspend()
358 	for_each_gart_pte(gart, iova)  in tegra_gart_suspend()
359 		*(data++) = gart_read_pte(gart, iova);  in tegra_gart_suspend()
360 	spin_unlock_irqrestore(&gart->pte_lock, flags);  in tegra_gart_suspend()
366 	struct gart_device *gart = dev_get_drvdata(dev);  in tegra_gart_resume()  local
369 	spin_lock_irqsave(&gart->pte_lock, flags);  in tegra_gart_resume()
370 	do_gart_setup(gart, gart->savedata);  in tegra_gart_resume()
371 	spin_unlock_irqrestore(&gart->pte_lock, flags);  in tegra_gart_resume()
377 	struct gart_device *gart;  in tegra_gart_probe()  local
395 	gart = devm_kzalloc(dev, sizeof(*gart), GFP_KERNEL);  in tegra_gart_probe()
396 	if (!gart) {  in tegra_gart_probe()
407 	gart->dev = &pdev->dev;  in tegra_gart_probe()
408 	spin_lock_init(&gart->pte_lock);  in tegra_gart_probe()
409 	spin_lock_init(&gart->client_lock);  in tegra_gart_probe()
410 	INIT_LIST_HEAD(&gart->client);  in tegra_gart_probe()
411 	gart->regs = gart_regs;  in tegra_gart_probe()
412 	gart->iovmm_base = (dma_addr_t)res_remap->start;  in tegra_gart_probe()
413 	gart->page_count = (resource_size(res_remap) >> GART_PAGE_SHIFT);  in tegra_gart_probe()
415 	gart->savedata = vmalloc(sizeof(u32) * gart->page_count);  in tegra_gart_probe()
416 	if (!gart->savedata) {  in tegra_gart_probe()
421 	platform_set_drvdata(pdev, gart);  in tegra_gart_probe()
422 	do_gart_setup(gart, NULL);  in tegra_gart_probe()
424 	gart_handle = gart;  in tegra_gart_probe()
431 	struct gart_device *gart = platform_get_drvdata(pdev);  in tegra_gart_remove()  local
433 	writel(0, gart->regs + GART_CONFIG);  in tegra_gart_remove()
434 	if (gart->savedata)  in tegra_gart_remove()
435 		vfree(gart->savedata);  in tegra_gart_remove()