This source file includes following definitions.
- gart_set_pte
- gart_read_pte
- do_gart_setup
- gart_iova_range_invalid
- gart_pte_valid
- gart_iommu_attach_dev
- gart_iommu_detach_dev
- gart_iommu_domain_alloc
- gart_iommu_domain_free
- __gart_iommu_map
- gart_iommu_map
- __gart_iommu_unmap
- gart_iommu_unmap
- gart_iommu_iova_to_phys
- gart_iommu_capable
- gart_iommu_add_device
- gart_iommu_remove_device
- gart_iommu_of_xlate
- gart_iommu_sync_map
- gart_iommu_sync
- tegra_gart_suspend
- tegra_gart_resume
- tegra_gart_probe
1
2
3
4
5
6
7
8
9
10 #define dev_fmt(fmt) "gart: " fmt
11
12 #include <linux/io.h>
13 #include <linux/iommu.h>
14 #include <linux/moduleparam.h>
15 #include <linux/platform_device.h>
16 #include <linux/slab.h>
17 #include <linux/spinlock.h>
18 #include <linux/vmalloc.h>
19
20 #include <soc/tegra/mc.h>
21
22 #define GART_REG_BASE 0x24
23 #define GART_CONFIG (0x24 - GART_REG_BASE)
24 #define GART_ENTRY_ADDR (0x28 - GART_REG_BASE)
25 #define GART_ENTRY_DATA (0x2c - GART_REG_BASE)
26
27 #define GART_ENTRY_PHYS_ADDR_VALID BIT(31)
28
29 #define GART_PAGE_SHIFT 12
30 #define GART_PAGE_SIZE (1 << GART_PAGE_SHIFT)
31 #define GART_PAGE_MASK GENMASK(30, GART_PAGE_SHIFT)
32
33
34 #define GART_IOMMU_PGSIZES (GART_PAGE_SIZE)
35
36 struct gart_device {
37 void __iomem *regs;
38 u32 *savedata;
39 unsigned long iovmm_base;
40 unsigned long iovmm_end;
41 spinlock_t pte_lock;
42 spinlock_t dom_lock;
43 unsigned int active_devices;
44 struct iommu_domain *active_domain;
45 struct iommu_device iommu;
46 struct device *dev;
47 };
48
49 static struct gart_device *gart_handle;
50
51 static bool gart_debug;
52
53
54
55
56
57
58 #define FLUSH_GART_REGS(gart) readl_relaxed((gart)->regs + GART_CONFIG)
59
60 #define for_each_gart_pte(gart, iova) \
61 for (iova = gart->iovmm_base; \
62 iova < gart->iovmm_end; \
63 iova += GART_PAGE_SIZE)
64
65 static inline void gart_set_pte(struct gart_device *gart,
66 unsigned long iova, unsigned long pte)
67 {
68 writel_relaxed(iova, gart->regs + GART_ENTRY_ADDR);
69 writel_relaxed(pte, gart->regs + GART_ENTRY_DATA);
70 }
71
72 static inline unsigned long gart_read_pte(struct gart_device *gart,
73 unsigned long iova)
74 {
75 unsigned long pte;
76
77 writel_relaxed(iova, gart->regs + GART_ENTRY_ADDR);
78 pte = readl_relaxed(gart->regs + GART_ENTRY_DATA);
79
80 return pte;
81 }
82
83 static void do_gart_setup(struct gart_device *gart, const u32 *data)
84 {
85 unsigned long iova;
86
87 for_each_gart_pte(gart, iova)
88 gart_set_pte(gart, iova, data ? *(data++) : 0);
89
90 writel_relaxed(1, gart->regs + GART_CONFIG);
91 FLUSH_GART_REGS(gart);
92 }
93
94 static inline bool gart_iova_range_invalid(struct gart_device *gart,
95 unsigned long iova, size_t bytes)
96 {
97 return unlikely(iova < gart->iovmm_base || bytes != GART_PAGE_SIZE ||
98 iova + bytes > gart->iovmm_end);
99 }
100
101 static inline bool gart_pte_valid(struct gart_device *gart, unsigned long iova)
102 {
103 return !!(gart_read_pte(gart, iova) & GART_ENTRY_PHYS_ADDR_VALID);
104 }
105
106 static int gart_iommu_attach_dev(struct iommu_domain *domain,
107 struct device *dev)
108 {
109 struct gart_device *gart = gart_handle;
110 int ret = 0;
111
112 spin_lock(&gart->dom_lock);
113
114 if (gart->active_domain && gart->active_domain != domain) {
115 ret = -EBUSY;
116 } else if (dev->archdata.iommu != domain) {
117 dev->archdata.iommu = domain;
118 gart->active_domain = domain;
119 gart->active_devices++;
120 }
121
122 spin_unlock(&gart->dom_lock);
123
124 return ret;
125 }
126
127 static void gart_iommu_detach_dev(struct iommu_domain *domain,
128 struct device *dev)
129 {
130 struct gart_device *gart = gart_handle;
131
132 spin_lock(&gart->dom_lock);
133
134 if (dev->archdata.iommu == domain) {
135 dev->archdata.iommu = NULL;
136
137 if (--gart->active_devices == 0)
138 gart->active_domain = NULL;
139 }
140
141 spin_unlock(&gart->dom_lock);
142 }
143
144 static struct iommu_domain *gart_iommu_domain_alloc(unsigned type)
145 {
146 struct iommu_domain *domain;
147
148 if (type != IOMMU_DOMAIN_UNMANAGED)
149 return NULL;
150
151 domain = kzalloc(sizeof(*domain), GFP_KERNEL);
152 if (domain) {
153 domain->geometry.aperture_start = gart_handle->iovmm_base;
154 domain->geometry.aperture_end = gart_handle->iovmm_end - 1;
155 domain->geometry.force_aperture = true;
156 }
157
158 return domain;
159 }
160
161 static void gart_iommu_domain_free(struct iommu_domain *domain)
162 {
163 WARN_ON(gart_handle->active_domain == domain);
164 kfree(domain);
165 }
166
167 static inline int __gart_iommu_map(struct gart_device *gart, unsigned long iova,
168 unsigned long pa)
169 {
170 if (unlikely(gart_debug && gart_pte_valid(gart, iova))) {
171 dev_err(gart->dev, "Page entry is in-use\n");
172 return -EINVAL;
173 }
174
175 gart_set_pte(gart, iova, GART_ENTRY_PHYS_ADDR_VALID | pa);
176
177 return 0;
178 }
179
180 static int gart_iommu_map(struct iommu_domain *domain, unsigned long iova,
181 phys_addr_t pa, size_t bytes, int prot)
182 {
183 struct gart_device *gart = gart_handle;
184 int ret;
185
186 if (gart_iova_range_invalid(gart, iova, bytes))
187 return -EINVAL;
188
189 spin_lock(&gart->pte_lock);
190 ret = __gart_iommu_map(gart, iova, (unsigned long)pa);
191 spin_unlock(&gart->pte_lock);
192
193 return ret;
194 }
195
196 static inline int __gart_iommu_unmap(struct gart_device *gart,
197 unsigned long iova)
198 {
199 if (unlikely(gart_debug && !gart_pte_valid(gart, iova))) {
200 dev_err(gart->dev, "Page entry is invalid\n");
201 return -EINVAL;
202 }
203
204 gart_set_pte(gart, iova, 0);
205
206 return 0;
207 }
208
209 static size_t gart_iommu_unmap(struct iommu_domain *domain, unsigned long iova,
210 size_t bytes, struct iommu_iotlb_gather *gather)
211 {
212 struct gart_device *gart = gart_handle;
213 int err;
214
215 if (gart_iova_range_invalid(gart, iova, bytes))
216 return 0;
217
218 spin_lock(&gart->pte_lock);
219 err = __gart_iommu_unmap(gart, iova);
220 spin_unlock(&gart->pte_lock);
221
222 return err ? 0 : bytes;
223 }
224
225 static phys_addr_t gart_iommu_iova_to_phys(struct iommu_domain *domain,
226 dma_addr_t iova)
227 {
228 struct gart_device *gart = gart_handle;
229 unsigned long pte;
230
231 if (gart_iova_range_invalid(gart, iova, GART_PAGE_SIZE))
232 return -EINVAL;
233
234 spin_lock(&gart->pte_lock);
235 pte = gart_read_pte(gart, iova);
236 spin_unlock(&gart->pte_lock);
237
238 return pte & GART_PAGE_MASK;
239 }
240
241 static bool gart_iommu_capable(enum iommu_cap cap)
242 {
243 return false;
244 }
245
246 static int gart_iommu_add_device(struct device *dev)
247 {
248 struct iommu_group *group;
249
250 if (!dev->iommu_fwspec)
251 return -ENODEV;
252
253 group = iommu_group_get_for_dev(dev);
254 if (IS_ERR(group))
255 return PTR_ERR(group);
256
257 iommu_group_put(group);
258
259 iommu_device_link(&gart_handle->iommu, dev);
260
261 return 0;
262 }
263
264 static void gart_iommu_remove_device(struct device *dev)
265 {
266 iommu_group_remove_device(dev);
267 iommu_device_unlink(&gart_handle->iommu, dev);
268 }
269
270 static int gart_iommu_of_xlate(struct device *dev,
271 struct of_phandle_args *args)
272 {
273 return 0;
274 }
275
276 static void gart_iommu_sync_map(struct iommu_domain *domain)
277 {
278 FLUSH_GART_REGS(gart_handle);
279 }
280
281 static void gart_iommu_sync(struct iommu_domain *domain,
282 struct iommu_iotlb_gather *gather)
283 {
284 gart_iommu_sync_map(domain);
285 }
286
287 static const struct iommu_ops gart_iommu_ops = {
288 .capable = gart_iommu_capable,
289 .domain_alloc = gart_iommu_domain_alloc,
290 .domain_free = gart_iommu_domain_free,
291 .attach_dev = gart_iommu_attach_dev,
292 .detach_dev = gart_iommu_detach_dev,
293 .add_device = gart_iommu_add_device,
294 .remove_device = gart_iommu_remove_device,
295 .device_group = generic_device_group,
296 .map = gart_iommu_map,
297 .unmap = gart_iommu_unmap,
298 .iova_to_phys = gart_iommu_iova_to_phys,
299 .pgsize_bitmap = GART_IOMMU_PGSIZES,
300 .of_xlate = gart_iommu_of_xlate,
301 .iotlb_sync_map = gart_iommu_sync_map,
302 .iotlb_sync = gart_iommu_sync,
303 };
304
305 int tegra_gart_suspend(struct gart_device *gart)
306 {
307 u32 *data = gart->savedata;
308 unsigned long iova;
309
310
311
312
313
314
315 writel_relaxed(0, gart->regs + GART_CONFIG);
316 FLUSH_GART_REGS(gart);
317
318 for_each_gart_pte(gart, iova)
319 *(data++) = gart_read_pte(gart, iova);
320
321 return 0;
322 }
323
324 int tegra_gart_resume(struct gart_device *gart)
325 {
326 do_gart_setup(gart, gart->savedata);
327
328 return 0;
329 }
330
331 struct gart_device *tegra_gart_probe(struct device *dev, struct tegra_mc *mc)
332 {
333 struct gart_device *gart;
334 struct resource *res;
335 int err;
336
337 BUILD_BUG_ON(PAGE_SHIFT != GART_PAGE_SHIFT);
338
339
340 res = platform_get_resource(to_platform_device(dev), IORESOURCE_MEM, 1);
341 if (!res) {
342 dev_err(dev, "Memory aperture resource unavailable\n");
343 return ERR_PTR(-ENXIO);
344 }
345
346 gart = kzalloc(sizeof(*gart), GFP_KERNEL);
347 if (!gart)
348 return ERR_PTR(-ENOMEM);
349
350 gart_handle = gart;
351
352 gart->dev = dev;
353 gart->regs = mc->regs + GART_REG_BASE;
354 gart->iovmm_base = res->start;
355 gart->iovmm_end = res->end + 1;
356 spin_lock_init(&gart->pte_lock);
357 spin_lock_init(&gart->dom_lock);
358
359 do_gart_setup(gart, NULL);
360
361 err = iommu_device_sysfs_add(&gart->iommu, dev, NULL, "gart");
362 if (err)
363 goto free_gart;
364
365 iommu_device_set_ops(&gart->iommu, &gart_iommu_ops);
366 iommu_device_set_fwnode(&gart->iommu, dev->fwnode);
367
368 err = iommu_device_register(&gart->iommu);
369 if (err)
370 goto remove_sysfs;
371
372 gart->savedata = vmalloc(resource_size(res) / GART_PAGE_SIZE *
373 sizeof(u32));
374 if (!gart->savedata) {
375 err = -ENOMEM;
376 goto unregister_iommu;
377 }
378
379 return gart;
380
381 unregister_iommu:
382 iommu_device_unregister(&gart->iommu);
383 remove_sysfs:
384 iommu_device_sysfs_remove(&gart->iommu);
385 free_gart:
386 kfree(gart);
387
388 return ERR_PTR(err);
389 }
390
391 module_param(gart_debug, bool, 0644);
392 MODULE_PARM_DESC(gart_debug, "Enable GART debugging");