This source file includes following definitions.
- arm_smmu_page1_fixup
- to_smmu_domain
- parse_driver_options
- queue_has_space
- queue_full
- queue_empty
- queue_consumed
- queue_sync_cons_out
- queue_inc_cons
- queue_sync_prod_in
- queue_inc_prod_n
- queue_poll_init
- queue_poll
- queue_write
- queue_read
- queue_remove_raw
- arm_smmu_cmdq_build_cmd
- arm_smmu_cmdq_build_sync_cmd
- arm_smmu_cmdq_skip_err
- arm_smmu_cmdq_shared_lock
- arm_smmu_cmdq_shared_unlock
- arm_smmu_cmdq_shared_tryunlock
- __arm_smmu_cmdq_poll_set_valid_map
- arm_smmu_cmdq_set_valid_map
- arm_smmu_cmdq_poll_valid_map
- arm_smmu_cmdq_poll_until_not_full
- __arm_smmu_cmdq_poll_until_msi
- __arm_smmu_cmdq_poll_until_consumed
- arm_smmu_cmdq_poll_until_sync
- arm_smmu_cmdq_write_entries
- arm_smmu_cmdq_issue_cmdlist
- arm_smmu_cmdq_issue_cmd
- arm_smmu_cmdq_issue_sync
- arm_smmu_cpu_tcr_to_cd
- arm_smmu_write_ctx_desc
- arm_smmu_write_strtab_l1_desc
- arm_smmu_sync_ste_for_sid
- arm_smmu_write_strtab_ent
- arm_smmu_init_bypass_stes
- arm_smmu_init_l2_strtab
- arm_smmu_evtq_thread
- arm_smmu_handle_ppr
- arm_smmu_priq_thread
- arm_smmu_gerror_handler
- arm_smmu_combined_irq_thread
- arm_smmu_combined_irq_handler
- arm_smmu_atc_inv_to_cmd
- arm_smmu_atc_inv_master
- arm_smmu_atc_inv_domain
- arm_smmu_tlb_inv_context
- arm_smmu_tlb_inv_range
- arm_smmu_tlb_inv_page_nosync
- arm_smmu_tlb_inv_walk
- arm_smmu_tlb_inv_leaf
- arm_smmu_capable
- arm_smmu_domain_alloc
- arm_smmu_bitmap_alloc
- arm_smmu_bitmap_free
- arm_smmu_domain_free
- arm_smmu_domain_finalise_s1
- arm_smmu_domain_finalise_s2
- arm_smmu_domain_finalise
- arm_smmu_get_step_for_sid
- arm_smmu_install_ste_for_dev
- arm_smmu_ats_supported
- arm_smmu_ats_supported
- arm_smmu_enable_ats
- arm_smmu_disable_ats
- arm_smmu_detach_dev
- arm_smmu_attach_dev
- arm_smmu_map
- arm_smmu_unmap
- arm_smmu_flush_iotlb_all
- arm_smmu_iotlb_sync
- arm_smmu_iova_to_phys
- arm_smmu_get_by_fwnode
- arm_smmu_sid_in_range
- arm_smmu_add_device
- arm_smmu_remove_device
- arm_smmu_device_group
- arm_smmu_domain_get_attr
- arm_smmu_domain_set_attr
- arm_smmu_of_xlate
- arm_smmu_get_resv_regions
- arm_smmu_put_resv_regions
- arm_smmu_init_one_queue
- arm_smmu_cmdq_free_bitmap
- arm_smmu_cmdq_init
- arm_smmu_init_queues
- arm_smmu_init_l1_strtab
- arm_smmu_init_strtab_2lvl
- arm_smmu_init_strtab_linear
- arm_smmu_init_strtab
- arm_smmu_init_structures
- arm_smmu_write_reg_sync
- arm_smmu_update_gbpa
- arm_smmu_free_msis
- arm_smmu_write_msi_msg
- arm_smmu_setup_msis
- arm_smmu_setup_unique_irqs
- arm_smmu_setup_irqs
- arm_smmu_device_disable
- arm_smmu_device_reset
- arm_smmu_device_hw_probe
- acpi_smmu_get_options
- arm_smmu_device_acpi_probe
- arm_smmu_device_acpi_probe
- arm_smmu_device_dt_probe
- arm_smmu_resource_size
- arm_smmu_device_probe
- arm_smmu_device_shutdown
1
2
3
4
5
6
7
8
9
10
11
12 #include <linux/acpi.h>
13 #include <linux/acpi_iort.h>
14 #include <linux/bitfield.h>
15 #include <linux/bitops.h>
16 #include <linux/crash_dump.h>
17 #include <linux/delay.h>
18 #include <linux/dma-iommu.h>
19 #include <linux/err.h>
20 #include <linux/interrupt.h>
21 #include <linux/io-pgtable.h>
22 #include <linux/iommu.h>
23 #include <linux/iopoll.h>
24 #include <linux/init.h>
25 #include <linux/moduleparam.h>
26 #include <linux/msi.h>
27 #include <linux/of.h>
28 #include <linux/of_address.h>
29 #include <linux/of_iommu.h>
30 #include <linux/of_platform.h>
31 #include <linux/pci.h>
32 #include <linux/pci-ats.h>
33 #include <linux/platform_device.h>
34
35 #include <linux/amba/bus.h>
36
37
38 #define ARM_SMMU_IDR0 0x0
39 #define IDR0_ST_LVL GENMASK(28, 27)
40 #define IDR0_ST_LVL_2LVL 1
41 #define IDR0_STALL_MODEL GENMASK(25, 24)
42 #define IDR0_STALL_MODEL_STALL 0
43 #define IDR0_STALL_MODEL_FORCE 2
44 #define IDR0_TTENDIAN GENMASK(22, 21)
45 #define IDR0_TTENDIAN_MIXED 0
46 #define IDR0_TTENDIAN_LE 2
47 #define IDR0_TTENDIAN_BE 3
48 #define IDR0_CD2L (1 << 19)
49 #define IDR0_VMID16 (1 << 18)
50 #define IDR0_PRI (1 << 16)
51 #define IDR0_SEV (1 << 14)
52 #define IDR0_MSI (1 << 13)
53 #define IDR0_ASID16 (1 << 12)
54 #define IDR0_ATS (1 << 10)
55 #define IDR0_HYP (1 << 9)
56 #define IDR0_COHACC (1 << 4)
57 #define IDR0_TTF GENMASK(3, 2)
58 #define IDR0_TTF_AARCH64 2
59 #define IDR0_TTF_AARCH32_64 3
60 #define IDR0_S1P (1 << 1)
61 #define IDR0_S2P (1 << 0)
62
63 #define ARM_SMMU_IDR1 0x4
64 #define IDR1_TABLES_PRESET (1 << 30)
65 #define IDR1_QUEUES_PRESET (1 << 29)
66 #define IDR1_REL (1 << 28)
67 #define IDR1_CMDQS GENMASK(25, 21)
68 #define IDR1_EVTQS GENMASK(20, 16)
69 #define IDR1_PRIQS GENMASK(15, 11)
70 #define IDR1_SSIDSIZE GENMASK(10, 6)
71 #define IDR1_SIDSIZE GENMASK(5, 0)
72
73 #define ARM_SMMU_IDR5 0x14
74 #define IDR5_STALL_MAX GENMASK(31, 16)
75 #define IDR5_GRAN64K (1 << 6)
76 #define IDR5_GRAN16K (1 << 5)
77 #define IDR5_GRAN4K (1 << 4)
78 #define IDR5_OAS GENMASK(2, 0)
79 #define IDR5_OAS_32_BIT 0
80 #define IDR5_OAS_36_BIT 1
81 #define IDR5_OAS_40_BIT 2
82 #define IDR5_OAS_42_BIT 3
83 #define IDR5_OAS_44_BIT 4
84 #define IDR5_OAS_48_BIT 5
85 #define IDR5_OAS_52_BIT 6
86 #define IDR5_VAX GENMASK(11, 10)
87 #define IDR5_VAX_52_BIT 1
88
89 #define ARM_SMMU_CR0 0x20
90 #define CR0_ATSCHK (1 << 4)
91 #define CR0_CMDQEN (1 << 3)
92 #define CR0_EVTQEN (1 << 2)
93 #define CR0_PRIQEN (1 << 1)
94 #define CR0_SMMUEN (1 << 0)
95
96 #define ARM_SMMU_CR0ACK 0x24
97
98 #define ARM_SMMU_CR1 0x28
99 #define CR1_TABLE_SH GENMASK(11, 10)
100 #define CR1_TABLE_OC GENMASK(9, 8)
101 #define CR1_TABLE_IC GENMASK(7, 6)
102 #define CR1_QUEUE_SH GENMASK(5, 4)
103 #define CR1_QUEUE_OC GENMASK(3, 2)
104 #define CR1_QUEUE_IC GENMASK(1, 0)
105
106 #define CR1_CACHE_NC 0
107 #define CR1_CACHE_WB 1
108 #define CR1_CACHE_WT 2
109
110 #define ARM_SMMU_CR2 0x2c
111 #define CR2_PTM (1 << 2)
112 #define CR2_RECINVSID (1 << 1)
113 #define CR2_E2H (1 << 0)
114
115 #define ARM_SMMU_GBPA 0x44
116 #define GBPA_UPDATE (1 << 31)
117 #define GBPA_ABORT (1 << 20)
118
119 #define ARM_SMMU_IRQ_CTRL 0x50
120 #define IRQ_CTRL_EVTQ_IRQEN (1 << 2)
121 #define IRQ_CTRL_PRIQ_IRQEN (1 << 1)
122 #define IRQ_CTRL_GERROR_IRQEN (1 << 0)
123
124 #define ARM_SMMU_IRQ_CTRLACK 0x54
125
126 #define ARM_SMMU_GERROR 0x60
127 #define GERROR_SFM_ERR (1 << 8)
128 #define GERROR_MSI_GERROR_ABT_ERR (1 << 7)
129 #define GERROR_MSI_PRIQ_ABT_ERR (1 << 6)
130 #define GERROR_MSI_EVTQ_ABT_ERR (1 << 5)
131 #define GERROR_MSI_CMDQ_ABT_ERR (1 << 4)
132 #define GERROR_PRIQ_ABT_ERR (1 << 3)
133 #define GERROR_EVTQ_ABT_ERR (1 << 2)
134 #define GERROR_CMDQ_ERR (1 << 0)
135 #define GERROR_ERR_MASK 0xfd
136
137 #define ARM_SMMU_GERRORN 0x64
138
139 #define ARM_SMMU_GERROR_IRQ_CFG0 0x68
140 #define ARM_SMMU_GERROR_IRQ_CFG1 0x70
141 #define ARM_SMMU_GERROR_IRQ_CFG2 0x74
142
143 #define ARM_SMMU_STRTAB_BASE 0x80
144 #define STRTAB_BASE_RA (1UL << 62)
145 #define STRTAB_BASE_ADDR_MASK GENMASK_ULL(51, 6)
146
147 #define ARM_SMMU_STRTAB_BASE_CFG 0x88
148 #define STRTAB_BASE_CFG_FMT GENMASK(17, 16)
149 #define STRTAB_BASE_CFG_FMT_LINEAR 0
150 #define STRTAB_BASE_CFG_FMT_2LVL 1
151 #define STRTAB_BASE_CFG_SPLIT GENMASK(10, 6)
152 #define STRTAB_BASE_CFG_LOG2SIZE GENMASK(5, 0)
153
154 #define ARM_SMMU_CMDQ_BASE 0x90
155 #define ARM_SMMU_CMDQ_PROD 0x98
156 #define ARM_SMMU_CMDQ_CONS 0x9c
157
158 #define ARM_SMMU_EVTQ_BASE 0xa0
159 #define ARM_SMMU_EVTQ_PROD 0x100a8
160 #define ARM_SMMU_EVTQ_CONS 0x100ac
161 #define ARM_SMMU_EVTQ_IRQ_CFG0 0xb0
162 #define ARM_SMMU_EVTQ_IRQ_CFG1 0xb8
163 #define ARM_SMMU_EVTQ_IRQ_CFG2 0xbc
164
165 #define ARM_SMMU_PRIQ_BASE 0xc0
166 #define ARM_SMMU_PRIQ_PROD 0x100c8
167 #define ARM_SMMU_PRIQ_CONS 0x100cc
168 #define ARM_SMMU_PRIQ_IRQ_CFG0 0xd0
169 #define ARM_SMMU_PRIQ_IRQ_CFG1 0xd8
170 #define ARM_SMMU_PRIQ_IRQ_CFG2 0xdc
171
172
173 #define MSI_CFG0_ADDR_MASK GENMASK_ULL(51, 2)
174 #define MSI_CFG2_SH GENMASK(5, 4)
175 #define MSI_CFG2_MEMATTR GENMASK(3, 0)
176
177
178 #define ARM_SMMU_SH_NSH 0
179 #define ARM_SMMU_SH_OSH 2
180 #define ARM_SMMU_SH_ISH 3
181 #define ARM_SMMU_MEMATTR_DEVICE_nGnRE 0x1
182 #define ARM_SMMU_MEMATTR_OIWB 0xf
183
184 #define Q_IDX(llq, p) ((p) & ((1 << (llq)->max_n_shift) - 1))
185 #define Q_WRP(llq, p) ((p) & (1 << (llq)->max_n_shift))
186 #define Q_OVERFLOW_FLAG (1U << 31)
187 #define Q_OVF(p) ((p) & Q_OVERFLOW_FLAG)
188 #define Q_ENT(q, p) ((q)->base + \
189 Q_IDX(&((q)->llq), p) * \
190 (q)->ent_dwords)
191
192 #define Q_BASE_RWA (1UL << 62)
193 #define Q_BASE_ADDR_MASK GENMASK_ULL(51, 5)
194 #define Q_BASE_LOG2SIZE GENMASK(4, 0)
195
196
197 #ifdef CONFIG_CMA_ALIGNMENT
198 #define Q_MAX_SZ_SHIFT (PAGE_SHIFT + CONFIG_CMA_ALIGNMENT)
199 #else
200 #define Q_MAX_SZ_SHIFT (PAGE_SHIFT + MAX_ORDER - 1)
201 #endif
202
203
204
205
206
207
208
209
210 #define STRTAB_L1_SZ_SHIFT 20
211 #define STRTAB_SPLIT 8
212
213 #define STRTAB_L1_DESC_DWORDS 1
214 #define STRTAB_L1_DESC_SPAN GENMASK_ULL(4, 0)
215 #define STRTAB_L1_DESC_L2PTR_MASK GENMASK_ULL(51, 6)
216
217 #define STRTAB_STE_DWORDS 8
218 #define STRTAB_STE_0_V (1UL << 0)
219 #define STRTAB_STE_0_CFG GENMASK_ULL(3, 1)
220 #define STRTAB_STE_0_CFG_ABORT 0
221 #define STRTAB_STE_0_CFG_BYPASS 4
222 #define STRTAB_STE_0_CFG_S1_TRANS 5
223 #define STRTAB_STE_0_CFG_S2_TRANS 6
224
225 #define STRTAB_STE_0_S1FMT GENMASK_ULL(5, 4)
226 #define STRTAB_STE_0_S1FMT_LINEAR 0
227 #define STRTAB_STE_0_S1CTXPTR_MASK GENMASK_ULL(51, 6)
228 #define STRTAB_STE_0_S1CDMAX GENMASK_ULL(63, 59)
229
230 #define STRTAB_STE_1_S1C_CACHE_NC 0UL
231 #define STRTAB_STE_1_S1C_CACHE_WBRA 1UL
232 #define STRTAB_STE_1_S1C_CACHE_WT 2UL
233 #define STRTAB_STE_1_S1C_CACHE_WB 3UL
234 #define STRTAB_STE_1_S1CIR GENMASK_ULL(3, 2)
235 #define STRTAB_STE_1_S1COR GENMASK_ULL(5, 4)
236 #define STRTAB_STE_1_S1CSH GENMASK_ULL(7, 6)
237
238 #define STRTAB_STE_1_S1STALLD (1UL << 27)
239
240 #define STRTAB_STE_1_EATS GENMASK_ULL(29, 28)
241 #define STRTAB_STE_1_EATS_ABT 0UL
242 #define STRTAB_STE_1_EATS_TRANS 1UL
243 #define STRTAB_STE_1_EATS_S1CHK 2UL
244
245 #define STRTAB_STE_1_STRW GENMASK_ULL(31, 30)
246 #define STRTAB_STE_1_STRW_NSEL1 0UL
247 #define STRTAB_STE_1_STRW_EL2 2UL
248
249 #define STRTAB_STE_1_SHCFG GENMASK_ULL(45, 44)
250 #define STRTAB_STE_1_SHCFG_INCOMING 1UL
251
252 #define STRTAB_STE_2_S2VMID GENMASK_ULL(15, 0)
253 #define STRTAB_STE_2_VTCR GENMASK_ULL(50, 32)
254 #define STRTAB_STE_2_S2AA64 (1UL << 51)
255 #define STRTAB_STE_2_S2ENDI (1UL << 52)
256 #define STRTAB_STE_2_S2PTW (1UL << 54)
257 #define STRTAB_STE_2_S2R (1UL << 58)
258
259 #define STRTAB_STE_3_S2TTB_MASK GENMASK_ULL(51, 4)
260
261
262 #define CTXDESC_CD_DWORDS 8
263 #define CTXDESC_CD_0_TCR_T0SZ GENMASK_ULL(5, 0)
264 #define ARM64_TCR_T0SZ GENMASK_ULL(5, 0)
265 #define CTXDESC_CD_0_TCR_TG0 GENMASK_ULL(7, 6)
266 #define ARM64_TCR_TG0 GENMASK_ULL(15, 14)
267 #define CTXDESC_CD_0_TCR_IRGN0 GENMASK_ULL(9, 8)
268 #define ARM64_TCR_IRGN0 GENMASK_ULL(9, 8)
269 #define CTXDESC_CD_0_TCR_ORGN0 GENMASK_ULL(11, 10)
270 #define ARM64_TCR_ORGN0 GENMASK_ULL(11, 10)
271 #define CTXDESC_CD_0_TCR_SH0 GENMASK_ULL(13, 12)
272 #define ARM64_TCR_SH0 GENMASK_ULL(13, 12)
273 #define CTXDESC_CD_0_TCR_EPD0 (1ULL << 14)
274 #define ARM64_TCR_EPD0 (1ULL << 7)
275 #define CTXDESC_CD_0_TCR_EPD1 (1ULL << 30)
276 #define ARM64_TCR_EPD1 (1ULL << 23)
277
278 #define CTXDESC_CD_0_ENDI (1UL << 15)
279 #define CTXDESC_CD_0_V (1UL << 31)
280
281 #define CTXDESC_CD_0_TCR_IPS GENMASK_ULL(34, 32)
282 #define ARM64_TCR_IPS GENMASK_ULL(34, 32)
283 #define CTXDESC_CD_0_TCR_TBI0 (1ULL << 38)
284 #define ARM64_TCR_TBI0 (1ULL << 37)
285
286 #define CTXDESC_CD_0_AA64 (1UL << 41)
287 #define CTXDESC_CD_0_S (1UL << 44)
288 #define CTXDESC_CD_0_R (1UL << 45)
289 #define CTXDESC_CD_0_A (1UL << 46)
290 #define CTXDESC_CD_0_ASET (1UL << 47)
291 #define CTXDESC_CD_0_ASID GENMASK_ULL(63, 48)
292
293 #define CTXDESC_CD_1_TTB0_MASK GENMASK_ULL(51, 4)
294
295
296 #define ARM_SMMU_TCR2CD(tcr, fld) FIELD_PREP(CTXDESC_CD_0_TCR_##fld, \
297 FIELD_GET(ARM64_TCR_##fld, tcr))
298
299
300 #define CMDQ_ENT_SZ_SHIFT 4
301 #define CMDQ_ENT_DWORDS ((1 << CMDQ_ENT_SZ_SHIFT) >> 3)
302 #define CMDQ_MAX_SZ_SHIFT (Q_MAX_SZ_SHIFT - CMDQ_ENT_SZ_SHIFT)
303
304 #define CMDQ_CONS_ERR GENMASK(30, 24)
305 #define CMDQ_ERR_CERROR_NONE_IDX 0
306 #define CMDQ_ERR_CERROR_ILL_IDX 1
307 #define CMDQ_ERR_CERROR_ABT_IDX 2
308 #define CMDQ_ERR_CERROR_ATC_INV_IDX 3
309
310 #define CMDQ_PROD_OWNED_FLAG Q_OVERFLOW_FLAG
311
312
313
314
315
316
317 #define CMDQ_BATCH_ENTRIES BITS_PER_LONG
318
319 #define CMDQ_0_OP GENMASK_ULL(7, 0)
320 #define CMDQ_0_SSV (1UL << 11)
321
322 #define CMDQ_PREFETCH_0_SID GENMASK_ULL(63, 32)
323 #define CMDQ_PREFETCH_1_SIZE GENMASK_ULL(4, 0)
324 #define CMDQ_PREFETCH_1_ADDR_MASK GENMASK_ULL(63, 12)
325
326 #define CMDQ_CFGI_0_SID GENMASK_ULL(63, 32)
327 #define CMDQ_CFGI_1_LEAF (1UL << 0)
328 #define CMDQ_CFGI_1_RANGE GENMASK_ULL(4, 0)
329
330 #define CMDQ_TLBI_0_VMID GENMASK_ULL(47, 32)
331 #define CMDQ_TLBI_0_ASID GENMASK_ULL(63, 48)
332 #define CMDQ_TLBI_1_LEAF (1UL << 0)
333 #define CMDQ_TLBI_1_VA_MASK GENMASK_ULL(63, 12)
334 #define CMDQ_TLBI_1_IPA_MASK GENMASK_ULL(51, 12)
335
336 #define CMDQ_ATC_0_SSID GENMASK_ULL(31, 12)
337 #define CMDQ_ATC_0_SID GENMASK_ULL(63, 32)
338 #define CMDQ_ATC_0_GLOBAL (1UL << 9)
339 #define CMDQ_ATC_1_SIZE GENMASK_ULL(5, 0)
340 #define CMDQ_ATC_1_ADDR_MASK GENMASK_ULL(63, 12)
341
342 #define CMDQ_PRI_0_SSID GENMASK_ULL(31, 12)
343 #define CMDQ_PRI_0_SID GENMASK_ULL(63, 32)
344 #define CMDQ_PRI_1_GRPID GENMASK_ULL(8, 0)
345 #define CMDQ_PRI_1_RESP GENMASK_ULL(13, 12)
346
347 #define CMDQ_SYNC_0_CS GENMASK_ULL(13, 12)
348 #define CMDQ_SYNC_0_CS_NONE 0
349 #define CMDQ_SYNC_0_CS_IRQ 1
350 #define CMDQ_SYNC_0_CS_SEV 2
351 #define CMDQ_SYNC_0_MSH GENMASK_ULL(23, 22)
352 #define CMDQ_SYNC_0_MSIATTR GENMASK_ULL(27, 24)
353 #define CMDQ_SYNC_0_MSIDATA GENMASK_ULL(63, 32)
354 #define CMDQ_SYNC_1_MSIADDR_MASK GENMASK_ULL(51, 2)
355
356
357 #define EVTQ_ENT_SZ_SHIFT 5
358 #define EVTQ_ENT_DWORDS ((1 << EVTQ_ENT_SZ_SHIFT) >> 3)
359 #define EVTQ_MAX_SZ_SHIFT (Q_MAX_SZ_SHIFT - EVTQ_ENT_SZ_SHIFT)
360
361 #define EVTQ_0_ID GENMASK_ULL(7, 0)
362
363
364 #define PRIQ_ENT_SZ_SHIFT 4
365 #define PRIQ_ENT_DWORDS ((1 << PRIQ_ENT_SZ_SHIFT) >> 3)
366 #define PRIQ_MAX_SZ_SHIFT (Q_MAX_SZ_SHIFT - PRIQ_ENT_SZ_SHIFT)
367
368 #define PRIQ_0_SID GENMASK_ULL(31, 0)
369 #define PRIQ_0_SSID GENMASK_ULL(51, 32)
370 #define PRIQ_0_PERM_PRIV (1UL << 58)
371 #define PRIQ_0_PERM_EXEC (1UL << 59)
372 #define PRIQ_0_PERM_READ (1UL << 60)
373 #define PRIQ_0_PERM_WRITE (1UL << 61)
374 #define PRIQ_0_PRG_LAST (1UL << 62)
375 #define PRIQ_0_SSID_V (1UL << 63)
376
377 #define PRIQ_1_PRG_IDX GENMASK_ULL(8, 0)
378 #define PRIQ_1_ADDR_MASK GENMASK_ULL(63, 12)
379
380
381 #define ARM_SMMU_POLL_TIMEOUT_US 1000000
382 #define ARM_SMMU_POLL_SPIN_COUNT 10
383
384 #define MSI_IOVA_BASE 0x8000000
385 #define MSI_IOVA_LENGTH 0x100000
386
387
388
389
390
391 static bool disable_bypass = 1;
392 module_param_named(disable_bypass, disable_bypass, bool, S_IRUGO);
393 MODULE_PARM_DESC(disable_bypass,
394 "Disable bypass streams such that incoming transactions from devices that are not attached to an iommu domain will report an abort back to the device and will not be allowed to pass through the SMMU.");
395
396 enum pri_resp {
397 PRI_RESP_DENY = 0,
398 PRI_RESP_FAIL = 1,
399 PRI_RESP_SUCC = 2,
400 };
401
402 enum arm_smmu_msi_index {
403 EVTQ_MSI_INDEX,
404 GERROR_MSI_INDEX,
405 PRIQ_MSI_INDEX,
406 ARM_SMMU_MAX_MSIS,
407 };
408
409 static phys_addr_t arm_smmu_msi_cfg[ARM_SMMU_MAX_MSIS][3] = {
410 [EVTQ_MSI_INDEX] = {
411 ARM_SMMU_EVTQ_IRQ_CFG0,
412 ARM_SMMU_EVTQ_IRQ_CFG1,
413 ARM_SMMU_EVTQ_IRQ_CFG2,
414 },
415 [GERROR_MSI_INDEX] = {
416 ARM_SMMU_GERROR_IRQ_CFG0,
417 ARM_SMMU_GERROR_IRQ_CFG1,
418 ARM_SMMU_GERROR_IRQ_CFG2,
419 },
420 [PRIQ_MSI_INDEX] = {
421 ARM_SMMU_PRIQ_IRQ_CFG0,
422 ARM_SMMU_PRIQ_IRQ_CFG1,
423 ARM_SMMU_PRIQ_IRQ_CFG2,
424 },
425 };
426
427 struct arm_smmu_cmdq_ent {
428
429 u8 opcode;
430 bool substream_valid;
431
432
433 union {
434 #define CMDQ_OP_PREFETCH_CFG 0x1
435 struct {
436 u32 sid;
437 u8 size;
438 u64 addr;
439 } prefetch;
440
441 #define CMDQ_OP_CFGI_STE 0x3
442 #define CMDQ_OP_CFGI_ALL 0x4
443 struct {
444 u32 sid;
445 union {
446 bool leaf;
447 u8 span;
448 };
449 } cfgi;
450
451 #define CMDQ_OP_TLBI_NH_ASID 0x11
452 #define CMDQ_OP_TLBI_NH_VA 0x12
453 #define CMDQ_OP_TLBI_EL2_ALL 0x20
454 #define CMDQ_OP_TLBI_S12_VMALL 0x28
455 #define CMDQ_OP_TLBI_S2_IPA 0x2a
456 #define CMDQ_OP_TLBI_NSNH_ALL 0x30
457 struct {
458 u16 asid;
459 u16 vmid;
460 bool leaf;
461 u64 addr;
462 } tlbi;
463
464 #define CMDQ_OP_ATC_INV 0x40
465 #define ATC_INV_SIZE_ALL 52
466 struct {
467 u32 sid;
468 u32 ssid;
469 u64 addr;
470 u8 size;
471 bool global;
472 } atc;
473
474 #define CMDQ_OP_PRI_RESP 0x41
475 struct {
476 u32 sid;
477 u32 ssid;
478 u16 grpid;
479 enum pri_resp resp;
480 } pri;
481
482 #define CMDQ_OP_CMD_SYNC 0x46
483 struct {
484 u64 msiaddr;
485 } sync;
486 };
487 };
488
489 struct arm_smmu_ll_queue {
490 union {
491 u64 val;
492 struct {
493 u32 prod;
494 u32 cons;
495 };
496 struct {
497 atomic_t prod;
498 atomic_t cons;
499 } atomic;
500 u8 __pad[SMP_CACHE_BYTES];
501 } ____cacheline_aligned_in_smp;
502 u32 max_n_shift;
503 };
504
505 struct arm_smmu_queue {
506 struct arm_smmu_ll_queue llq;
507 int irq;
508
509 __le64 *base;
510 dma_addr_t base_dma;
511 u64 q_base;
512
513 size_t ent_dwords;
514
515 u32 __iomem *prod_reg;
516 u32 __iomem *cons_reg;
517 };
518
519 struct arm_smmu_queue_poll {
520 ktime_t timeout;
521 unsigned int delay;
522 unsigned int spin_cnt;
523 bool wfe;
524 };
525
526 struct arm_smmu_cmdq {
527 struct arm_smmu_queue q;
528 atomic_long_t *valid_map;
529 atomic_t owner_prod;
530 atomic_t lock;
531 };
532
533 struct arm_smmu_evtq {
534 struct arm_smmu_queue q;
535 u32 max_stalls;
536 };
537
538 struct arm_smmu_priq {
539 struct arm_smmu_queue q;
540 };
541
542
543 struct arm_smmu_strtab_l1_desc {
544 u8 span;
545
546 __le64 *l2ptr;
547 dma_addr_t l2ptr_dma;
548 };
549
550 struct arm_smmu_s1_cfg {
551 __le64 *cdptr;
552 dma_addr_t cdptr_dma;
553
554 struct arm_smmu_ctx_desc {
555 u16 asid;
556 u64 ttbr;
557 u64 tcr;
558 u64 mair;
559 } cd;
560 };
561
562 struct arm_smmu_s2_cfg {
563 u16 vmid;
564 u64 vttbr;
565 u64 vtcr;
566 };
567
568 struct arm_smmu_strtab_cfg {
569 __le64 *strtab;
570 dma_addr_t strtab_dma;
571 struct arm_smmu_strtab_l1_desc *l1_desc;
572 unsigned int num_l1_ents;
573
574 u64 strtab_base;
575 u32 strtab_base_cfg;
576 };
577
578
579 struct arm_smmu_device {
580 struct device *dev;
581 void __iomem *base;
582
583 #define ARM_SMMU_FEAT_2_LVL_STRTAB (1 << 0)
584 #define ARM_SMMU_FEAT_2_LVL_CDTAB (1 << 1)
585 #define ARM_SMMU_FEAT_TT_LE (1 << 2)
586 #define ARM_SMMU_FEAT_TT_BE (1 << 3)
587 #define ARM_SMMU_FEAT_PRI (1 << 4)
588 #define ARM_SMMU_FEAT_ATS (1 << 5)
589 #define ARM_SMMU_FEAT_SEV (1 << 6)
590 #define ARM_SMMU_FEAT_MSI (1 << 7)
591 #define ARM_SMMU_FEAT_COHERENCY (1 << 8)
592 #define ARM_SMMU_FEAT_TRANS_S1 (1 << 9)
593 #define ARM_SMMU_FEAT_TRANS_S2 (1 << 10)
594 #define ARM_SMMU_FEAT_STALLS (1 << 11)
595 #define ARM_SMMU_FEAT_HYP (1 << 12)
596 #define ARM_SMMU_FEAT_STALL_FORCE (1 << 13)
597 #define ARM_SMMU_FEAT_VAX (1 << 14)
598 u32 features;
599
600 #define ARM_SMMU_OPT_SKIP_PREFETCH (1 << 0)
601 #define ARM_SMMU_OPT_PAGE0_REGS_ONLY (1 << 1)
602 u32 options;
603
604 struct arm_smmu_cmdq cmdq;
605 struct arm_smmu_evtq evtq;
606 struct arm_smmu_priq priq;
607
608 int gerr_irq;
609 int combined_irq;
610
611 unsigned long ias;
612 unsigned long oas;
613 unsigned long pgsize_bitmap;
614
615 #define ARM_SMMU_MAX_ASIDS (1 << 16)
616 unsigned int asid_bits;
617 DECLARE_BITMAP(asid_map, ARM_SMMU_MAX_ASIDS);
618
619 #define ARM_SMMU_MAX_VMIDS (1 << 16)
620 unsigned int vmid_bits;
621 DECLARE_BITMAP(vmid_map, ARM_SMMU_MAX_VMIDS);
622
623 unsigned int ssid_bits;
624 unsigned int sid_bits;
625
626 struct arm_smmu_strtab_cfg strtab_cfg;
627
628
629 struct iommu_device iommu;
630 };
631
632
633 struct arm_smmu_master {
634 struct arm_smmu_device *smmu;
635 struct device *dev;
636 struct arm_smmu_domain *domain;
637 struct list_head domain_head;
638 u32 *sids;
639 unsigned int num_sids;
640 bool ats_enabled;
641 };
642
643
644 enum arm_smmu_domain_stage {
645 ARM_SMMU_DOMAIN_S1 = 0,
646 ARM_SMMU_DOMAIN_S2,
647 ARM_SMMU_DOMAIN_NESTED,
648 ARM_SMMU_DOMAIN_BYPASS,
649 };
650
651 struct arm_smmu_domain {
652 struct arm_smmu_device *smmu;
653 struct mutex init_mutex;
654
655 struct io_pgtable_ops *pgtbl_ops;
656 bool non_strict;
657 atomic_t nr_ats_masters;
658
659 enum arm_smmu_domain_stage stage;
660 union {
661 struct arm_smmu_s1_cfg s1_cfg;
662 struct arm_smmu_s2_cfg s2_cfg;
663 };
664
665 struct iommu_domain domain;
666
667 struct list_head devices;
668 spinlock_t devices_lock;
669 };
670
671 struct arm_smmu_option_prop {
672 u32 opt;
673 const char *prop;
674 };
675
676 static struct arm_smmu_option_prop arm_smmu_options[] = {
677 { ARM_SMMU_OPT_SKIP_PREFETCH, "hisilicon,broken-prefetch-cmd" },
678 { ARM_SMMU_OPT_PAGE0_REGS_ONLY, "cavium,cn9900-broken-page1-regspace"},
679 { 0, NULL},
680 };
681
682 static inline void __iomem *arm_smmu_page1_fixup(unsigned long offset,
683 struct arm_smmu_device *smmu)
684 {
685 if ((offset > SZ_64K) &&
686 (smmu->options & ARM_SMMU_OPT_PAGE0_REGS_ONLY))
687 offset -= SZ_64K;
688
689 return smmu->base + offset;
690 }
691
692 static struct arm_smmu_domain *to_smmu_domain(struct iommu_domain *dom)
693 {
694 return container_of(dom, struct arm_smmu_domain, domain);
695 }
696
697 static void parse_driver_options(struct arm_smmu_device *smmu)
698 {
699 int i = 0;
700
701 do {
702 if (of_property_read_bool(smmu->dev->of_node,
703 arm_smmu_options[i].prop)) {
704 smmu->options |= arm_smmu_options[i].opt;
705 dev_notice(smmu->dev, "option %s\n",
706 arm_smmu_options[i].prop);
707 }
708 } while (arm_smmu_options[++i].opt);
709 }
710
711
712 static bool queue_has_space(struct arm_smmu_ll_queue *q, u32 n)
713 {
714 u32 space, prod, cons;
715
716 prod = Q_IDX(q, q->prod);
717 cons = Q_IDX(q, q->cons);
718
719 if (Q_WRP(q, q->prod) == Q_WRP(q, q->cons))
720 space = (1 << q->max_n_shift) - (prod - cons);
721 else
722 space = cons - prod;
723
724 return space >= n;
725 }
726
727 static bool queue_full(struct arm_smmu_ll_queue *q)
728 {
729 return Q_IDX(q, q->prod) == Q_IDX(q, q->cons) &&
730 Q_WRP(q, q->prod) != Q_WRP(q, q->cons);
731 }
732
733 static bool queue_empty(struct arm_smmu_ll_queue *q)
734 {
735 return Q_IDX(q, q->prod) == Q_IDX(q, q->cons) &&
736 Q_WRP(q, q->prod) == Q_WRP(q, q->cons);
737 }
738
739 static bool queue_consumed(struct arm_smmu_ll_queue *q, u32 prod)
740 {
741 return ((Q_WRP(q, q->cons) == Q_WRP(q, prod)) &&
742 (Q_IDX(q, q->cons) > Q_IDX(q, prod))) ||
743 ((Q_WRP(q, q->cons) != Q_WRP(q, prod)) &&
744 (Q_IDX(q, q->cons) <= Q_IDX(q, prod)));
745 }
746
747 static void queue_sync_cons_out(struct arm_smmu_queue *q)
748 {
749
750
751
752
753 mb();
754 writel_relaxed(q->llq.cons, q->cons_reg);
755 }
756
757 static void queue_inc_cons(struct arm_smmu_ll_queue *q)
758 {
759 u32 cons = (Q_WRP(q, q->cons) | Q_IDX(q, q->cons)) + 1;
760 q->cons = Q_OVF(q->cons) | Q_WRP(q, cons) | Q_IDX(q, cons);
761 }
762
763 static int queue_sync_prod_in(struct arm_smmu_queue *q)
764 {
765 int ret = 0;
766 u32 prod = readl_relaxed(q->prod_reg);
767
768 if (Q_OVF(prod) != Q_OVF(q->llq.prod))
769 ret = -EOVERFLOW;
770
771 q->llq.prod = prod;
772 return ret;
773 }
774
775 static u32 queue_inc_prod_n(struct arm_smmu_ll_queue *q, int n)
776 {
777 u32 prod = (Q_WRP(q, q->prod) | Q_IDX(q, q->prod)) + n;
778 return Q_OVF(q->prod) | Q_WRP(q, prod) | Q_IDX(q, prod);
779 }
780
781 static void queue_poll_init(struct arm_smmu_device *smmu,
782 struct arm_smmu_queue_poll *qp)
783 {
784 qp->delay = 1;
785 qp->spin_cnt = 0;
786 qp->wfe = !!(smmu->features & ARM_SMMU_FEAT_SEV);
787 qp->timeout = ktime_add_us(ktime_get(), ARM_SMMU_POLL_TIMEOUT_US);
788 }
789
790 static int queue_poll(struct arm_smmu_queue_poll *qp)
791 {
792 if (ktime_compare(ktime_get(), qp->timeout) > 0)
793 return -ETIMEDOUT;
794
795 if (qp->wfe) {
796 wfe();
797 } else if (++qp->spin_cnt < ARM_SMMU_POLL_SPIN_COUNT) {
798 cpu_relax();
799 } else {
800 udelay(qp->delay);
801 qp->delay *= 2;
802 qp->spin_cnt = 0;
803 }
804
805 return 0;
806 }
807
808 static void queue_write(__le64 *dst, u64 *src, size_t n_dwords)
809 {
810 int i;
811
812 for (i = 0; i < n_dwords; ++i)
813 *dst++ = cpu_to_le64(*src++);
814 }
815
816 static void queue_read(__le64 *dst, u64 *src, size_t n_dwords)
817 {
818 int i;
819
820 for (i = 0; i < n_dwords; ++i)
821 *dst++ = le64_to_cpu(*src++);
822 }
823
824 static int queue_remove_raw(struct arm_smmu_queue *q, u64 *ent)
825 {
826 if (queue_empty(&q->llq))
827 return -EAGAIN;
828
829 queue_read(ent, Q_ENT(q, q->llq.cons), q->ent_dwords);
830 queue_inc_cons(&q->llq);
831 queue_sync_cons_out(q);
832 return 0;
833 }
834
835
836 static int arm_smmu_cmdq_build_cmd(u64 *cmd, struct arm_smmu_cmdq_ent *ent)
837 {
838 memset(cmd, 0, 1 << CMDQ_ENT_SZ_SHIFT);
839 cmd[0] |= FIELD_PREP(CMDQ_0_OP, ent->opcode);
840
841 switch (ent->opcode) {
842 case CMDQ_OP_TLBI_EL2_ALL:
843 case CMDQ_OP_TLBI_NSNH_ALL:
844 break;
845 case CMDQ_OP_PREFETCH_CFG:
846 cmd[0] |= FIELD_PREP(CMDQ_PREFETCH_0_SID, ent->prefetch.sid);
847 cmd[1] |= FIELD_PREP(CMDQ_PREFETCH_1_SIZE, ent->prefetch.size);
848 cmd[1] |= ent->prefetch.addr & CMDQ_PREFETCH_1_ADDR_MASK;
849 break;
850 case CMDQ_OP_CFGI_STE:
851 cmd[0] |= FIELD_PREP(CMDQ_CFGI_0_SID, ent->cfgi.sid);
852 cmd[1] |= FIELD_PREP(CMDQ_CFGI_1_LEAF, ent->cfgi.leaf);
853 break;
854 case CMDQ_OP_CFGI_ALL:
855
856 cmd[1] |= FIELD_PREP(CMDQ_CFGI_1_RANGE, 31);
857 break;
858 case CMDQ_OP_TLBI_NH_VA:
859 cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_VMID, ent->tlbi.vmid);
860 cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_ASID, ent->tlbi.asid);
861 cmd[1] |= FIELD_PREP(CMDQ_TLBI_1_LEAF, ent->tlbi.leaf);
862 cmd[1] |= ent->tlbi.addr & CMDQ_TLBI_1_VA_MASK;
863 break;
864 case CMDQ_OP_TLBI_S2_IPA:
865 cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_VMID, ent->tlbi.vmid);
866 cmd[1] |= FIELD_PREP(CMDQ_TLBI_1_LEAF, ent->tlbi.leaf);
867 cmd[1] |= ent->tlbi.addr & CMDQ_TLBI_1_IPA_MASK;
868 break;
869 case CMDQ_OP_TLBI_NH_ASID:
870 cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_ASID, ent->tlbi.asid);
871
872 case CMDQ_OP_TLBI_S12_VMALL:
873 cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_VMID, ent->tlbi.vmid);
874 break;
875 case CMDQ_OP_ATC_INV:
876 cmd[0] |= FIELD_PREP(CMDQ_0_SSV, ent->substream_valid);
877 cmd[0] |= FIELD_PREP(CMDQ_ATC_0_GLOBAL, ent->atc.global);
878 cmd[0] |= FIELD_PREP(CMDQ_ATC_0_SSID, ent->atc.ssid);
879 cmd[0] |= FIELD_PREP(CMDQ_ATC_0_SID, ent->atc.sid);
880 cmd[1] |= FIELD_PREP(CMDQ_ATC_1_SIZE, ent->atc.size);
881 cmd[1] |= ent->atc.addr & CMDQ_ATC_1_ADDR_MASK;
882 break;
883 case CMDQ_OP_PRI_RESP:
884 cmd[0] |= FIELD_PREP(CMDQ_0_SSV, ent->substream_valid);
885 cmd[0] |= FIELD_PREP(CMDQ_PRI_0_SSID, ent->pri.ssid);
886 cmd[0] |= FIELD_PREP(CMDQ_PRI_0_SID, ent->pri.sid);
887 cmd[1] |= FIELD_PREP(CMDQ_PRI_1_GRPID, ent->pri.grpid);
888 switch (ent->pri.resp) {
889 case PRI_RESP_DENY:
890 case PRI_RESP_FAIL:
891 case PRI_RESP_SUCC:
892 break;
893 default:
894 return -EINVAL;
895 }
896 cmd[1] |= FIELD_PREP(CMDQ_PRI_1_RESP, ent->pri.resp);
897 break;
898 case CMDQ_OP_CMD_SYNC:
899 if (ent->sync.msiaddr) {
900 cmd[0] |= FIELD_PREP(CMDQ_SYNC_0_CS, CMDQ_SYNC_0_CS_IRQ);
901 cmd[1] |= ent->sync.msiaddr & CMDQ_SYNC_1_MSIADDR_MASK;
902 } else {
903 cmd[0] |= FIELD_PREP(CMDQ_SYNC_0_CS, CMDQ_SYNC_0_CS_SEV);
904 }
905 cmd[0] |= FIELD_PREP(CMDQ_SYNC_0_MSH, ARM_SMMU_SH_ISH);
906 cmd[0] |= FIELD_PREP(CMDQ_SYNC_0_MSIATTR, ARM_SMMU_MEMATTR_OIWB);
907 break;
908 default:
909 return -ENOENT;
910 }
911
912 return 0;
913 }
914
915 static void arm_smmu_cmdq_build_sync_cmd(u64 *cmd, struct arm_smmu_device *smmu,
916 u32 prod)
917 {
918 struct arm_smmu_queue *q = &smmu->cmdq.q;
919 struct arm_smmu_cmdq_ent ent = {
920 .opcode = CMDQ_OP_CMD_SYNC,
921 };
922
923
924
925
926
927 if (smmu->features & ARM_SMMU_FEAT_MSI &&
928 smmu->features & ARM_SMMU_FEAT_COHERENCY) {
929 ent.sync.msiaddr = q->base_dma + Q_IDX(&q->llq, prod) *
930 q->ent_dwords * 8;
931 }
932
933 arm_smmu_cmdq_build_cmd(cmd, &ent);
934 }
935
936 static void arm_smmu_cmdq_skip_err(struct arm_smmu_device *smmu)
937 {
938 static const char *cerror_str[] = {
939 [CMDQ_ERR_CERROR_NONE_IDX] = "No error",
940 [CMDQ_ERR_CERROR_ILL_IDX] = "Illegal command",
941 [CMDQ_ERR_CERROR_ABT_IDX] = "Abort on command fetch",
942 [CMDQ_ERR_CERROR_ATC_INV_IDX] = "ATC invalidate timeout",
943 };
944
945 int i;
946 u64 cmd[CMDQ_ENT_DWORDS];
947 struct arm_smmu_queue *q = &smmu->cmdq.q;
948 u32 cons = readl_relaxed(q->cons_reg);
949 u32 idx = FIELD_GET(CMDQ_CONS_ERR, cons);
950 struct arm_smmu_cmdq_ent cmd_sync = {
951 .opcode = CMDQ_OP_CMD_SYNC,
952 };
953
954 dev_err(smmu->dev, "CMDQ error (cons 0x%08x): %s\n", cons,
955 idx < ARRAY_SIZE(cerror_str) ? cerror_str[idx] : "Unknown");
956
957 switch (idx) {
958 case CMDQ_ERR_CERROR_ABT_IDX:
959 dev_err(smmu->dev, "retrying command fetch\n");
960 case CMDQ_ERR_CERROR_NONE_IDX:
961 return;
962 case CMDQ_ERR_CERROR_ATC_INV_IDX:
963
964
965
966
967
968
969 return;
970 case CMDQ_ERR_CERROR_ILL_IDX:
971
972 default:
973 break;
974 }
975
976
977
978
979
980 queue_read(cmd, Q_ENT(q, cons), q->ent_dwords);
981 dev_err(smmu->dev, "skipping command in error state:\n");
982 for (i = 0; i < ARRAY_SIZE(cmd); ++i)
983 dev_err(smmu->dev, "\t0x%016llx\n", (unsigned long long)cmd[i]);
984
985
986 if (arm_smmu_cmdq_build_cmd(cmd, &cmd_sync)) {
987 dev_err(smmu->dev, "failed to convert to CMD_SYNC\n");
988 return;
989 }
990
991 queue_write(Q_ENT(q, cons), cmd, q->ent_dwords);
992 }
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006 static void arm_smmu_cmdq_shared_lock(struct arm_smmu_cmdq *cmdq)
1007 {
1008 int val;
1009
1010
1011
1012
1013
1014
1015
1016 if (atomic_fetch_inc_relaxed(&cmdq->lock) >= 0)
1017 return;
1018
1019 do {
1020 val = atomic_cond_read_relaxed(&cmdq->lock, VAL >= 0);
1021 } while (atomic_cmpxchg_relaxed(&cmdq->lock, val, val + 1) != val);
1022 }
1023
1024 static void arm_smmu_cmdq_shared_unlock(struct arm_smmu_cmdq *cmdq)
1025 {
1026 (void)atomic_dec_return_release(&cmdq->lock);
1027 }
1028
1029 static bool arm_smmu_cmdq_shared_tryunlock(struct arm_smmu_cmdq *cmdq)
1030 {
1031 if (atomic_read(&cmdq->lock) == 1)
1032 return false;
1033
1034 arm_smmu_cmdq_shared_unlock(cmdq);
1035 return true;
1036 }
1037
1038 #define arm_smmu_cmdq_exclusive_trylock_irqsave(cmdq, flags) \
1039 ({ \
1040 bool __ret; \
1041 local_irq_save(flags); \
1042 __ret = !atomic_cmpxchg_relaxed(&cmdq->lock, 0, INT_MIN); \
1043 if (!__ret) \
1044 local_irq_restore(flags); \
1045 __ret; \
1046 })
1047
1048 #define arm_smmu_cmdq_exclusive_unlock_irqrestore(cmdq, flags) \
1049 ({ \
1050 atomic_set_release(&cmdq->lock, 0); \
1051 local_irq_restore(flags); \
1052 })
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094 static void __arm_smmu_cmdq_poll_set_valid_map(struct arm_smmu_cmdq *cmdq,
1095 u32 sprod, u32 eprod, bool set)
1096 {
1097 u32 swidx, sbidx, ewidx, ebidx;
1098 struct arm_smmu_ll_queue llq = {
1099 .max_n_shift = cmdq->q.llq.max_n_shift,
1100 .prod = sprod,
1101 };
1102
1103 ewidx = BIT_WORD(Q_IDX(&llq, eprod));
1104 ebidx = Q_IDX(&llq, eprod) % BITS_PER_LONG;
1105
1106 while (llq.prod != eprod) {
1107 unsigned long mask;
1108 atomic_long_t *ptr;
1109 u32 limit = BITS_PER_LONG;
1110
1111 swidx = BIT_WORD(Q_IDX(&llq, llq.prod));
1112 sbidx = Q_IDX(&llq, llq.prod) % BITS_PER_LONG;
1113
1114 ptr = &cmdq->valid_map[swidx];
1115
1116 if ((swidx == ewidx) && (sbidx < ebidx))
1117 limit = ebidx;
1118
1119 mask = GENMASK(limit - 1, sbidx);
1120
1121
1122
1123
1124
1125
1126
1127 if (set) {
1128 atomic_long_xor(mask, ptr);
1129 } else {
1130 unsigned long valid;
1131
1132 valid = (ULONG_MAX + !!Q_WRP(&llq, llq.prod)) & mask;
1133 atomic_long_cond_read_relaxed(ptr, (VAL & mask) == valid);
1134 }
1135
1136 llq.prod = queue_inc_prod_n(&llq, limit - sbidx);
1137 }
1138 }
1139
1140
1141 static void arm_smmu_cmdq_set_valid_map(struct arm_smmu_cmdq *cmdq,
1142 u32 sprod, u32 eprod)
1143 {
1144 __arm_smmu_cmdq_poll_set_valid_map(cmdq, sprod, eprod, true);
1145 }
1146
1147
1148 static void arm_smmu_cmdq_poll_valid_map(struct arm_smmu_cmdq *cmdq,
1149 u32 sprod, u32 eprod)
1150 {
1151 __arm_smmu_cmdq_poll_set_valid_map(cmdq, sprod, eprod, false);
1152 }
1153
1154
1155 static int arm_smmu_cmdq_poll_until_not_full(struct arm_smmu_device *smmu,
1156 struct arm_smmu_ll_queue *llq)
1157 {
1158 unsigned long flags;
1159 struct arm_smmu_queue_poll qp;
1160 struct arm_smmu_cmdq *cmdq = &smmu->cmdq;
1161 int ret = 0;
1162
1163
1164
1165
1166
1167 if (arm_smmu_cmdq_exclusive_trylock_irqsave(cmdq, flags)) {
1168 WRITE_ONCE(cmdq->q.llq.cons, readl_relaxed(cmdq->q.cons_reg));
1169 arm_smmu_cmdq_exclusive_unlock_irqrestore(cmdq, flags);
1170 llq->val = READ_ONCE(cmdq->q.llq.val);
1171 return 0;
1172 }
1173
1174 queue_poll_init(smmu, &qp);
1175 do {
1176 llq->val = READ_ONCE(smmu->cmdq.q.llq.val);
1177 if (!queue_full(llq))
1178 break;
1179
1180 ret = queue_poll(&qp);
1181 } while (!ret);
1182
1183 return ret;
1184 }
1185
1186
1187
1188
1189
1190 static int __arm_smmu_cmdq_poll_until_msi(struct arm_smmu_device *smmu,
1191 struct arm_smmu_ll_queue *llq)
1192 {
1193 int ret = 0;
1194 struct arm_smmu_queue_poll qp;
1195 struct arm_smmu_cmdq *cmdq = &smmu->cmdq;
1196 u32 *cmd = (u32 *)(Q_ENT(&cmdq->q, llq->prod));
1197
1198 queue_poll_init(smmu, &qp);
1199
1200
1201
1202
1203
1204 qp.wfe = false;
1205 smp_cond_load_relaxed(cmd, !VAL || (ret = queue_poll(&qp)));
1206 llq->cons = ret ? llq->prod : queue_inc_prod_n(llq, 1);
1207 return ret;
1208 }
1209
1210
1211
1212
1213
1214 static int __arm_smmu_cmdq_poll_until_consumed(struct arm_smmu_device *smmu,
1215 struct arm_smmu_ll_queue *llq)
1216 {
1217 struct arm_smmu_queue_poll qp;
1218 struct arm_smmu_cmdq *cmdq = &smmu->cmdq;
1219 u32 prod = llq->prod;
1220 int ret = 0;
1221
1222 queue_poll_init(smmu, &qp);
1223 llq->val = READ_ONCE(smmu->cmdq.q.llq.val);
1224 do {
1225 if (queue_consumed(llq, prod))
1226 break;
1227
1228 ret = queue_poll(&qp);
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258 llq->cons = readl(cmdq->q.cons_reg);
1259 } while (!ret);
1260
1261 return ret;
1262 }
1263
1264 static int arm_smmu_cmdq_poll_until_sync(struct arm_smmu_device *smmu,
1265 struct arm_smmu_ll_queue *llq)
1266 {
1267 if (smmu->features & ARM_SMMU_FEAT_MSI &&
1268 smmu->features & ARM_SMMU_FEAT_COHERENCY)
1269 return __arm_smmu_cmdq_poll_until_msi(smmu, llq);
1270
1271 return __arm_smmu_cmdq_poll_until_consumed(smmu, llq);
1272 }
1273
1274 static void arm_smmu_cmdq_write_entries(struct arm_smmu_cmdq *cmdq, u64 *cmds,
1275 u32 prod, int n)
1276 {
1277 int i;
1278 struct arm_smmu_ll_queue llq = {
1279 .max_n_shift = cmdq->q.llq.max_n_shift,
1280 .prod = prod,
1281 };
1282
1283 for (i = 0; i < n; ++i) {
1284 u64 *cmd = &cmds[i * CMDQ_ENT_DWORDS];
1285
1286 prod = queue_inc_prod_n(&llq, i);
1287 queue_write(Q_ENT(&cmdq->q, prod), cmd, CMDQ_ENT_DWORDS);
1288 }
1289 }
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307 static int arm_smmu_cmdq_issue_cmdlist(struct arm_smmu_device *smmu,
1308 u64 *cmds, int n, bool sync)
1309 {
1310 u64 cmd_sync[CMDQ_ENT_DWORDS];
1311 u32 prod;
1312 unsigned long flags;
1313 bool owner;
1314 struct arm_smmu_cmdq *cmdq = &smmu->cmdq;
1315 struct arm_smmu_ll_queue llq = {
1316 .max_n_shift = cmdq->q.llq.max_n_shift,
1317 }, head = llq;
1318 int ret = 0;
1319
1320
1321 local_irq_save(flags);
1322 llq.val = READ_ONCE(cmdq->q.llq.val);
1323 do {
1324 u64 old;
1325
1326 while (!queue_has_space(&llq, n + sync)) {
1327 local_irq_restore(flags);
1328 if (arm_smmu_cmdq_poll_until_not_full(smmu, &llq))
1329 dev_err_ratelimited(smmu->dev, "CMDQ timeout\n");
1330 local_irq_save(flags);
1331 }
1332
1333 head.cons = llq.cons;
1334 head.prod = queue_inc_prod_n(&llq, n + sync) |
1335 CMDQ_PROD_OWNED_FLAG;
1336
1337 old = cmpxchg_relaxed(&cmdq->q.llq.val, llq.val, head.val);
1338 if (old == llq.val)
1339 break;
1340
1341 llq.val = old;
1342 } while (1);
1343 owner = !(llq.prod & CMDQ_PROD_OWNED_FLAG);
1344 head.prod &= ~CMDQ_PROD_OWNED_FLAG;
1345 llq.prod &= ~CMDQ_PROD_OWNED_FLAG;
1346
1347
1348
1349
1350
1351 arm_smmu_cmdq_write_entries(cmdq, cmds, llq.prod, n);
1352 if (sync) {
1353 prod = queue_inc_prod_n(&llq, n);
1354 arm_smmu_cmdq_build_sync_cmd(cmd_sync, smmu, prod);
1355 queue_write(Q_ENT(&cmdq->q, prod), cmd_sync, CMDQ_ENT_DWORDS);
1356
1357
1358
1359
1360
1361
1362
1363 arm_smmu_cmdq_shared_lock(cmdq);
1364 }
1365
1366
1367 dma_wmb();
1368 arm_smmu_cmdq_set_valid_map(cmdq, llq.prod, head.prod);
1369
1370
1371 if (owner) {
1372
1373 atomic_cond_read_relaxed(&cmdq->owner_prod, VAL == llq.prod);
1374
1375
1376 prod = atomic_fetch_andnot_relaxed(CMDQ_PROD_OWNED_FLAG,
1377 &cmdq->q.llq.atomic.prod);
1378 prod &= ~CMDQ_PROD_OWNED_FLAG;
1379
1380
1381
1382
1383
1384
1385 arm_smmu_cmdq_poll_valid_map(cmdq, llq.prod, prod);
1386
1387
1388
1389
1390
1391 writel_relaxed(prod, cmdq->q.prod_reg);
1392
1393
1394
1395
1396
1397
1398 atomic_set_release(&cmdq->owner_prod, prod);
1399 }
1400
1401
1402 if (sync) {
1403 llq.prod = queue_inc_prod_n(&llq, n);
1404 ret = arm_smmu_cmdq_poll_until_sync(smmu, &llq);
1405 if (ret) {
1406 dev_err_ratelimited(smmu->dev,
1407 "CMD_SYNC timeout at 0x%08x [hwprod 0x%08x, hwcons 0x%08x]\n",
1408 llq.prod,
1409 readl_relaxed(cmdq->q.prod_reg),
1410 readl_relaxed(cmdq->q.cons_reg));
1411 }
1412
1413
1414
1415
1416
1417 if (!arm_smmu_cmdq_shared_tryunlock(cmdq)) {
1418 WRITE_ONCE(cmdq->q.llq.cons, llq.cons);
1419 arm_smmu_cmdq_shared_unlock(cmdq);
1420 }
1421 }
1422
1423 local_irq_restore(flags);
1424 return ret;
1425 }
1426
1427 static int arm_smmu_cmdq_issue_cmd(struct arm_smmu_device *smmu,
1428 struct arm_smmu_cmdq_ent *ent)
1429 {
1430 u64 cmd[CMDQ_ENT_DWORDS];
1431
1432 if (arm_smmu_cmdq_build_cmd(cmd, ent)) {
1433 dev_warn(smmu->dev, "ignoring unknown CMDQ opcode 0x%x\n",
1434 ent->opcode);
1435 return -EINVAL;
1436 }
1437
1438 return arm_smmu_cmdq_issue_cmdlist(smmu, cmd, 1, false);
1439 }
1440
1441 static int arm_smmu_cmdq_issue_sync(struct arm_smmu_device *smmu)
1442 {
1443 return arm_smmu_cmdq_issue_cmdlist(smmu, NULL, 0, true);
1444 }
1445
1446
1447 static u64 arm_smmu_cpu_tcr_to_cd(u64 tcr)
1448 {
1449 u64 val = 0;
1450
1451
1452 val |= ARM_SMMU_TCR2CD(tcr, T0SZ);
1453 val |= ARM_SMMU_TCR2CD(tcr, TG0);
1454 val |= ARM_SMMU_TCR2CD(tcr, IRGN0);
1455 val |= ARM_SMMU_TCR2CD(tcr, ORGN0);
1456 val |= ARM_SMMU_TCR2CD(tcr, SH0);
1457 val |= ARM_SMMU_TCR2CD(tcr, EPD0);
1458 val |= ARM_SMMU_TCR2CD(tcr, EPD1);
1459 val |= ARM_SMMU_TCR2CD(tcr, IPS);
1460
1461 return val;
1462 }
1463
1464 static void arm_smmu_write_ctx_desc(struct arm_smmu_device *smmu,
1465 struct arm_smmu_s1_cfg *cfg)
1466 {
1467 u64 val;
1468
1469
1470
1471
1472
1473 val = arm_smmu_cpu_tcr_to_cd(cfg->cd.tcr) |
1474 #ifdef __BIG_ENDIAN
1475 CTXDESC_CD_0_ENDI |
1476 #endif
1477 CTXDESC_CD_0_R | CTXDESC_CD_0_A | CTXDESC_CD_0_ASET |
1478 CTXDESC_CD_0_AA64 | FIELD_PREP(CTXDESC_CD_0_ASID, cfg->cd.asid) |
1479 CTXDESC_CD_0_V;
1480
1481
1482 if (smmu->features & ARM_SMMU_FEAT_STALL_FORCE)
1483 val |= CTXDESC_CD_0_S;
1484
1485 cfg->cdptr[0] = cpu_to_le64(val);
1486
1487 val = cfg->cd.ttbr & CTXDESC_CD_1_TTB0_MASK;
1488 cfg->cdptr[1] = cpu_to_le64(val);
1489
1490 cfg->cdptr[3] = cpu_to_le64(cfg->cd.mair);
1491 }
1492
1493
1494 static void
1495 arm_smmu_write_strtab_l1_desc(__le64 *dst, struct arm_smmu_strtab_l1_desc *desc)
1496 {
1497 u64 val = 0;
1498
1499 val |= FIELD_PREP(STRTAB_L1_DESC_SPAN, desc->span);
1500 val |= desc->l2ptr_dma & STRTAB_L1_DESC_L2PTR_MASK;
1501
1502 *dst = cpu_to_le64(val);
1503 }
1504
1505 static void arm_smmu_sync_ste_for_sid(struct arm_smmu_device *smmu, u32 sid)
1506 {
1507 struct arm_smmu_cmdq_ent cmd = {
1508 .opcode = CMDQ_OP_CFGI_STE,
1509 .cfgi = {
1510 .sid = sid,
1511 .leaf = true,
1512 },
1513 };
1514
1515 arm_smmu_cmdq_issue_cmd(smmu, &cmd);
1516 arm_smmu_cmdq_issue_sync(smmu);
1517 }
1518
1519 static void arm_smmu_write_strtab_ent(struct arm_smmu_master *master, u32 sid,
1520 __le64 *dst)
1521 {
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538 u64 val = le64_to_cpu(dst[0]);
1539 bool ste_live = false;
1540 struct arm_smmu_device *smmu = NULL;
1541 struct arm_smmu_s1_cfg *s1_cfg = NULL;
1542 struct arm_smmu_s2_cfg *s2_cfg = NULL;
1543 struct arm_smmu_domain *smmu_domain = NULL;
1544 struct arm_smmu_cmdq_ent prefetch_cmd = {
1545 .opcode = CMDQ_OP_PREFETCH_CFG,
1546 .prefetch = {
1547 .sid = sid,
1548 },
1549 };
1550
1551 if (master) {
1552 smmu_domain = master->domain;
1553 smmu = master->smmu;
1554 }
1555
1556 if (smmu_domain) {
1557 switch (smmu_domain->stage) {
1558 case ARM_SMMU_DOMAIN_S1:
1559 s1_cfg = &smmu_domain->s1_cfg;
1560 break;
1561 case ARM_SMMU_DOMAIN_S2:
1562 case ARM_SMMU_DOMAIN_NESTED:
1563 s2_cfg = &smmu_domain->s2_cfg;
1564 break;
1565 default:
1566 break;
1567 }
1568 }
1569
1570 if (val & STRTAB_STE_0_V) {
1571 switch (FIELD_GET(STRTAB_STE_0_CFG, val)) {
1572 case STRTAB_STE_0_CFG_BYPASS:
1573 break;
1574 case STRTAB_STE_0_CFG_S1_TRANS:
1575 case STRTAB_STE_0_CFG_S2_TRANS:
1576 ste_live = true;
1577 break;
1578 case STRTAB_STE_0_CFG_ABORT:
1579 BUG_ON(!disable_bypass);
1580 break;
1581 default:
1582 BUG();
1583 }
1584 }
1585
1586
1587 val = STRTAB_STE_0_V;
1588
1589
1590 if (!smmu_domain || !(s1_cfg || s2_cfg)) {
1591 if (!smmu_domain && disable_bypass)
1592 val |= FIELD_PREP(STRTAB_STE_0_CFG, STRTAB_STE_0_CFG_ABORT);
1593 else
1594 val |= FIELD_PREP(STRTAB_STE_0_CFG, STRTAB_STE_0_CFG_BYPASS);
1595
1596 dst[0] = cpu_to_le64(val);
1597 dst[1] = cpu_to_le64(FIELD_PREP(STRTAB_STE_1_SHCFG,
1598 STRTAB_STE_1_SHCFG_INCOMING));
1599 dst[2] = 0;
1600
1601
1602
1603
1604 if (smmu)
1605 arm_smmu_sync_ste_for_sid(smmu, sid);
1606 return;
1607 }
1608
1609 if (s1_cfg) {
1610 BUG_ON(ste_live);
1611 dst[1] = cpu_to_le64(
1612 FIELD_PREP(STRTAB_STE_1_S1CIR, STRTAB_STE_1_S1C_CACHE_WBRA) |
1613 FIELD_PREP(STRTAB_STE_1_S1COR, STRTAB_STE_1_S1C_CACHE_WBRA) |
1614 FIELD_PREP(STRTAB_STE_1_S1CSH, ARM_SMMU_SH_ISH) |
1615 FIELD_PREP(STRTAB_STE_1_STRW, STRTAB_STE_1_STRW_NSEL1));
1616
1617 if (smmu->features & ARM_SMMU_FEAT_STALLS &&
1618 !(smmu->features & ARM_SMMU_FEAT_STALL_FORCE))
1619 dst[1] |= cpu_to_le64(STRTAB_STE_1_S1STALLD);
1620
1621 val |= (s1_cfg->cdptr_dma & STRTAB_STE_0_S1CTXPTR_MASK) |
1622 FIELD_PREP(STRTAB_STE_0_CFG, STRTAB_STE_0_CFG_S1_TRANS);
1623 }
1624
1625 if (s2_cfg) {
1626 BUG_ON(ste_live);
1627 dst[2] = cpu_to_le64(
1628 FIELD_PREP(STRTAB_STE_2_S2VMID, s2_cfg->vmid) |
1629 FIELD_PREP(STRTAB_STE_2_VTCR, s2_cfg->vtcr) |
1630 #ifdef __BIG_ENDIAN
1631 STRTAB_STE_2_S2ENDI |
1632 #endif
1633 STRTAB_STE_2_S2PTW | STRTAB_STE_2_S2AA64 |
1634 STRTAB_STE_2_S2R);
1635
1636 dst[3] = cpu_to_le64(s2_cfg->vttbr & STRTAB_STE_3_S2TTB_MASK);
1637
1638 val |= FIELD_PREP(STRTAB_STE_0_CFG, STRTAB_STE_0_CFG_S2_TRANS);
1639 }
1640
1641 if (master->ats_enabled)
1642 dst[1] |= cpu_to_le64(FIELD_PREP(STRTAB_STE_1_EATS,
1643 STRTAB_STE_1_EATS_TRANS));
1644
1645 arm_smmu_sync_ste_for_sid(smmu, sid);
1646
1647 WRITE_ONCE(dst[0], cpu_to_le64(val));
1648 arm_smmu_sync_ste_for_sid(smmu, sid);
1649
1650
1651 if (!(smmu->options & ARM_SMMU_OPT_SKIP_PREFETCH))
1652 arm_smmu_cmdq_issue_cmd(smmu, &prefetch_cmd);
1653 }
1654
1655 static void arm_smmu_init_bypass_stes(u64 *strtab, unsigned int nent)
1656 {
1657 unsigned int i;
1658
1659 for (i = 0; i < nent; ++i) {
1660 arm_smmu_write_strtab_ent(NULL, -1, strtab);
1661 strtab += STRTAB_STE_DWORDS;
1662 }
1663 }
1664
1665 static int arm_smmu_init_l2_strtab(struct arm_smmu_device *smmu, u32 sid)
1666 {
1667 size_t size;
1668 void *strtab;
1669 struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg;
1670 struct arm_smmu_strtab_l1_desc *desc = &cfg->l1_desc[sid >> STRTAB_SPLIT];
1671
1672 if (desc->l2ptr)
1673 return 0;
1674
1675 size = 1 << (STRTAB_SPLIT + ilog2(STRTAB_STE_DWORDS) + 3);
1676 strtab = &cfg->strtab[(sid >> STRTAB_SPLIT) * STRTAB_L1_DESC_DWORDS];
1677
1678 desc->span = STRTAB_SPLIT + 1;
1679 desc->l2ptr = dmam_alloc_coherent(smmu->dev, size, &desc->l2ptr_dma,
1680 GFP_KERNEL | __GFP_ZERO);
1681 if (!desc->l2ptr) {
1682 dev_err(smmu->dev,
1683 "failed to allocate l2 stream table for SID %u\n",
1684 sid);
1685 return -ENOMEM;
1686 }
1687
1688 arm_smmu_init_bypass_stes(desc->l2ptr, 1 << STRTAB_SPLIT);
1689 arm_smmu_write_strtab_l1_desc(strtab, desc);
1690 return 0;
1691 }
1692
1693
1694 static irqreturn_t arm_smmu_evtq_thread(int irq, void *dev)
1695 {
1696 int i;
1697 struct arm_smmu_device *smmu = dev;
1698 struct arm_smmu_queue *q = &smmu->evtq.q;
1699 struct arm_smmu_ll_queue *llq = &q->llq;
1700 u64 evt[EVTQ_ENT_DWORDS];
1701
1702 do {
1703 while (!queue_remove_raw(q, evt)) {
1704 u8 id = FIELD_GET(EVTQ_0_ID, evt[0]);
1705
1706 dev_info(smmu->dev, "event 0x%02x received:\n", id);
1707 for (i = 0; i < ARRAY_SIZE(evt); ++i)
1708 dev_info(smmu->dev, "\t0x%016llx\n",
1709 (unsigned long long)evt[i]);
1710
1711 }
1712
1713
1714
1715
1716
1717 if (queue_sync_prod_in(q) == -EOVERFLOW)
1718 dev_err(smmu->dev, "EVTQ overflow detected -- events lost\n");
1719 } while (!queue_empty(llq));
1720
1721
1722 llq->cons = Q_OVF(llq->prod) | Q_WRP(llq, llq->cons) |
1723 Q_IDX(llq, llq->cons);
1724 return IRQ_HANDLED;
1725 }
1726
1727 static void arm_smmu_handle_ppr(struct arm_smmu_device *smmu, u64 *evt)
1728 {
1729 u32 sid, ssid;
1730 u16 grpid;
1731 bool ssv, last;
1732
1733 sid = FIELD_GET(PRIQ_0_SID, evt[0]);
1734 ssv = FIELD_GET(PRIQ_0_SSID_V, evt[0]);
1735 ssid = ssv ? FIELD_GET(PRIQ_0_SSID, evt[0]) : 0;
1736 last = FIELD_GET(PRIQ_0_PRG_LAST, evt[0]);
1737 grpid = FIELD_GET(PRIQ_1_PRG_IDX, evt[1]);
1738
1739 dev_info(smmu->dev, "unexpected PRI request received:\n");
1740 dev_info(smmu->dev,
1741 "\tsid 0x%08x.0x%05x: [%u%s] %sprivileged %s%s%s access at iova 0x%016llx\n",
1742 sid, ssid, grpid, last ? "L" : "",
1743 evt[0] & PRIQ_0_PERM_PRIV ? "" : "un",
1744 evt[0] & PRIQ_0_PERM_READ ? "R" : "",
1745 evt[0] & PRIQ_0_PERM_WRITE ? "W" : "",
1746 evt[0] & PRIQ_0_PERM_EXEC ? "X" : "",
1747 evt[1] & PRIQ_1_ADDR_MASK);
1748
1749 if (last) {
1750 struct arm_smmu_cmdq_ent cmd = {
1751 .opcode = CMDQ_OP_PRI_RESP,
1752 .substream_valid = ssv,
1753 .pri = {
1754 .sid = sid,
1755 .ssid = ssid,
1756 .grpid = grpid,
1757 .resp = PRI_RESP_DENY,
1758 },
1759 };
1760
1761 arm_smmu_cmdq_issue_cmd(smmu, &cmd);
1762 }
1763 }
1764
1765 static irqreturn_t arm_smmu_priq_thread(int irq, void *dev)
1766 {
1767 struct arm_smmu_device *smmu = dev;
1768 struct arm_smmu_queue *q = &smmu->priq.q;
1769 struct arm_smmu_ll_queue *llq = &q->llq;
1770 u64 evt[PRIQ_ENT_DWORDS];
1771
1772 do {
1773 while (!queue_remove_raw(q, evt))
1774 arm_smmu_handle_ppr(smmu, evt);
1775
1776 if (queue_sync_prod_in(q) == -EOVERFLOW)
1777 dev_err(smmu->dev, "PRIQ overflow detected -- requests lost\n");
1778 } while (!queue_empty(llq));
1779
1780
1781 llq->cons = Q_OVF(llq->prod) | Q_WRP(llq, llq->cons) |
1782 Q_IDX(llq, llq->cons);
1783 queue_sync_cons_out(q);
1784 return IRQ_HANDLED;
1785 }
1786
1787 static int arm_smmu_device_disable(struct arm_smmu_device *smmu);
1788
1789 static irqreturn_t arm_smmu_gerror_handler(int irq, void *dev)
1790 {
1791 u32 gerror, gerrorn, active;
1792 struct arm_smmu_device *smmu = dev;
1793
1794 gerror = readl_relaxed(smmu->base + ARM_SMMU_GERROR);
1795 gerrorn = readl_relaxed(smmu->base + ARM_SMMU_GERRORN);
1796
1797 active = gerror ^ gerrorn;
1798 if (!(active & GERROR_ERR_MASK))
1799 return IRQ_NONE;
1800
1801 dev_warn(smmu->dev,
1802 "unexpected global error reported (0x%08x), this could be serious\n",
1803 active);
1804
1805 if (active & GERROR_SFM_ERR) {
1806 dev_err(smmu->dev, "device has entered Service Failure Mode!\n");
1807 arm_smmu_device_disable(smmu);
1808 }
1809
1810 if (active & GERROR_MSI_GERROR_ABT_ERR)
1811 dev_warn(smmu->dev, "GERROR MSI write aborted\n");
1812
1813 if (active & GERROR_MSI_PRIQ_ABT_ERR)
1814 dev_warn(smmu->dev, "PRIQ MSI write aborted\n");
1815
1816 if (active & GERROR_MSI_EVTQ_ABT_ERR)
1817 dev_warn(smmu->dev, "EVTQ MSI write aborted\n");
1818
1819 if (active & GERROR_MSI_CMDQ_ABT_ERR)
1820 dev_warn(smmu->dev, "CMDQ MSI write aborted\n");
1821
1822 if (active & GERROR_PRIQ_ABT_ERR)
1823 dev_err(smmu->dev, "PRIQ write aborted -- events may have been lost\n");
1824
1825 if (active & GERROR_EVTQ_ABT_ERR)
1826 dev_err(smmu->dev, "EVTQ write aborted -- events may have been lost\n");
1827
1828 if (active & GERROR_CMDQ_ERR)
1829 arm_smmu_cmdq_skip_err(smmu);
1830
1831 writel(gerror, smmu->base + ARM_SMMU_GERRORN);
1832 return IRQ_HANDLED;
1833 }
1834
1835 static irqreturn_t arm_smmu_combined_irq_thread(int irq, void *dev)
1836 {
1837 struct arm_smmu_device *smmu = dev;
1838
1839 arm_smmu_evtq_thread(irq, dev);
1840 if (smmu->features & ARM_SMMU_FEAT_PRI)
1841 arm_smmu_priq_thread(irq, dev);
1842
1843 return IRQ_HANDLED;
1844 }
1845
1846 static irqreturn_t arm_smmu_combined_irq_handler(int irq, void *dev)
1847 {
1848 arm_smmu_gerror_handler(irq, dev);
1849 return IRQ_WAKE_THREAD;
1850 }
1851
1852 static void
1853 arm_smmu_atc_inv_to_cmd(int ssid, unsigned long iova, size_t size,
1854 struct arm_smmu_cmdq_ent *cmd)
1855 {
1856 size_t log2_span;
1857 size_t span_mask;
1858
1859 size_t inval_grain_shift = 12;
1860 unsigned long page_start, page_end;
1861
1862 *cmd = (struct arm_smmu_cmdq_ent) {
1863 .opcode = CMDQ_OP_ATC_INV,
1864 .substream_valid = !!ssid,
1865 .atc.ssid = ssid,
1866 };
1867
1868 if (!size) {
1869 cmd->atc.size = ATC_INV_SIZE_ALL;
1870 return;
1871 }
1872
1873 page_start = iova >> inval_grain_shift;
1874 page_end = (iova + size - 1) >> inval_grain_shift;
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896 log2_span = fls_long(page_start ^ page_end);
1897 span_mask = (1ULL << log2_span) - 1;
1898
1899 page_start &= ~span_mask;
1900
1901 cmd->atc.addr = page_start << inval_grain_shift;
1902 cmd->atc.size = log2_span;
1903 }
1904
1905 static int arm_smmu_atc_inv_master(struct arm_smmu_master *master,
1906 struct arm_smmu_cmdq_ent *cmd)
1907 {
1908 int i;
1909
1910 if (!master->ats_enabled)
1911 return 0;
1912
1913 for (i = 0; i < master->num_sids; i++) {
1914 cmd->atc.sid = master->sids[i];
1915 arm_smmu_cmdq_issue_cmd(master->smmu, cmd);
1916 }
1917
1918 return arm_smmu_cmdq_issue_sync(master->smmu);
1919 }
1920
1921 static int arm_smmu_atc_inv_domain(struct arm_smmu_domain *smmu_domain,
1922 int ssid, unsigned long iova, size_t size)
1923 {
1924 int ret = 0;
1925 unsigned long flags;
1926 struct arm_smmu_cmdq_ent cmd;
1927 struct arm_smmu_master *master;
1928
1929 if (!(smmu_domain->smmu->features & ARM_SMMU_FEAT_ATS))
1930 return 0;
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945 smp_mb();
1946 if (!atomic_read(&smmu_domain->nr_ats_masters))
1947 return 0;
1948
1949 arm_smmu_atc_inv_to_cmd(ssid, iova, size, &cmd);
1950
1951 spin_lock_irqsave(&smmu_domain->devices_lock, flags);
1952 list_for_each_entry(master, &smmu_domain->devices, domain_head)
1953 ret |= arm_smmu_atc_inv_master(master, &cmd);
1954 spin_unlock_irqrestore(&smmu_domain->devices_lock, flags);
1955
1956 return ret ? -ETIMEDOUT : 0;
1957 }
1958
1959
1960 static void arm_smmu_tlb_inv_context(void *cookie)
1961 {
1962 struct arm_smmu_domain *smmu_domain = cookie;
1963 struct arm_smmu_device *smmu = smmu_domain->smmu;
1964 struct arm_smmu_cmdq_ent cmd;
1965
1966 if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1) {
1967 cmd.opcode = CMDQ_OP_TLBI_NH_ASID;
1968 cmd.tlbi.asid = smmu_domain->s1_cfg.cd.asid;
1969 cmd.tlbi.vmid = 0;
1970 } else {
1971 cmd.opcode = CMDQ_OP_TLBI_S12_VMALL;
1972 cmd.tlbi.vmid = smmu_domain->s2_cfg.vmid;
1973 }
1974
1975
1976
1977
1978
1979
1980
1981
1982 arm_smmu_cmdq_issue_cmd(smmu, &cmd);
1983 arm_smmu_cmdq_issue_sync(smmu);
1984 arm_smmu_atc_inv_domain(smmu_domain, 0, 0, 0);
1985 }
1986
1987 static void arm_smmu_tlb_inv_range(unsigned long iova, size_t size,
1988 size_t granule, bool leaf,
1989 struct arm_smmu_domain *smmu_domain)
1990 {
1991 u64 cmds[CMDQ_BATCH_ENTRIES * CMDQ_ENT_DWORDS];
1992 struct arm_smmu_device *smmu = smmu_domain->smmu;
1993 unsigned long start = iova, end = iova + size;
1994 int i = 0;
1995 struct arm_smmu_cmdq_ent cmd = {
1996 .tlbi = {
1997 .leaf = leaf,
1998 },
1999 };
2000
2001 if (!size)
2002 return;
2003
2004 if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1) {
2005 cmd.opcode = CMDQ_OP_TLBI_NH_VA;
2006 cmd.tlbi.asid = smmu_domain->s1_cfg.cd.asid;
2007 } else {
2008 cmd.opcode = CMDQ_OP_TLBI_S2_IPA;
2009 cmd.tlbi.vmid = smmu_domain->s2_cfg.vmid;
2010 }
2011
2012 while (iova < end) {
2013 if (i == CMDQ_BATCH_ENTRIES) {
2014 arm_smmu_cmdq_issue_cmdlist(smmu, cmds, i, false);
2015 i = 0;
2016 }
2017
2018 cmd.tlbi.addr = iova;
2019 arm_smmu_cmdq_build_cmd(&cmds[i * CMDQ_ENT_DWORDS], &cmd);
2020 iova += granule;
2021 i++;
2022 }
2023
2024 arm_smmu_cmdq_issue_cmdlist(smmu, cmds, i, true);
2025
2026
2027
2028
2029
2030 arm_smmu_atc_inv_domain(smmu_domain, 0, start, size);
2031 }
2032
2033 static void arm_smmu_tlb_inv_page_nosync(struct iommu_iotlb_gather *gather,
2034 unsigned long iova, size_t granule,
2035 void *cookie)
2036 {
2037 struct arm_smmu_domain *smmu_domain = cookie;
2038 struct iommu_domain *domain = &smmu_domain->domain;
2039
2040 iommu_iotlb_gather_add_page(domain, gather, iova, granule);
2041 }
2042
2043 static void arm_smmu_tlb_inv_walk(unsigned long iova, size_t size,
2044 size_t granule, void *cookie)
2045 {
2046 arm_smmu_tlb_inv_range(iova, size, granule, false, cookie);
2047 }
2048
2049 static void arm_smmu_tlb_inv_leaf(unsigned long iova, size_t size,
2050 size_t granule, void *cookie)
2051 {
2052 arm_smmu_tlb_inv_range(iova, size, granule, true, cookie);
2053 }
2054
2055 static const struct iommu_flush_ops arm_smmu_flush_ops = {
2056 .tlb_flush_all = arm_smmu_tlb_inv_context,
2057 .tlb_flush_walk = arm_smmu_tlb_inv_walk,
2058 .tlb_flush_leaf = arm_smmu_tlb_inv_leaf,
2059 .tlb_add_page = arm_smmu_tlb_inv_page_nosync,
2060 };
2061
2062
2063 static bool arm_smmu_capable(enum iommu_cap cap)
2064 {
2065 switch (cap) {
2066 case IOMMU_CAP_CACHE_COHERENCY:
2067 return true;
2068 case IOMMU_CAP_NOEXEC:
2069 return true;
2070 default:
2071 return false;
2072 }
2073 }
2074
2075 static struct iommu_domain *arm_smmu_domain_alloc(unsigned type)
2076 {
2077 struct arm_smmu_domain *smmu_domain;
2078
2079 if (type != IOMMU_DOMAIN_UNMANAGED &&
2080 type != IOMMU_DOMAIN_DMA &&
2081 type != IOMMU_DOMAIN_IDENTITY)
2082 return NULL;
2083
2084
2085
2086
2087
2088
2089 smmu_domain = kzalloc(sizeof(*smmu_domain), GFP_KERNEL);
2090 if (!smmu_domain)
2091 return NULL;
2092
2093 if (type == IOMMU_DOMAIN_DMA &&
2094 iommu_get_dma_cookie(&smmu_domain->domain)) {
2095 kfree(smmu_domain);
2096 return NULL;
2097 }
2098
2099 mutex_init(&smmu_domain->init_mutex);
2100 INIT_LIST_HEAD(&smmu_domain->devices);
2101 spin_lock_init(&smmu_domain->devices_lock);
2102
2103 return &smmu_domain->domain;
2104 }
2105
2106 static int arm_smmu_bitmap_alloc(unsigned long *map, int span)
2107 {
2108 int idx, size = 1 << span;
2109
2110 do {
2111 idx = find_first_zero_bit(map, size);
2112 if (idx == size)
2113 return -ENOSPC;
2114 } while (test_and_set_bit(idx, map));
2115
2116 return idx;
2117 }
2118
2119 static void arm_smmu_bitmap_free(unsigned long *map, int idx)
2120 {
2121 clear_bit(idx, map);
2122 }
2123
2124 static void arm_smmu_domain_free(struct iommu_domain *domain)
2125 {
2126 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
2127 struct arm_smmu_device *smmu = smmu_domain->smmu;
2128
2129 iommu_put_dma_cookie(domain);
2130 free_io_pgtable_ops(smmu_domain->pgtbl_ops);
2131
2132
2133 if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1) {
2134 struct arm_smmu_s1_cfg *cfg = &smmu_domain->s1_cfg;
2135
2136 if (cfg->cdptr) {
2137 dmam_free_coherent(smmu_domain->smmu->dev,
2138 CTXDESC_CD_DWORDS << 3,
2139 cfg->cdptr,
2140 cfg->cdptr_dma);
2141
2142 arm_smmu_bitmap_free(smmu->asid_map, cfg->cd.asid);
2143 }
2144 } else {
2145 struct arm_smmu_s2_cfg *cfg = &smmu_domain->s2_cfg;
2146 if (cfg->vmid)
2147 arm_smmu_bitmap_free(smmu->vmid_map, cfg->vmid);
2148 }
2149
2150 kfree(smmu_domain);
2151 }
2152
2153 static int arm_smmu_domain_finalise_s1(struct arm_smmu_domain *smmu_domain,
2154 struct io_pgtable_cfg *pgtbl_cfg)
2155 {
2156 int ret;
2157 int asid;
2158 struct arm_smmu_device *smmu = smmu_domain->smmu;
2159 struct arm_smmu_s1_cfg *cfg = &smmu_domain->s1_cfg;
2160
2161 asid = arm_smmu_bitmap_alloc(smmu->asid_map, smmu->asid_bits);
2162 if (asid < 0)
2163 return asid;
2164
2165 cfg->cdptr = dmam_alloc_coherent(smmu->dev, CTXDESC_CD_DWORDS << 3,
2166 &cfg->cdptr_dma,
2167 GFP_KERNEL | __GFP_ZERO);
2168 if (!cfg->cdptr) {
2169 dev_warn(smmu->dev, "failed to allocate context descriptor\n");
2170 ret = -ENOMEM;
2171 goto out_free_asid;
2172 }
2173
2174 cfg->cd.asid = (u16)asid;
2175 cfg->cd.ttbr = pgtbl_cfg->arm_lpae_s1_cfg.ttbr[0];
2176 cfg->cd.tcr = pgtbl_cfg->arm_lpae_s1_cfg.tcr;
2177 cfg->cd.mair = pgtbl_cfg->arm_lpae_s1_cfg.mair[0];
2178 return 0;
2179
2180 out_free_asid:
2181 arm_smmu_bitmap_free(smmu->asid_map, asid);
2182 return ret;
2183 }
2184
2185 static int arm_smmu_domain_finalise_s2(struct arm_smmu_domain *smmu_domain,
2186 struct io_pgtable_cfg *pgtbl_cfg)
2187 {
2188 int vmid;
2189 struct arm_smmu_device *smmu = smmu_domain->smmu;
2190 struct arm_smmu_s2_cfg *cfg = &smmu_domain->s2_cfg;
2191
2192 vmid = arm_smmu_bitmap_alloc(smmu->vmid_map, smmu->vmid_bits);
2193 if (vmid < 0)
2194 return vmid;
2195
2196 cfg->vmid = (u16)vmid;
2197 cfg->vttbr = pgtbl_cfg->arm_lpae_s2_cfg.vttbr;
2198 cfg->vtcr = pgtbl_cfg->arm_lpae_s2_cfg.vtcr;
2199 return 0;
2200 }
2201
2202 static int arm_smmu_domain_finalise(struct iommu_domain *domain)
2203 {
2204 int ret;
2205 unsigned long ias, oas;
2206 enum io_pgtable_fmt fmt;
2207 struct io_pgtable_cfg pgtbl_cfg;
2208 struct io_pgtable_ops *pgtbl_ops;
2209 int (*finalise_stage_fn)(struct arm_smmu_domain *,
2210 struct io_pgtable_cfg *);
2211 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
2212 struct arm_smmu_device *smmu = smmu_domain->smmu;
2213
2214 if (domain->type == IOMMU_DOMAIN_IDENTITY) {
2215 smmu_domain->stage = ARM_SMMU_DOMAIN_BYPASS;
2216 return 0;
2217 }
2218
2219
2220 if (!(smmu->features & ARM_SMMU_FEAT_TRANS_S1))
2221 smmu_domain->stage = ARM_SMMU_DOMAIN_S2;
2222 if (!(smmu->features & ARM_SMMU_FEAT_TRANS_S2))
2223 smmu_domain->stage = ARM_SMMU_DOMAIN_S1;
2224
2225 switch (smmu_domain->stage) {
2226 case ARM_SMMU_DOMAIN_S1:
2227 ias = (smmu->features & ARM_SMMU_FEAT_VAX) ? 52 : 48;
2228 ias = min_t(unsigned long, ias, VA_BITS);
2229 oas = smmu->ias;
2230 fmt = ARM_64_LPAE_S1;
2231 finalise_stage_fn = arm_smmu_domain_finalise_s1;
2232 break;
2233 case ARM_SMMU_DOMAIN_NESTED:
2234 case ARM_SMMU_DOMAIN_S2:
2235 ias = smmu->ias;
2236 oas = smmu->oas;
2237 fmt = ARM_64_LPAE_S2;
2238 finalise_stage_fn = arm_smmu_domain_finalise_s2;
2239 break;
2240 default:
2241 return -EINVAL;
2242 }
2243
2244 pgtbl_cfg = (struct io_pgtable_cfg) {
2245 .pgsize_bitmap = smmu->pgsize_bitmap,
2246 .ias = ias,
2247 .oas = oas,
2248 .coherent_walk = smmu->features & ARM_SMMU_FEAT_COHERENCY,
2249 .tlb = &arm_smmu_flush_ops,
2250 .iommu_dev = smmu->dev,
2251 };
2252
2253 if (smmu_domain->non_strict)
2254 pgtbl_cfg.quirks |= IO_PGTABLE_QUIRK_NON_STRICT;
2255
2256 pgtbl_ops = alloc_io_pgtable_ops(fmt, &pgtbl_cfg, smmu_domain);
2257 if (!pgtbl_ops)
2258 return -ENOMEM;
2259
2260 domain->pgsize_bitmap = pgtbl_cfg.pgsize_bitmap;
2261 domain->geometry.aperture_end = (1UL << pgtbl_cfg.ias) - 1;
2262 domain->geometry.force_aperture = true;
2263
2264 ret = finalise_stage_fn(smmu_domain, &pgtbl_cfg);
2265 if (ret < 0) {
2266 free_io_pgtable_ops(pgtbl_ops);
2267 return ret;
2268 }
2269
2270 smmu_domain->pgtbl_ops = pgtbl_ops;
2271 return 0;
2272 }
2273
2274 static __le64 *arm_smmu_get_step_for_sid(struct arm_smmu_device *smmu, u32 sid)
2275 {
2276 __le64 *step;
2277 struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg;
2278
2279 if (smmu->features & ARM_SMMU_FEAT_2_LVL_STRTAB) {
2280 struct arm_smmu_strtab_l1_desc *l1_desc;
2281 int idx;
2282
2283
2284 idx = (sid >> STRTAB_SPLIT) * STRTAB_L1_DESC_DWORDS;
2285 l1_desc = &cfg->l1_desc[idx];
2286 idx = (sid & ((1 << STRTAB_SPLIT) - 1)) * STRTAB_STE_DWORDS;
2287 step = &l1_desc->l2ptr[idx];
2288 } else {
2289
2290 step = &cfg->strtab[sid * STRTAB_STE_DWORDS];
2291 }
2292
2293 return step;
2294 }
2295
2296 static void arm_smmu_install_ste_for_dev(struct arm_smmu_master *master)
2297 {
2298 int i, j;
2299 struct arm_smmu_device *smmu = master->smmu;
2300
2301 for (i = 0; i < master->num_sids; ++i) {
2302 u32 sid = master->sids[i];
2303 __le64 *step = arm_smmu_get_step_for_sid(smmu, sid);
2304
2305
2306 for (j = 0; j < i; j++)
2307 if (master->sids[j] == sid)
2308 break;
2309 if (j < i)
2310 continue;
2311
2312 arm_smmu_write_strtab_ent(master, sid, step);
2313 }
2314 }
2315
2316 #ifdef CONFIG_PCI_ATS
2317 static bool arm_smmu_ats_supported(struct arm_smmu_master *master)
2318 {
2319 struct pci_dev *pdev;
2320 struct arm_smmu_device *smmu = master->smmu;
2321 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(master->dev);
2322
2323 if (!(smmu->features & ARM_SMMU_FEAT_ATS) || !dev_is_pci(master->dev) ||
2324 !(fwspec->flags & IOMMU_FWSPEC_PCI_RC_ATS) || pci_ats_disabled())
2325 return false;
2326
2327 pdev = to_pci_dev(master->dev);
2328 return !pdev->untrusted && pdev->ats_cap;
2329 }
2330 #else
2331 static bool arm_smmu_ats_supported(struct arm_smmu_master *master)
2332 {
2333 return false;
2334 }
2335 #endif
2336
2337 static void arm_smmu_enable_ats(struct arm_smmu_master *master)
2338 {
2339 size_t stu;
2340 struct pci_dev *pdev;
2341 struct arm_smmu_device *smmu = master->smmu;
2342 struct arm_smmu_domain *smmu_domain = master->domain;
2343
2344
2345 if (!master->ats_enabled)
2346 return;
2347
2348
2349 stu = __ffs(smmu->pgsize_bitmap);
2350 pdev = to_pci_dev(master->dev);
2351
2352 atomic_inc(&smmu_domain->nr_ats_masters);
2353 arm_smmu_atc_inv_domain(smmu_domain, 0, 0, 0);
2354 if (pci_enable_ats(pdev, stu))
2355 dev_err(master->dev, "Failed to enable ATS (STU %zu)\n", stu);
2356 }
2357
2358 static void arm_smmu_disable_ats(struct arm_smmu_master *master)
2359 {
2360 struct arm_smmu_cmdq_ent cmd;
2361 struct arm_smmu_domain *smmu_domain = master->domain;
2362
2363 if (!master->ats_enabled)
2364 return;
2365
2366 pci_disable_ats(to_pci_dev(master->dev));
2367
2368
2369
2370
2371 wmb();
2372 arm_smmu_atc_inv_to_cmd(0, 0, 0, &cmd);
2373 arm_smmu_atc_inv_master(master, &cmd);
2374 atomic_dec(&smmu_domain->nr_ats_masters);
2375 }
2376
2377 static void arm_smmu_detach_dev(struct arm_smmu_master *master)
2378 {
2379 unsigned long flags;
2380 struct arm_smmu_domain *smmu_domain = master->domain;
2381
2382 if (!smmu_domain)
2383 return;
2384
2385 arm_smmu_disable_ats(master);
2386
2387 spin_lock_irqsave(&smmu_domain->devices_lock, flags);
2388 list_del(&master->domain_head);
2389 spin_unlock_irqrestore(&smmu_domain->devices_lock, flags);
2390
2391 master->domain = NULL;
2392 master->ats_enabled = false;
2393 arm_smmu_install_ste_for_dev(master);
2394 }
2395
2396 static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev)
2397 {
2398 int ret = 0;
2399 unsigned long flags;
2400 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
2401 struct arm_smmu_device *smmu;
2402 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
2403 struct arm_smmu_master *master;
2404
2405 if (!fwspec)
2406 return -ENOENT;
2407
2408 master = fwspec->iommu_priv;
2409 smmu = master->smmu;
2410
2411 arm_smmu_detach_dev(master);
2412
2413 mutex_lock(&smmu_domain->init_mutex);
2414
2415 if (!smmu_domain->smmu) {
2416 smmu_domain->smmu = smmu;
2417 ret = arm_smmu_domain_finalise(domain);
2418 if (ret) {
2419 smmu_domain->smmu = NULL;
2420 goto out_unlock;
2421 }
2422 } else if (smmu_domain->smmu != smmu) {
2423 dev_err(dev,
2424 "cannot attach to SMMU %s (upstream of %s)\n",
2425 dev_name(smmu_domain->smmu->dev),
2426 dev_name(smmu->dev));
2427 ret = -ENXIO;
2428 goto out_unlock;
2429 }
2430
2431 master->domain = smmu_domain;
2432
2433 if (smmu_domain->stage != ARM_SMMU_DOMAIN_BYPASS)
2434 master->ats_enabled = arm_smmu_ats_supported(master);
2435
2436 if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1)
2437 arm_smmu_write_ctx_desc(smmu, &smmu_domain->s1_cfg);
2438
2439 arm_smmu_install_ste_for_dev(master);
2440
2441 spin_lock_irqsave(&smmu_domain->devices_lock, flags);
2442 list_add(&master->domain_head, &smmu_domain->devices);
2443 spin_unlock_irqrestore(&smmu_domain->devices_lock, flags);
2444
2445 arm_smmu_enable_ats(master);
2446
2447 out_unlock:
2448 mutex_unlock(&smmu_domain->init_mutex);
2449 return ret;
2450 }
2451
2452 static int arm_smmu_map(struct iommu_domain *domain, unsigned long iova,
2453 phys_addr_t paddr, size_t size, int prot)
2454 {
2455 struct io_pgtable_ops *ops = to_smmu_domain(domain)->pgtbl_ops;
2456
2457 if (!ops)
2458 return -ENODEV;
2459
2460 return ops->map(ops, iova, paddr, size, prot);
2461 }
2462
2463 static size_t arm_smmu_unmap(struct iommu_domain *domain, unsigned long iova,
2464 size_t size, struct iommu_iotlb_gather *gather)
2465 {
2466 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
2467 struct io_pgtable_ops *ops = smmu_domain->pgtbl_ops;
2468
2469 if (!ops)
2470 return 0;
2471
2472 return ops->unmap(ops, iova, size, gather);
2473 }
2474
2475 static void arm_smmu_flush_iotlb_all(struct iommu_domain *domain)
2476 {
2477 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
2478
2479 if (smmu_domain->smmu)
2480 arm_smmu_tlb_inv_context(smmu_domain);
2481 }
2482
2483 static void arm_smmu_iotlb_sync(struct iommu_domain *domain,
2484 struct iommu_iotlb_gather *gather)
2485 {
2486 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
2487
2488 arm_smmu_tlb_inv_range(gather->start, gather->end - gather->start,
2489 gather->pgsize, true, smmu_domain);
2490 }
2491
2492 static phys_addr_t
2493 arm_smmu_iova_to_phys(struct iommu_domain *domain, dma_addr_t iova)
2494 {
2495 struct io_pgtable_ops *ops = to_smmu_domain(domain)->pgtbl_ops;
2496
2497 if (domain->type == IOMMU_DOMAIN_IDENTITY)
2498 return iova;
2499
2500 if (!ops)
2501 return 0;
2502
2503 return ops->iova_to_phys(ops, iova);
2504 }
2505
2506 static struct platform_driver arm_smmu_driver;
2507
2508 static
2509 struct arm_smmu_device *arm_smmu_get_by_fwnode(struct fwnode_handle *fwnode)
2510 {
2511 struct device *dev = driver_find_device_by_fwnode(&arm_smmu_driver.driver,
2512 fwnode);
2513 put_device(dev);
2514 return dev ? dev_get_drvdata(dev) : NULL;
2515 }
2516
2517 static bool arm_smmu_sid_in_range(struct arm_smmu_device *smmu, u32 sid)
2518 {
2519 unsigned long limit = smmu->strtab_cfg.num_l1_ents;
2520
2521 if (smmu->features & ARM_SMMU_FEAT_2_LVL_STRTAB)
2522 limit *= 1UL << STRTAB_SPLIT;
2523
2524 return sid < limit;
2525 }
2526
2527 static struct iommu_ops arm_smmu_ops;
2528
2529 static int arm_smmu_add_device(struct device *dev)
2530 {
2531 int i, ret;
2532 struct arm_smmu_device *smmu;
2533 struct arm_smmu_master *master;
2534 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
2535 struct iommu_group *group;
2536
2537 if (!fwspec || fwspec->ops != &arm_smmu_ops)
2538 return -ENODEV;
2539
2540
2541
2542
2543
2544 if (WARN_ON_ONCE(fwspec->iommu_priv)) {
2545 master = fwspec->iommu_priv;
2546 smmu = master->smmu;
2547 } else {
2548 smmu = arm_smmu_get_by_fwnode(fwspec->iommu_fwnode);
2549 if (!smmu)
2550 return -ENODEV;
2551 master = kzalloc(sizeof(*master), GFP_KERNEL);
2552 if (!master)
2553 return -ENOMEM;
2554
2555 master->dev = dev;
2556 master->smmu = smmu;
2557 master->sids = fwspec->ids;
2558 master->num_sids = fwspec->num_ids;
2559 fwspec->iommu_priv = master;
2560 }
2561
2562
2563 for (i = 0; i < master->num_sids; i++) {
2564 u32 sid = master->sids[i];
2565
2566 if (!arm_smmu_sid_in_range(smmu, sid))
2567 return -ERANGE;
2568
2569
2570 if (smmu->features & ARM_SMMU_FEAT_2_LVL_STRTAB) {
2571 ret = arm_smmu_init_l2_strtab(smmu, sid);
2572 if (ret)
2573 return ret;
2574 }
2575 }
2576
2577 group = iommu_group_get_for_dev(dev);
2578 if (!IS_ERR(group)) {
2579 iommu_group_put(group);
2580 iommu_device_link(&smmu->iommu, dev);
2581 }
2582
2583 return PTR_ERR_OR_ZERO(group);
2584 }
2585
2586 static void arm_smmu_remove_device(struct device *dev)
2587 {
2588 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
2589 struct arm_smmu_master *master;
2590 struct arm_smmu_device *smmu;
2591
2592 if (!fwspec || fwspec->ops != &arm_smmu_ops)
2593 return;
2594
2595 master = fwspec->iommu_priv;
2596 smmu = master->smmu;
2597 arm_smmu_detach_dev(master);
2598 iommu_group_remove_device(dev);
2599 iommu_device_unlink(&smmu->iommu, dev);
2600 kfree(master);
2601 iommu_fwspec_free(dev);
2602 }
2603
2604 static struct iommu_group *arm_smmu_device_group(struct device *dev)
2605 {
2606 struct iommu_group *group;
2607
2608
2609
2610
2611
2612
2613 if (dev_is_pci(dev))
2614 group = pci_device_group(dev);
2615 else
2616 group = generic_device_group(dev);
2617
2618 return group;
2619 }
2620
2621 static int arm_smmu_domain_get_attr(struct iommu_domain *domain,
2622 enum iommu_attr attr, void *data)
2623 {
2624 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
2625
2626 switch (domain->type) {
2627 case IOMMU_DOMAIN_UNMANAGED:
2628 switch (attr) {
2629 case DOMAIN_ATTR_NESTING:
2630 *(int *)data = (smmu_domain->stage == ARM_SMMU_DOMAIN_NESTED);
2631 return 0;
2632 default:
2633 return -ENODEV;
2634 }
2635 break;
2636 case IOMMU_DOMAIN_DMA:
2637 switch (attr) {
2638 case DOMAIN_ATTR_DMA_USE_FLUSH_QUEUE:
2639 *(int *)data = smmu_domain->non_strict;
2640 return 0;
2641 default:
2642 return -ENODEV;
2643 }
2644 break;
2645 default:
2646 return -EINVAL;
2647 }
2648 }
2649
2650 static int arm_smmu_domain_set_attr(struct iommu_domain *domain,
2651 enum iommu_attr attr, void *data)
2652 {
2653 int ret = 0;
2654 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
2655
2656 mutex_lock(&smmu_domain->init_mutex);
2657
2658 switch (domain->type) {
2659 case IOMMU_DOMAIN_UNMANAGED:
2660 switch (attr) {
2661 case DOMAIN_ATTR_NESTING:
2662 if (smmu_domain->smmu) {
2663 ret = -EPERM;
2664 goto out_unlock;
2665 }
2666
2667 if (*(int *)data)
2668 smmu_domain->stage = ARM_SMMU_DOMAIN_NESTED;
2669 else
2670 smmu_domain->stage = ARM_SMMU_DOMAIN_S1;
2671 break;
2672 default:
2673 ret = -ENODEV;
2674 }
2675 break;
2676 case IOMMU_DOMAIN_DMA:
2677 switch(attr) {
2678 case DOMAIN_ATTR_DMA_USE_FLUSH_QUEUE:
2679 smmu_domain->non_strict = *(int *)data;
2680 break;
2681 default:
2682 ret = -ENODEV;
2683 }
2684 break;
2685 default:
2686 ret = -EINVAL;
2687 }
2688
2689 out_unlock:
2690 mutex_unlock(&smmu_domain->init_mutex);
2691 return ret;
2692 }
2693
2694 static int arm_smmu_of_xlate(struct device *dev, struct of_phandle_args *args)
2695 {
2696 return iommu_fwspec_add_ids(dev, args->args, 1);
2697 }
2698
2699 static void arm_smmu_get_resv_regions(struct device *dev,
2700 struct list_head *head)
2701 {
2702 struct iommu_resv_region *region;
2703 int prot = IOMMU_WRITE | IOMMU_NOEXEC | IOMMU_MMIO;
2704
2705 region = iommu_alloc_resv_region(MSI_IOVA_BASE, MSI_IOVA_LENGTH,
2706 prot, IOMMU_RESV_SW_MSI);
2707 if (!region)
2708 return;
2709
2710 list_add_tail(®ion->list, head);
2711
2712 iommu_dma_get_resv_regions(dev, head);
2713 }
2714
2715 static void arm_smmu_put_resv_regions(struct device *dev,
2716 struct list_head *head)
2717 {
2718 struct iommu_resv_region *entry, *next;
2719
2720 list_for_each_entry_safe(entry, next, head, list)
2721 kfree(entry);
2722 }
2723
2724 static struct iommu_ops arm_smmu_ops = {
2725 .capable = arm_smmu_capable,
2726 .domain_alloc = arm_smmu_domain_alloc,
2727 .domain_free = arm_smmu_domain_free,
2728 .attach_dev = arm_smmu_attach_dev,
2729 .map = arm_smmu_map,
2730 .unmap = arm_smmu_unmap,
2731 .flush_iotlb_all = arm_smmu_flush_iotlb_all,
2732 .iotlb_sync = arm_smmu_iotlb_sync,
2733 .iova_to_phys = arm_smmu_iova_to_phys,
2734 .add_device = arm_smmu_add_device,
2735 .remove_device = arm_smmu_remove_device,
2736 .device_group = arm_smmu_device_group,
2737 .domain_get_attr = arm_smmu_domain_get_attr,
2738 .domain_set_attr = arm_smmu_domain_set_attr,
2739 .of_xlate = arm_smmu_of_xlate,
2740 .get_resv_regions = arm_smmu_get_resv_regions,
2741 .put_resv_regions = arm_smmu_put_resv_regions,
2742 .pgsize_bitmap = -1UL,
2743 };
2744
2745
2746 static int arm_smmu_init_one_queue(struct arm_smmu_device *smmu,
2747 struct arm_smmu_queue *q,
2748 unsigned long prod_off,
2749 unsigned long cons_off,
2750 size_t dwords, const char *name)
2751 {
2752 size_t qsz;
2753
2754 do {
2755 qsz = ((1 << q->llq.max_n_shift) * dwords) << 3;
2756 q->base = dmam_alloc_coherent(smmu->dev, qsz, &q->base_dma,
2757 GFP_KERNEL);
2758 if (q->base || qsz < PAGE_SIZE)
2759 break;
2760
2761 q->llq.max_n_shift--;
2762 } while (1);
2763
2764 if (!q->base) {
2765 dev_err(smmu->dev,
2766 "failed to allocate queue (0x%zx bytes) for %s\n",
2767 qsz, name);
2768 return -ENOMEM;
2769 }
2770
2771 if (!WARN_ON(q->base_dma & (qsz - 1))) {
2772 dev_info(smmu->dev, "allocated %u entries for %s\n",
2773 1 << q->llq.max_n_shift, name);
2774 }
2775
2776 q->prod_reg = arm_smmu_page1_fixup(prod_off, smmu);
2777 q->cons_reg = arm_smmu_page1_fixup(cons_off, smmu);
2778 q->ent_dwords = dwords;
2779
2780 q->q_base = Q_BASE_RWA;
2781 q->q_base |= q->base_dma & Q_BASE_ADDR_MASK;
2782 q->q_base |= FIELD_PREP(Q_BASE_LOG2SIZE, q->llq.max_n_shift);
2783
2784 q->llq.prod = q->llq.cons = 0;
2785 return 0;
2786 }
2787
2788 static void arm_smmu_cmdq_free_bitmap(void *data)
2789 {
2790 unsigned long *bitmap = data;
2791 bitmap_free(bitmap);
2792 }
2793
2794 static int arm_smmu_cmdq_init(struct arm_smmu_device *smmu)
2795 {
2796 int ret = 0;
2797 struct arm_smmu_cmdq *cmdq = &smmu->cmdq;
2798 unsigned int nents = 1 << cmdq->q.llq.max_n_shift;
2799 atomic_long_t *bitmap;
2800
2801 atomic_set(&cmdq->owner_prod, 0);
2802 atomic_set(&cmdq->lock, 0);
2803
2804 bitmap = (atomic_long_t *)bitmap_zalloc(nents, GFP_KERNEL);
2805 if (!bitmap) {
2806 dev_err(smmu->dev, "failed to allocate cmdq bitmap\n");
2807 ret = -ENOMEM;
2808 } else {
2809 cmdq->valid_map = bitmap;
2810 devm_add_action(smmu->dev, arm_smmu_cmdq_free_bitmap, bitmap);
2811 }
2812
2813 return ret;
2814 }
2815
2816 static int arm_smmu_init_queues(struct arm_smmu_device *smmu)
2817 {
2818 int ret;
2819
2820
2821 ret = arm_smmu_init_one_queue(smmu, &smmu->cmdq.q, ARM_SMMU_CMDQ_PROD,
2822 ARM_SMMU_CMDQ_CONS, CMDQ_ENT_DWORDS,
2823 "cmdq");
2824 if (ret)
2825 return ret;
2826
2827 ret = arm_smmu_cmdq_init(smmu);
2828 if (ret)
2829 return ret;
2830
2831
2832 ret = arm_smmu_init_one_queue(smmu, &smmu->evtq.q, ARM_SMMU_EVTQ_PROD,
2833 ARM_SMMU_EVTQ_CONS, EVTQ_ENT_DWORDS,
2834 "evtq");
2835 if (ret)
2836 return ret;
2837
2838
2839 if (!(smmu->features & ARM_SMMU_FEAT_PRI))
2840 return 0;
2841
2842 return arm_smmu_init_one_queue(smmu, &smmu->priq.q, ARM_SMMU_PRIQ_PROD,
2843 ARM_SMMU_PRIQ_CONS, PRIQ_ENT_DWORDS,
2844 "priq");
2845 }
2846
2847 static int arm_smmu_init_l1_strtab(struct arm_smmu_device *smmu)
2848 {
2849 unsigned int i;
2850 struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg;
2851 size_t size = sizeof(*cfg->l1_desc) * cfg->num_l1_ents;
2852 void *strtab = smmu->strtab_cfg.strtab;
2853
2854 cfg->l1_desc = devm_kzalloc(smmu->dev, size, GFP_KERNEL);
2855 if (!cfg->l1_desc) {
2856 dev_err(smmu->dev, "failed to allocate l1 stream table desc\n");
2857 return -ENOMEM;
2858 }
2859
2860 for (i = 0; i < cfg->num_l1_ents; ++i) {
2861 arm_smmu_write_strtab_l1_desc(strtab, &cfg->l1_desc[i]);
2862 strtab += STRTAB_L1_DESC_DWORDS << 3;
2863 }
2864
2865 return 0;
2866 }
2867
2868 static int arm_smmu_init_strtab_2lvl(struct arm_smmu_device *smmu)
2869 {
2870 void *strtab;
2871 u64 reg;
2872 u32 size, l1size;
2873 struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg;
2874
2875
2876 size = STRTAB_L1_SZ_SHIFT - (ilog2(STRTAB_L1_DESC_DWORDS) + 3);
2877 size = min(size, smmu->sid_bits - STRTAB_SPLIT);
2878 cfg->num_l1_ents = 1 << size;
2879
2880 size += STRTAB_SPLIT;
2881 if (size < smmu->sid_bits)
2882 dev_warn(smmu->dev,
2883 "2-level strtab only covers %u/%u bits of SID\n",
2884 size, smmu->sid_bits);
2885
2886 l1size = cfg->num_l1_ents * (STRTAB_L1_DESC_DWORDS << 3);
2887 strtab = dmam_alloc_coherent(smmu->dev, l1size, &cfg->strtab_dma,
2888 GFP_KERNEL | __GFP_ZERO);
2889 if (!strtab) {
2890 dev_err(smmu->dev,
2891 "failed to allocate l1 stream table (%u bytes)\n",
2892 size);
2893 return -ENOMEM;
2894 }
2895 cfg->strtab = strtab;
2896
2897
2898 reg = FIELD_PREP(STRTAB_BASE_CFG_FMT, STRTAB_BASE_CFG_FMT_2LVL);
2899 reg |= FIELD_PREP(STRTAB_BASE_CFG_LOG2SIZE, size);
2900 reg |= FIELD_PREP(STRTAB_BASE_CFG_SPLIT, STRTAB_SPLIT);
2901 cfg->strtab_base_cfg = reg;
2902
2903 return arm_smmu_init_l1_strtab(smmu);
2904 }
2905
2906 static int arm_smmu_init_strtab_linear(struct arm_smmu_device *smmu)
2907 {
2908 void *strtab;
2909 u64 reg;
2910 u32 size;
2911 struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg;
2912
2913 size = (1 << smmu->sid_bits) * (STRTAB_STE_DWORDS << 3);
2914 strtab = dmam_alloc_coherent(smmu->dev, size, &cfg->strtab_dma,
2915 GFP_KERNEL | __GFP_ZERO);
2916 if (!strtab) {
2917 dev_err(smmu->dev,
2918 "failed to allocate linear stream table (%u bytes)\n",
2919 size);
2920 return -ENOMEM;
2921 }
2922 cfg->strtab = strtab;
2923 cfg->num_l1_ents = 1 << smmu->sid_bits;
2924
2925
2926 reg = FIELD_PREP(STRTAB_BASE_CFG_FMT, STRTAB_BASE_CFG_FMT_LINEAR);
2927 reg |= FIELD_PREP(STRTAB_BASE_CFG_LOG2SIZE, smmu->sid_bits);
2928 cfg->strtab_base_cfg = reg;
2929
2930 arm_smmu_init_bypass_stes(strtab, cfg->num_l1_ents);
2931 return 0;
2932 }
2933
2934 static int arm_smmu_init_strtab(struct arm_smmu_device *smmu)
2935 {
2936 u64 reg;
2937 int ret;
2938
2939 if (smmu->features & ARM_SMMU_FEAT_2_LVL_STRTAB)
2940 ret = arm_smmu_init_strtab_2lvl(smmu);
2941 else
2942 ret = arm_smmu_init_strtab_linear(smmu);
2943
2944 if (ret)
2945 return ret;
2946
2947
2948 reg = smmu->strtab_cfg.strtab_dma & STRTAB_BASE_ADDR_MASK;
2949 reg |= STRTAB_BASE_RA;
2950 smmu->strtab_cfg.strtab_base = reg;
2951
2952
2953 set_bit(0, smmu->vmid_map);
2954 return 0;
2955 }
2956
2957 static int arm_smmu_init_structures(struct arm_smmu_device *smmu)
2958 {
2959 int ret;
2960
2961 ret = arm_smmu_init_queues(smmu);
2962 if (ret)
2963 return ret;
2964
2965 return arm_smmu_init_strtab(smmu);
2966 }
2967
2968 static int arm_smmu_write_reg_sync(struct arm_smmu_device *smmu, u32 val,
2969 unsigned int reg_off, unsigned int ack_off)
2970 {
2971 u32 reg;
2972
2973 writel_relaxed(val, smmu->base + reg_off);
2974 return readl_relaxed_poll_timeout(smmu->base + ack_off, reg, reg == val,
2975 1, ARM_SMMU_POLL_TIMEOUT_US);
2976 }
2977
2978
2979 static int arm_smmu_update_gbpa(struct arm_smmu_device *smmu, u32 set, u32 clr)
2980 {
2981 int ret;
2982 u32 reg, __iomem *gbpa = smmu->base + ARM_SMMU_GBPA;
2983
2984 ret = readl_relaxed_poll_timeout(gbpa, reg, !(reg & GBPA_UPDATE),
2985 1, ARM_SMMU_POLL_TIMEOUT_US);
2986 if (ret)
2987 return ret;
2988
2989 reg &= ~clr;
2990 reg |= set;
2991 writel_relaxed(reg | GBPA_UPDATE, gbpa);
2992 ret = readl_relaxed_poll_timeout(gbpa, reg, !(reg & GBPA_UPDATE),
2993 1, ARM_SMMU_POLL_TIMEOUT_US);
2994
2995 if (ret)
2996 dev_err(smmu->dev, "GBPA not responding to update\n");
2997 return ret;
2998 }
2999
3000 static void arm_smmu_free_msis(void *data)
3001 {
3002 struct device *dev = data;
3003 platform_msi_domain_free_irqs(dev);
3004 }
3005
3006 static void arm_smmu_write_msi_msg(struct msi_desc *desc, struct msi_msg *msg)
3007 {
3008 phys_addr_t doorbell;
3009 struct device *dev = msi_desc_to_dev(desc);
3010 struct arm_smmu_device *smmu = dev_get_drvdata(dev);
3011 phys_addr_t *cfg = arm_smmu_msi_cfg[desc->platform.msi_index];
3012
3013 doorbell = (((u64)msg->address_hi) << 32) | msg->address_lo;
3014 doorbell &= MSI_CFG0_ADDR_MASK;
3015
3016 writeq_relaxed(doorbell, smmu->base + cfg[0]);
3017 writel_relaxed(msg->data, smmu->base + cfg[1]);
3018 writel_relaxed(ARM_SMMU_MEMATTR_DEVICE_nGnRE, smmu->base + cfg[2]);
3019 }
3020
3021 static void arm_smmu_setup_msis(struct arm_smmu_device *smmu)
3022 {
3023 struct msi_desc *desc;
3024 int ret, nvec = ARM_SMMU_MAX_MSIS;
3025 struct device *dev = smmu->dev;
3026
3027
3028 writeq_relaxed(0, smmu->base + ARM_SMMU_GERROR_IRQ_CFG0);
3029 writeq_relaxed(0, smmu->base + ARM_SMMU_EVTQ_IRQ_CFG0);
3030
3031 if (smmu->features & ARM_SMMU_FEAT_PRI)
3032 writeq_relaxed(0, smmu->base + ARM_SMMU_PRIQ_IRQ_CFG0);
3033 else
3034 nvec--;
3035
3036 if (!(smmu->features & ARM_SMMU_FEAT_MSI))
3037 return;
3038
3039 if (!dev->msi_domain) {
3040 dev_info(smmu->dev, "msi_domain absent - falling back to wired irqs\n");
3041 return;
3042 }
3043
3044
3045 ret = platform_msi_domain_alloc_irqs(dev, nvec, arm_smmu_write_msi_msg);
3046 if (ret) {
3047 dev_warn(dev, "failed to allocate MSIs - falling back to wired irqs\n");
3048 return;
3049 }
3050
3051 for_each_msi_entry(desc, dev) {
3052 switch (desc->platform.msi_index) {
3053 case EVTQ_MSI_INDEX:
3054 smmu->evtq.q.irq = desc->irq;
3055 break;
3056 case GERROR_MSI_INDEX:
3057 smmu->gerr_irq = desc->irq;
3058 break;
3059 case PRIQ_MSI_INDEX:
3060 smmu->priq.q.irq = desc->irq;
3061 break;
3062 default:
3063 continue;
3064 }
3065 }
3066
3067
3068 devm_add_action(dev, arm_smmu_free_msis, dev);
3069 }
3070
3071 static void arm_smmu_setup_unique_irqs(struct arm_smmu_device *smmu)
3072 {
3073 int irq, ret;
3074
3075 arm_smmu_setup_msis(smmu);
3076
3077
3078 irq = smmu->evtq.q.irq;
3079 if (irq) {
3080 ret = devm_request_threaded_irq(smmu->dev, irq, NULL,
3081 arm_smmu_evtq_thread,
3082 IRQF_ONESHOT,
3083 "arm-smmu-v3-evtq", smmu);
3084 if (ret < 0)
3085 dev_warn(smmu->dev, "failed to enable evtq irq\n");
3086 } else {
3087 dev_warn(smmu->dev, "no evtq irq - events will not be reported!\n");
3088 }
3089
3090 irq = smmu->gerr_irq;
3091 if (irq) {
3092 ret = devm_request_irq(smmu->dev, irq, arm_smmu_gerror_handler,
3093 0, "arm-smmu-v3-gerror", smmu);
3094 if (ret < 0)
3095 dev_warn(smmu->dev, "failed to enable gerror irq\n");
3096 } else {
3097 dev_warn(smmu->dev, "no gerr irq - errors will not be reported!\n");
3098 }
3099
3100 if (smmu->features & ARM_SMMU_FEAT_PRI) {
3101 irq = smmu->priq.q.irq;
3102 if (irq) {
3103 ret = devm_request_threaded_irq(smmu->dev, irq, NULL,
3104 arm_smmu_priq_thread,
3105 IRQF_ONESHOT,
3106 "arm-smmu-v3-priq",
3107 smmu);
3108 if (ret < 0)
3109 dev_warn(smmu->dev,
3110 "failed to enable priq irq\n");
3111 } else {
3112 dev_warn(smmu->dev, "no priq irq - PRI will be broken\n");
3113 }
3114 }
3115 }
3116
3117 static int arm_smmu_setup_irqs(struct arm_smmu_device *smmu)
3118 {
3119 int ret, irq;
3120 u32 irqen_flags = IRQ_CTRL_EVTQ_IRQEN | IRQ_CTRL_GERROR_IRQEN;
3121
3122
3123 ret = arm_smmu_write_reg_sync(smmu, 0, ARM_SMMU_IRQ_CTRL,
3124 ARM_SMMU_IRQ_CTRLACK);
3125 if (ret) {
3126 dev_err(smmu->dev, "failed to disable irqs\n");
3127 return ret;
3128 }
3129
3130 irq = smmu->combined_irq;
3131 if (irq) {
3132
3133
3134
3135
3136 ret = devm_request_threaded_irq(smmu->dev, irq,
3137 arm_smmu_combined_irq_handler,
3138 arm_smmu_combined_irq_thread,
3139 IRQF_ONESHOT,
3140 "arm-smmu-v3-combined-irq", smmu);
3141 if (ret < 0)
3142 dev_warn(smmu->dev, "failed to enable combined irq\n");
3143 } else
3144 arm_smmu_setup_unique_irqs(smmu);
3145
3146 if (smmu->features & ARM_SMMU_FEAT_PRI)
3147 irqen_flags |= IRQ_CTRL_PRIQ_IRQEN;
3148
3149
3150 ret = arm_smmu_write_reg_sync(smmu, irqen_flags,
3151 ARM_SMMU_IRQ_CTRL, ARM_SMMU_IRQ_CTRLACK);
3152 if (ret)
3153 dev_warn(smmu->dev, "failed to enable irqs\n");
3154
3155 return 0;
3156 }
3157
3158 static int arm_smmu_device_disable(struct arm_smmu_device *smmu)
3159 {
3160 int ret;
3161
3162 ret = arm_smmu_write_reg_sync(smmu, 0, ARM_SMMU_CR0, ARM_SMMU_CR0ACK);
3163 if (ret)
3164 dev_err(smmu->dev, "failed to clear cr0\n");
3165
3166 return ret;
3167 }
3168
3169 static int arm_smmu_device_reset(struct arm_smmu_device *smmu, bool bypass)
3170 {
3171 int ret;
3172 u32 reg, enables;
3173 struct arm_smmu_cmdq_ent cmd;
3174
3175
3176 reg = readl_relaxed(smmu->base + ARM_SMMU_CR0);
3177 if (reg & CR0_SMMUEN) {
3178 dev_warn(smmu->dev, "SMMU currently enabled! Resetting...\n");
3179 WARN_ON(is_kdump_kernel() && !disable_bypass);
3180 arm_smmu_update_gbpa(smmu, GBPA_ABORT, 0);
3181 }
3182
3183 ret = arm_smmu_device_disable(smmu);
3184 if (ret)
3185 return ret;
3186
3187
3188 reg = FIELD_PREP(CR1_TABLE_SH, ARM_SMMU_SH_ISH) |
3189 FIELD_PREP(CR1_TABLE_OC, CR1_CACHE_WB) |
3190 FIELD_PREP(CR1_TABLE_IC, CR1_CACHE_WB) |
3191 FIELD_PREP(CR1_QUEUE_SH, ARM_SMMU_SH_ISH) |
3192 FIELD_PREP(CR1_QUEUE_OC, CR1_CACHE_WB) |
3193 FIELD_PREP(CR1_QUEUE_IC, CR1_CACHE_WB);
3194 writel_relaxed(reg, smmu->base + ARM_SMMU_CR1);
3195
3196
3197 reg = CR2_PTM | CR2_RECINVSID | CR2_E2H;
3198 writel_relaxed(reg, smmu->base + ARM_SMMU_CR2);
3199
3200
3201 writeq_relaxed(smmu->strtab_cfg.strtab_base,
3202 smmu->base + ARM_SMMU_STRTAB_BASE);
3203 writel_relaxed(smmu->strtab_cfg.strtab_base_cfg,
3204 smmu->base + ARM_SMMU_STRTAB_BASE_CFG);
3205
3206
3207 writeq_relaxed(smmu->cmdq.q.q_base, smmu->base + ARM_SMMU_CMDQ_BASE);
3208 writel_relaxed(smmu->cmdq.q.llq.prod, smmu->base + ARM_SMMU_CMDQ_PROD);
3209 writel_relaxed(smmu->cmdq.q.llq.cons, smmu->base + ARM_SMMU_CMDQ_CONS);
3210
3211 enables = CR0_CMDQEN;
3212 ret = arm_smmu_write_reg_sync(smmu, enables, ARM_SMMU_CR0,
3213 ARM_SMMU_CR0ACK);
3214 if (ret) {
3215 dev_err(smmu->dev, "failed to enable command queue\n");
3216 return ret;
3217 }
3218
3219
3220 cmd.opcode = CMDQ_OP_CFGI_ALL;
3221 arm_smmu_cmdq_issue_cmd(smmu, &cmd);
3222 arm_smmu_cmdq_issue_sync(smmu);
3223
3224
3225 if (smmu->features & ARM_SMMU_FEAT_HYP) {
3226 cmd.opcode = CMDQ_OP_TLBI_EL2_ALL;
3227 arm_smmu_cmdq_issue_cmd(smmu, &cmd);
3228 }
3229
3230 cmd.opcode = CMDQ_OP_TLBI_NSNH_ALL;
3231 arm_smmu_cmdq_issue_cmd(smmu, &cmd);
3232 arm_smmu_cmdq_issue_sync(smmu);
3233
3234
3235 writeq_relaxed(smmu->evtq.q.q_base, smmu->base + ARM_SMMU_EVTQ_BASE);
3236 writel_relaxed(smmu->evtq.q.llq.prod,
3237 arm_smmu_page1_fixup(ARM_SMMU_EVTQ_PROD, smmu));
3238 writel_relaxed(smmu->evtq.q.llq.cons,
3239 arm_smmu_page1_fixup(ARM_SMMU_EVTQ_CONS, smmu));
3240
3241 enables |= CR0_EVTQEN;
3242 ret = arm_smmu_write_reg_sync(smmu, enables, ARM_SMMU_CR0,
3243 ARM_SMMU_CR0ACK);
3244 if (ret) {
3245 dev_err(smmu->dev, "failed to enable event queue\n");
3246 return ret;
3247 }
3248
3249
3250 if (smmu->features & ARM_SMMU_FEAT_PRI) {
3251 writeq_relaxed(smmu->priq.q.q_base,
3252 smmu->base + ARM_SMMU_PRIQ_BASE);
3253 writel_relaxed(smmu->priq.q.llq.prod,
3254 arm_smmu_page1_fixup(ARM_SMMU_PRIQ_PROD, smmu));
3255 writel_relaxed(smmu->priq.q.llq.cons,
3256 arm_smmu_page1_fixup(ARM_SMMU_PRIQ_CONS, smmu));
3257
3258 enables |= CR0_PRIQEN;
3259 ret = arm_smmu_write_reg_sync(smmu, enables, ARM_SMMU_CR0,
3260 ARM_SMMU_CR0ACK);
3261 if (ret) {
3262 dev_err(smmu->dev, "failed to enable PRI queue\n");
3263 return ret;
3264 }
3265 }
3266
3267 if (smmu->features & ARM_SMMU_FEAT_ATS) {
3268 enables |= CR0_ATSCHK;
3269 ret = arm_smmu_write_reg_sync(smmu, enables, ARM_SMMU_CR0,
3270 ARM_SMMU_CR0ACK);
3271 if (ret) {
3272 dev_err(smmu->dev, "failed to enable ATS check\n");
3273 return ret;
3274 }
3275 }
3276
3277 ret = arm_smmu_setup_irqs(smmu);
3278 if (ret) {
3279 dev_err(smmu->dev, "failed to setup irqs\n");
3280 return ret;
3281 }
3282
3283 if (is_kdump_kernel())
3284 enables &= ~(CR0_EVTQEN | CR0_PRIQEN);
3285
3286
3287 if (!bypass || disable_bypass) {
3288 enables |= CR0_SMMUEN;
3289 } else {
3290 ret = arm_smmu_update_gbpa(smmu, 0, GBPA_ABORT);
3291 if (ret)
3292 return ret;
3293 }
3294 ret = arm_smmu_write_reg_sync(smmu, enables, ARM_SMMU_CR0,
3295 ARM_SMMU_CR0ACK);
3296 if (ret) {
3297 dev_err(smmu->dev, "failed to enable SMMU interface\n");
3298 return ret;
3299 }
3300
3301 return 0;
3302 }
3303
3304 static int arm_smmu_device_hw_probe(struct arm_smmu_device *smmu)
3305 {
3306 u32 reg;
3307 bool coherent = smmu->features & ARM_SMMU_FEAT_COHERENCY;
3308
3309
3310 reg = readl_relaxed(smmu->base + ARM_SMMU_IDR0);
3311
3312
3313 if (FIELD_GET(IDR0_ST_LVL, reg) == IDR0_ST_LVL_2LVL)
3314 smmu->features |= ARM_SMMU_FEAT_2_LVL_STRTAB;
3315
3316 if (reg & IDR0_CD2L)
3317 smmu->features |= ARM_SMMU_FEAT_2_LVL_CDTAB;
3318
3319
3320
3321
3322
3323
3324 switch (FIELD_GET(IDR0_TTENDIAN, reg)) {
3325 case IDR0_TTENDIAN_MIXED:
3326 smmu->features |= ARM_SMMU_FEAT_TT_LE | ARM_SMMU_FEAT_TT_BE;
3327 break;
3328 #ifdef __BIG_ENDIAN
3329 case IDR0_TTENDIAN_BE:
3330 smmu->features |= ARM_SMMU_FEAT_TT_BE;
3331 break;
3332 #else
3333 case IDR0_TTENDIAN_LE:
3334 smmu->features |= ARM_SMMU_FEAT_TT_LE;
3335 break;
3336 #endif
3337 default:
3338 dev_err(smmu->dev, "unknown/unsupported TT endianness!\n");
3339 return -ENXIO;
3340 }
3341
3342
3343 if (IS_ENABLED(CONFIG_PCI_PRI) && reg & IDR0_PRI)
3344 smmu->features |= ARM_SMMU_FEAT_PRI;
3345
3346 if (IS_ENABLED(CONFIG_PCI_ATS) && reg & IDR0_ATS)
3347 smmu->features |= ARM_SMMU_FEAT_ATS;
3348
3349 if (reg & IDR0_SEV)
3350 smmu->features |= ARM_SMMU_FEAT_SEV;
3351
3352 if (reg & IDR0_MSI)
3353 smmu->features |= ARM_SMMU_FEAT_MSI;
3354
3355 if (reg & IDR0_HYP)
3356 smmu->features |= ARM_SMMU_FEAT_HYP;
3357
3358
3359
3360
3361
3362 if (!!(reg & IDR0_COHACC) != coherent)
3363 dev_warn(smmu->dev, "IDR0.COHACC overridden by FW configuration (%s)\n",
3364 coherent ? "true" : "false");
3365
3366 switch (FIELD_GET(IDR0_STALL_MODEL, reg)) {
3367 case IDR0_STALL_MODEL_FORCE:
3368 smmu->features |= ARM_SMMU_FEAT_STALL_FORCE;
3369
3370 case IDR0_STALL_MODEL_STALL:
3371 smmu->features |= ARM_SMMU_FEAT_STALLS;
3372 }
3373
3374 if (reg & IDR0_S1P)
3375 smmu->features |= ARM_SMMU_FEAT_TRANS_S1;
3376
3377 if (reg & IDR0_S2P)
3378 smmu->features |= ARM_SMMU_FEAT_TRANS_S2;
3379
3380 if (!(reg & (IDR0_S1P | IDR0_S2P))) {
3381 dev_err(smmu->dev, "no translation support!\n");
3382 return -ENXIO;
3383 }
3384
3385
3386 switch (FIELD_GET(IDR0_TTF, reg)) {
3387 case IDR0_TTF_AARCH32_64:
3388 smmu->ias = 40;
3389
3390 case IDR0_TTF_AARCH64:
3391 break;
3392 default:
3393 dev_err(smmu->dev, "AArch64 table format not supported!\n");
3394 return -ENXIO;
3395 }
3396
3397
3398 smmu->asid_bits = reg & IDR0_ASID16 ? 16 : 8;
3399 smmu->vmid_bits = reg & IDR0_VMID16 ? 16 : 8;
3400
3401
3402 reg = readl_relaxed(smmu->base + ARM_SMMU_IDR1);
3403 if (reg & (IDR1_TABLES_PRESET | IDR1_QUEUES_PRESET | IDR1_REL)) {
3404 dev_err(smmu->dev, "embedded implementation not supported\n");
3405 return -ENXIO;
3406 }
3407
3408
3409 smmu->cmdq.q.llq.max_n_shift = min_t(u32, CMDQ_MAX_SZ_SHIFT,
3410 FIELD_GET(IDR1_CMDQS, reg));
3411 if (smmu->cmdq.q.llq.max_n_shift <= ilog2(CMDQ_BATCH_ENTRIES)) {
3412
3413
3414
3415
3416
3417
3418 dev_err(smmu->dev, "command queue size <= %d entries not supported\n",
3419 CMDQ_BATCH_ENTRIES);
3420 return -ENXIO;
3421 }
3422
3423 smmu->evtq.q.llq.max_n_shift = min_t(u32, EVTQ_MAX_SZ_SHIFT,
3424 FIELD_GET(IDR1_EVTQS, reg));
3425 smmu->priq.q.llq.max_n_shift = min_t(u32, PRIQ_MAX_SZ_SHIFT,
3426 FIELD_GET(IDR1_PRIQS, reg));
3427
3428
3429 smmu->ssid_bits = FIELD_GET(IDR1_SSIDSIZE, reg);
3430 smmu->sid_bits = FIELD_GET(IDR1_SIDSIZE, reg);
3431
3432
3433
3434
3435
3436 if (smmu->sid_bits <= STRTAB_SPLIT)
3437 smmu->features &= ~ARM_SMMU_FEAT_2_LVL_STRTAB;
3438
3439
3440 reg = readl_relaxed(smmu->base + ARM_SMMU_IDR5);
3441
3442
3443 smmu->evtq.max_stalls = FIELD_GET(IDR5_STALL_MAX, reg);
3444
3445
3446 if (reg & IDR5_GRAN64K)
3447 smmu->pgsize_bitmap |= SZ_64K | SZ_512M;
3448 if (reg & IDR5_GRAN16K)
3449 smmu->pgsize_bitmap |= SZ_16K | SZ_32M;
3450 if (reg & IDR5_GRAN4K)
3451 smmu->pgsize_bitmap |= SZ_4K | SZ_2M | SZ_1G;
3452
3453
3454 if (FIELD_GET(IDR5_VAX, reg) == IDR5_VAX_52_BIT)
3455 smmu->features |= ARM_SMMU_FEAT_VAX;
3456
3457
3458 switch (FIELD_GET(IDR5_OAS, reg)) {
3459 case IDR5_OAS_32_BIT:
3460 smmu->oas = 32;
3461 break;
3462 case IDR5_OAS_36_BIT:
3463 smmu->oas = 36;
3464 break;
3465 case IDR5_OAS_40_BIT:
3466 smmu->oas = 40;
3467 break;
3468 case IDR5_OAS_42_BIT:
3469 smmu->oas = 42;
3470 break;
3471 case IDR5_OAS_44_BIT:
3472 smmu->oas = 44;
3473 break;
3474 case IDR5_OAS_52_BIT:
3475 smmu->oas = 52;
3476 smmu->pgsize_bitmap |= 1ULL << 42;
3477 break;
3478 default:
3479 dev_info(smmu->dev,
3480 "unknown output address size. Truncating to 48-bit\n");
3481
3482 case IDR5_OAS_48_BIT:
3483 smmu->oas = 48;
3484 }
3485
3486 if (arm_smmu_ops.pgsize_bitmap == -1UL)
3487 arm_smmu_ops.pgsize_bitmap = smmu->pgsize_bitmap;
3488 else
3489 arm_smmu_ops.pgsize_bitmap |= smmu->pgsize_bitmap;
3490
3491
3492 if (dma_set_mask_and_coherent(smmu->dev, DMA_BIT_MASK(smmu->oas)))
3493 dev_warn(smmu->dev,
3494 "failed to set DMA mask for table walker\n");
3495
3496 smmu->ias = max(smmu->ias, smmu->oas);
3497
3498 dev_info(smmu->dev, "ias %lu-bit, oas %lu-bit (features 0x%08x)\n",
3499 smmu->ias, smmu->oas, smmu->features);
3500 return 0;
3501 }
3502
3503 #ifdef CONFIG_ACPI
3504 static void acpi_smmu_get_options(u32 model, struct arm_smmu_device *smmu)
3505 {
3506 switch (model) {
3507 case ACPI_IORT_SMMU_V3_CAVIUM_CN99XX:
3508 smmu->options |= ARM_SMMU_OPT_PAGE0_REGS_ONLY;
3509 break;
3510 case ACPI_IORT_SMMU_V3_HISILICON_HI161X:
3511 smmu->options |= ARM_SMMU_OPT_SKIP_PREFETCH;
3512 break;
3513 }
3514
3515 dev_notice(smmu->dev, "option mask 0x%x\n", smmu->options);
3516 }
3517
3518 static int arm_smmu_device_acpi_probe(struct platform_device *pdev,
3519 struct arm_smmu_device *smmu)
3520 {
3521 struct acpi_iort_smmu_v3 *iort_smmu;
3522 struct device *dev = smmu->dev;
3523 struct acpi_iort_node *node;
3524
3525 node = *(struct acpi_iort_node **)dev_get_platdata(dev);
3526
3527
3528 iort_smmu = (struct acpi_iort_smmu_v3 *)node->node_data;
3529
3530 acpi_smmu_get_options(iort_smmu->model, smmu);
3531
3532 if (iort_smmu->flags & ACPI_IORT_SMMU_V3_COHACC_OVERRIDE)
3533 smmu->features |= ARM_SMMU_FEAT_COHERENCY;
3534
3535 return 0;
3536 }
3537 #else
3538 static inline int arm_smmu_device_acpi_probe(struct platform_device *pdev,
3539 struct arm_smmu_device *smmu)
3540 {
3541 return -ENODEV;
3542 }
3543 #endif
3544
3545 static int arm_smmu_device_dt_probe(struct platform_device *pdev,
3546 struct arm_smmu_device *smmu)
3547 {
3548 struct device *dev = &pdev->dev;
3549 u32 cells;
3550 int ret = -EINVAL;
3551
3552 if (of_property_read_u32(dev->of_node, "#iommu-cells", &cells))
3553 dev_err(dev, "missing #iommu-cells property\n");
3554 else if (cells != 1)
3555 dev_err(dev, "invalid #iommu-cells value (%d)\n", cells);
3556 else
3557 ret = 0;
3558
3559 parse_driver_options(smmu);
3560
3561 if (of_dma_is_coherent(dev->of_node))
3562 smmu->features |= ARM_SMMU_FEAT_COHERENCY;
3563
3564 return ret;
3565 }
3566
3567 static unsigned long arm_smmu_resource_size(struct arm_smmu_device *smmu)
3568 {
3569 if (smmu->options & ARM_SMMU_OPT_PAGE0_REGS_ONLY)
3570 return SZ_64K;
3571 else
3572 return SZ_128K;
3573 }
3574
3575 static int arm_smmu_device_probe(struct platform_device *pdev)
3576 {
3577 int irq, ret;
3578 struct resource *res;
3579 resource_size_t ioaddr;
3580 struct arm_smmu_device *smmu;
3581 struct device *dev = &pdev->dev;
3582 bool bypass;
3583
3584 smmu = devm_kzalloc(dev, sizeof(*smmu), GFP_KERNEL);
3585 if (!smmu) {
3586 dev_err(dev, "failed to allocate arm_smmu_device\n");
3587 return -ENOMEM;
3588 }
3589 smmu->dev = dev;
3590
3591 if (dev->of_node) {
3592 ret = arm_smmu_device_dt_probe(pdev, smmu);
3593 } else {
3594 ret = arm_smmu_device_acpi_probe(pdev, smmu);
3595 if (ret == -ENODEV)
3596 return ret;
3597 }
3598
3599
3600 bypass = !!ret;
3601
3602
3603 res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
3604 if (resource_size(res) + 1 < arm_smmu_resource_size(smmu)) {
3605 dev_err(dev, "MMIO region too small (%pr)\n", res);
3606 return -EINVAL;
3607 }
3608 ioaddr = res->start;
3609
3610 smmu->base = devm_ioremap_resource(dev, res);
3611 if (IS_ERR(smmu->base))
3612 return PTR_ERR(smmu->base);
3613
3614
3615
3616 irq = platform_get_irq_byname_optional(pdev, "combined");
3617 if (irq > 0)
3618 smmu->combined_irq = irq;
3619 else {
3620 irq = platform_get_irq_byname_optional(pdev, "eventq");
3621 if (irq > 0)
3622 smmu->evtq.q.irq = irq;
3623
3624 irq = platform_get_irq_byname_optional(pdev, "priq");
3625 if (irq > 0)
3626 smmu->priq.q.irq = irq;
3627
3628 irq = platform_get_irq_byname_optional(pdev, "gerror");
3629 if (irq > 0)
3630 smmu->gerr_irq = irq;
3631 }
3632
3633 ret = arm_smmu_device_hw_probe(smmu);
3634 if (ret)
3635 return ret;
3636
3637
3638 ret = arm_smmu_init_structures(smmu);
3639 if (ret)
3640 return ret;
3641
3642
3643 platform_set_drvdata(pdev, smmu);
3644
3645
3646 ret = arm_smmu_device_reset(smmu, bypass);
3647 if (ret)
3648 return ret;
3649
3650
3651 ret = iommu_device_sysfs_add(&smmu->iommu, dev, NULL,
3652 "smmu3.%pa", &ioaddr);
3653 if (ret)
3654 return ret;
3655
3656 iommu_device_set_ops(&smmu->iommu, &arm_smmu_ops);
3657 iommu_device_set_fwnode(&smmu->iommu, dev->fwnode);
3658
3659 ret = iommu_device_register(&smmu->iommu);
3660 if (ret) {
3661 dev_err(dev, "Failed to register iommu\n");
3662 return ret;
3663 }
3664
3665 #ifdef CONFIG_PCI
3666 if (pci_bus_type.iommu_ops != &arm_smmu_ops) {
3667 pci_request_acs();
3668 ret = bus_set_iommu(&pci_bus_type, &arm_smmu_ops);
3669 if (ret)
3670 return ret;
3671 }
3672 #endif
3673 #ifdef CONFIG_ARM_AMBA
3674 if (amba_bustype.iommu_ops != &arm_smmu_ops) {
3675 ret = bus_set_iommu(&amba_bustype, &arm_smmu_ops);
3676 if (ret)
3677 return ret;
3678 }
3679 #endif
3680 if (platform_bus_type.iommu_ops != &arm_smmu_ops) {
3681 ret = bus_set_iommu(&platform_bus_type, &arm_smmu_ops);
3682 if (ret)
3683 return ret;
3684 }
3685 return 0;
3686 }
3687
3688 static void arm_smmu_device_shutdown(struct platform_device *pdev)
3689 {
3690 struct arm_smmu_device *smmu = platform_get_drvdata(pdev);
3691
3692 arm_smmu_device_disable(smmu);
3693 }
3694
3695 static const struct of_device_id arm_smmu_of_match[] = {
3696 { .compatible = "arm,smmu-v3", },
3697 { },
3698 };
3699
3700 static struct platform_driver arm_smmu_driver = {
3701 .driver = {
3702 .name = "arm-smmu-v3",
3703 .of_match_table = of_match_ptr(arm_smmu_of_match),
3704 .suppress_bind_attrs = true,
3705 },
3706 .probe = arm_smmu_device_probe,
3707 .shutdown = arm_smmu_device_shutdown,
3708 };
3709 builtin_platform_driver(arm_smmu_driver);