Lines Matching refs:smmu
604 struct arm_smmu_device *smmu; member
619 struct arm_smmu_device *smmu; member
649 static void parse_driver_options(struct arm_smmu_device *smmu) in parse_driver_options() argument
654 if (of_property_read_bool(smmu->dev->of_node, in parse_driver_options()
656 smmu->options |= arm_smmu_options[i].opt; in parse_driver_options()
657 dev_notice(smmu->dev, "option %s\n", in parse_driver_options()
840 static void arm_smmu_cmdq_skip_err(struct arm_smmu_device *smmu) in arm_smmu_cmdq_skip_err() argument
850 struct arm_smmu_queue *q = &smmu->cmdq.q; in arm_smmu_cmdq_skip_err()
857 dev_err(smmu->dev, "CMDQ error (cons 0x%08x): %s\n", cons, in arm_smmu_cmdq_skip_err()
864 dev_err(smmu->dev, "retrying command fetch\n"); in arm_smmu_cmdq_skip_err()
874 dev_err(smmu->dev, "skipping command in error state:\n"); in arm_smmu_cmdq_skip_err()
876 dev_err(smmu->dev, "\t0x%016llx\n", (unsigned long long)cmd[i]); in arm_smmu_cmdq_skip_err()
880 dev_err(smmu->dev, "failed to convert to CMD_SYNC\n"); in arm_smmu_cmdq_skip_err()
887 static void arm_smmu_cmdq_issue_cmd(struct arm_smmu_device *smmu, in arm_smmu_cmdq_issue_cmd() argument
892 bool wfe = !!(smmu->features & ARM_SMMU_FEAT_SEV); in arm_smmu_cmdq_issue_cmd()
893 struct arm_smmu_queue *q = &smmu->cmdq.q; in arm_smmu_cmdq_issue_cmd()
896 dev_warn(smmu->dev, "ignoring unknown CMDQ opcode 0x%x\n", in arm_smmu_cmdq_issue_cmd()
901 spin_lock(&smmu->cmdq.lock); in arm_smmu_cmdq_issue_cmd()
909 dev_err_ratelimited(smmu->dev, "CMDQ timeout\n"); in arm_smmu_cmdq_issue_cmd()
913 dev_err_ratelimited(smmu->dev, "CMD_SYNC timeout\n"); in arm_smmu_cmdq_issue_cmd()
914 spin_unlock(&smmu->cmdq.lock); in arm_smmu_cmdq_issue_cmd()
936 static void arm_smmu_write_ctx_desc(struct arm_smmu_device *smmu, in arm_smmu_write_ctx_desc() argument
974 static void arm_smmu_sync_ste_for_sid(struct arm_smmu_device *smmu, u32 sid) in arm_smmu_sync_ste_for_sid() argument
984 arm_smmu_cmdq_issue_cmd(smmu, &cmd); in arm_smmu_sync_ste_for_sid()
986 arm_smmu_cmdq_issue_cmd(smmu, &cmd); in arm_smmu_sync_ste_for_sid()
989 static void arm_smmu_write_strtab_ent(struct arm_smmu_device *smmu, u32 sid, in arm_smmu_write_strtab_ent() argument
1047 arm_smmu_sync_ste_for_sid(smmu, sid); in arm_smmu_write_strtab_ent()
1089 arm_smmu_sync_ste_for_sid(smmu, sid); in arm_smmu_write_strtab_ent()
1091 arm_smmu_sync_ste_for_sid(smmu, sid); in arm_smmu_write_strtab_ent()
1094 if (!(smmu->options & ARM_SMMU_OPT_SKIP_PREFETCH)) in arm_smmu_write_strtab_ent()
1095 arm_smmu_cmdq_issue_cmd(smmu, &prefetch_cmd); in arm_smmu_write_strtab_ent()
1112 static int arm_smmu_init_l2_strtab(struct arm_smmu_device *smmu, u32 sid) in arm_smmu_init_l2_strtab() argument
1116 struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg; in arm_smmu_init_l2_strtab()
1126 desc->l2ptr = dma_zalloc_coherent(smmu->dev, size, &desc->l2ptr_dma, in arm_smmu_init_l2_strtab()
1129 dev_err(smmu->dev, in arm_smmu_init_l2_strtab()
1144 struct arm_smmu_device *smmu = dev; in arm_smmu_evtq_thread() local
1145 struct arm_smmu_queue *q = &smmu->evtq.q; in arm_smmu_evtq_thread()
1151 dev_info(smmu->dev, "event 0x%02x received:\n", id); in arm_smmu_evtq_thread()
1153 dev_info(smmu->dev, "\t0x%016llx\n", in arm_smmu_evtq_thread()
1165 struct arm_smmu_device *smmu = dev; in arm_smmu_evtq_handler() local
1166 struct arm_smmu_queue *q = &smmu->evtq.q; in arm_smmu_evtq_handler()
1173 dev_err(smmu->dev, "EVTQ overflow detected -- events lost\n"); in arm_smmu_evtq_handler()
1182 struct arm_smmu_device *smmu = dev; in arm_smmu_priq_thread() local
1183 struct arm_smmu_queue *q = &smmu->priq.q; in arm_smmu_priq_thread()
1197 dev_info(smmu->dev, "unexpected PRI request received:\n"); in arm_smmu_priq_thread()
1198 dev_info(smmu->dev, in arm_smmu_priq_thread()
1219 arm_smmu_cmdq_issue_cmd(smmu, &cmd); in arm_smmu_priq_thread()
1231 struct arm_smmu_device *smmu = dev; in arm_smmu_priq_handler() local
1232 struct arm_smmu_queue *q = &smmu->priq.q; in arm_smmu_priq_handler()
1236 dev_err(smmu->dev, "PRIQ overflow detected -- requests lost\n"); in arm_smmu_priq_handler()
1249 static int arm_smmu_device_disable(struct arm_smmu_device *smmu);
1254 struct arm_smmu_device *smmu = dev; in arm_smmu_gerror_handler() local
1256 gerror = readl_relaxed(smmu->base + ARM_SMMU_GERROR); in arm_smmu_gerror_handler()
1257 gerrorn = readl_relaxed(smmu->base + ARM_SMMU_GERRORN); in arm_smmu_gerror_handler()
1263 dev_warn(smmu->dev, in arm_smmu_gerror_handler()
1268 dev_err(smmu->dev, "device has entered Service Failure Mode!\n"); in arm_smmu_gerror_handler()
1269 arm_smmu_device_disable(smmu); in arm_smmu_gerror_handler()
1273 dev_warn(smmu->dev, "GERROR MSI write aborted\n"); in arm_smmu_gerror_handler()
1276 dev_warn(smmu->dev, "PRIQ MSI write aborted\n"); in arm_smmu_gerror_handler()
1277 arm_smmu_priq_handler(irq, smmu->dev); in arm_smmu_gerror_handler()
1281 dev_warn(smmu->dev, "EVTQ MSI write aborted\n"); in arm_smmu_gerror_handler()
1282 arm_smmu_evtq_handler(irq, smmu->dev); in arm_smmu_gerror_handler()
1286 dev_warn(smmu->dev, "CMDQ MSI write aborted\n"); in arm_smmu_gerror_handler()
1287 arm_smmu_cmdq_sync_handler(irq, smmu->dev); in arm_smmu_gerror_handler()
1291 dev_err(smmu->dev, "PRIQ write aborted -- events may have been lost\n"); in arm_smmu_gerror_handler()
1294 dev_err(smmu->dev, "EVTQ write aborted -- events may have been lost\n"); in arm_smmu_gerror_handler()
1297 arm_smmu_cmdq_skip_err(smmu); in arm_smmu_gerror_handler()
1299 writel(gerror, smmu->base + ARM_SMMU_GERRORN); in arm_smmu_gerror_handler()
1304 static void __arm_smmu_tlb_sync(struct arm_smmu_device *smmu) in __arm_smmu_tlb_sync() argument
1309 arm_smmu_cmdq_issue_cmd(smmu, &cmd); in __arm_smmu_tlb_sync()
1315 __arm_smmu_tlb_sync(smmu_domain->smmu); in arm_smmu_tlb_sync()
1321 struct arm_smmu_device *smmu = smmu_domain->smmu; in arm_smmu_tlb_inv_context() local
1333 arm_smmu_cmdq_issue_cmd(smmu, &cmd); in arm_smmu_tlb_inv_context()
1334 __arm_smmu_tlb_sync(smmu); in arm_smmu_tlb_inv_context()
1341 struct arm_smmu_device *smmu = smmu_domain->smmu; in arm_smmu_tlb_inv_range_nosync() local
1357 arm_smmu_cmdq_issue_cmd(smmu, &cmd); in arm_smmu_tlb_inv_range_nosync()
1423 struct arm_smmu_device *smmu = smmu_domain->smmu; in arm_smmu_domain_free() local
1432 dma_free_coherent(smmu_domain->smmu->dev, in arm_smmu_domain_free()
1437 arm_smmu_bitmap_free(smmu->asid_map, cfg->cd.asid); in arm_smmu_domain_free()
1442 arm_smmu_bitmap_free(smmu->vmid_map, cfg->vmid); in arm_smmu_domain_free()
1453 struct arm_smmu_device *smmu = smmu_domain->smmu; in arm_smmu_domain_finalise_s1() local
1456 asid = arm_smmu_bitmap_alloc(smmu->asid_map, smmu->asid_bits); in arm_smmu_domain_finalise_s1()
1460 cfg->cdptr = dma_zalloc_coherent(smmu->dev, CTXDESC_CD_DWORDS << 3, in arm_smmu_domain_finalise_s1()
1463 dev_warn(smmu->dev, "failed to allocate context descriptor\n"); in arm_smmu_domain_finalise_s1()
1475 arm_smmu_bitmap_free(smmu->asid_map, asid); in arm_smmu_domain_finalise_s1()
1483 struct arm_smmu_device *smmu = smmu_domain->smmu; in arm_smmu_domain_finalise_s2() local
1486 vmid = arm_smmu_bitmap_alloc(smmu->vmid_map, smmu->vmid_bits); in arm_smmu_domain_finalise_s2()
1508 struct arm_smmu_device *smmu = smmu_domain->smmu; in arm_smmu_domain_finalise() local
1511 if (!(smmu->features & ARM_SMMU_FEAT_TRANS_S1)) in arm_smmu_domain_finalise()
1513 if (!(smmu->features & ARM_SMMU_FEAT_TRANS_S2)) in arm_smmu_domain_finalise()
1519 oas = smmu->ias; in arm_smmu_domain_finalise()
1525 ias = smmu->ias; in arm_smmu_domain_finalise()
1526 oas = smmu->oas; in arm_smmu_domain_finalise()
1539 .iommu_dev = smmu->dev, in arm_smmu_domain_finalise()
1570 static __le64 *arm_smmu_get_step_for_sid(struct arm_smmu_device *smmu, u32 sid) in arm_smmu_get_step_for_sid() argument
1573 struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg; in arm_smmu_get_step_for_sid()
1575 if (smmu->features & ARM_SMMU_FEAT_2_LVL_STRTAB) { in arm_smmu_get_step_for_sid()
1597 struct arm_smmu_device *smmu = smmu_group->smmu; in arm_smmu_install_ste_for_group() local
1602 arm_smmu_write_ctx_desc(smmu, ste->s1_cfg); in arm_smmu_install_ste_for_group()
1610 __le64 *step = arm_smmu_get_step_for_sid(smmu, sid); in arm_smmu_install_ste_for_group()
1612 arm_smmu_write_strtab_ent(smmu, sid, step, ste); in arm_smmu_install_ste_for_group()
1621 struct arm_smmu_device *smmu; in arm_smmu_attach_dev() local
1632 smmu = smmu_group->smmu; in arm_smmu_attach_dev()
1635 if (!smmu_domain->smmu) { in arm_smmu_attach_dev()
1636 smmu_domain->smmu = smmu; in arm_smmu_attach_dev()
1639 smmu_domain->smmu = NULL; in arm_smmu_attach_dev()
1642 } else if (smmu_domain->smmu != smmu) { in arm_smmu_attach_dev()
1645 dev_name(smmu_domain->smmu->dev), in arm_smmu_attach_dev()
1646 dev_name(smmu->dev)); in arm_smmu_attach_dev()
1753 struct arm_smmu_device *smmu = NULL; in arm_smmu_get_for_pci_dev() local
1768 smmu = platform_get_drvdata(smmu_pdev); in arm_smmu_get_for_pci_dev()
1771 return smmu; in arm_smmu_get_for_pci_dev()
1774 static bool arm_smmu_sid_in_range(struct arm_smmu_device *smmu, u32 sid) in arm_smmu_sid_in_range() argument
1776 unsigned long limit = smmu->strtab_cfg.num_l1_ents; in arm_smmu_sid_in_range()
1778 if (smmu->features & ARM_SMMU_FEAT_2_LVL_STRTAB) in arm_smmu_sid_in_range()
1791 struct arm_smmu_device *smmu; in arm_smmu_add_device() local
1804 smmu = arm_smmu_get_for_pci_dev(pdev); in arm_smmu_add_device()
1805 if (!smmu) { in arm_smmu_add_device()
1817 smmu_group->smmu = smmu; in arm_smmu_add_device()
1821 smmu = smmu_group->smmu; in arm_smmu_add_device()
1833 if (!arm_smmu_sid_in_range(smmu, sid)) { in arm_smmu_add_device()
1839 if (smmu->features & ARM_SMMU_FEAT_2_LVL_STRTAB) { in arm_smmu_add_device()
1840 ret = arm_smmu_init_l2_strtab(smmu, sid); in arm_smmu_add_device()
1894 if (smmu_domain->smmu) { in arm_smmu_domain_set_attr()
1932 static int arm_smmu_init_one_queue(struct arm_smmu_device *smmu, in arm_smmu_init_one_queue() argument
1940 q->base = dma_alloc_coherent(smmu->dev, qsz, &q->base_dma, GFP_KERNEL); in arm_smmu_init_one_queue()
1942 dev_err(smmu->dev, "failed to allocate queue (0x%zx bytes)\n", in arm_smmu_init_one_queue()
1947 q->prod_reg = smmu->base + prod_off; in arm_smmu_init_one_queue()
1948 q->cons_reg = smmu->base + cons_off; in arm_smmu_init_one_queue()
1960 static void arm_smmu_free_one_queue(struct arm_smmu_device *smmu, in arm_smmu_free_one_queue() argument
1965 dma_free_coherent(smmu->dev, qsz, q->base, q->base_dma); in arm_smmu_free_one_queue()
1968 static void arm_smmu_free_queues(struct arm_smmu_device *smmu) in arm_smmu_free_queues() argument
1970 arm_smmu_free_one_queue(smmu, &smmu->cmdq.q); in arm_smmu_free_queues()
1971 arm_smmu_free_one_queue(smmu, &smmu->evtq.q); in arm_smmu_free_queues()
1973 if (smmu->features & ARM_SMMU_FEAT_PRI) in arm_smmu_free_queues()
1974 arm_smmu_free_one_queue(smmu, &smmu->priq.q); in arm_smmu_free_queues()
1977 static int arm_smmu_init_queues(struct arm_smmu_device *smmu) in arm_smmu_init_queues() argument
1982 spin_lock_init(&smmu->cmdq.lock); in arm_smmu_init_queues()
1983 ret = arm_smmu_init_one_queue(smmu, &smmu->cmdq.q, ARM_SMMU_CMDQ_PROD, in arm_smmu_init_queues()
1989 ret = arm_smmu_init_one_queue(smmu, &smmu->evtq.q, ARM_SMMU_EVTQ_PROD, in arm_smmu_init_queues()
1995 if (!(smmu->features & ARM_SMMU_FEAT_PRI)) in arm_smmu_init_queues()
1998 ret = arm_smmu_init_one_queue(smmu, &smmu->priq.q, ARM_SMMU_PRIQ_PROD, in arm_smmu_init_queues()
2006 arm_smmu_free_one_queue(smmu, &smmu->evtq.q); in arm_smmu_init_queues()
2008 arm_smmu_free_one_queue(smmu, &smmu->cmdq.q); in arm_smmu_init_queues()
2013 static void arm_smmu_free_l2_strtab(struct arm_smmu_device *smmu) in arm_smmu_free_l2_strtab() argument
2017 struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg; in arm_smmu_free_l2_strtab()
2026 dma_free_coherent(smmu->dev, size, desc->l2ptr, in arm_smmu_free_l2_strtab()
2031 static int arm_smmu_init_l1_strtab(struct arm_smmu_device *smmu) in arm_smmu_init_l1_strtab() argument
2034 struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg; in arm_smmu_init_l1_strtab()
2036 void *strtab = smmu->strtab_cfg.strtab; in arm_smmu_init_l1_strtab()
2038 cfg->l1_desc = devm_kzalloc(smmu->dev, size, GFP_KERNEL); in arm_smmu_init_l1_strtab()
2040 dev_err(smmu->dev, "failed to allocate l1 stream table desc\n"); in arm_smmu_init_l1_strtab()
2052 static int arm_smmu_init_strtab_2lvl(struct arm_smmu_device *smmu) in arm_smmu_init_strtab_2lvl() argument
2058 struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg; in arm_smmu_init_strtab_2lvl()
2065 if (smmu->sid_bits < STRTAB_SPLIT) { in arm_smmu_init_strtab_2lvl()
2069 size = min(size, smmu->sid_bits - STRTAB_SPLIT); in arm_smmu_init_strtab_2lvl()
2074 if (size < smmu->sid_bits) in arm_smmu_init_strtab_2lvl()
2075 dev_warn(smmu->dev, in arm_smmu_init_strtab_2lvl()
2077 size, smmu->sid_bits); in arm_smmu_init_strtab_2lvl()
2080 strtab = dma_zalloc_coherent(smmu->dev, l1size, &cfg->strtab_dma, in arm_smmu_init_strtab_2lvl()
2083 dev_err(smmu->dev, in arm_smmu_init_strtab_2lvl()
2098 ret = arm_smmu_init_l1_strtab(smmu); in arm_smmu_init_strtab_2lvl()
2100 dma_free_coherent(smmu->dev, in arm_smmu_init_strtab_2lvl()
2107 static int arm_smmu_init_strtab_linear(struct arm_smmu_device *smmu) in arm_smmu_init_strtab_linear() argument
2112 struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg; in arm_smmu_init_strtab_linear()
2114 size = (1 << smmu->sid_bits) * (STRTAB_STE_DWORDS << 3); in arm_smmu_init_strtab_linear()
2115 strtab = dma_zalloc_coherent(smmu->dev, size, &cfg->strtab_dma, in arm_smmu_init_strtab_linear()
2118 dev_err(smmu->dev, in arm_smmu_init_strtab_linear()
2124 cfg->num_l1_ents = 1 << smmu->sid_bits; in arm_smmu_init_strtab_linear()
2128 reg |= (smmu->sid_bits & STRTAB_BASE_CFG_LOG2SIZE_MASK) in arm_smmu_init_strtab_linear()
2136 static int arm_smmu_init_strtab(struct arm_smmu_device *smmu) in arm_smmu_init_strtab() argument
2141 if (smmu->features & ARM_SMMU_FEAT_2_LVL_STRTAB) in arm_smmu_init_strtab()
2142 ret = arm_smmu_init_strtab_2lvl(smmu); in arm_smmu_init_strtab()
2144 ret = arm_smmu_init_strtab_linear(smmu); in arm_smmu_init_strtab()
2150 reg = smmu->strtab_cfg.strtab_dma & in arm_smmu_init_strtab()
2153 smmu->strtab_cfg.strtab_base = reg; in arm_smmu_init_strtab()
2156 set_bit(0, smmu->vmid_map); in arm_smmu_init_strtab()
2160 static void arm_smmu_free_strtab(struct arm_smmu_device *smmu) in arm_smmu_free_strtab() argument
2162 struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg; in arm_smmu_free_strtab()
2165 if (smmu->features & ARM_SMMU_FEAT_2_LVL_STRTAB) { in arm_smmu_free_strtab()
2166 arm_smmu_free_l2_strtab(smmu); in arm_smmu_free_strtab()
2172 dma_free_coherent(smmu->dev, size, cfg->strtab, cfg->strtab_dma); in arm_smmu_free_strtab()
2175 static int arm_smmu_init_structures(struct arm_smmu_device *smmu) in arm_smmu_init_structures() argument
2179 ret = arm_smmu_init_queues(smmu); in arm_smmu_init_structures()
2183 ret = arm_smmu_init_strtab(smmu); in arm_smmu_init_structures()
2190 arm_smmu_free_queues(smmu); in arm_smmu_init_structures()
2194 static void arm_smmu_free_structures(struct arm_smmu_device *smmu) in arm_smmu_free_structures() argument
2196 arm_smmu_free_strtab(smmu); in arm_smmu_free_structures()
2197 arm_smmu_free_queues(smmu); in arm_smmu_free_structures()
2200 static int arm_smmu_write_reg_sync(struct arm_smmu_device *smmu, u32 val, in arm_smmu_write_reg_sync() argument
2205 writel_relaxed(val, smmu->base + reg_off); in arm_smmu_write_reg_sync()
2206 return readl_relaxed_poll_timeout(smmu->base + ack_off, reg, reg == val, in arm_smmu_write_reg_sync()
2220 struct arm_smmu_device *smmu = dev_get_drvdata(dev); in arm_smmu_write_msi_msg() local
2226 writeq_relaxed(doorbell, smmu->base + cfg[0]); in arm_smmu_write_msi_msg()
2227 writel_relaxed(msg->data, smmu->base + cfg[1]); in arm_smmu_write_msi_msg()
2228 writel_relaxed(MSI_CFG2_MEMATTR_DEVICE_nGnRE, smmu->base + cfg[2]); in arm_smmu_write_msi_msg()
2231 static void arm_smmu_setup_msis(struct arm_smmu_device *smmu) in arm_smmu_setup_msis() argument
2235 struct device *dev = smmu->dev; in arm_smmu_setup_msis()
2238 writeq_relaxed(0, smmu->base + ARM_SMMU_GERROR_IRQ_CFG0); in arm_smmu_setup_msis()
2239 writeq_relaxed(0, smmu->base + ARM_SMMU_EVTQ_IRQ_CFG0); in arm_smmu_setup_msis()
2241 if (smmu->features & ARM_SMMU_FEAT_PRI) in arm_smmu_setup_msis()
2242 writeq_relaxed(0, smmu->base + ARM_SMMU_PRIQ_IRQ_CFG0); in arm_smmu_setup_msis()
2246 if (!(smmu->features & ARM_SMMU_FEAT_MSI)) in arm_smmu_setup_msis()
2259 smmu->evtq.q.irq = desc->irq; in arm_smmu_setup_msis()
2262 smmu->gerr_irq = desc->irq; in arm_smmu_setup_msis()
2265 smmu->priq.q.irq = desc->irq; in arm_smmu_setup_msis()
2276 static int arm_smmu_setup_irqs(struct arm_smmu_device *smmu) in arm_smmu_setup_irqs() argument
2282 ret = arm_smmu_write_reg_sync(smmu, 0, ARM_SMMU_IRQ_CTRL, in arm_smmu_setup_irqs()
2285 dev_err(smmu->dev, "failed to disable irqs\n"); in arm_smmu_setup_irqs()
2289 arm_smmu_setup_msis(smmu); in arm_smmu_setup_irqs()
2292 irq = smmu->evtq.q.irq; in arm_smmu_setup_irqs()
2294 ret = devm_request_threaded_irq(smmu->dev, irq, in arm_smmu_setup_irqs()
2297 0, "arm-smmu-v3-evtq", smmu); in arm_smmu_setup_irqs()
2299 dev_warn(smmu->dev, "failed to enable evtq irq\n"); in arm_smmu_setup_irqs()
2302 irq = smmu->cmdq.q.irq; in arm_smmu_setup_irqs()
2304 ret = devm_request_irq(smmu->dev, irq, in arm_smmu_setup_irqs()
2306 "arm-smmu-v3-cmdq-sync", smmu); in arm_smmu_setup_irqs()
2308 dev_warn(smmu->dev, "failed to enable cmdq-sync irq\n"); in arm_smmu_setup_irqs()
2311 irq = smmu->gerr_irq; in arm_smmu_setup_irqs()
2313 ret = devm_request_irq(smmu->dev, irq, arm_smmu_gerror_handler, in arm_smmu_setup_irqs()
2314 0, "arm-smmu-v3-gerror", smmu); in arm_smmu_setup_irqs()
2316 dev_warn(smmu->dev, "failed to enable gerror irq\n"); in arm_smmu_setup_irqs()
2319 if (smmu->features & ARM_SMMU_FEAT_PRI) { in arm_smmu_setup_irqs()
2320 irq = smmu->priq.q.irq; in arm_smmu_setup_irqs()
2322 ret = devm_request_threaded_irq(smmu->dev, irq, in arm_smmu_setup_irqs()
2326 smmu); in arm_smmu_setup_irqs()
2328 dev_warn(smmu->dev, in arm_smmu_setup_irqs()
2336 ret = arm_smmu_write_reg_sync(smmu, irqen_flags, in arm_smmu_setup_irqs()
2339 dev_warn(smmu->dev, "failed to enable irqs\n"); in arm_smmu_setup_irqs()
2344 static int arm_smmu_device_disable(struct arm_smmu_device *smmu) in arm_smmu_device_disable() argument
2348 ret = arm_smmu_write_reg_sync(smmu, 0, ARM_SMMU_CR0, ARM_SMMU_CR0ACK); in arm_smmu_device_disable()
2350 dev_err(smmu->dev, "failed to clear cr0\n"); in arm_smmu_device_disable()
2355 static int arm_smmu_device_reset(struct arm_smmu_device *smmu) in arm_smmu_device_reset() argument
2362 reg = readl_relaxed(smmu->base + ARM_SMMU_CR0); in arm_smmu_device_reset()
2364 dev_warn(smmu->dev, "SMMU currently enabled! Resetting...\n"); in arm_smmu_device_reset()
2366 ret = arm_smmu_device_disable(smmu); in arm_smmu_device_reset()
2377 writel_relaxed(reg, smmu->base + ARM_SMMU_CR1); in arm_smmu_device_reset()
2381 writel_relaxed(reg, smmu->base + ARM_SMMU_CR2); in arm_smmu_device_reset()
2384 writeq_relaxed(smmu->strtab_cfg.strtab_base, in arm_smmu_device_reset()
2385 smmu->base + ARM_SMMU_STRTAB_BASE); in arm_smmu_device_reset()
2386 writel_relaxed(smmu->strtab_cfg.strtab_base_cfg, in arm_smmu_device_reset()
2387 smmu->base + ARM_SMMU_STRTAB_BASE_CFG); in arm_smmu_device_reset()
2390 writeq_relaxed(smmu->cmdq.q.q_base, smmu->base + ARM_SMMU_CMDQ_BASE); in arm_smmu_device_reset()
2391 writel_relaxed(smmu->cmdq.q.prod, smmu->base + ARM_SMMU_CMDQ_PROD); in arm_smmu_device_reset()
2392 writel_relaxed(smmu->cmdq.q.cons, smmu->base + ARM_SMMU_CMDQ_CONS); in arm_smmu_device_reset()
2395 ret = arm_smmu_write_reg_sync(smmu, enables, ARM_SMMU_CR0, in arm_smmu_device_reset()
2398 dev_err(smmu->dev, "failed to enable command queue\n"); in arm_smmu_device_reset()
2404 arm_smmu_cmdq_issue_cmd(smmu, &cmd); in arm_smmu_device_reset()
2406 arm_smmu_cmdq_issue_cmd(smmu, &cmd); in arm_smmu_device_reset()
2409 if (smmu->features & ARM_SMMU_FEAT_HYP) { in arm_smmu_device_reset()
2411 arm_smmu_cmdq_issue_cmd(smmu, &cmd); in arm_smmu_device_reset()
2415 arm_smmu_cmdq_issue_cmd(smmu, &cmd); in arm_smmu_device_reset()
2417 arm_smmu_cmdq_issue_cmd(smmu, &cmd); in arm_smmu_device_reset()
2420 writeq_relaxed(smmu->evtq.q.q_base, smmu->base + ARM_SMMU_EVTQ_BASE); in arm_smmu_device_reset()
2421 writel_relaxed(smmu->evtq.q.prod, smmu->base + ARM_SMMU_EVTQ_PROD); in arm_smmu_device_reset()
2422 writel_relaxed(smmu->evtq.q.cons, smmu->base + ARM_SMMU_EVTQ_CONS); in arm_smmu_device_reset()
2425 ret = arm_smmu_write_reg_sync(smmu, enables, ARM_SMMU_CR0, in arm_smmu_device_reset()
2428 dev_err(smmu->dev, "failed to enable event queue\n"); in arm_smmu_device_reset()
2433 if (smmu->features & ARM_SMMU_FEAT_PRI) { in arm_smmu_device_reset()
2434 writeq_relaxed(smmu->priq.q.q_base, in arm_smmu_device_reset()
2435 smmu->base + ARM_SMMU_PRIQ_BASE); in arm_smmu_device_reset()
2436 writel_relaxed(smmu->priq.q.prod, in arm_smmu_device_reset()
2437 smmu->base + ARM_SMMU_PRIQ_PROD); in arm_smmu_device_reset()
2438 writel_relaxed(smmu->priq.q.cons, in arm_smmu_device_reset()
2439 smmu->base + ARM_SMMU_PRIQ_CONS); in arm_smmu_device_reset()
2442 ret = arm_smmu_write_reg_sync(smmu, enables, ARM_SMMU_CR0, in arm_smmu_device_reset()
2445 dev_err(smmu->dev, "failed to enable PRI queue\n"); in arm_smmu_device_reset()
2450 ret = arm_smmu_setup_irqs(smmu); in arm_smmu_device_reset()
2452 dev_err(smmu->dev, "failed to setup irqs\n"); in arm_smmu_device_reset()
2458 ret = arm_smmu_write_reg_sync(smmu, enables, ARM_SMMU_CR0, in arm_smmu_device_reset()
2461 dev_err(smmu->dev, "failed to enable SMMU interface\n"); in arm_smmu_device_reset()
2468 static int arm_smmu_device_probe(struct arm_smmu_device *smmu) in arm_smmu_device_probe() argument
2475 reg = readl_relaxed(smmu->base + ARM_SMMU_IDR0); in arm_smmu_device_probe()
2479 smmu->features |= ARM_SMMU_FEAT_2_LVL_STRTAB; in arm_smmu_device_probe()
2482 smmu->features |= ARM_SMMU_FEAT_2_LVL_CDTAB; in arm_smmu_device_probe()
2491 smmu->features |= ARM_SMMU_FEAT_TT_LE | ARM_SMMU_FEAT_TT_BE; in arm_smmu_device_probe()
2495 smmu->features |= ARM_SMMU_FEAT_TT_BE; in arm_smmu_device_probe()
2499 smmu->features |= ARM_SMMU_FEAT_TT_LE; in arm_smmu_device_probe()
2503 dev_err(smmu->dev, "unknown/unsupported TT endianness!\n"); in arm_smmu_device_probe()
2509 smmu->features |= ARM_SMMU_FEAT_PRI; in arm_smmu_device_probe()
2512 smmu->features |= ARM_SMMU_FEAT_ATS; in arm_smmu_device_probe()
2515 smmu->features |= ARM_SMMU_FEAT_SEV; in arm_smmu_device_probe()
2518 smmu->features |= ARM_SMMU_FEAT_MSI; in arm_smmu_device_probe()
2521 smmu->features |= ARM_SMMU_FEAT_HYP; in arm_smmu_device_probe()
2527 coherent = of_dma_is_coherent(smmu->dev->of_node); in arm_smmu_device_probe()
2529 smmu->features |= ARM_SMMU_FEAT_COHERENCY; in arm_smmu_device_probe()
2532 dev_warn(smmu->dev, "IDR0.COHACC overridden by dma-coherent property (%s)\n", in arm_smmu_device_probe()
2536 smmu->features |= ARM_SMMU_FEAT_STALLS; in arm_smmu_device_probe()
2539 smmu->features |= ARM_SMMU_FEAT_TRANS_S1; in arm_smmu_device_probe()
2542 smmu->features |= ARM_SMMU_FEAT_TRANS_S2; in arm_smmu_device_probe()
2545 dev_err(smmu->dev, "no translation support!\n"); in arm_smmu_device_probe()
2552 smmu->ias = 40; in arm_smmu_device_probe()
2557 dev_err(smmu->dev, "AArch64 table format not supported!\n"); in arm_smmu_device_probe()
2562 smmu->asid_bits = reg & IDR0_ASID16 ? 16 : 8; in arm_smmu_device_probe()
2563 smmu->vmid_bits = reg & IDR0_VMID16 ? 16 : 8; in arm_smmu_device_probe()
2566 reg = readl_relaxed(smmu->base + ARM_SMMU_IDR1); in arm_smmu_device_probe()
2568 dev_err(smmu->dev, "embedded implementation not supported\n"); in arm_smmu_device_probe()
2573 smmu->cmdq.q.max_n_shift = min((u32)CMDQ_MAX_SZ_SHIFT, in arm_smmu_device_probe()
2575 if (!smmu->cmdq.q.max_n_shift) { in arm_smmu_device_probe()
2577 dev_err(smmu->dev, "unit-length command queue not supported\n"); in arm_smmu_device_probe()
2581 smmu->evtq.q.max_n_shift = min((u32)EVTQ_MAX_SZ_SHIFT, in arm_smmu_device_probe()
2583 smmu->priq.q.max_n_shift = min((u32)PRIQ_MAX_SZ_SHIFT, in arm_smmu_device_probe()
2587 smmu->ssid_bits = reg >> IDR1_SSID_SHIFT & IDR1_SSID_MASK; in arm_smmu_device_probe()
2588 smmu->sid_bits = reg >> IDR1_SID_SHIFT & IDR1_SID_MASK; in arm_smmu_device_probe()
2591 reg = readl_relaxed(smmu->base + ARM_SMMU_IDR5); in arm_smmu_device_probe()
2594 smmu->evtq.max_stalls = reg >> IDR5_STALL_MAX_SHIFT in arm_smmu_device_probe()
2610 smmu->oas = 32; in arm_smmu_device_probe()
2613 smmu->oas = 36; in arm_smmu_device_probe()
2616 smmu->oas = 40; in arm_smmu_device_probe()
2619 smmu->oas = 42; in arm_smmu_device_probe()
2622 smmu->oas = 44; in arm_smmu_device_probe()
2625 dev_info(smmu->dev, in arm_smmu_device_probe()
2629 smmu->oas = 48; in arm_smmu_device_probe()
2633 if (dma_set_mask_and_coherent(smmu->dev, DMA_BIT_MASK(smmu->oas))) in arm_smmu_device_probe()
2634 dev_warn(smmu->dev, in arm_smmu_device_probe()
2637 smmu->ias = max(smmu->ias, smmu->oas); in arm_smmu_device_probe()
2639 dev_info(smmu->dev, "ias %lu-bit, oas %lu-bit (features 0x%08x)\n", in arm_smmu_device_probe()
2640 smmu->ias, smmu->oas, smmu->features); in arm_smmu_device_probe()
2648 struct arm_smmu_device *smmu; in arm_smmu_device_dt_probe() local
2651 smmu = devm_kzalloc(dev, sizeof(*smmu), GFP_KERNEL); in arm_smmu_device_dt_probe()
2652 if (!smmu) { in arm_smmu_device_dt_probe()
2656 smmu->dev = dev; in arm_smmu_device_dt_probe()
2665 smmu->base = devm_ioremap_resource(dev, res); in arm_smmu_device_dt_probe()
2666 if (IS_ERR(smmu->base)) in arm_smmu_device_dt_probe()
2667 return PTR_ERR(smmu->base); in arm_smmu_device_dt_probe()
2672 smmu->evtq.q.irq = irq; in arm_smmu_device_dt_probe()
2676 smmu->priq.q.irq = irq; in arm_smmu_device_dt_probe()
2680 smmu->cmdq.q.irq = irq; in arm_smmu_device_dt_probe()
2684 smmu->gerr_irq = irq; in arm_smmu_device_dt_probe()
2686 parse_driver_options(smmu); in arm_smmu_device_dt_probe()
2689 ret = arm_smmu_device_probe(smmu); in arm_smmu_device_dt_probe()
2694 ret = arm_smmu_init_structures(smmu); in arm_smmu_device_dt_probe()
2699 platform_set_drvdata(pdev, smmu); in arm_smmu_device_dt_probe()
2702 ret = arm_smmu_device_reset(smmu); in arm_smmu_device_dt_probe()
2709 arm_smmu_free_structures(smmu); in arm_smmu_device_dt_probe()
2715 struct arm_smmu_device *smmu = platform_get_drvdata(pdev); in arm_smmu_device_remove() local
2717 arm_smmu_device_disable(smmu); in arm_smmu_device_remove()
2718 arm_smmu_free_structures(smmu); in arm_smmu_device_remove()