1/*
2 * Marvell Armada 370 and Armada XP SoC IRQ handling
3 *
4 * Copyright (C) 2012 Marvell
5 *
6 * Lior Amsalem <alior@marvell.com>
7 * Gregory CLEMENT <gregory.clement@free-electrons.com>
8 * Thomas Petazzoni <thomas.petazzoni@free-electrons.com>
9 * Ben Dooks <ben.dooks@codethink.co.uk>
10 *
11 * This file is licensed under the terms of the GNU General Public
12 * License version 2.  This program is licensed "as is" without any
13 * warranty of any kind, whether express or implied.
14 */
15
16#include <linux/kernel.h>
17#include <linux/module.h>
18#include <linux/init.h>
19#include <linux/irq.h>
20#include <linux/interrupt.h>
21#include <linux/irqchip.h>
22#include <linux/irqchip/chained_irq.h>
23#include <linux/cpu.h>
24#include <linux/io.h>
25#include <linux/of_address.h>
26#include <linux/of_irq.h>
27#include <linux/of_pci.h>
28#include <linux/irqdomain.h>
29#include <linux/slab.h>
30#include <linux/syscore_ops.h>
31#include <linux/msi.h>
32#include <asm/mach/arch.h>
33#include <asm/exception.h>
34#include <asm/smp_plat.h>
35#include <asm/mach/irq.h>
36
37/* Interrupt Controller Registers Map */
38#define ARMADA_370_XP_INT_SET_MASK_OFFS		(0x48)
39#define ARMADA_370_XP_INT_CLEAR_MASK_OFFS	(0x4C)
40#define ARMADA_370_XP_INT_FABRIC_MASK_OFFS	(0x54)
41#define ARMADA_370_XP_INT_CAUSE_PERF(cpu)	(1 << cpu)
42
43#define ARMADA_370_XP_INT_CONTROL		(0x00)
44#define ARMADA_370_XP_INT_SET_ENABLE_OFFS	(0x30)
45#define ARMADA_370_XP_INT_CLEAR_ENABLE_OFFS	(0x34)
46#define ARMADA_370_XP_INT_SOURCE_CTL(irq)	(0x100 + irq*4)
47#define ARMADA_370_XP_INT_SOURCE_CPU_MASK	0xF
48#define ARMADA_370_XP_INT_IRQ_FIQ_MASK(cpuid)	((BIT(0) | BIT(8)) << cpuid)
49
50#define ARMADA_370_XP_CPU_INTACK_OFFS		(0x44)
51#define ARMADA_375_PPI_CAUSE			(0x10)
52
53#define ARMADA_370_XP_SW_TRIG_INT_OFFS           (0x4)
54#define ARMADA_370_XP_IN_DRBEL_MSK_OFFS          (0xc)
55#define ARMADA_370_XP_IN_DRBEL_CAUSE_OFFS        (0x8)
56
57#define ARMADA_370_XP_MAX_PER_CPU_IRQS		(28)
58
59#define IPI_DOORBELL_START                      (0)
60#define IPI_DOORBELL_END                        (8)
61#define IPI_DOORBELL_MASK                       0xFF
62#define PCI_MSI_DOORBELL_START                  (16)
63#define PCI_MSI_DOORBELL_NR                     (16)
64#define PCI_MSI_DOORBELL_END                    (32)
65#define PCI_MSI_DOORBELL_MASK                   0xFFFF0000
66
67static void __iomem *per_cpu_int_base;
68static void __iomem *main_int_base;
69static struct irq_domain *armada_370_xp_mpic_domain;
70static u32 doorbell_mask_reg;
71static int parent_irq;
72#ifdef CONFIG_PCI_MSI
73static struct irq_domain *armada_370_xp_msi_domain;
74static DECLARE_BITMAP(msi_used, PCI_MSI_DOORBELL_NR);
75static DEFINE_MUTEX(msi_used_lock);
76static phys_addr_t msi_doorbell_addr;
77#endif
78
79static inline bool is_percpu_irq(irq_hw_number_t irq)
80{
81	if (irq <= ARMADA_370_XP_MAX_PER_CPU_IRQS)
82		return true;
83
84	return false;
85}
86
87/*
88 * In SMP mode:
89 * For shared global interrupts, mask/unmask global enable bit
90 * For CPU interrupts, mask/unmask the calling CPU's bit
91 */
92static void armada_370_xp_irq_mask(struct irq_data *d)
93{
94	irq_hw_number_t hwirq = irqd_to_hwirq(d);
95
96	if (!is_percpu_irq(hwirq))
97		writel(hwirq, main_int_base +
98				ARMADA_370_XP_INT_CLEAR_ENABLE_OFFS);
99	else
100		writel(hwirq, per_cpu_int_base +
101				ARMADA_370_XP_INT_SET_MASK_OFFS);
102}
103
104static void armada_370_xp_irq_unmask(struct irq_data *d)
105{
106	irq_hw_number_t hwirq = irqd_to_hwirq(d);
107
108	if (!is_percpu_irq(hwirq))
109		writel(hwirq, main_int_base +
110				ARMADA_370_XP_INT_SET_ENABLE_OFFS);
111	else
112		writel(hwirq, per_cpu_int_base +
113				ARMADA_370_XP_INT_CLEAR_MASK_OFFS);
114}
115
116#ifdef CONFIG_PCI_MSI
117
118static int armada_370_xp_alloc_msi(void)
119{
120	int hwirq;
121
122	mutex_lock(&msi_used_lock);
123	hwirq = find_first_zero_bit(&msi_used, PCI_MSI_DOORBELL_NR);
124	if (hwirq >= PCI_MSI_DOORBELL_NR)
125		hwirq = -ENOSPC;
126	else
127		set_bit(hwirq, msi_used);
128	mutex_unlock(&msi_used_lock);
129
130	return hwirq;
131}
132
133static void armada_370_xp_free_msi(int hwirq)
134{
135	mutex_lock(&msi_used_lock);
136	if (!test_bit(hwirq, msi_used))
137		pr_err("trying to free unused MSI#%d\n", hwirq);
138	else
139		clear_bit(hwirq, msi_used);
140	mutex_unlock(&msi_used_lock);
141}
142
143static int armada_370_xp_setup_msi_irq(struct msi_controller *chip,
144				       struct pci_dev *pdev,
145				       struct msi_desc *desc)
146{
147	struct msi_msg msg;
148	int virq, hwirq;
149
150	/* We support MSI, but not MSI-X */
151	if (desc->msi_attrib.is_msix)
152		return -EINVAL;
153
154	hwirq = armada_370_xp_alloc_msi();
155	if (hwirq < 0)
156		return hwirq;
157
158	virq = irq_create_mapping(armada_370_xp_msi_domain, hwirq);
159	if (!virq) {
160		armada_370_xp_free_msi(hwirq);
161		return -EINVAL;
162	}
163
164	irq_set_msi_desc(virq, desc);
165
166	msg.address_lo = msi_doorbell_addr;
167	msg.address_hi = 0;
168	msg.data = 0xf00 | (hwirq + 16);
169
170	pci_write_msi_msg(virq, &msg);
171	return 0;
172}
173
174static void armada_370_xp_teardown_msi_irq(struct msi_controller *chip,
175					   unsigned int irq)
176{
177	struct irq_data *d = irq_get_irq_data(irq);
178	unsigned long hwirq = d->hwirq;
179
180	irq_dispose_mapping(irq);
181	armada_370_xp_free_msi(hwirq);
182}
183
184static struct irq_chip armada_370_xp_msi_irq_chip = {
185	.name = "armada_370_xp_msi_irq",
186	.irq_enable = pci_msi_unmask_irq,
187	.irq_disable = pci_msi_mask_irq,
188	.irq_mask = pci_msi_mask_irq,
189	.irq_unmask = pci_msi_unmask_irq,
190};
191
192static int armada_370_xp_msi_map(struct irq_domain *domain, unsigned int virq,
193				 irq_hw_number_t hw)
194{
195	irq_set_chip_and_handler(virq, &armada_370_xp_msi_irq_chip,
196				 handle_simple_irq);
197
198	return 0;
199}
200
201static const struct irq_domain_ops armada_370_xp_msi_irq_ops = {
202	.map = armada_370_xp_msi_map,
203};
204
205static int armada_370_xp_msi_init(struct device_node *node,
206				  phys_addr_t main_int_phys_base)
207{
208	struct msi_controller *msi_chip;
209	u32 reg;
210	int ret;
211
212	msi_doorbell_addr = main_int_phys_base +
213		ARMADA_370_XP_SW_TRIG_INT_OFFS;
214
215	msi_chip = kzalloc(sizeof(*msi_chip), GFP_KERNEL);
216	if (!msi_chip)
217		return -ENOMEM;
218
219	msi_chip->setup_irq = armada_370_xp_setup_msi_irq;
220	msi_chip->teardown_irq = armada_370_xp_teardown_msi_irq;
221	msi_chip->of_node = node;
222
223	armada_370_xp_msi_domain =
224		irq_domain_add_linear(NULL, PCI_MSI_DOORBELL_NR,
225				      &armada_370_xp_msi_irq_ops,
226				      NULL);
227	if (!armada_370_xp_msi_domain) {
228		kfree(msi_chip);
229		return -ENOMEM;
230	}
231
232	ret = of_pci_msi_chip_add(msi_chip);
233	if (ret < 0) {
234		irq_domain_remove(armada_370_xp_msi_domain);
235		kfree(msi_chip);
236		return ret;
237	}
238
239	reg = readl(per_cpu_int_base + ARMADA_370_XP_IN_DRBEL_MSK_OFFS)
240		| PCI_MSI_DOORBELL_MASK;
241
242	writel(reg, per_cpu_int_base +
243	       ARMADA_370_XP_IN_DRBEL_MSK_OFFS);
244
245	/* Unmask IPI interrupt */
246	writel(1, per_cpu_int_base + ARMADA_370_XP_INT_CLEAR_MASK_OFFS);
247
248	return 0;
249}
250#else
251static inline int armada_370_xp_msi_init(struct device_node *node,
252					 phys_addr_t main_int_phys_base)
253{
254	return 0;
255}
256#endif
257
258#ifdef CONFIG_SMP
259static DEFINE_RAW_SPINLOCK(irq_controller_lock);
260
261static int armada_xp_set_affinity(struct irq_data *d,
262				  const struct cpumask *mask_val, bool force)
263{
264	irq_hw_number_t hwirq = irqd_to_hwirq(d);
265	unsigned long reg, mask;
266	int cpu;
267
268	/* Select a single core from the affinity mask which is online */
269	cpu = cpumask_any_and(mask_val, cpu_online_mask);
270	mask = 1UL << cpu_logical_map(cpu);
271
272	raw_spin_lock(&irq_controller_lock);
273	reg = readl(main_int_base + ARMADA_370_XP_INT_SOURCE_CTL(hwirq));
274	reg = (reg & (~ARMADA_370_XP_INT_SOURCE_CPU_MASK)) | mask;
275	writel(reg, main_int_base + ARMADA_370_XP_INT_SOURCE_CTL(hwirq));
276	raw_spin_unlock(&irq_controller_lock);
277
278	return IRQ_SET_MASK_OK;
279}
280#endif
281
282static struct irq_chip armada_370_xp_irq_chip = {
283	.name		= "armada_370_xp_irq",
284	.irq_mask       = armada_370_xp_irq_mask,
285	.irq_mask_ack   = armada_370_xp_irq_mask,
286	.irq_unmask     = armada_370_xp_irq_unmask,
287#ifdef CONFIG_SMP
288	.irq_set_affinity = armada_xp_set_affinity,
289#endif
290	.flags		= IRQCHIP_SKIP_SET_WAKE | IRQCHIP_MASK_ON_SUSPEND,
291};
292
293static int armada_370_xp_mpic_irq_map(struct irq_domain *h,
294				      unsigned int virq, irq_hw_number_t hw)
295{
296	armada_370_xp_irq_mask(irq_get_irq_data(virq));
297	if (!is_percpu_irq(hw))
298		writel(hw, per_cpu_int_base +
299			ARMADA_370_XP_INT_CLEAR_MASK_OFFS);
300	else
301		writel(hw, main_int_base + ARMADA_370_XP_INT_SET_ENABLE_OFFS);
302	irq_set_status_flags(virq, IRQ_LEVEL);
303
304	if (is_percpu_irq(hw)) {
305		irq_set_percpu_devid(virq);
306		irq_set_chip_and_handler(virq, &armada_370_xp_irq_chip,
307					handle_percpu_devid_irq);
308
309	} else {
310		irq_set_chip_and_handler(virq, &armada_370_xp_irq_chip,
311					handle_level_irq);
312	}
313	irq_set_probe(virq);
314	irq_clear_status_flags(virq, IRQ_NOAUTOEN);
315
316	return 0;
317}
318
319static void armada_xp_mpic_smp_cpu_init(void)
320{
321	u32 control;
322	int nr_irqs, i;
323
324	control = readl(main_int_base + ARMADA_370_XP_INT_CONTROL);
325	nr_irqs = (control >> 2) & 0x3ff;
326
327	for (i = 0; i < nr_irqs; i++)
328		writel(i, per_cpu_int_base + ARMADA_370_XP_INT_SET_MASK_OFFS);
329
330	/* Clear pending IPIs */
331	writel(0, per_cpu_int_base + ARMADA_370_XP_IN_DRBEL_CAUSE_OFFS);
332
333	/* Enable first 8 IPIs */
334	writel(IPI_DOORBELL_MASK, per_cpu_int_base +
335		ARMADA_370_XP_IN_DRBEL_MSK_OFFS);
336
337	/* Unmask IPI interrupt */
338	writel(0, per_cpu_int_base + ARMADA_370_XP_INT_CLEAR_MASK_OFFS);
339}
340
341static void armada_xp_mpic_perf_init(void)
342{
343	unsigned long cpuid = cpu_logical_map(smp_processor_id());
344
345	/* Enable Performance Counter Overflow interrupts */
346	writel(ARMADA_370_XP_INT_CAUSE_PERF(cpuid),
347	       per_cpu_int_base + ARMADA_370_XP_INT_FABRIC_MASK_OFFS);
348}
349
350#ifdef CONFIG_SMP
351static void armada_mpic_send_doorbell(const struct cpumask *mask,
352				      unsigned int irq)
353{
354	int cpu;
355	unsigned long map = 0;
356
357	/* Convert our logical CPU mask into a physical one. */
358	for_each_cpu(cpu, mask)
359		map |= 1 << cpu_logical_map(cpu);
360
361	/*
362	 * Ensure that stores to Normal memory are visible to the
363	 * other CPUs before issuing the IPI.
364	 */
365	dsb();
366
367	/* submit softirq */
368	writel((map << 8) | irq, main_int_base +
369		ARMADA_370_XP_SW_TRIG_INT_OFFS);
370}
371
372static int armada_xp_mpic_secondary_init(struct notifier_block *nfb,
373					 unsigned long action, void *hcpu)
374{
375	if (action == CPU_STARTING || action == CPU_STARTING_FROZEN) {
376		armada_xp_mpic_perf_init();
377		armada_xp_mpic_smp_cpu_init();
378	}
379
380	return NOTIFY_OK;
381}
382
383static struct notifier_block armada_370_xp_mpic_cpu_notifier = {
384	.notifier_call = armada_xp_mpic_secondary_init,
385	.priority = 100,
386};
387
388static int mpic_cascaded_secondary_init(struct notifier_block *nfb,
389					unsigned long action, void *hcpu)
390{
391	if (action == CPU_STARTING || action == CPU_STARTING_FROZEN) {
392		armada_xp_mpic_perf_init();
393		enable_percpu_irq(parent_irq, IRQ_TYPE_NONE);
394	}
395
396	return NOTIFY_OK;
397}
398
399static struct notifier_block mpic_cascaded_cpu_notifier = {
400	.notifier_call = mpic_cascaded_secondary_init,
401	.priority = 100,
402};
403#endif /* CONFIG_SMP */
404
405static const struct irq_domain_ops armada_370_xp_mpic_irq_ops = {
406	.map = armada_370_xp_mpic_irq_map,
407	.xlate = irq_domain_xlate_onecell,
408};
409
410#ifdef CONFIG_PCI_MSI
411static void armada_370_xp_handle_msi_irq(struct pt_regs *regs, bool is_chained)
412{
413	u32 msimask, msinr;
414
415	msimask = readl_relaxed(per_cpu_int_base +
416				ARMADA_370_XP_IN_DRBEL_CAUSE_OFFS)
417		& PCI_MSI_DOORBELL_MASK;
418
419	writel(~msimask, per_cpu_int_base +
420	       ARMADA_370_XP_IN_DRBEL_CAUSE_OFFS);
421
422	for (msinr = PCI_MSI_DOORBELL_START;
423	     msinr < PCI_MSI_DOORBELL_END; msinr++) {
424		int irq;
425
426		if (!(msimask & BIT(msinr)))
427			continue;
428
429		if (is_chained) {
430			irq = irq_find_mapping(armada_370_xp_msi_domain,
431					       msinr - 16);
432			generic_handle_irq(irq);
433		} else {
434			irq = msinr - 16;
435			handle_domain_irq(armada_370_xp_msi_domain,
436					  irq, regs);
437		}
438	}
439}
440#else
441static void armada_370_xp_handle_msi_irq(struct pt_regs *r, bool b) {}
442#endif
443
444static void armada_370_xp_mpic_handle_cascade_irq(struct irq_desc *desc)
445{
446	struct irq_chip *chip = irq_desc_get_chip(desc);
447	unsigned long irqmap, irqn, irqsrc, cpuid;
448	unsigned int cascade_irq;
449
450	chained_irq_enter(chip, desc);
451
452	irqmap = readl_relaxed(per_cpu_int_base + ARMADA_375_PPI_CAUSE);
453	cpuid = cpu_logical_map(smp_processor_id());
454
455	for_each_set_bit(irqn, &irqmap, BITS_PER_LONG) {
456		irqsrc = readl_relaxed(main_int_base +
457				       ARMADA_370_XP_INT_SOURCE_CTL(irqn));
458
459		/* Check if the interrupt is not masked on current CPU.
460		 * Test IRQ (0-1) and FIQ (8-9) mask bits.
461		 */
462		if (!(irqsrc & ARMADA_370_XP_INT_IRQ_FIQ_MASK(cpuid)))
463			continue;
464
465		if (irqn == 1) {
466			armada_370_xp_handle_msi_irq(NULL, true);
467			continue;
468		}
469
470		cascade_irq = irq_find_mapping(armada_370_xp_mpic_domain, irqn);
471		generic_handle_irq(cascade_irq);
472	}
473
474	chained_irq_exit(chip, desc);
475}
476
477static void __exception_irq_entry
478armada_370_xp_handle_irq(struct pt_regs *regs)
479{
480	u32 irqstat, irqnr;
481
482	do {
483		irqstat = readl_relaxed(per_cpu_int_base +
484					ARMADA_370_XP_CPU_INTACK_OFFS);
485		irqnr = irqstat & 0x3FF;
486
487		if (irqnr > 1022)
488			break;
489
490		if (irqnr > 1) {
491			handle_domain_irq(armada_370_xp_mpic_domain,
492					  irqnr, regs);
493			continue;
494		}
495
496		/* MSI handling */
497		if (irqnr == 1)
498			armada_370_xp_handle_msi_irq(regs, false);
499
500#ifdef CONFIG_SMP
501		/* IPI Handling */
502		if (irqnr == 0) {
503			u32 ipimask, ipinr;
504
505			ipimask = readl_relaxed(per_cpu_int_base +
506						ARMADA_370_XP_IN_DRBEL_CAUSE_OFFS)
507				& IPI_DOORBELL_MASK;
508
509			writel(~ipimask, per_cpu_int_base +
510				ARMADA_370_XP_IN_DRBEL_CAUSE_OFFS);
511
512			/* Handle all pending doorbells */
513			for (ipinr = IPI_DOORBELL_START;
514			     ipinr < IPI_DOORBELL_END; ipinr++) {
515				if (ipimask & (0x1 << ipinr))
516					handle_IPI(ipinr, regs);
517			}
518			continue;
519		}
520#endif
521
522	} while (1);
523}
524
525static int armada_370_xp_mpic_suspend(void)
526{
527	doorbell_mask_reg = readl(per_cpu_int_base +
528				  ARMADA_370_XP_IN_DRBEL_MSK_OFFS);
529	return 0;
530}
531
532static void armada_370_xp_mpic_resume(void)
533{
534	int nirqs;
535	irq_hw_number_t irq;
536
537	/* Re-enable interrupts */
538	nirqs = (readl(main_int_base + ARMADA_370_XP_INT_CONTROL) >> 2) & 0x3ff;
539	for (irq = 0; irq < nirqs; irq++) {
540		struct irq_data *data;
541		int virq;
542
543		virq = irq_linear_revmap(armada_370_xp_mpic_domain, irq);
544		if (virq == 0)
545			continue;
546
547		if (!is_percpu_irq(irq))
548			writel(irq, per_cpu_int_base +
549			       ARMADA_370_XP_INT_CLEAR_MASK_OFFS);
550		else
551			writel(irq, main_int_base +
552			       ARMADA_370_XP_INT_SET_ENABLE_OFFS);
553
554		data = irq_get_irq_data(virq);
555		if (!irqd_irq_disabled(data))
556			armada_370_xp_irq_unmask(data);
557	}
558
559	/* Reconfigure doorbells for IPIs and MSIs */
560	writel(doorbell_mask_reg,
561	       per_cpu_int_base + ARMADA_370_XP_IN_DRBEL_MSK_OFFS);
562	if (doorbell_mask_reg & IPI_DOORBELL_MASK)
563		writel(0, per_cpu_int_base + ARMADA_370_XP_INT_CLEAR_MASK_OFFS);
564	if (doorbell_mask_reg & PCI_MSI_DOORBELL_MASK)
565		writel(1, per_cpu_int_base + ARMADA_370_XP_INT_CLEAR_MASK_OFFS);
566}
567
568struct syscore_ops armada_370_xp_mpic_syscore_ops = {
569	.suspend	= armada_370_xp_mpic_suspend,
570	.resume		= armada_370_xp_mpic_resume,
571};
572
573static int __init armada_370_xp_mpic_of_init(struct device_node *node,
574					     struct device_node *parent)
575{
576	struct resource main_int_res, per_cpu_int_res;
577	int nr_irqs, i;
578	u32 control;
579
580	BUG_ON(of_address_to_resource(node, 0, &main_int_res));
581	BUG_ON(of_address_to_resource(node, 1, &per_cpu_int_res));
582
583	BUG_ON(!request_mem_region(main_int_res.start,
584				   resource_size(&main_int_res),
585				   node->full_name));
586	BUG_ON(!request_mem_region(per_cpu_int_res.start,
587				   resource_size(&per_cpu_int_res),
588				   node->full_name));
589
590	main_int_base = ioremap(main_int_res.start,
591				resource_size(&main_int_res));
592	BUG_ON(!main_int_base);
593
594	per_cpu_int_base = ioremap(per_cpu_int_res.start,
595				   resource_size(&per_cpu_int_res));
596	BUG_ON(!per_cpu_int_base);
597
598	control = readl(main_int_base + ARMADA_370_XP_INT_CONTROL);
599	nr_irqs = (control >> 2) & 0x3ff;
600
601	for (i = 0; i < nr_irqs; i++)
602		writel(i, main_int_base + ARMADA_370_XP_INT_CLEAR_ENABLE_OFFS);
603
604	armada_370_xp_mpic_domain =
605		irq_domain_add_linear(node, nr_irqs,
606				&armada_370_xp_mpic_irq_ops, NULL);
607
608	BUG_ON(!armada_370_xp_mpic_domain);
609
610	/* Setup for the boot CPU */
611	armada_xp_mpic_perf_init();
612	armada_xp_mpic_smp_cpu_init();
613
614	armada_370_xp_msi_init(node, main_int_res.start);
615
616	parent_irq = irq_of_parse_and_map(node, 0);
617	if (parent_irq <= 0) {
618		irq_set_default_host(armada_370_xp_mpic_domain);
619		set_handle_irq(armada_370_xp_handle_irq);
620#ifdef CONFIG_SMP
621		set_smp_cross_call(armada_mpic_send_doorbell);
622		register_cpu_notifier(&armada_370_xp_mpic_cpu_notifier);
623#endif
624	} else {
625#ifdef CONFIG_SMP
626		register_cpu_notifier(&mpic_cascaded_cpu_notifier);
627#endif
628		irq_set_chained_handler(parent_irq,
629					armada_370_xp_mpic_handle_cascade_irq);
630	}
631
632	register_syscore_ops(&armada_370_xp_mpic_syscore_ops);
633
634	return 0;
635}
636
637IRQCHIP_DECLARE(armada_370_xp_mpic, "marvell,mpic", armada_370_xp_mpic_of_init);
638