This source file includes following definitions.
- to_rcar_msi
- rcar_pci_write_reg
- rcar_pci_read_reg
- rcar_rmw32
- rcar_read_conf
- rcar_pcie_config_access
- rcar_pcie_read_conf
- rcar_pcie_write_conf
- rcar_pcie_setup_window
- rcar_pcie_setup
- rcar_pcie_force_speedup
- rcar_pcie_enable
- phy_wait_for_ack
- phy_write_reg
- rcar_pcie_wait_for_phyrdy
- rcar_pcie_wait_for_dl
- rcar_pcie_hw_init
- rcar_pcie_phy_init_h1
- rcar_pcie_phy_init_gen2
- rcar_pcie_phy_init_gen3
- rcar_msi_alloc
- rcar_msi_alloc_region
- rcar_msi_free
- rcar_pcie_msi_irq
- rcar_msi_setup_irq
- rcar_msi_setup_irqs
- rcar_msi_teardown_irq
- rcar_msi_map
- rcar_pcie_unmap_msi
- rcar_pcie_enable_msi
- rcar_pcie_teardown_msi
- rcar_pcie_get_resources
- rcar_pcie_inbound_ranges
- rcar_pcie_parse_map_dma_ranges
- rcar_pcie_probe
- rcar_pcie_resume_noirq
1
2
3
4
5
6
7
8
9
10
11
12
13
14 #include <linux/bitops.h>
15 #include <linux/clk.h>
16 #include <linux/delay.h>
17 #include <linux/interrupt.h>
18 #include <linux/irq.h>
19 #include <linux/irqdomain.h>
20 #include <linux/kernel.h>
21 #include <linux/init.h>
22 #include <linux/msi.h>
23 #include <linux/of_address.h>
24 #include <linux/of_irq.h>
25 #include <linux/of_pci.h>
26 #include <linux/of_platform.h>
27 #include <linux/pci.h>
28 #include <linux/phy/phy.h>
29 #include <linux/platform_device.h>
30 #include <linux/pm_runtime.h>
31 #include <linux/slab.h>
32
33 #include "../pci.h"
34
35 #define PCIECAR 0x000010
36 #define PCIECCTLR 0x000018
37 #define CONFIG_SEND_ENABLE BIT(31)
38 #define TYPE0 (0 << 8)
39 #define TYPE1 BIT(8)
40 #define PCIECDR 0x000020
41 #define PCIEMSR 0x000028
42 #define PCIEINTXR 0x000400
43 #define PCIEPHYSR 0x0007f0
44 #define PHYRDY BIT(0)
45 #define PCIEMSITXR 0x000840
46
47
48 #define PCIETCTLR 0x02000
49 #define DL_DOWN BIT(3)
50 #define CFINIT BIT(0)
51 #define PCIETSTR 0x02004
52 #define DATA_LINK_ACTIVE BIT(0)
53 #define PCIEERRFR 0x02020
54 #define UNSUPPORTED_REQUEST BIT(4)
55 #define PCIEMSIFR 0x02044
56 #define PCIEMSIALR 0x02048
57 #define MSIFE BIT(0)
58 #define PCIEMSIAUR 0x0204c
59 #define PCIEMSIIER 0x02050
60
61
62 #define PCIEPRAR(x) (0x02080 + ((x) * 0x4))
63
64
65 #define PCIELAR(x) (0x02200 + ((x) * 0x20))
66 #define PCIELAMR(x) (0x02208 + ((x) * 0x20))
67 #define LAM_PREFETCH BIT(3)
68 #define LAM_64BIT BIT(2)
69 #define LAR_ENABLE BIT(1)
70
71
72 #define PCIEPALR(x) (0x03400 + ((x) * 0x20))
73 #define PCIEPAUR(x) (0x03404 + ((x) * 0x20))
74 #define PCIEPAMR(x) (0x03408 + ((x) * 0x20))
75 #define PCIEPTCTLR(x) (0x0340c + ((x) * 0x20))
76 #define PAR_ENABLE BIT(31)
77 #define IO_SPACE BIT(8)
78
79
80 #define PCICONF(x) (0x010000 + ((x) * 0x4))
81 #define PMCAP(x) (0x010040 + ((x) * 0x4))
82 #define EXPCAP(x) (0x010070 + ((x) * 0x4))
83 #define VCCAP(x) (0x010100 + ((x) * 0x4))
84
85
86 #define IDSETR1 0x011004
87 #define TLCTLR 0x011048
88 #define MACSR 0x011054
89 #define SPCHGFIN BIT(4)
90 #define SPCHGFAIL BIT(6)
91 #define SPCHGSUC BIT(7)
92 #define LINK_SPEED (0xf << 16)
93 #define LINK_SPEED_2_5GTS (1 << 16)
94 #define LINK_SPEED_5_0GTS (2 << 16)
95 #define MACCTLR 0x011058
96 #define MACCTLR_NFTS_MASK GENMASK(23, 16)
97 #define SPEED_CHANGE BIT(24)
98 #define SCRAMBLE_DISABLE BIT(27)
99 #define LTSMDIS BIT(31)
100 #define MACCTLR_INIT_VAL (LTSMDIS | MACCTLR_NFTS_MASK)
101 #define PMSR 0x01105c
102 #define MACS2R 0x011078
103 #define MACCGSPSETR 0x011084
104 #define SPCNGRSN BIT(31)
105
106
107 #define H1_PCIEPHYADRR 0x04000c
108 #define WRITE_CMD BIT(16)
109 #define PHY_ACK BIT(24)
110 #define RATE_POS 12
111 #define LANE_POS 8
112 #define ADR_POS 0
113 #define H1_PCIEPHYDOUTR 0x040014
114
115
116 #define GEN2_PCIEPHYADDR 0x780
117 #define GEN2_PCIEPHYDATA 0x784
118 #define GEN2_PCIEPHYCTRL 0x78c
119
120 #define INT_PCI_MSI_NR 32
121
122 #define RCONF(x) (PCICONF(0) + (x))
123 #define RPMCAP(x) (PMCAP(0) + (x))
124 #define REXPCAP(x) (EXPCAP(0) + (x))
125 #define RVCCAP(x) (VCCAP(0) + (x))
126
127 #define PCIE_CONF_BUS(b) (((b) & 0xff) << 24)
128 #define PCIE_CONF_DEV(d) (((d) & 0x1f) << 19)
129 #define PCIE_CONF_FUNC(f) (((f) & 0x7) << 16)
130
131 #define RCAR_PCI_MAX_RESOURCES 4
132 #define MAX_NR_INBOUND_MAPS 6
133
134 struct rcar_msi {
135 DECLARE_BITMAP(used, INT_PCI_MSI_NR);
136 struct irq_domain *domain;
137 struct msi_controller chip;
138 unsigned long pages;
139 struct mutex lock;
140 int irq1;
141 int irq2;
142 };
143
144 static inline struct rcar_msi *to_rcar_msi(struct msi_controller *chip)
145 {
146 return container_of(chip, struct rcar_msi, chip);
147 }
148
149
150 struct rcar_pcie {
151 struct device *dev;
152 struct phy *phy;
153 void __iomem *base;
154 struct list_head resources;
155 int root_bus_nr;
156 struct clk *bus_clk;
157 struct rcar_msi msi;
158 };
159
160 static void rcar_pci_write_reg(struct rcar_pcie *pcie, u32 val,
161 unsigned int reg)
162 {
163 writel(val, pcie->base + reg);
164 }
165
166 static u32 rcar_pci_read_reg(struct rcar_pcie *pcie, unsigned int reg)
167 {
168 return readl(pcie->base + reg);
169 }
170
171 enum {
172 RCAR_PCI_ACCESS_READ,
173 RCAR_PCI_ACCESS_WRITE,
174 };
175
176 static void rcar_rmw32(struct rcar_pcie *pcie, int where, u32 mask, u32 data)
177 {
178 unsigned int shift = BITS_PER_BYTE * (where & 3);
179 u32 val = rcar_pci_read_reg(pcie, where & ~3);
180
181 val &= ~(mask << shift);
182 val |= data << shift;
183 rcar_pci_write_reg(pcie, val, where & ~3);
184 }
185
186 static u32 rcar_read_conf(struct rcar_pcie *pcie, int where)
187 {
188 unsigned int shift = BITS_PER_BYTE * (where & 3);
189 u32 val = rcar_pci_read_reg(pcie, where & ~3);
190
191 return val >> shift;
192 }
193
194
195 static int rcar_pcie_config_access(struct rcar_pcie *pcie,
196 unsigned char access_type, struct pci_bus *bus,
197 unsigned int devfn, int where, u32 *data)
198 {
199 unsigned int dev, func, reg, index;
200
201 dev = PCI_SLOT(devfn);
202 func = PCI_FUNC(devfn);
203 reg = where & ~3;
204 index = reg / 4;
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221 if (pci_is_root_bus(bus)) {
222 if (dev != 0)
223 return PCIBIOS_DEVICE_NOT_FOUND;
224
225 if (access_type == RCAR_PCI_ACCESS_READ) {
226 *data = rcar_pci_read_reg(pcie, PCICONF(index));
227 } else {
228
229 if (pci_is_root_bus(bus) && (reg == PCI_PRIMARY_BUS))
230 pcie->root_bus_nr = *data & 0xff;
231
232 rcar_pci_write_reg(pcie, *data, PCICONF(index));
233 }
234
235 return PCIBIOS_SUCCESSFUL;
236 }
237
238 if (pcie->root_bus_nr < 0)
239 return PCIBIOS_DEVICE_NOT_FOUND;
240
241
242 rcar_pci_write_reg(pcie, rcar_pci_read_reg(pcie, PCIEERRFR), PCIEERRFR);
243
244
245 rcar_pci_write_reg(pcie, PCIE_CONF_BUS(bus->number) |
246 PCIE_CONF_DEV(dev) | PCIE_CONF_FUNC(func) | reg, PCIECAR);
247
248
249 if (bus->parent->number == pcie->root_bus_nr)
250 rcar_pci_write_reg(pcie, CONFIG_SEND_ENABLE | TYPE0, PCIECCTLR);
251 else
252 rcar_pci_write_reg(pcie, CONFIG_SEND_ENABLE | TYPE1, PCIECCTLR);
253
254
255 if (rcar_pci_read_reg(pcie, PCIEERRFR) & UNSUPPORTED_REQUEST)
256 return PCIBIOS_DEVICE_NOT_FOUND;
257
258
259 if (rcar_read_conf(pcie, RCONF(PCI_STATUS)) &
260 (PCI_STATUS_REC_MASTER_ABORT | PCI_STATUS_REC_TARGET_ABORT))
261 return PCIBIOS_DEVICE_NOT_FOUND;
262
263 if (access_type == RCAR_PCI_ACCESS_READ)
264 *data = rcar_pci_read_reg(pcie, PCIECDR);
265 else
266 rcar_pci_write_reg(pcie, *data, PCIECDR);
267
268
269 rcar_pci_write_reg(pcie, 0, PCIECCTLR);
270
271 return PCIBIOS_SUCCESSFUL;
272 }
273
274 static int rcar_pcie_read_conf(struct pci_bus *bus, unsigned int devfn,
275 int where, int size, u32 *val)
276 {
277 struct rcar_pcie *pcie = bus->sysdata;
278 int ret;
279
280 ret = rcar_pcie_config_access(pcie, RCAR_PCI_ACCESS_READ,
281 bus, devfn, where, val);
282 if (ret != PCIBIOS_SUCCESSFUL) {
283 *val = 0xffffffff;
284 return ret;
285 }
286
287 if (size == 1)
288 *val = (*val >> (BITS_PER_BYTE * (where & 3))) & 0xff;
289 else if (size == 2)
290 *val = (*val >> (BITS_PER_BYTE * (where & 2))) & 0xffff;
291
292 dev_dbg(&bus->dev, "pcie-config-read: bus=%3d devfn=0x%04x where=0x%04x size=%d val=0x%08x\n",
293 bus->number, devfn, where, size, *val);
294
295 return ret;
296 }
297
298
299 static int rcar_pcie_write_conf(struct pci_bus *bus, unsigned int devfn,
300 int where, int size, u32 val)
301 {
302 struct rcar_pcie *pcie = bus->sysdata;
303 unsigned int shift;
304 u32 data;
305 int ret;
306
307 ret = rcar_pcie_config_access(pcie, RCAR_PCI_ACCESS_READ,
308 bus, devfn, where, &data);
309 if (ret != PCIBIOS_SUCCESSFUL)
310 return ret;
311
312 dev_dbg(&bus->dev, "pcie-config-write: bus=%3d devfn=0x%04x where=0x%04x size=%d val=0x%08x\n",
313 bus->number, devfn, where, size, val);
314
315 if (size == 1) {
316 shift = BITS_PER_BYTE * (where & 3);
317 data &= ~(0xff << shift);
318 data |= ((val & 0xff) << shift);
319 } else if (size == 2) {
320 shift = BITS_PER_BYTE * (where & 2);
321 data &= ~(0xffff << shift);
322 data |= ((val & 0xffff) << shift);
323 } else
324 data = val;
325
326 ret = rcar_pcie_config_access(pcie, RCAR_PCI_ACCESS_WRITE,
327 bus, devfn, where, &data);
328
329 return ret;
330 }
331
332 static struct pci_ops rcar_pcie_ops = {
333 .read = rcar_pcie_read_conf,
334 .write = rcar_pcie_write_conf,
335 };
336
337 static void rcar_pcie_setup_window(int win, struct rcar_pcie *pcie,
338 struct resource *res)
339 {
340
341 resource_size_t size;
342 resource_size_t res_start;
343 u32 mask;
344
345 rcar_pci_write_reg(pcie, 0x00000000, PCIEPTCTLR(win));
346
347
348
349
350
351 size = resource_size(res);
352 mask = (roundup_pow_of_two(size) / SZ_128) - 1;
353 rcar_pci_write_reg(pcie, mask << 7, PCIEPAMR(win));
354
355 if (res->flags & IORESOURCE_IO)
356 res_start = pci_pio_to_address(res->start);
357 else
358 res_start = res->start;
359
360 rcar_pci_write_reg(pcie, upper_32_bits(res_start), PCIEPAUR(win));
361 rcar_pci_write_reg(pcie, lower_32_bits(res_start) & ~0x7F,
362 PCIEPALR(win));
363
364
365 mask = PAR_ENABLE;
366 if (res->flags & IORESOURCE_IO)
367 mask |= IO_SPACE;
368
369 rcar_pci_write_reg(pcie, mask, PCIEPTCTLR(win));
370 }
371
372 static int rcar_pcie_setup(struct list_head *resource, struct rcar_pcie *pci)
373 {
374 struct resource_entry *win;
375 int i = 0;
376
377
378 resource_list_for_each_entry(win, &pci->resources) {
379 struct resource *res = win->res;
380
381 if (!res->flags)
382 continue;
383
384 switch (resource_type(res)) {
385 case IORESOURCE_IO:
386 case IORESOURCE_MEM:
387 rcar_pcie_setup_window(i, pci, res);
388 i++;
389 break;
390 case IORESOURCE_BUS:
391 pci->root_bus_nr = res->start;
392 break;
393 default:
394 continue;
395 }
396
397 pci_add_resource(resource, res);
398 }
399
400 return 1;
401 }
402
403 static void rcar_pcie_force_speedup(struct rcar_pcie *pcie)
404 {
405 struct device *dev = pcie->dev;
406 unsigned int timeout = 1000;
407 u32 macsr;
408
409 if ((rcar_pci_read_reg(pcie, MACS2R) & LINK_SPEED) != LINK_SPEED_5_0GTS)
410 return;
411
412 if (rcar_pci_read_reg(pcie, MACCTLR) & SPEED_CHANGE) {
413 dev_err(dev, "Speed change already in progress\n");
414 return;
415 }
416
417 macsr = rcar_pci_read_reg(pcie, MACSR);
418 if ((macsr & LINK_SPEED) == LINK_SPEED_5_0GTS)
419 goto done;
420
421
422 rcar_rmw32(pcie, EXPCAP(12), PCI_EXP_LNKSTA_CLS,
423 PCI_EXP_LNKSTA_CLS_5_0GB);
424
425
426 rcar_rmw32(pcie, MACCGSPSETR, SPCNGRSN, 0);
427
428
429 if (macsr & (SPCHGFIN | SPCHGSUC | SPCHGFAIL))
430 rcar_pci_write_reg(pcie, macsr, MACSR);
431
432
433 rcar_rmw32(pcie, MACCTLR, SPEED_CHANGE, SPEED_CHANGE);
434
435 while (timeout--) {
436 macsr = rcar_pci_read_reg(pcie, MACSR);
437 if (macsr & SPCHGFIN) {
438
439 rcar_pci_write_reg(pcie, macsr, MACSR);
440
441 if (macsr & SPCHGFAIL)
442 dev_err(dev, "Speed change failed\n");
443
444 goto done;
445 }
446
447 msleep(1);
448 }
449
450 dev_err(dev, "Speed change timed out\n");
451
452 done:
453 dev_info(dev, "Current link speed is %s GT/s\n",
454 (macsr & LINK_SPEED) == LINK_SPEED_5_0GTS ? "5" : "2.5");
455 }
456
457 static int rcar_pcie_enable(struct rcar_pcie *pcie)
458 {
459 struct device *dev = pcie->dev;
460 struct pci_host_bridge *bridge = pci_host_bridge_from_priv(pcie);
461 struct pci_bus *bus, *child;
462 int ret;
463
464
465 rcar_pcie_force_speedup(pcie);
466
467 rcar_pcie_setup(&bridge->windows, pcie);
468
469 pci_add_flags(PCI_REASSIGN_ALL_BUS);
470
471 bridge->dev.parent = dev;
472 bridge->sysdata = pcie;
473 bridge->busnr = pcie->root_bus_nr;
474 bridge->ops = &rcar_pcie_ops;
475 bridge->map_irq = of_irq_parse_and_map_pci;
476 bridge->swizzle_irq = pci_common_swizzle;
477 if (IS_ENABLED(CONFIG_PCI_MSI))
478 bridge->msi = &pcie->msi.chip;
479
480 ret = pci_scan_root_bus_bridge(bridge);
481 if (ret < 0)
482 return ret;
483
484 bus = bridge->bus;
485
486 pci_bus_size_bridges(bus);
487 pci_bus_assign_resources(bus);
488
489 list_for_each_entry(child, &bus->children, node)
490 pcie_bus_configure_settings(child);
491
492 pci_bus_add_devices(bus);
493
494 return 0;
495 }
496
497 static int phy_wait_for_ack(struct rcar_pcie *pcie)
498 {
499 struct device *dev = pcie->dev;
500 unsigned int timeout = 100;
501
502 while (timeout--) {
503 if (rcar_pci_read_reg(pcie, H1_PCIEPHYADRR) & PHY_ACK)
504 return 0;
505
506 udelay(100);
507 }
508
509 dev_err(dev, "Access to PCIe phy timed out\n");
510
511 return -ETIMEDOUT;
512 }
513
514 static void phy_write_reg(struct rcar_pcie *pcie,
515 unsigned int rate, u32 addr,
516 unsigned int lane, u32 data)
517 {
518 u32 phyaddr;
519
520 phyaddr = WRITE_CMD |
521 ((rate & 1) << RATE_POS) |
522 ((lane & 0xf) << LANE_POS) |
523 ((addr & 0xff) << ADR_POS);
524
525
526 rcar_pci_write_reg(pcie, data, H1_PCIEPHYDOUTR);
527 rcar_pci_write_reg(pcie, phyaddr, H1_PCIEPHYADRR);
528
529
530 phy_wait_for_ack(pcie);
531
532
533 rcar_pci_write_reg(pcie, 0, H1_PCIEPHYDOUTR);
534 rcar_pci_write_reg(pcie, 0, H1_PCIEPHYADRR);
535
536
537 phy_wait_for_ack(pcie);
538 }
539
540 static int rcar_pcie_wait_for_phyrdy(struct rcar_pcie *pcie)
541 {
542 unsigned int timeout = 10;
543
544 while (timeout--) {
545 if (rcar_pci_read_reg(pcie, PCIEPHYSR) & PHYRDY)
546 return 0;
547
548 msleep(5);
549 }
550
551 return -ETIMEDOUT;
552 }
553
554 static int rcar_pcie_wait_for_dl(struct rcar_pcie *pcie)
555 {
556 unsigned int timeout = 10000;
557
558 while (timeout--) {
559 if ((rcar_pci_read_reg(pcie, PCIETSTR) & DATA_LINK_ACTIVE))
560 return 0;
561
562 udelay(5);
563 cpu_relax();
564 }
565
566 return -ETIMEDOUT;
567 }
568
569 static int rcar_pcie_hw_init(struct rcar_pcie *pcie)
570 {
571 int err;
572
573
574 rcar_pci_write_reg(pcie, 0, PCIETCTLR);
575
576
577 rcar_pci_write_reg(pcie, 1, PCIEMSR);
578
579 err = rcar_pcie_wait_for_phyrdy(pcie);
580 if (err)
581 return err;
582
583
584
585
586
587
588 rcar_pci_write_reg(pcie, PCI_CLASS_BRIDGE_PCI << 16, IDSETR1);
589
590
591
592
593
594 rcar_rmw32(pcie, RCONF(PCI_SECONDARY_BUS), 0xff, 1);
595 rcar_rmw32(pcie, RCONF(PCI_SUBORDINATE_BUS), 0xff, 1);
596
597
598 rcar_rmw32(pcie, REXPCAP(0), 0xff, PCI_CAP_ID_EXP);
599 rcar_rmw32(pcie, REXPCAP(PCI_EXP_FLAGS),
600 PCI_EXP_FLAGS_TYPE, PCI_EXP_TYPE_ROOT_PORT << 4);
601 rcar_rmw32(pcie, RCONF(PCI_HEADER_TYPE), 0x7f,
602 PCI_HEADER_TYPE_BRIDGE);
603
604
605 rcar_rmw32(pcie, REXPCAP(PCI_EXP_LNKCAP), PCI_EXP_LNKCAP_DLLLARC,
606 PCI_EXP_LNKCAP_DLLLARC);
607
608
609 rcar_rmw32(pcie, REXPCAP(PCI_EXP_SLTCAP), PCI_EXP_SLTCAP_PSN, 0);
610
611
612 rcar_rmw32(pcie, TLCTLR + 1, 0x3f, 50);
613
614
615 rcar_rmw32(pcie, RVCCAP(0), 0xfff00000, 0);
616
617
618 if (IS_ENABLED(CONFIG_PCI_MSI))
619 rcar_pci_write_reg(pcie, 0x801f0000, PCIEMSITXR);
620
621 rcar_pci_write_reg(pcie, MACCTLR_INIT_VAL, MACCTLR);
622
623
624 rcar_pci_write_reg(pcie, CFINIT, PCIETCTLR);
625
626
627 err = rcar_pcie_wait_for_dl(pcie);
628 if (err)
629 return err;
630
631
632 rcar_rmw32(pcie, PCIEINTXR, 0, 0xF << 8);
633
634 wmb();
635
636 return 0;
637 }
638
639 static int rcar_pcie_phy_init_h1(struct rcar_pcie *pcie)
640 {
641
642 phy_write_reg(pcie, 0, 0x42, 0x1, 0x0EC34191);
643 phy_write_reg(pcie, 1, 0x42, 0x1, 0x0EC34180);
644 phy_write_reg(pcie, 0, 0x43, 0x1, 0x00210188);
645 phy_write_reg(pcie, 1, 0x43, 0x1, 0x00210188);
646 phy_write_reg(pcie, 0, 0x44, 0x1, 0x015C0014);
647 phy_write_reg(pcie, 1, 0x44, 0x1, 0x015C0014);
648 phy_write_reg(pcie, 1, 0x4C, 0x1, 0x786174A0);
649 phy_write_reg(pcie, 1, 0x4D, 0x1, 0x048000BB);
650 phy_write_reg(pcie, 0, 0x51, 0x1, 0x079EC062);
651 phy_write_reg(pcie, 0, 0x52, 0x1, 0x20000000);
652 phy_write_reg(pcie, 1, 0x52, 0x1, 0x20000000);
653 phy_write_reg(pcie, 1, 0x56, 0x1, 0x00003806);
654
655 phy_write_reg(pcie, 0, 0x60, 0x1, 0x004B03A5);
656 phy_write_reg(pcie, 0, 0x64, 0x1, 0x3F0F1F0F);
657 phy_write_reg(pcie, 0, 0x66, 0x1, 0x00008000);
658
659 return 0;
660 }
661
662 static int rcar_pcie_phy_init_gen2(struct rcar_pcie *pcie)
663 {
664
665
666
667
668 rcar_pci_write_reg(pcie, 0x000f0030, GEN2_PCIEPHYADDR);
669 rcar_pci_write_reg(pcie, 0x00381203, GEN2_PCIEPHYDATA);
670 rcar_pci_write_reg(pcie, 0x00000001, GEN2_PCIEPHYCTRL);
671 rcar_pci_write_reg(pcie, 0x00000006, GEN2_PCIEPHYCTRL);
672
673 rcar_pci_write_reg(pcie, 0x000f0054, GEN2_PCIEPHYADDR);
674
675 rcar_pci_write_reg(pcie, 0x13802007, GEN2_PCIEPHYDATA);
676 rcar_pci_write_reg(pcie, 0x00000001, GEN2_PCIEPHYCTRL);
677 rcar_pci_write_reg(pcie, 0x00000006, GEN2_PCIEPHYCTRL);
678
679 return 0;
680 }
681
682 static int rcar_pcie_phy_init_gen3(struct rcar_pcie *pcie)
683 {
684 int err;
685
686 err = phy_init(pcie->phy);
687 if (err)
688 return err;
689
690 err = phy_power_on(pcie->phy);
691 if (err)
692 phy_exit(pcie->phy);
693
694 return err;
695 }
696
697 static int rcar_msi_alloc(struct rcar_msi *chip)
698 {
699 int msi;
700
701 mutex_lock(&chip->lock);
702
703 msi = find_first_zero_bit(chip->used, INT_PCI_MSI_NR);
704 if (msi < INT_PCI_MSI_NR)
705 set_bit(msi, chip->used);
706 else
707 msi = -ENOSPC;
708
709 mutex_unlock(&chip->lock);
710
711 return msi;
712 }
713
714 static int rcar_msi_alloc_region(struct rcar_msi *chip, int no_irqs)
715 {
716 int msi;
717
718 mutex_lock(&chip->lock);
719 msi = bitmap_find_free_region(chip->used, INT_PCI_MSI_NR,
720 order_base_2(no_irqs));
721 mutex_unlock(&chip->lock);
722
723 return msi;
724 }
725
726 static void rcar_msi_free(struct rcar_msi *chip, unsigned long irq)
727 {
728 mutex_lock(&chip->lock);
729 clear_bit(irq, chip->used);
730 mutex_unlock(&chip->lock);
731 }
732
733 static irqreturn_t rcar_pcie_msi_irq(int irq, void *data)
734 {
735 struct rcar_pcie *pcie = data;
736 struct rcar_msi *msi = &pcie->msi;
737 struct device *dev = pcie->dev;
738 unsigned long reg;
739
740 reg = rcar_pci_read_reg(pcie, PCIEMSIFR);
741
742
743 if (!reg)
744 return IRQ_NONE;
745
746 while (reg) {
747 unsigned int index = find_first_bit(®, 32);
748 unsigned int msi_irq;
749
750
751 rcar_pci_write_reg(pcie, 1 << index, PCIEMSIFR);
752
753 msi_irq = irq_find_mapping(msi->domain, index);
754 if (msi_irq) {
755 if (test_bit(index, msi->used))
756 generic_handle_irq(msi_irq);
757 else
758 dev_info(dev, "unhandled MSI\n");
759 } else {
760
761 dev_dbg(dev, "unexpected MSI\n");
762 }
763
764
765 reg = rcar_pci_read_reg(pcie, PCIEMSIFR);
766 }
767
768 return IRQ_HANDLED;
769 }
770
771 static int rcar_msi_setup_irq(struct msi_controller *chip, struct pci_dev *pdev,
772 struct msi_desc *desc)
773 {
774 struct rcar_msi *msi = to_rcar_msi(chip);
775 struct rcar_pcie *pcie = container_of(chip, struct rcar_pcie, msi.chip);
776 struct msi_msg msg;
777 unsigned int irq;
778 int hwirq;
779
780 hwirq = rcar_msi_alloc(msi);
781 if (hwirq < 0)
782 return hwirq;
783
784 irq = irq_find_mapping(msi->domain, hwirq);
785 if (!irq) {
786 rcar_msi_free(msi, hwirq);
787 return -EINVAL;
788 }
789
790 irq_set_msi_desc(irq, desc);
791
792 msg.address_lo = rcar_pci_read_reg(pcie, PCIEMSIALR) & ~MSIFE;
793 msg.address_hi = rcar_pci_read_reg(pcie, PCIEMSIAUR);
794 msg.data = hwirq;
795
796 pci_write_msi_msg(irq, &msg);
797
798 return 0;
799 }
800
801 static int rcar_msi_setup_irqs(struct msi_controller *chip,
802 struct pci_dev *pdev, int nvec, int type)
803 {
804 struct rcar_pcie *pcie = container_of(chip, struct rcar_pcie, msi.chip);
805 struct rcar_msi *msi = to_rcar_msi(chip);
806 struct msi_desc *desc;
807 struct msi_msg msg;
808 unsigned int irq;
809 int hwirq;
810 int i;
811
812
813 if (type == PCI_CAP_ID_MSIX)
814 return -EINVAL;
815
816 WARN_ON(!list_is_singular(&pdev->dev.msi_list));
817 desc = list_entry(pdev->dev.msi_list.next, struct msi_desc, list);
818
819 hwirq = rcar_msi_alloc_region(msi, nvec);
820 if (hwirq < 0)
821 return -ENOSPC;
822
823 irq = irq_find_mapping(msi->domain, hwirq);
824 if (!irq)
825 return -ENOSPC;
826
827 for (i = 0; i < nvec; i++) {
828
829
830
831
832
833
834
835 if (irq_set_msi_desc_off(irq, i, desc)) {
836
837 return -EINVAL;
838 }
839 }
840
841 desc->nvec_used = nvec;
842 desc->msi_attrib.multiple = order_base_2(nvec);
843
844 msg.address_lo = rcar_pci_read_reg(pcie, PCIEMSIALR) & ~MSIFE;
845 msg.address_hi = rcar_pci_read_reg(pcie, PCIEMSIAUR);
846 msg.data = hwirq;
847
848 pci_write_msi_msg(irq, &msg);
849
850 return 0;
851 }
852
853 static void rcar_msi_teardown_irq(struct msi_controller *chip, unsigned int irq)
854 {
855 struct rcar_msi *msi = to_rcar_msi(chip);
856 struct irq_data *d = irq_get_irq_data(irq);
857
858 rcar_msi_free(msi, d->hwirq);
859 }
860
861 static struct irq_chip rcar_msi_irq_chip = {
862 .name = "R-Car PCIe MSI",
863 .irq_enable = pci_msi_unmask_irq,
864 .irq_disable = pci_msi_mask_irq,
865 .irq_mask = pci_msi_mask_irq,
866 .irq_unmask = pci_msi_unmask_irq,
867 };
868
869 static int rcar_msi_map(struct irq_domain *domain, unsigned int irq,
870 irq_hw_number_t hwirq)
871 {
872 irq_set_chip_and_handler(irq, &rcar_msi_irq_chip, handle_simple_irq);
873 irq_set_chip_data(irq, domain->host_data);
874
875 return 0;
876 }
877
878 static const struct irq_domain_ops msi_domain_ops = {
879 .map = rcar_msi_map,
880 };
881
882 static void rcar_pcie_unmap_msi(struct rcar_pcie *pcie)
883 {
884 struct rcar_msi *msi = &pcie->msi;
885 int i, irq;
886
887 for (i = 0; i < INT_PCI_MSI_NR; i++) {
888 irq = irq_find_mapping(msi->domain, i);
889 if (irq > 0)
890 irq_dispose_mapping(irq);
891 }
892
893 irq_domain_remove(msi->domain);
894 }
895
896 static int rcar_pcie_enable_msi(struct rcar_pcie *pcie)
897 {
898 struct device *dev = pcie->dev;
899 struct rcar_msi *msi = &pcie->msi;
900 phys_addr_t base;
901 int err, i;
902
903 mutex_init(&msi->lock);
904
905 msi->chip.dev = dev;
906 msi->chip.setup_irq = rcar_msi_setup_irq;
907 msi->chip.setup_irqs = rcar_msi_setup_irqs;
908 msi->chip.teardown_irq = rcar_msi_teardown_irq;
909
910 msi->domain = irq_domain_add_linear(dev->of_node, INT_PCI_MSI_NR,
911 &msi_domain_ops, &msi->chip);
912 if (!msi->domain) {
913 dev_err(dev, "failed to create IRQ domain\n");
914 return -ENOMEM;
915 }
916
917 for (i = 0; i < INT_PCI_MSI_NR; i++)
918 irq_create_mapping(msi->domain, i);
919
920
921 err = devm_request_irq(dev, msi->irq1, rcar_pcie_msi_irq,
922 IRQF_SHARED | IRQF_NO_THREAD,
923 rcar_msi_irq_chip.name, pcie);
924 if (err < 0) {
925 dev_err(dev, "failed to request IRQ: %d\n", err);
926 goto err;
927 }
928
929 err = devm_request_irq(dev, msi->irq2, rcar_pcie_msi_irq,
930 IRQF_SHARED | IRQF_NO_THREAD,
931 rcar_msi_irq_chip.name, pcie);
932 if (err < 0) {
933 dev_err(dev, "failed to request IRQ: %d\n", err);
934 goto err;
935 }
936
937
938 msi->pages = __get_free_pages(GFP_KERNEL, 0);
939 if (!msi->pages) {
940 err = -ENOMEM;
941 goto err;
942 }
943 base = virt_to_phys((void *)msi->pages);
944
945 rcar_pci_write_reg(pcie, lower_32_bits(base) | MSIFE, PCIEMSIALR);
946 rcar_pci_write_reg(pcie, upper_32_bits(base), PCIEMSIAUR);
947
948
949 rcar_pci_write_reg(pcie, 0xffffffff, PCIEMSIIER);
950
951 return 0;
952
953 err:
954 rcar_pcie_unmap_msi(pcie);
955 return err;
956 }
957
958 static void rcar_pcie_teardown_msi(struct rcar_pcie *pcie)
959 {
960 struct rcar_msi *msi = &pcie->msi;
961
962
963 rcar_pci_write_reg(pcie, 0, PCIEMSIIER);
964
965
966 rcar_pci_write_reg(pcie, 0, PCIEMSIALR);
967
968 free_pages(msi->pages, 0);
969
970 rcar_pcie_unmap_msi(pcie);
971 }
972
973 static int rcar_pcie_get_resources(struct rcar_pcie *pcie)
974 {
975 struct device *dev = pcie->dev;
976 struct resource res;
977 int err, i;
978
979 pcie->phy = devm_phy_optional_get(dev, "pcie");
980 if (IS_ERR(pcie->phy))
981 return PTR_ERR(pcie->phy);
982
983 err = of_address_to_resource(dev->of_node, 0, &res);
984 if (err)
985 return err;
986
987 pcie->base = devm_ioremap_resource(dev, &res);
988 if (IS_ERR(pcie->base))
989 return PTR_ERR(pcie->base);
990
991 pcie->bus_clk = devm_clk_get(dev, "pcie_bus");
992 if (IS_ERR(pcie->bus_clk)) {
993 dev_err(dev, "cannot get pcie bus clock\n");
994 return PTR_ERR(pcie->bus_clk);
995 }
996
997 i = irq_of_parse_and_map(dev->of_node, 0);
998 if (!i) {
999 dev_err(dev, "cannot get platform resources for msi interrupt\n");
1000 err = -ENOENT;
1001 goto err_irq1;
1002 }
1003 pcie->msi.irq1 = i;
1004
1005 i = irq_of_parse_and_map(dev->of_node, 1);
1006 if (!i) {
1007 dev_err(dev, "cannot get platform resources for msi interrupt\n");
1008 err = -ENOENT;
1009 goto err_irq2;
1010 }
1011 pcie->msi.irq2 = i;
1012
1013 return 0;
1014
1015 err_irq2:
1016 irq_dispose_mapping(pcie->msi.irq1);
1017 err_irq1:
1018 return err;
1019 }
1020
1021 static int rcar_pcie_inbound_ranges(struct rcar_pcie *pcie,
1022 struct of_pci_range *range,
1023 int *index)
1024 {
1025 u64 restype = range->flags;
1026 u64 cpu_addr = range->cpu_addr;
1027 u64 cpu_end = range->cpu_addr + range->size;
1028 u64 pci_addr = range->pci_addr;
1029 u32 flags = LAM_64BIT | LAR_ENABLE;
1030 u64 mask;
1031 u64 size;
1032 int idx = *index;
1033
1034 if (restype & IORESOURCE_PREFETCH)
1035 flags |= LAM_PREFETCH;
1036
1037
1038
1039
1040
1041 if (cpu_addr > 0) {
1042 unsigned long nr_zeros = __ffs64(cpu_addr);
1043 u64 alignment = 1ULL << nr_zeros;
1044
1045 size = min(range->size, alignment);
1046 } else {
1047 size = range->size;
1048 }
1049
1050 size = min(size, 1ULL << 32);
1051
1052 mask = roundup_pow_of_two(size) - 1;
1053 mask &= ~0xf;
1054
1055 while (cpu_addr < cpu_end) {
1056
1057
1058
1059
1060 rcar_pci_write_reg(pcie, lower_32_bits(pci_addr),
1061 PCIEPRAR(idx));
1062 rcar_pci_write_reg(pcie, lower_32_bits(cpu_addr), PCIELAR(idx));
1063 rcar_pci_write_reg(pcie, lower_32_bits(mask) | flags,
1064 PCIELAMR(idx));
1065
1066 rcar_pci_write_reg(pcie, upper_32_bits(pci_addr),
1067 PCIEPRAR(idx + 1));
1068 rcar_pci_write_reg(pcie, upper_32_bits(cpu_addr),
1069 PCIELAR(idx + 1));
1070 rcar_pci_write_reg(pcie, 0, PCIELAMR(idx + 1));
1071
1072 pci_addr += size;
1073 cpu_addr += size;
1074 idx += 2;
1075
1076 if (idx > MAX_NR_INBOUND_MAPS) {
1077 dev_err(pcie->dev, "Failed to map inbound regions!\n");
1078 return -EINVAL;
1079 }
1080 }
1081 *index = idx;
1082
1083 return 0;
1084 }
1085
1086 static int rcar_pcie_parse_map_dma_ranges(struct rcar_pcie *pcie,
1087 struct device_node *np)
1088 {
1089 struct of_pci_range range;
1090 struct of_pci_range_parser parser;
1091 int index = 0;
1092 int err;
1093
1094 if (of_pci_dma_range_parser_init(&parser, np))
1095 return -EINVAL;
1096
1097
1098 for_each_of_pci_range(&parser, &range) {
1099 u64 end = range.cpu_addr + range.size - 1;
1100
1101 dev_dbg(pcie->dev, "0x%08x 0x%016llx..0x%016llx -> 0x%016llx\n",
1102 range.flags, range.cpu_addr, end, range.pci_addr);
1103
1104 err = rcar_pcie_inbound_ranges(pcie, &range, &index);
1105 if (err)
1106 return err;
1107 }
1108
1109 return 0;
1110 }
1111
1112 static const struct of_device_id rcar_pcie_of_match[] = {
1113 { .compatible = "renesas,pcie-r8a7779",
1114 .data = rcar_pcie_phy_init_h1 },
1115 { .compatible = "renesas,pcie-r8a7790",
1116 .data = rcar_pcie_phy_init_gen2 },
1117 { .compatible = "renesas,pcie-r8a7791",
1118 .data = rcar_pcie_phy_init_gen2 },
1119 { .compatible = "renesas,pcie-rcar-gen2",
1120 .data = rcar_pcie_phy_init_gen2 },
1121 { .compatible = "renesas,pcie-r8a7795",
1122 .data = rcar_pcie_phy_init_gen3 },
1123 { .compatible = "renesas,pcie-rcar-gen3",
1124 .data = rcar_pcie_phy_init_gen3 },
1125 {},
1126 };
1127
1128 static int rcar_pcie_probe(struct platform_device *pdev)
1129 {
1130 struct device *dev = &pdev->dev;
1131 struct rcar_pcie *pcie;
1132 u32 data;
1133 int err;
1134 int (*phy_init_fn)(struct rcar_pcie *);
1135 struct pci_host_bridge *bridge;
1136
1137 bridge = pci_alloc_host_bridge(sizeof(*pcie));
1138 if (!bridge)
1139 return -ENOMEM;
1140
1141 pcie = pci_host_bridge_priv(bridge);
1142
1143 pcie->dev = dev;
1144 platform_set_drvdata(pdev, pcie);
1145
1146 err = pci_parse_request_of_pci_ranges(dev, &pcie->resources, NULL);
1147 if (err)
1148 goto err_free_bridge;
1149
1150 pm_runtime_enable(pcie->dev);
1151 err = pm_runtime_get_sync(pcie->dev);
1152 if (err < 0) {
1153 dev_err(pcie->dev, "pm_runtime_get_sync failed\n");
1154 goto err_pm_disable;
1155 }
1156
1157 err = rcar_pcie_get_resources(pcie);
1158 if (err < 0) {
1159 dev_err(dev, "failed to request resources: %d\n", err);
1160 goto err_pm_put;
1161 }
1162
1163 err = clk_prepare_enable(pcie->bus_clk);
1164 if (err) {
1165 dev_err(dev, "failed to enable bus clock: %d\n", err);
1166 goto err_unmap_msi_irqs;
1167 }
1168
1169 err = rcar_pcie_parse_map_dma_ranges(pcie, dev->of_node);
1170 if (err)
1171 goto err_clk_disable;
1172
1173 phy_init_fn = of_device_get_match_data(dev);
1174 err = phy_init_fn(pcie);
1175 if (err) {
1176 dev_err(dev, "failed to init PCIe PHY\n");
1177 goto err_clk_disable;
1178 }
1179
1180
1181 if (rcar_pcie_hw_init(pcie)) {
1182 dev_info(dev, "PCIe link down\n");
1183 err = -ENODEV;
1184 goto err_phy_shutdown;
1185 }
1186
1187 data = rcar_pci_read_reg(pcie, MACSR);
1188 dev_info(dev, "PCIe x%d: link up\n", (data >> 20) & 0x3f);
1189
1190 if (IS_ENABLED(CONFIG_PCI_MSI)) {
1191 err = rcar_pcie_enable_msi(pcie);
1192 if (err < 0) {
1193 dev_err(dev,
1194 "failed to enable MSI support: %d\n",
1195 err);
1196 goto err_phy_shutdown;
1197 }
1198 }
1199
1200 err = rcar_pcie_enable(pcie);
1201 if (err)
1202 goto err_msi_teardown;
1203
1204 return 0;
1205
1206 err_msi_teardown:
1207 if (IS_ENABLED(CONFIG_PCI_MSI))
1208 rcar_pcie_teardown_msi(pcie);
1209
1210 err_phy_shutdown:
1211 if (pcie->phy) {
1212 phy_power_off(pcie->phy);
1213 phy_exit(pcie->phy);
1214 }
1215
1216 err_clk_disable:
1217 clk_disable_unprepare(pcie->bus_clk);
1218
1219 err_unmap_msi_irqs:
1220 irq_dispose_mapping(pcie->msi.irq2);
1221 irq_dispose_mapping(pcie->msi.irq1);
1222
1223 err_pm_put:
1224 pm_runtime_put(dev);
1225
1226 err_pm_disable:
1227 pm_runtime_disable(dev);
1228 pci_free_resource_list(&pcie->resources);
1229
1230 err_free_bridge:
1231 pci_free_host_bridge(bridge);
1232
1233 return err;
1234 }
1235
1236 static int rcar_pcie_resume_noirq(struct device *dev)
1237 {
1238 struct rcar_pcie *pcie = dev_get_drvdata(dev);
1239
1240 if (rcar_pci_read_reg(pcie, PMSR) &&
1241 !(rcar_pci_read_reg(pcie, PCIETCTLR) & DL_DOWN))
1242 return 0;
1243
1244
1245 rcar_pci_write_reg(pcie, MACCTLR_INIT_VAL, MACCTLR);
1246 rcar_pci_write_reg(pcie, CFINIT, PCIETCTLR);
1247 return rcar_pcie_wait_for_dl(pcie);
1248 }
1249
1250 static const struct dev_pm_ops rcar_pcie_pm_ops = {
1251 .resume_noirq = rcar_pcie_resume_noirq,
1252 };
1253
1254 static struct platform_driver rcar_pcie_driver = {
1255 .driver = {
1256 .name = "rcar-pcie",
1257 .of_match_table = rcar_pcie_of_match,
1258 .pm = &rcar_pcie_pm_ops,
1259 .suppress_bind_attrs = true,
1260 },
1261 .probe = rcar_pcie_probe,
1262 };
1263 builtin_platform_driver(rcar_pcie_driver);