361 static void pl08x_write_lli(struct pl08x_driver_data *pl08x,  in pl08x_write_lli()  argument
364 	if (pl08x->vd->pl080s)  in pl08x_write_lli()
365 		dev_vdbg(&pl08x->adev->dev,  in pl08x_write_lli()
372 		dev_vdbg(&pl08x->adev->dev,  in pl08x_write_lli()
383 	if (pl08x->vd->pl080s)  in pl08x_write_lli()
398 	struct pl08x_driver_data *pl08x = plchan->host;  in pl08x_start_next_txd()  local
412 	pl08x_write_lli(pl08x, phychan, &txd->llis_va[0], txd->ccfg);  in pl08x_start_next_txd()
416 	while (readl(pl08x->base + PL080_EN_CHAN) & (1 << phychan->id))  in pl08x_start_next_txd()
473 static void pl08x_terminate_phy_chan(struct pl08x_driver_data *pl08x,  in pl08x_terminate_phy_chan()  argument
483 	writel(1 << ch->id, pl08x->base + PL080_ERR_CLEAR);  in pl08x_terminate_phy_chan()
484 	writel(1 << ch->id, pl08x->base + PL080_TC_CLEAR);  in pl08x_terminate_phy_chan()
530 	struct pl08x_driver_data *pl08x = plchan->host;  in pl08x_getbytes_chan()  local
552 	if (pl08x->vd->pl080s)  in pl08x_getbytes_chan()
565 	llis_max_words = pl08x->lli_words * MAX_NUM_TSFR_LLIS;  in pl08x_getbytes_chan()
577 	for (; llis_va < llis_va_limit; llis_va += pl08x->lli_words) {  in pl08x_getbytes_chan()
578 		if (pl08x->vd->pl080s)  in pl08x_getbytes_chan()
603 pl08x_get_phy_channel(struct pl08x_driver_data *pl08x,  in pl08x_get_phy_channel()  argument
610 	for (i = 0; i < pl08x->vd->channels; i++) {  in pl08x_get_phy_channel()
611 		ch = &pl08x->phy_chans[i];  in pl08x_get_phy_channel()
624 	if (i == pl08x->vd->channels) {  in pl08x_get_phy_channel()
633 static inline void pl08x_put_phy_channel(struct pl08x_driver_data *pl08x,  in pl08x_put_phy_channel()  argument
646 	struct pl08x_driver_data *pl08x = plchan->host;  in pl08x_phy_alloc_and_start()  local
649 	ch = pl08x_get_phy_channel(pl08x, plchan);  in pl08x_phy_alloc_and_start()
651 		dev_dbg(&pl08x->adev->dev, "no physical channel available for xfer on %s\n", plchan->name);  in pl08x_phy_alloc_and_start()
656 	dev_dbg(&pl08x->adev->dev, "allocated physical channel %d for xfer on %s\n",  in pl08x_phy_alloc_and_start()
667 	struct pl08x_driver_data *pl08x = plchan->host;  in pl08x_phy_reassign_start()  local
669 	dev_dbg(&pl08x->adev->dev, "reassigned physical channel %d for xfer on %s\n",  in pl08x_phy_reassign_start()
689 	struct pl08x_driver_data *pl08x = plchan->host;  in pl08x_phy_free()  local
696 	list_for_each_entry(p, &pl08x->memcpy.channels, vc.chan.device_node)  in pl08x_phy_free()
703 		list_for_each_entry(p, &pl08x->slave.channels, vc.chan.device_node)  in pl08x_phy_free()
711 	pl08x_terminate_phy_chan(pl08x, plchan->phychan);  in pl08x_phy_free()
732 		pl08x_put_phy_channel(pl08x, plchan->phychan);  in pl08x_phy_free()
845 static void pl08x_fill_lli_for_desc(struct pl08x_driver_data *pl08x,  in pl08x_fill_lli_for_desc()  argument
849 	u32 offset = num_llis * pl08x->lli_words;  in pl08x_fill_lli_for_desc()
856 	offset += pl08x->lli_words;  in pl08x_fill_lli_for_desc()
863 	if (pl08x->vd->pl080s)  in pl08x_fill_lli_for_desc()
876 static inline void prep_byte_width_lli(struct pl08x_driver_data *pl08x,  in prep_byte_width_lli()  argument
881 	pl08x_fill_lli_for_desc(pl08x, bd, num_llis, len, *cctl, len);  in prep_byte_width_lli()
886 static void pl08x_dump_lli(struct pl08x_driver_data *pl08x,  in pl08x_dump_lli()  argument
891 	if (pl08x->vd->pl080s) {  in pl08x_dump_lli()
892 		dev_vdbg(&pl08x->adev->dev,  in pl08x_dump_lli()
896 			dev_vdbg(&pl08x->adev->dev,  in pl08x_dump_lli()
902 			llis_va += pl08x->lli_words;  in pl08x_dump_lli()
905 		dev_vdbg(&pl08x->adev->dev,  in pl08x_dump_lli()
909 			dev_vdbg(&pl08x->adev->dev,  in pl08x_dump_lli()
914 			llis_va += pl08x->lli_words;  in pl08x_dump_lli()
919 static inline void pl08x_dump_lli(struct pl08x_driver_data *pl08x,  in pl08x_dump_lli()  argument
928 static int pl08x_fill_llis_for_desc(struct pl08x_driver_data *pl08x,  in pl08x_fill_llis_for_desc()  argument
939 	txd->llis_va = dma_pool_alloc(pl08x->pool, GFP_NOWAIT, &txd->llis_bus);  in pl08x_fill_llis_for_desc()
941 		dev_err(&pl08x->adev->dev, "%s no memory for llis\n", __func__);  in pl08x_fill_llis_for_desc()
946 	bd.lli_bus = (pl08x->lli_buses & PL08X_AHB2) ? PL080_LLI_LM_AHB2 : 0;  in pl08x_fill_llis_for_desc()
971 		dev_vdbg(&pl08x->adev->dev,  in pl08x_fill_llis_for_desc()
980 		dev_vdbg(&pl08x->adev->dev, "mbus=%s sbus=%s\n",  in pl08x_fill_llis_for_desc()
1010 				dev_err(&pl08x->adev->dev, "%s sg len can't be zero",  in pl08x_fill_llis_for_desc()
1017 				dev_err(&pl08x->adev->dev,  in pl08x_fill_llis_for_desc()
1026 			pl08x_fill_lli_for_desc(pl08x, &bd, num_llis++,  in pl08x_fill_llis_for_desc()
1046 			dev_vdbg(&pl08x->adev->dev,  in pl08x_fill_llis_for_desc()
1049 			prep_byte_width_lli(pl08x, &bd, &cctl, early_bytes,  in pl08x_fill_llis_for_desc()
1059 				dev_dbg(&pl08x->adev->dev,  in pl08x_fill_llis_for_desc()
1071 						pl08x->vd->max_transfer_size;  in pl08x_fill_llis_for_desc()
1072 			dev_vdbg(&pl08x->adev->dev,  in pl08x_fill_llis_for_desc()
1099 				dev_vdbg(&pl08x->adev->dev,  in pl08x_fill_llis_for_desc()
1106 				pl08x_fill_lli_for_desc(pl08x, &bd, num_llis++,  in pl08x_fill_llis_for_desc()
1115 				dev_vdbg(&pl08x->adev->dev,  in pl08x_fill_llis_for_desc()
1118 				prep_byte_width_lli(pl08x, &bd, &cctl,  in pl08x_fill_llis_for_desc()
1124 			dev_err(&pl08x->adev->dev,  in pl08x_fill_llis_for_desc()
1131 			dev_err(&pl08x->adev->dev,  in pl08x_fill_llis_for_desc()
1139 	last_lli = llis_va + (num_llis - 1) * pl08x->lli_words;  in pl08x_fill_llis_for_desc()
1151 	pl08x_dump_lli(pl08x, llis_va, num_llis);  in pl08x_fill_llis_for_desc()
1156 static void pl08x_free_txd(struct pl08x_driver_data *pl08x,  in pl08x_free_txd()  argument
1162 		dma_pool_free(pl08x->pool, txd->llis_va, txd->llis_bus);  in pl08x_free_txd()
1184 static void pl08x_free_txd_list(struct pl08x_driver_data *pl08x,  in pl08x_free_txd_list()  argument
1427 	struct pl08x_driver_data *pl08x = plchan->host;  in pl08x_prep_dma_memcpy()  local
1434 		dev_err(&pl08x->adev->dev,  in pl08x_prep_dma_memcpy()
1441 		pl08x_free_txd(pl08x, txd);  in pl08x_prep_dma_memcpy()
1442 		dev_err(&pl08x->adev->dev, "%s no memory for pl080 sg\n",  in pl08x_prep_dma_memcpy()
1454 	txd->cctl = pl08x->pd->memcpy_channel.cctl_memcpy &  in pl08x_prep_dma_memcpy()
1460 	if (pl08x->vd->dualmaster)  in pl08x_prep_dma_memcpy()
1461 		txd->cctl |= pl08x_select_bus(pl08x->mem_buses,  in pl08x_prep_dma_memcpy()
1462 					      pl08x->mem_buses);  in pl08x_prep_dma_memcpy()
1466 		pl08x_free_txd(pl08x, txd);  in pl08x_prep_dma_memcpy()
1479 	struct pl08x_driver_data *pl08x = plchan->host;  in pl08x_init_txd()  local
1488 		dev_err(&pl08x->adev->dev, "%s no txd\n", __func__);  in pl08x_init_txd()
1502 		src_buses = pl08x->mem_buses;  in pl08x_init_txd()
1510 		dst_buses = pl08x->mem_buses;  in pl08x_init_txd()
1512 		pl08x_free_txd(pl08x, txd);  in pl08x_init_txd()
1513 		dev_err(&pl08x->adev->dev,  in pl08x_init_txd()
1520 		pl08x_free_txd(pl08x, txd);  in pl08x_init_txd()
1521 		dev_err(&pl08x->adev->dev,  in pl08x_init_txd()
1539 		pl08x_free_txd(pl08x, txd);  in pl08x_init_txd()
1540 		dev_dbg(&pl08x->adev->dev,  in pl08x_init_txd()
1546 	dev_dbg(&pl08x->adev->dev, "allocated DMA request signal %d for xfer on %s\n",  in pl08x_init_txd()
1590 	struct pl08x_driver_data *pl08x = plchan->host;  in pl08x_prep_slave_sg()  local
1596 	dev_dbg(&pl08x->adev->dev, "%s prepare transaction of %d bytes from %s\n",  in pl08x_prep_slave_sg()
1609 			pl08x_free_txd(pl08x, txd);  in pl08x_prep_slave_sg()
1610 			dev_err(&pl08x->adev->dev, "%s no mem for pl080 sg\n",  in pl08x_prep_slave_sg()
1619 		pl08x_free_txd(pl08x, txd);  in pl08x_prep_slave_sg()
1632 	struct pl08x_driver_data *pl08x = plchan->host;  in pl08x_prep_dma_cyclic()  local
1637 	dev_dbg(&pl08x->adev->dev,  in pl08x_prep_dma_cyclic()
1654 			pl08x_free_txd(pl08x, txd);  in pl08x_prep_dma_cyclic()
1662 		pl08x_free_txd(pl08x, txd);  in pl08x_prep_dma_cyclic()
1673 	struct pl08x_driver_data *pl08x = plchan->host;  in pl08x_config()  local
1683 	if (config->device_fc && pl08x->vd->pl080s) {  in pl08x_config()
1684 		dev_err(&pl08x->adev->dev,  in pl08x_config()
1698 	struct pl08x_driver_data *pl08x = plchan->host;  in pl08x_terminate_all()  local
1722 	pl08x_free_txd_list(pl08x, plchan);  in pl08x_terminate_all()
1800 static void pl08x_ensure_on(struct pl08x_driver_data *pl08x)  in pl08x_ensure_on()  argument
1803 	if (pl08x->vd->nomadik)  in pl08x_ensure_on()
1805 	writel(PL080_CONFIG_ENABLE, pl08x->base + PL080_CONFIG);  in pl08x_ensure_on()
1810 	struct pl08x_driver_data *pl08x = dev;  in pl08x_irq()  local
1814 	err = readl(pl08x->base + PL080_ERR_STATUS);  in pl08x_irq()
1816 		dev_err(&pl08x->adev->dev, "%s error interrupt, register value 0x%08x\n",  in pl08x_irq()
1818 		writel(err, pl08x->base + PL080_ERR_CLEAR);  in pl08x_irq()
1820 	tc = readl(pl08x->base + PL080_TC_STATUS);  in pl08x_irq()
1822 		writel(tc, pl08x->base + PL080_TC_CLEAR);  in pl08x_irq()
1827 	for (i = 0; i < pl08x->vd->channels; i++) {  in pl08x_irq()
1830 			struct pl08x_phy_chan *phychan = &pl08x->phy_chans[i];  in pl08x_irq()
1835 				dev_err(&pl08x->adev->dev,  in pl08x_irq()
1885 static int pl08x_dma_init_virtual_channels(struct pl08x_driver_data *pl08x,  in pl08x_dma_init_virtual_channels()  argument
1901 			dev_err(&pl08x->adev->dev,  in pl08x_dma_init_virtual_channels()
1906 		chan->host = pl08x;  in pl08x_dma_init_virtual_channels()
1911 			chan->cd = &pl08x->pd->slave_channels[i];  in pl08x_dma_init_virtual_channels()
1914 			chan->cd = &pl08x->pd->memcpy_channel;  in pl08x_dma_init_virtual_channels()
1921 		dev_dbg(&pl08x->adev->dev,  in pl08x_dma_init_virtual_channels()
1928 	dev_info(&pl08x->adev->dev, "initialized %d virtual %s channels\n",  in pl08x_dma_init_virtual_channels()
1965 	struct pl08x_driver_data *pl08x = s->private;  in pl08x_debugfs_show()  local
1974 	for (i = 0; i < pl08x->vd->channels; i++) {  in pl08x_debugfs_show()
1977 		ch = &pl08x->phy_chans[i];  in pl08x_debugfs_show()
1993 	list_for_each_entry(chan, &pl08x->memcpy.channels, vc.chan.device_node) {  in pl08x_debugfs_show()
2001 	list_for_each_entry(chan, &pl08x->slave.channels, vc.chan.device_node) {  in pl08x_debugfs_show()
2021 static void init_pl08x_debugfs(struct pl08x_driver_data *pl08x)  in init_pl08x_debugfs()  argument
2024 	(void) debugfs_create_file(dev_name(&pl08x->adev->dev),  in init_pl08x_debugfs()
2025 			S_IFREG | S_IRUGO, NULL, pl08x,  in init_pl08x_debugfs()
2030 static inline void init_pl08x_debugfs(struct pl08x_driver_data *pl08x)  in init_pl08x_debugfs()  argument
2036 static struct dma_chan *pl08x_find_chan_id(struct pl08x_driver_data *pl08x,  in pl08x_find_chan_id()  argument
2041 	list_for_each_entry(chan, &pl08x->slave.channels, vc.chan.device_node) {  in pl08x_find_chan_id()
2052 	struct pl08x_driver_data *pl08x = ofdma->of_dma_data;  in pl08x_of_xlate()  local
2057 	if (!pl08x)  in pl08x_of_xlate()
2063 	dma_chan = pl08x_find_chan_id(pl08x, dma_spec->args[0]);  in pl08x_of_xlate()
2067 	chan = devm_kzalloc(pl08x->slave.dev, sizeof(*chan) + sizeof(*data),  in pl08x_of_xlate()
2077 	chan->host = pl08x;  in pl08x_of_xlate()
2084 	vchan_init(&chan->vc, &pl08x->slave);  in pl08x_of_xlate()
2090 			  struct pl08x_driver_data *pl08x,  in pl08x_of_probe()  argument
2198 	pl08x->pd = pd;  in pl08x_of_probe()
2201 					  pl08x);  in pl08x_of_probe()
2205 				 struct pl08x_driver_data *pl08x,  in pl08x_of_probe()  argument
2214 	struct pl08x_driver_data *pl08x;  in pl08x_probe()  local
2231 	pl08x = kzalloc(sizeof(*pl08x), GFP_KERNEL);  in pl08x_probe()
2232 	if (!pl08x) {  in pl08x_probe()
2238 	dma_cap_set(DMA_MEMCPY, pl08x->memcpy.cap_mask);  in pl08x_probe()
2239 	pl08x->memcpy.dev = &adev->dev;  in pl08x_probe()
2240 	pl08x->memcpy.device_free_chan_resources = pl08x_free_chan_resources;  in pl08x_probe()
2241 	pl08x->memcpy.device_prep_dma_memcpy = pl08x_prep_dma_memcpy;  in pl08x_probe()
2242 	pl08x->memcpy.device_prep_dma_interrupt = pl08x_prep_dma_interrupt;  in pl08x_probe()
2243 	pl08x->memcpy.device_tx_status = pl08x_dma_tx_status;  in pl08x_probe()
2244 	pl08x->memcpy.device_issue_pending = pl08x_issue_pending;  in pl08x_probe()
2245 	pl08x->memcpy.device_config = pl08x_config;  in pl08x_probe()
2246 	pl08x->memcpy.device_pause = pl08x_pause;  in pl08x_probe()
2247 	pl08x->memcpy.device_resume = pl08x_resume;  in pl08x_probe()
2248 	pl08x->memcpy.device_terminate_all = pl08x_terminate_all;  in pl08x_probe()
2249 	pl08x->memcpy.src_addr_widths = PL80X_DMA_BUSWIDTHS;  in pl08x_probe()
2250 	pl08x->memcpy.dst_addr_widths = PL80X_DMA_BUSWIDTHS;  in pl08x_probe()
2251 	pl08x->memcpy.directions = BIT(DMA_MEM_TO_MEM);  in pl08x_probe()
2252 	pl08x->memcpy.residue_granularity = DMA_RESIDUE_GRANULARITY_SEGMENT;  in pl08x_probe()
2255 	dma_cap_set(DMA_SLAVE, pl08x->slave.cap_mask);  in pl08x_probe()
2256 	dma_cap_set(DMA_CYCLIC, pl08x->slave.cap_mask);  in pl08x_probe()
2257 	pl08x->slave.dev = &adev->dev;  in pl08x_probe()
2258 	pl08x->slave.device_free_chan_resources = pl08x_free_chan_resources;  in pl08x_probe()
2259 	pl08x->slave.device_prep_dma_interrupt = pl08x_prep_dma_interrupt;  in pl08x_probe()
2260 	pl08x->slave.device_tx_status = pl08x_dma_tx_status;  in pl08x_probe()
2261 	pl08x->slave.device_issue_pending = pl08x_issue_pending;  in pl08x_probe()
2262 	pl08x->slave.device_prep_slave_sg = pl08x_prep_slave_sg;  in pl08x_probe()
2263 	pl08x->slave.device_prep_dma_cyclic = pl08x_prep_dma_cyclic;  in pl08x_probe()
2264 	pl08x->slave.device_config = pl08x_config;  in pl08x_probe()
2265 	pl08x->slave.device_pause = pl08x_pause;  in pl08x_probe()
2266 	pl08x->slave.device_resume = pl08x_resume;  in pl08x_probe()
2267 	pl08x->slave.device_terminate_all = pl08x_terminate_all;  in pl08x_probe()
2268 	pl08x->slave.src_addr_widths = PL80X_DMA_BUSWIDTHS;  in pl08x_probe()
2269 	pl08x->slave.dst_addr_widths = PL80X_DMA_BUSWIDTHS;  in pl08x_probe()
2270 	pl08x->slave.directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV);  in pl08x_probe()
2271 	pl08x->slave.residue_granularity = DMA_RESIDUE_GRANULARITY_SEGMENT;  in pl08x_probe()
2274 	pl08x->pd = dev_get_platdata(&adev->dev);  in pl08x_probe()
2275 	if (!pl08x->pd) {  in pl08x_probe()
2277 			ret = pl08x_of_probe(adev, pl08x, np);  in pl08x_probe()
2288 	pl08x->adev = adev;  in pl08x_probe()
2289 	pl08x->vd = vd;  in pl08x_probe()
2292 	pl08x->lli_buses = PL08X_AHB1;  in pl08x_probe()
2293 	pl08x->mem_buses = PL08X_AHB1;  in pl08x_probe()
2294 	if (pl08x->vd->dualmaster) {  in pl08x_probe()
2295 		pl08x->lli_buses = pl08x->pd->lli_buses;  in pl08x_probe()
2296 		pl08x->mem_buses = pl08x->pd->mem_buses;  in pl08x_probe()
2300 		pl08x->lli_words = PL080S_LLI_WORDS;  in pl08x_probe()
2302 		pl08x->lli_words = PL080_LLI_WORDS;  in pl08x_probe()
2303 	tsfr_size = MAX_NUM_TSFR_LLIS * pl08x->lli_words * sizeof(u32);  in pl08x_probe()
2306 	pl08x->pool = dma_pool_create(DRIVER_NAME, &pl08x->adev->dev,  in pl08x_probe()
2308 	if (!pl08x->pool) {  in pl08x_probe()
2313 	pl08x->base = ioremap(adev->res.start, resource_size(&adev->res));  in pl08x_probe()
2314 	if (!pl08x->base) {  in pl08x_probe()
2320 	pl08x_ensure_on(pl08x);  in pl08x_probe()
2323 	writel(0x000000FF, pl08x->base + PL080_ERR_CLEAR);  in pl08x_probe()
2324 	writel(0x000000FF, pl08x->base + PL080_TC_CLEAR);  in pl08x_probe()
2326 	ret = request_irq(adev->irq[0], pl08x_irq, 0, DRIVER_NAME, pl08x);  in pl08x_probe()
2334 	pl08x->phy_chans = kzalloc((vd->channels * sizeof(*pl08x->phy_chans)),  in pl08x_probe()
2336 	if (!pl08x->phy_chans) {  in pl08x_probe()
2345 		struct pl08x_phy_chan *ch = &pl08x->phy_chans[i];  in pl08x_probe()
2348 		ch->base = pl08x->base + PL080_Cx_BASE(i);  in pl08x_probe()
2372 	ret = pl08x_dma_init_virtual_channels(pl08x, &pl08x->memcpy,  in pl08x_probe()
2373 					      pl08x->vd->channels, false);  in pl08x_probe()
2375 		dev_warn(&pl08x->adev->dev,  in pl08x_probe()
2382 	ret = pl08x_dma_init_virtual_channels(pl08x, &pl08x->slave,  in pl08x_probe()
2383 			pl08x->pd->num_slave_channels, true);  in pl08x_probe()
2385 		dev_warn(&pl08x->adev->dev,  in pl08x_probe()
2391 	ret = dma_async_device_register(&pl08x->memcpy);  in pl08x_probe()
2393 		dev_warn(&pl08x->adev->dev,  in pl08x_probe()
2399 	ret = dma_async_device_register(&pl08x->slave);  in pl08x_probe()
2401 		dev_warn(&pl08x->adev->dev,  in pl08x_probe()
2407 	amba_set_drvdata(adev, pl08x);  in pl08x_probe()
2408 	init_pl08x_debugfs(pl08x);  in pl08x_probe()
2409 	dev_info(&pl08x->adev->dev, "DMA: PL%03x%s rev%u at 0x%08llx irq %d\n",  in pl08x_probe()
2410 		 amba_part(adev), pl08x->vd->pl080s ? "s" : "", amba_rev(adev),  in pl08x_probe()
2416 	dma_async_device_unregister(&pl08x->memcpy);  in pl08x_probe()
2418 	pl08x_free_virtual_channels(&pl08x->slave);  in pl08x_probe()
2420 	pl08x_free_virtual_channels(&pl08x->memcpy);  in pl08x_probe()
2422 	kfree(pl08x->phy_chans);  in pl08x_probe()
2424 	free_irq(adev->irq[0], pl08x);  in pl08x_probe()
2426 	iounmap(pl08x->base);  in pl08x_probe()
2428 	dma_pool_destroy(pl08x->pool);  in pl08x_probe()
2431 	kfree(pl08x);  in pl08x_probe()