1/* 2 * Copyright(c) 2004 - 2006 Intel Corporation. All rights reserved. 3 * 4 * This program is free software; you can redistribute it and/or modify it 5 * under the terms of the GNU General Public License as published by the Free 6 * Software Foundation; either version 2 of the License, or (at your option) 7 * any later version. 8 * 9 * This program is distributed in the hope that it will be useful, but WITHOUT 10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 12 * more details. 13 * 14 * The full GNU General Public License is included in this distribution in the 15 * file called COPYING. 16 */ 17 18/* 19 * This code implements the DMA subsystem. It provides a HW-neutral interface 20 * for other kernel code to use asynchronous memory copy capabilities, 21 * if present, and allows different HW DMA drivers to register as providing 22 * this capability. 23 * 24 * Due to the fact we are accelerating what is already a relatively fast 25 * operation, the code goes to great lengths to avoid additional overhead, 26 * such as locking. 27 * 28 * LOCKING: 29 * 30 * The subsystem keeps a global list of dma_device structs it is protected by a 31 * mutex, dma_list_mutex. 32 * 33 * A subsystem can get access to a channel by calling dmaengine_get() followed 34 * by dma_find_channel(), or if it has need for an exclusive channel it can call 35 * dma_request_channel(). Once a channel is allocated a reference is taken 36 * against its corresponding driver to disable removal. 37 * 38 * Each device has a channels list, which runs unlocked but is never modified 39 * once the device is registered, it's just setup by the driver. 40 * 41 * See Documentation/dmaengine.txt for more details 42 */ 43 44#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 45 46#include <linux/dma-mapping.h> 47#include <linux/init.h> 48#include <linux/module.h> 49#include <linux/mm.h> 50#include <linux/device.h> 51#include <linux/dmaengine.h> 52#include <linux/hardirq.h> 53#include <linux/spinlock.h> 54#include <linux/percpu.h> 55#include <linux/rcupdate.h> 56#include <linux/mutex.h> 57#include <linux/jiffies.h> 58#include <linux/rculist.h> 59#include <linux/idr.h> 60#include <linux/slab.h> 61#include <linux/acpi.h> 62#include <linux/acpi_dma.h> 63#include <linux/of_dma.h> 64#include <linux/mempool.h> 65 66static DEFINE_MUTEX(dma_list_mutex); 67static DEFINE_IDR(dma_idr); 68static LIST_HEAD(dma_device_list); 69static long dmaengine_ref_count; 70 71/* --- sysfs implementation --- */ 72 73/** 74 * dev_to_dma_chan - convert a device pointer to the its sysfs container object 75 * @dev - device node 76 * 77 * Must be called under dma_list_mutex 78 */ 79static struct dma_chan *dev_to_dma_chan(struct device *dev) 80{ 81 struct dma_chan_dev *chan_dev; 82 83 chan_dev = container_of(dev, typeof(*chan_dev), device); 84 return chan_dev->chan; 85} 86 87static ssize_t memcpy_count_show(struct device *dev, 88 struct device_attribute *attr, char *buf) 89{ 90 struct dma_chan *chan; 91 unsigned long count = 0; 92 int i; 93 int err; 94 95 mutex_lock(&dma_list_mutex); 96 chan = dev_to_dma_chan(dev); 97 if (chan) { 98 for_each_possible_cpu(i) 99 count += per_cpu_ptr(chan->local, i)->memcpy_count; 100 err = sprintf(buf, "%lu\n", count); 101 } else 102 err = -ENODEV; 103 mutex_unlock(&dma_list_mutex); 104 105 return err; 106} 107static DEVICE_ATTR_RO(memcpy_count); 108 109static ssize_t bytes_transferred_show(struct device *dev, 110 struct device_attribute *attr, char *buf) 111{ 112 struct dma_chan *chan; 113 unsigned long count = 0; 114 int i; 115 int err; 116 117 mutex_lock(&dma_list_mutex); 118 chan = dev_to_dma_chan(dev); 119 if (chan) { 120 for_each_possible_cpu(i) 121 count += per_cpu_ptr(chan->local, i)->bytes_transferred; 122 err = sprintf(buf, "%lu\n", count); 123 } else 124 err = -ENODEV; 125 mutex_unlock(&dma_list_mutex); 126 127 return err; 128} 129static DEVICE_ATTR_RO(bytes_transferred); 130 131static ssize_t in_use_show(struct device *dev, struct device_attribute *attr, 132 char *buf) 133{ 134 struct dma_chan *chan; 135 int err; 136 137 mutex_lock(&dma_list_mutex); 138 chan = dev_to_dma_chan(dev); 139 if (chan) 140 err = sprintf(buf, "%d\n", chan->client_count); 141 else 142 err = -ENODEV; 143 mutex_unlock(&dma_list_mutex); 144 145 return err; 146} 147static DEVICE_ATTR_RO(in_use); 148 149static struct attribute *dma_dev_attrs[] = { 150 &dev_attr_memcpy_count.attr, 151 &dev_attr_bytes_transferred.attr, 152 &dev_attr_in_use.attr, 153 NULL, 154}; 155ATTRIBUTE_GROUPS(dma_dev); 156 157static void chan_dev_release(struct device *dev) 158{ 159 struct dma_chan_dev *chan_dev; 160 161 chan_dev = container_of(dev, typeof(*chan_dev), device); 162 if (atomic_dec_and_test(chan_dev->idr_ref)) { 163 mutex_lock(&dma_list_mutex); 164 idr_remove(&dma_idr, chan_dev->dev_id); 165 mutex_unlock(&dma_list_mutex); 166 kfree(chan_dev->idr_ref); 167 } 168 kfree(chan_dev); 169} 170 171static struct class dma_devclass = { 172 .name = "dma", 173 .dev_groups = dma_dev_groups, 174 .dev_release = chan_dev_release, 175}; 176 177/* --- client and device registration --- */ 178 179#define dma_device_satisfies_mask(device, mask) \ 180 __dma_device_satisfies_mask((device), &(mask)) 181static int 182__dma_device_satisfies_mask(struct dma_device *device, 183 const dma_cap_mask_t *want) 184{ 185 dma_cap_mask_t has; 186 187 bitmap_and(has.bits, want->bits, device->cap_mask.bits, 188 DMA_TX_TYPE_END); 189 return bitmap_equal(want->bits, has.bits, DMA_TX_TYPE_END); 190} 191 192static struct module *dma_chan_to_owner(struct dma_chan *chan) 193{ 194 return chan->device->dev->driver->owner; 195} 196 197/** 198 * balance_ref_count - catch up the channel reference count 199 * @chan - channel to balance ->client_count versus dmaengine_ref_count 200 * 201 * balance_ref_count must be called under dma_list_mutex 202 */ 203static void balance_ref_count(struct dma_chan *chan) 204{ 205 struct module *owner = dma_chan_to_owner(chan); 206 207 while (chan->client_count < dmaengine_ref_count) { 208 __module_get(owner); 209 chan->client_count++; 210 } 211} 212 213/** 214 * dma_chan_get - try to grab a dma channel's parent driver module 215 * @chan - channel to grab 216 * 217 * Must be called under dma_list_mutex 218 */ 219static int dma_chan_get(struct dma_chan *chan) 220{ 221 struct module *owner = dma_chan_to_owner(chan); 222 int ret; 223 224 /* The channel is already in use, update client count */ 225 if (chan->client_count) { 226 __module_get(owner); 227 goto out; 228 } 229 230 if (!try_module_get(owner)) 231 return -ENODEV; 232 233 /* allocate upon first client reference */ 234 if (chan->device->device_alloc_chan_resources) { 235 ret = chan->device->device_alloc_chan_resources(chan); 236 if (ret < 0) 237 goto err_out; 238 } 239 240 if (!dma_has_cap(DMA_PRIVATE, chan->device->cap_mask)) 241 balance_ref_count(chan); 242 243out: 244 chan->client_count++; 245 return 0; 246 247err_out: 248 module_put(owner); 249 return ret; 250} 251 252/** 253 * dma_chan_put - drop a reference to a dma channel's parent driver module 254 * @chan - channel to release 255 * 256 * Must be called under dma_list_mutex 257 */ 258static void dma_chan_put(struct dma_chan *chan) 259{ 260 /* This channel is not in use, bail out */ 261 if (!chan->client_count) 262 return; 263 264 chan->client_count--; 265 module_put(dma_chan_to_owner(chan)); 266 267 /* This channel is not in use anymore, free it */ 268 if (!chan->client_count && chan->device->device_free_chan_resources) 269 chan->device->device_free_chan_resources(chan); 270 271 /* If the channel is used via a DMA request router, free the mapping */ 272 if (chan->router && chan->router->route_free) { 273 chan->router->route_free(chan->router->dev, chan->route_data); 274 chan->router = NULL; 275 chan->route_data = NULL; 276 } 277} 278 279enum dma_status dma_sync_wait(struct dma_chan *chan, dma_cookie_t cookie) 280{ 281 enum dma_status status; 282 unsigned long dma_sync_wait_timeout = jiffies + msecs_to_jiffies(5000); 283 284 dma_async_issue_pending(chan); 285 do { 286 status = dma_async_is_tx_complete(chan, cookie, NULL, NULL); 287 if (time_after_eq(jiffies, dma_sync_wait_timeout)) { 288 pr_err("%s: timeout!\n", __func__); 289 return DMA_ERROR; 290 } 291 if (status != DMA_IN_PROGRESS) 292 break; 293 cpu_relax(); 294 } while (1); 295 296 return status; 297} 298EXPORT_SYMBOL(dma_sync_wait); 299 300/** 301 * dma_cap_mask_all - enable iteration over all operation types 302 */ 303static dma_cap_mask_t dma_cap_mask_all; 304 305/** 306 * dma_chan_tbl_ent - tracks channel allocations per core/operation 307 * @chan - associated channel for this entry 308 */ 309struct dma_chan_tbl_ent { 310 struct dma_chan *chan; 311}; 312 313/** 314 * channel_table - percpu lookup table for memory-to-memory offload providers 315 */ 316static struct dma_chan_tbl_ent __percpu *channel_table[DMA_TX_TYPE_END]; 317 318static int __init dma_channel_table_init(void) 319{ 320 enum dma_transaction_type cap; 321 int err = 0; 322 323 bitmap_fill(dma_cap_mask_all.bits, DMA_TX_TYPE_END); 324 325 /* 'interrupt', 'private', and 'slave' are channel capabilities, 326 * but are not associated with an operation so they do not need 327 * an entry in the channel_table 328 */ 329 clear_bit(DMA_INTERRUPT, dma_cap_mask_all.bits); 330 clear_bit(DMA_PRIVATE, dma_cap_mask_all.bits); 331 clear_bit(DMA_SLAVE, dma_cap_mask_all.bits); 332 333 for_each_dma_cap_mask(cap, dma_cap_mask_all) { 334 channel_table[cap] = alloc_percpu(struct dma_chan_tbl_ent); 335 if (!channel_table[cap]) { 336 err = -ENOMEM; 337 break; 338 } 339 } 340 341 if (err) { 342 pr_err("initialization failure\n"); 343 for_each_dma_cap_mask(cap, dma_cap_mask_all) 344 free_percpu(channel_table[cap]); 345 } 346 347 return err; 348} 349arch_initcall(dma_channel_table_init); 350 351/** 352 * dma_find_channel - find a channel to carry out the operation 353 * @tx_type: transaction type 354 */ 355struct dma_chan *dma_find_channel(enum dma_transaction_type tx_type) 356{ 357 return this_cpu_read(channel_table[tx_type]->chan); 358} 359EXPORT_SYMBOL(dma_find_channel); 360 361/** 362 * dma_issue_pending_all - flush all pending operations across all channels 363 */ 364void dma_issue_pending_all(void) 365{ 366 struct dma_device *device; 367 struct dma_chan *chan; 368 369 rcu_read_lock(); 370 list_for_each_entry_rcu(device, &dma_device_list, global_node) { 371 if (dma_has_cap(DMA_PRIVATE, device->cap_mask)) 372 continue; 373 list_for_each_entry(chan, &device->channels, device_node) 374 if (chan->client_count) 375 device->device_issue_pending(chan); 376 } 377 rcu_read_unlock(); 378} 379EXPORT_SYMBOL(dma_issue_pending_all); 380 381/** 382 * dma_chan_is_local - returns true if the channel is in the same numa-node as the cpu 383 */ 384static bool dma_chan_is_local(struct dma_chan *chan, int cpu) 385{ 386 int node = dev_to_node(chan->device->dev); 387 return node == -1 || cpumask_test_cpu(cpu, cpumask_of_node(node)); 388} 389 390/** 391 * min_chan - returns the channel with min count and in the same numa-node as the cpu 392 * @cap: capability to match 393 * @cpu: cpu index which the channel should be close to 394 * 395 * If some channels are close to the given cpu, the one with the lowest 396 * reference count is returned. Otherwise, cpu is ignored and only the 397 * reference count is taken into account. 398 * Must be called under dma_list_mutex. 399 */ 400static struct dma_chan *min_chan(enum dma_transaction_type cap, int cpu) 401{ 402 struct dma_device *device; 403 struct dma_chan *chan; 404 struct dma_chan *min = NULL; 405 struct dma_chan *localmin = NULL; 406 407 list_for_each_entry(device, &dma_device_list, global_node) { 408 if (!dma_has_cap(cap, device->cap_mask) || 409 dma_has_cap(DMA_PRIVATE, device->cap_mask)) 410 continue; 411 list_for_each_entry(chan, &device->channels, device_node) { 412 if (!chan->client_count) 413 continue; 414 if (!min || chan->table_count < min->table_count) 415 min = chan; 416 417 if (dma_chan_is_local(chan, cpu)) 418 if (!localmin || 419 chan->table_count < localmin->table_count) 420 localmin = chan; 421 } 422 } 423 424 chan = localmin ? localmin : min; 425 426 if (chan) 427 chan->table_count++; 428 429 return chan; 430} 431 432/** 433 * dma_channel_rebalance - redistribute the available channels 434 * 435 * Optimize for cpu isolation (each cpu gets a dedicated channel for an 436 * operation type) in the SMP case, and operation isolation (avoid 437 * multi-tasking channels) in the non-SMP case. Must be called under 438 * dma_list_mutex. 439 */ 440static void dma_channel_rebalance(void) 441{ 442 struct dma_chan *chan; 443 struct dma_device *device; 444 int cpu; 445 int cap; 446 447 /* undo the last distribution */ 448 for_each_dma_cap_mask(cap, dma_cap_mask_all) 449 for_each_possible_cpu(cpu) 450 per_cpu_ptr(channel_table[cap], cpu)->chan = NULL; 451 452 list_for_each_entry(device, &dma_device_list, global_node) { 453 if (dma_has_cap(DMA_PRIVATE, device->cap_mask)) 454 continue; 455 list_for_each_entry(chan, &device->channels, device_node) 456 chan->table_count = 0; 457 } 458 459 /* don't populate the channel_table if no clients are available */ 460 if (!dmaengine_ref_count) 461 return; 462 463 /* redistribute available channels */ 464 for_each_dma_cap_mask(cap, dma_cap_mask_all) 465 for_each_online_cpu(cpu) { 466 chan = min_chan(cap, cpu); 467 per_cpu_ptr(channel_table[cap], cpu)->chan = chan; 468 } 469} 470 471int dma_get_slave_caps(struct dma_chan *chan, struct dma_slave_caps *caps) 472{ 473 struct dma_device *device; 474 475 if (!chan || !caps) 476 return -EINVAL; 477 478 device = chan->device; 479 480 /* check if the channel supports slave transactions */ 481 if (!test_bit(DMA_SLAVE, device->cap_mask.bits)) 482 return -ENXIO; 483 484 /* 485 * Check whether it reports it uses the generic slave 486 * capabilities, if not, that means it doesn't support any 487 * kind of slave capabilities reporting. 488 */ 489 if (!device->directions) 490 return -ENXIO; 491 492 caps->src_addr_widths = device->src_addr_widths; 493 caps->dst_addr_widths = device->dst_addr_widths; 494 caps->directions = device->directions; 495 caps->residue_granularity = device->residue_granularity; 496 497 /* 498 * Some devices implement only pause (e.g. to get residuum) but no 499 * resume. However cmd_pause is advertised as pause AND resume. 500 */ 501 caps->cmd_pause = !!(device->device_pause && device->device_resume); 502 caps->cmd_terminate = !!device->device_terminate_all; 503 504 return 0; 505} 506EXPORT_SYMBOL_GPL(dma_get_slave_caps); 507 508static struct dma_chan *private_candidate(const dma_cap_mask_t *mask, 509 struct dma_device *dev, 510 dma_filter_fn fn, void *fn_param) 511{ 512 struct dma_chan *chan; 513 514 if (!__dma_device_satisfies_mask(dev, mask)) { 515 pr_debug("%s: wrong capabilities\n", __func__); 516 return NULL; 517 } 518 /* devices with multiple channels need special handling as we need to 519 * ensure that all channels are either private or public. 520 */ 521 if (dev->chancnt > 1 && !dma_has_cap(DMA_PRIVATE, dev->cap_mask)) 522 list_for_each_entry(chan, &dev->channels, device_node) { 523 /* some channels are already publicly allocated */ 524 if (chan->client_count) 525 return NULL; 526 } 527 528 list_for_each_entry(chan, &dev->channels, device_node) { 529 if (chan->client_count) { 530 pr_debug("%s: %s busy\n", 531 __func__, dma_chan_name(chan)); 532 continue; 533 } 534 if (fn && !fn(chan, fn_param)) { 535 pr_debug("%s: %s filter said false\n", 536 __func__, dma_chan_name(chan)); 537 continue; 538 } 539 return chan; 540 } 541 542 return NULL; 543} 544 545/** 546 * dma_get_slave_channel - try to get specific channel exclusively 547 * @chan: target channel 548 */ 549struct dma_chan *dma_get_slave_channel(struct dma_chan *chan) 550{ 551 int err = -EBUSY; 552 553 /* lock against __dma_request_channel */ 554 mutex_lock(&dma_list_mutex); 555 556 if (chan->client_count == 0) { 557 struct dma_device *device = chan->device; 558 559 dma_cap_set(DMA_PRIVATE, device->cap_mask); 560 device->privatecnt++; 561 err = dma_chan_get(chan); 562 if (err) { 563 pr_debug("%s: failed to get %s: (%d)\n", 564 __func__, dma_chan_name(chan), err); 565 chan = NULL; 566 if (--device->privatecnt == 0) 567 dma_cap_clear(DMA_PRIVATE, device->cap_mask); 568 } 569 } else 570 chan = NULL; 571 572 mutex_unlock(&dma_list_mutex); 573 574 575 return chan; 576} 577EXPORT_SYMBOL_GPL(dma_get_slave_channel); 578 579struct dma_chan *dma_get_any_slave_channel(struct dma_device *device) 580{ 581 dma_cap_mask_t mask; 582 struct dma_chan *chan; 583 int err; 584 585 dma_cap_zero(mask); 586 dma_cap_set(DMA_SLAVE, mask); 587 588 /* lock against __dma_request_channel */ 589 mutex_lock(&dma_list_mutex); 590 591 chan = private_candidate(&mask, device, NULL, NULL); 592 if (chan) { 593 dma_cap_set(DMA_PRIVATE, device->cap_mask); 594 device->privatecnt++; 595 err = dma_chan_get(chan); 596 if (err) { 597 pr_debug("%s: failed to get %s: (%d)\n", 598 __func__, dma_chan_name(chan), err); 599 chan = NULL; 600 if (--device->privatecnt == 0) 601 dma_cap_clear(DMA_PRIVATE, device->cap_mask); 602 } 603 } 604 605 mutex_unlock(&dma_list_mutex); 606 607 return chan; 608} 609EXPORT_SYMBOL_GPL(dma_get_any_slave_channel); 610 611/** 612 * __dma_request_channel - try to allocate an exclusive channel 613 * @mask: capabilities that the channel must satisfy 614 * @fn: optional callback to disposition available channels 615 * @fn_param: opaque parameter to pass to dma_filter_fn 616 * 617 * Returns pointer to appropriate DMA channel on success or NULL. 618 */ 619struct dma_chan *__dma_request_channel(const dma_cap_mask_t *mask, 620 dma_filter_fn fn, void *fn_param) 621{ 622 struct dma_device *device, *_d; 623 struct dma_chan *chan = NULL; 624 int err; 625 626 /* Find a channel */ 627 mutex_lock(&dma_list_mutex); 628 list_for_each_entry_safe(device, _d, &dma_device_list, global_node) { 629 chan = private_candidate(mask, device, fn, fn_param); 630 if (chan) { 631 /* Found a suitable channel, try to grab, prep, and 632 * return it. We first set DMA_PRIVATE to disable 633 * balance_ref_count as this channel will not be 634 * published in the general-purpose allocator 635 */ 636 dma_cap_set(DMA_PRIVATE, device->cap_mask); 637 device->privatecnt++; 638 err = dma_chan_get(chan); 639 640 if (err == -ENODEV) { 641 pr_debug("%s: %s module removed\n", 642 __func__, dma_chan_name(chan)); 643 list_del_rcu(&device->global_node); 644 } else if (err) 645 pr_debug("%s: failed to get %s: (%d)\n", 646 __func__, dma_chan_name(chan), err); 647 else 648 break; 649 if (--device->privatecnt == 0) 650 dma_cap_clear(DMA_PRIVATE, device->cap_mask); 651 chan = NULL; 652 } 653 } 654 mutex_unlock(&dma_list_mutex); 655 656 pr_debug("%s: %s (%s)\n", 657 __func__, 658 chan ? "success" : "fail", 659 chan ? dma_chan_name(chan) : NULL); 660 661 return chan; 662} 663EXPORT_SYMBOL_GPL(__dma_request_channel); 664 665/** 666 * dma_request_slave_channel_reason - try to allocate an exclusive slave channel 667 * @dev: pointer to client device structure 668 * @name: slave channel name 669 * 670 * Returns pointer to appropriate DMA channel on success or an error pointer. 671 */ 672struct dma_chan *dma_request_slave_channel_reason(struct device *dev, 673 const char *name) 674{ 675 /* If device-tree is present get slave info from here */ 676 if (dev->of_node) 677 return of_dma_request_slave_channel(dev->of_node, name); 678 679 /* If device was enumerated by ACPI get slave info from here */ 680 if (ACPI_HANDLE(dev)) 681 return acpi_dma_request_slave_chan_by_name(dev, name); 682 683 return ERR_PTR(-ENODEV); 684} 685EXPORT_SYMBOL_GPL(dma_request_slave_channel_reason); 686 687/** 688 * dma_request_slave_channel - try to allocate an exclusive slave channel 689 * @dev: pointer to client device structure 690 * @name: slave channel name 691 * 692 * Returns pointer to appropriate DMA channel on success or NULL. 693 */ 694struct dma_chan *dma_request_slave_channel(struct device *dev, 695 const char *name) 696{ 697 struct dma_chan *ch = dma_request_slave_channel_reason(dev, name); 698 if (IS_ERR(ch)) 699 return NULL; 700 701 dma_cap_set(DMA_PRIVATE, ch->device->cap_mask); 702 ch->device->privatecnt++; 703 704 return ch; 705} 706EXPORT_SYMBOL_GPL(dma_request_slave_channel); 707 708void dma_release_channel(struct dma_chan *chan) 709{ 710 mutex_lock(&dma_list_mutex); 711 WARN_ONCE(chan->client_count != 1, 712 "chan reference count %d != 1\n", chan->client_count); 713 dma_chan_put(chan); 714 /* drop PRIVATE cap enabled by __dma_request_channel() */ 715 if (--chan->device->privatecnt == 0) 716 dma_cap_clear(DMA_PRIVATE, chan->device->cap_mask); 717 mutex_unlock(&dma_list_mutex); 718} 719EXPORT_SYMBOL_GPL(dma_release_channel); 720 721/** 722 * dmaengine_get - register interest in dma_channels 723 */ 724void dmaengine_get(void) 725{ 726 struct dma_device *device, *_d; 727 struct dma_chan *chan; 728 int err; 729 730 mutex_lock(&dma_list_mutex); 731 dmaengine_ref_count++; 732 733 /* try to grab channels */ 734 list_for_each_entry_safe(device, _d, &dma_device_list, global_node) { 735 if (dma_has_cap(DMA_PRIVATE, device->cap_mask)) 736 continue; 737 list_for_each_entry(chan, &device->channels, device_node) { 738 err = dma_chan_get(chan); 739 if (err == -ENODEV) { 740 /* module removed before we could use it */ 741 list_del_rcu(&device->global_node); 742 break; 743 } else if (err) 744 pr_debug("%s: failed to get %s: (%d)\n", 745 __func__, dma_chan_name(chan), err); 746 } 747 } 748 749 /* if this is the first reference and there were channels 750 * waiting we need to rebalance to get those channels 751 * incorporated into the channel table 752 */ 753 if (dmaengine_ref_count == 1) 754 dma_channel_rebalance(); 755 mutex_unlock(&dma_list_mutex); 756} 757EXPORT_SYMBOL(dmaengine_get); 758 759/** 760 * dmaengine_put - let dma drivers be removed when ref_count == 0 761 */ 762void dmaengine_put(void) 763{ 764 struct dma_device *device; 765 struct dma_chan *chan; 766 767 mutex_lock(&dma_list_mutex); 768 dmaengine_ref_count--; 769 BUG_ON(dmaengine_ref_count < 0); 770 /* drop channel references */ 771 list_for_each_entry(device, &dma_device_list, global_node) { 772 if (dma_has_cap(DMA_PRIVATE, device->cap_mask)) 773 continue; 774 list_for_each_entry(chan, &device->channels, device_node) 775 dma_chan_put(chan); 776 } 777 mutex_unlock(&dma_list_mutex); 778} 779EXPORT_SYMBOL(dmaengine_put); 780 781static bool device_has_all_tx_types(struct dma_device *device) 782{ 783 /* A device that satisfies this test has channels that will never cause 784 * an async_tx channel switch event as all possible operation types can 785 * be handled. 786 */ 787 #ifdef CONFIG_ASYNC_TX_DMA 788 if (!dma_has_cap(DMA_INTERRUPT, device->cap_mask)) 789 return false; 790 #endif 791 792 #if defined(CONFIG_ASYNC_MEMCPY) || defined(CONFIG_ASYNC_MEMCPY_MODULE) 793 if (!dma_has_cap(DMA_MEMCPY, device->cap_mask)) 794 return false; 795 #endif 796 797 #if defined(CONFIG_ASYNC_XOR) || defined(CONFIG_ASYNC_XOR_MODULE) 798 if (!dma_has_cap(DMA_XOR, device->cap_mask)) 799 return false; 800 801 #ifndef CONFIG_ASYNC_TX_DISABLE_XOR_VAL_DMA 802 if (!dma_has_cap(DMA_XOR_VAL, device->cap_mask)) 803 return false; 804 #endif 805 #endif 806 807 #if defined(CONFIG_ASYNC_PQ) || defined(CONFIG_ASYNC_PQ_MODULE) 808 if (!dma_has_cap(DMA_PQ, device->cap_mask)) 809 return false; 810 811 #ifndef CONFIG_ASYNC_TX_DISABLE_PQ_VAL_DMA 812 if (!dma_has_cap(DMA_PQ_VAL, device->cap_mask)) 813 return false; 814 #endif 815 #endif 816 817 return true; 818} 819 820static int get_dma_id(struct dma_device *device) 821{ 822 int rc; 823 824 mutex_lock(&dma_list_mutex); 825 826 rc = idr_alloc(&dma_idr, NULL, 0, 0, GFP_KERNEL); 827 if (rc >= 0) 828 device->dev_id = rc; 829 830 mutex_unlock(&dma_list_mutex); 831 return rc < 0 ? rc : 0; 832} 833 834/** 835 * dma_async_device_register - registers DMA devices found 836 * @device: &dma_device 837 */ 838int dma_async_device_register(struct dma_device *device) 839{ 840 int chancnt = 0, rc; 841 struct dma_chan* chan; 842 atomic_t *idr_ref; 843 844 if (!device) 845 return -ENODEV; 846 847 /* validate device routines */ 848 BUG_ON(dma_has_cap(DMA_MEMCPY, device->cap_mask) && 849 !device->device_prep_dma_memcpy); 850 BUG_ON(dma_has_cap(DMA_XOR, device->cap_mask) && 851 !device->device_prep_dma_xor); 852 BUG_ON(dma_has_cap(DMA_XOR_VAL, device->cap_mask) && 853 !device->device_prep_dma_xor_val); 854 BUG_ON(dma_has_cap(DMA_PQ, device->cap_mask) && 855 !device->device_prep_dma_pq); 856 BUG_ON(dma_has_cap(DMA_PQ_VAL, device->cap_mask) && 857 !device->device_prep_dma_pq_val); 858 BUG_ON(dma_has_cap(DMA_MEMSET, device->cap_mask) && 859 !device->device_prep_dma_memset); 860 BUG_ON(dma_has_cap(DMA_INTERRUPT, device->cap_mask) && 861 !device->device_prep_dma_interrupt); 862 BUG_ON(dma_has_cap(DMA_SG, device->cap_mask) && 863 !device->device_prep_dma_sg); 864 BUG_ON(dma_has_cap(DMA_CYCLIC, device->cap_mask) && 865 !device->device_prep_dma_cyclic); 866 BUG_ON(dma_has_cap(DMA_INTERLEAVE, device->cap_mask) && 867 !device->device_prep_interleaved_dma); 868 869 BUG_ON(!device->device_tx_status); 870 BUG_ON(!device->device_issue_pending); 871 BUG_ON(!device->dev); 872 873 /* note: this only matters in the 874 * CONFIG_ASYNC_TX_ENABLE_CHANNEL_SWITCH=n case 875 */ 876 if (device_has_all_tx_types(device)) 877 dma_cap_set(DMA_ASYNC_TX, device->cap_mask); 878 879 idr_ref = kmalloc(sizeof(*idr_ref), GFP_KERNEL); 880 if (!idr_ref) 881 return -ENOMEM; 882 rc = get_dma_id(device); 883 if (rc != 0) { 884 kfree(idr_ref); 885 return rc; 886 } 887 888 atomic_set(idr_ref, 0); 889 890 /* represent channels in sysfs. Probably want devs too */ 891 list_for_each_entry(chan, &device->channels, device_node) { 892 rc = -ENOMEM; 893 chan->local = alloc_percpu(typeof(*chan->local)); 894 if (chan->local == NULL) 895 goto err_out; 896 chan->dev = kzalloc(sizeof(*chan->dev), GFP_KERNEL); 897 if (chan->dev == NULL) { 898 free_percpu(chan->local); 899 chan->local = NULL; 900 goto err_out; 901 } 902 903 chan->chan_id = chancnt++; 904 chan->dev->device.class = &dma_devclass; 905 chan->dev->device.parent = device->dev; 906 chan->dev->chan = chan; 907 chan->dev->idr_ref = idr_ref; 908 chan->dev->dev_id = device->dev_id; 909 atomic_inc(idr_ref); 910 dev_set_name(&chan->dev->device, "dma%dchan%d", 911 device->dev_id, chan->chan_id); 912 913 rc = device_register(&chan->dev->device); 914 if (rc) { 915 free_percpu(chan->local); 916 chan->local = NULL; 917 kfree(chan->dev); 918 atomic_dec(idr_ref); 919 goto err_out; 920 } 921 chan->client_count = 0; 922 } 923 device->chancnt = chancnt; 924 925 mutex_lock(&dma_list_mutex); 926 /* take references on public channels */ 927 if (dmaengine_ref_count && !dma_has_cap(DMA_PRIVATE, device->cap_mask)) 928 list_for_each_entry(chan, &device->channels, device_node) { 929 /* if clients are already waiting for channels we need 930 * to take references on their behalf 931 */ 932 if (dma_chan_get(chan) == -ENODEV) { 933 /* note we can only get here for the first 934 * channel as the remaining channels are 935 * guaranteed to get a reference 936 */ 937 rc = -ENODEV; 938 mutex_unlock(&dma_list_mutex); 939 goto err_out; 940 } 941 } 942 list_add_tail_rcu(&device->global_node, &dma_device_list); 943 if (dma_has_cap(DMA_PRIVATE, device->cap_mask)) 944 device->privatecnt++; /* Always private */ 945 dma_channel_rebalance(); 946 mutex_unlock(&dma_list_mutex); 947 948 return 0; 949 950err_out: 951 /* if we never registered a channel just release the idr */ 952 if (atomic_read(idr_ref) == 0) { 953 mutex_lock(&dma_list_mutex); 954 idr_remove(&dma_idr, device->dev_id); 955 mutex_unlock(&dma_list_mutex); 956 kfree(idr_ref); 957 return rc; 958 } 959 960 list_for_each_entry(chan, &device->channels, device_node) { 961 if (chan->local == NULL) 962 continue; 963 mutex_lock(&dma_list_mutex); 964 chan->dev->chan = NULL; 965 mutex_unlock(&dma_list_mutex); 966 device_unregister(&chan->dev->device); 967 free_percpu(chan->local); 968 } 969 return rc; 970} 971EXPORT_SYMBOL(dma_async_device_register); 972 973/** 974 * dma_async_device_unregister - unregister a DMA device 975 * @device: &dma_device 976 * 977 * This routine is called by dma driver exit routines, dmaengine holds module 978 * references to prevent it being called while channels are in use. 979 */ 980void dma_async_device_unregister(struct dma_device *device) 981{ 982 struct dma_chan *chan; 983 984 mutex_lock(&dma_list_mutex); 985 list_del_rcu(&device->global_node); 986 dma_channel_rebalance(); 987 mutex_unlock(&dma_list_mutex); 988 989 list_for_each_entry(chan, &device->channels, device_node) { 990 WARN_ONCE(chan->client_count, 991 "%s called while %d clients hold a reference\n", 992 __func__, chan->client_count); 993 mutex_lock(&dma_list_mutex); 994 chan->dev->chan = NULL; 995 mutex_unlock(&dma_list_mutex); 996 device_unregister(&chan->dev->device); 997 free_percpu(chan->local); 998 } 999} 1000EXPORT_SYMBOL(dma_async_device_unregister); 1001 1002struct dmaengine_unmap_pool { 1003 struct kmem_cache *cache; 1004 const char *name; 1005 mempool_t *pool; 1006 size_t size; 1007}; 1008 1009#define __UNMAP_POOL(x) { .size = x, .name = "dmaengine-unmap-" __stringify(x) } 1010static struct dmaengine_unmap_pool unmap_pool[] = { 1011 __UNMAP_POOL(2), 1012 #if IS_ENABLED(CONFIG_DMA_ENGINE_RAID) 1013 __UNMAP_POOL(16), 1014 __UNMAP_POOL(128), 1015 __UNMAP_POOL(256), 1016 #endif 1017}; 1018 1019static struct dmaengine_unmap_pool *__get_unmap_pool(int nr) 1020{ 1021 int order = get_count_order(nr); 1022 1023 switch (order) { 1024 case 0 ... 1: 1025 return &unmap_pool[0]; 1026 case 2 ... 4: 1027 return &unmap_pool[1]; 1028 case 5 ... 7: 1029 return &unmap_pool[2]; 1030 case 8: 1031 return &unmap_pool[3]; 1032 default: 1033 BUG(); 1034 return NULL; 1035 } 1036} 1037 1038static void dmaengine_unmap(struct kref *kref) 1039{ 1040 struct dmaengine_unmap_data *unmap = container_of(kref, typeof(*unmap), kref); 1041 struct device *dev = unmap->dev; 1042 int cnt, i; 1043 1044 cnt = unmap->to_cnt; 1045 for (i = 0; i < cnt; i++) 1046 dma_unmap_page(dev, unmap->addr[i], unmap->len, 1047 DMA_TO_DEVICE); 1048 cnt += unmap->from_cnt; 1049 for (; i < cnt; i++) 1050 dma_unmap_page(dev, unmap->addr[i], unmap->len, 1051 DMA_FROM_DEVICE); 1052 cnt += unmap->bidi_cnt; 1053 for (; i < cnt; i++) { 1054 if (unmap->addr[i] == 0) 1055 continue; 1056 dma_unmap_page(dev, unmap->addr[i], unmap->len, 1057 DMA_BIDIRECTIONAL); 1058 } 1059 cnt = unmap->map_cnt; 1060 mempool_free(unmap, __get_unmap_pool(cnt)->pool); 1061} 1062 1063void dmaengine_unmap_put(struct dmaengine_unmap_data *unmap) 1064{ 1065 if (unmap) 1066 kref_put(&unmap->kref, dmaengine_unmap); 1067} 1068EXPORT_SYMBOL_GPL(dmaengine_unmap_put); 1069 1070static void dmaengine_destroy_unmap_pool(void) 1071{ 1072 int i; 1073 1074 for (i = 0; i < ARRAY_SIZE(unmap_pool); i++) { 1075 struct dmaengine_unmap_pool *p = &unmap_pool[i]; 1076 1077 mempool_destroy(p->pool); 1078 p->pool = NULL; 1079 kmem_cache_destroy(p->cache); 1080 p->cache = NULL; 1081 } 1082} 1083 1084static int __init dmaengine_init_unmap_pool(void) 1085{ 1086 int i; 1087 1088 for (i = 0; i < ARRAY_SIZE(unmap_pool); i++) { 1089 struct dmaengine_unmap_pool *p = &unmap_pool[i]; 1090 size_t size; 1091 1092 size = sizeof(struct dmaengine_unmap_data) + 1093 sizeof(dma_addr_t) * p->size; 1094 1095 p->cache = kmem_cache_create(p->name, size, 0, 1096 SLAB_HWCACHE_ALIGN, NULL); 1097 if (!p->cache) 1098 break; 1099 p->pool = mempool_create_slab_pool(1, p->cache); 1100 if (!p->pool) 1101 break; 1102 } 1103 1104 if (i == ARRAY_SIZE(unmap_pool)) 1105 return 0; 1106 1107 dmaengine_destroy_unmap_pool(); 1108 return -ENOMEM; 1109} 1110 1111struct dmaengine_unmap_data * 1112dmaengine_get_unmap_data(struct device *dev, int nr, gfp_t flags) 1113{ 1114 struct dmaengine_unmap_data *unmap; 1115 1116 unmap = mempool_alloc(__get_unmap_pool(nr)->pool, flags); 1117 if (!unmap) 1118 return NULL; 1119 1120 memset(unmap, 0, sizeof(*unmap)); 1121 kref_init(&unmap->kref); 1122 unmap->dev = dev; 1123 unmap->map_cnt = nr; 1124 1125 return unmap; 1126} 1127EXPORT_SYMBOL(dmaengine_get_unmap_data); 1128 1129void dma_async_tx_descriptor_init(struct dma_async_tx_descriptor *tx, 1130 struct dma_chan *chan) 1131{ 1132 tx->chan = chan; 1133 #ifdef CONFIG_ASYNC_TX_ENABLE_CHANNEL_SWITCH 1134 spin_lock_init(&tx->lock); 1135 #endif 1136} 1137EXPORT_SYMBOL(dma_async_tx_descriptor_init); 1138 1139/* dma_wait_for_async_tx - spin wait for a transaction to complete 1140 * @tx: in-flight transaction to wait on 1141 */ 1142enum dma_status 1143dma_wait_for_async_tx(struct dma_async_tx_descriptor *tx) 1144{ 1145 unsigned long dma_sync_wait_timeout = jiffies + msecs_to_jiffies(5000); 1146 1147 if (!tx) 1148 return DMA_COMPLETE; 1149 1150 while (tx->cookie == -EBUSY) { 1151 if (time_after_eq(jiffies, dma_sync_wait_timeout)) { 1152 pr_err("%s timeout waiting for descriptor submission\n", 1153 __func__); 1154 return DMA_ERROR; 1155 } 1156 cpu_relax(); 1157 } 1158 return dma_sync_wait(tx->chan, tx->cookie); 1159} 1160EXPORT_SYMBOL_GPL(dma_wait_for_async_tx); 1161 1162/* dma_run_dependencies - helper routine for dma drivers to process 1163 * (start) dependent operations on their target channel 1164 * @tx: transaction with dependencies 1165 */ 1166void dma_run_dependencies(struct dma_async_tx_descriptor *tx) 1167{ 1168 struct dma_async_tx_descriptor *dep = txd_next(tx); 1169 struct dma_async_tx_descriptor *dep_next; 1170 struct dma_chan *chan; 1171 1172 if (!dep) 1173 return; 1174 1175 /* we'll submit tx->next now, so clear the link */ 1176 txd_clear_next(tx); 1177 chan = dep->chan; 1178 1179 /* keep submitting up until a channel switch is detected 1180 * in that case we will be called again as a result of 1181 * processing the interrupt from async_tx_channel_switch 1182 */ 1183 for (; dep; dep = dep_next) { 1184 txd_lock(dep); 1185 txd_clear_parent(dep); 1186 dep_next = txd_next(dep); 1187 if (dep_next && dep_next->chan == chan) 1188 txd_clear_next(dep); /* ->next will be submitted */ 1189 else 1190 dep_next = NULL; /* submit current dep and terminate */ 1191 txd_unlock(dep); 1192 1193 dep->tx_submit(dep); 1194 } 1195 1196 chan->device->device_issue_pending(chan); 1197} 1198EXPORT_SYMBOL_GPL(dma_run_dependencies); 1199 1200static int __init dma_bus_init(void) 1201{ 1202 int err = dmaengine_init_unmap_pool(); 1203 1204 if (err) 1205 return err; 1206 return class_register(&dma_devclass); 1207} 1208arch_initcall(dma_bus_init); 1209 1210 1211