1/* 2 * linux/fs/block_dev.c 3 * 4 * Copyright (C) 1991, 1992 Linus Torvalds 5 * Copyright (C) 2001 Andrea Arcangeli <andrea@suse.de> SuSE 6 */ 7 8#include <linux/init.h> 9#include <linux/mm.h> 10#include <linux/fcntl.h> 11#include <linux/slab.h> 12#include <linux/kmod.h> 13#include <linux/major.h> 14#include <linux/device_cgroup.h> 15#include <linux/highmem.h> 16#include <linux/blkdev.h> 17#include <linux/backing-dev.h> 18#include <linux/module.h> 19#include <linux/blkpg.h> 20#include <linux/magic.h> 21#include <linux/buffer_head.h> 22#include <linux/swap.h> 23#include <linux/pagevec.h> 24#include <linux/writeback.h> 25#include <linux/mpage.h> 26#include <linux/mount.h> 27#include <linux/uio.h> 28#include <linux/namei.h> 29#include <linux/log2.h> 30#include <linux/cleancache.h> 31#include <linux/dax.h> 32#include <asm/uaccess.h> 33#include "internal.h" 34 35struct bdev_inode { 36 struct block_device bdev; 37 struct inode vfs_inode; 38}; 39 40static const struct address_space_operations def_blk_aops; 41 42static inline struct bdev_inode *BDEV_I(struct inode *inode) 43{ 44 return container_of(inode, struct bdev_inode, vfs_inode); 45} 46 47struct block_device *I_BDEV(struct inode *inode) 48{ 49 return &BDEV_I(inode)->bdev; 50} 51EXPORT_SYMBOL(I_BDEV); 52 53static void bdev_write_inode(struct block_device *bdev) 54{ 55 struct inode *inode = bdev->bd_inode; 56 int ret; 57 58 spin_lock(&inode->i_lock); 59 while (inode->i_state & I_DIRTY) { 60 spin_unlock(&inode->i_lock); 61 ret = write_inode_now(inode, true); 62 if (ret) { 63 char name[BDEVNAME_SIZE]; 64 pr_warn_ratelimited("VFS: Dirty inode writeback failed " 65 "for block device %s (err=%d).\n", 66 bdevname(bdev, name), ret); 67 } 68 spin_lock(&inode->i_lock); 69 } 70 spin_unlock(&inode->i_lock); 71} 72 73/* Kill _all_ buffers and pagecache , dirty or not.. */ 74void kill_bdev(struct block_device *bdev) 75{ 76 struct address_space *mapping = bdev->bd_inode->i_mapping; 77 78 if (mapping->nrpages == 0 && mapping->nrshadows == 0) 79 return; 80 81 invalidate_bh_lrus(); 82 truncate_inode_pages(mapping, 0); 83} 84EXPORT_SYMBOL(kill_bdev); 85 86/* Invalidate clean unused buffers and pagecache. */ 87void invalidate_bdev(struct block_device *bdev) 88{ 89 struct address_space *mapping = bdev->bd_inode->i_mapping; 90 91 if (mapping->nrpages == 0) 92 return; 93 94 invalidate_bh_lrus(); 95 lru_add_drain_all(); /* make sure all lru add caches are flushed */ 96 invalidate_mapping_pages(mapping, 0, -1); 97 /* 99% of the time, we don't need to flush the cleancache on the bdev. 98 * But, for the strange corners, lets be cautious 99 */ 100 cleancache_invalidate_inode(mapping); 101} 102EXPORT_SYMBOL(invalidate_bdev); 103 104int set_blocksize(struct block_device *bdev, int size) 105{ 106 /* Size must be a power of two, and between 512 and PAGE_SIZE */ 107 if (size > PAGE_SIZE || size < 512 || !is_power_of_2(size)) 108 return -EINVAL; 109 110 /* Size cannot be smaller than the size supported by the device */ 111 if (size < bdev_logical_block_size(bdev)) 112 return -EINVAL; 113 114 /* Don't change the size if it is same as current */ 115 if (bdev->bd_block_size != size) { 116 sync_blockdev(bdev); 117 bdev->bd_block_size = size; 118 bdev->bd_inode->i_blkbits = blksize_bits(size); 119 kill_bdev(bdev); 120 } 121 return 0; 122} 123 124EXPORT_SYMBOL(set_blocksize); 125 126int sb_set_blocksize(struct super_block *sb, int size) 127{ 128 if (set_blocksize(sb->s_bdev, size)) 129 return 0; 130 /* If we get here, we know size is power of two 131 * and it's value is between 512 and PAGE_SIZE */ 132 sb->s_blocksize = size; 133 sb->s_blocksize_bits = blksize_bits(size); 134 return sb->s_blocksize; 135} 136 137EXPORT_SYMBOL(sb_set_blocksize); 138 139int sb_min_blocksize(struct super_block *sb, int size) 140{ 141 int minsize = bdev_logical_block_size(sb->s_bdev); 142 if (size < minsize) 143 size = minsize; 144 return sb_set_blocksize(sb, size); 145} 146 147EXPORT_SYMBOL(sb_min_blocksize); 148 149static int 150blkdev_get_block(struct inode *inode, sector_t iblock, 151 struct buffer_head *bh, int create) 152{ 153 bh->b_bdev = I_BDEV(inode); 154 bh->b_blocknr = iblock; 155 set_buffer_mapped(bh); 156 return 0; 157} 158 159static ssize_t 160blkdev_direct_IO(struct kiocb *iocb, struct iov_iter *iter, loff_t offset) 161{ 162 struct file *file = iocb->ki_filp; 163 struct inode *inode = file->f_mapping->host; 164 165 if (IS_DAX(inode)) 166 return dax_do_io(iocb, inode, iter, offset, blkdev_get_block, 167 NULL, DIO_SKIP_DIO_COUNT); 168 return __blockdev_direct_IO(iocb, inode, I_BDEV(inode), iter, offset, 169 blkdev_get_block, NULL, NULL, 170 DIO_SKIP_DIO_COUNT); 171} 172 173int __sync_blockdev(struct block_device *bdev, int wait) 174{ 175 if (!bdev) 176 return 0; 177 if (!wait) 178 return filemap_flush(bdev->bd_inode->i_mapping); 179 return filemap_write_and_wait(bdev->bd_inode->i_mapping); 180} 181 182/* 183 * Write out and wait upon all the dirty data associated with a block 184 * device via its mapping. Does not take the superblock lock. 185 */ 186int sync_blockdev(struct block_device *bdev) 187{ 188 return __sync_blockdev(bdev, 1); 189} 190EXPORT_SYMBOL(sync_blockdev); 191 192/* 193 * Write out and wait upon all dirty data associated with this 194 * device. Filesystem data as well as the underlying block 195 * device. Takes the superblock lock. 196 */ 197int fsync_bdev(struct block_device *bdev) 198{ 199 struct super_block *sb = get_super(bdev); 200 if (sb) { 201 int res = sync_filesystem(sb); 202 drop_super(sb); 203 return res; 204 } 205 return sync_blockdev(bdev); 206} 207EXPORT_SYMBOL(fsync_bdev); 208 209/** 210 * freeze_bdev -- lock a filesystem and force it into a consistent state 211 * @bdev: blockdevice to lock 212 * 213 * If a superblock is found on this device, we take the s_umount semaphore 214 * on it to make sure nobody unmounts until the snapshot creation is done. 215 * The reference counter (bd_fsfreeze_count) guarantees that only the last 216 * unfreeze process can unfreeze the frozen filesystem actually when multiple 217 * freeze requests arrive simultaneously. It counts up in freeze_bdev() and 218 * count down in thaw_bdev(). When it becomes 0, thaw_bdev() will unfreeze 219 * actually. 220 */ 221struct super_block *freeze_bdev(struct block_device *bdev) 222{ 223 struct super_block *sb; 224 int error = 0; 225 226 mutex_lock(&bdev->bd_fsfreeze_mutex); 227 if (++bdev->bd_fsfreeze_count > 1) { 228 /* 229 * We don't even need to grab a reference - the first call 230 * to freeze_bdev grab an active reference and only the last 231 * thaw_bdev drops it. 232 */ 233 sb = get_super(bdev); 234 drop_super(sb); 235 mutex_unlock(&bdev->bd_fsfreeze_mutex); 236 return sb; 237 } 238 239 sb = get_active_super(bdev); 240 if (!sb) 241 goto out; 242 if (sb->s_op->freeze_super) 243 error = sb->s_op->freeze_super(sb); 244 else 245 error = freeze_super(sb); 246 if (error) { 247 deactivate_super(sb); 248 bdev->bd_fsfreeze_count--; 249 mutex_unlock(&bdev->bd_fsfreeze_mutex); 250 return ERR_PTR(error); 251 } 252 deactivate_super(sb); 253 out: 254 sync_blockdev(bdev); 255 mutex_unlock(&bdev->bd_fsfreeze_mutex); 256 return sb; /* thaw_bdev releases s->s_umount */ 257} 258EXPORT_SYMBOL(freeze_bdev); 259 260/** 261 * thaw_bdev -- unlock filesystem 262 * @bdev: blockdevice to unlock 263 * @sb: associated superblock 264 * 265 * Unlocks the filesystem and marks it writeable again after freeze_bdev(). 266 */ 267int thaw_bdev(struct block_device *bdev, struct super_block *sb) 268{ 269 int error = -EINVAL; 270 271 mutex_lock(&bdev->bd_fsfreeze_mutex); 272 if (!bdev->bd_fsfreeze_count) 273 goto out; 274 275 error = 0; 276 if (--bdev->bd_fsfreeze_count > 0) 277 goto out; 278 279 if (!sb) 280 goto out; 281 282 if (sb->s_op->thaw_super) 283 error = sb->s_op->thaw_super(sb); 284 else 285 error = thaw_super(sb); 286 if (error) { 287 bdev->bd_fsfreeze_count++; 288 mutex_unlock(&bdev->bd_fsfreeze_mutex); 289 return error; 290 } 291out: 292 mutex_unlock(&bdev->bd_fsfreeze_mutex); 293 return 0; 294} 295EXPORT_SYMBOL(thaw_bdev); 296 297static int blkdev_writepage(struct page *page, struct writeback_control *wbc) 298{ 299 return block_write_full_page(page, blkdev_get_block, wbc); 300} 301 302static int blkdev_readpage(struct file * file, struct page * page) 303{ 304 return block_read_full_page(page, blkdev_get_block); 305} 306 307static int blkdev_readpages(struct file *file, struct address_space *mapping, 308 struct list_head *pages, unsigned nr_pages) 309{ 310 return mpage_readpages(mapping, pages, nr_pages, blkdev_get_block); 311} 312 313static int blkdev_write_begin(struct file *file, struct address_space *mapping, 314 loff_t pos, unsigned len, unsigned flags, 315 struct page **pagep, void **fsdata) 316{ 317 return block_write_begin(mapping, pos, len, flags, pagep, 318 blkdev_get_block); 319} 320 321static int blkdev_write_end(struct file *file, struct address_space *mapping, 322 loff_t pos, unsigned len, unsigned copied, 323 struct page *page, void *fsdata) 324{ 325 int ret; 326 ret = block_write_end(file, mapping, pos, len, copied, page, fsdata); 327 328 unlock_page(page); 329 page_cache_release(page); 330 331 return ret; 332} 333 334/* 335 * private llseek: 336 * for a block special file file_inode(file)->i_size is zero 337 * so we compute the size by hand (just as in block_read/write above) 338 */ 339static loff_t block_llseek(struct file *file, loff_t offset, int whence) 340{ 341 struct inode *bd_inode = file->f_mapping->host; 342 loff_t retval; 343 344 mutex_lock(&bd_inode->i_mutex); 345 retval = fixed_size_llseek(file, offset, whence, i_size_read(bd_inode)); 346 mutex_unlock(&bd_inode->i_mutex); 347 return retval; 348} 349 350int blkdev_fsync(struct file *filp, loff_t start, loff_t end, int datasync) 351{ 352 struct inode *bd_inode = filp->f_mapping->host; 353 struct block_device *bdev = I_BDEV(bd_inode); 354 int error; 355 356 error = filemap_write_and_wait_range(filp->f_mapping, start, end); 357 if (error) 358 return error; 359 360 /* 361 * There is no need to serialise calls to blkdev_issue_flush with 362 * i_mutex and doing so causes performance issues with concurrent 363 * O_SYNC writers to a block device. 364 */ 365 error = blkdev_issue_flush(bdev, GFP_KERNEL, NULL); 366 if (error == -EOPNOTSUPP) 367 error = 0; 368 369 return error; 370} 371EXPORT_SYMBOL(blkdev_fsync); 372 373/** 374 * bdev_read_page() - Start reading a page from a block device 375 * @bdev: The device to read the page from 376 * @sector: The offset on the device to read the page to (need not be aligned) 377 * @page: The page to read 378 * 379 * On entry, the page should be locked. It will be unlocked when the page 380 * has been read. If the block driver implements rw_page synchronously, 381 * that will be true on exit from this function, but it need not be. 382 * 383 * Errors returned by this function are usually "soft", eg out of memory, or 384 * queue full; callers should try a different route to read this page rather 385 * than propagate an error back up the stack. 386 * 387 * Return: negative errno if an error occurs, 0 if submission was successful. 388 */ 389int bdev_read_page(struct block_device *bdev, sector_t sector, 390 struct page *page) 391{ 392 const struct block_device_operations *ops = bdev->bd_disk->fops; 393 int result = -EOPNOTSUPP; 394 395 if (!ops->rw_page || bdev_get_integrity(bdev)) 396 return result; 397 398 result = blk_queue_enter(bdev->bd_queue, GFP_KERNEL); 399 if (result) 400 return result; 401 result = ops->rw_page(bdev, sector + get_start_sect(bdev), page, READ); 402 blk_queue_exit(bdev->bd_queue); 403 return result; 404} 405EXPORT_SYMBOL_GPL(bdev_read_page); 406 407/** 408 * bdev_write_page() - Start writing a page to a block device 409 * @bdev: The device to write the page to 410 * @sector: The offset on the device to write the page to (need not be aligned) 411 * @page: The page to write 412 * @wbc: The writeback_control for the write 413 * 414 * On entry, the page should be locked and not currently under writeback. 415 * On exit, if the write started successfully, the page will be unlocked and 416 * under writeback. If the write failed already (eg the driver failed to 417 * queue the page to the device), the page will still be locked. If the 418 * caller is a ->writepage implementation, it will need to unlock the page. 419 * 420 * Errors returned by this function are usually "soft", eg out of memory, or 421 * queue full; callers should try a different route to write this page rather 422 * than propagate an error back up the stack. 423 * 424 * Return: negative errno if an error occurs, 0 if submission was successful. 425 */ 426int bdev_write_page(struct block_device *bdev, sector_t sector, 427 struct page *page, struct writeback_control *wbc) 428{ 429 int result; 430 int rw = (wbc->sync_mode == WB_SYNC_ALL) ? WRITE_SYNC : WRITE; 431 const struct block_device_operations *ops = bdev->bd_disk->fops; 432 433 if (!ops->rw_page || bdev_get_integrity(bdev)) 434 return -EOPNOTSUPP; 435 result = blk_queue_enter(bdev->bd_queue, GFP_KERNEL); 436 if (result) 437 return result; 438 439 set_page_writeback(page); 440 result = ops->rw_page(bdev, sector + get_start_sect(bdev), page, rw); 441 if (result) 442 end_page_writeback(page); 443 else 444 unlock_page(page); 445 blk_queue_exit(bdev->bd_queue); 446 return result; 447} 448EXPORT_SYMBOL_GPL(bdev_write_page); 449 450/** 451 * bdev_direct_access() - Get the address for directly-accessibly memory 452 * @bdev: The device containing the memory 453 * @sector: The offset within the device 454 * @addr: Where to put the address of the memory 455 * @pfn: The Page Frame Number for the memory 456 * @size: The number of bytes requested 457 * 458 * If a block device is made up of directly addressable memory, this function 459 * will tell the caller the PFN and the address of the memory. The address 460 * may be directly dereferenced within the kernel without the need to call 461 * ioremap(), kmap() or similar. The PFN is suitable for inserting into 462 * page tables. 463 * 464 * Return: negative errno if an error occurs, otherwise the number of bytes 465 * accessible at this address. 466 */ 467long bdev_direct_access(struct block_device *bdev, sector_t sector, 468 void __pmem **addr, unsigned long *pfn, long size) 469{ 470 long avail; 471 const struct block_device_operations *ops = bdev->bd_disk->fops; 472 473 /* 474 * The device driver is allowed to sleep, in order to make the 475 * memory directly accessible. 476 */ 477 might_sleep(); 478 479 if (size < 0) 480 return size; 481 if (!ops->direct_access) 482 return -EOPNOTSUPP; 483 if ((sector + DIV_ROUND_UP(size, 512)) > 484 part_nr_sects_read(bdev->bd_part)) 485 return -ERANGE; 486 sector += get_start_sect(bdev); 487 if (sector % (PAGE_SIZE / 512)) 488 return -EINVAL; 489 avail = ops->direct_access(bdev, sector, addr, pfn); 490 if (!avail) 491 return -ERANGE; 492 return min(avail, size); 493} 494EXPORT_SYMBOL_GPL(bdev_direct_access); 495 496/* 497 * pseudo-fs 498 */ 499 500static __cacheline_aligned_in_smp DEFINE_SPINLOCK(bdev_lock); 501static struct kmem_cache * bdev_cachep __read_mostly; 502 503static struct inode *bdev_alloc_inode(struct super_block *sb) 504{ 505 struct bdev_inode *ei = kmem_cache_alloc(bdev_cachep, GFP_KERNEL); 506 if (!ei) 507 return NULL; 508 return &ei->vfs_inode; 509} 510 511static void bdev_i_callback(struct rcu_head *head) 512{ 513 struct inode *inode = container_of(head, struct inode, i_rcu); 514 struct bdev_inode *bdi = BDEV_I(inode); 515 516 kmem_cache_free(bdev_cachep, bdi); 517} 518 519static void bdev_destroy_inode(struct inode *inode) 520{ 521 call_rcu(&inode->i_rcu, bdev_i_callback); 522} 523 524static void init_once(void *foo) 525{ 526 struct bdev_inode *ei = (struct bdev_inode *) foo; 527 struct block_device *bdev = &ei->bdev; 528 529 memset(bdev, 0, sizeof(*bdev)); 530 mutex_init(&bdev->bd_mutex); 531 INIT_LIST_HEAD(&bdev->bd_inodes); 532 INIT_LIST_HEAD(&bdev->bd_list); 533#ifdef CONFIG_SYSFS 534 INIT_LIST_HEAD(&bdev->bd_holder_disks); 535#endif 536 inode_init_once(&ei->vfs_inode); 537 /* Initialize mutex for freeze. */ 538 mutex_init(&bdev->bd_fsfreeze_mutex); 539} 540 541static inline void __bd_forget(struct inode *inode) 542{ 543 list_del_init(&inode->i_devices); 544 inode->i_bdev = NULL; 545 inode->i_mapping = &inode->i_data; 546} 547 548static void bdev_evict_inode(struct inode *inode) 549{ 550 struct block_device *bdev = &BDEV_I(inode)->bdev; 551 struct list_head *p; 552 truncate_inode_pages_final(&inode->i_data); 553 invalidate_inode_buffers(inode); /* is it needed here? */ 554 clear_inode(inode); 555 spin_lock(&bdev_lock); 556 while ( (p = bdev->bd_inodes.next) != &bdev->bd_inodes ) { 557 __bd_forget(list_entry(p, struct inode, i_devices)); 558 } 559 list_del_init(&bdev->bd_list); 560 spin_unlock(&bdev_lock); 561} 562 563static const struct super_operations bdev_sops = { 564 .statfs = simple_statfs, 565 .alloc_inode = bdev_alloc_inode, 566 .destroy_inode = bdev_destroy_inode, 567 .drop_inode = generic_delete_inode, 568 .evict_inode = bdev_evict_inode, 569}; 570 571static struct dentry *bd_mount(struct file_system_type *fs_type, 572 int flags, const char *dev_name, void *data) 573{ 574 return mount_pseudo(fs_type, "bdev:", &bdev_sops, NULL, BDEVFS_MAGIC); 575} 576 577static struct file_system_type bd_type = { 578 .name = "bdev", 579 .mount = bd_mount, 580 .kill_sb = kill_anon_super, 581}; 582 583struct super_block *blockdev_superblock __read_mostly; 584EXPORT_SYMBOL_GPL(blockdev_superblock); 585 586void __init bdev_cache_init(void) 587{ 588 int err; 589 static struct vfsmount *bd_mnt; 590 591 bdev_cachep = kmem_cache_create("bdev_cache", sizeof(struct bdev_inode), 592 0, (SLAB_HWCACHE_ALIGN|SLAB_RECLAIM_ACCOUNT| 593 SLAB_MEM_SPREAD|SLAB_PANIC), 594 init_once); 595 err = register_filesystem(&bd_type); 596 if (err) 597 panic("Cannot register bdev pseudo-fs"); 598 bd_mnt = kern_mount(&bd_type); 599 if (IS_ERR(bd_mnt)) 600 panic("Cannot create bdev pseudo-fs"); 601 blockdev_superblock = bd_mnt->mnt_sb; /* For writeback */ 602} 603 604/* 605 * Most likely _very_ bad one - but then it's hardly critical for small 606 * /dev and can be fixed when somebody will need really large one. 607 * Keep in mind that it will be fed through icache hash function too. 608 */ 609static inline unsigned long hash(dev_t dev) 610{ 611 return MAJOR(dev)+MINOR(dev); 612} 613 614static int bdev_test(struct inode *inode, void *data) 615{ 616 return BDEV_I(inode)->bdev.bd_dev == *(dev_t *)data; 617} 618 619static int bdev_set(struct inode *inode, void *data) 620{ 621 BDEV_I(inode)->bdev.bd_dev = *(dev_t *)data; 622 return 0; 623} 624 625static LIST_HEAD(all_bdevs); 626 627struct block_device *bdget(dev_t dev) 628{ 629 struct block_device *bdev; 630 struct inode *inode; 631 632 inode = iget5_locked(blockdev_superblock, hash(dev), 633 bdev_test, bdev_set, &dev); 634 635 if (!inode) 636 return NULL; 637 638 bdev = &BDEV_I(inode)->bdev; 639 640 if (inode->i_state & I_NEW) { 641 bdev->bd_contains = NULL; 642 bdev->bd_super = NULL; 643 bdev->bd_inode = inode; 644 bdev->bd_block_size = (1 << inode->i_blkbits); 645 bdev->bd_part_count = 0; 646 bdev->bd_invalidated = 0; 647 inode->i_mode = S_IFBLK; 648 inode->i_rdev = dev; 649 inode->i_bdev = bdev; 650 inode->i_data.a_ops = &def_blk_aops; 651 mapping_set_gfp_mask(&inode->i_data, GFP_USER); 652 spin_lock(&bdev_lock); 653 list_add(&bdev->bd_list, &all_bdevs); 654 spin_unlock(&bdev_lock); 655 unlock_new_inode(inode); 656 } 657 return bdev; 658} 659 660EXPORT_SYMBOL(bdget); 661 662/** 663 * bdgrab -- Grab a reference to an already referenced block device 664 * @bdev: Block device to grab a reference to. 665 */ 666struct block_device *bdgrab(struct block_device *bdev) 667{ 668 ihold(bdev->bd_inode); 669 return bdev; 670} 671EXPORT_SYMBOL(bdgrab); 672 673long nr_blockdev_pages(void) 674{ 675 struct block_device *bdev; 676 long ret = 0; 677 spin_lock(&bdev_lock); 678 list_for_each_entry(bdev, &all_bdevs, bd_list) { 679 ret += bdev->bd_inode->i_mapping->nrpages; 680 } 681 spin_unlock(&bdev_lock); 682 return ret; 683} 684 685void bdput(struct block_device *bdev) 686{ 687 iput(bdev->bd_inode); 688} 689 690EXPORT_SYMBOL(bdput); 691 692static struct block_device *bd_acquire(struct inode *inode) 693{ 694 struct block_device *bdev; 695 696 spin_lock(&bdev_lock); 697 bdev = inode->i_bdev; 698 if (bdev) { 699 ihold(bdev->bd_inode); 700 spin_unlock(&bdev_lock); 701 return bdev; 702 } 703 spin_unlock(&bdev_lock); 704 705 bdev = bdget(inode->i_rdev); 706 if (bdev) { 707 spin_lock(&bdev_lock); 708 if (!inode->i_bdev) { 709 /* 710 * We take an additional reference to bd_inode, 711 * and it's released in clear_inode() of inode. 712 * So, we can access it via ->i_mapping always 713 * without igrab(). 714 */ 715 ihold(bdev->bd_inode); 716 inode->i_bdev = bdev; 717 inode->i_mapping = bdev->bd_inode->i_mapping; 718 list_add(&inode->i_devices, &bdev->bd_inodes); 719 } 720 spin_unlock(&bdev_lock); 721 } 722 return bdev; 723} 724 725/* Call when you free inode */ 726 727void bd_forget(struct inode *inode) 728{ 729 struct block_device *bdev = NULL; 730 731 spin_lock(&bdev_lock); 732 if (!sb_is_blkdev_sb(inode->i_sb)) 733 bdev = inode->i_bdev; 734 __bd_forget(inode); 735 spin_unlock(&bdev_lock); 736 737 if (bdev) 738 iput(bdev->bd_inode); 739} 740 741/** 742 * bd_may_claim - test whether a block device can be claimed 743 * @bdev: block device of interest 744 * @whole: whole block device containing @bdev, may equal @bdev 745 * @holder: holder trying to claim @bdev 746 * 747 * Test whether @bdev can be claimed by @holder. 748 * 749 * CONTEXT: 750 * spin_lock(&bdev_lock). 751 * 752 * RETURNS: 753 * %true if @bdev can be claimed, %false otherwise. 754 */ 755static bool bd_may_claim(struct block_device *bdev, struct block_device *whole, 756 void *holder) 757{ 758 if (bdev->bd_holder == holder) 759 return true; /* already a holder */ 760 else if (bdev->bd_holder != NULL) 761 return false; /* held by someone else */ 762 else if (bdev->bd_contains == bdev) 763 return true; /* is a whole device which isn't held */ 764 765 else if (whole->bd_holder == bd_may_claim) 766 return true; /* is a partition of a device that is being partitioned */ 767 else if (whole->bd_holder != NULL) 768 return false; /* is a partition of a held device */ 769 else 770 return true; /* is a partition of an un-held device */ 771} 772 773/** 774 * bd_prepare_to_claim - prepare to claim a block device 775 * @bdev: block device of interest 776 * @whole: the whole device containing @bdev, may equal @bdev 777 * @holder: holder trying to claim @bdev 778 * 779 * Prepare to claim @bdev. This function fails if @bdev is already 780 * claimed by another holder and waits if another claiming is in 781 * progress. This function doesn't actually claim. On successful 782 * return, the caller has ownership of bd_claiming and bd_holder[s]. 783 * 784 * CONTEXT: 785 * spin_lock(&bdev_lock). Might release bdev_lock, sleep and regrab 786 * it multiple times. 787 * 788 * RETURNS: 789 * 0 if @bdev can be claimed, -EBUSY otherwise. 790 */ 791static int bd_prepare_to_claim(struct block_device *bdev, 792 struct block_device *whole, void *holder) 793{ 794retry: 795 /* if someone else claimed, fail */ 796 if (!bd_may_claim(bdev, whole, holder)) 797 return -EBUSY; 798 799 /* if claiming is already in progress, wait for it to finish */ 800 if (whole->bd_claiming) { 801 wait_queue_head_t *wq = bit_waitqueue(&whole->bd_claiming, 0); 802 DEFINE_WAIT(wait); 803 804 prepare_to_wait(wq, &wait, TASK_UNINTERRUPTIBLE); 805 spin_unlock(&bdev_lock); 806 schedule(); 807 finish_wait(wq, &wait); 808 spin_lock(&bdev_lock); 809 goto retry; 810 } 811 812 /* yay, all mine */ 813 return 0; 814} 815 816/** 817 * bd_start_claiming - start claiming a block device 818 * @bdev: block device of interest 819 * @holder: holder trying to claim @bdev 820 * 821 * @bdev is about to be opened exclusively. Check @bdev can be opened 822 * exclusively and mark that an exclusive open is in progress. Each 823 * successful call to this function must be matched with a call to 824 * either bd_finish_claiming() or bd_abort_claiming() (which do not 825 * fail). 826 * 827 * This function is used to gain exclusive access to the block device 828 * without actually causing other exclusive open attempts to fail. It 829 * should be used when the open sequence itself requires exclusive 830 * access but may subsequently fail. 831 * 832 * CONTEXT: 833 * Might sleep. 834 * 835 * RETURNS: 836 * Pointer to the block device containing @bdev on success, ERR_PTR() 837 * value on failure. 838 */ 839static struct block_device *bd_start_claiming(struct block_device *bdev, 840 void *holder) 841{ 842 struct gendisk *disk; 843 struct block_device *whole; 844 int partno, err; 845 846 might_sleep(); 847 848 /* 849 * @bdev might not have been initialized properly yet, look up 850 * and grab the outer block device the hard way. 851 */ 852 disk = get_gendisk(bdev->bd_dev, &partno); 853 if (!disk) 854 return ERR_PTR(-ENXIO); 855 856 /* 857 * Normally, @bdev should equal what's returned from bdget_disk() 858 * if partno is 0; however, some drivers (floppy) use multiple 859 * bdev's for the same physical device and @bdev may be one of the 860 * aliases. Keep @bdev if partno is 0. This means claimer 861 * tracking is broken for those devices but it has always been that 862 * way. 863 */ 864 if (partno) 865 whole = bdget_disk(disk, 0); 866 else 867 whole = bdgrab(bdev); 868 869 module_put(disk->fops->owner); 870 put_disk(disk); 871 if (!whole) 872 return ERR_PTR(-ENOMEM); 873 874 /* prepare to claim, if successful, mark claiming in progress */ 875 spin_lock(&bdev_lock); 876 877 err = bd_prepare_to_claim(bdev, whole, holder); 878 if (err == 0) { 879 whole->bd_claiming = holder; 880 spin_unlock(&bdev_lock); 881 return whole; 882 } else { 883 spin_unlock(&bdev_lock); 884 bdput(whole); 885 return ERR_PTR(err); 886 } 887} 888 889#ifdef CONFIG_SYSFS 890struct bd_holder_disk { 891 struct list_head list; 892 struct gendisk *disk; 893 int refcnt; 894}; 895 896static struct bd_holder_disk *bd_find_holder_disk(struct block_device *bdev, 897 struct gendisk *disk) 898{ 899 struct bd_holder_disk *holder; 900 901 list_for_each_entry(holder, &bdev->bd_holder_disks, list) 902 if (holder->disk == disk) 903 return holder; 904 return NULL; 905} 906 907static int add_symlink(struct kobject *from, struct kobject *to) 908{ 909 return sysfs_create_link(from, to, kobject_name(to)); 910} 911 912static void del_symlink(struct kobject *from, struct kobject *to) 913{ 914 sysfs_remove_link(from, kobject_name(to)); 915} 916 917/** 918 * bd_link_disk_holder - create symlinks between holding disk and slave bdev 919 * @bdev: the claimed slave bdev 920 * @disk: the holding disk 921 * 922 * DON'T USE THIS UNLESS YOU'RE ALREADY USING IT. 923 * 924 * This functions creates the following sysfs symlinks. 925 * 926 * - from "slaves" directory of the holder @disk to the claimed @bdev 927 * - from "holders" directory of the @bdev to the holder @disk 928 * 929 * For example, if /dev/dm-0 maps to /dev/sda and disk for dm-0 is 930 * passed to bd_link_disk_holder(), then: 931 * 932 * /sys/block/dm-0/slaves/sda --> /sys/block/sda 933 * /sys/block/sda/holders/dm-0 --> /sys/block/dm-0 934 * 935 * The caller must have claimed @bdev before calling this function and 936 * ensure that both @bdev and @disk are valid during the creation and 937 * lifetime of these symlinks. 938 * 939 * CONTEXT: 940 * Might sleep. 941 * 942 * RETURNS: 943 * 0 on success, -errno on failure. 944 */ 945int bd_link_disk_holder(struct block_device *bdev, struct gendisk *disk) 946{ 947 struct bd_holder_disk *holder; 948 int ret = 0; 949 950 mutex_lock(&bdev->bd_mutex); 951 952 WARN_ON_ONCE(!bdev->bd_holder); 953 954 /* FIXME: remove the following once add_disk() handles errors */ 955 if (WARN_ON(!disk->slave_dir || !bdev->bd_part->holder_dir)) 956 goto out_unlock; 957 958 holder = bd_find_holder_disk(bdev, disk); 959 if (holder) { 960 holder->refcnt++; 961 goto out_unlock; 962 } 963 964 holder = kzalloc(sizeof(*holder), GFP_KERNEL); 965 if (!holder) { 966 ret = -ENOMEM; 967 goto out_unlock; 968 } 969 970 INIT_LIST_HEAD(&holder->list); 971 holder->disk = disk; 972 holder->refcnt = 1; 973 974 ret = add_symlink(disk->slave_dir, &part_to_dev(bdev->bd_part)->kobj); 975 if (ret) 976 goto out_free; 977 978 ret = add_symlink(bdev->bd_part->holder_dir, &disk_to_dev(disk)->kobj); 979 if (ret) 980 goto out_del; 981 /* 982 * bdev could be deleted beneath us which would implicitly destroy 983 * the holder directory. Hold on to it. 984 */ 985 kobject_get(bdev->bd_part->holder_dir); 986 987 list_add(&holder->list, &bdev->bd_holder_disks); 988 goto out_unlock; 989 990out_del: 991 del_symlink(disk->slave_dir, &part_to_dev(bdev->bd_part)->kobj); 992out_free: 993 kfree(holder); 994out_unlock: 995 mutex_unlock(&bdev->bd_mutex); 996 return ret; 997} 998EXPORT_SYMBOL_GPL(bd_link_disk_holder); 999 1000/** 1001 * bd_unlink_disk_holder - destroy symlinks created by bd_link_disk_holder() 1002 * @bdev: the calimed slave bdev 1003 * @disk: the holding disk 1004 * 1005 * DON'T USE THIS UNLESS YOU'RE ALREADY USING IT. 1006 * 1007 * CONTEXT: 1008 * Might sleep. 1009 */ 1010void bd_unlink_disk_holder(struct block_device *bdev, struct gendisk *disk) 1011{ 1012 struct bd_holder_disk *holder; 1013 1014 mutex_lock(&bdev->bd_mutex); 1015 1016 holder = bd_find_holder_disk(bdev, disk); 1017 1018 if (!WARN_ON_ONCE(holder == NULL) && !--holder->refcnt) { 1019 del_symlink(disk->slave_dir, &part_to_dev(bdev->bd_part)->kobj); 1020 del_symlink(bdev->bd_part->holder_dir, 1021 &disk_to_dev(disk)->kobj); 1022 kobject_put(bdev->bd_part->holder_dir); 1023 list_del_init(&holder->list); 1024 kfree(holder); 1025 } 1026 1027 mutex_unlock(&bdev->bd_mutex); 1028} 1029EXPORT_SYMBOL_GPL(bd_unlink_disk_holder); 1030#endif 1031 1032/** 1033 * flush_disk - invalidates all buffer-cache entries on a disk 1034 * 1035 * @bdev: struct block device to be flushed 1036 * @kill_dirty: flag to guide handling of dirty inodes 1037 * 1038 * Invalidates all buffer-cache entries on a disk. It should be called 1039 * when a disk has been changed -- either by a media change or online 1040 * resize. 1041 */ 1042static void flush_disk(struct block_device *bdev, bool kill_dirty) 1043{ 1044 if (__invalidate_device(bdev, kill_dirty)) { 1045 char name[BDEVNAME_SIZE] = ""; 1046 1047 if (bdev->bd_disk) 1048 disk_name(bdev->bd_disk, 0, name); 1049 printk(KERN_WARNING "VFS: busy inodes on changed media or " 1050 "resized disk %s\n", name); 1051 } 1052 1053 if (!bdev->bd_disk) 1054 return; 1055 if (disk_part_scan_enabled(bdev->bd_disk)) 1056 bdev->bd_invalidated = 1; 1057} 1058 1059/** 1060 * check_disk_size_change - checks for disk size change and adjusts bdev size. 1061 * @disk: struct gendisk to check 1062 * @bdev: struct bdev to adjust. 1063 * 1064 * This routine checks to see if the bdev size does not match the disk size 1065 * and adjusts it if it differs. 1066 */ 1067void check_disk_size_change(struct gendisk *disk, struct block_device *bdev) 1068{ 1069 loff_t disk_size, bdev_size; 1070 1071 disk_size = (loff_t)get_capacity(disk) << 9; 1072 bdev_size = i_size_read(bdev->bd_inode); 1073 if (disk_size != bdev_size) { 1074 char name[BDEVNAME_SIZE]; 1075 1076 disk_name(disk, 0, name); 1077 printk(KERN_INFO 1078 "%s: detected capacity change from %lld to %lld\n", 1079 name, bdev_size, disk_size); 1080 i_size_write(bdev->bd_inode, disk_size); 1081 flush_disk(bdev, false); 1082 } 1083} 1084EXPORT_SYMBOL(check_disk_size_change); 1085 1086/** 1087 * revalidate_disk - wrapper for lower-level driver's revalidate_disk call-back 1088 * @disk: struct gendisk to be revalidated 1089 * 1090 * This routine is a wrapper for lower-level driver's revalidate_disk 1091 * call-backs. It is used to do common pre and post operations needed 1092 * for all revalidate_disk operations. 1093 */ 1094int revalidate_disk(struct gendisk *disk) 1095{ 1096 struct block_device *bdev; 1097 int ret = 0; 1098 1099 if (disk->fops->revalidate_disk) 1100 ret = disk->fops->revalidate_disk(disk); 1101 blk_integrity_revalidate(disk); 1102 bdev = bdget_disk(disk, 0); 1103 if (!bdev) 1104 return ret; 1105 1106 mutex_lock(&bdev->bd_mutex); 1107 check_disk_size_change(disk, bdev); 1108 bdev->bd_invalidated = 0; 1109 mutex_unlock(&bdev->bd_mutex); 1110 bdput(bdev); 1111 return ret; 1112} 1113EXPORT_SYMBOL(revalidate_disk); 1114 1115/* 1116 * This routine checks whether a removable media has been changed, 1117 * and invalidates all buffer-cache-entries in that case. This 1118 * is a relatively slow routine, so we have to try to minimize using 1119 * it. Thus it is called only upon a 'mount' or 'open'. This 1120 * is the best way of combining speed and utility, I think. 1121 * People changing diskettes in the middle of an operation deserve 1122 * to lose :-) 1123 */ 1124int check_disk_change(struct block_device *bdev) 1125{ 1126 struct gendisk *disk = bdev->bd_disk; 1127 const struct block_device_operations *bdops = disk->fops; 1128 unsigned int events; 1129 1130 events = disk_clear_events(disk, DISK_EVENT_MEDIA_CHANGE | 1131 DISK_EVENT_EJECT_REQUEST); 1132 if (!(events & DISK_EVENT_MEDIA_CHANGE)) 1133 return 0; 1134 1135 flush_disk(bdev, true); 1136 if (bdops->revalidate_disk) 1137 bdops->revalidate_disk(bdev->bd_disk); 1138 return 1; 1139} 1140 1141EXPORT_SYMBOL(check_disk_change); 1142 1143void bd_set_size(struct block_device *bdev, loff_t size) 1144{ 1145 unsigned bsize = bdev_logical_block_size(bdev); 1146 1147 mutex_lock(&bdev->bd_inode->i_mutex); 1148 i_size_write(bdev->bd_inode, size); 1149 mutex_unlock(&bdev->bd_inode->i_mutex); 1150 while (bsize < PAGE_CACHE_SIZE) { 1151 if (size & bsize) 1152 break; 1153 bsize <<= 1; 1154 } 1155 bdev->bd_block_size = bsize; 1156 bdev->bd_inode->i_blkbits = blksize_bits(bsize); 1157} 1158EXPORT_SYMBOL(bd_set_size); 1159 1160static void __blkdev_put(struct block_device *bdev, fmode_t mode, int for_part); 1161 1162/* 1163 * bd_mutex locking: 1164 * 1165 * mutex_lock(part->bd_mutex) 1166 * mutex_lock_nested(whole->bd_mutex, 1) 1167 */ 1168 1169static int __blkdev_get(struct block_device *bdev, fmode_t mode, int for_part) 1170{ 1171 struct gendisk *disk; 1172 struct module *owner; 1173 int ret; 1174 int partno; 1175 int perm = 0; 1176 1177 if (mode & FMODE_READ) 1178 perm |= MAY_READ; 1179 if (mode & FMODE_WRITE) 1180 perm |= MAY_WRITE; 1181 /* 1182 * hooks: /n/, see "layering violations". 1183 */ 1184 if (!for_part) { 1185 ret = devcgroup_inode_permission(bdev->bd_inode, perm); 1186 if (ret != 0) { 1187 bdput(bdev); 1188 return ret; 1189 } 1190 } 1191 1192 restart: 1193 1194 ret = -ENXIO; 1195 disk = get_gendisk(bdev->bd_dev, &partno); 1196 if (!disk) 1197 goto out; 1198 owner = disk->fops->owner; 1199 1200 disk_block_events(disk); 1201 mutex_lock_nested(&bdev->bd_mutex, for_part); 1202 if (!bdev->bd_openers) { 1203 bdev->bd_disk = disk; 1204 bdev->bd_queue = disk->queue; 1205 bdev->bd_contains = bdev; 1206 bdev->bd_inode->i_flags = disk->fops->direct_access ? S_DAX : 0; 1207 if (!partno) { 1208 ret = -ENXIO; 1209 bdev->bd_part = disk_get_part(disk, partno); 1210 if (!bdev->bd_part) 1211 goto out_clear; 1212 1213 ret = 0; 1214 if (disk->fops->open) { 1215 ret = disk->fops->open(bdev, mode); 1216 if (ret == -ERESTARTSYS) { 1217 /* Lost a race with 'disk' being 1218 * deleted, try again. 1219 * See md.c 1220 */ 1221 disk_put_part(bdev->bd_part); 1222 bdev->bd_part = NULL; 1223 bdev->bd_disk = NULL; 1224 bdev->bd_queue = NULL; 1225 mutex_unlock(&bdev->bd_mutex); 1226 disk_unblock_events(disk); 1227 put_disk(disk); 1228 module_put(owner); 1229 goto restart; 1230 } 1231 } 1232 1233 if (!ret) 1234 bd_set_size(bdev,(loff_t)get_capacity(disk)<<9); 1235 1236 /* 1237 * If the device is invalidated, rescan partition 1238 * if open succeeded or failed with -ENOMEDIUM. 1239 * The latter is necessary to prevent ghost 1240 * partitions on a removed medium. 1241 */ 1242 if (bdev->bd_invalidated) { 1243 if (!ret) 1244 rescan_partitions(disk, bdev); 1245 else if (ret == -ENOMEDIUM) 1246 invalidate_partitions(disk, bdev); 1247 } 1248 if (ret) 1249 goto out_clear; 1250 } else { 1251 struct block_device *whole; 1252 whole = bdget_disk(disk, 0); 1253 ret = -ENOMEM; 1254 if (!whole) 1255 goto out_clear; 1256 BUG_ON(for_part); 1257 ret = __blkdev_get(whole, mode, 1); 1258 if (ret) 1259 goto out_clear; 1260 bdev->bd_contains = whole; 1261 bdev->bd_part = disk_get_part(disk, partno); 1262 if (!(disk->flags & GENHD_FL_UP) || 1263 !bdev->bd_part || !bdev->bd_part->nr_sects) { 1264 ret = -ENXIO; 1265 goto out_clear; 1266 } 1267 bd_set_size(bdev, (loff_t)bdev->bd_part->nr_sects << 9); 1268 /* 1269 * If the partition is not aligned on a page 1270 * boundary, we can't do dax I/O to it. 1271 */ 1272 if ((bdev->bd_part->start_sect % (PAGE_SIZE / 512)) || 1273 (bdev->bd_part->nr_sects % (PAGE_SIZE / 512))) 1274 bdev->bd_inode->i_flags &= ~S_DAX; 1275 } 1276 } else { 1277 if (bdev->bd_contains == bdev) { 1278 ret = 0; 1279 if (bdev->bd_disk->fops->open) 1280 ret = bdev->bd_disk->fops->open(bdev, mode); 1281 /* the same as first opener case, read comment there */ 1282 if (bdev->bd_invalidated) { 1283 if (!ret) 1284 rescan_partitions(bdev->bd_disk, bdev); 1285 else if (ret == -ENOMEDIUM) 1286 invalidate_partitions(bdev->bd_disk, bdev); 1287 } 1288 if (ret) 1289 goto out_unlock_bdev; 1290 } 1291 /* only one opener holds refs to the module and disk */ 1292 put_disk(disk); 1293 module_put(owner); 1294 } 1295 bdev->bd_openers++; 1296 if (for_part) 1297 bdev->bd_part_count++; 1298 mutex_unlock(&bdev->bd_mutex); 1299 disk_unblock_events(disk); 1300 return 0; 1301 1302 out_clear: 1303 disk_put_part(bdev->bd_part); 1304 bdev->bd_disk = NULL; 1305 bdev->bd_part = NULL; 1306 bdev->bd_queue = NULL; 1307 if (bdev != bdev->bd_contains) 1308 __blkdev_put(bdev->bd_contains, mode, 1); 1309 bdev->bd_contains = NULL; 1310 out_unlock_bdev: 1311 mutex_unlock(&bdev->bd_mutex); 1312 disk_unblock_events(disk); 1313 put_disk(disk); 1314 module_put(owner); 1315 out: 1316 bdput(bdev); 1317 1318 return ret; 1319} 1320 1321/** 1322 * blkdev_get - open a block device 1323 * @bdev: block_device to open 1324 * @mode: FMODE_* mask 1325 * @holder: exclusive holder identifier 1326 * 1327 * Open @bdev with @mode. If @mode includes %FMODE_EXCL, @bdev is 1328 * open with exclusive access. Specifying %FMODE_EXCL with %NULL 1329 * @holder is invalid. Exclusive opens may nest for the same @holder. 1330 * 1331 * On success, the reference count of @bdev is unchanged. On failure, 1332 * @bdev is put. 1333 * 1334 * CONTEXT: 1335 * Might sleep. 1336 * 1337 * RETURNS: 1338 * 0 on success, -errno on failure. 1339 */ 1340int blkdev_get(struct block_device *bdev, fmode_t mode, void *holder) 1341{ 1342 struct block_device *whole = NULL; 1343 int res; 1344 1345 WARN_ON_ONCE((mode & FMODE_EXCL) && !holder); 1346 1347 if ((mode & FMODE_EXCL) && holder) { 1348 whole = bd_start_claiming(bdev, holder); 1349 if (IS_ERR(whole)) { 1350 bdput(bdev); 1351 return PTR_ERR(whole); 1352 } 1353 } 1354 1355 res = __blkdev_get(bdev, mode, 0); 1356 1357 if (whole) { 1358 struct gendisk *disk = whole->bd_disk; 1359 1360 /* finish claiming */ 1361 mutex_lock(&bdev->bd_mutex); 1362 spin_lock(&bdev_lock); 1363 1364 if (!res) { 1365 BUG_ON(!bd_may_claim(bdev, whole, holder)); 1366 /* 1367 * Note that for a whole device bd_holders 1368 * will be incremented twice, and bd_holder 1369 * will be set to bd_may_claim before being 1370 * set to holder 1371 */ 1372 whole->bd_holders++; 1373 whole->bd_holder = bd_may_claim; 1374 bdev->bd_holders++; 1375 bdev->bd_holder = holder; 1376 } 1377 1378 /* tell others that we're done */ 1379 BUG_ON(whole->bd_claiming != holder); 1380 whole->bd_claiming = NULL; 1381 wake_up_bit(&whole->bd_claiming, 0); 1382 1383 spin_unlock(&bdev_lock); 1384 1385 /* 1386 * Block event polling for write claims if requested. Any 1387 * write holder makes the write_holder state stick until 1388 * all are released. This is good enough and tracking 1389 * individual writeable reference is too fragile given the 1390 * way @mode is used in blkdev_get/put(). 1391 */ 1392 if (!res && (mode & FMODE_WRITE) && !bdev->bd_write_holder && 1393 (disk->flags & GENHD_FL_BLOCK_EVENTS_ON_EXCL_WRITE)) { 1394 bdev->bd_write_holder = true; 1395 disk_block_events(disk); 1396 } 1397 1398 mutex_unlock(&bdev->bd_mutex); 1399 bdput(whole); 1400 } 1401 1402 return res; 1403} 1404EXPORT_SYMBOL(blkdev_get); 1405 1406/** 1407 * blkdev_get_by_path - open a block device by name 1408 * @path: path to the block device to open 1409 * @mode: FMODE_* mask 1410 * @holder: exclusive holder identifier 1411 * 1412 * Open the blockdevice described by the device file at @path. @mode 1413 * and @holder are identical to blkdev_get(). 1414 * 1415 * On success, the returned block_device has reference count of one. 1416 * 1417 * CONTEXT: 1418 * Might sleep. 1419 * 1420 * RETURNS: 1421 * Pointer to block_device on success, ERR_PTR(-errno) on failure. 1422 */ 1423struct block_device *blkdev_get_by_path(const char *path, fmode_t mode, 1424 void *holder) 1425{ 1426 struct block_device *bdev; 1427 int err; 1428 1429 bdev = lookup_bdev(path); 1430 if (IS_ERR(bdev)) 1431 return bdev; 1432 1433 err = blkdev_get(bdev, mode, holder); 1434 if (err) 1435 return ERR_PTR(err); 1436 1437 if ((mode & FMODE_WRITE) && bdev_read_only(bdev)) { 1438 blkdev_put(bdev, mode); 1439 return ERR_PTR(-EACCES); 1440 } 1441 1442 return bdev; 1443} 1444EXPORT_SYMBOL(blkdev_get_by_path); 1445 1446/** 1447 * blkdev_get_by_dev - open a block device by device number 1448 * @dev: device number of block device to open 1449 * @mode: FMODE_* mask 1450 * @holder: exclusive holder identifier 1451 * 1452 * Open the blockdevice described by device number @dev. @mode and 1453 * @holder are identical to blkdev_get(). 1454 * 1455 * Use it ONLY if you really do not have anything better - i.e. when 1456 * you are behind a truly sucky interface and all you are given is a 1457 * device number. _Never_ to be used for internal purposes. If you 1458 * ever need it - reconsider your API. 1459 * 1460 * On success, the returned block_device has reference count of one. 1461 * 1462 * CONTEXT: 1463 * Might sleep. 1464 * 1465 * RETURNS: 1466 * Pointer to block_device on success, ERR_PTR(-errno) on failure. 1467 */ 1468struct block_device *blkdev_get_by_dev(dev_t dev, fmode_t mode, void *holder) 1469{ 1470 struct block_device *bdev; 1471 int err; 1472 1473 bdev = bdget(dev); 1474 if (!bdev) 1475 return ERR_PTR(-ENOMEM); 1476 1477 err = blkdev_get(bdev, mode, holder); 1478 if (err) 1479 return ERR_PTR(err); 1480 1481 return bdev; 1482} 1483EXPORT_SYMBOL(blkdev_get_by_dev); 1484 1485static int blkdev_open(struct inode * inode, struct file * filp) 1486{ 1487 struct block_device *bdev; 1488 1489 /* 1490 * Preserve backwards compatibility and allow large file access 1491 * even if userspace doesn't ask for it explicitly. Some mkfs 1492 * binary needs it. We might want to drop this workaround 1493 * during an unstable branch. 1494 */ 1495 filp->f_flags |= O_LARGEFILE; 1496 1497 if (filp->f_flags & O_NDELAY) 1498 filp->f_mode |= FMODE_NDELAY; 1499 if (filp->f_flags & O_EXCL) 1500 filp->f_mode |= FMODE_EXCL; 1501 if ((filp->f_flags & O_ACCMODE) == 3) 1502 filp->f_mode |= FMODE_WRITE_IOCTL; 1503 1504 bdev = bd_acquire(inode); 1505 if (bdev == NULL) 1506 return -ENOMEM; 1507 1508 filp->f_mapping = bdev->bd_inode->i_mapping; 1509 1510 return blkdev_get(bdev, filp->f_mode, filp); 1511} 1512 1513static void __blkdev_put(struct block_device *bdev, fmode_t mode, int for_part) 1514{ 1515 struct gendisk *disk = bdev->bd_disk; 1516 struct block_device *victim = NULL; 1517 1518 mutex_lock_nested(&bdev->bd_mutex, for_part); 1519 if (for_part) 1520 bdev->bd_part_count--; 1521 1522 if (!--bdev->bd_openers) { 1523 WARN_ON_ONCE(bdev->bd_holders); 1524 sync_blockdev(bdev); 1525 kill_bdev(bdev); 1526 1527 bdev_write_inode(bdev); 1528 /* 1529 * Detaching bdev inode from its wb in __destroy_inode() 1530 * is too late: the queue which embeds its bdi (along with 1531 * root wb) can be gone as soon as we put_disk() below. 1532 */ 1533 inode_detach_wb(bdev->bd_inode); 1534 } 1535 if (bdev->bd_contains == bdev) { 1536 if (disk->fops->release) 1537 disk->fops->release(disk, mode); 1538 } 1539 if (!bdev->bd_openers) { 1540 struct module *owner = disk->fops->owner; 1541 1542 disk_put_part(bdev->bd_part); 1543 bdev->bd_part = NULL; 1544 bdev->bd_disk = NULL; 1545 if (bdev != bdev->bd_contains) 1546 victim = bdev->bd_contains; 1547 bdev->bd_contains = NULL; 1548 1549 put_disk(disk); 1550 module_put(owner); 1551 } 1552 mutex_unlock(&bdev->bd_mutex); 1553 bdput(bdev); 1554 if (victim) 1555 __blkdev_put(victim, mode, 1); 1556} 1557 1558void blkdev_put(struct block_device *bdev, fmode_t mode) 1559{ 1560 mutex_lock(&bdev->bd_mutex); 1561 1562 if (mode & FMODE_EXCL) { 1563 bool bdev_free; 1564 1565 /* 1566 * Release a claim on the device. The holder fields 1567 * are protected with bdev_lock. bd_mutex is to 1568 * synchronize disk_holder unlinking. 1569 */ 1570 spin_lock(&bdev_lock); 1571 1572 WARN_ON_ONCE(--bdev->bd_holders < 0); 1573 WARN_ON_ONCE(--bdev->bd_contains->bd_holders < 0); 1574 1575 /* bd_contains might point to self, check in a separate step */ 1576 if ((bdev_free = !bdev->bd_holders)) 1577 bdev->bd_holder = NULL; 1578 if (!bdev->bd_contains->bd_holders) 1579 bdev->bd_contains->bd_holder = NULL; 1580 1581 spin_unlock(&bdev_lock); 1582 1583 /* 1584 * If this was the last claim, remove holder link and 1585 * unblock evpoll if it was a write holder. 1586 */ 1587 if (bdev_free && bdev->bd_write_holder) { 1588 disk_unblock_events(bdev->bd_disk); 1589 bdev->bd_write_holder = false; 1590 } 1591 } 1592 1593 /* 1594 * Trigger event checking and tell drivers to flush MEDIA_CHANGE 1595 * event. This is to ensure detection of media removal commanded 1596 * from userland - e.g. eject(1). 1597 */ 1598 disk_flush_events(bdev->bd_disk, DISK_EVENT_MEDIA_CHANGE); 1599 1600 mutex_unlock(&bdev->bd_mutex); 1601 1602 __blkdev_put(bdev, mode, 0); 1603} 1604EXPORT_SYMBOL(blkdev_put); 1605 1606static int blkdev_close(struct inode * inode, struct file * filp) 1607{ 1608 struct block_device *bdev = I_BDEV(filp->f_mapping->host); 1609 blkdev_put(bdev, filp->f_mode); 1610 return 0; 1611} 1612 1613static long block_ioctl(struct file *file, unsigned cmd, unsigned long arg) 1614{ 1615 struct block_device *bdev = I_BDEV(file->f_mapping->host); 1616 fmode_t mode = file->f_mode; 1617 1618 /* 1619 * O_NDELAY can be altered using fcntl(.., F_SETFL, ..), so we have 1620 * to updated it before every ioctl. 1621 */ 1622 if (file->f_flags & O_NDELAY) 1623 mode |= FMODE_NDELAY; 1624 else 1625 mode &= ~FMODE_NDELAY; 1626 1627 return blkdev_ioctl(bdev, mode, cmd, arg); 1628} 1629 1630/* 1631 * Write data to the block device. Only intended for the block device itself 1632 * and the raw driver which basically is a fake block device. 1633 * 1634 * Does not take i_mutex for the write and thus is not for general purpose 1635 * use. 1636 */ 1637ssize_t blkdev_write_iter(struct kiocb *iocb, struct iov_iter *from) 1638{ 1639 struct file *file = iocb->ki_filp; 1640 struct inode *bd_inode = file->f_mapping->host; 1641 loff_t size = i_size_read(bd_inode); 1642 struct blk_plug plug; 1643 ssize_t ret; 1644 1645 if (bdev_read_only(I_BDEV(bd_inode))) 1646 return -EPERM; 1647 1648 if (!iov_iter_count(from)) 1649 return 0; 1650 1651 if (iocb->ki_pos >= size) 1652 return -ENOSPC; 1653 1654 iov_iter_truncate(from, size - iocb->ki_pos); 1655 1656 blk_start_plug(&plug); 1657 ret = __generic_file_write_iter(iocb, from); 1658 if (ret > 0) { 1659 ssize_t err; 1660 err = generic_write_sync(file, iocb->ki_pos - ret, ret); 1661 if (err < 0) 1662 ret = err; 1663 } 1664 blk_finish_plug(&plug); 1665 return ret; 1666} 1667EXPORT_SYMBOL_GPL(blkdev_write_iter); 1668 1669ssize_t blkdev_read_iter(struct kiocb *iocb, struct iov_iter *to) 1670{ 1671 struct file *file = iocb->ki_filp; 1672 struct inode *bd_inode = file->f_mapping->host; 1673 loff_t size = i_size_read(bd_inode); 1674 loff_t pos = iocb->ki_pos; 1675 1676 if (pos >= size) 1677 return 0; 1678 1679 size -= pos; 1680 iov_iter_truncate(to, size); 1681 return generic_file_read_iter(iocb, to); 1682} 1683EXPORT_SYMBOL_GPL(blkdev_read_iter); 1684 1685/* 1686 * Try to release a page associated with block device when the system 1687 * is under memory pressure. 1688 */ 1689static int blkdev_releasepage(struct page *page, gfp_t wait) 1690{ 1691 struct super_block *super = BDEV_I(page->mapping->host)->bdev.bd_super; 1692 1693 if (super && super->s_op->bdev_try_to_free_page) 1694 return super->s_op->bdev_try_to_free_page(super, page, wait); 1695 1696 return try_to_free_buffers(page); 1697} 1698 1699static const struct address_space_operations def_blk_aops = { 1700 .readpage = blkdev_readpage, 1701 .readpages = blkdev_readpages, 1702 .writepage = blkdev_writepage, 1703 .write_begin = blkdev_write_begin, 1704 .write_end = blkdev_write_end, 1705 .writepages = generic_writepages, 1706 .releasepage = blkdev_releasepage, 1707 .direct_IO = blkdev_direct_IO, 1708 .is_dirty_writeback = buffer_check_dirty_writeback, 1709}; 1710 1711const struct file_operations def_blk_fops = { 1712 .open = blkdev_open, 1713 .release = blkdev_close, 1714 .llseek = block_llseek, 1715 .read_iter = blkdev_read_iter, 1716 .write_iter = blkdev_write_iter, 1717 .mmap = generic_file_mmap, 1718 .fsync = blkdev_fsync, 1719 .unlocked_ioctl = block_ioctl, 1720#ifdef CONFIG_COMPAT 1721 .compat_ioctl = compat_blkdev_ioctl, 1722#endif 1723 .splice_read = generic_file_splice_read, 1724 .splice_write = iter_file_splice_write, 1725}; 1726 1727int ioctl_by_bdev(struct block_device *bdev, unsigned cmd, unsigned long arg) 1728{ 1729 int res; 1730 mm_segment_t old_fs = get_fs(); 1731 set_fs(KERNEL_DS); 1732 res = blkdev_ioctl(bdev, 0, cmd, arg); 1733 set_fs(old_fs); 1734 return res; 1735} 1736 1737EXPORT_SYMBOL(ioctl_by_bdev); 1738 1739/** 1740 * lookup_bdev - lookup a struct block_device by name 1741 * @pathname: special file representing the block device 1742 * 1743 * Get a reference to the blockdevice at @pathname in the current 1744 * namespace if possible and return it. Return ERR_PTR(error) 1745 * otherwise. 1746 */ 1747struct block_device *lookup_bdev(const char *pathname) 1748{ 1749 struct block_device *bdev; 1750 struct inode *inode; 1751 struct path path; 1752 int error; 1753 1754 if (!pathname || !*pathname) 1755 return ERR_PTR(-EINVAL); 1756 1757 error = kern_path(pathname, LOOKUP_FOLLOW, &path); 1758 if (error) 1759 return ERR_PTR(error); 1760 1761 inode = d_backing_inode(path.dentry); 1762 error = -ENOTBLK; 1763 if (!S_ISBLK(inode->i_mode)) 1764 goto fail; 1765 error = -EACCES; 1766 if (path.mnt->mnt_flags & MNT_NODEV) 1767 goto fail; 1768 error = -ENOMEM; 1769 bdev = bd_acquire(inode); 1770 if (!bdev) 1771 goto fail; 1772out: 1773 path_put(&path); 1774 return bdev; 1775fail: 1776 bdev = ERR_PTR(error); 1777 goto out; 1778} 1779EXPORT_SYMBOL(lookup_bdev); 1780 1781int __invalidate_device(struct block_device *bdev, bool kill_dirty) 1782{ 1783 struct super_block *sb = get_super(bdev); 1784 int res = 0; 1785 1786 if (sb) { 1787 /* 1788 * no need to lock the super, get_super holds the 1789 * read mutex so the filesystem cannot go away 1790 * under us (->put_super runs with the write lock 1791 * hold). 1792 */ 1793 shrink_dcache_sb(sb); 1794 res = invalidate_inodes(sb, kill_dirty); 1795 drop_super(sb); 1796 } 1797 invalidate_bdev(bdev); 1798 return res; 1799} 1800EXPORT_SYMBOL(__invalidate_device); 1801 1802void iterate_bdevs(void (*func)(struct block_device *, void *), void *arg) 1803{ 1804 struct inode *inode, *old_inode = NULL; 1805 1806 spin_lock(&blockdev_superblock->s_inode_list_lock); 1807 list_for_each_entry(inode, &blockdev_superblock->s_inodes, i_sb_list) { 1808 struct address_space *mapping = inode->i_mapping; 1809 1810 spin_lock(&inode->i_lock); 1811 if (inode->i_state & (I_FREEING|I_WILL_FREE|I_NEW) || 1812 mapping->nrpages == 0) { 1813 spin_unlock(&inode->i_lock); 1814 continue; 1815 } 1816 __iget(inode); 1817 spin_unlock(&inode->i_lock); 1818 spin_unlock(&blockdev_superblock->s_inode_list_lock); 1819 /* 1820 * We hold a reference to 'inode' so it couldn't have been 1821 * removed from s_inodes list while we dropped the 1822 * s_inode_list_lock We cannot iput the inode now as we can 1823 * be holding the last reference and we cannot iput it under 1824 * s_inode_list_lock. So we keep the reference and iput it 1825 * later. 1826 */ 1827 iput(old_inode); 1828 old_inode = inode; 1829 1830 func(I_BDEV(inode), arg); 1831 1832 spin_lock(&blockdev_superblock->s_inode_list_lock); 1833 } 1834 spin_unlock(&blockdev_superblock->s_inode_list_lock); 1835 iput(old_inode); 1836} 1837