root/fs/btrfs/file.c

DEFINITIONS

1. __compare_inode_defrag
2. __btrfs_add_inode_defrag
3. __need_auto_defrag
4. btrfs_add_inode_defrag
5. btrfs_requeue_inode_defrag
6. btrfs_pick_defrag_inode
7. btrfs_cleanup_defrag_inodes
8. __btrfs_run_defrag_inode
9. btrfs_run_defrag_inodes
10. btrfs_copy_from_user
11. btrfs_drop_pages
12. btrfs_find_new_delalloc_bytes
13. btrfs_dirty_pages
14. btrfs_drop_extent_cache
15. __btrfs_drop_extents
16. btrfs_drop_extents
17. extent_mergeable
18. btrfs_mark_extent_written
19. prepare_uptodate_page
20. prepare_pages
21. lock_and_cleanup_extent_if_need
22. check_can_nocow
23. btrfs_buffered_write
24. __btrfs_direct_write
25. update_time_for_write
26. btrfs_file_write_iter
27. btrfs_release_file
28. start_ordered_ops
29. btrfs_sync_file
30. btrfs_file_mmap
31. hole_mergeable
32. fill_holes
33. find_first_non_hole
34. btrfs_punch_hole_lock_range
35. btrfs_insert_clone_extent
36. btrfs_punch_hole_range
37. btrfs_punch_hole
38. add_falloc_range
39. btrfs_fallocate_update_isize
40. btrfs_zero_range_check_range_boundary
41. btrfs_zero_range
42. btrfs_fallocate
43. find_desired_extent
44. btrfs_file_llseek
45. btrfs_file_open
46. btrfs_auto_defrag_exit
47. btrfs_auto_defrag_init
48. btrfs_fdatawrite_range

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Copyright (C) 2007 Oracle.  All rights reserved.
   4  */
   5 
   6 #include <linux/fs.h>
   7 #include <linux/pagemap.h>
   8 #include <linux/time.h>
   9 #include <linux/init.h>
  10 #include <linux/string.h>
  11 #include <linux/backing-dev.h>
  12 #include <linux/falloc.h>
  13 #include <linux/writeback.h>
  14 #include <linux/compat.h>
  15 #include <linux/slab.h>
  16 #include <linux/btrfs.h>
  17 #include <linux/uio.h>
  18 #include <linux/iversion.h>
  19 #include "ctree.h"
  20 #include "disk-io.h"
  21 #include "transaction.h"
  22 #include "btrfs_inode.h"
  23 #include "print-tree.h"
  24 #include "tree-log.h"
  25 #include "locking.h"
  26 #include "volumes.h"
  27 #include "qgroup.h"
  28 #include "compression.h"
  29 #include "delalloc-space.h"
  30 
  31 static struct kmem_cache *btrfs_inode_defrag_cachep;
  32 /*
  33  * when auto defrag is enabled we
  34  * queue up these defrag structs to remember which
  35  * inodes need defragging passes
  36  */
  37 struct inode_defrag {
  38         struct rb_node rb_node;
  39         /* objectid */
  40         u64 ino;
  41         /*
  42          * transid where the defrag was added, we search for
  43          * extents newer than this
  44          */
  45         u64 transid;
  46 
  47         /* root objectid */
  48         u64 root;
  49 
  50         /* last offset we were able to defrag */
  51         u64 last_offset;
  52 
  53         /* if we've wrapped around back to zero once already */
  54         int cycled;
  55 };
  56 
  57 static int __compare_inode_defrag(struct inode_defrag *defrag1,
  58                                   struct inode_defrag *defrag2)
  59 {
  60         if (defrag1->root > defrag2->root)
  61                 return 1;
  62         else if (defrag1->root < defrag2->root)
  63                 return -1;
  64         else if (defrag1->ino > defrag2->ino)
  65                 return 1;
  66         else if (defrag1->ino < defrag2->ino)
  67                 return -1;
  68         else
  69                 return 0;
  70 }
  71 
  72 /* pop a record for an inode into the defrag tree.  The lock
  73  * must be held already
  74  *
  75  * If you're inserting a record for an older transid than an
  76  * existing record, the transid already in the tree is lowered
  77  *
  78  * If an existing record is found the defrag item you
  79  * pass in is freed
  80  */
  81 static int __btrfs_add_inode_defrag(struct btrfs_inode *inode,
  82                                     struct inode_defrag *defrag)
  83 {
  84         struct btrfs_fs_info *fs_info = inode->root->fs_info;
  85         struct inode_defrag *entry;
  86         struct rb_node **p;
  87         struct rb_node *parent = NULL;
  88         int ret;
  89 
  90         p = &fs_info->defrag_inodes.rb_node;
  91         while (*p) {
  92                 parent = *p;
  93                 entry = rb_entry(parent, struct inode_defrag, rb_node);
  94 
  95                 ret = __compare_inode_defrag(defrag, entry);
  96                 if (ret < 0)
  97                         p = &parent->rb_left;
  98                 else if (ret > 0)
  99                         p = &parent->rb_right;
 100                 else {
 101                         /* if we're reinserting an entry for
 102                          * an old defrag run, make sure to
 103                          * lower the transid of our existing record
 104                          */
 105                         if (defrag->transid < entry->transid)
 106                                 entry->transid = defrag->transid;
 107                         if (defrag->last_offset > entry->last_offset)
 108                                 entry->last_offset = defrag->last_offset;
 109                         return -EEXIST;
 110                 }
 111         }
 112         set_bit(BTRFS_INODE_IN_DEFRAG, &inode->runtime_flags);
 113         rb_link_node(&defrag->rb_node, parent, p);
 114         rb_insert_color(&defrag->rb_node, &fs_info->defrag_inodes);
 115         return 0;
 116 }
 117 
 118 static inline int __need_auto_defrag(struct btrfs_fs_info *fs_info)
 119 {
 120         if (!btrfs_test_opt(fs_info, AUTO_DEFRAG))
 121                 return 0;
 122 
 123         if (btrfs_fs_closing(fs_info))
 124                 return 0;
 125 
 126         return 1;
 127 }
 128 
 129 /*
 130  * insert a defrag record for this inode if auto defrag is
 131  * enabled
 132  */
 133 int btrfs_add_inode_defrag(struct btrfs_trans_handle *trans,
 134                            struct btrfs_inode *inode)
 135 {
 136         struct btrfs_root *root = inode->root;
 137         struct btrfs_fs_info *fs_info = root->fs_info;
 138         struct inode_defrag *defrag;
 139         u64 transid;
 140         int ret;
 141 
 142         if (!__need_auto_defrag(fs_info))
 143                 return 0;
 144 
 145         if (test_bit(BTRFS_INODE_IN_DEFRAG, &inode->runtime_flags))
 146                 return 0;
 147 
 148         if (trans)
 149                 transid = trans->transid;
 150         else
 151                 transid = inode->root->last_trans;
 152 
 153         defrag = kmem_cache_zalloc(btrfs_inode_defrag_cachep, GFP_NOFS);
 154         if (!defrag)
 155                 return -ENOMEM;
 156 
 157         defrag->ino = btrfs_ino(inode);
 158         defrag->transid = transid;
 159         defrag->root = root->root_key.objectid;
 160 
 161         spin_lock(&fs_info->defrag_inodes_lock);
 162         if (!test_bit(BTRFS_INODE_IN_DEFRAG, &inode->runtime_flags)) {
 163                 /*
 164                  * If we set IN_DEFRAG flag and evict the inode from memory,
 165                  * and then re-read this inode, this new inode doesn't have
 166                  * IN_DEFRAG flag. At the case, we may find the existed defrag.
 167                  */
 168                 ret = __btrfs_add_inode_defrag(inode, defrag);
 169                 if (ret)
 170                         kmem_cache_free(btrfs_inode_defrag_cachep, defrag);
 171         } else {
 172                 kmem_cache_free(btrfs_inode_defrag_cachep, defrag);
 173         }
 174         spin_unlock(&fs_info->defrag_inodes_lock);
 175         return 0;
 176 }
 177 
 178 /*
 179  * Requeue the defrag object. If there is a defrag object that points to
 180  * the same inode in the tree, we will merge them together (by
 181  * __btrfs_add_inode_defrag()) and free the one that we want to requeue.
 182  */
 183 static void btrfs_requeue_inode_defrag(struct btrfs_inode *inode,
 184                                        struct inode_defrag *defrag)
 185 {
 186         struct btrfs_fs_info *fs_info = inode->root->fs_info;
 187         int ret;
 188 
 189         if (!__need_auto_defrag(fs_info))
 190                 goto out;
 191 
 192         /*
 193          * Here we don't check the IN_DEFRAG flag, because we need merge
 194          * them together.
 195          */
 196         spin_lock(&fs_info->defrag_inodes_lock);
 197         ret = __btrfs_add_inode_defrag(inode, defrag);
 198         spin_unlock(&fs_info->defrag_inodes_lock);
 199         if (ret)
 200                 goto out;
 201         return;
 202 out:
 203         kmem_cache_free(btrfs_inode_defrag_cachep, defrag);
 204 }
 205 
 206 /*
 207  * pick the defragable inode that we want, if it doesn't exist, we will get
 208  * the next one.
 209  */
 210 static struct inode_defrag *
 211 btrfs_pick_defrag_inode(struct btrfs_fs_info *fs_info, u64 root, u64 ino)
 212 {
 213         struct inode_defrag *entry = NULL;
 214         struct inode_defrag tmp;
 215         struct rb_node *p;
 216         struct rb_node *parent = NULL;
 217         int ret;
 218 
 219         tmp.ino = ino;
 220         tmp.root = root;
 221 
 222         spin_lock(&fs_info->defrag_inodes_lock);
 223         p = fs_info->defrag_inodes.rb_node;
 224         while (p) {
 225                 parent = p;
 226                 entry = rb_entry(parent, struct inode_defrag, rb_node);
 227 
 228                 ret = __compare_inode_defrag(&tmp, entry);
 229                 if (ret < 0)
 230                         p = parent->rb_left;
 231                 else if (ret > 0)
 232                         p = parent->rb_right;
 233                 else
 234                         goto out;
 235         }
 236 
 237         if (parent && __compare_inode_defrag(&tmp, entry) > 0) {
 238                 parent = rb_next(parent);
 239                 if (parent)
 240                         entry = rb_entry(parent, struct inode_defrag, rb_node);
 241                 else
 242                         entry = NULL;
 243         }
 244 out:
 245         if (entry)
 246                 rb_erase(parent, &fs_info->defrag_inodes);
 247         spin_unlock(&fs_info->defrag_inodes_lock);
 248         return entry;
 249 }
 250 
 251 void btrfs_cleanup_defrag_inodes(struct btrfs_fs_info *fs_info)
 252 {
 253         struct inode_defrag *defrag;
 254         struct rb_node *node;
 255 
 256         spin_lock(&fs_info->defrag_inodes_lock);
 257         node = rb_first(&fs_info->defrag_inodes);
 258         while (node) {
 259                 rb_erase(node, &fs_info->defrag_inodes);
 260                 defrag = rb_entry(node, struct inode_defrag, rb_node);
 261                 kmem_cache_free(btrfs_inode_defrag_cachep, defrag);
 262 
 263                 cond_resched_lock(&fs_info->defrag_inodes_lock);
 264 
 265                 node = rb_first(&fs_info->defrag_inodes);
 266         }
 267         spin_unlock(&fs_info->defrag_inodes_lock);
 268 }
 269 
 270 #define BTRFS_DEFRAG_BATCH      1024
 271 
 272 static int __btrfs_run_defrag_inode(struct btrfs_fs_info *fs_info,
 273                                     struct inode_defrag *defrag)
 274 {
 275         struct btrfs_root *inode_root;
 276         struct inode *inode;
 277         struct btrfs_key key;
 278         struct btrfs_ioctl_defrag_range_args range;
 279         int num_defrag;
 280         int index;
 281         int ret;
 282 
 283         /* get the inode */
 284         key.objectid = defrag->root;
 285         key.type = BTRFS_ROOT_ITEM_KEY;
 286         key.offset = (u64)-1;
 287 
 288         index = srcu_read_lock(&fs_info->subvol_srcu);
 289 
 290         inode_root = btrfs_read_fs_root_no_name(fs_info, &key);
 291         if (IS_ERR(inode_root)) {
 292                 ret = PTR_ERR(inode_root);
 293                 goto cleanup;
 294         }
 295 
 296         key.objectid = defrag->ino;
 297         key.type = BTRFS_INODE_ITEM_KEY;
 298         key.offset = 0;
 299         inode = btrfs_iget(fs_info->sb, &key, inode_root, NULL);
 300         if (IS_ERR(inode)) {
 301                 ret = PTR_ERR(inode);
 302                 goto cleanup;
 303         }
 304         srcu_read_unlock(&fs_info->subvol_srcu, index);
 305 
 306         /* do a chunk of defrag */
 307         clear_bit(BTRFS_INODE_IN_DEFRAG, &BTRFS_I(inode)->runtime_flags);
 308         memset(&range, 0, sizeof(range));
 309         range.len = (u64)-1;
 310         range.start = defrag->last_offset;
 311 
 312         sb_start_write(fs_info->sb);
 313         num_defrag = btrfs_defrag_file(inode, NULL, &range, defrag->transid,
 314                                        BTRFS_DEFRAG_BATCH);
 315         sb_end_write(fs_info->sb);
 316         /*
 317          * if we filled the whole defrag batch, there
 318          * must be more work to do.  Queue this defrag
 319          * again
 320          */
 321         if (num_defrag == BTRFS_DEFRAG_BATCH) {
 322                 defrag->last_offset = range.start;
 323                 btrfs_requeue_inode_defrag(BTRFS_I(inode), defrag);
 324         } else if (defrag->last_offset && !defrag->cycled) {
 325                 /*
 326                  * we didn't fill our defrag batch, but
 327                  * we didn't start at zero.  Make sure we loop
 328                  * around to the start of the file.
 329                  */
 330                 defrag->last_offset = 0;
 331                 defrag->cycled = 1;
 332                 btrfs_requeue_inode_defrag(BTRFS_I(inode), defrag);
 333         } else {
 334                 kmem_cache_free(btrfs_inode_defrag_cachep, defrag);
 335         }
 336 
 337         iput(inode);
 338         return 0;
 339 cleanup:
 340         srcu_read_unlock(&fs_info->subvol_srcu, index);
 341         kmem_cache_free(btrfs_inode_defrag_cachep, defrag);
 342         return ret;
 343 }
 344 
 345 /*
 346  * run through the list of inodes in the FS that need
 347  * defragging
 348  */
 349 int btrfs_run_defrag_inodes(struct btrfs_fs_info *fs_info)
 350 {
 351         struct inode_defrag *defrag;
 352         u64 first_ino = 0;
 353         u64 root_objectid = 0;
 354 
 355         atomic_inc(&fs_info->defrag_running);
 356         while (1) {
 357                 /* Pause the auto defragger. */
 358                 if (test_bit(BTRFS_FS_STATE_REMOUNTING,
 359                              &fs_info->fs_state))
 360                         break;
 361 
 362                 if (!__need_auto_defrag(fs_info))
 363                         break;
 364 
 365                 /* find an inode to defrag */
 366                 defrag = btrfs_pick_defrag_inode(fs_info, root_objectid,
 367                                                  first_ino);
 368                 if (!defrag) {
 369                         if (root_objectid || first_ino) {
 370                                 root_objectid = 0;
 371                                 first_ino = 0;
 372                                 continue;
 373                         } else {
 374                                 break;
 375                         }
 376                 }
 377 
 378                 first_ino = defrag->ino + 1;
 379                 root_objectid = defrag->root;
 380 
 381                 __btrfs_run_defrag_inode(fs_info, defrag);
 382         }
 383         atomic_dec(&fs_info->defrag_running);
 384 
 385         /*
 386          * during unmount, we use the transaction_wait queue to
 387          * wait for the defragger to stop
 388          */
 389         wake_up(&fs_info->transaction_wait);
 390         return 0;
 391 }
 392 
 393 /* simple helper to fault in pages and copy.  This should go away
 394  * and be replaced with calls into generic code.
 395  */
 396 static noinline int btrfs_copy_from_user(loff_t pos, size_t write_bytes,
 397                                          struct page **prepared_pages,
 398                                          struct iov_iter *i)
 399 {
 400         size_t copied = 0;
 401         size_t total_copied = 0;
 402         int pg = 0;
 403         int offset = offset_in_page(pos);
 404 
 405         while (write_bytes > 0) {
 406                 size_t count = min_t(size_t,
 407                                      PAGE_SIZE - offset, write_bytes);
 408                 struct page *page = prepared_pages[pg];
 409                 /*
 410                  * Copy data from userspace to the current page
 411                  */
 412                 copied = iov_iter_copy_from_user_atomic(page, i, offset, count);
 413 
 414                 /* Flush processor's dcache for this page */
 415                 flush_dcache_page(page);
 416 
 417                 /*
 418                  * if we get a partial write, we can end up with
 419                  * partially up to date pages.  These add
 420                  * a lot of complexity, so make sure they don't
 421                  * happen by forcing this copy to be retried.
 422                  *
 423                  * The rest of the btrfs_file_write code will fall
 424                  * back to page at a time copies after we return 0.
 425                  */
 426                 if (!PageUptodate(page) && copied < count)
 427                         copied = 0;
 428 
 429                 iov_iter_advance(i, copied);
 430                 write_bytes -= copied;
 431                 total_copied += copied;
 432 
 433                 /* Return to btrfs_file_write_iter to fault page */
 434                 if (unlikely(copied == 0))
 435                         break;
 436 
 437                 if (copied < PAGE_SIZE - offset) {
 438                         offset += copied;
 439                 } else {
 440                         pg++;
 441                         offset = 0;
 442                 }
 443         }
 444         return total_copied;
 445 }
 446 
 447 /*
 448  * unlocks pages after btrfs_file_write is done with them
 449  */
 450 static void btrfs_drop_pages(struct page **pages, size_t num_pages)
 451 {
 452         size_t i;
 453         for (i = 0; i < num_pages; i++) {
 454                 /* page checked is some magic around finding pages that
 455                  * have been modified without going through btrfs_set_page_dirty
 456                  * clear it here. There should be no need to mark the pages
 457                  * accessed as prepare_pages should have marked them accessed
 458                  * in prepare_pages via find_or_create_page()
 459                  */
 460                 ClearPageChecked(pages[i]);
 461                 unlock_page(pages[i]);
 462                 put_page(pages[i]);
 463         }
 464 }
 465 
 466 static int btrfs_find_new_delalloc_bytes(struct btrfs_inode *inode,
 467                                          const u64 start,
 468                                          const u64 len,
 469                                          struct extent_state **cached_state)
 470 {
 471         u64 search_start = start;
 472         const u64 end = start + len - 1;
 473 
 474         while (search_start < end) {
 475                 const u64 search_len = end - search_start + 1;
 476                 struct extent_map *em;
 477                 u64 em_len;
 478                 int ret = 0;
 479 
 480                 em = btrfs_get_extent(inode, NULL, 0, search_start,
 481                                       search_len, 0);
 482                 if (IS_ERR(em))
 483                         return PTR_ERR(em);
 484 
 485                 if (em->block_start != EXTENT_MAP_HOLE)
 486                         goto next;
 487 
 488                 em_len = em->len;
 489                 if (em->start < search_start)
 490                         em_len -= search_start - em->start;
 491                 if (em_len > search_len)
 492                         em_len = search_len;
 493 
 494                 ret = set_extent_bit(&inode->io_tree, search_start,
 495                                      search_start + em_len - 1,
 496                                      EXTENT_DELALLOC_NEW,
 497                                      NULL, cached_state, GFP_NOFS);
 498 next:
 499                 search_start = extent_map_end(em);
 500                 free_extent_map(em);
 501                 if (ret)
 502                         return ret;
 503         }
 504         return 0;
 505 }
 506 
 507 /*
 508  * after copy_from_user, pages need to be dirtied and we need to make
 509  * sure holes are created between the current EOF and the start of
 510  * any next extents (if required).
 511  *
 512  * this also makes the decision about creating an inline extent vs
 513  * doing real data extents, marking pages dirty and delalloc as required.
 514  */
 515 int btrfs_dirty_pages(struct inode *inode, struct page **pages,
 516                       size_t num_pages, loff_t pos, size_t write_bytes,
 517                       struct extent_state **cached)
 518 {
 519         struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
 520         int err = 0;
 521         int i;
 522         u64 num_bytes;
 523         u64 start_pos;
 524         u64 end_of_last_block;
 525         u64 end_pos = pos + write_bytes;
 526         loff_t isize = i_size_read(inode);
 527         unsigned int extra_bits = 0;
 528 
 529         start_pos = pos & ~((u64) fs_info->sectorsize - 1);
 530         num_bytes = round_up(write_bytes + pos - start_pos,
 531                              fs_info->sectorsize);
 532 
 533         end_of_last_block = start_pos + num_bytes - 1;
 534 
 535         /*
 536          * The pages may have already been dirty, clear out old accounting so
 537          * we can set things up properly
 538          */
 539         clear_extent_bit(&BTRFS_I(inode)->io_tree, start_pos, end_of_last_block,
 540                          EXTENT_DELALLOC | EXTENT_DO_ACCOUNTING | EXTENT_DEFRAG,
 541                          0, 0, cached);
 542 
 543         if (!btrfs_is_free_space_inode(BTRFS_I(inode))) {
 544                 if (start_pos >= isize &&
 545                     !(BTRFS_I(inode)->flags & BTRFS_INODE_PREALLOC)) {
 546                         /*
 547                          * There can't be any extents following eof in this case
 548                          * so just set the delalloc new bit for the range
 549                          * directly.
 550                          */
 551                         extra_bits |= EXTENT_DELALLOC_NEW;
 552                 } else {
 553                         err = btrfs_find_new_delalloc_bytes(BTRFS_I(inode),
 554                                                             start_pos,
 555                                                             num_bytes, cached);
 556                         if (err)
 557                                 return err;
 558                 }
 559         }
 560 
 561         err = btrfs_set_extent_delalloc(inode, start_pos, end_of_last_block,
 562                                         extra_bits, cached);
 563         if (err)
 564                 return err;
 565 
 566         for (i = 0; i < num_pages; i++) {
 567                 struct page *p = pages[i];
 568                 SetPageUptodate(p);
 569                 ClearPageChecked(p);
 570                 set_page_dirty(p);
 571         }
 572 
 573         /*
 574          * we've only changed i_size in ram, and we haven't updated
 575          * the disk i_size.  There is no need to log the inode
 576          * at this time.
 577          */
 578         if (end_pos > isize)
 579                 i_size_write(inode, end_pos);
 580         return 0;
 581 }
 582 
 583 /*
 584  * this drops all the extents in the cache that intersect the range
 585  * [start, end].  Existing extents are split as required.
 586  */
 587 void btrfs_drop_extent_cache(struct btrfs_inode *inode, u64 start, u64 end,
 588                              int skip_pinned)
 589 {
 590         struct extent_map *em;
 591         struct extent_map *split = NULL;
 592         struct extent_map *split2 = NULL;
 593         struct extent_map_tree *em_tree = &inode->extent_tree;
 594         u64 len = end - start + 1;
 595         u64 gen;
 596         int ret;
 597         int testend = 1;
 598         unsigned long flags;
 599         int compressed = 0;
 600         bool modified;
 601 
 602         WARN_ON(end < start);
 603         if (end == (u64)-1) {
 604                 len = (u64)-1;
 605                 testend = 0;
 606         }
 607         while (1) {
 608                 int no_splits = 0;
 609 
 610                 modified = false;
 611                 if (!split)
 612                         split = alloc_extent_map();
 613                 if (!split2)
 614                         split2 = alloc_extent_map();
 615                 if (!split || !split2)
 616                         no_splits = 1;
 617 
 618                 write_lock(&em_tree->lock);
 619                 em = lookup_extent_mapping(em_tree, start, len);
 620                 if (!em) {
 621                         write_unlock(&em_tree->lock);
 622                         break;
 623                 }
 624                 flags = em->flags;
 625                 gen = em->generation;
 626                 if (skip_pinned && test_bit(EXTENT_FLAG_PINNED, &em->flags)) {
 627                         if (testend && em->start + em->len >= start + len) {
 628                                 free_extent_map(em);
 629                                 write_unlock(&em_tree->lock);
 630                                 break;
 631                         }
 632                         start = em->start + em->len;
 633                         if (testend)
 634                                 len = start + len - (em->start + em->len);
 635                         free_extent_map(em);
 636                         write_unlock(&em_tree->lock);
 637                         continue;
 638                 }
 639                 compressed = test_bit(EXTENT_FLAG_COMPRESSED, &em->flags);
 640                 clear_bit(EXTENT_FLAG_PINNED, &em->flags);
 641                 clear_bit(EXTENT_FLAG_LOGGING, &flags);
 642                 modified = !list_empty(&em->list);
 643                 if (no_splits)
 644                         goto next;
 645 
 646                 if (em->start < start) {
 647                         split->start = em->start;
 648                         split->len = start - em->start;
 649 
 650                         if (em->block_start < EXTENT_MAP_LAST_BYTE) {
 651                                 split->orig_start = em->orig_start;
 652                                 split->block_start = em->block_start;
 653 
 654                                 if (compressed)
 655                                         split->block_len = em->block_len;
 656                                 else
 657                                         split->block_len = split->len;
 658                                 split->orig_block_len = max(split->block_len,
 659                                                 em->orig_block_len);
 660                                 split->ram_bytes = em->ram_bytes;
 661                         } else {
 662                                 split->orig_start = split->start;
 663                                 split->block_len = 0;
 664                                 split->block_start = em->block_start;
 665                                 split->orig_block_len = 0;
 666                                 split->ram_bytes = split->len;
 667                         }
 668 
 669                         split->generation = gen;
 670                         split->bdev = em->bdev;
 671                         split->flags = flags;
 672                         split->compress_type = em->compress_type;
 673                         replace_extent_mapping(em_tree, em, split, modified);
 674                         free_extent_map(split);
 675                         split = split2;
 676                         split2 = NULL;
 677                 }
 678                 if (testend && em->start + em->len > start + len) {
 679                         u64 diff = start + len - em->start;
 680 
 681                         split->start = start + len;
 682                         split->len = em->start + em->len - (start + len);
 683                         split->bdev = em->bdev;
 684                         split->flags = flags;
 685                         split->compress_type = em->compress_type;
 686                         split->generation = gen;
 687 
 688                         if (em->block_start < EXTENT_MAP_LAST_BYTE) {
 689                                 split->orig_block_len = max(em->block_len,
 690                                                     em->orig_block_len);
 691 
 692                                 split->ram_bytes = em->ram_bytes;
 693                                 if (compressed) {
 694                                         split->block_len = em->block_len;
 695                                         split->block_start = em->block_start;
 696                                         split->orig_start = em->orig_start;
 697                                 } else {
 698                                         split->block_len = split->len;
 699                                         split->block_start = em->block_start
 700                                                 + diff;
 701                                         split->orig_start = em->orig_start;
 702                                 }
 703                         } else {
 704                                 split->ram_bytes = split->len;
 705                                 split->orig_start = split->start;
 706                                 split->block_len = 0;
 707                                 split->block_start = em->block_start;
 708                                 split->orig_block_len = 0;
 709                         }
 710 
 711                         if (extent_map_in_tree(em)) {
 712                                 replace_extent_mapping(em_tree, em, split,
 713                                                        modified);
 714                         } else {
 715                                 ret = add_extent_mapping(em_tree, split,
 716                                                          modified);
 717                                 ASSERT(ret == 0); /* Logic error */
 718                         }
 719                         free_extent_map(split);
 720                         split = NULL;
 721                 }
 722 next:
 723                 if (extent_map_in_tree(em))
 724                         remove_extent_mapping(em_tree, em);
 725                 write_unlock(&em_tree->lock);
 726 
 727                 /* once for us */
 728                 free_extent_map(em);
 729                 /* once for the tree*/
 730                 free_extent_map(em);
 731         }
 732         if (split)
 733                 free_extent_map(split);
 734         if (split2)
 735                 free_extent_map(split2);
 736 }
 737 
 738 /*
 739  * this is very complex, but the basic idea is to drop all extents
 740  * in the range start - end.  hint_block is filled in with a block number
 741  * that would be a good hint to the block allocator for this file.
 742  *
 743  * If an extent intersects the range but is not entirely inside the range
 744  * it is either truncated or split.  Anything entirely inside the range
 745  * is deleted from the tree.
 746  */
 747 int __btrfs_drop_extents(struct btrfs_trans_handle *trans,
 748                          struct btrfs_root *root, struct inode *inode,
 749                          struct btrfs_path *path, u64 start, u64 end,
 750                          u64 *drop_end, int drop_cache,
 751                          int replace_extent,
 752                          u32 extent_item_size,
 753                          int *key_inserted)
 754 {
 755         struct btrfs_fs_info *fs_info = root->fs_info;
 756         struct extent_buffer *leaf;
 757         struct btrfs_file_extent_item *fi;
 758         struct btrfs_ref ref = { 0 };
 759         struct btrfs_key key;
 760         struct btrfs_key new_key;
 761         u64 ino = btrfs_ino(BTRFS_I(inode));
 762         u64 search_start = start;
 763         u64 disk_bytenr = 0;
 764         u64 num_bytes = 0;
 765         u64 extent_offset = 0;
 766         u64 extent_end = 0;
 767         u64 last_end = start;
 768         int del_nr = 0;
 769         int del_slot = 0;
 770         int extent_type;
 771         int recow;
 772         int ret;
 773         int modify_tree = -1;
 774         int update_refs;
 775         int found = 0;
 776         int leafs_visited = 0;
 777 
 778         if (drop_cache)
 779                 btrfs_drop_extent_cache(BTRFS_I(inode), start, end - 1, 0);
 780 
 781         if (start >= BTRFS_I(inode)->disk_i_size && !replace_extent)
 782                 modify_tree = 0;
 783 
 784         update_refs = (test_bit(BTRFS_ROOT_REF_COWS, &root->state) ||
 785                        root == fs_info->tree_root);
 786         while (1) {
 787                 recow = 0;
 788                 ret = btrfs_lookup_file_extent(trans, root, path, ino,
 789                                                search_start, modify_tree);
 790                 if (ret < 0)
 791                         break;
 792                 if (ret > 0 && path->slots[0] > 0 && search_start == start) {
 793                         leaf = path->nodes[0];
 794                         btrfs_item_key_to_cpu(leaf, &key, path->slots[0] - 1);
 795                         if (key.objectid == ino &&
 796                             key.type == BTRFS_EXTENT_DATA_KEY)
 797                                 path->slots[0]--;
 798                 }
 799                 ret = 0;
 800                 leafs_visited++;
 801 next_slot:
 802                 leaf = path->nodes[0];
 803                 if (path->slots[0] >= btrfs_header_nritems(leaf)) {
 804                         BUG_ON(del_nr > 0);
 805                         ret = btrfs_next_leaf(root, path);
 806                         if (ret < 0)
 807                                 break;
 808                         if (ret > 0) {
 809                                 ret = 0;
 810                                 break;
 811                         }
 812                         leafs_visited++;
 813                         leaf = path->nodes[0];
 814                         recow = 1;
 815                 }
 816 
 817                 btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);
 818 
 819                 if (key.objectid > ino)
 820                         break;
 821                 if (WARN_ON_ONCE(key.objectid < ino) ||
 822                     key.type < BTRFS_EXTENT_DATA_KEY) {
 823                         ASSERT(del_nr == 0);
 824                         path->slots[0]++;
 825                         goto next_slot;
 826                 }
 827                 if (key.type > BTRFS_EXTENT_DATA_KEY || key.offset >= end)
 828                         break;
 829 
 830                 fi = btrfs_item_ptr(leaf, path->slots[0],
 831                                     struct btrfs_file_extent_item);
 832                 extent_type = btrfs_file_extent_type(leaf, fi);
 833 
 834                 if (extent_type == BTRFS_FILE_EXTENT_REG ||
 835                     extent_type == BTRFS_FILE_EXTENT_PREALLOC) {
 836                         disk_bytenr = btrfs_file_extent_disk_bytenr(leaf, fi);
 837                         num_bytes = btrfs_file_extent_disk_num_bytes(leaf, fi);
 838                         extent_offset = btrfs_file_extent_offset(leaf, fi);
 839                         extent_end = key.offset +
 840                                 btrfs_file_extent_num_bytes(leaf, fi);
 841                 } else if (extent_type == BTRFS_FILE_EXTENT_INLINE) {
 842                         extent_end = key.offset +
 843                                 btrfs_file_extent_ram_bytes(leaf, fi);
 844                 } else {
 845                         /* can't happen */
 846                         BUG();
 847                 }
 848 
 849                 /*
 850                  * Don't skip extent items representing 0 byte lengths. They
 851                  * used to be created (bug) if while punching holes we hit
 852                  * -ENOSPC condition. So if we find one here, just ensure we
 853                  * delete it, otherwise we would insert a new file extent item
 854                  * with the same key (offset) as that 0 bytes length file
 855                  * extent item in the call to setup_items_for_insert() later
 856                  * in this function.
 857                  */
 858                 if (extent_end == key.offset && extent_end >= search_start) {
 859                         last_end = extent_end;
 860                         goto delete_extent_item;
 861                 }
 862 
 863                 if (extent_end <= search_start) {
 864                         path->slots[0]++;
 865                         goto next_slot;
 866                 }
 867 
 868                 found = 1;
 869                 search_start = max(key.offset, start);
 870                 if (recow || !modify_tree) {
 871                         modify_tree = -1;
 872                         btrfs_release_path(path);
 873                         continue;
 874                 }
 875 
 876                 /*
 877                  *     | - range to drop - |
 878                  *  | -------- extent -------- |
 879                  */
 880                 if (start > key.offset && end < extent_end) {
 881                         BUG_ON(del_nr > 0);
 882                         if (extent_type == BTRFS_FILE_EXTENT_INLINE) {
 883                                 ret = -EOPNOTSUPP;
 884                                 break;
 885                         }
 886 
 887                         memcpy(&new_key, &key, sizeof(new_key));
 888                         new_key.offset = start;
 889                         ret = btrfs_duplicate_item(trans, root, path,
 890                                                    &new_key);
 891                         if (ret == -EAGAIN) {
 892                                 btrfs_release_path(path);
 893                                 continue;
 894                         }
 895                         if (ret < 0)
 896                                 break;
 897 
 898                         leaf = path->nodes[0];
 899                         fi = btrfs_item_ptr(leaf, path->slots[0] - 1,
 900                                             struct btrfs_file_extent_item);
 901                         btrfs_set_file_extent_num_bytes(leaf, fi,
 902                                                         start - key.offset);
 903 
 904                         fi = btrfs_item_ptr(leaf, path->slots[0],
 905                                             struct btrfs_file_extent_item);
 906 
 907                         extent_offset += start - key.offset;
 908                         btrfs_set_file_extent_offset(leaf, fi, extent_offset);
 909                         btrfs_set_file_extent_num_bytes(leaf, fi,
 910                                                         extent_end - start);
 911                         btrfs_mark_buffer_dirty(leaf);
 912 
 913                         if (update_refs && disk_bytenr > 0) {
 914                                 btrfs_init_generic_ref(&ref,
 915                                                 BTRFS_ADD_DELAYED_REF,
 916                                                 disk_bytenr, num_bytes, 0);
 917                                 btrfs_init_data_ref(&ref,
 918                                                 root->root_key.objectid,
 919                                                 new_key.objectid,
 920                                                 start - extent_offset);
 921                                 ret = btrfs_inc_extent_ref(trans, &ref);
 922                                 BUG_ON(ret); /* -ENOMEM */
 923                         }
 924                         key.offset = start;
 925                 }
 926                 /*
 927                  * From here on out we will have actually dropped something, so
 928                  * last_end can be updated.
 929                  */
 930                 last_end = extent_end;
 931 
 932                 /*
 933                  *  | ---- range to drop ----- |
 934                  *      | -------- extent -------- |
 935                  */
 936                 if (start <= key.offset && end < extent_end) {
 937                         if (extent_type == BTRFS_FILE_EXTENT_INLINE) {
 938                                 ret = -EOPNOTSUPP;
 939                                 break;
 940                         }
 941 
 942                         memcpy(&new_key, &key, sizeof(new_key));
 943                         new_key.offset = end;
 944                         btrfs_set_item_key_safe(fs_info, path, &new_key);
 945 
 946                         extent_offset += end - key.offset;
 947                         btrfs_set_file_extent_offset(leaf, fi, extent_offset);
 948                         btrfs_set_file_extent_num_bytes(leaf, fi,
 949                                                         extent_end - end);
 950                         btrfs_mark_buffer_dirty(leaf);
 951                         if (update_refs && disk_bytenr > 0)
 952                                 inode_sub_bytes(inode, end - key.offset);
 953                         break;
 954                 }
 955 
 956                 search_start = extent_end;
 957                 /*
 958                  *       | ---- range to drop ----- |
 959                  *  | -------- extent -------- |
 960                  */
 961                 if (start > key.offset && end >= extent_end) {
 962                         BUG_ON(del_nr > 0);
 963                         if (extent_type == BTRFS_FILE_EXTENT_INLINE) {
 964                                 ret = -EOPNOTSUPP;
 965                                 break;
 966                         }
 967 
 968                         btrfs_set_file_extent_num_bytes(leaf, fi,
 969                                                         start - key.offset);
 970                         btrfs_mark_buffer_dirty(leaf);
 971                         if (update_refs && disk_bytenr > 0)
 972                                 inode_sub_bytes(inode, extent_end - start);
 973                         if (end == extent_end)
 974                                 break;
 975 
 976                         path->slots[0]++;
 977                         goto next_slot;
 978                 }
 979 
 980                 /*
 981                  *  | ---- range to drop ----- |
 982                  *    | ------ extent ------ |
 983                  */
 984                 if (start <= key.offset && end >= extent_end) {
 985 delete_extent_item:
 986                         if (del_nr == 0) {
 987                                 del_slot = path->slots[0];
 988                                 del_nr = 1;
 989                         } else {
 990                                 BUG_ON(del_slot + del_nr != path->slots[0]);
 991                                 del_nr++;
 992                         }
 993 
 994                         if (update_refs &&
 995                             extent_type == BTRFS_FILE_EXTENT_INLINE) {
 996                                 inode_sub_bytes(inode,
 997                                                 extent_end - key.offset);
 998                                 extent_end = ALIGN(extent_end,
 999                                                    fs_info->sectorsize);
1000                         } else if (update_refs && disk_bytenr > 0) {
1001                                 btrfs_init_generic_ref(&ref,
1002                                                 BTRFS_DROP_DELAYED_REF,
1003                                                 disk_bytenr, num_bytes, 0);
1004                                 btrfs_init_data_ref(&ref,
1005                                                 root->root_key.objectid,
1006                                                 key.objectid,
1007                                                 key.offset - extent_offset);
1008                                 ret = btrfs_free_extent(trans, &ref);
1009                                 BUG_ON(ret); /* -ENOMEM */
1010                                 inode_sub_bytes(inode,
1011                                                 extent_end - key.offset);
1012                         }
1013 
1014                         if (end == extent_end)
1015                                 break;
1016 
1017                         if (path->slots[0] + 1 < btrfs_header_nritems(leaf)) {
1018                                 path->slots[0]++;
1019                                 goto next_slot;
1020                         }
1021 
1022                         ret = btrfs_del_items(trans, root, path, del_slot,
1023                                               del_nr);
1024                         if (ret) {
1025                                 btrfs_abort_transaction(trans, ret);
1026                                 break;
1027                         }
1028 
1029                         del_nr = 0;
1030                         del_slot = 0;
1031 
1032                         btrfs_release_path(path);
1033                         continue;
1034                 }
1035 
1036                 BUG();
1037         }
1038 
1039         if (!ret && del_nr > 0) {
1040                 /*
1041                  * Set path->slots[0] to first slot, so that after the delete
1042                  * if items are move off from our leaf to its immediate left or
1043                  * right neighbor leafs, we end up with a correct and adjusted
1044                  * path->slots[0] for our insertion (if replace_extent != 0).
1045                  */
1046                 path->slots[0] = del_slot;
1047                 ret = btrfs_del_items(trans, root, path, del_slot, del_nr);
1048                 if (ret)
1049                         btrfs_abort_transaction(trans, ret);
1050         }
1051 
1052         leaf = path->nodes[0];
1053         /*
1054          * If btrfs_del_items() was called, it might have deleted a leaf, in
1055          * which case it unlocked our path, so check path->locks[0] matches a
1056          * write lock.
1057          */
1058         if (!ret && replace_extent && leafs_visited == 1 &&
1059             (path->locks[0] == BTRFS_WRITE_LOCK_BLOCKING ||
1060              path->locks[0] == BTRFS_WRITE_LOCK) &&
1061             btrfs_leaf_free_space(leaf) >=
1062             sizeof(struct btrfs_item) + extent_item_size) {
1063 
1064                 key.objectid = ino;
1065                 key.type = BTRFS_EXTENT_DATA_KEY;
1066                 key.offset = start;
1067                 if (!del_nr && path->slots[0] < btrfs_header_nritems(leaf)) {
1068                         struct btrfs_key slot_key;
1069 
1070                         btrfs_item_key_to_cpu(leaf, &slot_key, path->slots[0]);
1071                         if (btrfs_comp_cpu_keys(&key, &slot_key) > 0)
1072                                 path->slots[0]++;
1073                 }
1074                 setup_items_for_insert(root, path, &key,
1075                                        &extent_item_size,
1076                                        extent_item_size,
1077                                        sizeof(struct btrfs_item) +
1078                                        extent_item_size, 1);
1079                 *key_inserted = 1;
1080         }
1081 
1082         if (!replace_extent || !(*key_inserted))
1083                 btrfs_release_path(path);
1084         if (drop_end)
1085                 *drop_end = found ? min(end, last_end) : end;
1086         return ret;
1087 }
1088 
1089 int btrfs_drop_extents(struct btrfs_trans_handle *trans,
1090                        struct btrfs_root *root, struct inode *inode, u64 start,
1091                        u64 end, int drop_cache)
1092 {
1093         struct btrfs_path *path;
1094         int ret;
1095 
1096         path = btrfs_alloc_path();
1097         if (!path)
1098                 return -ENOMEM;
1099         ret = __btrfs_drop_extents(trans, root, inode, path, start, end, NULL,
1100                                    drop_cache, 0, 0, NULL);
1101         btrfs_free_path(path);
1102         return ret;
1103 }
1104 
1105 static int extent_mergeable(struct extent_buffer *leaf, int slot,
1106                             u64 objectid, u64 bytenr, u64 orig_offset,
1107                             u64 *start, u64 *end)
1108 {
1109         struct btrfs_file_extent_item *fi;
1110         struct btrfs_key key;
1111         u64 extent_end;
1112 
1113         if (slot < 0 || slot >= btrfs_header_nritems(leaf))
1114                 return 0;
1115 
1116         btrfs_item_key_to_cpu(leaf, &key, slot);
1117         if (key.objectid != objectid || key.type != BTRFS_EXTENT_DATA_KEY)
1118                 return 0;
1119 
1120         fi = btrfs_item_ptr(leaf, slot, struct btrfs_file_extent_item);
1121         if (btrfs_file_extent_type(leaf, fi) != BTRFS_FILE_EXTENT_REG ||
1122             btrfs_file_extent_disk_bytenr(leaf, fi) != bytenr ||
1123             btrfs_file_extent_offset(leaf, fi) != key.offset - orig_offset ||
1124             btrfs_file_extent_compression(leaf, fi) ||
1125             btrfs_file_extent_encryption(leaf, fi) ||
1126             btrfs_file_extent_other_encoding(leaf, fi))
1127                 return 0;
1128 
1129         extent_end = key.offset + btrfs_file_extent_num_bytes(leaf, fi);
1130         if ((*start && *start != key.offset) || (*end && *end != extent_end))
1131                 return 0;
1132 
1133         *start = key.offset;
1134         *end = extent_end;
1135         return 1;
1136 }
1137 
1138 /*
1139  * Mark extent in the range start - end as written.
1140  *
1141  * This changes extent type from 'pre-allocated' to 'regular'. If only
1142  * part of extent is marked as written, the extent will be split into
1143  * two or three.
1144  */
1145 int btrfs_mark_extent_written(struct btrfs_trans_handle *trans,
1146                               struct btrfs_inode *inode, u64 start, u64 end)
1147 {
1148         struct btrfs_fs_info *fs_info = trans->fs_info;
1149         struct btrfs_root *root = inode->root;
1150         struct extent_buffer *leaf;
1151         struct btrfs_path *path;
1152         struct btrfs_file_extent_item *fi;
1153         struct btrfs_ref ref = { 0 };
1154         struct btrfs_key key;
1155         struct btrfs_key new_key;
1156         u64 bytenr;
1157         u64 num_bytes;
1158         u64 extent_end;
1159         u64 orig_offset;
1160         u64 other_start;
1161         u64 other_end;
1162         u64 split;
1163         int del_nr = 0;
1164         int del_slot = 0;
1165         int recow;
1166         int ret;
1167         u64 ino = btrfs_ino(inode);
1168 
1169         path = btrfs_alloc_path();
1170         if (!path)
1171                 return -ENOMEM;
1172 again:
1173         recow = 0;
1174         split = start;
1175         key.objectid = ino;
1176         key.type = BTRFS_EXTENT_DATA_KEY;
1177         key.offset = split;
1178 
1179         ret = btrfs_search_slot(trans, root, &key, path, -1, 1);
1180         if (ret < 0)
1181                 goto out;
1182         if (ret > 0 && path->slots[0] > 0)
1183                 path->slots[0]--;
1184 
1185         leaf = path->nodes[0];
1186         btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);
1187         if (key.objectid != ino ||
1188             key.type != BTRFS_EXTENT_DATA_KEY) {
1189                 ret = -EINVAL;
1190                 btrfs_abort_transaction(trans, ret);
1191                 goto out;
1192         }
1193         fi = btrfs_item_ptr(leaf, path->slots[0],
1194                             struct btrfs_file_extent_item);
1195         if (btrfs_file_extent_type(leaf, fi) != BTRFS_FILE_EXTENT_PREALLOC) {
1196                 ret = -EINVAL;
1197                 btrfs_abort_transaction(trans, ret);
1198                 goto out;
1199         }
1200         extent_end = key.offset + btrfs_file_extent_num_bytes(leaf, fi);
1201         if (key.offset > start || extent_end < end) {
1202                 ret = -EINVAL;
1203                 btrfs_abort_transaction(trans, ret);
1204                 goto out;
1205         }
1206 
1207         bytenr = btrfs_file_extent_disk_bytenr(leaf, fi);
1208         num_bytes = btrfs_file_extent_disk_num_bytes(leaf, fi);
1209         orig_offset = key.offset - btrfs_file_extent_offset(leaf, fi);
1210         memcpy(&new_key, &key, sizeof(new_key));
1211 
1212         if (start == key.offset && end < extent_end) {
1213                 other_start = 0;
1214                 other_end = start;
1215                 if (extent_mergeable(leaf, path->slots[0] - 1,
1216                                      ino, bytenr, orig_offset,
1217                                      &other_start, &other_end)) {
1218                         new_key.offset = end;
1219                         btrfs_set_item_key_safe(fs_info, path, &new_key);
1220                         fi = btrfs_item_ptr(leaf, path->slots[0],
1221                                             struct btrfs_file_extent_item);
1222                         btrfs_set_file_extent_generation(leaf, fi,
1223                                                          trans->transid);
1224                         btrfs_set_file_extent_num_bytes(leaf, fi,
1225                                                         extent_end - end);
1226                         btrfs_set_file_extent_offset(leaf, fi,
1227                                                      end - orig_offset);
1228                         fi = btrfs_item_ptr(leaf, path->slots[0] - 1,
1229                                             struct btrfs_file_extent_item);
1230                         btrfs_set_file_extent_generation(leaf, fi,
1231                                                          trans->transid);
1232                         btrfs_set_file_extent_num_bytes(leaf, fi,
1233                                                         end - other_start);
1234                         btrfs_mark_buffer_dirty(leaf);
1235                         goto out;
1236                 }
1237         }
1238 
1239         if (start > key.offset && end == extent_end) {
1240                 other_start = end;
1241                 other_end = 0;
1242                 if (extent_mergeable(leaf, path->slots[0] + 1,
1243                                      ino, bytenr, orig_offset,
1244                                      &other_start, &other_end)) {
1245                         fi = btrfs_item_ptr(leaf, path->slots[0],
1246                                             struct btrfs_file_extent_item);
1247                         btrfs_set_file_extent_num_bytes(leaf, fi,
1248                                                         start - key.offset);
1249                         btrfs_set_file_extent_generation(leaf, fi,
1250                                                          trans->transid);
1251                         path->slots[0]++;
1252                         new_key.offset = start;
1253                         btrfs_set_item_key_safe(fs_info, path, &new_key);
1254 
1255                         fi = btrfs_item_ptr(leaf, path->slots[0],
1256                                             struct btrfs_file_extent_item);
1257                         btrfs_set_file_extent_generation(leaf, fi,
1258                                                          trans->transid);
1259                         btrfs_set_file_extent_num_bytes(leaf, fi,
1260                                                         other_end - start);
1261                         btrfs_set_file_extent_offset(leaf, fi,
1262                                                      start - orig_offset);
1263                         btrfs_mark_buffer_dirty(leaf);
1264                         goto out;
1265                 }
1266         }
1267 
1268         while (start > key.offset || end < extent_end) {
1269                 if (key.offset == start)
1270                         split = end;
1271 
1272                 new_key.offset = split;
1273                 ret = btrfs_duplicate_item(trans, root, path, &new_key);
1274                 if (ret == -EAGAIN) {
1275                         btrfs_release_path(path);
1276                         goto again;
1277                 }
1278                 if (ret < 0) {
1279                         btrfs_abort_transaction(trans, ret);
1280                         goto out;
1281                 }
1282 
1283                 leaf = path->nodes[0];
1284                 fi = btrfs_item_ptr(leaf, path->slots[0] - 1,
1285                                     struct btrfs_file_extent_item);
1286                 btrfs_set_file_extent_generation(leaf, fi, trans->transid);
1287                 btrfs_set_file_extent_num_bytes(leaf, fi,
1288                                                 split - key.offset);
1289 
1290                 fi = btrfs_item_ptr(leaf, path->slots[0],
1291                                     struct btrfs_file_extent_item);
1292 
1293                 btrfs_set_file_extent_generation(leaf, fi, trans->transid);
1294                 btrfs_set_file_extent_offset(leaf, fi, split - orig_offset);
1295                 btrfs_set_file_extent_num_bytes(leaf, fi,
1296                                                 extent_end - split);
1297                 btrfs_mark_buffer_dirty(leaf);
1298 
1299                 btrfs_init_generic_ref(&ref, BTRFS_ADD_DELAYED_REF, bytenr,
1300                                        num_bytes, 0);
1301                 btrfs_init_data_ref(&ref, root->root_key.objectid, ino,
1302                                     orig_offset);
1303                 ret = btrfs_inc_extent_ref(trans, &ref);
1304                 if (ret) {
1305                         btrfs_abort_transaction(trans, ret);
1306                         goto out;
1307                 }
1308 
1309                 if (split == start) {
1310                         key.offset = start;
1311                 } else {
1312                         if (start != key.offset) {
1313                                 ret = -EINVAL;
1314                                 btrfs_abort_transaction(trans, ret);
1315                                 goto out;
1316                         }
1317                         path->slots[0]--;
1318                         extent_end = end;
1319                 }
1320                 recow = 1;
1321         }
1322 
1323         other_start = end;
1324         other_end = 0;
1325         btrfs_init_generic_ref(&ref, BTRFS_DROP_DELAYED_REF, bytenr,
1326                                num_bytes, 0);
1327         btrfs_init_data_ref(&ref, root->root_key.objectid, ino, orig_offset);
1328         if (extent_mergeable(leaf, path->slots[0] + 1,
1329                              ino, bytenr, orig_offset,
1330                              &other_start, &other_end)) {
1331                 if (recow) {
1332                         btrfs_release_path(path);
1333                         goto again;
1334                 }
1335                 extent_end = other_end;
1336                 del_slot = path->slots[0] + 1;
1337                 del_nr++;
1338                 ret = btrfs_free_extent(trans, &ref);
1339                 if (ret) {
1340                         btrfs_abort_transaction(trans, ret);
1341                         goto out;
1342                 }
1343         }
1344         other_start = 0;
1345         other_end = start;
1346         if (extent_mergeable(leaf, path->slots[0] - 1,
1347                              ino, bytenr, orig_offset,
1348                              &other_start, &other_end)) {
1349                 if (recow) {
1350                         btrfs_release_path(path);
1351                         goto again;
1352                 }
1353                 key.offset = other_start;
1354                 del_slot = path->slots[0];
1355                 del_nr++;
1356                 ret = btrfs_free_extent(trans, &ref);
1357                 if (ret) {
1358                         btrfs_abort_transaction(trans, ret);
1359                         goto out;
1360                 }
1361         }
1362         if (del_nr == 0) {
1363                 fi = btrfs_item_ptr(leaf, path->slots[0],
1364                            struct btrfs_file_extent_item);
1365                 btrfs_set_file_extent_type(leaf, fi,
1366                                            BTRFS_FILE_EXTENT_REG);
1367                 btrfs_set_file_extent_generation(leaf, fi, trans->transid);
1368                 btrfs_mark_buffer_dirty(leaf);
1369         } else {
1370                 fi = btrfs_item_ptr(leaf, del_slot - 1,
1371                            struct btrfs_file_extent_item);
1372                 btrfs_set_file_extent_type(leaf, fi,
1373                                            BTRFS_FILE_EXTENT_REG);
1374                 btrfs_set_file_extent_generation(leaf, fi, trans->transid);
1375                 btrfs_set_file_extent_num_bytes(leaf, fi,
1376                                                 extent_end - key.offset);
1377                 btrfs_mark_buffer_dirty(leaf);
1378 
1379                 ret = btrfs_del_items(trans, root, path, del_slot, del_nr);
1380                 if (ret < 0) {
1381                         btrfs_abort_transaction(trans, ret);
1382                         goto out;
1383                 }
1384         }
1385 out:
1386         btrfs_free_path(path);
1387         return 0;
1388 }
1389 
1390 /*
1391  * on error we return an unlocked page and the error value
1392  * on success we return a locked page and 0
1393  */
1394 static int prepare_uptodate_page(struct inode *inode,
1395                                  struct page *page, u64 pos,
1396                                  bool force_uptodate)
1397 {
1398         int ret = 0;
1399 
1400         if (((pos & (PAGE_SIZE - 1)) || force_uptodate) &&
1401             !PageUptodate(page)) {
1402                 ret = btrfs_readpage(NULL, page);
1403                 if (ret)
1404                         return ret;
1405                 lock_page(page);
1406                 if (!PageUptodate(page)) {
1407                         unlock_page(page);
1408                         return -EIO;
1409                 }
1410                 if (page->mapping != inode->i_mapping) {
1411                         unlock_page(page);
1412                         return -EAGAIN;
1413                 }
1414         }
1415         return 0;
1416 }
1417 
1418 /*
1419  * this just gets pages into the page cache and locks them down.
1420  */
1421 static noinline int prepare_pages(struct inode *inode, struct page **pages,
1422                                   size_t num_pages, loff_t pos,
1423                                   size_t write_bytes, bool force_uptodate)
1424 {
1425         int i;
1426         unsigned long index = pos >> PAGE_SHIFT;
1427         gfp_t mask = btrfs_alloc_write_mask(inode->i_mapping);
1428         int err = 0;
1429         int faili;
1430 
1431         for (i = 0; i < num_pages; i++) {
1432 again:
1433                 pages[i] = find_or_create_page(inode->i_mapping, index + i,
1434                                                mask | __GFP_WRITE);
1435                 if (!pages[i]) {
1436                         faili = i - 1;
1437                         err = -ENOMEM;
1438                         goto fail;
1439                 }
1440 
1441                 if (i == 0)
1442                         err = prepare_uptodate_page(inode, pages[i], pos,
1443                                                     force_uptodate);
1444                 if (!err && i == num_pages - 1)
1445                         err = prepare_uptodate_page(inode, pages[i],
1446                                                     pos + write_bytes, false);
1447                 if (err) {
1448                         put_page(pages[i]);
1449                         if (err == -EAGAIN) {
1450                                 err = 0;
1451                                 goto again;
1452                         }
1453                         faili = i - 1;
1454                         goto fail;
1455                 }
1456                 wait_on_page_writeback(pages[i]);
1457         }
1458 
1459         return 0;
1460 fail:
1461         while (faili >= 0) {
1462                 unlock_page(pages[faili]);
1463                 put_page(pages[faili]);
1464                 faili--;
1465         }
1466         return err;
1467 
1468 }
1469 
1470 /*
1471  * This function locks the extent and properly waits for data=ordered extents
1472  * to finish before allowing the pages to be modified if need.
1473  *
1474  * The return value:
1475  * 1 - the extent is locked
1476  * 0 - the extent is not locked, and everything is OK
1477  * -EAGAIN - need re-prepare the pages
1478  * the other < 0 number - Something wrong happens
1479  */
1480 static noinline int
1481 lock_and_cleanup_extent_if_need(struct btrfs_inode *inode, struct page **pages,
1482                                 size_t num_pages, loff_t pos,
1483                                 size_t write_bytes,
1484                                 u64 *lockstart, u64 *lockend,
1485                                 struct extent_state **cached_state)
1486 {
1487         struct btrfs_fs_info *fs_info = inode->root->fs_info;
1488         u64 start_pos;
1489         u64 last_pos;
1490         int i;
1491         int ret = 0;
1492 
1493         start_pos = round_down(pos, fs_info->sectorsize);
1494         last_pos = start_pos
1495                 + round_up(pos + write_bytes - start_pos,
1496                            fs_info->sectorsize) - 1;
1497 
1498         if (start_pos < inode->vfs_inode.i_size) {
1499                 struct btrfs_ordered_extent *ordered;
1500 
1501                 lock_extent_bits(&inode->io_tree, start_pos, last_pos,
1502                                 cached_state);
1503                 ordered = btrfs_lookup_ordered_range(inode, start_pos,
1504                                                      last_pos - start_pos + 1);
1505                 if (ordered &&
1506                     ordered->file_offset + ordered->len > start_pos &&
1507                     ordered->file_offset <= last_pos) {
1508                         unlock_extent_cached(&inode->io_tree, start_pos,
1509                                         last_pos, cached_state);
1510                         for (i = 0; i < num_pages; i++) {
1511                                 unlock_page(pages[i]);
1512                                 put_page(pages[i]);
1513                         }
1514                         btrfs_start_ordered_extent(&inode->vfs_inode,
1515                                         ordered, 1);
1516                         btrfs_put_ordered_extent(ordered);
1517                         return -EAGAIN;
1518                 }
1519                 if (ordered)
1520                         btrfs_put_ordered_extent(ordered);
1521 
1522                 *lockstart = start_pos;
1523                 *lockend = last_pos;
1524                 ret = 1;
1525         }
1526 
1527         /*
1528          * It's possible the pages are dirty right now, but we don't want
1529          * to clean them yet because copy_from_user may catch a page fault
1530          * and we might have to fall back to one page at a time.  If that
1531          * happens, we'll unlock these pages and we'd have a window where
1532          * reclaim could sneak in and drop the once-dirty page on the floor
1533          * without writing it.
1534          *
1535          * We have the pages locked and the extent range locked, so there's
1536          * no way someone can start IO on any dirty pages in this range.
1537          *
1538          * We'll call btrfs_dirty_pages() later on, and that will flip around
1539          * delalloc bits and dirty the pages as required.
1540          */
1541         for (i = 0; i < num_pages; i++) {
1542                 set_page_extent_mapped(pages[i]);
1543                 WARN_ON(!PageLocked(pages[i]));
1544         }
1545 
1546         return ret;
1547 }
1548 
1549 static noinline int check_can_nocow(struct btrfs_inode *inode, loff_t pos,
1550                                     size_t *write_bytes)
1551 {
1552         struct btrfs_fs_info *fs_info = inode->root->fs_info;
1553         struct btrfs_root *root = inode->root;
1554         u64 lockstart, lockend;
1555         u64 num_bytes;
1556         int ret;
1557 
1558         ret = btrfs_start_write_no_snapshotting(root);
1559         if (!ret)
1560                 return -EAGAIN;
1561 
1562         lockstart = round_down(pos, fs_info->sectorsize);
1563         lockend = round_up(pos + *write_bytes,
1564                            fs_info->sectorsize) - 1;
1565 
1566         btrfs_lock_and_flush_ordered_range(&inode->io_tree, inode, lockstart,
1567                                            lockend, NULL);
1568 
1569         num_bytes = lockend - lockstart + 1;
1570         ret = can_nocow_extent(&inode->vfs_inode, lockstart, &num_bytes,
1571                         NULL, NULL, NULL);
1572         if (ret <= 0) {
1573                 ret = 0;
1574                 btrfs_end_write_no_snapshotting(root);
1575         } else {
1576                 *write_bytes = min_t(size_t, *write_bytes ,
1577                                      num_bytes - pos + lockstart);
1578         }
1579 
1580         unlock_extent(&inode->io_tree, lockstart, lockend);
1581 
1582         return ret;
1583 }
1584 
1585 static noinline ssize_t btrfs_buffered_write(struct kiocb *iocb,
1586                                                struct iov_iter *i)
1587 {
1588         struct file *file = iocb->ki_filp;
1589         loff_t pos = iocb->ki_pos;
1590         struct inode *inode = file_inode(file);
1591         struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
1592         struct btrfs_root *root = BTRFS_I(inode)->root;
1593         struct page **pages = NULL;
1594         struct extent_changeset *data_reserved = NULL;
1595         u64 release_bytes = 0;
1596         u64 lockstart;
1597         u64 lockend;
1598         size_t num_written = 0;
1599         int nrptrs;
1600         int ret = 0;
1601         bool only_release_metadata = false;
1602         bool force_page_uptodate = false;
1603 
1604         nrptrs = min(DIV_ROUND_UP(iov_iter_count(i), PAGE_SIZE),
1605                         PAGE_SIZE / (sizeof(struct page *)));
1606         nrptrs = min(nrptrs, current->nr_dirtied_pause - current->nr_dirtied);
1607         nrptrs = max(nrptrs, 8);
1608         pages = kmalloc_array(nrptrs, sizeof(struct page *), GFP_KERNEL);
1609         if (!pages)
1610                 return -ENOMEM;
1611 
1612         while (iov_iter_count(i) > 0) {
1613                 struct extent_state *cached_state = NULL;
1614                 size_t offset = offset_in_page(pos);
1615                 size_t sector_offset;
1616                 size_t write_bytes = min(iov_iter_count(i),
1617                                          nrptrs * (size_t)PAGE_SIZE -
1618                                          offset);
1619                 size_t num_pages = DIV_ROUND_UP(write_bytes + offset,
1620                                                 PAGE_SIZE);
1621                 size_t reserve_bytes;
1622                 size_t dirty_pages;
1623                 size_t copied;
1624                 size_t dirty_sectors;
1625                 size_t num_sectors;
1626                 int extents_locked;
1627 
1628                 WARN_ON(num_pages > nrptrs);
1629 
1630                 /*
1631                  * Fault pages before locking them in prepare_pages
1632                  * to avoid recursive lock
1633                  */
1634                 if (unlikely(iov_iter_fault_in_readable(i, write_bytes))) {
1635                         ret = -EFAULT;
1636                         break;
1637                 }
1638 
1639                 only_release_metadata = false;
1640                 sector_offset = pos & (fs_info->sectorsize - 1);
1641                 reserve_bytes = round_up(write_bytes + sector_offset,
1642                                 fs_info->sectorsize);
1643 
1644                 extent_changeset_release(data_reserved);
1645                 ret = btrfs_check_data_free_space(inode, &data_reserved, pos,
1646                                                   write_bytes);
1647                 if (ret < 0) {
1648                         if ((BTRFS_I(inode)->flags & (BTRFS_INODE_NODATACOW |
1649                                                       BTRFS_INODE_PREALLOC)) &&
1650                             check_can_nocow(BTRFS_I(inode), pos,
1651                                         &write_bytes) > 0) {
1652                                 /*
1653                                  * For nodata cow case, no need to reserve
1654                                  * data space.
1655                                  */
1656                                 only_release_metadata = true;
1657                                 /*
1658                                  * our prealloc extent may be smaller than
1659                                  * write_bytes, so scale down.
1660                                  */
1661                                 num_pages = DIV_ROUND_UP(write_bytes + offset,
1662                                                          PAGE_SIZE);
1663                                 reserve_bytes = round_up(write_bytes +
1664                                                          sector_offset,
1665                                                          fs_info->sectorsize);
1666                         } else {
1667                                 break;
1668                         }
1669                 }
1670 
1671                 WARN_ON(reserve_bytes == 0);
1672                 ret = btrfs_delalloc_reserve_metadata(BTRFS_I(inode),
1673                                 reserve_bytes);
1674                 if (ret) {
1675                         if (!only_release_metadata)
1676                                 btrfs_free_reserved_data_space(inode,
1677                                                 data_reserved, pos,
1678                                                 write_bytes);
1679                         else
1680                                 btrfs_end_write_no_snapshotting(root);
1681                         break;
1682                 }
1683 
1684                 release_bytes = reserve_bytes;
1685 again:
1686                 /*
1687                  * This is going to setup the pages array with the number of
1688                  * pages we want, so we don't really need to worry about the
1689                  * contents of pages from loop to loop
1690                  */
1691                 ret = prepare_pages(inode, pages, num_pages,
1692                                     pos, write_bytes,
1693                                     force_page_uptodate);
1694                 if (ret) {
1695                         btrfs_delalloc_release_extents(BTRFS_I(inode),
1696                                                        reserve_bytes);
1697                         break;
1698                 }
1699 
1700                 extents_locked = lock_and_cleanup_extent_if_need(
1701                                 BTRFS_I(inode), pages,
1702                                 num_pages, pos, write_bytes, &lockstart,
1703                                 &lockend, &cached_state);
1704                 if (extents_locked < 0) {
1705                         if (extents_locked == -EAGAIN)
1706                                 goto again;
1707                         btrfs_delalloc_release_extents(BTRFS_I(inode),
1708                                                        reserve_bytes);
1709                         ret = extents_locked;
1710                         break;
1711                 }
1712 
1713                 copied = btrfs_copy_from_user(pos, write_bytes, pages, i);
1714 
1715                 num_sectors = BTRFS_BYTES_TO_BLKS(fs_info, reserve_bytes);
1716                 dirty_sectors = round_up(copied + sector_offset,
1717                                         fs_info->sectorsize);
1718                 dirty_sectors = BTRFS_BYTES_TO_BLKS(fs_info, dirty_sectors);
1719 
1720                 /*
1721                  * if we have trouble faulting in the pages, fall
1722                  * back to one page at a time
1723                  */
1724                 if (copied < write_bytes)
1725                         nrptrs = 1;
1726 
1727                 if (copied == 0) {
1728                         force_page_uptodate = true;
1729                         dirty_sectors = 0;
1730                         dirty_pages = 0;
1731                 } else {
1732                         force_page_uptodate = false;
1733                         dirty_pages = DIV_ROUND_UP(copied + offset,
1734                                                    PAGE_SIZE);
1735                 }
1736 
1737                 if (num_sectors > dirty_sectors) {
1738                         /* release everything except the sectors we dirtied */
1739                         release_bytes -= dirty_sectors <<
1740                                                 fs_info->sb->s_blocksize_bits;
1741                         if (only_release_metadata) {
1742                                 btrfs_delalloc_release_metadata(BTRFS_I(inode),
1743                                                         release_bytes, true);
1744                         } else {
1745                                 u64 __pos;
1746 
1747                                 __pos = round_down(pos,
1748                                                    fs_info->sectorsize) +
1749                                         (dirty_pages << PAGE_SHIFT);
1750                                 btrfs_delalloc_release_space(inode,
1751                                                 data_reserved, __pos,
1752                                                 release_bytes, true);
1753                         }
1754                 }
1755 
1756                 release_bytes = round_up(copied + sector_offset,
1757                                         fs_info->sectorsize);
1758 
1759                 if (copied > 0)
1760                         ret = btrfs_dirty_pages(inode, pages, dirty_pages,
1761                                                 pos, copied, &cached_state);
1762 
1763                 /*
1764                  * If we have not locked the extent range, because the range's
1765                  * start offset is >= i_size, we might still have a non-NULL
1766                  * cached extent state, acquired while marking the extent range
1767                  * as delalloc through btrfs_dirty_pages(). Therefore free any
1768                  * possible cached extent state to avoid a memory leak.
1769                  */
1770                 if (extents_locked)
1771                         unlock_extent_cached(&BTRFS_I(inode)->io_tree,
1772                                              lockstart, lockend, &cached_state);
1773                 else
1774                         free_extent_state(cached_state);
1775 
1776                 btrfs_delalloc_release_extents(BTRFS_I(inode), reserve_bytes);
1777                 if (ret) {
1778                         btrfs_drop_pages(pages, num_pages);
1779                         break;
1780                 }
1781 
1782                 release_bytes = 0;
1783                 if (only_release_metadata)
1784                         btrfs_end_write_no_snapshotting(root);
1785 
1786                 if (only_release_metadata && copied > 0) {
1787                         lockstart = round_down(pos,
1788                                                fs_info->sectorsize);
1789                         lockend = round_up(pos + copied,
1790                                            fs_info->sectorsize) - 1;
1791 
1792                         set_extent_bit(&BTRFS_I(inode)->io_tree, lockstart,
1793                                        lockend, EXTENT_NORESERVE, NULL,
1794                                        NULL, GFP_NOFS);
1795                 }
1796 
1797                 btrfs_drop_pages(pages, num_pages);
1798 
1799                 cond_resched();
1800 
1801                 balance_dirty_pages_ratelimited(inode->i_mapping);
1802                 if (dirty_pages < (fs_info->nodesize >> PAGE_SHIFT) + 1)
1803                         btrfs_btree_balance_dirty(fs_info);
1804 
1805                 pos += copied;
1806                 num_written += copied;
1807         }
1808 
1809         kfree(pages);
1810 
1811         if (release_bytes) {
1812                 if (only_release_metadata) {
1813                         btrfs_end_write_no_snapshotting(root);
1814                         btrfs_delalloc_release_metadata(BTRFS_I(inode),
1815                                         release_bytes, true);
1816                 } else {
1817                         btrfs_delalloc_release_space(inode, data_reserved,
1818                                         round_down(pos, fs_info->sectorsize),
1819                                         release_bytes, true);
1820                 }
1821         }
1822 
1823         extent_changeset_free(data_reserved);
1824         return num_written ? num_written : ret;
1825 }
1826 
1827 static ssize_t __btrfs_direct_write(struct kiocb *iocb, struct iov_iter *from)
1828 {
1829         struct file *file = iocb->ki_filp;
1830         struct inode *inode = file_inode(file);
1831         loff_t pos;
1832         ssize_t written;
1833         ssize_t written_buffered;
1834         loff_t endbyte;
1835         int err;
1836 
1837         written = generic_file_direct_write(iocb, from);
1838 
1839         if (written < 0 || !iov_iter_count(from))
1840                 return written;
1841 
1842         pos = iocb->ki_pos;
1843         written_buffered = btrfs_buffered_write(iocb, from);
1844         if (written_buffered < 0) {
1845                 err = written_buffered;
1846                 goto out;
1847         }
1848         /*
1849          * Ensure all data is persisted. We want the next direct IO read to be
1850          * able to read what was just written.
1851          */
1852         endbyte = pos + written_buffered - 1;
1853         err = btrfs_fdatawrite_range(inode, pos, endbyte);
1854         if (err)
1855                 goto out;
1856         err = filemap_fdatawait_range(inode->i_mapping, pos, endbyte);
1857         if (err)
1858                 goto out;
1859         written += written_buffered;
1860         iocb->ki_pos = pos + written_buffered;
1861         invalidate_mapping_pages(file->f_mapping, pos >> PAGE_SHIFT,
1862                                  endbyte >> PAGE_SHIFT);
1863 out:
1864         return written ? written : err;
1865 }
1866 
1867 static void update_time_for_write(struct inode *inode)
1868 {
1869         struct timespec64 now;
1870 
1871         if (IS_NOCMTIME(inode))
1872                 return;
1873 
1874         now = current_time(inode);
1875         if (!timespec64_equal(&inode->i_mtime, &now))
1876                 inode->i_mtime = now;
1877 
1878         if (!timespec64_equal(&inode->i_ctime, &now))
1879                 inode->i_ctime = now;
1880 
1881         if (IS_I_VERSION(inode))
1882                 inode_inc_iversion(inode);
1883 }
1884 
1885 static ssize_t btrfs_file_write_iter(struct kiocb *iocb,
1886                                     struct iov_iter *from)
1887 {
1888         struct file *file = iocb->ki_filp;
1889         struct inode *inode = file_inode(file);
1890         struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
1891         struct btrfs_root *root = BTRFS_I(inode)->root;
1892         u64 start_pos;
1893         u64 end_pos;
1894         ssize_t num_written = 0;
1895         const bool sync = iocb->ki_flags & IOCB_DSYNC;
1896         ssize_t err;
1897         loff_t pos;
1898         size_t count;
1899         loff_t oldsize;
1900         int clean_page = 0;
1901 
1902         if (!(iocb->ki_flags & IOCB_DIRECT) &&
1903             (iocb->ki_flags & IOCB_NOWAIT))
1904                 return -EOPNOTSUPP;
1905 
1906         if (iocb->ki_flags & IOCB_NOWAIT) {
1907                 if (!inode_trylock(inode))
1908                         return -EAGAIN;
1909         } else {
1910                 inode_lock(inode);
1911         }
1912 
1913         err = generic_write_checks(iocb, from);
1914         if (err <= 0) {
1915                 inode_unlock(inode);
1916                 return err;
1917         }
1918 
1919         pos = iocb->ki_pos;
1920         count = iov_iter_count(from);
1921         if (iocb->ki_flags & IOCB_NOWAIT) {
1922                 /*
1923                  * We will allocate space in case nodatacow is not set,
1924                  * so bail
1925                  */
1926                 if (!(BTRFS_I(inode)->flags & (BTRFS_INODE_NODATACOW |
1927                                               BTRFS_INODE_PREALLOC)) ||
1928                     check_can_nocow(BTRFS_I(inode), pos, &count) <= 0) {
1929                         inode_unlock(inode);
1930                         return -EAGAIN;
1931                 }
1932         }
1933 
1934         current->backing_dev_info = inode_to_bdi(inode);
1935         err = file_remove_privs(file);
1936         if (err) {
1937                 inode_unlock(inode);
1938                 goto out;
1939         }
1940 
1941         /*
1942          * If BTRFS flips readonly due to some impossible error
1943          * (fs_info->fs_state now has BTRFS_SUPER_FLAG_ERROR),
1944          * although we have opened a file as writable, we have
1945          * to stop this write operation to ensure FS consistency.
1946          */
1947         if (test_bit(BTRFS_FS_STATE_ERROR, &fs_info->fs_state)) {
1948                 inode_unlock(inode);
1949                 err = -EROFS;
1950                 goto out;
1951         }
1952 
1953         /*
1954          * We reserve space for updating the inode when we reserve space for the
1955          * extent we are going to write, so we will enospc out there.  We don't
1956          * need to start yet another transaction to update the inode as we will
1957          * update the inode when we finish writing whatever data we write.
1958          */
1959         update_time_for_write(inode);
1960 
1961         start_pos = round_down(pos, fs_info->sectorsize);
1962         oldsize = i_size_read(inode);
1963         if (start_pos > oldsize) {
1964                 /* Expand hole size to cover write data, preventing empty gap */
1965                 end_pos = round_up(pos + count,
1966                                    fs_info->sectorsize);
1967                 err = btrfs_cont_expand(inode, oldsize, end_pos);
1968                 if (err) {
1969                         inode_unlock(inode);
1970                         goto out;
1971                 }
1972                 if (start_pos > round_up(oldsize, fs_info->sectorsize))
1973                         clean_page = 1;
1974         }
1975 
1976         if (sync)
1977                 atomic_inc(&BTRFS_I(inode)->sync_writers);
1978 
1979         if (iocb->ki_flags & IOCB_DIRECT) {
1980                 num_written = __btrfs_direct_write(iocb, from);
1981         } else {
1982                 num_written = btrfs_buffered_write(iocb, from);
1983                 if (num_written > 0)
1984                         iocb->ki_pos = pos + num_written;
1985                 if (clean_page)
1986                         pagecache_isize_extended(inode, oldsize,
1987                                                 i_size_read(inode));
1988         }
1989 
1990         inode_unlock(inode);
1991 
1992         /*
1993          * We also have to set last_sub_trans to the current log transid,
1994          * otherwise subsequent syncs to a file that's been synced in this
1995          * transaction will appear to have already occurred.
1996          */
1997         spin_lock(&BTRFS_I(inode)->lock);
1998         BTRFS_I(inode)->last_sub_trans = root->log_transid;
1999         spin_unlock(&BTRFS_I(inode)->lock);
2000         if (num_written > 0)
2001                 num_written = generic_write_sync(iocb, num_written);
2002 
2003         if (sync)
2004                 atomic_dec(&BTRFS_I(inode)->sync_writers);
2005 out:
2006         current->backing_dev_info = NULL;
2007         return num_written ? num_written : err;
2008 }
2009 
2010 int btrfs_release_file(struct inode *inode, struct file *filp)
2011 {
2012         struct btrfs_file_private *private = filp->private_data;
2013 
2014         if (private && private->filldir_buf)
2015                 kfree(private->filldir_buf);
2016         kfree(private);
2017         filp->private_data = NULL;
2018 
2019         /*
2020          * ordered_data_close is set by setattr when we are about to truncate
2021          * a file from a non-zero size to a zero size.  This tries to
2022          * flush down new bytes that may have been written if the
2023          * application were using truncate to replace a file in place.
2024          */
2025         if (test_and_clear_bit(BTRFS_INODE_ORDERED_DATA_CLOSE,
2026                                &BTRFS_I(inode)->runtime_flags))
2027                         filemap_flush(inode->i_mapping);
2028         return 0;
2029 }
2030 
2031 static int start_ordered_ops(struct inode *inode, loff_t start, loff_t end)
2032 {
2033         int ret;
2034         struct blk_plug plug;
2035 
2036         /*
2037          * This is only called in fsync, which would do synchronous writes, so
2038          * a plug can merge adjacent IOs as much as possible.  Esp. in case of
2039          * multiple disks using raid profile, a large IO can be split to
2040          * several segments of stripe length (currently 64K).
2041          */
2042         blk_start_plug(&plug);
2043         atomic_inc(&BTRFS_I(inode)->sync_writers);
2044         ret = btrfs_fdatawrite_range(inode, start, end);
2045         atomic_dec(&BTRFS_I(inode)->sync_writers);
2046         blk_finish_plug(&plug);
2047 
2048         return ret;
2049 }
2050 
2051 /*
2052  * fsync call for both files and directories.  This logs the inode into
2053  * the tree log instead of forcing full commits whenever possible.
2054  *
2055  * It needs to call filemap_fdatawait so that all ordered extent updates are
2056  * in the metadata btree are up to date for copying to the log.
2057  *
2058  * It drops the inode mutex before doing the tree log commit.  This is an
2059  * important optimization for directories because holding the mutex prevents
2060  * new operations on the dir while we write to disk.
2061  */
2062 int btrfs_sync_file(struct file *file, loff_t start, loff_t end, int datasync)
2063 {
2064         struct dentry *dentry = file_dentry(file);
2065         struct inode *inode = d_inode(dentry);
2066         struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
2067         struct btrfs_root *root = BTRFS_I(inode)->root;
2068         struct btrfs_trans_handle *trans;
2069         struct btrfs_log_ctx ctx;
2070         int ret = 0, err;
2071 
2072         trace_btrfs_sync_file(file, datasync);
2073 
2074         btrfs_init_log_ctx(&ctx, inode);
2075 
2076         /*
2077          * Set the range to full if the NO_HOLES feature is not enabled.
2078          * This is to avoid missing file extent items representing holes after
2079          * replaying the log.
2080          */
2081         if (!btrfs_fs_incompat(fs_info, NO_HOLES)) {
2082                 start = 0;
2083                 end = LLONG_MAX;
2084         }
2085 
2086         /*
2087          * We write the dirty pages in the range and wait until they complete
2088          * out of the ->i_mutex. If so, we can flush the dirty pages by
2089          * multi-task, and make the performance up.  See
2090          * btrfs_wait_ordered_range for an explanation of the ASYNC check.
2091          */
2092         ret = start_ordered_ops(inode, start, end);
2093         if (ret)
2094                 goto out;
2095 
2096         inode_lock(inode);
2097 
2098         /*
2099          * We take the dio_sem here because the tree log stuff can race with
2100          * lockless dio writes and get an extent map logged for an extent we
2101          * never waited on.  We need it this high up for lockdep reasons.
2102          */
2103         down_write(&BTRFS_I(inode)->dio_sem);
2104 
2105         atomic_inc(&root->log_batch);
2106 
2107         /*
2108          * If the inode needs a full sync, make sure we use a full range to
2109          * avoid log tree corruption, due to hole detection racing with ordered
2110          * extent completion for adjacent ranges, and assertion failures during
2111          * hole detection. Do this while holding the inode lock, to avoid races
2112          * with other tasks.
2113          */
2114         if (test_bit(BTRFS_INODE_NEEDS_FULL_SYNC,
2115                      &BTRFS_I(inode)->runtime_flags)) {
2116                 start = 0;
2117                 end = LLONG_MAX;
2118         }
2119 
2120         /*
2121          * Before we acquired the inode's lock, someone may have dirtied more
2122          * pages in the target range. We need to make sure that writeback for
2123          * any such pages does not start while we are logging the inode, because
2124          * if it does, any of the following might happen when we are not doing a
2125          * full inode sync:
2126          *
2127          * 1) We log an extent after its writeback finishes but before its
2128          *    checksums are added to the csum tree, leading to -EIO errors
2129          *    when attempting to read the extent after a log replay.
2130          *
2131          * 2) We can end up logging an extent before its writeback finishes.
2132          *    Therefore after the log replay we will have a file extent item
2133          *    pointing to an unwritten extent (and no data checksums as well).
2134          *
2135          * So trigger writeback for any eventual new dirty pages and then we
2136          * wait for all ordered extents to complete below.
2137          */
2138         ret = start_ordered_ops(inode, start, end);
2139         if (ret) {
2140                 up_write(&BTRFS_I(inode)->dio_sem);
2141                 inode_unlock(inode);
2142                 goto out;
2143         }
2144 
2145         /*
2146          * We have to do this here to avoid the priority inversion of waiting on
2147          * IO of a lower priority task while holding a transaction open.
2148          *
2149          * Also, the range length can be represented by u64, we have to do the
2150          * typecasts to avoid signed overflow if it's [0, LLONG_MAX].
2151          */
2152         ret = btrfs_wait_ordered_range(inode, start, (u64)end - (u64)start + 1);
2153         if (ret) {
2154                 up_write(&BTRFS_I(inode)->dio_sem);
2155                 inode_unlock(inode);
2156                 goto out;
2157         }
2158         atomic_inc(&root->log_batch);
2159 
2160         smp_mb();
2161         if (btrfs_inode_in_log(BTRFS_I(inode), fs_info->generation) ||
2162             BTRFS_I(inode)->last_trans <= fs_info->last_trans_committed) {
2163                 /*
2164                  * We've had everything committed since the last time we were
2165                  * modified so clear this flag in case it was set for whatever
2166                  * reason, it's no longer relevant.
2167                  */
2168                 clear_bit(BTRFS_INODE_NEEDS_FULL_SYNC,
2169                           &BTRFS_I(inode)->runtime_flags);
2170                 /*
2171                  * An ordered extent might have started before and completed
2172                  * already with io errors, in which case the inode was not
2173                  * updated and we end up here. So check the inode's mapping
2174                  * for any errors that might have happened since we last
2175                  * checked called fsync.
2176                  */
2177                 ret = filemap_check_wb_err(inode->i_mapping, file->f_wb_err);
2178                 up_write(&BTRFS_I(inode)->dio_sem);
2179                 inode_unlock(inode);
2180                 goto out;
2181         }
2182 
2183         /*
2184          * We use start here because we will need to wait on the IO to complete
2185          * in btrfs_sync_log, which could require joining a transaction (for
2186          * example checking cross references in the nocow path).  If we use join
2187          * here we could get into a situation where we're waiting on IO to
2188          * happen that is blocked on a transaction trying to commit.  With start
2189          * we inc the extwriter counter, so we wait for all extwriters to exit
2190          * before we start blocking joiners.  This comment is to keep somebody
2191          * from thinking they are super smart and changing this to
2192          * btrfs_join_transaction *cough*Josef*cough*.
2193          */
2194         trans = btrfs_start_transaction(root, 0);
2195         if (IS_ERR(trans)) {
2196                 ret = PTR_ERR(trans);
2197                 up_write(&BTRFS_I(inode)->dio_sem);
2198                 inode_unlock(inode);
2199                 goto out;
2200         }
2201 
2202         ret = btrfs_log_dentry_safe(trans, dentry, start, end, &ctx);
2203         if (ret < 0) {
2204                 /* Fallthrough and commit/free transaction. */
2205                 ret = 1;
2206         }
2207 
2208         /* we've logged all the items and now have a consistent
2209          * version of the file in the log.  It is possible that
2210          * someone will come in and modify the file, but that's
2211          * fine because the log is consistent on disk, and we
2212          * have references to all of the file's extents
2213          *
2214          * It is possible that someone will come in and log the
2215          * file again, but that will end up using the synchronization
2216          * inside btrfs_sync_log to keep things safe.
2217          */
2218         up_write(&BTRFS_I(inode)->dio_sem);
2219         inode_unlock(inode);
2220 
2221         if (ret != BTRFS_NO_LOG_SYNC) {
2222                 if (!ret) {
2223                         ret = btrfs_sync_log(trans, root, &ctx);
2224                         if (!ret) {
2225                                 ret = btrfs_end_transaction(trans);
2226                                 goto out;
2227                         }
2228                 }
2229                 ret = btrfs_commit_transaction(trans);
2230         } else {
2231                 ret = btrfs_end_transaction(trans);
2232         }
2233 out:
2234         ASSERT(list_empty(&ctx.list));
2235         err = file_check_and_advance_wb_err(file);
2236         if (!ret)
2237                 ret = err;
2238         return ret > 0 ? -EIO : ret;
2239 }
2240 
2241 static const struct vm_operations_struct btrfs_file_vm_ops = {
2242         .fault          = filemap_fault,
2243         .map_pages      = filemap_map_pages,
2244         .page_mkwrite   = btrfs_page_mkwrite,
2245 };
2246 
2247 static int btrfs_file_mmap(struct file  *filp, struct vm_area_struct *vma)
2248 {
2249         struct address_space *mapping = filp->f_mapping;
2250 
2251         if (!mapping->a_ops->readpage)
2252                 return -ENOEXEC;
2253 
2254         file_accessed(filp);
2255         vma->vm_ops = &btrfs_file_vm_ops;
2256 
2257         return 0;
2258 }
2259 
2260 static int hole_mergeable(struct btrfs_inode *inode, struct extent_buffer *leaf,
2261                           int slot, u64 start, u64 end)
2262 {
2263         struct btrfs_file_extent_item *fi;
2264         struct btrfs_key key;
2265 
2266         if (slot < 0 || slot >= btrfs_header_nritems(leaf))
2267                 return 0;
2268 
2269         btrfs_item_key_to_cpu(leaf, &key, slot);
2270         if (key.objectid != btrfs_ino(inode) ||
2271             key.type != BTRFS_EXTENT_DATA_KEY)
2272                 return 0;
2273 
2274         fi = btrfs_item_ptr(leaf, slot, struct btrfs_file_extent_item);
2275 
2276         if (btrfs_file_extent_type(leaf, fi) != BTRFS_FILE_EXTENT_REG)
2277                 return 0;
2278 
2279         if (btrfs_file_extent_disk_bytenr(leaf, fi))
2280                 return 0;
2281 
2282         if (key.offset == end)
2283                 return 1;
2284         if (key.offset + btrfs_file_extent_num_bytes(leaf, fi) == start)
2285                 return 1;
2286         return 0;
2287 }
2288 
2289 static int fill_holes(struct btrfs_trans_handle *trans,
2290                 struct btrfs_inode *inode,
2291                 struct btrfs_path *path, u64 offset, u64 end)
2292 {
2293         struct btrfs_fs_info *fs_info = trans->fs_info;
2294         struct btrfs_root *root = inode->root;
2295         struct extent_buffer *leaf;
2296         struct btrfs_file_extent_item *fi;
2297         struct extent_map *hole_em;
2298         struct extent_map_tree *em_tree = &inode->extent_tree;
2299         struct btrfs_key key;
2300         int ret;
2301 
2302         if (btrfs_fs_incompat(fs_info, NO_HOLES))
2303                 goto out;
2304 
2305         key.objectid = btrfs_ino(inode);
2306         key.type = BTRFS_EXTENT_DATA_KEY;
2307         key.offset = offset;
2308 
2309         ret = btrfs_search_slot(trans, root, &key, path, 0, 1);
2310         if (ret <= 0) {
2311                 /*
2312                  * We should have dropped this offset, so if we find it then
2313                  * something has gone horribly wrong.
2314                  */
2315                 if (ret == 0)
2316                         ret = -EINVAL;
2317                 return ret;
2318         }
2319 
2320         leaf = path->nodes[0];
2321         if (hole_mergeable(inode, leaf, path->slots[0] - 1, offset, end)) {
2322                 u64 num_bytes;
2323 
2324                 path->slots[0]--;
2325                 fi = btrfs_item_ptr(leaf, path->slots[0],
2326                                     struct btrfs_file_extent_item);
2327                 num_bytes = btrfs_file_extent_num_bytes(leaf, fi) +
2328                         end - offset;
2329                 btrfs_set_file_extent_num_bytes(leaf, fi, num_bytes);
2330                 btrfs_set_file_extent_ram_bytes(leaf, fi, num_bytes);
2331                 btrfs_set_file_extent_offset(leaf, fi, 0);
2332                 btrfs_mark_buffer_dirty(leaf);
2333                 goto out;
2334         }
2335 
2336         if (hole_mergeable(inode, leaf, path->slots[0], offset, end)) {
2337                 u64 num_bytes;
2338 
2339                 key.offset = offset;
2340                 btrfs_set_item_key_safe(fs_info, path, &key);
2341                 fi = btrfs_item_ptr(leaf, path->slots[0],
2342                                     struct btrfs_file_extent_item);
2343                 num_bytes = btrfs_file_extent_num_bytes(leaf, fi) + end -
2344                         offset;
2345                 btrfs_set_file_extent_num_bytes(leaf, fi, num_bytes);
2346                 btrfs_set_file_extent_ram_bytes(leaf, fi, num_bytes);
2347                 btrfs_set_file_extent_offset(leaf, fi, 0);
2348                 btrfs_mark_buffer_dirty(leaf);
2349                 goto out;
2350         }
2351         btrfs_release_path(path);
2352 
2353         ret = btrfs_insert_file_extent(trans, root, btrfs_ino(inode),
2354                         offset, 0, 0, end - offset, 0, end - offset, 0, 0, 0);
2355         if (ret)
2356                 return ret;
2357 
2358 out:
2359         btrfs_release_path(path);
2360 
2361         hole_em = alloc_extent_map();
2362         if (!hole_em) {
2363                 btrfs_drop_extent_cache(inode, offset, end - 1, 0);
2364                 set_bit(BTRFS_INODE_NEEDS_FULL_SYNC, &inode->runtime_flags);
2365         } else {
2366                 hole_em->start = offset;
2367                 hole_em->len = end - offset;
2368                 hole_em->ram_bytes = hole_em->len;
2369                 hole_em->orig_start = offset;
2370 
2371                 hole_em->block_start = EXTENT_MAP_HOLE;
2372                 hole_em->block_len = 0;
2373                 hole_em->orig_block_len = 0;
2374                 hole_em->bdev = fs_info->fs_devices->latest_bdev;
2375                 hole_em->compress_type = BTRFS_COMPRESS_NONE;
2376                 hole_em->generation = trans->transid;
2377 
2378                 do {
2379                         btrfs_drop_extent_cache(inode, offset, end - 1, 0);
2380                         write_lock(&em_tree->lock);
2381                         ret = add_extent_mapping(em_tree, hole_em, 1);
2382                         write_unlock(&em_tree->lock);
2383                 } while (ret == -EEXIST);
2384                 free_extent_map(hole_em);
2385                 if (ret)
2386                         set_bit(BTRFS_INODE_NEEDS_FULL_SYNC,
2387                                         &inode->runtime_flags);
2388         }
2389 
2390         return 0;
2391 }
2392 
2393 /*
2394  * Find a hole extent on given inode and change start/len to the end of hole
2395  * extent.(hole/vacuum extent whose em->start <= start &&
2396  *         em->start + em->len > start)
2397  * When a hole extent is found, return 1 and modify start/len.
2398  */
2399 static int find_first_non_hole(struct inode *inode, u64 *start, u64 *len)
2400 {
2401         struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
2402         struct extent_map *em;
2403         int ret = 0;
2404 
2405         em = btrfs_get_extent(BTRFS_I(inode), NULL, 0,
2406                               round_down(*start, fs_info->sectorsize),
2407                               round_up(*len, fs_info->sectorsize), 0);
2408         if (IS_ERR(em))
2409                 return PTR_ERR(em);
2410 
2411         /* Hole or vacuum extent(only exists in no-hole mode) */
2412         if (em->block_start == EXTENT_MAP_HOLE) {
2413                 ret = 1;
2414                 *len = em->start + em->len > *start + *len ?
2415                        0 : *start + *len - em->start - em->len;
2416                 *start = em->start + em->len;
2417         }
2418         free_extent_map(em);
2419         return ret;
2420 }
2421 
2422 static int btrfs_punch_hole_lock_range(struct inode *inode,
2423                                        const u64 lockstart,
2424                                        const u64 lockend,
2425                                        struct extent_state **cached_state)
2426 {
2427         while (1) {
2428                 struct btrfs_ordered_extent *ordered;
2429                 int ret;
2430 
2431                 truncate_pagecache_range(inode, lockstart, lockend);
2432 
2433                 lock_extent_bits(&BTRFS_I(inode)->io_tree, lockstart, lockend,
2434                                  cached_state);
2435                 ordered = btrfs_lookup_first_ordered_extent(inode, lockend);
2436 
2437                 /*
2438                  * We need to make sure we have no ordered extents in this range
2439                  * and nobody raced in and read a page in this range, if we did
2440                  * we need to try again.
2441                  */
2442                 if ((!ordered ||
2443                     (ordered->file_offset + ordered->len <= lockstart ||
2444                      ordered->file_offset > lockend)) &&
2445                      !filemap_range_has_page(inode->i_mapping,
2446                                              lockstart, lockend)) {
2447                         if (ordered)
2448                                 btrfs_put_ordered_extent(ordered);
2449                         break;
2450                 }
2451                 if (ordered)
2452                         btrfs_put_ordered_extent(ordered);
2453                 unlock_extent_cached(&BTRFS_I(inode)->io_tree, lockstart,
2454                                      lockend, cached_state);
2455                 ret = btrfs_wait_ordered_range(inode, lockstart,
2456                                                lockend - lockstart + 1);
2457                 if (ret)
2458                         return ret;
2459         }
2460         return 0;
2461 }
2462 
2463 static int btrfs_insert_clone_extent(struct btrfs_trans_handle *trans,
2464                                      struct inode *inode,
2465                                      struct btrfs_path *path,
2466                                      struct btrfs_clone_extent_info *clone_info,
2467                                      const u64 clone_len)
2468 {
2469         struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
2470         struct btrfs_root *root = BTRFS_I(inode)->root;
2471         struct btrfs_file_extent_item *extent;
2472         struct extent_buffer *leaf;
2473         struct btrfs_key key;
2474         int slot;
2475         struct btrfs_ref ref = { 0 };
2476         u64 ref_offset;
2477         int ret;
2478 
2479         if (clone_len == 0)
2480                 return 0;
2481 
2482         if (clone_info->disk_offset == 0 &&
2483             btrfs_fs_incompat(fs_info, NO_HOLES))
2484                 return 0;
2485 
2486         key.objectid = btrfs_ino(BTRFS_I(inode));
2487         key.type = BTRFS_EXTENT_DATA_KEY;
2488         key.offset = clone_info->file_offset;
2489         ret = btrfs_insert_empty_item(trans, root, path, &key,
2490                                       clone_info->item_size);
2491         if (ret)
2492                 return ret;
2493         leaf = path->nodes[0];
2494         slot = path->slots[0];
2495         write_extent_buffer(leaf, clone_info->extent_buf,
2496                             btrfs_item_ptr_offset(leaf, slot),
2497                             clone_info->item_size);
2498         extent = btrfs_item_ptr(leaf, slot, struct btrfs_file_extent_item);
2499         btrfs_set_file_extent_offset(leaf, extent, clone_info->data_offset);
2500         btrfs_set_file_extent_num_bytes(leaf, extent, clone_len);
2501         btrfs_mark_buffer_dirty(leaf);
2502         btrfs_release_path(path);
2503 
2504         /* If it's a hole, nothing more needs to be done. */
2505         if (clone_info->disk_offset == 0)
2506                 return 0;
2507 
2508         inode_add_bytes(inode, clone_len);
2509         btrfs_init_generic_ref(&ref, BTRFS_ADD_DELAYED_REF,
2510                                clone_info->disk_offset,
2511                                clone_info->disk_len, 0);
2512         ref_offset = clone_info->file_offset - clone_info->data_offset;
2513         btrfs_init_data_ref(&ref, root->root_key.objectid,
2514                             btrfs_ino(BTRFS_I(inode)), ref_offset);
2515         ret = btrfs_inc_extent_ref(trans, &ref);
2516 
2517         return ret;
2518 }
2519 
2520 /*
2521  * The respective range must have been previously locked, as well as the inode.
2522  * The end offset is inclusive (last byte of the range).
2523  * @clone_info is NULL for fallocate's hole punching and non-NULL for extent
2524  * cloning.
2525  * When cloning, we don't want to end up in a state where we dropped extents
2526  * without inserting a new one, so we must abort the transaction to avoid a
2527  * corruption.
2528  */
2529 int btrfs_punch_hole_range(struct inode *inode, struct btrfs_path *path,
2530                            const u64 start, const u64 end,
2531                            struct btrfs_clone_extent_info *clone_info,
2532                            struct btrfs_trans_handle **trans_out)
2533 {
2534         struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
2535         u64 min_size = btrfs_calc_insert_metadata_size(fs_info, 1);
2536         u64 ino_size = round_up(inode->i_size, fs_info->sectorsize);
2537         struct btrfs_root *root = BTRFS_I(inode)->root;
2538         struct btrfs_trans_handle *trans = NULL;
2539         struct btrfs_block_rsv *rsv;
2540         unsigned int rsv_count;
2541         u64 cur_offset;
2542         u64 drop_end;
2543         u64 len = end - start;
2544         int ret = 0;
2545 
2546         if (end <= start)
2547                 return -EINVAL;
2548 
2549         rsv = btrfs_alloc_block_rsv(fs_info, BTRFS_BLOCK_RSV_TEMP);
2550         if (!rsv) {
2551                 ret = -ENOMEM;
2552                 goto out;
2553         }
2554         rsv->size = btrfs_calc_insert_metadata_size(fs_info, 1);
2555         rsv->failfast = 1;
2556 
2557         /*
2558          * 1 - update the inode
2559          * 1 - removing the extents in the range
2560          * 1 - adding the hole extent if no_holes isn't set or if we are cloning
2561          *     an extent
2562          */
2563         if (!btrfs_fs_incompat(fs_info, NO_HOLES) || clone_info)
2564                 rsv_count = 3;
2565         else
2566                 rsv_count = 2;
2567 
2568         trans = btrfs_start_transaction(root, rsv_count);
2569         if (IS_ERR(trans)) {
2570                 ret = PTR_ERR(trans);
2571                 trans = NULL;
2572                 goto out_free;
2573         }
2574 
2575         ret = btrfs_block_rsv_migrate(&fs_info->trans_block_rsv, rsv,
2576                                       min_size, false);
2577         BUG_ON(ret);
2578         trans->block_rsv = rsv;
2579 
2580         cur_offset = start;
2581         while (cur_offset < end) {
2582                 ret = __btrfs_drop_extents(trans, root, inode, path,
2583                                            cur_offset, end + 1, &drop_end,
2584                                            1, 0, 0, NULL);
2585                 if (ret != -ENOSPC) {
2586                         /*
2587                          * When cloning we want to avoid transaction aborts when
2588                          * nothing was done and we are attempting to clone parts
2589                          * of inline extents, in such cases -EOPNOTSUPP is
2590                          * returned by __btrfs_drop_extents() without having
2591                          * changed anything in the file.
2592                          */
2593                         if (clone_info && ret && ret != -EOPNOTSUPP)
2594                                 btrfs_abort_transaction(trans, ret);
2595                         break;
2596                 }
2597 
2598                 trans->block_rsv = &fs_info->trans_block_rsv;
2599 
2600                 if (!clone_info && cur_offset < drop_end &&
2601                     cur_offset < ino_size) {
2602                         ret = fill_holes(trans, BTRFS_I(inode), path,
2603                                         cur_offset, drop_end);
2604                         if (ret) {
2605                                 /*
2606                                  * If we failed then we didn't insert our hole
2607                                  * entries for the area we dropped, so now the
2608                                  * fs is corrupted, so we must abort the
2609                                  * transaction.
2610                                  */
2611                                 btrfs_abort_transaction(trans, ret);
2612                                 break;
2613                         }
2614                 }
2615 
2616                 if (clone_info && drop_end > clone_info->file_offset) {
2617                         u64 clone_len = drop_end - clone_info->file_offset;
2618 
2619                         ret = btrfs_insert_clone_extent(trans, inode, path,
2620                                                         clone_info, clone_len);
2621                         if (ret) {
2622                                 btrfs_abort_transaction(trans, ret);
2623                                 break;
2624                         }
2625                         clone_info->data_len -= clone_len;
2626                         clone_info->data_offset += clone_len;
2627                         clone_info->file_offset += clone_len;
2628                 }
2629 
2630                 cur_offset = drop_end;
2631 
2632                 ret = btrfs_update_inode(trans, root, inode);
2633                 if (ret)
2634                         break;
2635 
2636                 btrfs_end_transaction(trans);
2637                 btrfs_btree_balance_dirty(fs_info);
2638 
2639                 trans = btrfs_start_transaction(root, rsv_count);
2640                 if (IS_ERR(trans)) {
2641                         ret = PTR_ERR(trans);
2642                         trans = NULL;
2643                         break;
2644                 }
2645 
2646                 ret = btrfs_block_rsv_migrate(&fs_info->trans_block_rsv,
2647                                               rsv, min_size, false);
2648                 BUG_ON(ret);    /* shouldn't happen */
2649                 trans->block_rsv = rsv;
2650 
2651                 if (!clone_info) {
2652                         ret = find_first_non_hole(inode, &cur_offset, &len);
2653                         if (unlikely(ret < 0))
2654                                 break;
2655                         if (ret && !len) {
2656                                 ret = 0;
2657                                 break;
2658                         }
2659                 }
2660         }
2661 
2662         /*
2663          * If we were cloning, force the next fsync to be a full one since we
2664          * we replaced (or just dropped in the case of cloning holes when
2665          * NO_HOLES is enabled) extents and extent maps.
2666          * This is for the sake of simplicity, and cloning into files larger
2667          * than 16Mb would force the full fsync any way (when
2668          * try_release_extent_mapping() is invoked during page cache truncation.
2669          */
2670         if (clone_info)
2671                 set_bit(BTRFS_INODE_NEEDS_FULL_SYNC,
2672                         &BTRFS_I(inode)->runtime_flags);
2673 
2674         if (ret)
2675                 goto out_trans;
2676 
2677         trans->block_rsv = &fs_info->trans_block_rsv;
2678         /*
2679          * If we are using the NO_HOLES feature we might have had already an
2680          * hole that overlaps a part of the region [lockstart, lockend] and
2681          * ends at (or beyond) lockend. Since we have no file extent items to
2682          * represent holes, drop_end can be less than lockend and so we must
2683          * make sure we have an extent map representing the existing hole (the
2684          * call to __btrfs_drop_extents() might have dropped the existing extent
2685          * map representing the existing hole), otherwise the fast fsync path
2686          * will not record the existence of the hole region
2687          * [existing_hole_start, lockend].
2688          */
2689         if (drop_end <= end)
2690                 drop_end = end + 1;
2691         /*
2692          * Don't insert file hole extent item if it's for a range beyond eof
2693          * (because it's useless) or if it represents a 0 bytes range (when
2694          * cur_offset == drop_end).
2695          */
2696         if (!clone_info && cur_offset < ino_size && cur_offset < drop_end) {
2697                 ret = fill_holes(trans, BTRFS_I(inode), path,
2698                                 cur_offset, drop_end);
2699                 if (ret) {
2700                         /* Same comment as above. */
2701                         btrfs_abort_transaction(trans, ret);
2702                         goto out_trans;
2703                 }
2704         }
2705         if (clone_info) {
2706                 ret = btrfs_insert_clone_extent(trans, inode, path, clone_info,
2707                                                 clone_info->data_len);
2708                 if (ret) {
2709                         btrfs_abort_transaction(trans, ret);
2710                         goto out_trans;
2711                 }
2712         }
2713 
2714 out_trans:
2715         if (!trans)
2716                 goto out_free;
2717 
2718         trans->block_rsv = &fs_info->trans_block_rsv;
2719         if (ret)
2720                 btrfs_end_transaction(trans);
2721         else
2722                 *trans_out = trans;
2723 out_free:
2724         btrfs_free_block_rsv(fs_info, rsv);
2725 out:
2726         return ret;
2727 }
2728 
2729 static int btrfs_punch_hole(struct inode *inode, loff_t offset, loff_t len)
2730 {
2731         struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
2732         struct btrfs_root *root = BTRFS_I(inode)->root;
2733         struct extent_state *cached_state = NULL;
2734         struct btrfs_path *path;
2735         struct btrfs_trans_handle *trans = NULL;
2736         u64 lockstart;
2737         u64 lockend;
2738         u64 tail_start;
2739         u64 tail_len;
2740         u64 orig_start = offset;
2741         int ret = 0;
2742         bool same_block;
2743         u64 ino_size;
2744         bool truncated_block = false;
2745         bool updated_inode = false;
2746 
2747         ret = btrfs_wait_ordered_range(inode, offset, len);
2748         if (ret)
2749                 return ret;
2750 
2751         inode_lock(inode);
2752         ino_size = round_up(inode->i_size, fs_info->sectorsize);
2753         ret = find_first_non_hole(inode, &offset, &len);
2754         if (ret < 0)
2755                 goto out_only_mutex;
2756         if (ret && !len) {
2757                 /* Already in a large hole */
2758                 ret = 0;
2759                 goto out_only_mutex;
2760         }
2761 
2762         lockstart = round_up(offset, btrfs_inode_sectorsize(inode));
2763         lockend = round_down(offset + len,
2764                              btrfs_inode_sectorsize(inode)) - 1;
2765         same_block = (BTRFS_BYTES_TO_BLKS(fs_info, offset))
2766                 == (BTRFS_BYTES_TO_BLKS(fs_info, offset + len - 1));
2767         /*
2768          * We needn't truncate any block which is beyond the end of the file
2769          * because we are sure there is no data there.
2770          */
2771         /*
2772          * Only do this if we are in the same block and we aren't doing the
2773          * entire block.
2774          */
2775         if (same_block && len < fs_info->sectorsize) {
2776                 if (offset < ino_size) {
2777                         truncated_block = true;
2778                         ret = btrfs_truncate_block(inode, offset, len, 0);
2779                 } else {
2780                         ret = 0;
2781                 }
2782                 goto out_only_mutex;
2783         }
2784 
2785         /* zero back part of the first block */
2786         if (offset < ino_size) {
2787                 truncated_block = true;
2788                 ret = btrfs_truncate_block(inode, offset, 0, 0);
2789                 if (ret) {
2790                         inode_unlock(inode);
2791                         return ret;
2792                 }
2793         }
2794 
2795         /* Check the aligned pages after the first unaligned page,
2796          * if offset != orig_start, which means the first unaligned page
2797          * including several following pages are already in holes,
2798          * the extra check can be skipped */
2799         if (offset == orig_start) {
2800                 /* after truncate page, check hole again */
2801                 len = offset + len - lockstart;
2802                 offset = lockstart;
2803                 ret = find_first_non_hole(inode, &offset, &len);
2804                 if (ret < 0)
2805                         goto out_only_mutex;
2806                 if (ret && !len) {
2807                         ret = 0;
2808                         goto out_only_mutex;
2809                 }
2810                 lockstart = offset;
2811         }
2812 
2813         /* Check the tail unaligned part is in a hole */
2814         tail_start = lockend + 1;
2815         tail_len = offset + len - tail_start;
2816         if (tail_len) {
2817                 ret = find_first_non_hole(inode, &tail_start, &tail_len);
2818                 if (unlikely(ret < 0))
2819                         goto out_only_mutex;
2820                 if (!ret) {
2821                         /* zero the front end of the last page */
2822                         if (tail_start + tail_len < ino_size) {
2823                                 truncated_block = true;
2824                                 ret = btrfs_truncate_block(inode,
2825                                                         tail_start + tail_len,
2826                                                         0, 1);
2827                                 if (ret)
2828                                         goto out_only_mutex;
2829                         }
2830                 }
2831         }
2832 
2833         if (lockend < lockstart) {
2834                 ret = 0;
2835                 goto out_only_mutex;
2836         }
2837 
2838         ret = btrfs_punch_hole_lock_range(inode, lockstart, lockend,
2839                                           &cached_state);
2840         if (ret)
2841                 goto out_only_mutex;
2842 
2843         path = btrfs_alloc_path();
2844         if (!path) {
2845                 ret = -ENOMEM;
2846                 goto out;
2847         }
2848 
2849         ret = btrfs_punch_hole_range(inode, path, lockstart, lockend, NULL,
2850                                      &trans);
2851         btrfs_free_path(path);
2852         if (ret)
2853                 goto out;
2854 
2855         ASSERT(trans != NULL);
2856         inode_inc_iversion(inode);
2857         inode->i_mtime = inode->i_ctime = current_time(inode);
2858         ret = btrfs_update_inode(trans, root, inode);
2859         updated_inode = true;
2860         btrfs_end_transaction(trans);
2861         btrfs_btree_balance_dirty(fs_info);
2862 out:
2863         unlock_extent_cached(&BTRFS_I(inode)->io_tree, lockstart, lockend,
2864                              &cached_state);
2865 out_only_mutex:
2866         if (!updated_inode && truncated_block && !ret) {
2867                 /*
2868                  * If we only end up zeroing part of a page, we still need to
2869                  * update the inode item, so that all the time fields are
2870                  * updated as well as the necessary btrfs inode in memory fields
2871                  * for detecting, at fsync time, if the inode isn't yet in the
2872                  * log tree or it's there but not up to date.
2873                  */
2874                 struct timespec64 now = current_time(inode);
2875 
2876                 inode_inc_iversion(inode);
2877                 inode->i_mtime = now;
2878                 inode->i_ctime = now;
2879                 trans = btrfs_start_transaction(root, 1);
2880                 if (IS_ERR(trans)) {
2881                         ret = PTR_ERR(trans);
2882                 } else {
2883                         int ret2;
2884 
2885                         ret = btrfs_update_inode(trans, root, inode);
2886                         ret2 = btrfs_end_transaction(trans);
2887                         if (!ret)
2888                                 ret = ret2;
2889                 }
2890         }
2891         inode_unlock(inode);
2892         return ret;
2893 }
2894 
2895 /* Helper structure to record which range is already reserved */
2896 struct falloc_range {
2897         struct list_head list;
2898         u64 start;
2899         u64 len;
2900 };
2901 
2902 /*
2903  * Helper function to add falloc range
2904  *
2905  * Caller should have locked the larger range of extent containing
2906  * [start, len)
2907  */
2908 static int add_falloc_range(struct list_head *head, u64 start, u64 len)
2909 {
2910         struct falloc_range *prev = NULL;
2911         struct falloc_range *range = NULL;
2912 
2913         if (list_empty(head))
2914                 goto insert;
2915 
2916         /*
2917          * As fallocate iterate by bytenr order, we only need to check
2918          * the last range.
2919          */
2920         prev = list_entry(head->prev, struct falloc_range, list);
2921         if (prev->start + prev->len == start) {
2922                 prev->len += len;
2923                 return 0;
2924         }
2925 insert:
2926         range = kmalloc(sizeof(*range), GFP_KERNEL);
2927         if (!range)
2928                 return -ENOMEM;
2929         range->start = start;
2930         range->len = len;
2931         list_add_tail(&range->list, head);
2932         return 0;
2933 }
2934 
2935 static int btrfs_fallocate_update_isize(struct inode *inode,
2936                                         const u64 end,
2937                                         const int mode)
2938 {
2939         struct btrfs_trans_handle *trans;
2940         struct btrfs_root *root = BTRFS_I(inode)->root;
2941         int ret;
2942         int ret2;
2943 
2944         if (mode & FALLOC_FL_KEEP_SIZE || end <= i_size_read(inode))
2945                 return 0;
2946 
2947         trans = btrfs_start_transaction(root, 1);
2948         if (IS_ERR(trans))
2949                 return PTR_ERR(trans);
2950 
2951         inode->i_ctime = current_time(inode);
2952         i_size_write(inode, end);
2953         btrfs_ordered_update_i_size(inode, end, NULL);
2954         ret = btrfs_update_inode(trans, root, inode);
2955         ret2 = btrfs_end_transaction(trans);
2956 
2957         return ret ? ret : ret2;
2958 }
2959 
2960 enum {
2961         RANGE_BOUNDARY_WRITTEN_EXTENT,
2962         RANGE_BOUNDARY_PREALLOC_EXTENT,
2963         RANGE_BOUNDARY_HOLE,
2964 };
2965 
2966 static int btrfs_zero_range_check_range_boundary(struct inode *inode,
2967                                                  u64 offset)
2968 {
2969         const u64 sectorsize = btrfs_inode_sectorsize(inode);
2970         struct extent_map *em;
2971         int ret;
2972 
2973         offset = round_down(offset, sectorsize);
2974         em = btrfs_get_extent(BTRFS_I(inode), NULL, 0, offset, sectorsize, 0);
2975         if (IS_ERR(em))
2976                 return PTR_ERR(em);
2977 
2978         if (em->block_start == EXTENT_MAP_HOLE)
2979                 ret = RANGE_BOUNDARY_HOLE;
2980         else if (test_bit(EXTENT_FLAG_PREALLOC, &em->flags))
2981                 ret = RANGE_BOUNDARY_PREALLOC_EXTENT;
2982         else
2983                 ret = RANGE_BOUNDARY_WRITTEN_EXTENT;
2984 
2985         free_extent_map(em);
2986         return ret;
2987 }
2988 
2989 static int btrfs_zero_range(struct inode *inode,
2990                             loff_t offset,
2991                             loff_t len,
2992                             const int mode)
2993 {
2994         struct btrfs_fs_info *fs_info = BTRFS_I(inode)->root->fs_info;
2995         struct extent_map *em;
2996         struct extent_changeset *data_reserved = NULL;
2997         int ret;
2998         u64 alloc_hint = 0;
2999         const u64 sectorsize = btrfs_inode_sectorsize(inode);
3000         u64 alloc_start = round_down(offset, sectorsize);
3001         u64 alloc_end = round_up(offset + len, sectorsize);
3002         u64 bytes_to_reserve = 0;
3003         bool space_reserved = false;
3004 
3005         inode_dio_wait(inode);
3006 
3007         em = btrfs_get_extent(BTRFS_I(inode), NULL, 0,
3008                               alloc_start, alloc_end - alloc_start, 0);
3009         if (IS_ERR(em)) {
3010                 ret = PTR_ERR(em);
3011                 goto out;
3012         }
3013 
3014         /*
3015          * Avoid hole punching and extent allocation for some cases. More cases
3016          * could be considered, but these are unlikely common and we keep things
3017          * as simple as possible for now. Also, intentionally, if the target
3018          * range contains one or more prealloc extents together with regular
3019          * extents and holes, we drop all the existing extents and allocate a
3020          * new prealloc extent, so that we get a larger contiguous disk extent.
3021          */
3022         if (em->start <= alloc_start &&
3023             test_bit(EXTENT_FLAG_PREALLOC, &em->flags)) {
3024                 const u64 em_end = em->start + em->len;
3025 
3026                 if (em_end >= offset + len) {
3027                         /*
3028                          * The whole range is already a prealloc extent,
3029                          * do nothing except updating the inode's i_size if
3030                          * needed.
3031                          */
3032                         free_extent_map(em);
3033                         ret = btrfs_fallocate_update_isize(inode, offset + len,
3034                                                            mode);
3035                         goto out;
3036                 }
3037                 /*
3038                  * Part of the range is already a prealloc extent, so operate
3039                  * only on the remaining part of the range.
3040                  */
3041                 alloc_start = em_end;
3042                 ASSERT(IS_ALIGNED(alloc_start, sectorsize));
3043                 len = offset + len - alloc_start;
3044                 offset = alloc_start;
3045                 alloc_hint = em->block_start + em->len;
3046         }
3047         free_extent_map(em);
3048 
3049         if (BTRFS_BYTES_TO_BLKS(fs_info, offset) ==
3050             BTRFS_BYTES_TO_BLKS(fs_info, offset + len - 1)) {
3051                 em = btrfs_get_extent(BTRFS_I(inode), NULL, 0,
3052                                       alloc_start, sectorsize, 0);
3053                 if (IS_ERR(em)) {
3054                         ret = PTR_ERR(em);
3055                         goto out;
3056                 }
3057 
3058                 if (test_bit(EXTENT_FLAG_PREALLOC, &em->flags)) {
3059                         free_extent_map(em);
3060                         ret = btrfs_fallocate_update_isize(inode, offset + len,
3061                                                            mode);
3062                         goto out;
3063                 }
3064                 if (len < sectorsize && em->block_start != EXTENT_MAP_HOLE) {
3065                         free_extent_map(em);
3066                         ret = btrfs_truncate_block(inode, offset, len, 0);
3067                         if (!ret)
3068                                 ret = btrfs_fallocate_update_isize(inode,
3069                                                                    offset + len,
3070                                                                    mode);
3071                         return ret;
3072                 }
3073                 free_extent_map(em);
3074                 alloc_start = round_down(offset, sectorsize);
3075                 alloc_end = alloc_start + sectorsize;
3076                 goto reserve_space;
3077         }
3078 
3079         alloc_start = round_up(offset, sectorsize);
3080         alloc_end = round_down(offset + len, sectorsize);
3081 
3082         /*
3083          * For unaligned ranges, check the pages at the boundaries, they might
3084          * map to an extent, in which case we need to partially zero them, or
3085          * they might map to a hole, in which case we need our allocation range
3086          * to cover them.
3087          */
3088         if (!IS_ALIGNED(offset, sectorsize)) {
3089                 ret = btrfs_zero_range_check_range_boundary(inode, offset);
3090                 if (ret < 0)
3091                         goto out;
3092                 if (ret == RANGE_BOUNDARY_HOLE) {
3093                         alloc_start = round_down(offset, sectorsize);
3094                         ret = 0;
3095                 } else if (ret == RANGE_BOUNDARY_WRITTEN_EXTENT) {
3096                         ret = btrfs_truncate_block(inode, offset, 0, 0);
3097                         if (ret)
3098                                 goto out;
3099                 } else {
3100                         ret = 0;
3101                 }
3102         }
3103 
3104         if (!IS_ALIGNED(offset + len, sectorsize)) {
3105                 ret = btrfs_zero_range_check_range_boundary(inode,
3106                                                             offset + len);
3107                 if (ret < 0)
3108                         goto out;
3109                 if (ret == RANGE_BOUNDARY_HOLE) {
3110                         alloc_end = round_up(offset + len, sectorsize);
3111                         ret = 0;
3112                 } else if (ret == RANGE_BOUNDARY_WRITTEN_EXTENT) {
3113                         ret = btrfs_truncate_block(inode, offset + len, 0, 1);
3114                         if (ret)
3115                                 goto out;
3116                 } else {
3117                         ret = 0;
3118                 }
3119         }
3120 
3121 reserve_space:
3122         if (alloc_start < alloc_end) {
3123                 struct extent_state *cached_state = NULL;
3124                 const u64 lockstart = alloc_start;
3125                 const u64 lockend = alloc_end - 1;
3126 
3127                 bytes_to_reserve = alloc_end - alloc_start;
3128                 ret = btrfs_alloc_data_chunk_ondemand(BTRFS_I(inode),
3129                                                       bytes_to_reserve);
3130                 if (ret < 0)
3131                         goto out;
3132                 space_reserved = true;
3133                 ret = btrfs_qgroup_reserve_data(inode, &data_reserved,
3134                                                 alloc_start, bytes_to_reserve);
3135                 if (ret)
3136                         goto out;
3137                 ret = btrfs_punch_hole_lock_range(inode, lockstart, lockend,
3138                                                   &cached_state);
3139                 if (ret)
3140                         goto out;
3141                 ret = btrfs_prealloc_file_range(inode, mode, alloc_start,
3142                                                 alloc_end - alloc_start,
3143                                                 i_blocksize(inode),
3144                                                 offset + len, &alloc_hint);
3145                 unlock_extent_cached(&BTRFS_I(inode)->io_tree, lockstart,
3146                                      lockend, &cached_state);
3147                 /* btrfs_prealloc_file_range releases reserved space on error */
3148                 if (ret) {
3149                         space_reserved = false;
3150                         goto out;
3151                 }
3152         }
3153         ret = btrfs_fallocate_update_isize(inode, offset + len, mode);
3154  out:
3155         if (ret && space_reserved)
3156                 btrfs_free_reserved_data_space(inode, data_reserved,
3157                                                alloc_start, bytes_to_reserve);
3158         extent_changeset_free(data_reserved);
3159 
3160         return ret;
3161 }
3162 
3163 static long btrfs_fallocate(struct file *file, int mode,
3164                             loff_t offset, loff_t len)
3165 {
3166         struct inode *inode = file_inode(file);
3167         struct extent_state *cached_state = NULL;
3168         struct extent_changeset *data_reserved = NULL;
3169         struct falloc_range *range;
3170         struct falloc_range *tmp;
3171         struct list_head reserve_list;
3172         u64 cur_offset;
3173         u64 last_byte;
3174         u64 alloc_start;
3175         u64 alloc_end;
3176         u64 alloc_hint = 0;
3177         u64 locked_end;
3178         u64 actual_end = 0;
3179         struct extent_map *em;
3180         int blocksize = btrfs_inode_sectorsize(inode);
3181         int ret;
3182 
3183         alloc_start = round_down(offset, blocksize);
3184         alloc_end = round_up(offset + len, blocksize);
3185         cur_offset = alloc_start;
3186 
3187         /* Make sure we aren't being give some crap mode */
3188         if (mode & ~(FALLOC_FL_KEEP_SIZE | FALLOC_FL_PUNCH_HOLE |
3189                      FALLOC_FL_ZERO_RANGE))
3190                 return -EOPNOTSUPP;
3191 
3192         if (mode & FALLOC_FL_PUNCH_HOLE)
3193                 return btrfs_punch_hole(inode, offset, len);
3194 
3195         /*
3196          * Only trigger disk allocation, don't trigger qgroup reserve
3197          *
3198          * For qgroup space, it will be checked later.
3199          */
3200         if (!(mode & FALLOC_FL_ZERO_RANGE)) {
3201                 ret = btrfs_alloc_data_chunk_ondemand(BTRFS_I(inode),
3202                                                       alloc_end - alloc_start);
3203                 if (ret < 0)
3204                         return ret;
3205         }
3206 
3207         inode_lock(inode);
3208 
3209         if (!(mode & FALLOC_FL_KEEP_SIZE) && offset + len > inode->i_size) {
3210                 ret = inode_newsize_ok(inode, offset + len);
3211                 if (ret)
3212                         goto out;
3213         }
3214 
3215         /*
3216          * TODO: Move these two operations after we have checked
3217          * accurate reserved space, or fallocate can still fail but
3218          * with page truncated or size expanded.
3219          *
3220          * But that's a minor problem and won't do much harm BTW.
3221          */
3222         if (alloc_start > inode->i_size) {
3223                 ret = btrfs_cont_expand(inode, i_size_read(inode),
3224                                         alloc_start);
3225                 if (ret)
3226                         goto out;
3227         } else if (offset + len > inode->i_size) {
3228                 /*
3229                  * If we are fallocating from the end of the file onward we
3230                  * need to zero out the end of the block if i_size lands in the
3231                  * middle of a block.
3232                  */
3233                 ret = btrfs_truncate_block(inode, inode->i_size, 0, 0);
3234                 if (ret)
3235                         goto out;
3236         }
3237 
3238         /*
3239          * wait for ordered IO before we have any locks.  We'll loop again
3240          * below with the locks held.
3241          */
3242         ret = btrfs_wait_ordered_range(inode, alloc_start,
3243                                        alloc_end - alloc_start);
3244         if (ret)
3245                 goto out;
3246 
3247         if (mode & FALLOC_FL_ZERO_RANGE) {
3248                 ret = btrfs_zero_range(inode, offset, len, mode);
3249                 inode_unlock(inode);
3250                 return ret;
3251         }
3252 
3253         locked_end = alloc_end - 1;
3254         while (1) {
3255                 struct btrfs_ordered_extent *ordered;
3256 
3257                 /* the extent lock is ordered inside the running
3258                  * transaction
3259                  */
3260                 lock_extent_bits(&BTRFS_I(inode)->io_tree, alloc_start,
3261                                  locked_end, &cached_state);
3262                 ordered = btrfs_lookup_first_ordered_extent(inode, locked_end);
3263 
3264                 if (ordered &&
3265                     ordered->file_offset + ordered->len > alloc_start &&
3266                     ordered->file_offset < alloc_end) {
3267                         btrfs_put_ordered_extent(ordered);
3268                         unlock_extent_cached(&BTRFS_I(inode)->io_tree,
3269                                              alloc_start, locked_end,
3270                                              &cached_state);
3271                         /*
3272                          * we can't wait on the range with the transaction
3273                          * running or with the extent lock held
3274                          */
3275                         ret = btrfs_wait_ordered_range(inode, alloc_start,
3276                                                        alloc_end - alloc_start);
3277                         if (ret)
3278                                 goto out;
3279                 } else {
3280                         if (ordered)
3281                                 btrfs_put_ordered_extent(ordered);
3282                         break;
3283                 }
3284         }
3285 
3286         /* First, check if we exceed the qgroup limit */
3287         INIT_LIST_HEAD(&reserve_list);
3288         while (cur_offset < alloc_end) {
3289                 em = btrfs_get_extent(BTRFS_I(inode), NULL, 0, cur_offset,
3290                                       alloc_end - cur_offset, 0);
3291                 if (IS_ERR(em)) {
3292                         ret = PTR_ERR(em);
3293                         break;
3294                 }
3295                 last_byte = min(extent_map_end(em), alloc_end);
3296                 actual_end = min_t(u64, extent_map_end(em), offset + len);
3297                 last_byte = ALIGN(last_byte, blocksize);
3298                 if (em->block_start == EXTENT_MAP_HOLE ||
3299                     (cur_offset >= inode->i_size &&
3300                      !test_bit(EXTENT_FLAG_PREALLOC, &em->flags))) {
3301                         ret = add_falloc_range(&reserve_list, cur_offset,
3302                                                last_byte - cur_offset);
3303                         if (ret < 0) {
3304                                 free_extent_map(em);
3305                                 break;
3306                         }
3307                         ret = btrfs_qgroup_reserve_data(inode, &data_reserved,
3308                                         cur_offset, last_byte - cur_offset);
3309                         if (ret < 0) {
3310                                 cur_offset = last_byte;
3311                                 free_extent_map(em);
3312                                 break;
3313                         }
3314                 } else {
3315                         /*
3316                          * Do not need to reserve unwritten extent for this
3317                          * range, free reserved data space first, otherwise
3318                          * it'll result in false ENOSPC error.
3319                          */
3320                         btrfs_free_reserved_data_space(inode, data_reserved,
3321                                         cur_offset, last_byte - cur_offset);
3322                 }
3323                 free_extent_map(em);
3324                 cur_offset = last_byte;
3325         }
3326 
3327         /*
3328          * If ret is still 0, means we're OK to fallocate.
3329          * Or just cleanup the list and exit.
3330          */
3331         list_for_each_entry_safe(range, tmp, &reserve_list, list) {
3332                 if (!ret)
3333                         ret = btrfs_prealloc_file_range(inode, mode,
3334                                         range->start,
3335                                         range->len, i_blocksize(inode),
3336                                         offset + len, &alloc_hint);
3337                 else
3338                         btrfs_free_reserved_data_space(inode,
3339                                         data_reserved, range->start,
3340                                         range->len);
3341                 list_del(&range->list);
3342                 kfree(range);
3343         }
3344         if (ret < 0)
3345                 goto out_unlock;
3346 
3347         /*
3348          * We didn't need to allocate any more space, but we still extended the
3349          * size of the file so we need to update i_size and the inode item.
3350          */
3351         ret = btrfs_fallocate_update_isize(inode, actual_end, mode);
3352 out_unlock:
3353         unlock_extent_cached(&BTRFS_I(inode)->io_tree, alloc_start, locked_end,
3354                              &cached_state);
3355 out:
3356         inode_unlock(inode);
3357         /* Let go of our reservation. */
3358         if (ret != 0 && !(mode & FALLOC_FL_ZERO_RANGE))
3359                 btrfs_free_reserved_data_space(inode, data_reserved,
3360                                 cur_offset, alloc_end - cur_offset);
3361         extent_changeset_free(data_reserved);
3362         return ret;
3363 }
3364 
3365 static int find_desired_extent(struct inode *inode, loff_t *offset, int whence)
3366 {
3367         struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
3368         struct extent_map *em = NULL;
3369         struct extent_state *cached_state = NULL;
3370         u64 lockstart;
3371         u64 lockend;
3372         u64 start;
3373         u64 len;
3374         int ret = 0;
3375 
3376         if (inode->i_size == 0)
3377                 return -ENXIO;
3378 
3379         /*
3380          * *offset can be negative, in this case we start finding DATA/HOLE from
3381          * the very start of the file.
3382          */
3383         start = max_t(loff_t, 0, *offset);
3384 
3385         lockstart = round_down(start, fs_info->sectorsize);
3386         lockend = round_up(i_size_read(inode),
3387                            fs_info->sectorsize);
3388         if (lockend <= lockstart)
3389                 lockend = lockstart + fs_info->sectorsize;
3390         lockend--;
3391         len = lockend - lockstart + 1;
3392 
3393         lock_extent_bits(&BTRFS_I(inode)->io_tree, lockstart, lockend,
3394                          &cached_state);
3395 
3396         while (start < inode->i_size) {
3397                 em = btrfs_get_extent_fiemap(BTRFS_I(inode), start, len);
3398                 if (IS_ERR(em)) {
3399                         ret = PTR_ERR(em);
3400                         em = NULL;
3401                         break;
3402                 }
3403 
3404                 if (whence == SEEK_HOLE &&
3405                     (em->block_start == EXTENT_MAP_HOLE ||
3406                      test_bit(EXTENT_FLAG_PREALLOC, &em->flags)))
3407                         break;
3408                 else if (whence == SEEK_DATA &&
3409                            (em->block_start != EXTENT_MAP_HOLE &&
3410                             !test_bit(EXTENT_FLAG_PREALLOC, &em->flags)))
3411                         break;
3412 
3413                 start = em->start + em->len;
3414                 free_extent_map(em);
3415                 em = NULL;
3416                 cond_resched();
3417         }
3418         free_extent_map(em);
3419         if (!ret) {
3420                 if (whence == SEEK_DATA && start >= inode->i_size)
3421                         ret = -ENXIO;
3422                 else
3423                         *offset = min_t(loff_t, start, inode->i_size);
3424         }
3425         unlock_extent_cached(&BTRFS_I(inode)->io_tree, lockstart, lockend,
3426                              &cached_state);
3427         return ret;
3428 }
3429 
3430 static loff_t btrfs_file_llseek(struct file *file, loff_t offset, int whence)
3431 {
3432         struct inode *inode = file->f_mapping->host;
3433         int ret;
3434 
3435         inode_lock(inode);
3436         switch (whence) {
3437         case SEEK_END:
3438         case SEEK_CUR:
3439                 offset = generic_file_llseek(file, offset, whence);
3440                 goto out;
3441         case SEEK_DATA:
3442         case SEEK_HOLE:
3443                 if (offset >= i_size_read(inode)) {
3444                         inode_unlock(inode);
3445                         return -ENXIO;
3446                 }
3447 
3448                 ret = find_desired_extent(inode, &offset, whence);
3449                 if (ret) {
3450                         inode_unlock(inode);
3451                         return ret;
3452                 }
3453         }
3454 
3455         offset = vfs_setpos(file, offset, inode->i_sb->s_maxbytes);
3456 out:
3457         inode_unlock(inode);
3458         return offset;
3459 }
3460 
3461 static int btrfs_file_open(struct inode *inode, struct file *filp)
3462 {
3463         filp->f_mode |= FMODE_NOWAIT;
3464         return generic_file_open(inode, filp);
3465 }
3466 
3467 const struct file_operations btrfs_file_operations = {
3468         .llseek         = btrfs_file_llseek,
3469         .read_iter      = generic_file_read_iter,
3470         .splice_read    = generic_file_splice_read,
3471         .write_iter     = btrfs_file_write_iter,
3472         .mmap           = btrfs_file_mmap,
3473         .open           = btrfs_file_open,
3474         .release        = btrfs_release_file,
3475         .fsync          = btrfs_sync_file,
3476         .fallocate      = btrfs_fallocate,
3477         .unlocked_ioctl = btrfs_ioctl,
3478 #ifdef CONFIG_COMPAT
3479         .compat_ioctl   = btrfs_compat_ioctl,
3480 #endif
3481         .remap_file_range = btrfs_remap_file_range,
3482 };
3483 
3484 void __cold btrfs_auto_defrag_exit(void)
3485 {
3486         kmem_cache_destroy(btrfs_inode_defrag_cachep);
3487 }
3488 
3489 int __init btrfs_auto_defrag_init(void)
3490 {
3491         btrfs_inode_defrag_cachep = kmem_cache_create("btrfs_inode_defrag",
3492                                         sizeof(struct inode_defrag), 0,
3493                                         SLAB_MEM_SPREAD,
3494                                         NULL);
3495         if (!btrfs_inode_defrag_cachep)
3496                 return -ENOMEM;
3497 
3498         return 0;
3499 }
3500 
3501 int btrfs_fdatawrite_range(struct inode *inode, loff_t start, loff_t end)
3502 {
3503         int ret;
3504 
3505         /*
3506          * So with compression we will find and lock a dirty page and clear the
3507          * first one as dirty, setup an async extent, and immediately return
3508          * with the entire range locked but with nobody actually marked with
3509          * writeback.  So we can't just filemap_write_and_wait_range() and
3510          * expect it to work since it will just kick off a thread to do the
3511          * actual work.  So we need to call filemap_fdatawrite_range _again_
3512          * since it will wait on the page lock, which won't be unlocked until
3513          * after the pages have been marked as writeback and so we're good to go
3514          * from there.  We have to do this otherwise we'll miss the ordered
3515          * extents and that results in badness.  Please Josef, do not think you
3516          * know better and pull this out at some point in the future, it is
3517          * right and you are wrong.
3518          */
3519         ret = filemap_fdatawrite_range(inode->i_mapping, start, end);
3520         if (!ret && test_bit(BTRFS_INODE_HAS_ASYNC_EXTENT,
3521                              &BTRFS_I(inode)->runtime_flags))
3522                 ret = filemap_fdatawrite_range(inode->i_mapping, start, end);
3523 
3524         return ret;
3525 }