1/* 2 * mm/page-writeback.c 3 * 4 * Copyright (C) 2002, Linus Torvalds. 5 * Copyright (C) 2007 Red Hat, Inc., Peter Zijlstra 6 * 7 * Contains functions related to writing back dirty pages at the 8 * address_space level. 9 * 10 * 10Apr2002 Andrew Morton 11 * Initial version 12 */ 13 14#include <linux/kernel.h> 15#include <linux/export.h> 16#include <linux/spinlock.h> 17#include <linux/fs.h> 18#include <linux/mm.h> 19#include <linux/swap.h> 20#include <linux/slab.h> 21#include <linux/pagemap.h> 22#include <linux/writeback.h> 23#include <linux/init.h> 24#include <linux/backing-dev.h> 25#include <linux/task_io_accounting_ops.h> 26#include <linux/blkdev.h> 27#include <linux/mpage.h> 28#include <linux/rmap.h> 29#include <linux/percpu.h> 30#include <linux/notifier.h> 31#include <linux/smp.h> 32#include <linux/sysctl.h> 33#include <linux/cpu.h> 34#include <linux/syscalls.h> 35#include <linux/buffer_head.h> /* __set_page_dirty_buffers */ 36#include <linux/pagevec.h> 37#include <linux/timer.h> 38#include <linux/sched/rt.h> 39#include <linux/mm_inline.h> 40#include <trace/events/writeback.h> 41 42#include "internal.h" 43 44/* 45 * Sleep at most 200ms at a time in balance_dirty_pages(). 46 */ 47#define MAX_PAUSE max(HZ/5, 1) 48 49/* 50 * Try to keep balance_dirty_pages() call intervals higher than this many pages 51 * by raising pause time to max_pause when falls below it. 52 */ 53#define DIRTY_POLL_THRESH (128 >> (PAGE_SHIFT - 10)) 54 55/* 56 * Estimate write bandwidth at 200ms intervals. 57 */ 58#define BANDWIDTH_INTERVAL max(HZ/5, 1) 59 60#define RATELIMIT_CALC_SHIFT 10 61 62/* 63 * After a CPU has dirtied this many pages, balance_dirty_pages_ratelimited 64 * will look to see if it needs to force writeback or throttling. 65 */ 66static long ratelimit_pages = 32; 67 68/* The following parameters are exported via /proc/sys/vm */ 69 70/* 71 * Start background writeback (via writeback threads) at this percentage 72 */ 73int dirty_background_ratio = 10; 74 75/* 76 * dirty_background_bytes starts at 0 (disabled) so that it is a function of 77 * dirty_background_ratio * the amount of dirtyable memory 78 */ 79unsigned long dirty_background_bytes; 80 81/* 82 * free highmem will not be subtracted from the total free memory 83 * for calculating free ratios if vm_highmem_is_dirtyable is true 84 */ 85int vm_highmem_is_dirtyable; 86 87/* 88 * The generator of dirty data starts writeback at this percentage 89 */ 90int vm_dirty_ratio = 20; 91 92/* 93 * vm_dirty_bytes starts at 0 (disabled) so that it is a function of 94 * vm_dirty_ratio * the amount of dirtyable memory 95 */ 96unsigned long vm_dirty_bytes; 97 98/* 99 * The interval between `kupdate'-style writebacks 100 */ 101unsigned int dirty_writeback_interval = 5 * 100; /* centiseconds */ 102 103EXPORT_SYMBOL_GPL(dirty_writeback_interval); 104 105/* 106 * The longest time for which data is allowed to remain dirty 107 */ 108unsigned int dirty_expire_interval = 30 * 100; /* centiseconds */ 109 110/* 111 * Flag that makes the machine dump writes/reads and block dirtyings. 112 */ 113int block_dump; 114 115/* 116 * Flag that puts the machine in "laptop mode". Doubles as a timeout in jiffies: 117 * a full sync is triggered after this time elapses without any disk activity. 118 */ 119int laptop_mode; 120 121EXPORT_SYMBOL(laptop_mode); 122 123/* End of sysctl-exported parameters */ 124 125struct wb_domain global_wb_domain; 126 127/* consolidated parameters for balance_dirty_pages() and its subroutines */ 128struct dirty_throttle_control { 129#ifdef CONFIG_CGROUP_WRITEBACK 130 struct wb_domain *dom; 131 struct dirty_throttle_control *gdtc; /* only set in memcg dtc's */ 132#endif 133 struct bdi_writeback *wb; 134 struct fprop_local_percpu *wb_completions; 135 136 unsigned long avail; /* dirtyable */ 137 unsigned long dirty; /* file_dirty + write + nfs */ 138 unsigned long thresh; /* dirty threshold */ 139 unsigned long bg_thresh; /* dirty background threshold */ 140 141 unsigned long wb_dirty; /* per-wb counterparts */ 142 unsigned long wb_thresh; 143 unsigned long wb_bg_thresh; 144 145 unsigned long pos_ratio; 146}; 147 148/* 149 * Length of period for aging writeout fractions of bdis. This is an 150 * arbitrarily chosen number. The longer the period, the slower fractions will 151 * reflect changes in current writeout rate. 152 */ 153#define VM_COMPLETIONS_PERIOD_LEN (3*HZ) 154 155#ifdef CONFIG_CGROUP_WRITEBACK 156 157#define GDTC_INIT(__wb) .wb = (__wb), \ 158 .dom = &global_wb_domain, \ 159 .wb_completions = &(__wb)->completions 160 161#define GDTC_INIT_NO_WB .dom = &global_wb_domain 162 163#define MDTC_INIT(__wb, __gdtc) .wb = (__wb), \ 164 .dom = mem_cgroup_wb_domain(__wb), \ 165 .wb_completions = &(__wb)->memcg_completions, \ 166 .gdtc = __gdtc 167 168static bool mdtc_valid(struct dirty_throttle_control *dtc) 169{ 170 return dtc->dom; 171} 172 173static struct wb_domain *dtc_dom(struct dirty_throttle_control *dtc) 174{ 175 return dtc->dom; 176} 177 178static struct dirty_throttle_control *mdtc_gdtc(struct dirty_throttle_control *mdtc) 179{ 180 return mdtc->gdtc; 181} 182 183static struct fprop_local_percpu *wb_memcg_completions(struct bdi_writeback *wb) 184{ 185 return &wb->memcg_completions; 186} 187 188static void wb_min_max_ratio(struct bdi_writeback *wb, 189 unsigned long *minp, unsigned long *maxp) 190{ 191 unsigned long this_bw = wb->avg_write_bandwidth; 192 unsigned long tot_bw = atomic_long_read(&wb->bdi->tot_write_bandwidth); 193 unsigned long long min = wb->bdi->min_ratio; 194 unsigned long long max = wb->bdi->max_ratio; 195 196 /* 197 * @wb may already be clean by the time control reaches here and 198 * the total may not include its bw. 199 */ 200 if (this_bw < tot_bw) { 201 if (min) { 202 min *= this_bw; 203 do_div(min, tot_bw); 204 } 205 if (max < 100) { 206 max *= this_bw; 207 do_div(max, tot_bw); 208 } 209 } 210 211 *minp = min; 212 *maxp = max; 213} 214 215#else /* CONFIG_CGROUP_WRITEBACK */ 216 217#define GDTC_INIT(__wb) .wb = (__wb), \ 218 .wb_completions = &(__wb)->completions 219#define GDTC_INIT_NO_WB 220#define MDTC_INIT(__wb, __gdtc) 221 222static bool mdtc_valid(struct dirty_throttle_control *dtc) 223{ 224 return false; 225} 226 227static struct wb_domain *dtc_dom(struct dirty_throttle_control *dtc) 228{ 229 return &global_wb_domain; 230} 231 232static struct dirty_throttle_control *mdtc_gdtc(struct dirty_throttle_control *mdtc) 233{ 234 return NULL; 235} 236 237static struct fprop_local_percpu *wb_memcg_completions(struct bdi_writeback *wb) 238{ 239 return NULL; 240} 241 242static void wb_min_max_ratio(struct bdi_writeback *wb, 243 unsigned long *minp, unsigned long *maxp) 244{ 245 *minp = wb->bdi->min_ratio; 246 *maxp = wb->bdi->max_ratio; 247} 248 249#endif /* CONFIG_CGROUP_WRITEBACK */ 250 251/* 252 * In a memory zone, there is a certain amount of pages we consider 253 * available for the page cache, which is essentially the number of 254 * free and reclaimable pages, minus some zone reserves to protect 255 * lowmem and the ability to uphold the zone's watermarks without 256 * requiring writeback. 257 * 258 * This number of dirtyable pages is the base value of which the 259 * user-configurable dirty ratio is the effictive number of pages that 260 * are allowed to be actually dirtied. Per individual zone, or 261 * globally by using the sum of dirtyable pages over all zones. 262 * 263 * Because the user is allowed to specify the dirty limit globally as 264 * absolute number of bytes, calculating the per-zone dirty limit can 265 * require translating the configured limit into a percentage of 266 * global dirtyable memory first. 267 */ 268 269/** 270 * zone_dirtyable_memory - number of dirtyable pages in a zone 271 * @zone: the zone 272 * 273 * Returns the zone's number of pages potentially available for dirty 274 * page cache. This is the base value for the per-zone dirty limits. 275 */ 276static unsigned long zone_dirtyable_memory(struct zone *zone) 277{ 278 unsigned long nr_pages; 279 280 nr_pages = zone_page_state(zone, NR_FREE_PAGES); 281 nr_pages -= min(nr_pages, zone->dirty_balance_reserve); 282 283 nr_pages += zone_page_state(zone, NR_INACTIVE_FILE); 284 nr_pages += zone_page_state(zone, NR_ACTIVE_FILE); 285 286 return nr_pages; 287} 288 289static unsigned long highmem_dirtyable_memory(unsigned long total) 290{ 291#ifdef CONFIG_HIGHMEM 292 int node; 293 unsigned long x = 0; 294 295 for_each_node_state(node, N_HIGH_MEMORY) { 296 struct zone *z = &NODE_DATA(node)->node_zones[ZONE_HIGHMEM]; 297 298 x += zone_dirtyable_memory(z); 299 } 300 /* 301 * Unreclaimable memory (kernel memory or anonymous memory 302 * without swap) can bring down the dirtyable pages below 303 * the zone's dirty balance reserve and the above calculation 304 * will underflow. However we still want to add in nodes 305 * which are below threshold (negative values) to get a more 306 * accurate calculation but make sure that the total never 307 * underflows. 308 */ 309 if ((long)x < 0) 310 x = 0; 311 312 /* 313 * Make sure that the number of highmem pages is never larger 314 * than the number of the total dirtyable memory. This can only 315 * occur in very strange VM situations but we want to make sure 316 * that this does not occur. 317 */ 318 return min(x, total); 319#else 320 return 0; 321#endif 322} 323 324/** 325 * global_dirtyable_memory - number of globally dirtyable pages 326 * 327 * Returns the global number of pages potentially available for dirty 328 * page cache. This is the base value for the global dirty limits. 329 */ 330static unsigned long global_dirtyable_memory(void) 331{ 332 unsigned long x; 333 334 x = global_page_state(NR_FREE_PAGES); 335 x -= min(x, dirty_balance_reserve); 336 337 x += global_page_state(NR_INACTIVE_FILE); 338 x += global_page_state(NR_ACTIVE_FILE); 339 340 if (!vm_highmem_is_dirtyable) 341 x -= highmem_dirtyable_memory(x); 342 343 return x + 1; /* Ensure that we never return 0 */ 344} 345 346/** 347 * domain_dirty_limits - calculate thresh and bg_thresh for a wb_domain 348 * @dtc: dirty_throttle_control of interest 349 * 350 * Calculate @dtc->thresh and ->bg_thresh considering 351 * vm_dirty_{bytes|ratio} and dirty_background_{bytes|ratio}. The caller 352 * must ensure that @dtc->avail is set before calling this function. The 353 * dirty limits will be lifted by 1/4 for PF_LESS_THROTTLE (ie. nfsd) and 354 * real-time tasks. 355 */ 356static void domain_dirty_limits(struct dirty_throttle_control *dtc) 357{ 358 const unsigned long available_memory = dtc->avail; 359 struct dirty_throttle_control *gdtc = mdtc_gdtc(dtc); 360 unsigned long bytes = vm_dirty_bytes; 361 unsigned long bg_bytes = dirty_background_bytes; 362 unsigned long ratio = vm_dirty_ratio; 363 unsigned long bg_ratio = dirty_background_ratio; 364 unsigned long thresh; 365 unsigned long bg_thresh; 366 struct task_struct *tsk; 367 368 /* gdtc is !NULL iff @dtc is for memcg domain */ 369 if (gdtc) { 370 unsigned long global_avail = gdtc->avail; 371 372 /* 373 * The byte settings can't be applied directly to memcg 374 * domains. Convert them to ratios by scaling against 375 * globally available memory. 376 */ 377 if (bytes) 378 ratio = min(DIV_ROUND_UP(bytes, PAGE_SIZE) * 100 / 379 global_avail, 100UL); 380 if (bg_bytes) 381 bg_ratio = min(DIV_ROUND_UP(bg_bytes, PAGE_SIZE) * 100 / 382 global_avail, 100UL); 383 bytes = bg_bytes = 0; 384 } 385 386 if (bytes) 387 thresh = DIV_ROUND_UP(bytes, PAGE_SIZE); 388 else 389 thresh = (ratio * available_memory) / 100; 390 391 if (bg_bytes) 392 bg_thresh = DIV_ROUND_UP(bg_bytes, PAGE_SIZE); 393 else 394 bg_thresh = (bg_ratio * available_memory) / 100; 395 396 if (bg_thresh >= thresh) 397 bg_thresh = thresh / 2; 398 tsk = current; 399 if (tsk->flags & PF_LESS_THROTTLE || rt_task(tsk)) { 400 bg_thresh += bg_thresh / 4; 401 thresh += thresh / 4; 402 } 403 dtc->thresh = thresh; 404 dtc->bg_thresh = bg_thresh; 405 406 /* we should eventually report the domain in the TP */ 407 if (!gdtc) 408 trace_global_dirty_state(bg_thresh, thresh); 409} 410 411/** 412 * global_dirty_limits - background-writeback and dirty-throttling thresholds 413 * @pbackground: out parameter for bg_thresh 414 * @pdirty: out parameter for thresh 415 * 416 * Calculate bg_thresh and thresh for global_wb_domain. See 417 * domain_dirty_limits() for details. 418 */ 419void global_dirty_limits(unsigned long *pbackground, unsigned long *pdirty) 420{ 421 struct dirty_throttle_control gdtc = { GDTC_INIT_NO_WB }; 422 423 gdtc.avail = global_dirtyable_memory(); 424 domain_dirty_limits(&gdtc); 425 426 *pbackground = gdtc.bg_thresh; 427 *pdirty = gdtc.thresh; 428} 429 430/** 431 * zone_dirty_limit - maximum number of dirty pages allowed in a zone 432 * @zone: the zone 433 * 434 * Returns the maximum number of dirty pages allowed in a zone, based 435 * on the zone's dirtyable memory. 436 */ 437static unsigned long zone_dirty_limit(struct zone *zone) 438{ 439 unsigned long zone_memory = zone_dirtyable_memory(zone); 440 struct task_struct *tsk = current; 441 unsigned long dirty; 442 443 if (vm_dirty_bytes) 444 dirty = DIV_ROUND_UP(vm_dirty_bytes, PAGE_SIZE) * 445 zone_memory / global_dirtyable_memory(); 446 else 447 dirty = vm_dirty_ratio * zone_memory / 100; 448 449 if (tsk->flags & PF_LESS_THROTTLE || rt_task(tsk)) 450 dirty += dirty / 4; 451 452 return dirty; 453} 454 455/** 456 * zone_dirty_ok - tells whether a zone is within its dirty limits 457 * @zone: the zone to check 458 * 459 * Returns %true when the dirty pages in @zone are within the zone's 460 * dirty limit, %false if the limit is exceeded. 461 */ 462bool zone_dirty_ok(struct zone *zone) 463{ 464 unsigned long limit = zone_dirty_limit(zone); 465 466 return zone_page_state(zone, NR_FILE_DIRTY) + 467 zone_page_state(zone, NR_UNSTABLE_NFS) + 468 zone_page_state(zone, NR_WRITEBACK) <= limit; 469} 470 471int dirty_background_ratio_handler(struct ctl_table *table, int write, 472 void __user *buffer, size_t *lenp, 473 loff_t *ppos) 474{ 475 int ret; 476 477 ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos); 478 if (ret == 0 && write) 479 dirty_background_bytes = 0; 480 return ret; 481} 482 483int dirty_background_bytes_handler(struct ctl_table *table, int write, 484 void __user *buffer, size_t *lenp, 485 loff_t *ppos) 486{ 487 int ret; 488 489 ret = proc_doulongvec_minmax(table, write, buffer, lenp, ppos); 490 if (ret == 0 && write) 491 dirty_background_ratio = 0; 492 return ret; 493} 494 495int dirty_ratio_handler(struct ctl_table *table, int write, 496 void __user *buffer, size_t *lenp, 497 loff_t *ppos) 498{ 499 int old_ratio = vm_dirty_ratio; 500 int ret; 501 502 ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos); 503 if (ret == 0 && write && vm_dirty_ratio != old_ratio) { 504 writeback_set_ratelimit(); 505 vm_dirty_bytes = 0; 506 } 507 return ret; 508} 509 510int dirty_bytes_handler(struct ctl_table *table, int write, 511 void __user *buffer, size_t *lenp, 512 loff_t *ppos) 513{ 514 unsigned long old_bytes = vm_dirty_bytes; 515 int ret; 516 517 ret = proc_doulongvec_minmax(table, write, buffer, lenp, ppos); 518 if (ret == 0 && write && vm_dirty_bytes != old_bytes) { 519 writeback_set_ratelimit(); 520 vm_dirty_ratio = 0; 521 } 522 return ret; 523} 524 525static unsigned long wp_next_time(unsigned long cur_time) 526{ 527 cur_time += VM_COMPLETIONS_PERIOD_LEN; 528 /* 0 has a special meaning... */ 529 if (!cur_time) 530 return 1; 531 return cur_time; 532} 533 534static void wb_domain_writeout_inc(struct wb_domain *dom, 535 struct fprop_local_percpu *completions, 536 unsigned int max_prop_frac) 537{ 538 __fprop_inc_percpu_max(&dom->completions, completions, 539 max_prop_frac); 540 /* First event after period switching was turned off? */ 541 if (!unlikely(dom->period_time)) { 542 /* 543 * We can race with other __bdi_writeout_inc calls here but 544 * it does not cause any harm since the resulting time when 545 * timer will fire and what is in writeout_period_time will be 546 * roughly the same. 547 */ 548 dom->period_time = wp_next_time(jiffies); 549 mod_timer(&dom->period_timer, dom->period_time); 550 } 551} 552 553/* 554 * Increment @wb's writeout completion count and the global writeout 555 * completion count. Called from test_clear_page_writeback(). 556 */ 557static inline void __wb_writeout_inc(struct bdi_writeback *wb) 558{ 559 struct wb_domain *cgdom; 560 561 __inc_wb_stat(wb, WB_WRITTEN); 562 wb_domain_writeout_inc(&global_wb_domain, &wb->completions, 563 wb->bdi->max_prop_frac); 564 565 cgdom = mem_cgroup_wb_domain(wb); 566 if (cgdom) 567 wb_domain_writeout_inc(cgdom, wb_memcg_completions(wb), 568 wb->bdi->max_prop_frac); 569} 570 571void wb_writeout_inc(struct bdi_writeback *wb) 572{ 573 unsigned long flags; 574 575 local_irq_save(flags); 576 __wb_writeout_inc(wb); 577 local_irq_restore(flags); 578} 579EXPORT_SYMBOL_GPL(wb_writeout_inc); 580 581/* 582 * On idle system, we can be called long after we scheduled because we use 583 * deferred timers so count with missed periods. 584 */ 585static void writeout_period(unsigned long t) 586{ 587 struct wb_domain *dom = (void *)t; 588 int miss_periods = (jiffies - dom->period_time) / 589 VM_COMPLETIONS_PERIOD_LEN; 590 591 if (fprop_new_period(&dom->completions, miss_periods + 1)) { 592 dom->period_time = wp_next_time(dom->period_time + 593 miss_periods * VM_COMPLETIONS_PERIOD_LEN); 594 mod_timer(&dom->period_timer, dom->period_time); 595 } else { 596 /* 597 * Aging has zeroed all fractions. Stop wasting CPU on period 598 * updates. 599 */ 600 dom->period_time = 0; 601 } 602} 603 604int wb_domain_init(struct wb_domain *dom, gfp_t gfp) 605{ 606 memset(dom, 0, sizeof(*dom)); 607 608 spin_lock_init(&dom->lock); 609 610 init_timer_deferrable(&dom->period_timer); 611 dom->period_timer.function = writeout_period; 612 dom->period_timer.data = (unsigned long)dom; 613 614 dom->dirty_limit_tstamp = jiffies; 615 616 return fprop_global_init(&dom->completions, gfp); 617} 618 619#ifdef CONFIG_CGROUP_WRITEBACK 620void wb_domain_exit(struct wb_domain *dom) 621{ 622 del_timer_sync(&dom->period_timer); 623 fprop_global_destroy(&dom->completions); 624} 625#endif 626 627/* 628 * bdi_min_ratio keeps the sum of the minimum dirty shares of all 629 * registered backing devices, which, for obvious reasons, can not 630 * exceed 100%. 631 */ 632static unsigned int bdi_min_ratio; 633 634int bdi_set_min_ratio(struct backing_dev_info *bdi, unsigned int min_ratio) 635{ 636 int ret = 0; 637 638 spin_lock_bh(&bdi_lock); 639 if (min_ratio > bdi->max_ratio) { 640 ret = -EINVAL; 641 } else { 642 min_ratio -= bdi->min_ratio; 643 if (bdi_min_ratio + min_ratio < 100) { 644 bdi_min_ratio += min_ratio; 645 bdi->min_ratio += min_ratio; 646 } else { 647 ret = -EINVAL; 648 } 649 } 650 spin_unlock_bh(&bdi_lock); 651 652 return ret; 653} 654 655int bdi_set_max_ratio(struct backing_dev_info *bdi, unsigned max_ratio) 656{ 657 int ret = 0; 658 659 if (max_ratio > 100) 660 return -EINVAL; 661 662 spin_lock_bh(&bdi_lock); 663 if (bdi->min_ratio > max_ratio) { 664 ret = -EINVAL; 665 } else { 666 bdi->max_ratio = max_ratio; 667 bdi->max_prop_frac = (FPROP_FRAC_BASE * max_ratio) / 100; 668 } 669 spin_unlock_bh(&bdi_lock); 670 671 return ret; 672} 673EXPORT_SYMBOL(bdi_set_max_ratio); 674 675static unsigned long dirty_freerun_ceiling(unsigned long thresh, 676 unsigned long bg_thresh) 677{ 678 return (thresh + bg_thresh) / 2; 679} 680 681static unsigned long hard_dirty_limit(struct wb_domain *dom, 682 unsigned long thresh) 683{ 684 return max(thresh, dom->dirty_limit); 685} 686 687/* 688 * Memory which can be further allocated to a memcg domain is capped by 689 * system-wide clean memory excluding the amount being used in the domain. 690 */ 691static void mdtc_calc_avail(struct dirty_throttle_control *mdtc, 692 unsigned long filepages, unsigned long headroom) 693{ 694 struct dirty_throttle_control *gdtc = mdtc_gdtc(mdtc); 695 unsigned long clean = filepages - min(filepages, mdtc->dirty); 696 unsigned long global_clean = gdtc->avail - min(gdtc->avail, gdtc->dirty); 697 unsigned long other_clean = global_clean - min(global_clean, clean); 698 699 mdtc->avail = filepages + min(headroom, other_clean); 700} 701 702/** 703 * __wb_calc_thresh - @wb's share of dirty throttling threshold 704 * @dtc: dirty_throttle_context of interest 705 * 706 * Returns @wb's dirty limit in pages. The term "dirty" in the context of 707 * dirty balancing includes all PG_dirty, PG_writeback and NFS unstable pages. 708 * 709 * Note that balance_dirty_pages() will only seriously take it as a hard limit 710 * when sleeping max_pause per page is not enough to keep the dirty pages under 711 * control. For example, when the device is completely stalled due to some error 712 * conditions, or when there are 1000 dd tasks writing to a slow 10MB/s USB key. 713 * In the other normal situations, it acts more gently by throttling the tasks 714 * more (rather than completely block them) when the wb dirty pages go high. 715 * 716 * It allocates high/low dirty limits to fast/slow devices, in order to prevent 717 * - starving fast devices 718 * - piling up dirty pages (that will take long time to sync) on slow devices 719 * 720 * The wb's share of dirty limit will be adapting to its throughput and 721 * bounded by the bdi->min_ratio and/or bdi->max_ratio parameters, if set. 722 */ 723static unsigned long __wb_calc_thresh(struct dirty_throttle_control *dtc) 724{ 725 struct wb_domain *dom = dtc_dom(dtc); 726 unsigned long thresh = dtc->thresh; 727 u64 wb_thresh; 728 long numerator, denominator; 729 unsigned long wb_min_ratio, wb_max_ratio; 730 731 /* 732 * Calculate this BDI's share of the thresh ratio. 733 */ 734 fprop_fraction_percpu(&dom->completions, dtc->wb_completions, 735 &numerator, &denominator); 736 737 wb_thresh = (thresh * (100 - bdi_min_ratio)) / 100; 738 wb_thresh *= numerator; 739 do_div(wb_thresh, denominator); 740 741 wb_min_max_ratio(dtc->wb, &wb_min_ratio, &wb_max_ratio); 742 743 wb_thresh += (thresh * wb_min_ratio) / 100; 744 if (wb_thresh > (thresh * wb_max_ratio) / 100) 745 wb_thresh = thresh * wb_max_ratio / 100; 746 747 return wb_thresh; 748} 749 750unsigned long wb_calc_thresh(struct bdi_writeback *wb, unsigned long thresh) 751{ 752 struct dirty_throttle_control gdtc = { GDTC_INIT(wb), 753 .thresh = thresh }; 754 return __wb_calc_thresh(&gdtc); 755} 756 757/* 758 * setpoint - dirty 3 759 * f(dirty) := 1.0 + (----------------) 760 * limit - setpoint 761 * 762 * it's a 3rd order polynomial that subjects to 763 * 764 * (1) f(freerun) = 2.0 => rampup dirty_ratelimit reasonably fast 765 * (2) f(setpoint) = 1.0 => the balance point 766 * (3) f(limit) = 0 => the hard limit 767 * (4) df/dx <= 0 => negative feedback control 768 * (5) the closer to setpoint, the smaller |df/dx| (and the reverse) 769 * => fast response on large errors; small oscillation near setpoint 770 */ 771static long long pos_ratio_polynom(unsigned long setpoint, 772 unsigned long dirty, 773 unsigned long limit) 774{ 775 long long pos_ratio; 776 long x; 777 778 x = div64_s64(((s64)setpoint - (s64)dirty) << RATELIMIT_CALC_SHIFT, 779 (limit - setpoint) | 1); 780 pos_ratio = x; 781 pos_ratio = pos_ratio * x >> RATELIMIT_CALC_SHIFT; 782 pos_ratio = pos_ratio * x >> RATELIMIT_CALC_SHIFT; 783 pos_ratio += 1 << RATELIMIT_CALC_SHIFT; 784 785 return clamp(pos_ratio, 0LL, 2LL << RATELIMIT_CALC_SHIFT); 786} 787 788/* 789 * Dirty position control. 790 * 791 * (o) global/bdi setpoints 792 * 793 * We want the dirty pages be balanced around the global/wb setpoints. 794 * When the number of dirty pages is higher/lower than the setpoint, the 795 * dirty position control ratio (and hence task dirty ratelimit) will be 796 * decreased/increased to bring the dirty pages back to the setpoint. 797 * 798 * pos_ratio = 1 << RATELIMIT_CALC_SHIFT 799 * 800 * if (dirty < setpoint) scale up pos_ratio 801 * if (dirty > setpoint) scale down pos_ratio 802 * 803 * if (wb_dirty < wb_setpoint) scale up pos_ratio 804 * if (wb_dirty > wb_setpoint) scale down pos_ratio 805 * 806 * task_ratelimit = dirty_ratelimit * pos_ratio >> RATELIMIT_CALC_SHIFT 807 * 808 * (o) global control line 809 * 810 * ^ pos_ratio 811 * | 812 * | |<===== global dirty control scope ======>| 813 * 2.0 .............* 814 * | .* 815 * | . * 816 * | . * 817 * | . * 818 * | . * 819 * | . * 820 * 1.0 ................................* 821 * | . . * 822 * | . . * 823 * | . . * 824 * | . . * 825 * | . . * 826 * 0 +------------.------------------.----------------------*-------------> 827 * freerun^ setpoint^ limit^ dirty pages 828 * 829 * (o) wb control line 830 * 831 * ^ pos_ratio 832 * | 833 * | * 834 * | * 835 * | * 836 * | * 837 * | * |<=========== span ============>| 838 * 1.0 .......................* 839 * | . * 840 * | . * 841 * | . * 842 * | . * 843 * | . * 844 * | . * 845 * | . * 846 * | . * 847 * | . * 848 * | . * 849 * | . * 850 * 1/4 ...............................................* * * * * * * * * * * * 851 * | . . 852 * | . . 853 * | . . 854 * 0 +----------------------.-------------------------------.-------------> 855 * wb_setpoint^ x_intercept^ 856 * 857 * The wb control line won't drop below pos_ratio=1/4, so that wb_dirty can 858 * be smoothly throttled down to normal if it starts high in situations like 859 * - start writing to a slow SD card and a fast disk at the same time. The SD 860 * card's wb_dirty may rush to many times higher than wb_setpoint. 861 * - the wb dirty thresh drops quickly due to change of JBOD workload 862 */ 863static void wb_position_ratio(struct dirty_throttle_control *dtc) 864{ 865 struct bdi_writeback *wb = dtc->wb; 866 unsigned long write_bw = wb->avg_write_bandwidth; 867 unsigned long freerun = dirty_freerun_ceiling(dtc->thresh, dtc->bg_thresh); 868 unsigned long limit = hard_dirty_limit(dtc_dom(dtc), dtc->thresh); 869 unsigned long wb_thresh = dtc->wb_thresh; 870 unsigned long x_intercept; 871 unsigned long setpoint; /* dirty pages' target balance point */ 872 unsigned long wb_setpoint; 873 unsigned long span; 874 long long pos_ratio; /* for scaling up/down the rate limit */ 875 long x; 876 877 dtc->pos_ratio = 0; 878 879 if (unlikely(dtc->dirty >= limit)) 880 return; 881 882 /* 883 * global setpoint 884 * 885 * See comment for pos_ratio_polynom(). 886 */ 887 setpoint = (freerun + limit) / 2; 888 pos_ratio = pos_ratio_polynom(setpoint, dtc->dirty, limit); 889 890 /* 891 * The strictlimit feature is a tool preventing mistrusted filesystems 892 * from growing a large number of dirty pages before throttling. For 893 * such filesystems balance_dirty_pages always checks wb counters 894 * against wb limits. Even if global "nr_dirty" is under "freerun". 895 * This is especially important for fuse which sets bdi->max_ratio to 896 * 1% by default. Without strictlimit feature, fuse writeback may 897 * consume arbitrary amount of RAM because it is accounted in 898 * NR_WRITEBACK_TEMP which is not involved in calculating "nr_dirty". 899 * 900 * Here, in wb_position_ratio(), we calculate pos_ratio based on 901 * two values: wb_dirty and wb_thresh. Let's consider an example: 902 * total amount of RAM is 16GB, bdi->max_ratio is equal to 1%, global 903 * limits are set by default to 10% and 20% (background and throttle). 904 * Then wb_thresh is 1% of 20% of 16GB. This amounts to ~8K pages. 905 * wb_calc_thresh(wb, bg_thresh) is about ~4K pages. wb_setpoint is 906 * about ~6K pages (as the average of background and throttle wb 907 * limits). The 3rd order polynomial will provide positive feedback if 908 * wb_dirty is under wb_setpoint and vice versa. 909 * 910 * Note, that we cannot use global counters in these calculations 911 * because we want to throttle process writing to a strictlimit wb 912 * much earlier than global "freerun" is reached (~23MB vs. ~2.3GB 913 * in the example above). 914 */ 915 if (unlikely(wb->bdi->capabilities & BDI_CAP_STRICTLIMIT)) { 916 long long wb_pos_ratio; 917 918 if (dtc->wb_dirty < 8) { 919 dtc->pos_ratio = min_t(long long, pos_ratio * 2, 920 2 << RATELIMIT_CALC_SHIFT); 921 return; 922 } 923 924 if (dtc->wb_dirty >= wb_thresh) 925 return; 926 927 wb_setpoint = dirty_freerun_ceiling(wb_thresh, 928 dtc->wb_bg_thresh); 929 930 if (wb_setpoint == 0 || wb_setpoint == wb_thresh) 931 return; 932 933 wb_pos_ratio = pos_ratio_polynom(wb_setpoint, dtc->wb_dirty, 934 wb_thresh); 935 936 /* 937 * Typically, for strictlimit case, wb_setpoint << setpoint 938 * and pos_ratio >> wb_pos_ratio. In the other words global 939 * state ("dirty") is not limiting factor and we have to 940 * make decision based on wb counters. But there is an 941 * important case when global pos_ratio should get precedence: 942 * global limits are exceeded (e.g. due to activities on other 943 * wb's) while given strictlimit wb is below limit. 944 * 945 * "pos_ratio * wb_pos_ratio" would work for the case above, 946 * but it would look too non-natural for the case of all 947 * activity in the system coming from a single strictlimit wb 948 * with bdi->max_ratio == 100%. 949 * 950 * Note that min() below somewhat changes the dynamics of the 951 * control system. Normally, pos_ratio value can be well over 3 952 * (when globally we are at freerun and wb is well below wb 953 * setpoint). Now the maximum pos_ratio in the same situation 954 * is 2. We might want to tweak this if we observe the control 955 * system is too slow to adapt. 956 */ 957 dtc->pos_ratio = min(pos_ratio, wb_pos_ratio); 958 return; 959 } 960 961 /* 962 * We have computed basic pos_ratio above based on global situation. If 963 * the wb is over/under its share of dirty pages, we want to scale 964 * pos_ratio further down/up. That is done by the following mechanism. 965 */ 966 967 /* 968 * wb setpoint 969 * 970 * f(wb_dirty) := 1.0 + k * (wb_dirty - wb_setpoint) 971 * 972 * x_intercept - wb_dirty 973 * := -------------------------- 974 * x_intercept - wb_setpoint 975 * 976 * The main wb control line is a linear function that subjects to 977 * 978 * (1) f(wb_setpoint) = 1.0 979 * (2) k = - 1 / (8 * write_bw) (in single wb case) 980 * or equally: x_intercept = wb_setpoint + 8 * write_bw 981 * 982 * For single wb case, the dirty pages are observed to fluctuate 983 * regularly within range 984 * [wb_setpoint - write_bw/2, wb_setpoint + write_bw/2] 985 * for various filesystems, where (2) can yield in a reasonable 12.5% 986 * fluctuation range for pos_ratio. 987 * 988 * For JBOD case, wb_thresh (not wb_dirty!) could fluctuate up to its 989 * own size, so move the slope over accordingly and choose a slope that 990 * yields 100% pos_ratio fluctuation on suddenly doubled wb_thresh. 991 */ 992 if (unlikely(wb_thresh > dtc->thresh)) 993 wb_thresh = dtc->thresh; 994 /* 995 * It's very possible that wb_thresh is close to 0 not because the 996 * device is slow, but that it has remained inactive for long time. 997 * Honour such devices a reasonable good (hopefully IO efficient) 998 * threshold, so that the occasional writes won't be blocked and active 999 * writes can rampup the threshold quickly. 1000 */ 1001 wb_thresh = max(wb_thresh, (limit - dtc->dirty) / 8); 1002 /* 1003 * scale global setpoint to wb's: 1004 * wb_setpoint = setpoint * wb_thresh / thresh 1005 */ 1006 x = div_u64((u64)wb_thresh << 16, dtc->thresh | 1); 1007 wb_setpoint = setpoint * (u64)x >> 16; 1008 /* 1009 * Use span=(8*write_bw) in single wb case as indicated by 1010 * (thresh - wb_thresh ~= 0) and transit to wb_thresh in JBOD case. 1011 * 1012 * wb_thresh thresh - wb_thresh 1013 * span = --------- * (8 * write_bw) + ------------------ * wb_thresh 1014 * thresh thresh 1015 */ 1016 span = (dtc->thresh - wb_thresh + 8 * write_bw) * (u64)x >> 16; 1017 x_intercept = wb_setpoint + span; 1018 1019 if (dtc->wb_dirty < x_intercept - span / 4) { 1020 pos_ratio = div64_u64(pos_ratio * (x_intercept - dtc->wb_dirty), 1021 (x_intercept - wb_setpoint) | 1); 1022 } else 1023 pos_ratio /= 4; 1024 1025 /* 1026 * wb reserve area, safeguard against dirty pool underrun and disk idle 1027 * It may push the desired control point of global dirty pages higher 1028 * than setpoint. 1029 */ 1030 x_intercept = wb_thresh / 2; 1031 if (dtc->wb_dirty < x_intercept) { 1032 if (dtc->wb_dirty > x_intercept / 8) 1033 pos_ratio = div_u64(pos_ratio * x_intercept, 1034 dtc->wb_dirty); 1035 else 1036 pos_ratio *= 8; 1037 } 1038 1039 dtc->pos_ratio = pos_ratio; 1040} 1041 1042static void wb_update_write_bandwidth(struct bdi_writeback *wb, 1043 unsigned long elapsed, 1044 unsigned long written) 1045{ 1046 const unsigned long period = roundup_pow_of_two(3 * HZ); 1047 unsigned long avg = wb->avg_write_bandwidth; 1048 unsigned long old = wb->write_bandwidth; 1049 u64 bw; 1050 1051 /* 1052 * bw = written * HZ / elapsed 1053 * 1054 * bw * elapsed + write_bandwidth * (period - elapsed) 1055 * write_bandwidth = --------------------------------------------------- 1056 * period 1057 * 1058 * @written may have decreased due to account_page_redirty(). 1059 * Avoid underflowing @bw calculation. 1060 */ 1061 bw = written - min(written, wb->written_stamp); 1062 bw *= HZ; 1063 if (unlikely(elapsed > period)) { 1064 do_div(bw, elapsed); 1065 avg = bw; 1066 goto out; 1067 } 1068 bw += (u64)wb->write_bandwidth * (period - elapsed); 1069 bw >>= ilog2(period); 1070 1071 /* 1072 * one more level of smoothing, for filtering out sudden spikes 1073 */ 1074 if (avg > old && old >= (unsigned long)bw) 1075 avg -= (avg - old) >> 3; 1076 1077 if (avg < old && old <= (unsigned long)bw) 1078 avg += (old - avg) >> 3; 1079 1080out: 1081 /* keep avg > 0 to guarantee that tot > 0 if there are dirty wbs */ 1082 avg = max(avg, 1LU); 1083 if (wb_has_dirty_io(wb)) { 1084 long delta = avg - wb->avg_write_bandwidth; 1085 WARN_ON_ONCE(atomic_long_add_return(delta, 1086 &wb->bdi->tot_write_bandwidth) <= 0); 1087 } 1088 wb->write_bandwidth = bw; 1089 wb->avg_write_bandwidth = avg; 1090} 1091 1092static void update_dirty_limit(struct dirty_throttle_control *dtc) 1093{ 1094 struct wb_domain *dom = dtc_dom(dtc); 1095 unsigned long thresh = dtc->thresh; 1096 unsigned long limit = dom->dirty_limit; 1097 1098 /* 1099 * Follow up in one step. 1100 */ 1101 if (limit < thresh) { 1102 limit = thresh; 1103 goto update; 1104 } 1105 1106 /* 1107 * Follow down slowly. Use the higher one as the target, because thresh 1108 * may drop below dirty. This is exactly the reason to introduce 1109 * dom->dirty_limit which is guaranteed to lie above the dirty pages. 1110 */ 1111 thresh = max(thresh, dtc->dirty); 1112 if (limit > thresh) { 1113 limit -= (limit - thresh) >> 5; 1114 goto update; 1115 } 1116 return; 1117update: 1118 dom->dirty_limit = limit; 1119} 1120 1121static void domain_update_bandwidth(struct dirty_throttle_control *dtc, 1122 unsigned long now) 1123{ 1124 struct wb_domain *dom = dtc_dom(dtc); 1125 1126 /* 1127 * check locklessly first to optimize away locking for the most time 1128 */ 1129 if (time_before(now, dom->dirty_limit_tstamp + BANDWIDTH_INTERVAL)) 1130 return; 1131 1132 spin_lock(&dom->lock); 1133 if (time_after_eq(now, dom->dirty_limit_tstamp + BANDWIDTH_INTERVAL)) { 1134 update_dirty_limit(dtc); 1135 dom->dirty_limit_tstamp = now; 1136 } 1137 spin_unlock(&dom->lock); 1138} 1139 1140/* 1141 * Maintain wb->dirty_ratelimit, the base dirty throttle rate. 1142 * 1143 * Normal wb tasks will be curbed at or below it in long term. 1144 * Obviously it should be around (write_bw / N) when there are N dd tasks. 1145 */ 1146static void wb_update_dirty_ratelimit(struct dirty_throttle_control *dtc, 1147 unsigned long dirtied, 1148 unsigned long elapsed) 1149{ 1150 struct bdi_writeback *wb = dtc->wb; 1151 unsigned long dirty = dtc->dirty; 1152 unsigned long freerun = dirty_freerun_ceiling(dtc->thresh, dtc->bg_thresh); 1153 unsigned long limit = hard_dirty_limit(dtc_dom(dtc), dtc->thresh); 1154 unsigned long setpoint = (freerun + limit) / 2; 1155 unsigned long write_bw = wb->avg_write_bandwidth; 1156 unsigned long dirty_ratelimit = wb->dirty_ratelimit; 1157 unsigned long dirty_rate; 1158 unsigned long task_ratelimit; 1159 unsigned long balanced_dirty_ratelimit; 1160 unsigned long step; 1161 unsigned long x; 1162 1163 /* 1164 * The dirty rate will match the writeout rate in long term, except 1165 * when dirty pages are truncated by userspace or re-dirtied by FS. 1166 */ 1167 dirty_rate = (dirtied - wb->dirtied_stamp) * HZ / elapsed; 1168 1169 /* 1170 * task_ratelimit reflects each dd's dirty rate for the past 200ms. 1171 */ 1172 task_ratelimit = (u64)dirty_ratelimit * 1173 dtc->pos_ratio >> RATELIMIT_CALC_SHIFT; 1174 task_ratelimit++; /* it helps rampup dirty_ratelimit from tiny values */ 1175 1176 /* 1177 * A linear estimation of the "balanced" throttle rate. The theory is, 1178 * if there are N dd tasks, each throttled at task_ratelimit, the wb's 1179 * dirty_rate will be measured to be (N * task_ratelimit). So the below 1180 * formula will yield the balanced rate limit (write_bw / N). 1181 * 1182 * Note that the expanded form is not a pure rate feedback: 1183 * rate_(i+1) = rate_(i) * (write_bw / dirty_rate) (1) 1184 * but also takes pos_ratio into account: 1185 * rate_(i+1) = rate_(i) * (write_bw / dirty_rate) * pos_ratio (2) 1186 * 1187 * (1) is not realistic because pos_ratio also takes part in balancing 1188 * the dirty rate. Consider the state 1189 * pos_ratio = 0.5 (3) 1190 * rate = 2 * (write_bw / N) (4) 1191 * If (1) is used, it will stuck in that state! Because each dd will 1192 * be throttled at 1193 * task_ratelimit = pos_ratio * rate = (write_bw / N) (5) 1194 * yielding 1195 * dirty_rate = N * task_ratelimit = write_bw (6) 1196 * put (6) into (1) we get 1197 * rate_(i+1) = rate_(i) (7) 1198 * 1199 * So we end up using (2) to always keep 1200 * rate_(i+1) ~= (write_bw / N) (8) 1201 * regardless of the value of pos_ratio. As long as (8) is satisfied, 1202 * pos_ratio is able to drive itself to 1.0, which is not only where 1203 * the dirty count meet the setpoint, but also where the slope of 1204 * pos_ratio is most flat and hence task_ratelimit is least fluctuated. 1205 */ 1206 balanced_dirty_ratelimit = div_u64((u64)task_ratelimit * write_bw, 1207 dirty_rate | 1); 1208 /* 1209 * balanced_dirty_ratelimit ~= (write_bw / N) <= write_bw 1210 */ 1211 if (unlikely(balanced_dirty_ratelimit > write_bw)) 1212 balanced_dirty_ratelimit = write_bw; 1213 1214 /* 1215 * We could safely do this and return immediately: 1216 * 1217 * wb->dirty_ratelimit = balanced_dirty_ratelimit; 1218 * 1219 * However to get a more stable dirty_ratelimit, the below elaborated 1220 * code makes use of task_ratelimit to filter out singular points and 1221 * limit the step size. 1222 * 1223 * The below code essentially only uses the relative value of 1224 * 1225 * task_ratelimit - dirty_ratelimit 1226 * = (pos_ratio - 1) * dirty_ratelimit 1227 * 1228 * which reflects the direction and size of dirty position error. 1229 */ 1230 1231 /* 1232 * dirty_ratelimit will follow balanced_dirty_ratelimit iff 1233 * task_ratelimit is on the same side of dirty_ratelimit, too. 1234 * For example, when 1235 * - dirty_ratelimit > balanced_dirty_ratelimit 1236 * - dirty_ratelimit > task_ratelimit (dirty pages are above setpoint) 1237 * lowering dirty_ratelimit will help meet both the position and rate 1238 * control targets. Otherwise, don't update dirty_ratelimit if it will 1239 * only help meet the rate target. After all, what the users ultimately 1240 * feel and care are stable dirty rate and small position error. 1241 * 1242 * |task_ratelimit - dirty_ratelimit| is used to limit the step size 1243 * and filter out the singular points of balanced_dirty_ratelimit. Which 1244 * keeps jumping around randomly and can even leap far away at times 1245 * due to the small 200ms estimation period of dirty_rate (we want to 1246 * keep that period small to reduce time lags). 1247 */ 1248 step = 0; 1249 1250 /* 1251 * For strictlimit case, calculations above were based on wb counters 1252 * and limits (starting from pos_ratio = wb_position_ratio() and up to 1253 * balanced_dirty_ratelimit = task_ratelimit * write_bw / dirty_rate). 1254 * Hence, to calculate "step" properly, we have to use wb_dirty as 1255 * "dirty" and wb_setpoint as "setpoint". 1256 * 1257 * We rampup dirty_ratelimit forcibly if wb_dirty is low because 1258 * it's possible that wb_thresh is close to zero due to inactivity 1259 * of backing device. 1260 */ 1261 if (unlikely(wb->bdi->capabilities & BDI_CAP_STRICTLIMIT)) { 1262 dirty = dtc->wb_dirty; 1263 if (dtc->wb_dirty < 8) 1264 setpoint = dtc->wb_dirty + 1; 1265 else 1266 setpoint = (dtc->wb_thresh + dtc->wb_bg_thresh) / 2; 1267 } 1268 1269 if (dirty < setpoint) { 1270 x = min3(wb->balanced_dirty_ratelimit, 1271 balanced_dirty_ratelimit, task_ratelimit); 1272 if (dirty_ratelimit < x) 1273 step = x - dirty_ratelimit; 1274 } else { 1275 x = max3(wb->balanced_dirty_ratelimit, 1276 balanced_dirty_ratelimit, task_ratelimit); 1277 if (dirty_ratelimit > x) 1278 step = dirty_ratelimit - x; 1279 } 1280 1281 /* 1282 * Don't pursue 100% rate matching. It's impossible since the balanced 1283 * rate itself is constantly fluctuating. So decrease the track speed 1284 * when it gets close to the target. Helps eliminate pointless tremors. 1285 */ 1286 step >>= dirty_ratelimit / (2 * step + 1); 1287 /* 1288 * Limit the tracking speed to avoid overshooting. 1289 */ 1290 step = (step + 7) / 8; 1291 1292 if (dirty_ratelimit < balanced_dirty_ratelimit) 1293 dirty_ratelimit += step; 1294 else 1295 dirty_ratelimit -= step; 1296 1297 wb->dirty_ratelimit = max(dirty_ratelimit, 1UL); 1298 wb->balanced_dirty_ratelimit = balanced_dirty_ratelimit; 1299 1300 trace_bdi_dirty_ratelimit(wb, dirty_rate, task_ratelimit); 1301} 1302 1303static void __wb_update_bandwidth(struct dirty_throttle_control *gdtc, 1304 struct dirty_throttle_control *mdtc, 1305 unsigned long start_time, 1306 bool update_ratelimit) 1307{ 1308 struct bdi_writeback *wb = gdtc->wb; 1309 unsigned long now = jiffies; 1310 unsigned long elapsed = now - wb->bw_time_stamp; 1311 unsigned long dirtied; 1312 unsigned long written; 1313 1314 lockdep_assert_held(&wb->list_lock); 1315 1316 /* 1317 * rate-limit, only update once every 200ms. 1318 */ 1319 if (elapsed < BANDWIDTH_INTERVAL) 1320 return; 1321 1322 dirtied = percpu_counter_read(&wb->stat[WB_DIRTIED]); 1323 written = percpu_counter_read(&wb->stat[WB_WRITTEN]); 1324 1325 /* 1326 * Skip quiet periods when disk bandwidth is under-utilized. 1327 * (at least 1s idle time between two flusher runs) 1328 */ 1329 if (elapsed > HZ && time_before(wb->bw_time_stamp, start_time)) 1330 goto snapshot; 1331 1332 if (update_ratelimit) { 1333 domain_update_bandwidth(gdtc, now); 1334 wb_update_dirty_ratelimit(gdtc, dirtied, elapsed); 1335 1336 /* 1337 * @mdtc is always NULL if !CGROUP_WRITEBACK but the 1338 * compiler has no way to figure that out. Help it. 1339 */ 1340 if (IS_ENABLED(CONFIG_CGROUP_WRITEBACK) && mdtc) { 1341 domain_update_bandwidth(mdtc, now); 1342 wb_update_dirty_ratelimit(mdtc, dirtied, elapsed); 1343 } 1344 } 1345 wb_update_write_bandwidth(wb, elapsed, written); 1346 1347snapshot: 1348 wb->dirtied_stamp = dirtied; 1349 wb->written_stamp = written; 1350 wb->bw_time_stamp = now; 1351} 1352 1353void wb_update_bandwidth(struct bdi_writeback *wb, unsigned long start_time) 1354{ 1355 struct dirty_throttle_control gdtc = { GDTC_INIT(wb) }; 1356 1357 __wb_update_bandwidth(&gdtc, NULL, start_time, false); 1358} 1359 1360/* 1361 * After a task dirtied this many pages, balance_dirty_pages_ratelimited() 1362 * will look to see if it needs to start dirty throttling. 1363 * 1364 * If dirty_poll_interval is too low, big NUMA machines will call the expensive 1365 * global_page_state() too often. So scale it near-sqrt to the safety margin 1366 * (the number of pages we may dirty without exceeding the dirty limits). 1367 */ 1368static unsigned long dirty_poll_interval(unsigned long dirty, 1369 unsigned long thresh) 1370{ 1371 if (thresh > dirty) 1372 return 1UL << (ilog2(thresh - dirty) >> 1); 1373 1374 return 1; 1375} 1376 1377static unsigned long wb_max_pause(struct bdi_writeback *wb, 1378 unsigned long wb_dirty) 1379{ 1380 unsigned long bw = wb->avg_write_bandwidth; 1381 unsigned long t; 1382 1383 /* 1384 * Limit pause time for small memory systems. If sleeping for too long 1385 * time, a small pool of dirty/writeback pages may go empty and disk go 1386 * idle. 1387 * 1388 * 8 serves as the safety ratio. 1389 */ 1390 t = wb_dirty / (1 + bw / roundup_pow_of_two(1 + HZ / 8)); 1391 t++; 1392 1393 return min_t(unsigned long, t, MAX_PAUSE); 1394} 1395 1396static long wb_min_pause(struct bdi_writeback *wb, 1397 long max_pause, 1398 unsigned long task_ratelimit, 1399 unsigned long dirty_ratelimit, 1400 int *nr_dirtied_pause) 1401{ 1402 long hi = ilog2(wb->avg_write_bandwidth); 1403 long lo = ilog2(wb->dirty_ratelimit); 1404 long t; /* target pause */ 1405 long pause; /* estimated next pause */ 1406 int pages; /* target nr_dirtied_pause */ 1407 1408 /* target for 10ms pause on 1-dd case */ 1409 t = max(1, HZ / 100); 1410 1411 /* 1412 * Scale up pause time for concurrent dirtiers in order to reduce CPU 1413 * overheads. 1414 * 1415 * (N * 10ms) on 2^N concurrent tasks. 1416 */ 1417 if (hi > lo) 1418 t += (hi - lo) * (10 * HZ) / 1024; 1419 1420 /* 1421 * This is a bit convoluted. We try to base the next nr_dirtied_pause 1422 * on the much more stable dirty_ratelimit. However the next pause time 1423 * will be computed based on task_ratelimit and the two rate limits may 1424 * depart considerably at some time. Especially if task_ratelimit goes 1425 * below dirty_ratelimit/2 and the target pause is max_pause, the next 1426 * pause time will be max_pause*2 _trimmed down_ to max_pause. As a 1427 * result task_ratelimit won't be executed faithfully, which could 1428 * eventually bring down dirty_ratelimit. 1429 * 1430 * We apply two rules to fix it up: 1431 * 1) try to estimate the next pause time and if necessary, use a lower 1432 * nr_dirtied_pause so as not to exceed max_pause. When this happens, 1433 * nr_dirtied_pause will be "dancing" with task_ratelimit. 1434 * 2) limit the target pause time to max_pause/2, so that the normal 1435 * small fluctuations of task_ratelimit won't trigger rule (1) and 1436 * nr_dirtied_pause will remain as stable as dirty_ratelimit. 1437 */ 1438 t = min(t, 1 + max_pause / 2); 1439 pages = dirty_ratelimit * t / roundup_pow_of_two(HZ); 1440 1441 /* 1442 * Tiny nr_dirtied_pause is found to hurt I/O performance in the test 1443 * case fio-mmap-randwrite-64k, which does 16*{sync read, async write}. 1444 * When the 16 consecutive reads are often interrupted by some dirty 1445 * throttling pause during the async writes, cfq will go into idles 1446 * (deadline is fine). So push nr_dirtied_pause as high as possible 1447 * until reaches DIRTY_POLL_THRESH=32 pages. 1448 */ 1449 if (pages < DIRTY_POLL_THRESH) { 1450 t = max_pause; 1451 pages = dirty_ratelimit * t / roundup_pow_of_two(HZ); 1452 if (pages > DIRTY_POLL_THRESH) { 1453 pages = DIRTY_POLL_THRESH; 1454 t = HZ * DIRTY_POLL_THRESH / dirty_ratelimit; 1455 } 1456 } 1457 1458 pause = HZ * pages / (task_ratelimit + 1); 1459 if (pause > max_pause) { 1460 t = max_pause; 1461 pages = task_ratelimit * t / roundup_pow_of_two(HZ); 1462 } 1463 1464 *nr_dirtied_pause = pages; 1465 /* 1466 * The minimal pause time will normally be half the target pause time. 1467 */ 1468 return pages >= DIRTY_POLL_THRESH ? 1 + t / 2 : t; 1469} 1470 1471static inline void wb_dirty_limits(struct dirty_throttle_control *dtc) 1472{ 1473 struct bdi_writeback *wb = dtc->wb; 1474 unsigned long wb_reclaimable; 1475 1476 /* 1477 * wb_thresh is not treated as some limiting factor as 1478 * dirty_thresh, due to reasons 1479 * - in JBOD setup, wb_thresh can fluctuate a lot 1480 * - in a system with HDD and USB key, the USB key may somehow 1481 * go into state (wb_dirty >> wb_thresh) either because 1482 * wb_dirty starts high, or because wb_thresh drops low. 1483 * In this case we don't want to hard throttle the USB key 1484 * dirtiers for 100 seconds until wb_dirty drops under 1485 * wb_thresh. Instead the auxiliary wb control line in 1486 * wb_position_ratio() will let the dirtier task progress 1487 * at some rate <= (write_bw / 2) for bringing down wb_dirty. 1488 */ 1489 dtc->wb_thresh = __wb_calc_thresh(dtc); 1490 dtc->wb_bg_thresh = dtc->thresh ? 1491 div_u64((u64)dtc->wb_thresh * dtc->bg_thresh, dtc->thresh) : 0; 1492 1493 /* 1494 * In order to avoid the stacked BDI deadlock we need 1495 * to ensure we accurately count the 'dirty' pages when 1496 * the threshold is low. 1497 * 1498 * Otherwise it would be possible to get thresh+n pages 1499 * reported dirty, even though there are thresh-m pages 1500 * actually dirty; with m+n sitting in the percpu 1501 * deltas. 1502 */ 1503 if (dtc->wb_thresh < 2 * wb_stat_error(wb)) { 1504 wb_reclaimable = wb_stat_sum(wb, WB_RECLAIMABLE); 1505 dtc->wb_dirty = wb_reclaimable + wb_stat_sum(wb, WB_WRITEBACK); 1506 } else { 1507 wb_reclaimable = wb_stat(wb, WB_RECLAIMABLE); 1508 dtc->wb_dirty = wb_reclaimable + wb_stat(wb, WB_WRITEBACK); 1509 } 1510} 1511 1512/* 1513 * balance_dirty_pages() must be called by processes which are generating dirty 1514 * data. It looks at the number of dirty pages in the machine and will force 1515 * the caller to wait once crossing the (background_thresh + dirty_thresh) / 2. 1516 * If we're over `background_thresh' then the writeback threads are woken to 1517 * perform some writeout. 1518 */ 1519static void balance_dirty_pages(struct address_space *mapping, 1520 struct bdi_writeback *wb, 1521 unsigned long pages_dirtied) 1522{ 1523 struct dirty_throttle_control gdtc_stor = { GDTC_INIT(wb) }; 1524 struct dirty_throttle_control mdtc_stor = { MDTC_INIT(wb, &gdtc_stor) }; 1525 struct dirty_throttle_control * const gdtc = &gdtc_stor; 1526 struct dirty_throttle_control * const mdtc = mdtc_valid(&mdtc_stor) ? 1527 &mdtc_stor : NULL; 1528 struct dirty_throttle_control *sdtc; 1529 unsigned long nr_reclaimable; /* = file_dirty + unstable_nfs */ 1530 long period; 1531 long pause; 1532 long max_pause; 1533 long min_pause; 1534 int nr_dirtied_pause; 1535 bool dirty_exceeded = false; 1536 unsigned long task_ratelimit; 1537 unsigned long dirty_ratelimit; 1538 struct backing_dev_info *bdi = wb->bdi; 1539 bool strictlimit = bdi->capabilities & BDI_CAP_STRICTLIMIT; 1540 unsigned long start_time = jiffies; 1541 1542 for (;;) { 1543 unsigned long now = jiffies; 1544 unsigned long dirty, thresh, bg_thresh; 1545 unsigned long m_dirty = 0; /* stop bogus uninit warnings */ 1546 unsigned long m_thresh = 0; 1547 unsigned long m_bg_thresh = 0; 1548 1549 /* 1550 * Unstable writes are a feature of certain networked 1551 * filesystems (i.e. NFS) in which data may have been 1552 * written to the server's write cache, but has not yet 1553 * been flushed to permanent storage. 1554 */ 1555 nr_reclaimable = global_page_state(NR_FILE_DIRTY) + 1556 global_page_state(NR_UNSTABLE_NFS); 1557 gdtc->avail = global_dirtyable_memory(); 1558 gdtc->dirty = nr_reclaimable + global_page_state(NR_WRITEBACK); 1559 1560 domain_dirty_limits(gdtc); 1561 1562 if (unlikely(strictlimit)) { 1563 wb_dirty_limits(gdtc); 1564 1565 dirty = gdtc->wb_dirty; 1566 thresh = gdtc->wb_thresh; 1567 bg_thresh = gdtc->wb_bg_thresh; 1568 } else { 1569 dirty = gdtc->dirty; 1570 thresh = gdtc->thresh; 1571 bg_thresh = gdtc->bg_thresh; 1572 } 1573 1574 if (mdtc) { 1575 unsigned long filepages, headroom, writeback; 1576 1577 /* 1578 * If @wb belongs to !root memcg, repeat the same 1579 * basic calculations for the memcg domain. 1580 */ 1581 mem_cgroup_wb_stats(wb, &filepages, &headroom, 1582 &mdtc->dirty, &writeback); 1583 mdtc->dirty += writeback; 1584 mdtc_calc_avail(mdtc, filepages, headroom); 1585 1586 domain_dirty_limits(mdtc); 1587 1588 if (unlikely(strictlimit)) { 1589 wb_dirty_limits(mdtc); 1590 m_dirty = mdtc->wb_dirty; 1591 m_thresh = mdtc->wb_thresh; 1592 m_bg_thresh = mdtc->wb_bg_thresh; 1593 } else { 1594 m_dirty = mdtc->dirty; 1595 m_thresh = mdtc->thresh; 1596 m_bg_thresh = mdtc->bg_thresh; 1597 } 1598 } 1599 1600 /* 1601 * Throttle it only when the background writeback cannot 1602 * catch-up. This avoids (excessively) small writeouts 1603 * when the wb limits are ramping up in case of !strictlimit. 1604 * 1605 * In strictlimit case make decision based on the wb counters 1606 * and limits. Small writeouts when the wb limits are ramping 1607 * up are the price we consciously pay for strictlimit-ing. 1608 * 1609 * If memcg domain is in effect, @dirty should be under 1610 * both global and memcg freerun ceilings. 1611 */ 1612 if (dirty <= dirty_freerun_ceiling(thresh, bg_thresh) && 1613 (!mdtc || 1614 m_dirty <= dirty_freerun_ceiling(m_thresh, m_bg_thresh))) { 1615 unsigned long intv = dirty_poll_interval(dirty, thresh); 1616 unsigned long m_intv = ULONG_MAX; 1617 1618 current->dirty_paused_when = now; 1619 current->nr_dirtied = 0; 1620 if (mdtc) 1621 m_intv = dirty_poll_interval(m_dirty, m_thresh); 1622 current->nr_dirtied_pause = min(intv, m_intv); 1623 break; 1624 } 1625 1626 if (unlikely(!writeback_in_progress(wb))) 1627 wb_start_background_writeback(wb); 1628 1629 /* 1630 * Calculate global domain's pos_ratio and select the 1631 * global dtc by default. 1632 */ 1633 if (!strictlimit) 1634 wb_dirty_limits(gdtc); 1635 1636 dirty_exceeded = (gdtc->wb_dirty > gdtc->wb_thresh) && 1637 ((gdtc->dirty > gdtc->thresh) || strictlimit); 1638 1639 wb_position_ratio(gdtc); 1640 sdtc = gdtc; 1641 1642 if (mdtc) { 1643 /* 1644 * If memcg domain is in effect, calculate its 1645 * pos_ratio. @wb should satisfy constraints from 1646 * both global and memcg domains. Choose the one 1647 * w/ lower pos_ratio. 1648 */ 1649 if (!strictlimit) 1650 wb_dirty_limits(mdtc); 1651 1652 dirty_exceeded |= (mdtc->wb_dirty > mdtc->wb_thresh) && 1653 ((mdtc->dirty > mdtc->thresh) || strictlimit); 1654 1655 wb_position_ratio(mdtc); 1656 if (mdtc->pos_ratio < gdtc->pos_ratio) 1657 sdtc = mdtc; 1658 } 1659 1660 if (dirty_exceeded && !wb->dirty_exceeded) 1661 wb->dirty_exceeded = 1; 1662 1663 if (time_is_before_jiffies(wb->bw_time_stamp + 1664 BANDWIDTH_INTERVAL)) { 1665 spin_lock(&wb->list_lock); 1666 __wb_update_bandwidth(gdtc, mdtc, start_time, true); 1667 spin_unlock(&wb->list_lock); 1668 } 1669 1670 /* throttle according to the chosen dtc */ 1671 dirty_ratelimit = wb->dirty_ratelimit; 1672 task_ratelimit = ((u64)dirty_ratelimit * sdtc->pos_ratio) >> 1673 RATELIMIT_CALC_SHIFT; 1674 max_pause = wb_max_pause(wb, sdtc->wb_dirty); 1675 min_pause = wb_min_pause(wb, max_pause, 1676 task_ratelimit, dirty_ratelimit, 1677 &nr_dirtied_pause); 1678 1679 if (unlikely(task_ratelimit == 0)) { 1680 period = max_pause; 1681 pause = max_pause; 1682 goto pause; 1683 } 1684 period = HZ * pages_dirtied / task_ratelimit; 1685 pause = period; 1686 if (current->dirty_paused_when) 1687 pause -= now - current->dirty_paused_when; 1688 /* 1689 * For less than 1s think time (ext3/4 may block the dirtier 1690 * for up to 800ms from time to time on 1-HDD; so does xfs, 1691 * however at much less frequency), try to compensate it in 1692 * future periods by updating the virtual time; otherwise just 1693 * do a reset, as it may be a light dirtier. 1694 */ 1695 if (pause < min_pause) { 1696 trace_balance_dirty_pages(wb, 1697 sdtc->thresh, 1698 sdtc->bg_thresh, 1699 sdtc->dirty, 1700 sdtc->wb_thresh, 1701 sdtc->wb_dirty, 1702 dirty_ratelimit, 1703 task_ratelimit, 1704 pages_dirtied, 1705 period, 1706 min(pause, 0L), 1707 start_time); 1708 if (pause < -HZ) { 1709 current->dirty_paused_when = now; 1710 current->nr_dirtied = 0; 1711 } else if (period) { 1712 current->dirty_paused_when += period; 1713 current->nr_dirtied = 0; 1714 } else if (current->nr_dirtied_pause <= pages_dirtied) 1715 current->nr_dirtied_pause += pages_dirtied; 1716 break; 1717 } 1718 if (unlikely(pause > max_pause)) { 1719 /* for occasional dropped task_ratelimit */ 1720 now += min(pause - max_pause, max_pause); 1721 pause = max_pause; 1722 } 1723 1724pause: 1725 trace_balance_dirty_pages(wb, 1726 sdtc->thresh, 1727 sdtc->bg_thresh, 1728 sdtc->dirty, 1729 sdtc->wb_thresh, 1730 sdtc->wb_dirty, 1731 dirty_ratelimit, 1732 task_ratelimit, 1733 pages_dirtied, 1734 period, 1735 pause, 1736 start_time); 1737 __set_current_state(TASK_KILLABLE); 1738 io_schedule_timeout(pause); 1739 1740 current->dirty_paused_when = now + pause; 1741 current->nr_dirtied = 0; 1742 current->nr_dirtied_pause = nr_dirtied_pause; 1743 1744 /* 1745 * This is typically equal to (dirty < thresh) and can also 1746 * keep "1000+ dd on a slow USB stick" under control. 1747 */ 1748 if (task_ratelimit) 1749 break; 1750 1751 /* 1752 * In the case of an unresponding NFS server and the NFS dirty 1753 * pages exceeds dirty_thresh, give the other good wb's a pipe 1754 * to go through, so that tasks on them still remain responsive. 1755 * 1756 * In theory 1 page is enough to keep the comsumer-producer 1757 * pipe going: the flusher cleans 1 page => the task dirties 1 1758 * more page. However wb_dirty has accounting errors. So use 1759 * the larger and more IO friendly wb_stat_error. 1760 */ 1761 if (sdtc->wb_dirty <= wb_stat_error(wb)) 1762 break; 1763 1764 if (fatal_signal_pending(current)) 1765 break; 1766 } 1767 1768 if (!dirty_exceeded && wb->dirty_exceeded) 1769 wb->dirty_exceeded = 0; 1770 1771 if (writeback_in_progress(wb)) 1772 return; 1773 1774 /* 1775 * In laptop mode, we wait until hitting the higher threshold before 1776 * starting background writeout, and then write out all the way down 1777 * to the lower threshold. So slow writers cause minimal disk activity. 1778 * 1779 * In normal mode, we start background writeout at the lower 1780 * background_thresh, to keep the amount of dirty memory low. 1781 */ 1782 if (laptop_mode) 1783 return; 1784 1785 if (nr_reclaimable > gdtc->bg_thresh) 1786 wb_start_background_writeback(wb); 1787} 1788 1789static DEFINE_PER_CPU(int, bdp_ratelimits); 1790 1791/* 1792 * Normal tasks are throttled by 1793 * loop { 1794 * dirty tsk->nr_dirtied_pause pages; 1795 * take a snap in balance_dirty_pages(); 1796 * } 1797 * However there is a worst case. If every task exit immediately when dirtied 1798 * (tsk->nr_dirtied_pause - 1) pages, balance_dirty_pages() will never be 1799 * called to throttle the page dirties. The solution is to save the not yet 1800 * throttled page dirties in dirty_throttle_leaks on task exit and charge them 1801 * randomly into the running tasks. This works well for the above worst case, 1802 * as the new task will pick up and accumulate the old task's leaked dirty 1803 * count and eventually get throttled. 1804 */ 1805DEFINE_PER_CPU(int, dirty_throttle_leaks) = 0; 1806 1807/** 1808 * balance_dirty_pages_ratelimited - balance dirty memory state 1809 * @mapping: address_space which was dirtied 1810 * 1811 * Processes which are dirtying memory should call in here once for each page 1812 * which was newly dirtied. The function will periodically check the system's 1813 * dirty state and will initiate writeback if needed. 1814 * 1815 * On really big machines, get_writeback_state is expensive, so try to avoid 1816 * calling it too often (ratelimiting). But once we're over the dirty memory 1817 * limit we decrease the ratelimiting by a lot, to prevent individual processes 1818 * from overshooting the limit by (ratelimit_pages) each. 1819 */ 1820void balance_dirty_pages_ratelimited(struct address_space *mapping) 1821{ 1822 struct inode *inode = mapping->host; 1823 struct backing_dev_info *bdi = inode_to_bdi(inode); 1824 struct bdi_writeback *wb = NULL; 1825 int ratelimit; 1826 int *p; 1827 1828 if (!bdi_cap_account_dirty(bdi)) 1829 return; 1830 1831 if (inode_cgwb_enabled(inode)) 1832 wb = wb_get_create_current(bdi, GFP_KERNEL); 1833 if (!wb) 1834 wb = &bdi->wb; 1835 1836 ratelimit = current->nr_dirtied_pause; 1837 if (wb->dirty_exceeded) 1838 ratelimit = min(ratelimit, 32 >> (PAGE_SHIFT - 10)); 1839 1840 preempt_disable(); 1841 /* 1842 * This prevents one CPU to accumulate too many dirtied pages without 1843 * calling into balance_dirty_pages(), which can happen when there are 1844 * 1000+ tasks, all of them start dirtying pages at exactly the same 1845 * time, hence all honoured too large initial task->nr_dirtied_pause. 1846 */ 1847 p = this_cpu_ptr(&bdp_ratelimits); 1848 if (unlikely(current->nr_dirtied >= ratelimit)) 1849 *p = 0; 1850 else if (unlikely(*p >= ratelimit_pages)) { 1851 *p = 0; 1852 ratelimit = 0; 1853 } 1854 /* 1855 * Pick up the dirtied pages by the exited tasks. This avoids lots of 1856 * short-lived tasks (eg. gcc invocations in a kernel build) escaping 1857 * the dirty throttling and livelock other long-run dirtiers. 1858 */ 1859 p = this_cpu_ptr(&dirty_throttle_leaks); 1860 if (*p > 0 && current->nr_dirtied < ratelimit) { 1861 unsigned long nr_pages_dirtied; 1862 nr_pages_dirtied = min(*p, ratelimit - current->nr_dirtied); 1863 *p -= nr_pages_dirtied; 1864 current->nr_dirtied += nr_pages_dirtied; 1865 } 1866 preempt_enable(); 1867 1868 if (unlikely(current->nr_dirtied >= ratelimit)) 1869 balance_dirty_pages(mapping, wb, current->nr_dirtied); 1870 1871 wb_put(wb); 1872} 1873EXPORT_SYMBOL(balance_dirty_pages_ratelimited); 1874 1875/** 1876 * wb_over_bg_thresh - does @wb need to be written back? 1877 * @wb: bdi_writeback of interest 1878 * 1879 * Determines whether background writeback should keep writing @wb or it's 1880 * clean enough. Returns %true if writeback should continue. 1881 */ 1882bool wb_over_bg_thresh(struct bdi_writeback *wb) 1883{ 1884 struct dirty_throttle_control gdtc_stor = { GDTC_INIT(wb) }; 1885 struct dirty_throttle_control mdtc_stor = { MDTC_INIT(wb, &gdtc_stor) }; 1886 struct dirty_throttle_control * const gdtc = &gdtc_stor; 1887 struct dirty_throttle_control * const mdtc = mdtc_valid(&mdtc_stor) ? 1888 &mdtc_stor : NULL; 1889 1890 /* 1891 * Similar to balance_dirty_pages() but ignores pages being written 1892 * as we're trying to decide whether to put more under writeback. 1893 */ 1894 gdtc->avail = global_dirtyable_memory(); 1895 gdtc->dirty = global_page_state(NR_FILE_DIRTY) + 1896 global_page_state(NR_UNSTABLE_NFS); 1897 domain_dirty_limits(gdtc); 1898 1899 if (gdtc->dirty > gdtc->bg_thresh) 1900 return true; 1901 1902 if (wb_stat(wb, WB_RECLAIMABLE) > 1903 wb_calc_thresh(gdtc->wb, gdtc->bg_thresh)) 1904 return true; 1905 1906 if (mdtc) { 1907 unsigned long filepages, headroom, writeback; 1908 1909 mem_cgroup_wb_stats(wb, &filepages, &headroom, &mdtc->dirty, 1910 &writeback); 1911 mdtc_calc_avail(mdtc, filepages, headroom); 1912 domain_dirty_limits(mdtc); /* ditto, ignore writeback */ 1913 1914 if (mdtc->dirty > mdtc->bg_thresh) 1915 return true; 1916 1917 if (wb_stat(wb, WB_RECLAIMABLE) > 1918 wb_calc_thresh(mdtc->wb, mdtc->bg_thresh)) 1919 return true; 1920 } 1921 1922 return false; 1923} 1924 1925void throttle_vm_writeout(gfp_t gfp_mask) 1926{ 1927 unsigned long background_thresh; 1928 unsigned long dirty_thresh; 1929 1930 for ( ; ; ) { 1931 global_dirty_limits(&background_thresh, &dirty_thresh); 1932 dirty_thresh = hard_dirty_limit(&global_wb_domain, dirty_thresh); 1933 1934 /* 1935 * Boost the allowable dirty threshold a bit for page 1936 * allocators so they don't get DoS'ed by heavy writers 1937 */ 1938 dirty_thresh += dirty_thresh / 10; /* wheeee... */ 1939 1940 if (global_page_state(NR_UNSTABLE_NFS) + 1941 global_page_state(NR_WRITEBACK) <= dirty_thresh) 1942 break; 1943 congestion_wait(BLK_RW_ASYNC, HZ/10); 1944 1945 /* 1946 * The caller might hold locks which can prevent IO completion 1947 * or progress in the filesystem. So we cannot just sit here 1948 * waiting for IO to complete. 1949 */ 1950 if ((gfp_mask & (__GFP_FS|__GFP_IO)) != (__GFP_FS|__GFP_IO)) 1951 break; 1952 } 1953} 1954 1955/* 1956 * sysctl handler for /proc/sys/vm/dirty_writeback_centisecs 1957 */ 1958int dirty_writeback_centisecs_handler(struct ctl_table *table, int write, 1959 void __user *buffer, size_t *length, loff_t *ppos) 1960{ 1961 proc_dointvec(table, write, buffer, length, ppos); 1962 return 0; 1963} 1964 1965#ifdef CONFIG_BLOCK 1966void laptop_mode_timer_fn(unsigned long data) 1967{ 1968 struct request_queue *q = (struct request_queue *)data; 1969 int nr_pages = global_page_state(NR_FILE_DIRTY) + 1970 global_page_state(NR_UNSTABLE_NFS); 1971 struct bdi_writeback *wb; 1972 1973 /* 1974 * We want to write everything out, not just down to the dirty 1975 * threshold 1976 */ 1977 if (!bdi_has_dirty_io(&q->backing_dev_info)) 1978 return; 1979 1980 rcu_read_lock(); 1981 list_for_each_entry_rcu(wb, &q->backing_dev_info.wb_list, bdi_node) 1982 if (wb_has_dirty_io(wb)) 1983 wb_start_writeback(wb, nr_pages, true, 1984 WB_REASON_LAPTOP_TIMER); 1985 rcu_read_unlock(); 1986} 1987 1988/* 1989 * We've spun up the disk and we're in laptop mode: schedule writeback 1990 * of all dirty data a few seconds from now. If the flush is already scheduled 1991 * then push it back - the user is still using the disk. 1992 */ 1993void laptop_io_completion(struct backing_dev_info *info) 1994{ 1995 mod_timer(&info->laptop_mode_wb_timer, jiffies + laptop_mode); 1996} 1997 1998/* 1999 * We're in laptop mode and we've just synced. The sync's writes will have 2000 * caused another writeback to be scheduled by laptop_io_completion. 2001 * Nothing needs to be written back anymore, so we unschedule the writeback. 2002 */ 2003void laptop_sync_completion(void) 2004{ 2005 struct backing_dev_info *bdi; 2006 2007 rcu_read_lock(); 2008 2009 list_for_each_entry_rcu(bdi, &bdi_list, bdi_list) 2010 del_timer(&bdi->laptop_mode_wb_timer); 2011 2012 rcu_read_unlock(); 2013} 2014#endif 2015 2016/* 2017 * If ratelimit_pages is too high then we can get into dirty-data overload 2018 * if a large number of processes all perform writes at the same time. 2019 * If it is too low then SMP machines will call the (expensive) 2020 * get_writeback_state too often. 2021 * 2022 * Here we set ratelimit_pages to a level which ensures that when all CPUs are 2023 * dirtying in parallel, we cannot go more than 3% (1/32) over the dirty memory 2024 * thresholds. 2025 */ 2026 2027void writeback_set_ratelimit(void) 2028{ 2029 struct wb_domain *dom = &global_wb_domain; 2030 unsigned long background_thresh; 2031 unsigned long dirty_thresh; 2032 2033 global_dirty_limits(&background_thresh, &dirty_thresh); 2034 dom->dirty_limit = dirty_thresh; 2035 ratelimit_pages = dirty_thresh / (num_online_cpus() * 32); 2036 if (ratelimit_pages < 16) 2037 ratelimit_pages = 16; 2038} 2039 2040static int 2041ratelimit_handler(struct notifier_block *self, unsigned long action, 2042 void *hcpu) 2043{ 2044 2045 switch (action & ~CPU_TASKS_FROZEN) { 2046 case CPU_ONLINE: 2047 case CPU_DEAD: 2048 writeback_set_ratelimit(); 2049 return NOTIFY_OK; 2050 default: 2051 return NOTIFY_DONE; 2052 } 2053} 2054 2055static struct notifier_block ratelimit_nb = { 2056 .notifier_call = ratelimit_handler, 2057 .next = NULL, 2058}; 2059 2060/* 2061 * Called early on to tune the page writeback dirty limits. 2062 * 2063 * We used to scale dirty pages according to how total memory 2064 * related to pages that could be allocated for buffers (by 2065 * comparing nr_free_buffer_pages() to vm_total_pages. 2066 * 2067 * However, that was when we used "dirty_ratio" to scale with 2068 * all memory, and we don't do that any more. "dirty_ratio" 2069 * is now applied to total non-HIGHPAGE memory (by subtracting 2070 * totalhigh_pages from vm_total_pages), and as such we can't 2071 * get into the old insane situation any more where we had 2072 * large amounts of dirty pages compared to a small amount of 2073 * non-HIGHMEM memory. 2074 * 2075 * But we might still want to scale the dirty_ratio by how 2076 * much memory the box has.. 2077 */ 2078void __init page_writeback_init(void) 2079{ 2080 BUG_ON(wb_domain_init(&global_wb_domain, GFP_KERNEL)); 2081 2082 writeback_set_ratelimit(); 2083 register_cpu_notifier(&ratelimit_nb); 2084} 2085 2086/** 2087 * tag_pages_for_writeback - tag pages to be written by write_cache_pages 2088 * @mapping: address space structure to write 2089 * @start: starting page index 2090 * @end: ending page index (inclusive) 2091 * 2092 * This function scans the page range from @start to @end (inclusive) and tags 2093 * all pages that have DIRTY tag set with a special TOWRITE tag. The idea is 2094 * that write_cache_pages (or whoever calls this function) will then use 2095 * TOWRITE tag to identify pages eligible for writeback. This mechanism is 2096 * used to avoid livelocking of writeback by a process steadily creating new 2097 * dirty pages in the file (thus it is important for this function to be quick 2098 * so that it can tag pages faster than a dirtying process can create them). 2099 */ 2100/* 2101 * We tag pages in batches of WRITEBACK_TAG_BATCH to reduce tree_lock latency. 2102 */ 2103void tag_pages_for_writeback(struct address_space *mapping, 2104 pgoff_t start, pgoff_t end) 2105{ 2106#define WRITEBACK_TAG_BATCH 4096 2107 unsigned long tagged; 2108 2109 do { 2110 spin_lock_irq(&mapping->tree_lock); 2111 tagged = radix_tree_range_tag_if_tagged(&mapping->page_tree, 2112 &start, end, WRITEBACK_TAG_BATCH, 2113 PAGECACHE_TAG_DIRTY, PAGECACHE_TAG_TOWRITE); 2114 spin_unlock_irq(&mapping->tree_lock); 2115 WARN_ON_ONCE(tagged > WRITEBACK_TAG_BATCH); 2116 cond_resched(); 2117 /* We check 'start' to handle wrapping when end == ~0UL */ 2118 } while (tagged >= WRITEBACK_TAG_BATCH && start); 2119} 2120EXPORT_SYMBOL(tag_pages_for_writeback); 2121 2122/** 2123 * write_cache_pages - walk the list of dirty pages of the given address space and write all of them. 2124 * @mapping: address space structure to write 2125 * @wbc: subtract the number of written pages from *@wbc->nr_to_write 2126 * @writepage: function called for each page 2127 * @data: data passed to writepage function 2128 * 2129 * If a page is already under I/O, write_cache_pages() skips it, even 2130 * if it's dirty. This is desirable behaviour for memory-cleaning writeback, 2131 * but it is INCORRECT for data-integrity system calls such as fsync(). fsync() 2132 * and msync() need to guarantee that all the data which was dirty at the time 2133 * the call was made get new I/O started against them. If wbc->sync_mode is 2134 * WB_SYNC_ALL then we were called for data integrity and we must wait for 2135 * existing IO to complete. 2136 * 2137 * To avoid livelocks (when other process dirties new pages), we first tag 2138 * pages which should be written back with TOWRITE tag and only then start 2139 * writing them. For data-integrity sync we have to be careful so that we do 2140 * not miss some pages (e.g., because some other process has cleared TOWRITE 2141 * tag we set). The rule we follow is that TOWRITE tag can be cleared only 2142 * by the process clearing the DIRTY tag (and submitting the page for IO). 2143 */ 2144int write_cache_pages(struct address_space *mapping, 2145 struct writeback_control *wbc, writepage_t writepage, 2146 void *data) 2147{ 2148 int ret = 0; 2149 int done = 0; 2150 struct pagevec pvec; 2151 int nr_pages; 2152 pgoff_t uninitialized_var(writeback_index); 2153 pgoff_t index; 2154 pgoff_t end; /* Inclusive */ 2155 pgoff_t done_index; 2156 int cycled; 2157 int range_whole = 0; 2158 int tag; 2159 2160 pagevec_init(&pvec, 0); 2161 if (wbc->range_cyclic) { 2162 writeback_index = mapping->writeback_index; /* prev offset */ 2163 index = writeback_index; 2164 if (index == 0) 2165 cycled = 1; 2166 else 2167 cycled = 0; 2168 end = -1; 2169 } else { 2170 index = wbc->range_start >> PAGE_CACHE_SHIFT; 2171 end = wbc->range_end >> PAGE_CACHE_SHIFT; 2172 if (wbc->range_start == 0 && wbc->range_end == LLONG_MAX) 2173 range_whole = 1; 2174 cycled = 1; /* ignore range_cyclic tests */ 2175 } 2176 if (wbc->sync_mode == WB_SYNC_ALL || wbc->tagged_writepages) 2177 tag = PAGECACHE_TAG_TOWRITE; 2178 else 2179 tag = PAGECACHE_TAG_DIRTY; 2180retry: 2181 if (wbc->sync_mode == WB_SYNC_ALL || wbc->tagged_writepages) 2182 tag_pages_for_writeback(mapping, index, end); 2183 done_index = index; 2184 while (!done && (index <= end)) { 2185 int i; 2186 2187 nr_pages = pagevec_lookup_tag(&pvec, mapping, &index, tag, 2188 min(end - index, (pgoff_t)PAGEVEC_SIZE-1) + 1); 2189 if (nr_pages == 0) 2190 break; 2191 2192 for (i = 0; i < nr_pages; i++) { 2193 struct page *page = pvec.pages[i]; 2194 2195 /* 2196 * At this point, the page may be truncated or 2197 * invalidated (changing page->mapping to NULL), or 2198 * even swizzled back from swapper_space to tmpfs file 2199 * mapping. However, page->index will not change 2200 * because we have a reference on the page. 2201 */ 2202 if (page->index > end) { 2203 /* 2204 * can't be range_cyclic (1st pass) because 2205 * end == -1 in that case. 2206 */ 2207 done = 1; 2208 break; 2209 } 2210 2211 done_index = page->index; 2212 2213 lock_page(page); 2214 2215 /* 2216 * Page truncated or invalidated. We can freely skip it 2217 * then, even for data integrity operations: the page 2218 * has disappeared concurrently, so there could be no 2219 * real expectation of this data interity operation 2220 * even if there is now a new, dirty page at the same 2221 * pagecache address. 2222 */ 2223 if (unlikely(page->mapping != mapping)) { 2224continue_unlock: 2225 unlock_page(page); 2226 continue; 2227 } 2228 2229 if (!PageDirty(page)) { 2230 /* someone wrote it for us */ 2231 goto continue_unlock; 2232 } 2233 2234 if (PageWriteback(page)) { 2235 if (wbc->sync_mode != WB_SYNC_NONE) 2236 wait_on_page_writeback(page); 2237 else 2238 goto continue_unlock; 2239 } 2240 2241 BUG_ON(PageWriteback(page)); 2242 if (!clear_page_dirty_for_io(page)) 2243 goto continue_unlock; 2244 2245 trace_wbc_writepage(wbc, inode_to_bdi(mapping->host)); 2246 ret = (*writepage)(page, wbc, data); 2247 if (unlikely(ret)) { 2248 if (ret == AOP_WRITEPAGE_ACTIVATE) { 2249 unlock_page(page); 2250 ret = 0; 2251 } else { 2252 /* 2253 * done_index is set past this page, 2254 * so media errors will not choke 2255 * background writeout for the entire 2256 * file. This has consequences for 2257 * range_cyclic semantics (ie. it may 2258 * not be suitable for data integrity 2259 * writeout). 2260 */ 2261 done_index = page->index + 1; 2262 done = 1; 2263 break; 2264 } 2265 } 2266 2267 /* 2268 * We stop writing back only if we are not doing 2269 * integrity sync. In case of integrity sync we have to 2270 * keep going until we have written all the pages 2271 * we tagged for writeback prior to entering this loop. 2272 */ 2273 if (--wbc->nr_to_write <= 0 && 2274 wbc->sync_mode == WB_SYNC_NONE) { 2275 done = 1; 2276 break; 2277 } 2278 } 2279 pagevec_release(&pvec); 2280 cond_resched(); 2281 } 2282 if (!cycled && !done) { 2283 /* 2284 * range_cyclic: 2285 * We hit the last page and there is more work to be done: wrap 2286 * back to the start of the file 2287 */ 2288 cycled = 1; 2289 index = 0; 2290 end = writeback_index - 1; 2291 goto retry; 2292 } 2293 if (wbc->range_cyclic || (range_whole && wbc->nr_to_write > 0)) 2294 mapping->writeback_index = done_index; 2295 2296 return ret; 2297} 2298EXPORT_SYMBOL(write_cache_pages); 2299 2300/* 2301 * Function used by generic_writepages to call the real writepage 2302 * function and set the mapping flags on error 2303 */ 2304static int __writepage(struct page *page, struct writeback_control *wbc, 2305 void *data) 2306{ 2307 struct address_space *mapping = data; 2308 int ret = mapping->a_ops->writepage(page, wbc); 2309 mapping_set_error(mapping, ret); 2310 return ret; 2311} 2312 2313/** 2314 * generic_writepages - walk the list of dirty pages of the given address space and writepage() all of them. 2315 * @mapping: address space structure to write 2316 * @wbc: subtract the number of written pages from *@wbc->nr_to_write 2317 * 2318 * This is a library function, which implements the writepages() 2319 * address_space_operation. 2320 */ 2321int generic_writepages(struct address_space *mapping, 2322 struct writeback_control *wbc) 2323{ 2324 struct blk_plug plug; 2325 int ret; 2326 2327 /* deal with chardevs and other special file */ 2328 if (!mapping->a_ops->writepage) 2329 return 0; 2330 2331 blk_start_plug(&plug); 2332 ret = write_cache_pages(mapping, wbc, __writepage, mapping); 2333 blk_finish_plug(&plug); 2334 return ret; 2335} 2336 2337EXPORT_SYMBOL(generic_writepages); 2338 2339int do_writepages(struct address_space *mapping, struct writeback_control *wbc) 2340{ 2341 int ret; 2342 2343 if (wbc->nr_to_write <= 0) 2344 return 0; 2345 if (mapping->a_ops->writepages) 2346 ret = mapping->a_ops->writepages(mapping, wbc); 2347 else 2348 ret = generic_writepages(mapping, wbc); 2349 return ret; 2350} 2351 2352/** 2353 * write_one_page - write out a single page and optionally wait on I/O 2354 * @page: the page to write 2355 * @wait: if true, wait on writeout 2356 * 2357 * The page must be locked by the caller and will be unlocked upon return. 2358 * 2359 * write_one_page() returns a negative error code if I/O failed. 2360 */ 2361int write_one_page(struct page *page, int wait) 2362{ 2363 struct address_space *mapping = page->mapping; 2364 int ret = 0; 2365 struct writeback_control wbc = { 2366 .sync_mode = WB_SYNC_ALL, 2367 .nr_to_write = 1, 2368 }; 2369 2370 BUG_ON(!PageLocked(page)); 2371 2372 if (wait) 2373 wait_on_page_writeback(page); 2374 2375 if (clear_page_dirty_for_io(page)) { 2376 page_cache_get(page); 2377 ret = mapping->a_ops->writepage(page, &wbc); 2378 if (ret == 0 && wait) { 2379 wait_on_page_writeback(page); 2380 if (PageError(page)) 2381 ret = -EIO; 2382 } 2383 page_cache_release(page); 2384 } else { 2385 unlock_page(page); 2386 } 2387 return ret; 2388} 2389EXPORT_SYMBOL(write_one_page); 2390 2391/* 2392 * For address_spaces which do not use buffers nor write back. 2393 */ 2394int __set_page_dirty_no_writeback(struct page *page) 2395{ 2396 if (!PageDirty(page)) 2397 return !TestSetPageDirty(page); 2398 return 0; 2399} 2400 2401/* 2402 * Helper function for set_page_dirty family. 2403 * 2404 * Caller must hold mem_cgroup_begin_page_stat(). 2405 * 2406 * NOTE: This relies on being atomic wrt interrupts. 2407 */ 2408void account_page_dirtied(struct page *page, struct address_space *mapping, 2409 struct mem_cgroup *memcg) 2410{ 2411 struct inode *inode = mapping->host; 2412 2413 trace_writeback_dirty_page(page, mapping); 2414 2415 if (mapping_cap_account_dirty(mapping)) { 2416 struct bdi_writeback *wb; 2417 2418 inode_attach_wb(inode, page); 2419 wb = inode_to_wb(inode); 2420 2421 mem_cgroup_inc_page_stat(memcg, MEM_CGROUP_STAT_DIRTY); 2422 __inc_zone_page_state(page, NR_FILE_DIRTY); 2423 __inc_zone_page_state(page, NR_DIRTIED); 2424 __inc_wb_stat(wb, WB_RECLAIMABLE); 2425 __inc_wb_stat(wb, WB_DIRTIED); 2426 task_io_account_write(PAGE_CACHE_SIZE); 2427 current->nr_dirtied++; 2428 this_cpu_inc(bdp_ratelimits); 2429 } 2430} 2431EXPORT_SYMBOL(account_page_dirtied); 2432 2433/* 2434 * Helper function for deaccounting dirty page without writeback. 2435 * 2436 * Caller must hold mem_cgroup_begin_page_stat(). 2437 */ 2438void account_page_cleaned(struct page *page, struct address_space *mapping, 2439 struct mem_cgroup *memcg, struct bdi_writeback *wb) 2440{ 2441 if (mapping_cap_account_dirty(mapping)) { 2442 mem_cgroup_dec_page_stat(memcg, MEM_CGROUP_STAT_DIRTY); 2443 dec_zone_page_state(page, NR_FILE_DIRTY); 2444 dec_wb_stat(wb, WB_RECLAIMABLE); 2445 task_io_account_cancelled_write(PAGE_CACHE_SIZE); 2446 } 2447} 2448 2449/* 2450 * For address_spaces which do not use buffers. Just tag the page as dirty in 2451 * its radix tree. 2452 * 2453 * This is also used when a single buffer is being dirtied: we want to set the 2454 * page dirty in that case, but not all the buffers. This is a "bottom-up" 2455 * dirtying, whereas __set_page_dirty_buffers() is a "top-down" dirtying. 2456 * 2457 * The caller must ensure this doesn't race with truncation. Most will simply 2458 * hold the page lock, but e.g. zap_pte_range() calls with the page mapped and 2459 * the pte lock held, which also locks out truncation. 2460 */ 2461int __set_page_dirty_nobuffers(struct page *page) 2462{ 2463 struct mem_cgroup *memcg; 2464 2465 memcg = mem_cgroup_begin_page_stat(page); 2466 if (!TestSetPageDirty(page)) { 2467 struct address_space *mapping = page_mapping(page); 2468 unsigned long flags; 2469 2470 if (!mapping) { 2471 mem_cgroup_end_page_stat(memcg); 2472 return 1; 2473 } 2474 2475 spin_lock_irqsave(&mapping->tree_lock, flags); 2476 BUG_ON(page_mapping(page) != mapping); 2477 WARN_ON_ONCE(!PagePrivate(page) && !PageUptodate(page)); 2478 account_page_dirtied(page, mapping, memcg); 2479 radix_tree_tag_set(&mapping->page_tree, page_index(page), 2480 PAGECACHE_TAG_DIRTY); 2481 spin_unlock_irqrestore(&mapping->tree_lock, flags); 2482 mem_cgroup_end_page_stat(memcg); 2483 2484 if (mapping->host) { 2485 /* !PageAnon && !swapper_space */ 2486 __mark_inode_dirty(mapping->host, I_DIRTY_PAGES); 2487 } 2488 return 1; 2489 } 2490 mem_cgroup_end_page_stat(memcg); 2491 return 0; 2492} 2493EXPORT_SYMBOL(__set_page_dirty_nobuffers); 2494 2495/* 2496 * Call this whenever redirtying a page, to de-account the dirty counters 2497 * (NR_DIRTIED, BDI_DIRTIED, tsk->nr_dirtied), so that they match the written 2498 * counters (NR_WRITTEN, BDI_WRITTEN) in long term. The mismatches will lead to 2499 * systematic errors in balanced_dirty_ratelimit and the dirty pages position 2500 * control. 2501 */ 2502void account_page_redirty(struct page *page) 2503{ 2504 struct address_space *mapping = page->mapping; 2505 2506 if (mapping && mapping_cap_account_dirty(mapping)) { 2507 struct inode *inode = mapping->host; 2508 struct bdi_writeback *wb; 2509 bool locked; 2510 2511 wb = unlocked_inode_to_wb_begin(inode, &locked); 2512 current->nr_dirtied--; 2513 dec_zone_page_state(page, NR_DIRTIED); 2514 dec_wb_stat(wb, WB_DIRTIED); 2515 unlocked_inode_to_wb_end(inode, locked); 2516 } 2517} 2518EXPORT_SYMBOL(account_page_redirty); 2519 2520/* 2521 * When a writepage implementation decides that it doesn't want to write this 2522 * page for some reason, it should redirty the locked page via 2523 * redirty_page_for_writepage() and it should then unlock the page and return 0 2524 */ 2525int redirty_page_for_writepage(struct writeback_control *wbc, struct page *page) 2526{ 2527 int ret; 2528 2529 wbc->pages_skipped++; 2530 ret = __set_page_dirty_nobuffers(page); 2531 account_page_redirty(page); 2532 return ret; 2533} 2534EXPORT_SYMBOL(redirty_page_for_writepage); 2535 2536/* 2537 * Dirty a page. 2538 * 2539 * For pages with a mapping this should be done under the page lock 2540 * for the benefit of asynchronous memory errors who prefer a consistent 2541 * dirty state. This rule can be broken in some special cases, 2542 * but should be better not to. 2543 * 2544 * If the mapping doesn't provide a set_page_dirty a_op, then 2545 * just fall through and assume that it wants buffer_heads. 2546 */ 2547int set_page_dirty(struct page *page) 2548{ 2549 struct address_space *mapping = page_mapping(page); 2550 2551 if (likely(mapping)) { 2552 int (*spd)(struct page *) = mapping->a_ops->set_page_dirty; 2553 /* 2554 * readahead/lru_deactivate_page could remain 2555 * PG_readahead/PG_reclaim due to race with end_page_writeback 2556 * About readahead, if the page is written, the flags would be 2557 * reset. So no problem. 2558 * About lru_deactivate_page, if the page is redirty, the flag 2559 * will be reset. So no problem. but if the page is used by readahead 2560 * it will confuse readahead and make it restart the size rampup 2561 * process. But it's a trivial problem. 2562 */ 2563 if (PageReclaim(page)) 2564 ClearPageReclaim(page); 2565#ifdef CONFIG_BLOCK 2566 if (!spd) 2567 spd = __set_page_dirty_buffers; 2568#endif 2569 return (*spd)(page); 2570 } 2571 if (!PageDirty(page)) { 2572 if (!TestSetPageDirty(page)) 2573 return 1; 2574 } 2575 return 0; 2576} 2577EXPORT_SYMBOL(set_page_dirty); 2578 2579/* 2580 * set_page_dirty() is racy if the caller has no reference against 2581 * page->mapping->host, and if the page is unlocked. This is because another 2582 * CPU could truncate the page off the mapping and then free the mapping. 2583 * 2584 * Usually, the page _is_ locked, or the caller is a user-space process which 2585 * holds a reference on the inode by having an open file. 2586 * 2587 * In other cases, the page should be locked before running set_page_dirty(). 2588 */ 2589int set_page_dirty_lock(struct page *page) 2590{ 2591 int ret; 2592 2593 lock_page(page); 2594 ret = set_page_dirty(page); 2595 unlock_page(page); 2596 return ret; 2597} 2598EXPORT_SYMBOL(set_page_dirty_lock); 2599 2600/* 2601 * This cancels just the dirty bit on the kernel page itself, it does NOT 2602 * actually remove dirty bits on any mmap's that may be around. It also 2603 * leaves the page tagged dirty, so any sync activity will still find it on 2604 * the dirty lists, and in particular, clear_page_dirty_for_io() will still 2605 * look at the dirty bits in the VM. 2606 * 2607 * Doing this should *normally* only ever be done when a page is truncated, 2608 * and is not actually mapped anywhere at all. However, fs/buffer.c does 2609 * this when it notices that somebody has cleaned out all the buffers on a 2610 * page without actually doing it through the VM. Can you say "ext3 is 2611 * horribly ugly"? Thought you could. 2612 */ 2613void cancel_dirty_page(struct page *page) 2614{ 2615 struct address_space *mapping = page_mapping(page); 2616 2617 if (mapping_cap_account_dirty(mapping)) { 2618 struct inode *inode = mapping->host; 2619 struct bdi_writeback *wb; 2620 struct mem_cgroup *memcg; 2621 bool locked; 2622 2623 memcg = mem_cgroup_begin_page_stat(page); 2624 wb = unlocked_inode_to_wb_begin(inode, &locked); 2625 2626 if (TestClearPageDirty(page)) 2627 account_page_cleaned(page, mapping, memcg, wb); 2628 2629 unlocked_inode_to_wb_end(inode, locked); 2630 mem_cgroup_end_page_stat(memcg); 2631 } else { 2632 ClearPageDirty(page); 2633 } 2634} 2635EXPORT_SYMBOL(cancel_dirty_page); 2636 2637/* 2638 * Clear a page's dirty flag, while caring for dirty memory accounting. 2639 * Returns true if the page was previously dirty. 2640 * 2641 * This is for preparing to put the page under writeout. We leave the page 2642 * tagged as dirty in the radix tree so that a concurrent write-for-sync 2643 * can discover it via a PAGECACHE_TAG_DIRTY walk. The ->writepage 2644 * implementation will run either set_page_writeback() or set_page_dirty(), 2645 * at which stage we bring the page's dirty flag and radix-tree dirty tag 2646 * back into sync. 2647 * 2648 * This incoherency between the page's dirty flag and radix-tree tag is 2649 * unfortunate, but it only exists while the page is locked. 2650 */ 2651int clear_page_dirty_for_io(struct page *page) 2652{ 2653 struct address_space *mapping = page_mapping(page); 2654 int ret = 0; 2655 2656 BUG_ON(!PageLocked(page)); 2657 2658 if (mapping && mapping_cap_account_dirty(mapping)) { 2659 struct inode *inode = mapping->host; 2660 struct bdi_writeback *wb; 2661 struct mem_cgroup *memcg; 2662 bool locked; 2663 2664 /* 2665 * Yes, Virginia, this is indeed insane. 2666 * 2667 * We use this sequence to make sure that 2668 * (a) we account for dirty stats properly 2669 * (b) we tell the low-level filesystem to 2670 * mark the whole page dirty if it was 2671 * dirty in a pagetable. Only to then 2672 * (c) clean the page again and return 1 to 2673 * cause the writeback. 2674 * 2675 * This way we avoid all nasty races with the 2676 * dirty bit in multiple places and clearing 2677 * them concurrently from different threads. 2678 * 2679 * Note! Normally the "set_page_dirty(page)" 2680 * has no effect on the actual dirty bit - since 2681 * that will already usually be set. But we 2682 * need the side effects, and it can help us 2683 * avoid races. 2684 * 2685 * We basically use the page "master dirty bit" 2686 * as a serialization point for all the different 2687 * threads doing their things. 2688 */ 2689 if (page_mkclean(page)) 2690 set_page_dirty(page); 2691 /* 2692 * We carefully synchronise fault handlers against 2693 * installing a dirty pte and marking the page dirty 2694 * at this point. We do this by having them hold the 2695 * page lock while dirtying the page, and pages are 2696 * always locked coming in here, so we get the desired 2697 * exclusion. 2698 */ 2699 memcg = mem_cgroup_begin_page_stat(page); 2700 wb = unlocked_inode_to_wb_begin(inode, &locked); 2701 if (TestClearPageDirty(page)) { 2702 mem_cgroup_dec_page_stat(memcg, MEM_CGROUP_STAT_DIRTY); 2703 dec_zone_page_state(page, NR_FILE_DIRTY); 2704 dec_wb_stat(wb, WB_RECLAIMABLE); 2705 ret = 1; 2706 } 2707 unlocked_inode_to_wb_end(inode, locked); 2708 mem_cgroup_end_page_stat(memcg); 2709 return ret; 2710 } 2711 return TestClearPageDirty(page); 2712} 2713EXPORT_SYMBOL(clear_page_dirty_for_io); 2714 2715int test_clear_page_writeback(struct page *page) 2716{ 2717 struct address_space *mapping = page_mapping(page); 2718 struct mem_cgroup *memcg; 2719 int ret; 2720 2721 memcg = mem_cgroup_begin_page_stat(page); 2722 if (mapping) { 2723 struct inode *inode = mapping->host; 2724 struct backing_dev_info *bdi = inode_to_bdi(inode); 2725 unsigned long flags; 2726 2727 spin_lock_irqsave(&mapping->tree_lock, flags); 2728 ret = TestClearPageWriteback(page); 2729 if (ret) { 2730 radix_tree_tag_clear(&mapping->page_tree, 2731 page_index(page), 2732 PAGECACHE_TAG_WRITEBACK); 2733 if (bdi_cap_account_writeback(bdi)) { 2734 struct bdi_writeback *wb = inode_to_wb(inode); 2735 2736 __dec_wb_stat(wb, WB_WRITEBACK); 2737 __wb_writeout_inc(wb); 2738 } 2739 } 2740 spin_unlock_irqrestore(&mapping->tree_lock, flags); 2741 } else { 2742 ret = TestClearPageWriteback(page); 2743 } 2744 if (ret) { 2745 mem_cgroup_dec_page_stat(memcg, MEM_CGROUP_STAT_WRITEBACK); 2746 dec_zone_page_state(page, NR_WRITEBACK); 2747 inc_zone_page_state(page, NR_WRITTEN); 2748 } 2749 mem_cgroup_end_page_stat(memcg); 2750 return ret; 2751} 2752 2753int __test_set_page_writeback(struct page *page, bool keep_write) 2754{ 2755 struct address_space *mapping = page_mapping(page); 2756 struct mem_cgroup *memcg; 2757 int ret; 2758 2759 memcg = mem_cgroup_begin_page_stat(page); 2760 if (mapping) { 2761 struct inode *inode = mapping->host; 2762 struct backing_dev_info *bdi = inode_to_bdi(inode); 2763 unsigned long flags; 2764 2765 spin_lock_irqsave(&mapping->tree_lock, flags); 2766 ret = TestSetPageWriteback(page); 2767 if (!ret) { 2768 radix_tree_tag_set(&mapping->page_tree, 2769 page_index(page), 2770 PAGECACHE_TAG_WRITEBACK); 2771 if (bdi_cap_account_writeback(bdi)) 2772 __inc_wb_stat(inode_to_wb(inode), WB_WRITEBACK); 2773 } 2774 if (!PageDirty(page)) 2775 radix_tree_tag_clear(&mapping->page_tree, 2776 page_index(page), 2777 PAGECACHE_TAG_DIRTY); 2778 if (!keep_write) 2779 radix_tree_tag_clear(&mapping->page_tree, 2780 page_index(page), 2781 PAGECACHE_TAG_TOWRITE); 2782 spin_unlock_irqrestore(&mapping->tree_lock, flags); 2783 } else { 2784 ret = TestSetPageWriteback(page); 2785 } 2786 if (!ret) { 2787 mem_cgroup_inc_page_stat(memcg, MEM_CGROUP_STAT_WRITEBACK); 2788 inc_zone_page_state(page, NR_WRITEBACK); 2789 } 2790 mem_cgroup_end_page_stat(memcg); 2791 return ret; 2792 2793} 2794EXPORT_SYMBOL(__test_set_page_writeback); 2795 2796/* 2797 * Return true if any of the pages in the mapping are marked with the 2798 * passed tag. 2799 */ 2800int mapping_tagged(struct address_space *mapping, int tag) 2801{ 2802 return radix_tree_tagged(&mapping->page_tree, tag); 2803} 2804EXPORT_SYMBOL(mapping_tagged); 2805 2806/** 2807 * wait_for_stable_page() - wait for writeback to finish, if necessary. 2808 * @page: The page to wait on. 2809 * 2810 * This function determines if the given page is related to a backing device 2811 * that requires page contents to be held stable during writeback. If so, then 2812 * it will wait for any pending writeback to complete. 2813 */ 2814void wait_for_stable_page(struct page *page) 2815{ 2816 if (bdi_cap_stable_pages_required(inode_to_bdi(page->mapping->host))) 2817 wait_on_page_writeback(page); 2818} 2819EXPORT_SYMBOL_GPL(wait_for_stable_page); 2820