Lines Matching refs:wb
102 static bool wb_io_lists_populated(struct bdi_writeback *wb) in wb_io_lists_populated() argument
104 if (wb_has_dirty_io(wb)) { in wb_io_lists_populated()
107 set_bit(WB_has_dirty_io, &wb->state); in wb_io_lists_populated()
108 WARN_ON_ONCE(!wb->avg_write_bandwidth); in wb_io_lists_populated()
109 atomic_long_add(wb->avg_write_bandwidth, in wb_io_lists_populated()
110 &wb->bdi->tot_write_bandwidth); in wb_io_lists_populated()
115 static void wb_io_lists_depopulated(struct bdi_writeback *wb) in wb_io_lists_depopulated() argument
117 if (wb_has_dirty_io(wb) && list_empty(&wb->b_dirty) && in wb_io_lists_depopulated()
118 list_empty(&wb->b_io) && list_empty(&wb->b_more_io)) { in wb_io_lists_depopulated()
119 clear_bit(WB_has_dirty_io, &wb->state); in wb_io_lists_depopulated()
120 WARN_ON_ONCE(atomic_long_sub_return(wb->avg_write_bandwidth, in wb_io_lists_depopulated()
121 &wb->bdi->tot_write_bandwidth) < 0); in wb_io_lists_depopulated()
136 struct bdi_writeback *wb, in inode_io_list_move_locked() argument
139 assert_spin_locked(&wb->list_lock); in inode_io_list_move_locked()
144 if (head != &wb->b_dirty_time) in inode_io_list_move_locked()
145 return wb_io_lists_populated(wb); in inode_io_list_move_locked()
147 wb_io_lists_depopulated(wb); in inode_io_list_move_locked()
160 struct bdi_writeback *wb) in inode_io_list_del_locked() argument
162 assert_spin_locked(&wb->list_lock); in inode_io_list_del_locked()
165 wb_io_lists_depopulated(wb); in inode_io_list_del_locked()
168 static void wb_wakeup(struct bdi_writeback *wb) in wb_wakeup() argument
170 spin_lock_bh(&wb->work_lock); in wb_wakeup()
171 if (test_bit(WB_registered, &wb->state)) in wb_wakeup()
172 mod_delayed_work(bdi_wq, &wb->dwork, 0); in wb_wakeup()
173 spin_unlock_bh(&wb->work_lock); in wb_wakeup()
176 static void wb_queue_work(struct bdi_writeback *wb, in wb_queue_work() argument
179 trace_writeback_queue(wb, work); in wb_queue_work()
181 spin_lock_bh(&wb->work_lock); in wb_queue_work()
182 if (!test_bit(WB_registered, &wb->state)) in wb_queue_work()
186 list_add_tail(&work->list, &wb->work_list); in wb_queue_work()
187 mod_delayed_work(bdi_wq, &wb->dwork, 0); in wb_queue_work()
189 spin_unlock_bh(&wb->work_lock); in wb_queue_work()
232 struct bdi_writeback *wb = NULL; in __inode_attach_wb() local
239 wb = wb_get_create(bdi, memcg_css, GFP_ATOMIC); in __inode_attach_wb()
243 wb = wb_get_create(bdi, memcg_css, GFP_ATOMIC); in __inode_attach_wb()
248 if (!wb) in __inode_attach_wb()
249 wb = &bdi->wb; in __inode_attach_wb()
255 if (unlikely(cmpxchg(&inode->i_wb, NULL, wb))) in __inode_attach_wb()
256 wb_put(wb); in __inode_attach_wb()
270 __acquires(&wb->list_lock) in locked_inode_to_wb_and_lock_list()
273 struct bdi_writeback *wb = inode_to_wb(inode); in locked_inode_to_wb_and_lock_list() local
281 wb_get(wb); in locked_inode_to_wb_and_lock_list()
283 spin_lock(&wb->list_lock); in locked_inode_to_wb_and_lock_list()
286 if (likely(wb == inode->i_wb)) { in locked_inode_to_wb_and_lock_list()
287 wb_put(wb); /* @inode already has ref */ in locked_inode_to_wb_and_lock_list()
288 return wb; in locked_inode_to_wb_and_lock_list()
291 spin_unlock(&wb->list_lock); in locked_inode_to_wb_and_lock_list()
292 wb_put(wb); in locked_inode_to_wb_and_lock_list()
306 __acquires(&wb->list_lock) in inode_to_wb_and_lock_list()
526 wbc->wb = inode_to_wb(inode); in wbc_attach_and_unlock_inode()
529 wbc->wb_id = wbc->wb->memcg_css->id; in wbc_attach_and_unlock_inode()
536 wb_get(wbc->wb); in wbc_attach_and_unlock_inode()
543 if (unlikely(wb_dying(wbc->wb))) in wbc_attach_and_unlock_inode()
586 struct bdi_writeback *wb = wbc->wb; in wbc_detach_inode() local
592 if (!wb) in wbc_detach_inode()
619 wb->avg_write_bandwidth); in wbc_detach_inode()
662 wb_put(wbc->wb); in wbc_detach_inode()
663 wbc->wb = NULL; in wbc_detach_inode()
687 if (!wbc->wb) in wbc_account_io()
735 struct bdi_writeback *wb; in inode_congested() local
738 wb = unlocked_inode_to_wb_begin(inode, &locked); in inode_congested()
739 congested = wb_congested(wb, cong_bits); in inode_congested()
744 return wb_congested(&inode_to_bdi(inode)->wb, cong_bits); in inode_congested()
757 static long wb_split_bdi_pages(struct bdi_writeback *wb, long nr_pages) in wb_split_bdi_pages() argument
759 unsigned long this_bw = wb->avg_write_bandwidth; in wb_split_bdi_pages()
760 unsigned long tot_bw = atomic_long_read(&wb->bdi->tot_write_bandwidth); in wb_split_bdi_pages()
792 struct bdi_writeback *wb = list_entry(&bdi->wb_list, in bdi_split_work_to_wbs() local
798 list_for_each_entry_continue_rcu(wb, &bdi->wb_list, bdi_node) { in bdi_split_work_to_wbs()
810 if (!wb_has_dirty_io(wb) && in bdi_split_work_to_wbs()
812 list_empty(&wb->b_dirty_time))) in bdi_split_work_to_wbs()
814 if (skip_if_busy && writeback_in_progress(wb)) in bdi_split_work_to_wbs()
817 nr_pages = wb_split_bdi_pages(wb, base_work->nr_pages); in bdi_split_work_to_wbs()
824 wb_queue_work(wb, work); in bdi_split_work_to_wbs()
835 wb_queue_work(wb, work); in bdi_split_work_to_wbs()
842 wb_get(wb); in bdi_split_work_to_wbs()
843 last_wb = wb; in bdi_split_work_to_wbs()
887 __acquires(&wb->list_lock) in locked_inode_to_wb_and_lock_list()
889 struct bdi_writeback *wb = inode_to_wb(inode); in locked_inode_to_wb_and_lock_list() local
892 spin_lock(&wb->list_lock); in locked_inode_to_wb_and_lock_list()
893 return wb; in locked_inode_to_wb_and_lock_list()
897 __acquires(&wb->list_lock) in inode_to_wb_and_lock_list()
899 struct bdi_writeback *wb = inode_to_wb(inode); in inode_to_wb_and_lock_list() local
901 spin_lock(&wb->list_lock); in inode_to_wb_and_lock_list()
902 return wb; in inode_to_wb_and_lock_list()
905 static long wb_split_bdi_pages(struct bdi_writeback *wb, long nr_pages) in wb_split_bdi_pages() argument
916 if (!skip_if_busy || !writeback_in_progress(&bdi->wb)) { in bdi_split_work_to_wbs()
918 wb_queue_work(&bdi->wb, base_work); in bdi_split_work_to_wbs()
924 void wb_start_writeback(struct bdi_writeback *wb, long nr_pages, in wb_start_writeback() argument
929 if (!wb_has_dirty_io(wb)) in wb_start_writeback()
938 trace_writeback_nowork(wb); in wb_start_writeback()
939 wb_wakeup(wb); in wb_start_writeback()
949 wb_queue_work(wb, work); in wb_start_writeback()
962 void wb_start_background_writeback(struct bdi_writeback *wb) in wb_start_background_writeback() argument
968 trace_writeback_wake_background(wb); in wb_start_background_writeback()
969 wb_wakeup(wb); in wb_start_background_writeback()
977 struct bdi_writeback *wb; in inode_io_list_del() local
979 wb = inode_to_wb_and_lock_list(inode); in inode_io_list_del()
980 inode_io_list_del_locked(inode, wb); in inode_io_list_del()
981 spin_unlock(&wb->list_lock); in inode_io_list_del()
993 static void redirty_tail(struct inode *inode, struct bdi_writeback *wb) in redirty_tail() argument
995 if (!list_empty(&wb->b_dirty)) { in redirty_tail()
998 tail = wb_inode(wb->b_dirty.next); in redirty_tail()
1002 inode_io_list_move_locked(inode, wb, &wb->b_dirty); in redirty_tail()
1008 static void requeue_io(struct inode *inode, struct bdi_writeback *wb) in requeue_io() argument
1010 inode_io_list_move_locked(inode, wb, &wb->b_more_io); in requeue_io()
1110 static void queue_io(struct bdi_writeback *wb, struct wb_writeback_work *work) in queue_io() argument
1114 assert_spin_locked(&wb->list_lock); in queue_io()
1115 list_splice_init(&wb->b_more_io, &wb->b_io); in queue_io()
1116 moved = move_expired_inodes(&wb->b_dirty, &wb->b_io, 0, work); in queue_io()
1117 moved += move_expired_inodes(&wb->b_dirty_time, &wb->b_io, in queue_io()
1120 wb_io_lists_populated(wb); in queue_io()
1121 trace_writeback_queue_io(wb, work, moved); in queue_io()
1195 static void requeue_inode(struct inode *inode, struct bdi_writeback *wb, in requeue_inode() argument
1215 redirty_tail(inode, wb); in requeue_inode()
1226 requeue_io(inode, wb); in requeue_inode()
1235 redirty_tail(inode, wb); in requeue_inode()
1243 redirty_tail(inode, wb); in requeue_inode()
1246 inode_io_list_move_locked(inode, wb, &wb->b_dirty_time); in requeue_inode()
1249 inode_io_list_del_locked(inode, wb); in requeue_inode()
1347 struct bdi_writeback *wb; in writeback_single_inode() local
1386 wb = inode_to_wb_and_lock_list(inode); in writeback_single_inode()
1393 inode_io_list_del_locked(inode, wb); in writeback_single_inode()
1394 spin_unlock(&wb->list_lock); in writeback_single_inode()
1401 static long writeback_chunk_size(struct bdi_writeback *wb, in writeback_chunk_size() argument
1422 pages = min(wb->avg_write_bandwidth / 2, in writeback_chunk_size()
1442 struct bdi_writeback *wb, in writeback_sb_inodes() argument
1459 while (!list_empty(&wb->b_io)) { in writeback_sb_inodes()
1460 struct inode *inode = wb_inode(wb->b_io.prev); in writeback_sb_inodes()
1470 redirty_tail(inode, wb); in writeback_sb_inodes()
1490 redirty_tail(inode, wb); in writeback_sb_inodes()
1504 requeue_io(inode, wb); in writeback_sb_inodes()
1508 spin_unlock(&wb->list_lock); in writeback_sb_inodes()
1519 spin_lock(&wb->list_lock); in writeback_sb_inodes()
1525 write_chunk = writeback_chunk_size(wb, work); in writeback_sb_inodes()
1564 if (unlikely(tmp_wb != wb)) { in writeback_sb_inodes()
1566 spin_lock(&wb->list_lock); in writeback_sb_inodes()
1583 static long __writeback_inodes_wb(struct bdi_writeback *wb, in __writeback_inodes_wb() argument
1589 while (!list_empty(&wb->b_io)) { in __writeback_inodes_wb()
1590 struct inode *inode = wb_inode(wb->b_io.prev); in __writeback_inodes_wb()
1599 redirty_tail(inode, wb); in __writeback_inodes_wb()
1602 wrote += writeback_sb_inodes(sb, wb, work); in __writeback_inodes_wb()
1617 static long writeback_inodes_wb(struct bdi_writeback *wb, long nr_pages, in writeback_inodes_wb() argument
1629 spin_lock(&wb->list_lock); in writeback_inodes_wb()
1630 if (list_empty(&wb->b_io)) in writeback_inodes_wb()
1631 queue_io(wb, &work); in writeback_inodes_wb()
1632 __writeback_inodes_wb(wb, &work); in writeback_inodes_wb()
1633 spin_unlock(&wb->list_lock); in writeback_inodes_wb()
1654 static long wb_writeback(struct bdi_writeback *wb, in wb_writeback() argument
1668 spin_lock(&wb->list_lock); in wb_writeback()
1683 !list_empty(&wb->work_list)) in wb_writeback()
1690 if (work->for_background && !wb_over_bg_thresh(wb)) in wb_writeback()
1705 trace_writeback_start(wb, work); in wb_writeback()
1706 if (list_empty(&wb->b_io)) in wb_writeback()
1707 queue_io(wb, work); in wb_writeback()
1709 progress = writeback_sb_inodes(work->sb, wb, work); in wb_writeback()
1711 progress = __writeback_inodes_wb(wb, work); in wb_writeback()
1712 trace_writeback_written(wb, work); in wb_writeback()
1714 wb_update_bandwidth(wb, wb_start); in wb_writeback()
1729 if (list_empty(&wb->b_more_io)) in wb_writeback()
1736 if (!list_empty(&wb->b_more_io)) { in wb_writeback()
1737 trace_writeback_wait(wb, work); in wb_writeback()
1738 inode = wb_inode(wb->b_more_io.prev); in wb_writeback()
1740 spin_unlock(&wb->list_lock); in wb_writeback()
1743 spin_lock(&wb->list_lock); in wb_writeback()
1746 spin_unlock(&wb->list_lock); in wb_writeback()
1755 static struct wb_writeback_work *get_next_work_item(struct bdi_writeback *wb) in get_next_work_item() argument
1759 spin_lock_bh(&wb->work_lock); in get_next_work_item()
1760 if (!list_empty(&wb->work_list)) { in get_next_work_item()
1761 work = list_entry(wb->work_list.next, in get_next_work_item()
1765 spin_unlock_bh(&wb->work_lock); in get_next_work_item()
1780 static long wb_check_background_flush(struct bdi_writeback *wb) in wb_check_background_flush() argument
1782 if (wb_over_bg_thresh(wb)) { in wb_check_background_flush()
1792 return wb_writeback(wb, &work); in wb_check_background_flush()
1798 static long wb_check_old_data_flush(struct bdi_writeback *wb) in wb_check_old_data_flush() argument
1809 expired = wb->last_old_flush + in wb_check_old_data_flush()
1814 wb->last_old_flush = jiffies; in wb_check_old_data_flush()
1826 return wb_writeback(wb, &work); in wb_check_old_data_flush()
1835 static long wb_do_writeback(struct bdi_writeback *wb) in wb_do_writeback() argument
1840 set_bit(WB_writeback_running, &wb->state); in wb_do_writeback()
1841 while ((work = get_next_work_item(wb)) != NULL) { in wb_do_writeback()
1844 trace_writeback_exec(wb, work); in wb_do_writeback()
1846 wrote += wb_writeback(wb, work); in wb_do_writeback()
1851 wake_up_all(&wb->bdi->wb_waitq); in wb_do_writeback()
1857 wrote += wb_check_old_data_flush(wb); in wb_do_writeback()
1858 wrote += wb_check_background_flush(wb); in wb_do_writeback()
1859 clear_bit(WB_writeback_running, &wb->state); in wb_do_writeback()
1870 struct bdi_writeback *wb = container_of(to_delayed_work(work), in wb_workfn() local
1874 set_worker_desc("flush-%s", dev_name(wb->bdi->dev)); in wb_workfn()
1878 !test_bit(WB_registered, &wb->state))) { in wb_workfn()
1886 pages_written = wb_do_writeback(wb); in wb_workfn()
1888 } while (!list_empty(&wb->work_list)); in wb_workfn()
1895 pages_written = writeback_inodes_wb(wb, 1024, in wb_workfn()
1900 if (!list_empty(&wb->work_list)) in wb_workfn()
1901 mod_delayed_work(bdi_wq, &wb->dwork, 0); in wb_workfn()
1902 else if (wb_has_dirty_io(wb) && dirty_writeback_interval) in wb_workfn()
1903 wb_wakeup_delayed(wb); in wb_workfn()
1921 struct bdi_writeback *wb; in wakeup_flusher_threads() local
1926 list_for_each_entry_rcu(wb, &bdi->wb_list, bdi_node) in wakeup_flusher_threads()
1927 wb_start_writeback(wb, wb_split_bdi_pages(wb, nr_pages), in wakeup_flusher_threads()
1957 struct bdi_writeback *wb; in wakeup_dirtytime_writeback() local
1959 list_for_each_entry_rcu(wb, &bdi->wb_list, bdi_node) in wakeup_dirtytime_writeback()
1960 if (!list_empty(&wb->b_dirty_time)) in wakeup_dirtytime_writeback()
1961 wb_wakeup(wb); in wakeup_dirtytime_writeback()
2104 struct bdi_writeback *wb; in __mark_inode_dirty() local
2108 wb = locked_inode_to_wb_and_lock_list(inode); in __mark_inode_dirty()
2110 WARN(bdi_cap_writeback_dirty(wb->bdi) && in __mark_inode_dirty()
2111 !test_bit(WB_registered, &wb->state), in __mark_inode_dirty()
2112 "bdi-%s not registered\n", wb->bdi->name); in __mark_inode_dirty()
2119 dirty_list = &wb->b_dirty; in __mark_inode_dirty()
2121 dirty_list = &wb->b_dirty_time; in __mark_inode_dirty()
2123 wakeup_bdi = inode_io_list_move_locked(inode, wb, in __mark_inode_dirty()
2126 spin_unlock(&wb->list_lock); in __mark_inode_dirty()
2135 if (bdi_cap_writeback_dirty(wb->bdi) && wakeup_bdi) in __mark_inode_dirty()
2136 wb_wakeup_delayed(wb); in __mark_inode_dirty()