87 static inline struct inode *wb_inode(struct list_head *head)  in wb_inode()
89 	return list_entry(head, struct inode, i_io_list);  in wb_inode()
135 static bool inode_io_list_move_locked(struct inode *inode,  in inode_io_list_move_locked()  argument
141 	list_move(&inode->i_io_list, head);  in inode_io_list_move_locked()
159 static void inode_io_list_del_locked(struct inode *inode,  in inode_io_list_del_locked()  argument
164 	list_del_init(&inode->i_io_list);  in inode_io_list_del_locked()
229 void __inode_attach_wb(struct inode *inode, struct page *page)  in __inode_attach_wb()  argument
231 	struct backing_dev_info *bdi = inode_to_bdi(inode);  in __inode_attach_wb()
234 	if (inode_cgwb_enabled(inode)) {  in __inode_attach_wb()
255 	if (unlikely(cmpxchg(&inode->i_wb, NULL, wb)))  in __inode_attach_wb()
268 locked_inode_to_wb_and_lock_list(struct inode *inode)  in locked_inode_to_wb_and_lock_list()  argument
269 	__releases(&inode->i_lock)  in locked_inode_to_wb_and_lock_list()
273 		struct bdi_writeback *wb = inode_to_wb(inode);  in locked_inode_to_wb_and_lock_list()
282 		spin_unlock(&inode->i_lock);  in locked_inode_to_wb_and_lock_list()
286 		if (likely(wb == inode->i_wb)) {  in locked_inode_to_wb_and_lock_list()
294 		spin_lock(&inode->i_lock);  in locked_inode_to_wb_and_lock_list()
305 static struct bdi_writeback *inode_to_wb_and_lock_list(struct inode *inode)  in inode_to_wb_and_lock_list()  argument
308 	spin_lock(&inode->i_lock);  in inode_to_wb_and_lock_list()
309 	return locked_inode_to_wb_and_lock_list(inode);  in inode_to_wb_and_lock_list()
313 	struct inode		*inode;  member
324 	struct inode *inode = isw->inode;  in inode_switch_wbs_work_fn()  local
325 	struct address_space *mapping = inode->i_mapping;  in inode_switch_wbs_work_fn()
326 	struct bdi_writeback *old_wb = inode->i_wb;  in inode_switch_wbs_work_fn()
349 	spin_lock(&inode->i_lock);  in inode_switch_wbs_work_fn()
356 	if (unlikely(inode->i_state & I_FREEING))  in inode_switch_wbs_work_fn()
393 	if (!list_empty(&inode->i_io_list)) {  in inode_switch_wbs_work_fn()
394 		struct inode *pos;  in inode_switch_wbs_work_fn()
396 		inode_io_list_del_locked(inode, old_wb);  in inode_switch_wbs_work_fn()
397 		inode->i_wb = new_wb;  in inode_switch_wbs_work_fn()
399 			if (time_after_eq(inode->dirtied_when,  in inode_switch_wbs_work_fn()
402 		inode_io_list_move_locked(inode, new_wb, pos->i_io_list.prev);  in inode_switch_wbs_work_fn()
404 		inode->i_wb = new_wb;  in inode_switch_wbs_work_fn()
408 	inode->i_wb_frn_winner = 0;  in inode_switch_wbs_work_fn()
409 	inode->i_wb_frn_avg_time = 0;  in inode_switch_wbs_work_fn()
410 	inode->i_wb_frn_history = 0;  in inode_switch_wbs_work_fn()
417 	smp_store_release(&inode->i_state, inode->i_state & ~I_WB_SWITCH);  in inode_switch_wbs_work_fn()
420 	spin_unlock(&inode->i_lock);  in inode_switch_wbs_work_fn()
430 	iput(inode);  in inode_switch_wbs_work_fn()
454 static void inode_switch_wbs(struct inode *inode, int new_wb_id)  in inode_switch_wbs()  argument
456 	struct backing_dev_info *bdi = inode_to_bdi(inode);  in inode_switch_wbs()
461 	if (inode->i_state & I_WB_SWITCH)  in inode_switch_wbs()
478 	spin_lock(&inode->i_lock);  in inode_switch_wbs()
479 	if (!(inode->i_sb->s_flags & MS_ACTIVE) ||  in inode_switch_wbs()
480 	    inode->i_state & (I_WB_SWITCH | I_FREEING) ||  in inode_switch_wbs()
481 	    inode_to_wb(inode) == isw->new_wb) {  in inode_switch_wbs()
482 		spin_unlock(&inode->i_lock);  in inode_switch_wbs()
485 	inode->i_state |= I_WB_SWITCH;  in inode_switch_wbs()
486 	spin_unlock(&inode->i_lock);  in inode_switch_wbs()
488 	ihold(inode);  in inode_switch_wbs()
489 	isw->inode = inode;  in inode_switch_wbs()
519 				 struct inode *inode)  in wbc_attach_and_unlock_inode()  argument
521 	if (!inode_cgwb_enabled(inode)) {  in wbc_attach_and_unlock_inode()
522 		spin_unlock(&inode->i_lock);  in wbc_attach_and_unlock_inode()
526 	wbc->wb = inode_to_wb(inode);  in wbc_attach_and_unlock_inode()
527 	wbc->inode = inode;  in wbc_attach_and_unlock_inode()
530 	wbc->wb_lcand_id = inode->i_wb_frn_winner;  in wbc_attach_and_unlock_inode()
537 	spin_unlock(&inode->i_lock);  in wbc_attach_and_unlock_inode()
544 		inode_switch_wbs(inode, wbc->wb_id);  in wbc_attach_and_unlock_inode()
587 	struct inode *inode = wbc->inode;  in wbc_detach_inode()  local
595 	history = inode->i_wb_frn_history;  in wbc_detach_inode()
596 	avg_time = inode->i_wb_frn_avg_time;  in wbc_detach_inode()
651 			inode_switch_wbs(inode, max_id);  in wbc_detach_inode()
658 	inode->i_wb_frn_winner = max_id;  in wbc_detach_inode()
659 	inode->i_wb_frn_avg_time = min(avg_time, (unsigned long)U16_MAX);  in wbc_detach_inode()
660 	inode->i_wb_frn_history = history;  in wbc_detach_inode()
728 int inode_congested(struct inode *inode, int cong_bits)  in inode_congested()  argument
734 	if (inode && inode_to_wb_is_valid(inode)) {  in inode_congested()
738 		wb = unlocked_inode_to_wb_begin(inode, &locked);  in inode_congested()
740 		unlocked_inode_to_wb_end(inode, locked);  in inode_congested()
744 	return wb_congested(&inode_to_bdi(inode)->wb, cong_bits);  in inode_congested()
885 locked_inode_to_wb_and_lock_list(struct inode *inode)  in locked_inode_to_wb_and_lock_list()  argument
886 	__releases(&inode->i_lock)  in locked_inode_to_wb_and_lock_list()
889 	struct bdi_writeback *wb = inode_to_wb(inode);  in locked_inode_to_wb_and_lock_list()
891 	spin_unlock(&inode->i_lock);  in locked_inode_to_wb_and_lock_list()
896 static struct bdi_writeback *inode_to_wb_and_lock_list(struct inode *inode)  in inode_to_wb_and_lock_list()  argument
899 	struct bdi_writeback *wb = inode_to_wb(inode);  in inode_to_wb_and_lock_list()
975 void inode_io_list_del(struct inode *inode)  in inode_io_list_del()  argument
979 	wb = inode_to_wb_and_lock_list(inode);  in inode_io_list_del()
980 	inode_io_list_del_locked(inode, wb);  in inode_io_list_del()
993 static void redirty_tail(struct inode *inode, struct bdi_writeback *wb)  in redirty_tail()  argument
996 		struct inode *tail;  in redirty_tail()
999 		if (time_before(inode->dirtied_when, tail->dirtied_when))  in redirty_tail()
1000 			inode->dirtied_when = jiffies;  in redirty_tail()
1002 	inode_io_list_move_locked(inode, wb, &wb->b_dirty);  in redirty_tail()
1008 static void requeue_io(struct inode *inode, struct bdi_writeback *wb)  in requeue_io()  argument
1010 	inode_io_list_move_locked(inode, wb, &wb->b_more_io);  in requeue_io()
1013 static void inode_sync_complete(struct inode *inode)  in inode_sync_complete()  argument
1015 	inode->i_state &= ~I_SYNC;  in inode_sync_complete()
1017 	inode_add_lru(inode);  in inode_sync_complete()
1020 	wake_up_bit(&inode->i_state, __I_SYNC);  in inode_sync_complete()
1023 static bool inode_dirtied_after(struct inode *inode, unsigned long t)  in inode_dirtied_after()  argument
1025 	bool ret = time_after(inode->dirtied_when, t);  in inode_dirtied_after()
1033 	ret = ret && time_before_eq(inode->dirtied_when, jiffies);  in inode_dirtied_after()
1054 	struct inode *inode;  in move_expired_inodes()  local
1065 		inode = wb_inode(delaying_queue->prev);  in move_expired_inodes()
1067 		    inode_dirtied_after(inode, *older_than_this))  in move_expired_inodes()
1069 		list_move(&inode->i_io_list, &tmp);  in move_expired_inodes()
1072 			set_bit(__I_DIRTY_TIME_EXPIRED, &inode->i_state);  in move_expired_inodes()
1073 		if (sb_is_blkdev_sb(inode->i_sb))  in move_expired_inodes()
1075 		if (sb && sb != inode->i_sb)  in move_expired_inodes()
1077 		sb = inode->i_sb;  in move_expired_inodes()
1090 			inode = wb_inode(pos);  in move_expired_inodes()
1091 			if (inode->i_sb == sb)  in move_expired_inodes()
1092 				list_move(&inode->i_io_list, dispatch_queue);  in move_expired_inodes()
1124 static int write_inode(struct inode *inode, struct writeback_control *wbc)  in write_inode()  argument
1128 	if (inode->i_sb->s_op->write_inode && !is_bad_inode(inode)) {  in write_inode()
1129 		trace_writeback_write_inode_start(inode, wbc);  in write_inode()
1130 		ret = inode->i_sb->s_op->write_inode(inode, wbc);  in write_inode()
1131 		trace_writeback_write_inode(inode, wbc);  in write_inode()
1141 static void __inode_wait_for_writeback(struct inode *inode)  in __inode_wait_for_writeback()  argument
1142 	__releases(inode->i_lock)  in __inode_wait_for_writeback()
1143 	__acquires(inode->i_lock)  in __inode_wait_for_writeback()
1145 	DEFINE_WAIT_BIT(wq, &inode->i_state, __I_SYNC);  in __inode_wait_for_writeback()
1148 	wqh = bit_waitqueue(&inode->i_state, __I_SYNC);  in __inode_wait_for_writeback()
1149 	while (inode->i_state & I_SYNC) {  in __inode_wait_for_writeback()
1150 		spin_unlock(&inode->i_lock);  in __inode_wait_for_writeback()
1153 		spin_lock(&inode->i_lock);  in __inode_wait_for_writeback()
1160 void inode_wait_for_writeback(struct inode *inode)  in inode_wait_for_writeback()  argument
1162 	spin_lock(&inode->i_lock);  in inode_wait_for_writeback()
1163 	__inode_wait_for_writeback(inode);  in inode_wait_for_writeback()
1164 	spin_unlock(&inode->i_lock);  in inode_wait_for_writeback()
1172 static void inode_sleep_on_writeback(struct inode *inode)  in inode_sleep_on_writeback()  argument
1173 	__releases(inode->i_lock)  in inode_sleep_on_writeback()
1176 	wait_queue_head_t *wqh = bit_waitqueue(&inode->i_state, __I_SYNC);  in inode_sleep_on_writeback()
1180 	sleep = inode->i_state & I_SYNC;  in inode_sleep_on_writeback()
1181 	spin_unlock(&inode->i_lock);  in inode_sleep_on_writeback()
1195 static void requeue_inode(struct inode *inode, struct bdi_writeback *wb,  in requeue_inode()  argument
1198 	if (inode->i_state & I_FREEING)  in requeue_inode()
1206 	if ((inode->i_state & I_DIRTY) &&  in requeue_inode()
1208 		inode->dirtied_when = jiffies;  in requeue_inode()
1215 		redirty_tail(inode, wb);  in requeue_inode()
1219 	if (mapping_tagged(inode->i_mapping, PAGECACHE_TAG_DIRTY)) {  in requeue_inode()
1226 			requeue_io(inode, wb);  in requeue_inode()
1235 			redirty_tail(inode, wb);  in requeue_inode()
1237 	} else if (inode->i_state & I_DIRTY) {  in requeue_inode()
1243 		redirty_tail(inode, wb);  in requeue_inode()
1244 	} else if (inode->i_state & I_DIRTY_TIME) {  in requeue_inode()
1245 		inode->dirtied_when = jiffies;  in requeue_inode()
1246 		inode_io_list_move_locked(inode, wb, &wb->b_dirty_time);  in requeue_inode()
1249 		inode_io_list_del_locked(inode, wb);  in requeue_inode()
1259 __writeback_single_inode(struct inode *inode, struct writeback_control *wbc)  in __writeback_single_inode()  argument
1261 	struct address_space *mapping = inode->i_mapping;  in __writeback_single_inode()
1266 	WARN_ON(!(inode->i_state & I_SYNC));  in __writeback_single_inode()
1268 	trace_writeback_single_inode_start(inode, wbc, nr_to_write);  in __writeback_single_inode()
1290 	spin_lock(&inode->i_lock);  in __writeback_single_inode()
1292 	dirty = inode->i_state & I_DIRTY;  in __writeback_single_inode()
1293 	if (inode->i_state & I_DIRTY_TIME) {  in __writeback_single_inode()
1295 		    unlikely(inode->i_state & I_DIRTY_TIME_EXPIRED) ||  in __writeback_single_inode()
1297 					(inode->dirtied_time_when +  in __writeback_single_inode()
1300 			trace_writeback_lazytime(inode);  in __writeback_single_inode()
1303 		inode->i_state &= ~I_DIRTY_TIME_EXPIRED;  in __writeback_single_inode()
1304 	inode->i_state &= ~dirty;  in __writeback_single_inode()
1320 		inode->i_state |= I_DIRTY_PAGES;  in __writeback_single_inode()
1322 	spin_unlock(&inode->i_lock);  in __writeback_single_inode()
1325 		mark_inode_dirty_sync(inode);  in __writeback_single_inode()
1328 		int err = write_inode(inode, wbc);  in __writeback_single_inode()
1332 	trace_writeback_single_inode(inode, wbc, nr_to_write);  in __writeback_single_inode()
1344 static int writeback_single_inode(struct inode *inode,  in writeback_single_inode()  argument
1350 	spin_lock(&inode->i_lock);  in writeback_single_inode()
1351 	if (!atomic_read(&inode->i_count))  in writeback_single_inode()
1352 		WARN_ON(!(inode->i_state & (I_WILL_FREE|I_FREEING)));  in writeback_single_inode()
1354 		WARN_ON(inode->i_state & I_WILL_FREE);  in writeback_single_inode()
1356 	if (inode->i_state & I_SYNC) {  in writeback_single_inode()
1364 		__inode_wait_for_writeback(inode);  in writeback_single_inode()
1366 	WARN_ON(inode->i_state & I_SYNC);  in writeback_single_inode()
1375 	if (!(inode->i_state & I_DIRTY_ALL) &&  in writeback_single_inode()
1377 	     !mapping_tagged(inode->i_mapping, PAGECACHE_TAG_WRITEBACK)))  in writeback_single_inode()
1379 	inode->i_state |= I_SYNC;  in writeback_single_inode()
1380 	wbc_attach_and_unlock_inode(wbc, inode);  in writeback_single_inode()
1382 	ret = __writeback_single_inode(inode, wbc);  in writeback_single_inode()
1386 	wb = inode_to_wb_and_lock_list(inode);  in writeback_single_inode()
1387 	spin_lock(&inode->i_lock);  in writeback_single_inode()
1392 	if (!(inode->i_state & I_DIRTY_ALL))  in writeback_single_inode()
1393 		inode_io_list_del_locked(inode, wb);  in writeback_single_inode()
1395 	inode_sync_complete(inode);  in writeback_single_inode()
1397 	spin_unlock(&inode->i_lock);  in writeback_single_inode()
1460 		struct inode *inode = wb_inode(wb->b_io.prev);  in writeback_sb_inodes()  local
1463 		if (inode->i_sb != sb) {  in writeback_sb_inodes()
1470 				redirty_tail(inode, wb);  in writeback_sb_inodes()
1487 		spin_lock(&inode->i_lock);  in writeback_sb_inodes()
1488 		if (inode->i_state & (I_NEW | I_FREEING | I_WILL_FREE)) {  in writeback_sb_inodes()
1489 			spin_unlock(&inode->i_lock);  in writeback_sb_inodes()
1490 			redirty_tail(inode, wb);  in writeback_sb_inodes()
1493 		if ((inode->i_state & I_SYNC) && wbc.sync_mode != WB_SYNC_ALL) {  in writeback_sb_inodes()
1503 			spin_unlock(&inode->i_lock);  in writeback_sb_inodes()
1504 			requeue_io(inode, wb);  in writeback_sb_inodes()
1505 			trace_writeback_sb_inodes_requeue(inode);  in writeback_sb_inodes()
1515 		if (inode->i_state & I_SYNC) {  in writeback_sb_inodes()
1517 			inode_sleep_on_writeback(inode);  in writeback_sb_inodes()
1522 		inode->i_state |= I_SYNC;  in writeback_sb_inodes()
1523 		wbc_attach_and_unlock_inode(&wbc, inode);  in writeback_sb_inodes()
1533 		__writeback_single_inode(inode, &wbc);  in writeback_sb_inodes()
1556 		tmp_wb = inode_to_wb_and_lock_list(inode);  in writeback_sb_inodes()
1557 		spin_lock(&inode->i_lock);  in writeback_sb_inodes()
1558 		if (!(inode->i_state & I_DIRTY_ALL))  in writeback_sb_inodes()
1560 		requeue_inode(inode, tmp_wb, &wbc);  in writeback_sb_inodes()
1561 		inode_sync_complete(inode);  in writeback_sb_inodes()
1562 		spin_unlock(&inode->i_lock);  in writeback_sb_inodes()
1590 		struct inode *inode = wb_inode(wb->b_io.prev);  in __writeback_inodes_wb()  local
1591 		struct super_block *sb = inode->i_sb;  in __writeback_inodes_wb()
1599 			redirty_tail(inode, wb);  in __writeback_inodes_wb()
1660 	struct inode *inode;  in wb_writeback()  local
1738 			inode = wb_inode(wb->b_more_io.prev);  in wb_writeback()
1739 			spin_lock(&inode->i_lock);  in wb_writeback()
1742 			inode_sleep_on_writeback(inode);  in wb_writeback()
1985 static noinline void block_dump___mark_inode_dirty(struct inode *inode)  in block_dump___mark_inode_dirty()  argument
1987 	if (inode->i_ino || strcmp(inode->i_sb->s_id, "bdev")) {  in block_dump___mark_inode_dirty()
1991 		dentry = d_find_alias(inode);  in block_dump___mark_inode_dirty()
1998 		       current->comm, task_pid_nr(current), inode->i_ino,  in block_dump___mark_inode_dirty()
1999 		       name, inode->i_sb->s_id);  in block_dump___mark_inode_dirty()
2031 void __mark_inode_dirty(struct inode *inode, int flags)  in __mark_inode_dirty()  argument
2034 	struct super_block *sb = inode->i_sb;  in __mark_inode_dirty()
2037 	trace_writeback_mark_inode_dirty(inode, flags);  in __mark_inode_dirty()
2044 		trace_writeback_dirty_inode_start(inode, flags);  in __mark_inode_dirty()
2047 			sb->s_op->dirty_inode(inode, flags);  in __mark_inode_dirty()
2049 		trace_writeback_dirty_inode(inode, flags);  in __mark_inode_dirty()
2061 	if (((inode->i_state & flags) == flags) ||  in __mark_inode_dirty()
2062 	    (dirtytime && (inode->i_state & I_DIRTY_INODE)))  in __mark_inode_dirty()
2066 		block_dump___mark_inode_dirty(inode);  in __mark_inode_dirty()
2068 	spin_lock(&inode->i_lock);  in __mark_inode_dirty()
2069 	if (dirtytime && (inode->i_state & I_DIRTY_INODE))  in __mark_inode_dirty()
2071 	if ((inode->i_state & flags) != flags) {  in __mark_inode_dirty()
2072 		const int was_dirty = inode->i_state & I_DIRTY;  in __mark_inode_dirty()
2074 		inode_attach_wb(inode, NULL);  in __mark_inode_dirty()
2077 			inode->i_state &= ~I_DIRTY_TIME;  in __mark_inode_dirty()
2078 		inode->i_state |= flags;  in __mark_inode_dirty()
2085 		if (inode->i_state & I_SYNC)  in __mark_inode_dirty()
2092 		if (!S_ISBLK(inode->i_mode)) {  in __mark_inode_dirty()
2093 			if (inode_unhashed(inode))  in __mark_inode_dirty()
2096 		if (inode->i_state & I_FREEING)  in __mark_inode_dirty()
2108 			wb = locked_inode_to_wb_and_lock_list(inode);  in __mark_inode_dirty()
2114 			inode->dirtied_when = jiffies;  in __mark_inode_dirty()
2116 				inode->dirtied_time_when = jiffies;  in __mark_inode_dirty()
2118 			if (inode->i_state & (I_DIRTY_INODE | I_DIRTY_PAGES))  in __mark_inode_dirty()
2123 			wakeup_bdi = inode_io_list_move_locked(inode, wb,  in __mark_inode_dirty()
2127 			trace_writeback_dirty_inode_enqueue(inode);  in __mark_inode_dirty()
2141 	spin_unlock(&inode->i_lock);  in __mark_inode_dirty()
2158 	struct inode *inode, *old_inode = NULL;  in wait_sb_inodes()  local
2176 	list_for_each_entry(inode, &sb->s_inodes, i_sb_list) {  in wait_sb_inodes()
2177 		struct address_space *mapping = inode->i_mapping;  in wait_sb_inodes()
2179 		spin_lock(&inode->i_lock);  in wait_sb_inodes()
2180 		if ((inode->i_state & (I_FREEING|I_WILL_FREE|I_NEW)) ||  in wait_sb_inodes()
2182 			spin_unlock(&inode->i_lock);  in wait_sb_inodes()
2185 		__iget(inode);  in wait_sb_inodes()
2186 		spin_unlock(&inode->i_lock);  in wait_sb_inodes()
2198 		old_inode = inode;  in wait_sb_inodes()
2353 int write_inode_now(struct inode *inode, int sync)  in write_inode_now()  argument
2362 	if (!mapping_cap_writeback_dirty(inode->i_mapping))  in write_inode_now()
2366 	return writeback_single_inode(inode, &wbc);  in write_inode_now()
2381 int sync_inode(struct inode *inode, struct writeback_control *wbc)  in sync_inode()  argument
2383 	return writeback_single_inode(inode, wbc);  in sync_inode()
2396 int sync_inode_metadata(struct inode *inode, int wait)  in sync_inode_metadata()  argument
2403 	return sync_inode(inode, &wbc);  in sync_inode_metadata()