79 	xfs_ioend_t		*ioend)  in xfs_destroy_ioend()  argument
83 	for (bh = ioend->io_buffer_head; bh; bh = next) {  in xfs_destroy_ioend()
85 		bh->b_end_io(bh, !ioend->io_error);  in xfs_destroy_ioend()
88 	mempool_free(ioend, xfs_ioend_pool);  in xfs_destroy_ioend()
94 static inline bool xfs_ioend_is_append(struct xfs_ioend *ioend)  in xfs_ioend_is_append()  argument
96 	return ioend->io_offset + ioend->io_size >  in xfs_ioend_is_append()
97 		XFS_I(ioend->io_inode)->i_d.di_size;  in xfs_ioend_is_append()
102 	struct xfs_ioend	*ioend)  in xfs_setfilesize_trans_alloc()  argument
104 	struct xfs_mount	*mp = XFS_I(ioend->io_inode)->i_mount;  in xfs_setfilesize_trans_alloc()
116 	ioend->io_append_trans = tp;  in xfs_setfilesize_trans_alloc()
122 	rwsem_release(&ioend->io_inode->i_sb->s_writers.lock_map[SB_FREEZE_FS-1],  in xfs_setfilesize_trans_alloc()
163 	struct xfs_ioend	*ioend)  in xfs_setfilesize_ioend()  argument
165 	struct xfs_inode	*ip = XFS_I(ioend->io_inode);  in xfs_setfilesize_ioend()
166 	struct xfs_trans	*tp = ioend->io_append_trans;  in xfs_setfilesize_ioend()
177 	return xfs_setfilesize(ip, tp, ioend->io_offset, ioend->io_size);  in xfs_setfilesize_ioend()
188 	struct xfs_ioend	*ioend)  in xfs_finish_ioend()  argument
190 	if (atomic_dec_and_test(&ioend->io_remaining)) {  in xfs_finish_ioend()
191 		struct xfs_mount	*mp = XFS_I(ioend->io_inode)->i_mount;  in xfs_finish_ioend()
193 		if (ioend->io_type == XFS_IO_UNWRITTEN)  in xfs_finish_ioend()
194 			queue_work(mp->m_unwritten_workqueue, &ioend->io_work);  in xfs_finish_ioend()
195 		else if (ioend->io_append_trans)  in xfs_finish_ioend()
196 			queue_work(mp->m_data_workqueue, &ioend->io_work);  in xfs_finish_ioend()
198 			xfs_destroy_ioend(ioend);  in xfs_finish_ioend()
209 	xfs_ioend_t	*ioend = container_of(work, xfs_ioend_t, io_work);  in xfs_end_io()  local
210 	struct xfs_inode *ip = XFS_I(ioend->io_inode);  in xfs_end_io()
214 		ioend->io_error = -EIO;  in xfs_end_io()
217 	if (ioend->io_error)  in xfs_end_io()
224 	if (ioend->io_type == XFS_IO_UNWRITTEN) {  in xfs_end_io()
225 		error = xfs_iomap_write_unwritten(ip, ioend->io_offset,  in xfs_end_io()
226 						  ioend->io_size);  in xfs_end_io()
227 	} else if (ioend->io_append_trans) {  in xfs_end_io()
228 		error = xfs_setfilesize_ioend(ioend);  in xfs_end_io()
230 		ASSERT(!xfs_ioend_is_append(ioend));  in xfs_end_io()
235 		ioend->io_error = error;  in xfs_end_io()
236 	xfs_destroy_ioend(ioend);  in xfs_end_io()
250 	xfs_ioend_t		*ioend;  in xfs_alloc_ioend()  local
252 	ioend = mempool_alloc(xfs_ioend_pool, GFP_NOFS);  in xfs_alloc_ioend()
259 	atomic_set(&ioend->io_remaining, 1);  in xfs_alloc_ioend()
260 	ioend->io_error = 0;  in xfs_alloc_ioend()
261 	ioend->io_list = NULL;  in xfs_alloc_ioend()
262 	ioend->io_type = type;  in xfs_alloc_ioend()
263 	ioend->io_inode = inode;  in xfs_alloc_ioend()
264 	ioend->io_buffer_head = NULL;  in xfs_alloc_ioend()
265 	ioend->io_buffer_tail = NULL;  in xfs_alloc_ioend()
266 	ioend->io_offset = 0;  in xfs_alloc_ioend()
267 	ioend->io_size = 0;  in xfs_alloc_ioend()
268 	ioend->io_append_trans = NULL;  in xfs_alloc_ioend()
270 	INIT_WORK(&ioend->io_work, xfs_end_io);  in xfs_alloc_ioend()
271 	return ioend;  in xfs_alloc_ioend()
357 	xfs_ioend_t		*ioend = bio->bi_private;  in xfs_end_bio()  local
360 	ioend->io_error = test_bit(BIO_UPTODATE, &bio->bi_flags) ? 0 : error;  in xfs_end_bio()
367 	xfs_finish_ioend(ioend);  in xfs_end_bio()
373 	xfs_ioend_t		*ioend,  in xfs_submit_ioend_bio()  argument
376 	atomic_inc(&ioend->io_remaining);  in xfs_submit_ioend_bio()
377 	bio->bi_private = ioend;  in xfs_submit_ioend_bio()
468 	xfs_ioend_t		*ioend,  in xfs_submit_ioend()  argument
471 	xfs_ioend_t		*head = ioend;  in xfs_submit_ioend()
479 		next = ioend->io_list;  in xfs_submit_ioend()
480 		for (bh = ioend->io_buffer_head; bh; bh = bh->b_private)  in xfs_submit_ioend()
482 	} while ((ioend = next) != NULL);  in xfs_submit_ioend()
485 	ioend = head;  in xfs_submit_ioend()
487 		next = ioend->io_list;  in xfs_submit_ioend()
497 			ioend->io_error = fail;  in xfs_submit_ioend()
498 			xfs_finish_ioend(ioend);  in xfs_submit_ioend()
502 		for (bh = ioend->io_buffer_head; bh; bh = bh->b_private) {  in xfs_submit_ioend()
508 				xfs_submit_ioend_bio(wbc, ioend, bio);  in xfs_submit_ioend()
513 				xfs_submit_ioend_bio(wbc, ioend, bio);  in xfs_submit_ioend()
520 			xfs_submit_ioend_bio(wbc, ioend, bio);  in xfs_submit_ioend()
521 		xfs_finish_ioend(ioend);  in xfs_submit_ioend()
522 	} while ((ioend = next) != NULL);  in xfs_submit_ioend()
532 	xfs_ioend_t		*ioend)  in xfs_cancel_ioend()  argument
538 		next = ioend->io_list;  in xfs_cancel_ioend()
539 		bh = ioend->io_buffer_head;  in xfs_cancel_ioend()
548 			if (ioend->io_type == XFS_IO_UNWRITTEN)  in xfs_cancel_ioend()
553 		mempool_free(ioend, xfs_ioend_pool);  in xfs_cancel_ioend()
554 	} while ((ioend = next) != NULL);  in xfs_cancel_ioend()
572 	xfs_ioend_t		*ioend = *result;  in xfs_add_to_ioend()  local
574 	if (!ioend || need_ioend || type != ioend->io_type) {  in xfs_add_to_ioend()
577 		ioend = xfs_alloc_ioend(inode, type);  in xfs_add_to_ioend()
578 		ioend->io_offset = offset;  in xfs_add_to_ioend()
579 		ioend->io_buffer_head = bh;  in xfs_add_to_ioend()
580 		ioend->io_buffer_tail = bh;  in xfs_add_to_ioend()
582 			previous->io_list = ioend;  in xfs_add_to_ioend()
583 		*result = ioend;  in xfs_add_to_ioend()
585 		ioend->io_buffer_tail->b_private = bh;  in xfs_add_to_ioend()
586 		ioend->io_buffer_tail = bh;  in xfs_add_to_ioend()
590 	ioend->io_size += bh->b_size;  in xfs_add_to_ioend()
945 	xfs_ioend_t		*ioend = NULL, *iohead = NULL;  in xfs_vm_writepage()  local
1125 			xfs_add_to_ioend(inode, bh, offset, type, &ioend,  in xfs_vm_writepage()
1131 			iohead = ioend;  in xfs_vm_writepage()
1141 	if (!ioend)  in xfs_vm_writepage()
1166 		xfs_cluster_write(inode, page->index + 1, &imap, &ioend,  in xfs_vm_writepage()
1175 	if (ioend->io_type != XFS_IO_UNWRITTEN && xfs_ioend_is_append(ioend))  in xfs_vm_writepage()
1176 		err = xfs_setfilesize_trans_alloc(ioend);  in xfs_vm_writepage()
1265 	struct xfs_ioend	*ioend;  in xfs_map_direct()  local
1277 		ioend = bh_result->b_private;  in xfs_map_direct()
1278 		ASSERT(ioend->io_size > 0);  in xfs_map_direct()
1279 		ASSERT(offset >= ioend->io_offset);  in xfs_map_direct()
1280 		if (offset + size > ioend->io_offset + ioend->io_size)  in xfs_map_direct()
1281 			ioend->io_size = offset - ioend->io_offset + size;  in xfs_map_direct()
1283 		if (type == XFS_IO_UNWRITTEN && type != ioend->io_type)  in xfs_map_direct()
1284 			ioend->io_type = XFS_IO_UNWRITTEN;  in xfs_map_direct()
1286 		trace_xfs_gbmap_direct_update(XFS_I(inode), ioend->io_offset,  in xfs_map_direct()
1287 					      ioend->io_size, ioend->io_type,  in xfs_map_direct()
1291 		ioend = xfs_alloc_ioend(inode, type);  in xfs_map_direct()
1292 		ioend->io_offset = offset;  in xfs_map_direct()
1293 		ioend->io_size = size;  in xfs_map_direct()
1295 		bh_result->b_private = ioend;  in xfs_map_direct()
1543 	struct xfs_ioend	*ioend = private;  in xfs_end_io_direct_write()  local
1546 				     ioend ? ioend->io_type : 0, NULL);  in xfs_end_io_direct_write()
1548 	if (!ioend) {  in xfs_end_io_direct_write()
1570 	ASSERT(offset + size <= ioend->io_offset + ioend->io_size);  in xfs_end_io_direct_write()
1571 	ioend->io_size = size;  in xfs_end_io_direct_write()
1572 	ioend->io_offset = offset;  in xfs_end_io_direct_write()
1602 	if (ioend->io_type == XFS_IO_OVERWRITE)  in xfs_end_io_direct_write()
1603 		ioend->io_error = xfs_setfilesize_trans_alloc(ioend);  in xfs_end_io_direct_write()
1606 	xfs_end_io(&ioend->io_work);  in xfs_end_io_direct_write()