104 static struct bio *
105 bl_submit_bio(int rw, struct bio *bio)  in bl_submit_bio()  argument
107 	if (bio) {  in bl_submit_bio()
108 		get_parallel(bio->bi_private);  in bl_submit_bio()
110 			rw == READ ? "read" : "write", bio->bi_iter.bi_size,  in bl_submit_bio()
111 			(unsigned long long)bio->bi_iter.bi_sector);  in bl_submit_bio()
112 		submit_bio(rw, bio);  in bl_submit_bio()
117 static struct bio *
119 		void (*end_io)(struct bio *, int err), struct parallel_io *par)  in bl_alloc_init_bio()  argument
121 	struct bio *bio;  in bl_alloc_init_bio()  local
124 	bio = bio_alloc(GFP_NOIO, npg);  in bl_alloc_init_bio()
125 	if (!bio && (current->flags & PF_MEMALLOC)) {  in bl_alloc_init_bio()
126 		while (!bio && (npg /= 2))  in bl_alloc_init_bio()
127 			bio = bio_alloc(GFP_NOIO, npg);  in bl_alloc_init_bio()
130 	if (bio) {  in bl_alloc_init_bio()
131 		bio->bi_iter.bi_sector = disk_sector;  in bl_alloc_init_bio()
132 		bio->bi_bdev = bdev;  in bl_alloc_init_bio()
133 		bio->bi_end_io = end_io;  in bl_alloc_init_bio()
134 		bio->bi_private = par;  in bl_alloc_init_bio()
136 	return bio;  in bl_alloc_init_bio()
139 static struct bio *
140 do_add_page_to_bio(struct bio *bio, int npg, int rw, sector_t isect,  in do_add_page_to_bio()  argument
143 		void (*end_io)(struct bio *, int err),  in do_add_page_to_bio()  argument
162 		bio = bl_submit_bio(rw, bio);  in do_add_page_to_bio()
173 	if (!bio) {  in do_add_page_to_bio()
174 		bio = bl_alloc_init_bio(npg, map->bdev,  in do_add_page_to_bio()
176 		if (!bio)  in do_add_page_to_bio()
179 	if (bio_add_page(bio, page, *len, offset) < *len) {  in do_add_page_to_bio()
180 		bio = bl_submit_bio(rw, bio);  in do_add_page_to_bio()
183 	return bio;  in do_add_page_to_bio()
186 static void bl_end_io_read(struct bio *bio, int err)  in bl_end_io_read()  argument
188 	struct parallel_io *par = bio->bi_private;  in bl_end_io_read()
198 	bio_put(bio);  in bl_end_io_read()
227 	struct bio *bio = NULL;  in bl_read_pagelist()  local
256 			bio = bl_submit_bio(READ, bio);  in bl_read_pagelist()
281 			bio = bl_submit_bio(READ, bio);  in bl_read_pagelist()
289 			bio = do_add_page_to_bio(bio,  in bl_read_pagelist()
295 			if (IS_ERR(bio)) {  in bl_read_pagelist()
296 				header->pnfs_error = PTR_ERR(bio);  in bl_read_pagelist()
297 				bio = NULL;  in bl_read_pagelist()
313 	bl_submit_bio(READ, bio);  in bl_read_pagelist()
319 static void bl_end_io_write(struct bio *bio, int err)  in bl_end_io_write()  argument
321 	struct parallel_io *par = bio->bi_private;  in bl_end_io_write()
322 	const int uptodate = test_bit(BIO_UPTODATE, &bio->bi_flags);  in bl_end_io_write()
330 	bio_put(bio);  in bl_end_io_write()
374 	struct bio *bio = NULL;  in bl_write_pagelist()  local
406 			bio = bl_submit_bio(WRITE, bio);  in bl_write_pagelist()
417 		bio = do_add_page_to_bio(bio, header->page_array.npages - i,  in bl_write_pagelist()
421 		if (IS_ERR(bio)) {  in bl_write_pagelist()
422 			header->pnfs_error = PTR_ERR(bio);  in bl_write_pagelist()
423 			bio = NULL;  in bl_write_pagelist()
435 	bl_submit_bio(WRITE, bio);  in bl_write_pagelist()