51 #define bio_prio(bio)	((bio)->bi_rw >> BIO_PRIO_SHIFT)  argument
52 #define bio_prio_valid(bio)	ioprio_valid(bio_prio(bio))  argument
54 #define bio_set_prio(bio, prio)		do {			\  argument
56 	(bio)->bi_rw &= ((1UL << BIO_PRIO_SHIFT) - 1);		\
57 	(bio)->bi_rw |= ((unsigned long) (prio) << BIO_PRIO_SHIFT);	\
83 #define bio_iter_iovec(bio, iter)				\  argument
84 	bvec_iter_bvec((bio)->bi_io_vec, (iter))
86 #define bio_iter_page(bio, iter)				\  argument
87 	bvec_iter_page((bio)->bi_io_vec, (iter))
88 #define bio_iter_len(bio, iter)					\  argument
89 	bvec_iter_len((bio)->bi_io_vec, (iter))
90 #define bio_iter_offset(bio, iter)				\  argument
91 	bvec_iter_offset((bio)->bi_io_vec, (iter))
93 #define bio_page(bio)		bio_iter_page((bio), (bio)->bi_iter)  argument
94 #define bio_offset(bio)		bio_iter_offset((bio), (bio)->bi_iter)  argument
95 #define bio_iovec(bio)		bio_iter_iovec((bio), (bio)->bi_iter)  argument
97 #define bio_multiple_segments(bio)				\  argument
98 	((bio)->bi_iter.bi_size != bio_iovec(bio).bv_len)
99 #define bio_sectors(bio)	((bio)->bi_iter.bi_size >> 9)  argument
100 #define bio_end_sector(bio)	((bio)->bi_iter.bi_sector + bio_sectors((bio)))  argument
105 static inline bool bio_has_data(struct bio *bio)  in bio_has_data()  argument
107 	if (bio &&  in bio_has_data()
108 	    bio->bi_iter.bi_size &&  in bio_has_data()
109 	    !(bio->bi_rw & REQ_DISCARD))  in bio_has_data()
115 static inline bool bio_is_rw(struct bio *bio)  in bio_is_rw()  argument
117 	if (!bio_has_data(bio))  in bio_is_rw()
120 	if (bio->bi_rw & BIO_NO_ADVANCE_ITER_MASK)  in bio_is_rw()
126 static inline bool bio_mergeable(struct bio *bio)  in bio_mergeable()  argument
128 	if (bio->bi_rw & REQ_NOMERGE_FLAGS)  in bio_mergeable()
134 static inline unsigned int bio_cur_bytes(struct bio *bio)  in bio_cur_bytes()  argument
136 	if (bio_has_data(bio))  in bio_cur_bytes()
137 		return bio_iovec(bio).bv_len;  in bio_cur_bytes()
139 		return bio->bi_iter.bi_size;  in bio_cur_bytes()
142 static inline void *bio_data(struct bio *bio)  in bio_data()  argument
144 	if (bio_has_data(bio))  in bio_data()
145 		return page_address(bio_page(bio)) + bio_offset(bio);  in bio_data()
153 #define bio_to_phys(bio)	(page_to_phys(bio_page((bio))) + (unsigned long) bio_offset((bio)))  argument
162 #define __bio_kmap_atomic(bio, iter)				\  argument
163 	(kmap_atomic(bio_iter_iovec((bio), (iter)).bv_page) +	\
164 		bio_iter_iovec((bio), (iter)).bv_offset)
193 #define bio_for_each_segment_all(bvl, bio, i)				\  argument
194 	for (i = 0, bvl = (bio)->bi_io_vec; i < (bio)->bi_vcnt; i++, bvl++)
223 static inline void bio_advance_iter(struct bio *bio, struct bvec_iter *iter,  in bio_advance_iter()  argument
228 	if (bio->bi_rw & BIO_NO_ADVANCE_ITER_MASK)  in bio_advance_iter()
231 		bvec_iter_advance(bio->bi_io_vec, iter, bytes);  in bio_advance_iter()
234 #define __bio_for_each_segment(bvl, bio, iter, start)			\  argument
237 		((bvl = bio_iter_iovec((bio), (iter))), 1);		\
238 	     bio_advance_iter((bio), &(iter), (bvl).bv_len))
240 #define bio_for_each_segment(bvl, bio, iter)				\  argument
241 	__bio_for_each_segment(bvl, bio, iter, (bio)->bi_iter)
245 static inline unsigned bio_segments(struct bio *bio)  in bio_segments()  argument
256 	if (bio->bi_rw & REQ_DISCARD)  in bio_segments()
259 	if (bio->bi_rw & REQ_WRITE_SAME)  in bio_segments()
262 	bio_for_each_segment(bv, bio, iter)  in bio_segments()
282 static inline void bio_get(struct bio *bio)  in bio_get()  argument
284 	bio->bi_flags |= (1 << BIO_REFFED);  in bio_get()
286 	atomic_inc(&bio->__bi_cnt);  in bio_get()
289 static inline void bio_cnt_set(struct bio *bio, unsigned int count)  in bio_cnt_set()  argument
292 		bio->bi_flags |= (1 << BIO_REFFED);  in bio_cnt_set()
295 	atomic_set(&bio->__bi_cnt, count);  in bio_cnt_set()
298 static inline bool bio_flagged(struct bio *bio, unsigned int bit)  in bio_flagged()  argument
300 	return (bio->bi_flags & (1U << bit)) != 0;  in bio_flagged()
303 static inline void bio_set_flag(struct bio *bio, unsigned int bit)  in bio_set_flag()  argument
305 	bio->bi_flags |= (1U << bit);  in bio_set_flag()
308 static inline void bio_clear_flag(struct bio *bio, unsigned int bit)  in bio_clear_flag()  argument
310 	bio->bi_flags &= ~(1U << bit);  in bio_clear_flag()
313 static inline void bio_get_first_bvec(struct bio *bio, struct bio_vec *bv)  in bio_get_first_bvec()  argument
315 	*bv = bio_iovec(bio);  in bio_get_first_bvec()
318 static inline void bio_get_last_bvec(struct bio *bio, struct bio_vec *bv)  in bio_get_last_bvec()  argument
320 	struct bvec_iter iter = bio->bi_iter;  in bio_get_last_bvec()
323 	if (unlikely(!bio_multiple_segments(bio))) {  in bio_get_last_bvec()
324 		*bv = bio_iovec(bio);  in bio_get_last_bvec()
328 	bio_advance_iter(bio, &iter, iter.bi_size);  in bio_get_last_bvec()
335 	*bv = bio->bi_io_vec[idx];  in bio_get_last_bvec()
355 static inline struct bio_integrity_payload *bio_integrity(struct bio *bio)  in bio_integrity()  argument
357 	if (bio->bi_rw & REQ_INTEGRITY)  in bio_integrity()
358 		return bio->bi_integrity;  in bio_integrity()
367 	struct bio		*bip_bio;	/* parent bio */
384 static inline bool bio_integrity_flagged(struct bio *bio, enum bip_flags flag)  in bio_integrity_flagged()  argument
386 	struct bio_integrity_payload *bip = bio_integrity(bio);  in bio_integrity_flagged()
407 extern void bio_trim(struct bio *bio, int offset, int size);
408 extern struct bio *bio_split(struct bio *bio, int sectors,
421 static inline struct bio *bio_next_split(struct bio *bio, int sectors,  in bio_next_split()  argument
424 	if (sectors >= bio_sectors(bio))  in bio_next_split()
425 		return bio;  in bio_next_split()
427 	return bio_split(bio, sectors, gfp, bs);  in bio_next_split()
435 extern struct bio *bio_alloc_bioset(gfp_t, int, struct bio_set *);
436 extern void bio_put(struct bio *);
438 extern void __bio_clone_fast(struct bio *, struct bio *);
439 extern struct bio *bio_clone_fast(struct bio *, gfp_t, struct bio_set *);
440 extern struct bio *bio_clone_bioset(struct bio *, gfp_t, struct bio_set *bs);
444 static inline struct bio *bio_alloc(gfp_t gfp_mask, unsigned int nr_iovecs)  in bio_alloc()
449 static inline struct bio *bio_clone(struct bio *bio, gfp_t gfp_mask)  in bio_clone()  argument
451 	return bio_clone_bioset(bio, gfp_mask, fs_bio_set);  in bio_clone()
454 static inline struct bio *bio_kmalloc(gfp_t gfp_mask, unsigned int nr_iovecs)  in bio_kmalloc()
459 static inline struct bio *bio_clone_kmalloc(struct bio *bio, gfp_t gfp_mask)  in bio_clone_kmalloc()  argument
461 	return bio_clone_bioset(bio, gfp_mask, NULL);  in bio_clone_kmalloc()
465 extern void bio_endio(struct bio *);
467 static inline void bio_io_error(struct bio *bio)  in bio_io_error()  argument
469 	bio->bi_error = -EIO;  in bio_io_error()
470 	bio_endio(bio);  in bio_io_error()
474 extern int bio_phys_segments(struct request_queue *, struct bio *);
476 extern int submit_bio_wait(int rw, struct bio *bio);
477 extern void bio_advance(struct bio *, unsigned);
479 extern void bio_init(struct bio *);
480 extern void bio_reset(struct bio *);
481 void bio_chain(struct bio *, struct bio *);
483 extern int bio_add_page(struct bio *, struct page *, unsigned int,unsigned int);
484 extern int bio_add_pc_page(struct request_queue *, struct bio *, struct page *,
487 extern struct bio *bio_map_user_iov(struct request_queue *,
489 extern void bio_unmap_user(struct bio *);
490 extern struct bio *bio_map_kern(struct request_queue *, void *, unsigned int,
492 extern struct bio *bio_copy_kern(struct request_queue *, void *, unsigned int,
494 extern void bio_set_pages_dirty(struct bio *bio);
495 extern void bio_check_pages_dirty(struct bio *bio);
506 extern void bio_flush_dcache_pages(struct bio *bi);
508 static inline void bio_flush_dcache_pages(struct bio *bi)  in bio_flush_dcache_pages()
513 extern void bio_copy_data(struct bio *dst, struct bio *src);
514 extern int bio_alloc_pages(struct bio *bio, gfp_t gfp);
516 extern struct bio *bio_copy_user_iov(struct request_queue *,
520 extern int bio_uncopy_user(struct bio *);
521 void zero_fill_bio(struct bio *bio);
527 int bio_associate_blkcg(struct bio *bio, struct cgroup_subsys_state *blkcg_css);
528 int bio_associate_current(struct bio *bio);
529 void bio_disassociate_task(struct bio *bio);
531 static inline int bio_associate_blkcg(struct bio *bio,  in bio_associate_blkcg()  argument
533 static inline int bio_associate_current(struct bio *bio) { return -ENOENT; }  in bio_associate_current()  argument
534 static inline void bio_disassociate_task(struct bio *bio) { }  in bio_disassociate_task()  argument
578 static inline char *__bio_kmap_irq(struct bio *bio, struct bvec_iter iter,  in __bio_kmap_irq()  argument
581 	return bvec_kmap_irq(&bio_iter_iovec(bio, iter), flags);  in __bio_kmap_irq()
585 #define bio_kmap_irq(bio, flags) \  argument
586 	__bio_kmap_irq((bio), (bio)->bi_iter, (flags))
597 	struct bio *head;
598 	struct bio *tail;
613 #define bio_list_for_each(bio, bl) \  argument
614 	for (bio = (bl)->head; bio; bio = bio->bi_next)
619 	struct bio *bio;  in bio_list_size()  local
621 	bio_list_for_each(bio, bl)  in bio_list_size()
627 static inline void bio_list_add(struct bio_list *bl, struct bio *bio)  in bio_list_add()  argument
629 	bio->bi_next = NULL;  in bio_list_add()
632 		bl->tail->bi_next = bio;  in bio_list_add()
634 		bl->head = bio;  in bio_list_add()
636 	bl->tail = bio;  in bio_list_add()
639 static inline void bio_list_add_head(struct bio_list *bl, struct bio *bio)  in bio_list_add_head()  argument
641 	bio->bi_next = bl->head;  in bio_list_add_head()
643 	bl->head = bio;  in bio_list_add_head()
646 		bl->tail = bio;  in bio_list_add_head()
676 static inline struct bio *bio_list_peek(struct bio_list *bl)  in bio_list_peek()
681 static inline struct bio *bio_list_pop(struct bio_list *bl)  in bio_list_pop()
683 	struct bio *bio = bl->head;  in bio_list_pop()  local
685 	if (bio) {  in bio_list_pop()
690 		bio->bi_next = NULL;  in bio_list_pop()
693 	return bio;  in bio_list_pop()
696 static inline struct bio *bio_list_get(struct bio_list *bl)  in bio_list_get()
698 	struct bio *bio = bl->head;  in bio_list_get()  local
702 	return bio;  in bio_list_get()
757 extern struct bio_integrity_payload *bio_integrity_alloc(struct bio *, gfp_t, unsigned int);
758 extern void bio_integrity_free(struct bio *);
759 extern int bio_integrity_add_page(struct bio *, struct page *, unsigned int, unsigned int);
760 extern bool bio_integrity_enabled(struct bio *bio);
761 extern int bio_integrity_prep(struct bio *);
762 extern void bio_integrity_endio(struct bio *);
763 extern void bio_integrity_advance(struct bio *, unsigned int);
764 extern void bio_integrity_trim(struct bio *, unsigned int, unsigned int);
765 extern int bio_integrity_clone(struct bio *, struct bio *, gfp_t);
772 static inline void *bio_integrity(struct bio *bio)  in bio_integrity()  argument
777 static inline bool bio_integrity_enabled(struct bio *bio)  in bio_integrity_enabled()  argument
792 static inline int bio_integrity_prep(struct bio *bio)  in bio_integrity_prep()  argument
797 static inline void bio_integrity_free(struct bio *bio)  in bio_integrity_free()  argument
802 static inline int bio_integrity_clone(struct bio *bio, struct bio *bio_src,  in bio_integrity_clone()  argument
808 static inline void bio_integrity_advance(struct bio *bio,  in bio_integrity_advance()  argument
814 static inline void bio_integrity_trim(struct bio *bio, unsigned int offset,  in bio_integrity_trim()  argument
825 static inline bool bio_integrity_flagged(struct bio *bio, enum bip_flags flag)  in bio_integrity_flagged()  argument