127 static void bio_put_slab(struct bio_set *bs)  in bio_put_slab()  argument
135 		if (bs->bio_slab == bio_slabs[i].slab) {  in bio_put_slab()
247 	struct bio_set *bs = bio->bi_pool;  in bio_free()  local
252 	if (bs) {  in bio_free()
254 			bvec_free(bs->bvec_pool, bio->bi_io_vec, BIO_POOL_IDX(bio));  in bio_free()
260 		p -= bs->front_pad;  in bio_free()
262 		mempool_free(p, bs->bio_pool);  in bio_free()
329 	struct bio_set *bs = container_of(work, struct bio_set, rescue_work);  in bio_alloc_rescue()  local
333 		spin_lock(&bs->rescue_lock);  in bio_alloc_rescue()
334 		bio = bio_list_pop(&bs->rescue_list);  in bio_alloc_rescue()
335 		spin_unlock(&bs->rescue_lock);  in bio_alloc_rescue()
344 static void punt_bios_to_rescuer(struct bio_set *bs)  in punt_bios_to_rescuer()  argument
364 		bio_list_add(bio->bi_pool == bs ? &punt : &nopunt, bio);  in punt_bios_to_rescuer()
368 	spin_lock(&bs->rescue_lock);  in punt_bios_to_rescuer()
369 	bio_list_merge(&bs->rescue_list, &punt);  in punt_bios_to_rescuer()
370 	spin_unlock(&bs->rescue_lock);  in punt_bios_to_rescuer()
372 	queue_work(bs->rescue_workqueue, &bs->rescue_work);  in punt_bios_to_rescuer()
410 struct bio *bio_alloc_bioset(gfp_t gfp_mask, int nr_iovecs, struct bio_set *bs)  in bio_alloc_bioset()  argument
420 	if (!bs) {  in bio_alloc_bioset()
431 		if (WARN_ON_ONCE(!bs->bvec_pool && nr_iovecs > 0))  in bio_alloc_bioset()
457 		p = mempool_alloc(bs->bio_pool, gfp_mask);  in bio_alloc_bioset()
459 			punt_bios_to_rescuer(bs);  in bio_alloc_bioset()
461 			p = mempool_alloc(bs->bio_pool, gfp_mask);  in bio_alloc_bioset()
464 		front_pad = bs->front_pad;  in bio_alloc_bioset()
475 		bvl = bvec_alloc(gfp_mask, nr_iovecs, &idx, bs->bvec_pool);  in bio_alloc_bioset()
477 			punt_bios_to_rescuer(bs);  in bio_alloc_bioset()
479 			bvl = bvec_alloc(gfp_mask, nr_iovecs, &idx, bs->bvec_pool);  in bio_alloc_bioset()
490 	bio->bi_pool = bs;  in bio_alloc_bioset()
497 	mempool_free(p, bs->bio_pool);  in bio_alloc_bioset()
581 struct bio *bio_clone_fast(struct bio *bio, gfp_t gfp_mask, struct bio_set *bs)  in bio_clone_fast()  argument
585 	b = bio_alloc_bioset(gfp_mask, 0, bs);  in bio_clone_fast()
616 			     struct bio_set *bs)  in bio_clone_bioset()  argument
644 	bio = bio_alloc_bioset(gfp_mask, bio_segments(bio_src), bs);  in bio_clone_bioset()
1825 		      gfp_t gfp, struct bio_set *bs)  in bio_split()  argument
1837 		split = bio_clone_bioset(bio, gfp, bs);  in bio_split()
1839 		split = bio_clone_fast(bio, gfp, bs);  in bio_split()
1890 void bioset_free(struct bio_set *bs)  in bioset_free()  argument
1892 	if (bs->rescue_workqueue)  in bioset_free()
1893 		destroy_workqueue(bs->rescue_workqueue);  in bioset_free()
1895 	if (bs->bio_pool)  in bioset_free()
1896 		mempool_destroy(bs->bio_pool);  in bioset_free()
1898 	if (bs->bvec_pool)  in bioset_free()
1899 		mempool_destroy(bs->bvec_pool);  in bioset_free()
1901 	bioset_integrity_free(bs);  in bioset_free()
1902 	bio_put_slab(bs);  in bioset_free()
1904 	kfree(bs);  in bioset_free()
1913 	struct bio_set *bs;  in __bioset_create()  local
1915 	bs = kzalloc(sizeof(*bs), GFP_KERNEL);  in __bioset_create()
1916 	if (!bs)  in __bioset_create()
1919 	bs->front_pad = front_pad;  in __bioset_create()
1921 	spin_lock_init(&bs->rescue_lock);  in __bioset_create()
1922 	bio_list_init(&bs->rescue_list);  in __bioset_create()
1923 	INIT_WORK(&bs->rescue_work, bio_alloc_rescue);  in __bioset_create()
1925 	bs->bio_slab = bio_find_or_create_slab(front_pad + back_pad);  in __bioset_create()
1926 	if (!bs->bio_slab) {  in __bioset_create()
1927 		kfree(bs);  in __bioset_create()
1931 	bs->bio_pool = mempool_create_slab_pool(pool_size, bs->bio_slab);  in __bioset_create()
1932 	if (!bs->bio_pool)  in __bioset_create()
1936 		bs->bvec_pool = biovec_create_pool(pool_size);  in __bioset_create()
1937 		if (!bs->bvec_pool)  in __bioset_create()
1941 	bs->rescue_workqueue = alloc_workqueue("bioset", WQ_MEM_RECLAIM, 0);  in __bioset_create()
1942 	if (!bs->rescue_workqueue)  in __bioset_create()
1945 	return bs;  in __bioset_create()
1947 	bioset_free(bs);  in __bioset_create()