127 static void bio_put_slab(struct bio_set *bs)  in bio_put_slab()  argument
135 		if (bs->bio_slab == bio_slabs[i].slab) {  in bio_put_slab()
247 	struct bio_set *bs = bio->bi_pool;  in bio_free()  local
252 	if (bs) {  in bio_free()
254 			bvec_free(bs->bvec_pool, bio->bi_io_vec, BIO_POOL_IDX(bio));  in bio_free()
260 		p -= bs->front_pad;  in bio_free()
262 		mempool_free(p, bs->bio_pool);  in bio_free()
342 	struct bio_set *bs = container_of(work, struct bio_set, rescue_work);  in bio_alloc_rescue()  local
346 		spin_lock(&bs->rescue_lock);  in bio_alloc_rescue()
347 		bio = bio_list_pop(&bs->rescue_list);  in bio_alloc_rescue()
348 		spin_unlock(&bs->rescue_lock);  in bio_alloc_rescue()
357 static void punt_bios_to_rescuer(struct bio_set *bs)  in punt_bios_to_rescuer()  argument
377 		bio_list_add(bio->bi_pool == bs ? &punt : &nopunt, bio);  in punt_bios_to_rescuer()
381 	spin_lock(&bs->rescue_lock);  in punt_bios_to_rescuer()
382 	bio_list_merge(&bs->rescue_list, &punt);  in punt_bios_to_rescuer()
383 	spin_unlock(&bs->rescue_lock);  in punt_bios_to_rescuer()
385 	queue_work(bs->rescue_workqueue, &bs->rescue_work);  in punt_bios_to_rescuer()
423 struct bio *bio_alloc_bioset(gfp_t gfp_mask, int nr_iovecs, struct bio_set *bs)  in bio_alloc_bioset()  argument
433 	if (!bs) {  in bio_alloc_bioset()
444 		if (WARN_ON_ONCE(!bs->bvec_pool && nr_iovecs > 0))  in bio_alloc_bioset()
470 		p = mempool_alloc(bs->bio_pool, gfp_mask);  in bio_alloc_bioset()
472 			punt_bios_to_rescuer(bs);  in bio_alloc_bioset()
474 			p = mempool_alloc(bs->bio_pool, gfp_mask);  in bio_alloc_bioset()
477 		front_pad = bs->front_pad;  in bio_alloc_bioset()
488 		bvl = bvec_alloc(gfp_mask, nr_iovecs, &idx, bs->bvec_pool);  in bio_alloc_bioset()
490 			punt_bios_to_rescuer(bs);  in bio_alloc_bioset()
492 			bvl = bvec_alloc(gfp_mask, nr_iovecs, &idx, bs->bvec_pool);  in bio_alloc_bioset()
503 	bio->bi_pool = bs;  in bio_alloc_bioset()
510 	mempool_free(p, bs->bio_pool);  in bio_alloc_bioset()
598 struct bio *bio_clone_fast(struct bio *bio, gfp_t gfp_mask, struct bio_set *bs)  in bio_clone_fast()  argument
602 	b = bio_alloc_bioset(gfp_mask, 0, bs);  in bio_clone_fast()
633 			     struct bio_set *bs)  in bio_clone_bioset()  argument
661 	bio = bio_alloc_bioset(gfp_mask, bio_segments(bio_src), bs);  in bio_clone_bioset()
1786 		      gfp_t gfp, struct bio_set *bs)  in bio_split()  argument
1798 		split = bio_clone_bioset(bio, gfp, bs);  in bio_split()
1800 		split = bio_clone_fast(bio, gfp, bs);  in bio_split()
1851 void bioset_free(struct bio_set *bs)  in bioset_free()  argument
1853 	if (bs->rescue_workqueue)  in bioset_free()
1854 		destroy_workqueue(bs->rescue_workqueue);  in bioset_free()
1856 	if (bs->bio_pool)  in bioset_free()
1857 		mempool_destroy(bs->bio_pool);  in bioset_free()
1859 	if (bs->bvec_pool)  in bioset_free()
1860 		mempool_destroy(bs->bvec_pool);  in bioset_free()
1862 	bioset_integrity_free(bs);  in bioset_free()
1863 	bio_put_slab(bs);  in bioset_free()
1865 	kfree(bs);  in bioset_free()
1874 	struct bio_set *bs;  in __bioset_create()  local
1876 	bs = kzalloc(sizeof(*bs), GFP_KERNEL);  in __bioset_create()
1877 	if (!bs)  in __bioset_create()
1880 	bs->front_pad = front_pad;  in __bioset_create()
1882 	spin_lock_init(&bs->rescue_lock);  in __bioset_create()
1883 	bio_list_init(&bs->rescue_list);  in __bioset_create()
1884 	INIT_WORK(&bs->rescue_work, bio_alloc_rescue);  in __bioset_create()
1886 	bs->bio_slab = bio_find_or_create_slab(front_pad + back_pad);  in __bioset_create()
1887 	if (!bs->bio_slab) {  in __bioset_create()
1888 		kfree(bs);  in __bioset_create()
1892 	bs->bio_pool = mempool_create_slab_pool(pool_size, bs->bio_slab);  in __bioset_create()
1893 	if (!bs->bio_pool)  in __bioset_create()
1897 		bs->bvec_pool = biovec_create_pool(pool_size);  in __bioset_create()
1898 		if (!bs->bvec_pool)  in __bioset_create()
1902 	bs->rescue_workqueue = alloc_workqueue("bioset", WQ_MEM_RECLAIM, 0);  in __bioset_create()
1903 	if (!bs->rescue_workqueue)  in __bioset_create()
1906 	return bs;  in __bioset_create()
1908 	bioset_free(bs);  in __bioset_create()