69 	struct mapped_device *md;  member
83 	struct mapped_device *md;  member
240 bool dm_use_blk_mq(struct mapped_device *md)  in dm_use_blk_mq()  argument
242 	return md->use_blk_mq;  in dm_use_blk_mq()
437 int dm_deleting_md(struct mapped_device *md)  in dm_deleting_md()  argument
439 	return test_bit(DMF_DELETING, &md->flags);  in dm_deleting_md()
444 	struct mapped_device *md;  in dm_blk_open()  local
448 	md = bdev->bd_disk->private_data;  in dm_blk_open()
449 	if (!md)  in dm_blk_open()
452 	if (test_bit(DMF_FREEING, &md->flags) ||  in dm_blk_open()
453 	    dm_deleting_md(md)) {  in dm_blk_open()
454 		md = NULL;  in dm_blk_open()
458 	dm_get(md);  in dm_blk_open()
459 	atomic_inc(&md->open_count);  in dm_blk_open()
463 	return md ? 0 : -ENXIO;  in dm_blk_open()
468 	struct mapped_device *md;  in dm_blk_close()  local
472 	md = disk->private_data;  in dm_blk_close()
473 	if (WARN_ON(!md))  in dm_blk_close()
476 	if (atomic_dec_and_test(&md->open_count) &&  in dm_blk_close()
477 	    (test_bit(DMF_DEFERRED_REMOVE, &md->flags)))  in dm_blk_close()
480 	dm_put(md);  in dm_blk_close()
485 int dm_open_count(struct mapped_device *md)  in dm_open_count()  argument
487 	return atomic_read(&md->open_count);  in dm_open_count()
493 int dm_lock_for_deletion(struct mapped_device *md, bool mark_deferred, bool only_deferred)  in dm_lock_for_deletion()  argument
499 	if (dm_open_count(md)) {  in dm_lock_for_deletion()
502 			set_bit(DMF_DEFERRED_REMOVE, &md->flags);  in dm_lock_for_deletion()
503 	} else if (only_deferred && !test_bit(DMF_DEFERRED_REMOVE, &md->flags))  in dm_lock_for_deletion()
506 		set_bit(DMF_DELETING, &md->flags);  in dm_lock_for_deletion()
513 int dm_cancel_deferred_remove(struct mapped_device *md)  in dm_cancel_deferred_remove()  argument
519 	if (test_bit(DMF_DELETING, &md->flags))  in dm_cancel_deferred_remove()
522 		clear_bit(DMF_DEFERRED_REMOVE, &md->flags);  in dm_cancel_deferred_remove()
534 sector_t dm_get_size(struct mapped_device *md)  in dm_get_size()  argument
536 	return get_capacity(md->disk);  in dm_get_size()
539 struct request_queue *dm_get_md_queue(struct mapped_device *md)  in dm_get_md_queue()  argument
541 	return md->queue;  in dm_get_md_queue()
544 struct dm_stats *dm_get_stats(struct mapped_device *md)  in dm_get_stats()  argument
546 	return &md->stats;  in dm_get_stats()
551 	struct mapped_device *md = bdev->bd_disk->private_data;  in dm_blk_getgeo()  local
553 	return dm_get_geometry(md, geo);  in dm_blk_getgeo()
559 	struct mapped_device *md = bdev->bd_disk->private_data;  in dm_blk_ioctl()  local
566 	map = dm_get_live_table(md, &srcu_idx);  in dm_blk_ioctl()
579 	if (dm_suspended_md(md)) {  in dm_blk_ioctl()
587 	dm_put_live_table(md, srcu_idx);  in dm_blk_ioctl()
597 static struct dm_io *alloc_io(struct mapped_device *md)  in alloc_io()  argument
599 	return mempool_alloc(md->io_pool, GFP_NOIO);  in alloc_io()
602 static void free_io(struct mapped_device *md, struct dm_io *io)  in free_io()  argument
604 	mempool_free(io, md->io_pool);  in free_io()
607 static void free_tio(struct mapped_device *md, struct dm_target_io *tio)  in free_tio()  argument
612 static struct dm_rq_target_io *alloc_rq_tio(struct mapped_device *md,  in alloc_rq_tio()  argument
615 	return mempool_alloc(md->io_pool, gfp_mask);  in alloc_rq_tio()
620 	mempool_free(tio, tio->md->io_pool);  in free_rq_tio()
623 static struct request *alloc_clone_request(struct mapped_device *md,  in alloc_clone_request()  argument
626 	return mempool_alloc(md->rq_pool, gfp_mask);  in alloc_clone_request()
629 static void free_clone_request(struct mapped_device *md, struct request *rq)  in free_clone_request()  argument
631 	mempool_free(rq, md->rq_pool);  in free_clone_request()
634 static int md_in_flight(struct mapped_device *md)  in md_in_flight()  argument
636 	return atomic_read(&md->pending[READ]) +  in md_in_flight()
637 	       atomic_read(&md->pending[WRITE]);  in md_in_flight()
642 	struct mapped_device *md = io->md;  in start_io_acct()  local
650 	part_round_stats(cpu, &dm_disk(md)->part0);  in start_io_acct()
652 	atomic_set(&dm_disk(md)->part0.in_flight[rw],  in start_io_acct()
653 		atomic_inc_return(&md->pending[rw]));  in start_io_acct()
655 	if (unlikely(dm_stats_used(&md->stats)))  in start_io_acct()
656 		dm_stats_account_io(&md->stats, bio->bi_rw, bio->bi_iter.bi_sector,  in start_io_acct()
662 	struct mapped_device *md = io->md;  in end_io_acct()  local
668 	generic_end_io_acct(rw, &dm_disk(md)->part0, io->start_time);  in end_io_acct()
670 	if (unlikely(dm_stats_used(&md->stats)))  in end_io_acct()
671 		dm_stats_account_io(&md->stats, bio->bi_rw, bio->bi_iter.bi_sector,  in end_io_acct()
678 	pending = atomic_dec_return(&md->pending[rw]);  in end_io_acct()
679 	atomic_set(&dm_disk(md)->part0.in_flight[rw], pending);  in end_io_acct()
680 	pending += atomic_read(&md->pending[rw^0x1]);  in end_io_acct()
684 		wake_up(&md->wait);  in end_io_acct()
690 static void queue_io(struct mapped_device *md, struct bio *bio)  in queue_io()  argument
694 	spin_lock_irqsave(&md->deferred_lock, flags);  in queue_io()
695 	bio_list_add(&md->deferred, bio);  in queue_io()
696 	spin_unlock_irqrestore(&md->deferred_lock, flags);  in queue_io()
697 	queue_work(md->wq, &md->work);  in queue_io()
705 struct dm_table *dm_get_live_table(struct mapped_device *md, int *srcu_idx) __acquires(md->io_barri…  in dm_get_live_table()  argument
707 	*srcu_idx = srcu_read_lock(&md->io_barrier);  in dm_get_live_table()
709 	return srcu_dereference(md->map, &md->io_barrier);  in dm_get_live_table()
712 void dm_put_live_table(struct mapped_device *md, int srcu_idx) __releases(md->io_barrier)  in dm_put_live_table()  argument
714 	srcu_read_unlock(&md->io_barrier, srcu_idx);  in dm_put_live_table()
717 void dm_sync_table(struct mapped_device *md)  in dm_sync_table()  argument
719 	synchronize_srcu(&md->io_barrier);  in dm_sync_table()
727 static struct dm_table *dm_get_live_table_fast(struct mapped_device *md) __acquires(RCU)  in dm_get_live_table_fast()  argument
730 	return rcu_dereference(md->map);  in dm_get_live_table_fast()
733 static void dm_put_live_table_fast(struct mapped_device *md) __releases(RCU)  in dm_put_live_table_fast()  argument
742 			     struct mapped_device *md)  in open_table_device()  argument
755 	r = bd_link_disk_holder(bdev, dm_disk(md));  in open_table_device()
768 static void close_table_device(struct table_device *td, struct mapped_device *md)  in close_table_device()  argument
773 	bd_unlink_disk_holder(td->dm_dev.bdev, dm_disk(md));  in close_table_device()
789 int dm_get_table_device(struct mapped_device *md, dev_t dev, fmode_t mode,  in dm_get_table_device()  argument
794 	mutex_lock(&md->table_devices_lock);  in dm_get_table_device()
795 	td = find_table_device(&md->table_devices, dev, mode);  in dm_get_table_device()
799 			mutex_unlock(&md->table_devices_lock);  in dm_get_table_device()
806 		if ((r = open_table_device(td, dev, md))) {  in dm_get_table_device()
807 			mutex_unlock(&md->table_devices_lock);  in dm_get_table_device()
815 		list_add(&td->list, &md->table_devices);  in dm_get_table_device()
818 	mutex_unlock(&md->table_devices_lock);  in dm_get_table_device()
825 void dm_put_table_device(struct mapped_device *md, struct dm_dev *d)  in dm_put_table_device()  argument
829 	mutex_lock(&md->table_devices_lock);  in dm_put_table_device()
831 		close_table_device(td, md);  in dm_put_table_device()
835 	mutex_unlock(&md->table_devices_lock);  in dm_put_table_device()
855 int dm_get_geometry(struct mapped_device *md, struct hd_geometry *geo)  in dm_get_geometry()  argument
857 	*geo = md->geometry;  in dm_get_geometry()
865 int dm_set_geometry(struct mapped_device *md, struct hd_geometry *geo)  in dm_set_geometry()  argument
874 	md->geometry = *geo;  in dm_set_geometry()
888 static int __noflush_suspending(struct mapped_device *md)  in __noflush_suspending()  argument
890 	return test_bit(DMF_NOFLUSH_SUSPENDING, &md->flags);  in __noflush_suspending()
902 	struct mapped_device *md = io->md;  in dec_pending()  local
907 		if (!(io->error > 0 && __noflush_suspending(md)))  in dec_pending()
917 			spin_lock_irqsave(&md->deferred_lock, flags);  in dec_pending()
918 			if (__noflush_suspending(md))  in dec_pending()
919 				bio_list_add_head(&md->deferred, io->bio);  in dec_pending()
923 			spin_unlock_irqrestore(&md->deferred_lock, flags);  in dec_pending()
929 		free_io(md, io);  in dec_pending()
940 			queue_io(md, bio);  in dec_pending()
943 			trace_block_bio_complete(md->queue, bio, io_error);  in dec_pending()
949 static void disable_write_same(struct mapped_device *md)  in disable_write_same()  argument
951 	struct queue_limits *limits = dm_get_queue_limits(md);  in disable_write_same()
962 	struct mapped_device *md = tio->io->md;  in clone_endio()  local
987 		disable_write_same(md);  in clone_endio()
989 	free_tio(md, tio);  in clone_endio()
1054 static void rq_completed(struct mapped_device *md, int rw, bool run_queue)  in rq_completed()  argument
1056 	atomic_dec(&md->pending[rw]);  in rq_completed()
1059 	if (!md_in_flight(md))  in rq_completed()
1060 		wake_up(&md->wait);  in rq_completed()
1068 	if (!md->queue->mq_ops && run_queue)  in rq_completed()
1069 		blk_run_queue_async(md->queue);  in rq_completed()
1074 	dm_put(md);  in rq_completed()
1080 	struct mapped_device *md = tio->md;  in free_rq_clone()  local
1084 	if (md->type == DM_TYPE_MQ_REQUEST_BASED)  in free_rq_clone()
1087 	else if (!md->queue->mq_ops)  in free_rq_clone()
1089 		free_clone_request(md, clone);  in free_rq_clone()
1096 	if (!md->queue->mq_ops)  in free_rq_clone()
1109 	struct mapped_device *md = tio->md;  in dm_end_request()  local
1130 	rq_completed(md, rw, true);  in dm_end_request()
1145 	else if (!tio->md->queue->mq_ops)  in dm_unprep_request()
1163 static void dm_requeue_unmapped_original_request(struct mapped_device *md,  in dm_requeue_unmapped_original_request()  argument
1177 	rq_completed(md, rw, false);  in dm_requeue_unmapped_original_request()
1184 	dm_requeue_unmapped_original_request(tio->md, tio->orig);  in dm_requeue_unmapped_request()
1240 		disable_write_same(tio->md);  in dm_done()
1271 			rq_completed(tio->md, rw, false);  in dm_softirq_done()
1275 			rq_completed(tio->md, rw, false);  in dm_softirq_done()
1434 	struct mapped_device *md;  in __map_bio()  local
1457 		md = tio->io->md;  in __map_bio()
1459 		free_tio(md, tio);  in __map_bio()
1467 	struct mapped_device *md;  member
1508 	clone = bio_alloc_bioset(GFP_NOIO, 0, ci->md->bs);  in alloc_tio()
1674 static void __split_and_process_bio(struct mapped_device *md,  in __split_and_process_bio()  argument
1686 	ci.md = md;  in __split_and_process_bio()
1687 	ci.io = alloc_io(md);  in __split_and_process_bio()
1691 	ci.io->md = md;  in __split_and_process_bio()
1698 		ci.bio = &ci.md->flush_bio;  in __split_and_process_bio()
1720 	struct mapped_device *md = q->queuedata;  in dm_merge_bvec()  local
1721 	struct dm_table *map = dm_get_live_table_fast(md);  in dm_merge_bvec()
1760 	dm_put_live_table_fast(md);  in dm_merge_bvec()
1777 	struct mapped_device *md = q->queuedata;  in dm_make_request()  local
1781 	map = dm_get_live_table(md, &srcu_idx);  in dm_make_request()
1783 	generic_start_io_acct(rw, bio_sectors(bio), &dm_disk(md)->part0);  in dm_make_request()
1786 	if (unlikely(test_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags))) {  in dm_make_request()
1787 		dm_put_live_table(md, srcu_idx);  in dm_make_request()
1790 			queue_io(md, bio);  in dm_make_request()
1796 	__split_and_process_bio(md, map, bio);  in dm_make_request()
1797 	dm_put_live_table(md, srcu_idx);  in dm_make_request()
1801 int dm_request_based(struct mapped_device *md)  in dm_request_based()  argument
1803 	return blk_queue_stackable(md->queue);  in dm_request_based()
1839 	r = blk_rq_prep_clone(clone, rq, tio->md->bs, gfp_mask,  in setup_clone()
1855 static struct request *clone_rq(struct request *rq, struct mapped_device *md,  in clone_rq()  argument
1866 		clone = alloc_clone_request(md, gfp_mask);  in clone_rq()
1876 			free_clone_request(md, clone);  in clone_rq()
1886 		     struct mapped_device *md)  in init_tio()  argument
1888 	tio->md = md;  in init_tio()
1894 	if (md->kworker_task)  in init_tio()
1899 					struct mapped_device *md, gfp_t gfp_mask)  in prep_tio()  argument
1905 	tio = alloc_rq_tio(md, gfp_mask);  in prep_tio()
1909 	init_tio(tio, rq, md);  in prep_tio()
1911 	table = dm_get_live_table(md, &srcu_idx);  in prep_tio()
1913 		if (!clone_rq(rq, md, tio, gfp_mask)) {  in prep_tio()
1914 			dm_put_live_table(md, srcu_idx);  in prep_tio()
1919 	dm_put_live_table(md, srcu_idx);  in prep_tio()
1929 	struct mapped_device *md = q->queuedata;  in dm_prep_fn()  local
1937 	tio = prep_tio(rq, md, GFP_ATOMIC);  in dm_prep_fn()
1954 		       struct mapped_device *md)  in map_request()  argument
1985 		trace_block_rq_remap(clone->q, clone, disk_devt(dm_disk(md)),  in map_request()
2011 	struct mapped_device *md = tio->md;  in map_tio_request()  local
2013 	if (map_request(tio, rq, md) == DM_MAPIO_REQUEUE)  in map_tio_request()
2014 		dm_requeue_unmapped_original_request(md, rq);  in map_tio_request()
2017 static void dm_start_request(struct mapped_device *md, struct request *orig)  in dm_start_request()  argument
2023 	atomic_inc(&md->pending[rq_data_dir(orig)]);  in dm_start_request()
2025 	if (md->seq_rq_merge_deadline_usecs) {  in dm_start_request()
2026 		md->last_rq_pos = rq_end_sector(orig);  in dm_start_request()
2027 		md->last_rq_rw = rq_data_dir(orig);  in dm_start_request()
2028 		md->last_rq_start_time = ktime_get();  in dm_start_request()
2038 	dm_get(md);  in dm_start_request()
2043 ssize_t dm_attr_rq_based_seq_io_merge_deadline_show(struct mapped_device *md, char *buf)  in dm_attr_rq_based_seq_io_merge_deadline_show()  argument
2045 	return sprintf(buf, "%u\n", md->seq_rq_merge_deadline_usecs);  in dm_attr_rq_based_seq_io_merge_deadline_show()
2048 ssize_t dm_attr_rq_based_seq_io_merge_deadline_store(struct mapped_device *md,  in dm_attr_rq_based_seq_io_merge_deadline_store()  argument
2053 	if (!dm_request_based(md) || md->use_blk_mq)  in dm_attr_rq_based_seq_io_merge_deadline_store()
2062 	md->seq_rq_merge_deadline_usecs = deadline;  in dm_attr_rq_based_seq_io_merge_deadline_store()
2067 static bool dm_request_peeked_before_merge_deadline(struct mapped_device *md)  in dm_request_peeked_before_merge_deadline()  argument
2071 	if (!md->seq_rq_merge_deadline_usecs)  in dm_request_peeked_before_merge_deadline()
2074 	kt_deadline = ns_to_ktime((u64)md->seq_rq_merge_deadline_usecs * NSEC_PER_USEC);  in dm_request_peeked_before_merge_deadline()
2075 	kt_deadline = ktime_add_safe(md->last_rq_start_time, kt_deadline);  in dm_request_peeked_before_merge_deadline()
2086 	struct mapped_device *md = q->queuedata;  in dm_request_fn()  local
2088 	struct dm_table *map = dm_get_live_table(md, &srcu_idx);  in dm_request_fn()
2117 			dm_start_request(md, rq);  in dm_request_fn()
2122 		if (dm_request_peeked_before_merge_deadline(md) &&  in dm_request_fn()
2123 		    md_in_flight(md) && rq->bio && rq->bio->bi_vcnt == 1 &&  in dm_request_fn()
2124 		    md->last_rq_pos == pos && md->last_rq_rw == rq_data_dir(rq))  in dm_request_fn()
2130 		dm_start_request(md, rq);  in dm_request_fn()
2135 		queue_kthread_work(&md->kworker, &tio->work);  in dm_request_fn()
2144 	dm_put_live_table(md, srcu_idx);  in dm_request_fn()
2150 	struct mapped_device *md = congested_data;  in dm_any_congested()  local
2153 	if (!test_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags)) {  in dm_any_congested()
2154 		map = dm_get_live_table_fast(md);  in dm_any_congested()
2160 			if (dm_request_based(md))  in dm_any_congested()
2161 				r = md->queue->backing_dev_info.state &  in dm_any_congested()
2166 		dm_put_live_table_fast(md);  in dm_any_congested()
2225 static void dm_init_md_queue(struct mapped_device *md)  in dm_init_md_queue()  argument
2236 	queue_flag_clear_unlocked(QUEUE_FLAG_STACKABLE, md->queue);  in dm_init_md_queue()
2239 static void dm_init_old_md_queue(struct mapped_device *md)  in dm_init_old_md_queue()  argument
2241 	md->use_blk_mq = false;  in dm_init_old_md_queue()
2242 	dm_init_md_queue(md);  in dm_init_old_md_queue()
2247 	md->queue->queuedata = md;  in dm_init_old_md_queue()
2248 	md->queue->backing_dev_info.congested_fn = dm_any_congested;  in dm_init_old_md_queue()
2249 	md->queue->backing_dev_info.congested_data = md;  in dm_init_old_md_queue()
2251 	blk_queue_bounce_limit(md->queue, BLK_BOUNCE_ANY);  in dm_init_old_md_queue()
2260 	struct mapped_device *md = kzalloc(sizeof(*md), GFP_KERNEL);  in alloc_dev()  local
2263 	if (!md) {  in alloc_dev()
2279 	r = init_srcu_struct(&md->io_barrier);  in alloc_dev()
2283 	md->use_blk_mq = use_blk_mq;  in alloc_dev()
2284 	md->type = DM_TYPE_NONE;  in alloc_dev()
2285 	mutex_init(&md->suspend_lock);  in alloc_dev()
2286 	mutex_init(&md->type_lock);  in alloc_dev()
2287 	mutex_init(&md->table_devices_lock);  in alloc_dev()
2288 	spin_lock_init(&md->deferred_lock);  in alloc_dev()
2289 	atomic_set(&md->holders, 1);  in alloc_dev()
2290 	atomic_set(&md->open_count, 0);  in alloc_dev()
2291 	atomic_set(&md->event_nr, 0);  in alloc_dev()
2292 	atomic_set(&md->uevent_seq, 0);  in alloc_dev()
2293 	INIT_LIST_HEAD(&md->uevent_list);  in alloc_dev()
2294 	INIT_LIST_HEAD(&md->table_devices);  in alloc_dev()
2295 	spin_lock_init(&md->uevent_lock);  in alloc_dev()
2297 	md->queue = blk_alloc_queue(GFP_KERNEL);  in alloc_dev()
2298 	if (!md->queue)  in alloc_dev()
2301 	dm_init_md_queue(md);  in alloc_dev()
2303 	md->disk = alloc_disk(1);  in alloc_dev()
2304 	if (!md->disk)  in alloc_dev()
2307 	atomic_set(&md->pending[0], 0);  in alloc_dev()
2308 	atomic_set(&md->pending[1], 0);  in alloc_dev()
2309 	init_waitqueue_head(&md->wait);  in alloc_dev()
2310 	INIT_WORK(&md->work, dm_wq_work);  in alloc_dev()
2311 	init_waitqueue_head(&md->eventq);  in alloc_dev()
2312 	init_completion(&md->kobj_holder.completion);  in alloc_dev()
2313 	md->kworker_task = NULL;  in alloc_dev()
2315 	md->disk->major = _major;  in alloc_dev()
2316 	md->disk->first_minor = minor;  in alloc_dev()
2317 	md->disk->fops = &dm_blk_dops;  in alloc_dev()
2318 	md->disk->queue = md->queue;  in alloc_dev()
2319 	md->disk->private_data = md;  in alloc_dev()
2320 	sprintf(md->disk->disk_name, "dm-%d", minor);  in alloc_dev()
2321 	add_disk(md->disk);  in alloc_dev()
2322 	format_dev_t(md->name, MKDEV(_major, minor));  in alloc_dev()
2324 	md->wq = alloc_workqueue("kdmflush", WQ_MEM_RECLAIM, 0);  in alloc_dev()
2325 	if (!md->wq)  in alloc_dev()
2328 	md->bdev = bdget_disk(md->disk, 0);  in alloc_dev()
2329 	if (!md->bdev)  in alloc_dev()
2332 	bio_init(&md->flush_bio);  in alloc_dev()
2333 	md->flush_bio.bi_bdev = md->bdev;  in alloc_dev()
2334 	md->flush_bio.bi_rw = WRITE_FLUSH;  in alloc_dev()
2336 	dm_stats_init(&md->stats);  in alloc_dev()
2340 	old_md = idr_replace(&_minor_idr, md, minor);  in alloc_dev()
2345 	return md;  in alloc_dev()
2348 	destroy_workqueue(md->wq);  in alloc_dev()
2350 	del_gendisk(md->disk);  in alloc_dev()
2351 	put_disk(md->disk);  in alloc_dev()
2353 	blk_cleanup_queue(md->queue);  in alloc_dev()
2355 	cleanup_srcu_struct(&md->io_barrier);  in alloc_dev()
2361 	kfree(md);  in alloc_dev()
2365 static void unlock_fs(struct mapped_device *md);
2367 static void free_dev(struct mapped_device *md)  in free_dev()  argument
2369 	int minor = MINOR(disk_devt(md->disk));  in free_dev()
2371 	unlock_fs(md);  in free_dev()
2372 	destroy_workqueue(md->wq);  in free_dev()
2374 	if (md->kworker_task)  in free_dev()
2375 		kthread_stop(md->kworker_task);  in free_dev()
2376 	if (md->io_pool)  in free_dev()
2377 		mempool_destroy(md->io_pool);  in free_dev()
2378 	if (md->rq_pool)  in free_dev()
2379 		mempool_destroy(md->rq_pool);  in free_dev()
2380 	if (md->bs)  in free_dev()
2381 		bioset_free(md->bs);  in free_dev()
2383 	cleanup_srcu_struct(&md->io_barrier);  in free_dev()
2384 	free_table_devices(&md->table_devices);  in free_dev()
2385 	dm_stats_cleanup(&md->stats);  in free_dev()
2388 	md->disk->private_data = NULL;  in free_dev()
2390 	if (blk_get_integrity(md->disk))  in free_dev()
2391 		blk_integrity_unregister(md->disk);  in free_dev()
2392 	del_gendisk(md->disk);  in free_dev()
2393 	put_disk(md->disk);  in free_dev()
2394 	blk_cleanup_queue(md->queue);  in free_dev()
2395 	if (md->use_blk_mq)  in free_dev()
2396 		blk_mq_free_tag_set(&md->tag_set);  in free_dev()
2397 	bdput(md->bdev);  in free_dev()
2401 	kfree(md);  in free_dev()
2404 static void __bind_mempools(struct mapped_device *md, struct dm_table *t)  in __bind_mempools()  argument
2408 	if (md->bs) {  in __bind_mempools()
2415 			bioset_free(md->bs);  in __bind_mempools()
2416 			md->bs = p->bs;  in __bind_mempools()
2430 	BUG_ON(!p || md->io_pool || md->rq_pool || md->bs);  in __bind_mempools()
2432 	md->io_pool = p->io_pool;  in __bind_mempools()
2434 	md->rq_pool = p->rq_pool;  in __bind_mempools()
2436 	md->bs = p->bs;  in __bind_mempools()
2451 	struct mapped_device *md = (struct mapped_device *) context;  in event_callback()  local
2453 	spin_lock_irqsave(&md->uevent_lock, flags);  in event_callback()
2454 	list_splice_init(&md->uevent_list, &uevents);  in event_callback()
2455 	spin_unlock_irqrestore(&md->uevent_lock, flags);  in event_callback()
2457 	dm_send_uevents(&uevents, &disk_to_dev(md->disk)->kobj);  in event_callback()
2459 	atomic_inc(&md->event_nr);  in event_callback()
2460 	wake_up(&md->eventq);  in event_callback()
2466 static void __set_size(struct mapped_device *md, sector_t size)  in __set_size()  argument
2468 	set_capacity(md->disk, size);  in __set_size()
2470 	i_size_write(md->bdev->bd_inode, (loff_t)size << SECTOR_SHIFT);  in __set_size()
2529 static struct dm_table *__bind(struct mapped_device *md, struct dm_table *t,  in __bind()  argument
2533 	struct request_queue *q = md->queue;  in __bind()
2542 	if (size != dm_get_size(md))  in __bind()
2543 		memset(&md->geometry, 0, sizeof(md->geometry));  in __bind()
2545 	__set_size(md, size);  in __bind()
2547 	dm_table_event_callback(t, event_callback, md);  in __bind()
2559 	__bind_mempools(md, t);  in __bind()
2563 	old_map = rcu_dereference_protected(md->map, lockdep_is_held(&md->suspend_lock));  in __bind()
2564 	rcu_assign_pointer(md->map, t);  in __bind()
2565 	md->immutable_target_type = dm_table_get_immutable_target_type(t);  in __bind()
2569 		set_bit(DMF_MERGE_IS_OPTIONAL, &md->flags);  in __bind()
2571 		clear_bit(DMF_MERGE_IS_OPTIONAL, &md->flags);  in __bind()
2573 		dm_sync_table(md);  in __bind()
2581 static struct dm_table *__unbind(struct mapped_device *md)  in __unbind()  argument
2583 	struct dm_table *map = rcu_dereference_protected(md->map, 1);  in __unbind()
2589 	RCU_INIT_POINTER(md->map, NULL);  in __unbind()
2590 	dm_sync_table(md);  in __unbind()
2600 	struct mapped_device *md;  in dm_create()  local
2602 	md = alloc_dev(minor);  in dm_create()
2603 	if (!md)  in dm_create()
2606 	dm_sysfs_init(md);  in dm_create()
2608 	*result = md;  in dm_create()
2616 void dm_lock_md_type(struct mapped_device *md)  in dm_lock_md_type()  argument
2618 	mutex_lock(&md->type_lock);  in dm_lock_md_type()
2621 void dm_unlock_md_type(struct mapped_device *md)  in dm_unlock_md_type()  argument
2623 	mutex_unlock(&md->type_lock);  in dm_unlock_md_type()
2626 void dm_set_md_type(struct mapped_device *md, unsigned type)  in dm_set_md_type()  argument
2628 	BUG_ON(!mutex_is_locked(&md->type_lock));  in dm_set_md_type()
2629 	md->type = type;  in dm_set_md_type()
2632 unsigned dm_get_md_type(struct mapped_device *md)  in dm_get_md_type()  argument
2634 	BUG_ON(!mutex_is_locked(&md->type_lock));  in dm_get_md_type()
2635 	return md->type;  in dm_get_md_type()
2638 struct target_type *dm_get_immutable_target_type(struct mapped_device *md)  in dm_get_immutable_target_type()  argument
2640 	return md->immutable_target_type;  in dm_get_immutable_target_type()
2647 struct queue_limits *dm_get_queue_limits(struct mapped_device *md)  in dm_get_queue_limits()  argument
2649 	BUG_ON(!atomic_read(&md->holders));  in dm_get_queue_limits()
2650 	return &md->queue->limits;  in dm_get_queue_limits()
2654 static void init_rq_based_worker_thread(struct mapped_device *md)  in init_rq_based_worker_thread()  argument
2657 	init_kthread_worker(&md->kworker);  in init_rq_based_worker_thread()
2658 	md->kworker_task = kthread_run(kthread_worker_fn, &md->kworker,  in init_rq_based_worker_thread()
2659 				       "kdmwork-%s", dm_device_name(md));  in init_rq_based_worker_thread()
2665 static int dm_init_request_based_queue(struct mapped_device *md)  in dm_init_request_based_queue()  argument
2670 	q = blk_init_allocated_queue(md->queue, dm_request_fn, NULL);  in dm_init_request_based_queue()
2675 	md->seq_rq_merge_deadline_usecs = 0;  in dm_init_request_based_queue()
2677 	md->queue = q;  in dm_init_request_based_queue()
2678 	dm_init_old_md_queue(md);  in dm_init_request_based_queue()
2679 	blk_queue_softirq_done(md->queue, dm_softirq_done);  in dm_init_request_based_queue()
2680 	blk_queue_prep_rq(md->queue, dm_prep_fn);  in dm_init_request_based_queue()
2682 	init_rq_based_worker_thread(md);  in dm_init_request_based_queue()
2684 	elv_register_queue(md->queue);  in dm_init_request_based_queue()
2693 	struct mapped_device *md = data;  in dm_mq_init_request()  local
2700 	tio->md = md;  in dm_mq_init_request()
2710 	struct mapped_device *md = tio->md;  in dm_mq_queue_rq()  local
2712 	struct dm_table *map = dm_get_live_table(md, &srcu_idx);  in dm_mq_queue_rq()
2723 		dm_put_live_table(md, srcu_idx);  in dm_mq_queue_rq()
2729 		dm_start_request(md, rq);  in dm_mq_queue_rq()
2732 	dm_put_live_table(md, srcu_idx);  in dm_mq_queue_rq()
2737 	dm_start_request(md, rq);  in dm_mq_queue_rq()
2740 	init_tio(tio, rq, md);  in dm_mq_queue_rq()
2752 		(void) clone_rq(rq, md, tio, GFP_ATOMIC);  in dm_mq_queue_rq()
2753 		queue_kthread_work(&md->kworker, &tio->work);  in dm_mq_queue_rq()
2756 		if (map_request(tio, rq, md) == DM_MAPIO_REQUEUE) {  in dm_mq_queue_rq()
2758 			rq_completed(md, rq_data_dir(rq), false);  in dm_mq_queue_rq()
2773 static int dm_init_request_based_blk_mq_queue(struct mapped_device *md)  in dm_init_request_based_blk_mq_queue()  argument
2775 	unsigned md_type = dm_get_md_type(md);  in dm_init_request_based_blk_mq_queue()
2779 	memset(&md->tag_set, 0, sizeof(md->tag_set));  in dm_init_request_based_blk_mq_queue()
2780 	md->tag_set.ops = &dm_mq_ops;  in dm_init_request_based_blk_mq_queue()
2781 	md->tag_set.queue_depth = BLKDEV_MAX_RQ;  in dm_init_request_based_blk_mq_queue()
2782 	md->tag_set.numa_node = NUMA_NO_NODE;  in dm_init_request_based_blk_mq_queue()
2783 	md->tag_set.flags = BLK_MQ_F_SHOULD_MERGE | BLK_MQ_F_SG_MERGE;  in dm_init_request_based_blk_mq_queue()
2784 	md->tag_set.nr_hw_queues = 1;  in dm_init_request_based_blk_mq_queue()
2787 		md->tag_set.cmd_size = sizeof(struct dm_rq_target_io) + sizeof(struct request);  in dm_init_request_based_blk_mq_queue()
2789 		md->tag_set.cmd_size = sizeof(struct dm_rq_target_io);  in dm_init_request_based_blk_mq_queue()
2790 	md->tag_set.driver_data = md;  in dm_init_request_based_blk_mq_queue()
2792 	err = blk_mq_alloc_tag_set(&md->tag_set);  in dm_init_request_based_blk_mq_queue()
2796 	q = blk_mq_init_allocated_queue(&md->tag_set, md->queue);  in dm_init_request_based_blk_mq_queue()
2801 	md->queue = q;  in dm_init_request_based_blk_mq_queue()
2802 	dm_init_md_queue(md);  in dm_init_request_based_blk_mq_queue()
2805 	blk_mq_register_disk(md->disk);  in dm_init_request_based_blk_mq_queue()
2808 		init_rq_based_worker_thread(md);  in dm_init_request_based_blk_mq_queue()
2813 	blk_mq_free_tag_set(&md->tag_set);  in dm_init_request_based_blk_mq_queue()
2817 static unsigned filter_md_type(unsigned type, struct mapped_device *md)  in filter_md_type()  argument
2822 	return !md->use_blk_mq ? DM_TYPE_REQUEST_BASED : DM_TYPE_MQ_REQUEST_BASED;  in filter_md_type()
2828 int dm_setup_md_queue(struct mapped_device *md)  in dm_setup_md_queue()  argument
2831 	unsigned md_type = filter_md_type(dm_get_md_type(md), md);  in dm_setup_md_queue()
2835 		r = dm_init_request_based_queue(md);  in dm_setup_md_queue()
2842 		r = dm_init_request_based_blk_mq_queue(md);  in dm_setup_md_queue()
2849 		dm_init_old_md_queue(md);  in dm_setup_md_queue()
2850 		blk_queue_make_request(md->queue, dm_make_request);  in dm_setup_md_queue()
2851 		blk_queue_merge_bvec(md->queue, dm_merge_bvec);  in dm_setup_md_queue()
2860 	struct mapped_device *md;  in dm_get_md()  local
2868 	md = idr_find(&_minor_idr, minor);  in dm_get_md()
2869 	if (md) {  in dm_get_md()
2870 		if ((md == MINOR_ALLOCED ||  in dm_get_md()
2871 		     (MINOR(disk_devt(dm_disk(md))) != minor) ||  in dm_get_md()
2872 		     dm_deleting_md(md) ||  in dm_get_md()
2873 		     test_bit(DMF_FREEING, &md->flags))) {  in dm_get_md()
2874 			md = NULL;  in dm_get_md()
2877 		dm_get(md);  in dm_get_md()
2883 	return md;  in dm_get_md()
2887 void *dm_get_mdptr(struct mapped_device *md)  in dm_get_mdptr()  argument
2889 	return md->interface_ptr;  in dm_get_mdptr()
2892 void dm_set_mdptr(struct mapped_device *md, void *ptr)  in dm_set_mdptr()  argument
2894 	md->interface_ptr = ptr;  in dm_set_mdptr()
2897 void dm_get(struct mapped_device *md)  in dm_get()  argument
2899 	atomic_inc(&md->holders);  in dm_get()
2900 	BUG_ON(test_bit(DMF_FREEING, &md->flags));  in dm_get()
2903 int dm_hold(struct mapped_device *md)  in dm_hold()  argument
2906 	if (test_bit(DMF_FREEING, &md->flags)) {  in dm_hold()
2910 	dm_get(md);  in dm_hold()
2916 const char *dm_device_name(struct mapped_device *md)  in dm_device_name()  argument
2918 	return md->name;  in dm_device_name()
2922 static void __dm_destroy(struct mapped_device *md, bool wait)  in __dm_destroy()  argument
2930 	idr_replace(&_minor_idr, MINOR_ALLOCED, MINOR(disk_devt(dm_disk(md))));  in __dm_destroy()
2931 	set_bit(DMF_FREEING, &md->flags);  in __dm_destroy()
2934 	if (dm_request_based(md) && md->kworker_task)  in __dm_destroy()
2935 		flush_kthread_worker(&md->kworker);  in __dm_destroy()
2941 	mutex_lock(&md->suspend_lock);  in __dm_destroy()
2942 	map = dm_get_live_table(md, &srcu_idx);  in __dm_destroy()
2943 	if (!dm_suspended_md(md)) {  in __dm_destroy()
2948 	dm_put_live_table(md, srcu_idx);  in __dm_destroy()
2949 	mutex_unlock(&md->suspend_lock);  in __dm_destroy()
2958 		while (atomic_read(&md->holders))  in __dm_destroy()
2960 	else if (atomic_read(&md->holders))  in __dm_destroy()
2962 		       dm_device_name(md), atomic_read(&md->holders));  in __dm_destroy()
2964 	dm_sysfs_exit(md);  in __dm_destroy()
2965 	dm_table_destroy(__unbind(md));  in __dm_destroy()
2966 	free_dev(md);  in __dm_destroy()
2969 void dm_destroy(struct mapped_device *md)  in dm_destroy()  argument
2971 	__dm_destroy(md, true);  in dm_destroy()
2974 void dm_destroy_immediate(struct mapped_device *md)  in dm_destroy_immediate()  argument
2976 	__dm_destroy(md, false);  in dm_destroy_immediate()
2979 void dm_put(struct mapped_device *md)  in dm_put()  argument
2981 	atomic_dec(&md->holders);  in dm_put()
2985 static int dm_wait_for_completion(struct mapped_device *md, int interruptible)  in dm_wait_for_completion()  argument
2990 	add_wait_queue(&md->wait, &wait);  in dm_wait_for_completion()
2995 		if (!md_in_flight(md))  in dm_wait_for_completion()
3008 	remove_wait_queue(&md->wait, &wait);  in dm_wait_for_completion()
3018 	struct mapped_device *md = container_of(work, struct mapped_device,  in dm_wq_work()  local
3024 	map = dm_get_live_table(md, &srcu_idx);  in dm_wq_work()
3026 	while (!test_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags)) {  in dm_wq_work()
3027 		spin_lock_irq(&md->deferred_lock);  in dm_wq_work()
3028 		c = bio_list_pop(&md->deferred);  in dm_wq_work()
3029 		spin_unlock_irq(&md->deferred_lock);  in dm_wq_work()
3034 		if (dm_request_based(md))  in dm_wq_work()
3037 			__split_and_process_bio(md, map, c);  in dm_wq_work()
3040 	dm_put_live_table(md, srcu_idx);  in dm_wq_work()
3043 static void dm_queue_flush(struct mapped_device *md)  in dm_queue_flush()  argument
3045 	clear_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags);  in dm_queue_flush()
3047 	queue_work(md->wq, &md->work);  in dm_queue_flush()
3053 struct dm_table *dm_swap_table(struct mapped_device *md, struct dm_table *table)  in dm_swap_table()  argument
3059 	mutex_lock(&md->suspend_lock);  in dm_swap_table()
3062 	if (!dm_suspended_md(md))  in dm_swap_table()
3072 		live_map = dm_get_live_table_fast(md);  in dm_swap_table()
3074 			limits = md->queue->limits;  in dm_swap_table()
3075 		dm_put_live_table_fast(md);  in dm_swap_table()
3086 	map = __bind(md, table, &limits);  in dm_swap_table()
3089 	mutex_unlock(&md->suspend_lock);  in dm_swap_table()
3097 static int lock_fs(struct mapped_device *md)  in lock_fs()  argument
3101 	WARN_ON(md->frozen_sb);  in lock_fs()
3103 	md->frozen_sb = freeze_bdev(md->bdev);  in lock_fs()
3104 	if (IS_ERR(md->frozen_sb)) {  in lock_fs()
3105 		r = PTR_ERR(md->frozen_sb);  in lock_fs()
3106 		md->frozen_sb = NULL;  in lock_fs()
3110 	set_bit(DMF_FROZEN, &md->flags);  in lock_fs()
3115 static void unlock_fs(struct mapped_device *md)  in unlock_fs()  argument
3117 	if (!test_bit(DMF_FROZEN, &md->flags))  in unlock_fs()
3120 	thaw_bdev(md->bdev, md->frozen_sb);  in unlock_fs()
3121 	md->frozen_sb = NULL;  in unlock_fs()
3122 	clear_bit(DMF_FROZEN, &md->flags);  in unlock_fs()
3132 static int __dm_suspend(struct mapped_device *md, struct dm_table *map,  in __dm_suspend()  argument
3144 		set_bit(DMF_NOFLUSH_SUSPENDING, &md->flags);  in __dm_suspend()
3159 		r = lock_fs(md);  in __dm_suspend()
3178 	set_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags);  in __dm_suspend()
3180 		synchronize_srcu(&md->io_barrier);  in __dm_suspend()
3186 	if (dm_request_based(md)) {  in __dm_suspend()
3187 		stop_queue(md->queue);  in __dm_suspend()
3188 		if (md->kworker_task)  in __dm_suspend()
3189 			flush_kthread_worker(&md->kworker);  in __dm_suspend()
3192 	flush_workqueue(md->wq);  in __dm_suspend()
3199 	r = dm_wait_for_completion(md, interruptible);  in __dm_suspend()
3202 		clear_bit(DMF_NOFLUSH_SUSPENDING, &md->flags);  in __dm_suspend()
3204 		synchronize_srcu(&md->io_barrier);  in __dm_suspend()
3208 		dm_queue_flush(md);  in __dm_suspend()
3210 		if (dm_request_based(md))  in __dm_suspend()
3211 			start_queue(md->queue);  in __dm_suspend()
3213 		unlock_fs(md);  in __dm_suspend()
3237 int dm_suspend(struct mapped_device *md, unsigned suspend_flags)  in dm_suspend()  argument
3243 	mutex_lock_nested(&md->suspend_lock, SINGLE_DEPTH_NESTING);  in dm_suspend()
3245 	if (dm_suspended_md(md)) {  in dm_suspend()
3250 	if (dm_suspended_internally_md(md)) {  in dm_suspend()
3252 		mutex_unlock(&md->suspend_lock);  in dm_suspend()
3253 		r = wait_on_bit(&md->flags, DMF_SUSPENDED_INTERNALLY, TASK_INTERRUPTIBLE);  in dm_suspend()
3259 	map = rcu_dereference_protected(md->map, lockdep_is_held(&md->suspend_lock));  in dm_suspend()
3261 	r = __dm_suspend(md, map, suspend_flags, TASK_INTERRUPTIBLE);  in dm_suspend()
3265 	set_bit(DMF_SUSPENDED, &md->flags);  in dm_suspend()
3270 	mutex_unlock(&md->suspend_lock);  in dm_suspend()
3274 static int __dm_resume(struct mapped_device *md, struct dm_table *map)  in __dm_resume()  argument
3282 	dm_queue_flush(md);  in __dm_resume()
3289 	if (dm_request_based(md))  in __dm_resume()
3290 		start_queue(md->queue);  in __dm_resume()
3292 	unlock_fs(md);  in __dm_resume()
3297 int dm_resume(struct mapped_device *md)  in dm_resume()  argument
3303 	mutex_lock_nested(&md->suspend_lock, SINGLE_DEPTH_NESTING);  in dm_resume()
3305 	if (!dm_suspended_md(md))  in dm_resume()
3308 	if (dm_suspended_internally_md(md)) {  in dm_resume()
3310 		mutex_unlock(&md->suspend_lock);  in dm_resume()
3311 		r = wait_on_bit(&md->flags, DMF_SUSPENDED_INTERNALLY, TASK_INTERRUPTIBLE);  in dm_resume()
3317 	map = rcu_dereference_protected(md->map, lockdep_is_held(&md->suspend_lock));  in dm_resume()
3321 	r = __dm_resume(md, map);  in dm_resume()
3325 	clear_bit(DMF_SUSPENDED, &md->flags);  in dm_resume()
3329 	mutex_unlock(&md->suspend_lock);  in dm_resume()
3340 static void __dm_internal_suspend(struct mapped_device *md, unsigned suspend_flags)  in __dm_internal_suspend()  argument
3344 	if (md->internal_suspend_count++)  in __dm_internal_suspend()
3347 	if (dm_suspended_md(md)) {  in __dm_internal_suspend()
3348 		set_bit(DMF_SUSPENDED_INTERNALLY, &md->flags);  in __dm_internal_suspend()
3352 	map = rcu_dereference_protected(md->map, lockdep_is_held(&md->suspend_lock));  in __dm_internal_suspend()
3360 	(void) __dm_suspend(md, map, suspend_flags, TASK_UNINTERRUPTIBLE);  in __dm_internal_suspend()
3362 	set_bit(DMF_SUSPENDED_INTERNALLY, &md->flags);  in __dm_internal_suspend()
3367 static void __dm_internal_resume(struct mapped_device *md)  in __dm_internal_resume()  argument
3369 	BUG_ON(!md->internal_suspend_count);  in __dm_internal_resume()
3371 	if (--md->internal_suspend_count)  in __dm_internal_resume()
3374 	if (dm_suspended_md(md))  in __dm_internal_resume()
3381 	(void) __dm_resume(md, NULL);  in __dm_internal_resume()
3384 	clear_bit(DMF_SUSPENDED_INTERNALLY, &md->flags);  in __dm_internal_resume()
3386 	wake_up_bit(&md->flags, DMF_SUSPENDED_INTERNALLY);  in __dm_internal_resume()
3389 void dm_internal_suspend_noflush(struct mapped_device *md)  in dm_internal_suspend_noflush()  argument
3391 	mutex_lock(&md->suspend_lock);  in dm_internal_suspend_noflush()
3392 	__dm_internal_suspend(md, DM_SUSPEND_NOFLUSH_FLAG);  in dm_internal_suspend_noflush()
3393 	mutex_unlock(&md->suspend_lock);  in dm_internal_suspend_noflush()
3397 void dm_internal_resume(struct mapped_device *md)  in dm_internal_resume()  argument
3399 	mutex_lock(&md->suspend_lock);  in dm_internal_resume()
3400 	__dm_internal_resume(md);  in dm_internal_resume()
3401 	mutex_unlock(&md->suspend_lock);  in dm_internal_resume()
3410 void dm_internal_suspend_fast(struct mapped_device *md)  in dm_internal_suspend_fast()  argument
3412 	mutex_lock(&md->suspend_lock);  in dm_internal_suspend_fast()
3413 	if (dm_suspended_md(md) || dm_suspended_internally_md(md))  in dm_internal_suspend_fast()
3416 	set_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags);  in dm_internal_suspend_fast()
3417 	synchronize_srcu(&md->io_barrier);  in dm_internal_suspend_fast()
3418 	flush_workqueue(md->wq);  in dm_internal_suspend_fast()
3419 	dm_wait_for_completion(md, TASK_UNINTERRUPTIBLE);  in dm_internal_suspend_fast()
3423 void dm_internal_resume_fast(struct mapped_device *md)  in dm_internal_resume_fast()  argument
3425 	if (dm_suspended_md(md) || dm_suspended_internally_md(md))  in dm_internal_resume_fast()
3428 	dm_queue_flush(md);  in dm_internal_resume_fast()
3431 	mutex_unlock(&md->suspend_lock);  in dm_internal_resume_fast()
3438 int dm_kobject_uevent(struct mapped_device *md, enum kobject_action action,  in dm_kobject_uevent()  argument
3445 		return kobject_uevent(&disk_to_dev(md->disk)->kobj, action);  in dm_kobject_uevent()
3449 		return kobject_uevent_env(&disk_to_dev(md->disk)->kobj,  in dm_kobject_uevent()
3454 uint32_t dm_next_uevent_seq(struct mapped_device *md)  in dm_next_uevent_seq()  argument
3456 	return atomic_add_return(1, &md->uevent_seq);  in dm_next_uevent_seq()
3459 uint32_t dm_get_event_nr(struct mapped_device *md)  in dm_get_event_nr()  argument
3461 	return atomic_read(&md->event_nr);  in dm_get_event_nr()
3464 int dm_wait_event(struct mapped_device *md, int event_nr)  in dm_wait_event()  argument
3466 	return wait_event_interruptible(md->eventq,  in dm_wait_event()
3467 			(event_nr != atomic_read(&md->event_nr)));  in dm_wait_event()
3470 void dm_uevent_add(struct mapped_device *md, struct list_head *elist)  in dm_uevent_add()  argument
3474 	spin_lock_irqsave(&md->uevent_lock, flags);  in dm_uevent_add()
3475 	list_add(elist, &md->uevent_list);  in dm_uevent_add()
3476 	spin_unlock_irqrestore(&md->uevent_lock, flags);  in dm_uevent_add()
3483 struct gendisk *dm_disk(struct mapped_device *md)  in dm_disk()  argument
3485 	return md->disk;  in dm_disk()
3489 struct kobject *dm_kobject(struct mapped_device *md)  in dm_kobject()  argument
3491 	return &md->kobj_holder.kobj;  in dm_kobject()
3496 	struct mapped_device *md;  in dm_get_from_kobject()  local
3498 	md = container_of(kobj, struct mapped_device, kobj_holder.kobj);  in dm_get_from_kobject()
3500 	if (test_bit(DMF_FREEING, &md->flags) ||  in dm_get_from_kobject()
3501 	    dm_deleting_md(md))  in dm_get_from_kobject()
3504 	dm_get(md);  in dm_get_from_kobject()
3505 	return md;  in dm_get_from_kobject()
3508 int dm_suspended_md(struct mapped_device *md)  in dm_suspended_md()  argument
3510 	return test_bit(DMF_SUSPENDED, &md->flags);  in dm_suspended_md()
3513 int dm_suspended_internally_md(struct mapped_device *md)  in dm_suspended_internally_md()  argument
3515 	return test_bit(DMF_SUSPENDED_INTERNALLY, &md->flags);  in dm_suspended_internally_md()
3518 int dm_test_deferred_remove_flag(struct mapped_device *md)  in dm_test_deferred_remove_flag()  argument
3520 	return test_bit(DMF_DEFERRED_REMOVE, &md->flags);  in dm_test_deferred_remove_flag()
3535 struct dm_md_mempools *dm_alloc_md_mempools(struct mapped_device *md, unsigned type,  in dm_alloc_md_mempools()  argument
3546 	type = filter_md_type(type, md);  in dm_alloc_md_mempools()