105 union map_info *dm_get_rq_mapinfo(struct request *rq)  in dm_get_rq_mapinfo()  argument
107 	if (rq && rq->end_io_data)  in dm_get_rq_mapinfo()
108 		return &((struct dm_rq_target_io *)rq->end_io_data)->info;  in dm_get_rq_mapinfo()
629 static void free_clone_request(struct mapped_device *md, struct request *rq)  in free_clone_request()  argument
631 	mempool_free(rq, md->rq_pool);  in free_clone_request()
1044 static struct dm_rq_target_io *tio_from_request(struct request *rq)  in tio_from_request()  argument
1046 	return (rq->q->mq_ops ? blk_mq_rq_to_pdu(rq) : rq->special);  in tio_from_request()
1110 	struct request *rq = tio->orig;  in dm_end_request()  local
1112 	if (rq->cmd_type == REQ_TYPE_BLOCK_PC) {  in dm_end_request()
1113 		rq->errors = clone->errors;  in dm_end_request()
1114 		rq->resid_len = clone->resid_len;  in dm_end_request()
1116 		if (rq->sense)  in dm_end_request()
1122 			rq->sense_len = clone->sense_len;  in dm_end_request()
1126 	if (!rq->q->mq_ops)  in dm_end_request()
1127 		blk_end_request_all(rq, error);  in dm_end_request()
1129 		blk_mq_end_request(rq, error);  in dm_end_request()
1133 static void dm_unprep_request(struct request *rq)  in dm_unprep_request()  argument
1135 	struct dm_rq_target_io *tio = tio_from_request(rq);  in dm_unprep_request()
1138 	if (!rq->q->mq_ops) {  in dm_unprep_request()
1139 		rq->special = NULL;  in dm_unprep_request()
1140 		rq->cmd_flags &= ~REQ_DONTPREP;  in dm_unprep_request()
1152 static void old_requeue_request(struct request *rq)  in old_requeue_request()  argument
1154 	struct request_queue *q = rq->q;  in old_requeue_request()
1158 	blk_requeue_request(q, rq);  in old_requeue_request()
1164 						 struct request *rq)  in dm_requeue_unmapped_original_request()  argument
1166 	int rw = rq_data_dir(rq);  in dm_requeue_unmapped_original_request()
1168 	dm_unprep_request(rq);  in dm_requeue_unmapped_original_request()
1170 	if (!rq->q->mq_ops)  in dm_requeue_unmapped_original_request()
1171 		old_requeue_request(rq);  in dm_requeue_unmapped_original_request()
1173 		blk_mq_requeue_request(rq);  in dm_requeue_unmapped_original_request()
1174 		blk_mq_kick_requeue_list(rq->q);  in dm_requeue_unmapped_original_request()
1260 static void dm_softirq_done(struct request *rq)  in dm_softirq_done()  argument
1263 	struct dm_rq_target_io *tio = tio_from_request(rq);  in dm_softirq_done()
1268 		rw = rq_data_dir(rq);  in dm_softirq_done()
1269 		if (!rq->q->mq_ops) {  in dm_softirq_done()
1270 			blk_end_request_all(rq, tio->error);  in dm_softirq_done()
1274 			blk_mq_end_request(rq, tio->error);  in dm_softirq_done()
1280 	if (rq->cmd_flags & REQ_FAILED)  in dm_softirq_done()
1290 static void dm_complete_request(struct request *rq, int error)  in dm_complete_request()  argument
1292 	struct dm_rq_target_io *tio = tio_from_request(rq);  in dm_complete_request()
1295 	if (!rq->q->mq_ops)  in dm_complete_request()
1296 		blk_complete_request(rq);  in dm_complete_request()
1298 		blk_mq_complete_request(rq);  in dm_complete_request()
1307 static void dm_kill_unmapped_request(struct request *rq, int error)  in dm_kill_unmapped_request()  argument
1309 	rq->cmd_flags |= REQ_FAILED;  in dm_kill_unmapped_request()
1310 	dm_complete_request(rq, error);  in dm_kill_unmapped_request()
1806 static void dm_dispatch_clone_request(struct request *clone, struct request *rq)  in dm_dispatch_clone_request()  argument
1817 		dm_complete_request(rq, r);  in dm_dispatch_clone_request()
1834 static int setup_clone(struct request *clone, struct request *rq,  in setup_clone()  argument
1839 	r = blk_rq_prep_clone(clone, rq, tio->md->bs, gfp_mask,  in setup_clone()
1844 	clone->cmd = rq->cmd;  in setup_clone()
1845 	clone->cmd_len = rq->cmd_len;  in setup_clone()
1846 	clone->sense = rq->sense;  in setup_clone()
1855 static struct request *clone_rq(struct request *rq, struct mapped_device *md,  in clone_rq()  argument
1873 	if (setup_clone(clone, rq, tio, gfp_mask)) {  in clone_rq()
1885 static void init_tio(struct dm_rq_target_io *tio, struct request *rq,  in init_tio()  argument
1891 	tio->orig = rq;  in init_tio()
1898 static struct dm_rq_target_io *prep_tio(struct request *rq,  in prep_tio()  argument
1909 	init_tio(tio, rq, md);  in prep_tio()
1913 		if (!clone_rq(rq, md, tio, gfp_mask)) {  in prep_tio()
1927 static int dm_prep_fn(struct request_queue *q, struct request *rq)  in dm_prep_fn()  argument
1932 	if (unlikely(rq->special)) {  in dm_prep_fn()
1937 	tio = prep_tio(rq, md, GFP_ATOMIC);  in dm_prep_fn()
1941 	rq->special = tio;  in dm_prep_fn()
1942 	rq->cmd_flags |= REQ_DONTPREP;  in dm_prep_fn()
1953 static int map_request(struct dm_rq_target_io *tio, struct request *rq,  in map_request()  argument
1964 		r = ti->type->clone_and_map_rq(ti, rq, &tio->info, &clone);  in map_request()
1967 			dm_kill_unmapped_request(rq, r);  in map_request()
1972 		if (setup_clone(clone, rq, tio, GFP_ATOMIC)) {  in map_request()
1986 				     blk_rq_pos(rq));  in map_request()
1987 		dm_dispatch_clone_request(clone, rq);  in map_request()
2000 		dm_kill_unmapped_request(rq, r);  in map_request()
2010 	struct request *rq = tio->orig;  in map_tio_request()  local
2013 	if (map_request(tio, rq, md) == DM_MAPIO_REQUEUE)  in map_tio_request()
2014 		dm_requeue_unmapped_original_request(md, rq);  in map_tio_request()
2090 	struct request *rq;  in dm_request_fn()  local
2101 		rq = blk_peek_request(q);  in dm_request_fn()
2102 		if (!rq)  in dm_request_fn()
2107 		if (!(rq->cmd_flags & REQ_FLUSH))  in dm_request_fn()
2108 			pos = blk_rq_pos(rq);  in dm_request_fn()
2117 			dm_start_request(md, rq);  in dm_request_fn()
2118 			dm_kill_unmapped_request(rq, -EIO);  in dm_request_fn()
2123 		    md_in_flight(md) && rq->bio && rq->bio->bi_vcnt == 1 &&  in dm_request_fn()
2124 		    md->last_rq_pos == pos && md->last_rq_rw == rq_data_dir(rq))  in dm_request_fn()
2130 		dm_start_request(md, rq);  in dm_request_fn()
2132 		tio = tio_from_request(rq);  in dm_request_fn()
2689 static int dm_mq_init_request(void *data, struct request *rq,  in dm_mq_init_request()  argument
2694 	struct dm_rq_target_io *tio = blk_mq_rq_to_pdu(rq);  in dm_mq_init_request()
2708 	struct request *rq = bd->rq;  in dm_mq_queue_rq()  local
2709 	struct dm_rq_target_io *tio = blk_mq_rq_to_pdu(rq);  in dm_mq_queue_rq()
2718 	if (!(rq->cmd_flags & REQ_FLUSH))  in dm_mq_queue_rq()
2719 		pos = blk_rq_pos(rq);  in dm_mq_queue_rq()
2729 		dm_start_request(md, rq);  in dm_mq_queue_rq()
2737 	dm_start_request(md, rq);  in dm_mq_queue_rq()
2740 	init_tio(tio, rq, md);  in dm_mq_queue_rq()
2751 		tio->clone = (void *)blk_mq_rq_to_pdu(rq) + sizeof(struct dm_rq_target_io);  in dm_mq_queue_rq()
2752 		(void) clone_rq(rq, md, tio, GFP_ATOMIC);  in dm_mq_queue_rq()
2756 		if (map_request(tio, rq, md) == DM_MAPIO_REQUEUE) {  in dm_mq_queue_rq()
2758 			rq_completed(md, rq_data_dir(rq), false);  in dm_mq_queue_rq()