220 	struct request *req;  in scsi_execute()  local
224 	req = blk_get_request(sdev->request_queue, write, __GFP_WAIT);  in scsi_execute()
225 	if (IS_ERR(req))  in scsi_execute()
227 	blk_rq_set_block_pc(req);  in scsi_execute()
229 	if (bufflen &&	blk_rq_map_kern(sdev->request_queue, req,  in scsi_execute()
233 	req->cmd_len = COMMAND_SIZE(cmd[0]);  in scsi_execute()
234 	memcpy(req->cmd, cmd, req->cmd_len);  in scsi_execute()
235 	req->sense = sense;  in scsi_execute()
236 	req->sense_len = 0;  in scsi_execute()
237 	req->retries = retries;  in scsi_execute()
238 	req->timeout = timeout;  in scsi_execute()
239 	req->cmd_flags |= flags | REQ_QUIET | REQ_PREEMPT;  in scsi_execute()
244 	blk_execute_rq(req->q, NULL, req, 1);  in scsi_execute()
252 	if (unlikely(req->resid_len > 0 && req->resid_len <= bufflen))  in scsi_execute()
253 		memset(buffer + (bufflen - req->resid_len), 0, req->resid_len);  in scsi_execute()
256 		*resid = req->resid_len;  in scsi_execute()
257 	ret = req->errors;  in scsi_execute()
259 	blk_put_request(req);  in scsi_execute()
531 	struct request *req = cmd->request;  in scsi_requeue_command()  local
535 	blk_unprep_request(req);  in scsi_requeue_command()
536 	req->special = NULL;  in scsi_requeue_command()
538 	blk_requeue_request(q, req);  in scsi_requeue_command()
687 static bool scsi_end_request(struct request *req, int error,  in scsi_end_request()  argument
690 	struct scsi_cmnd *cmd = req->special;  in scsi_end_request()
694 	if (blk_update_request(req, error, bytes))  in scsi_end_request()
699 	    blk_update_request(req->next_rq, error, bidi_bytes))  in scsi_end_request()
703 		add_disk_randomness(req->rq_disk);  in scsi_end_request()
705 	if (req->mq_ctx) {  in scsi_end_request()
715 		__blk_mq_end_request(req, error);  in scsi_end_request()
729 		blk_finish_request(req, error);  in scsi_end_request()
821 	struct request *req = cmd->request;  in scsi_io_completion()  local
828 	unsigned long wait_for = (cmd->allowed + 1) * req->timeout;  in scsi_io_completion()
836 	if (req->cmd_type == REQ_TYPE_BLOCK_PC) { /* SG_IO ioctl from block level */  in scsi_io_completion()
838 			if (sense_valid && req->sense) {  in scsi_io_completion()
846 				memcpy(req->sense, cmd->sense_buffer,  len);  in scsi_io_completion()
847 				req->sense_len = len;  in scsi_io_completion()
855 		req->errors = cmd->result;  in scsi_io_completion()
857 		req->resid_len = scsi_get_resid(cmd);  in scsi_io_completion()
864 			req->next_rq->resid_len = scsi_in(cmd)->resid;  in scsi_io_completion()
865 			if (scsi_end_request(req, 0, blk_rq_bytes(req),  in scsi_io_completion()
866 					blk_rq_bytes(req->next_rq)))  in scsi_io_completion()
870 	} else if (blk_rq_bytes(req) == 0 && result && !sense_deferred) {  in scsi_io_completion()
881 	BUG_ON(blk_bidi_rq(req));  in scsi_io_completion()
889 		blk_rq_sectors(req), good_bytes));  in scsi_io_completion()
904 		else if (!(req->cmd_flags & REQ_QUIET))  in scsi_io_completion()
916 	if (!(blk_rq_bytes(req) == 0 && error) &&  in scsi_io_completion()
917 	    !scsi_end_request(req, error, good_bytes, 0))  in scsi_io_completion()
924 		if (scsi_end_request(req, error, blk_rq_bytes(req), 0))  in scsi_io_completion()
1034 		if (!(req->cmd_flags & REQ_QUIET)) {  in scsi_io_completion()
1054 		if (!scsi_end_request(req, error, blk_rq_err_bytes(req), 0))  in scsi_io_completion()
1082 static int scsi_init_sgtable(struct request *req, struct scsi_data_buffer *sdb)  in scsi_init_sgtable()  argument
1089 	if (unlikely(scsi_alloc_sgtable(sdb, req->nr_phys_segments,  in scsi_init_sgtable()
1090 					req->mq_ctx != NULL)))  in scsi_init_sgtable()
1097 	count = blk_rq_map_sg(req->q, req, sdb->table.sgl);  in scsi_init_sgtable()
1100 	sdb->length = blk_rq_bytes(req);  in scsi_init_sgtable()
1191 		struct request *req)  in scsi_get_cmd_from_req()  argument
1195 	if (!req->special) {  in scsi_get_cmd_from_req()
1205 		req->special = cmd;  in scsi_get_cmd_from_req()
1207 		cmd = req->special;  in scsi_get_cmd_from_req()
1211 	cmd->tag = req->tag;  in scsi_get_cmd_from_req()
1212 	cmd->request = req;  in scsi_get_cmd_from_req()
1214 	cmd->cmnd = req->cmd;  in scsi_get_cmd_from_req()
1220 static int scsi_setup_blk_pc_cmnd(struct scsi_device *sdev, struct request *req)  in scsi_setup_blk_pc_cmnd()  argument
1222 	struct scsi_cmnd *cmd = req->special;  in scsi_setup_blk_pc_cmnd()
1230 	if (req->bio) {  in scsi_setup_blk_pc_cmnd()
1235 		BUG_ON(blk_rq_bytes(req));  in scsi_setup_blk_pc_cmnd()
1240 	cmd->cmd_len = req->cmd_len;  in scsi_setup_blk_pc_cmnd()
1241 	cmd->transfersize = blk_rq_bytes(req);  in scsi_setup_blk_pc_cmnd()
1242 	cmd->allowed = req->retries;  in scsi_setup_blk_pc_cmnd()
1250 static int scsi_setup_fs_cmnd(struct scsi_device *sdev, struct request *req)  in scsi_setup_fs_cmnd()  argument
1252 	struct scsi_cmnd *cmd = req->special;  in scsi_setup_fs_cmnd()
1256 		int ret = sdev->scsi_dh_data->scsi_dh->prep_fn(sdev, req);  in scsi_setup_fs_cmnd()
1265 static int scsi_setup_cmnd(struct scsi_device *sdev, struct request *req)  in scsi_setup_cmnd()  argument
1267 	struct scsi_cmnd *cmd = req->special;  in scsi_setup_cmnd()
1269 	if (!blk_rq_bytes(req))  in scsi_setup_cmnd()
1271 	else if (rq_data_dir(req) == WRITE)  in scsi_setup_cmnd()
1276 	switch (req->cmd_type) {  in scsi_setup_cmnd()
1278 		return scsi_setup_fs_cmnd(sdev, req);  in scsi_setup_cmnd()
1280 		return scsi_setup_blk_pc_cmnd(sdev, req);  in scsi_setup_cmnd()
1287 scsi_prep_state_check(struct scsi_device *sdev, struct request *req)  in scsi_prep_state_check()  argument
1325 			if (!(req->cmd_flags & REQ_PREEMPT))  in scsi_prep_state_check()
1334 			if (!(req->cmd_flags & REQ_PREEMPT))  in scsi_prep_state_check()
1343 scsi_prep_return(struct request_queue *q, struct request *req, int ret)  in scsi_prep_return()  argument
1349 		req->errors = DID_NO_CONNECT << 16;  in scsi_prep_return()
1351 		if (req->special) {  in scsi_prep_return()
1352 			struct scsi_cmnd *cmd = req->special;  in scsi_prep_return()
1356 			req->special = NULL;  in scsi_prep_return()
1369 		req->cmd_flags |= REQ_DONTPREP;  in scsi_prep_return()
1375 static int scsi_prep_fn(struct request_queue *q, struct request *req)  in scsi_prep_fn()  argument
1381 	ret = scsi_prep_state_check(sdev, req);  in scsi_prep_fn()
1385 	cmd = scsi_get_cmd_from_req(sdev, req);  in scsi_prep_fn()
1391 	ret = scsi_setup_cmnd(sdev, req);  in scsi_prep_fn()
1393 	return scsi_prep_return(q, req, ret);  in scsi_prep_fn()
1396 static void scsi_unprep_fn(struct request_queue *q, struct request *req)  in scsi_unprep_fn()  argument
1398 	scsi_uninit_cmd(req->special);  in scsi_unprep_fn()
1587 static void scsi_kill_request(struct request *req, struct request_queue *q)  in scsi_kill_request()  argument
1589 	struct scsi_cmnd *cmd = req->special;  in scsi_kill_request()
1594 	blk_start_request(req);  in scsi_kill_request()
1615 	blk_complete_request(req);  in scsi_kill_request()
1773 	struct request *req;  in scsi_request_fn()  local
1787 		req = blk_peek_request(q);  in scsi_request_fn()
1788 		if (!req)  in scsi_request_fn()
1794 			scsi_kill_request(req, q);  in scsi_request_fn()
1804 		if (!(blk_queue_tagged(q) && !blk_queue_start_tag(q, req)))  in scsi_request_fn()
1805 			blk_start_request(req);  in scsi_request_fn()
1808 		cmd = req->special;  in scsi_request_fn()
1814 			blk_dump_rq_flags(req, "foo");  in scsi_request_fn()
1826 		if (blk_queue_tagged(q) && !(req->cmd_flags & REQ_QUEUED)) {  in scsi_request_fn()
1880 	blk_requeue_request(q, req);  in scsi_request_fn()
1899 static int scsi_mq_prep_fn(struct request *req)  in scsi_mq_prep_fn()  argument
1901 	struct scsi_cmnd *cmd = blk_mq_rq_to_pdu(req);  in scsi_mq_prep_fn()
1902 	struct scsi_device *sdev = req->q->queuedata;  in scsi_mq_prep_fn()
1909 	req->special = cmd;  in scsi_mq_prep_fn()
1911 	cmd->request = req;  in scsi_mq_prep_fn()
1915 	cmd->tag = req->tag;  in scsi_mq_prep_fn()
1917 	cmd->cmnd = req->cmd;  in scsi_mq_prep_fn()
1944 	if (blk_bidi_rq(req)) {  in scsi_mq_prep_fn()
1945 		struct request *next_rq = req->next_rq;  in scsi_mq_prep_fn()
1955 	blk_mq_start_request(req);  in scsi_mq_prep_fn()
1957 	return scsi_setup_cmnd(sdev, req);  in scsi_mq_prep_fn()
1969 	struct request *req = bd->rq;  in scsi_queue_rq()  local
1970 	struct request_queue *q = req->q;  in scsi_queue_rq()
1973 	struct scsi_cmnd *cmd = blk_mq_rq_to_pdu(req);  in scsi_queue_rq()
1977 	ret = prep_to_mq(scsi_prep_state_check(sdev, req));  in scsi_queue_rq()
1993 	if (!(req->cmd_flags & REQ_DONTPREP)) {  in scsi_queue_rq()
1994 		ret = prep_to_mq(scsi_mq_prep_fn(req));  in scsi_queue_rq()
1997 		req->cmd_flags |= REQ_DONTPREP;  in scsi_queue_rq()
1999 		blk_mq_start_request(req);  in scsi_queue_rq()
2042 		if (req->cmd_flags & REQ_DONTPREP)  in scsi_queue_rq()
2051 static enum blk_eh_timer_return scsi_timeout(struct request *req,  in scsi_timeout()  argument
2056 	return scsi_times_out(req);  in scsi_timeout()