618 	struct srpt_ioctx *ioctx;  in srpt_alloc_ioctx()  local
620 	ioctx = kmalloc(ioctx_size, GFP_KERNEL);  in srpt_alloc_ioctx()
621 	if (!ioctx)  in srpt_alloc_ioctx()
624 	ioctx->buf = kmalloc(dma_size, GFP_KERNEL);  in srpt_alloc_ioctx()
625 	if (!ioctx->buf)  in srpt_alloc_ioctx()
628 	ioctx->dma = ib_dma_map_single(sdev->device, ioctx->buf, dma_size, dir);  in srpt_alloc_ioctx()
629 	if (ib_dma_mapping_error(sdev->device, ioctx->dma))  in srpt_alloc_ioctx()
632 	return ioctx;  in srpt_alloc_ioctx()
635 	kfree(ioctx->buf);  in srpt_alloc_ioctx()
637 	kfree(ioctx);  in srpt_alloc_ioctx()
645 static void srpt_free_ioctx(struct srpt_device *sdev, struct srpt_ioctx *ioctx,  in srpt_free_ioctx()  argument
648 	if (!ioctx)  in srpt_free_ioctx()
651 	ib_dma_unmap_single(sdev->device, ioctx->dma, dma_size, dir);  in srpt_free_ioctx()
652 	kfree(ioctx->buf);  in srpt_free_ioctx()
653 	kfree(ioctx);  in srpt_free_ioctx()
711 static enum srpt_command_state srpt_get_cmd_state(struct srpt_send_ioctx *ioctx)  in srpt_get_cmd_state()  argument
716 	BUG_ON(!ioctx);  in srpt_get_cmd_state()
718 	spin_lock_irqsave(&ioctx->spinlock, flags);  in srpt_get_cmd_state()
719 	state = ioctx->state;  in srpt_get_cmd_state()
720 	spin_unlock_irqrestore(&ioctx->spinlock, flags);  in srpt_get_cmd_state()
730 static enum srpt_command_state srpt_set_cmd_state(struct srpt_send_ioctx *ioctx,  in srpt_set_cmd_state()  argument
736 	BUG_ON(!ioctx);  in srpt_set_cmd_state()
738 	spin_lock_irqsave(&ioctx->spinlock, flags);  in srpt_set_cmd_state()
739 	previous = ioctx->state;  in srpt_set_cmd_state()
741 		ioctx->state = new;  in srpt_set_cmd_state()
742 	spin_unlock_irqrestore(&ioctx->spinlock, flags);  in srpt_set_cmd_state()
752 static bool srpt_test_and_set_cmd_state(struct srpt_send_ioctx *ioctx,  in srpt_test_and_set_cmd_state()  argument
759 	WARN_ON(!ioctx);  in srpt_test_and_set_cmd_state()
763 	spin_lock_irqsave(&ioctx->spinlock, flags);  in srpt_test_and_set_cmd_state()
764 	previous = ioctx->state;  in srpt_test_and_set_cmd_state()
766 		ioctx->state = new;  in srpt_test_and_set_cmd_state()
767 	spin_unlock_irqrestore(&ioctx->spinlock, flags);  in srpt_test_and_set_cmd_state()
775 			  struct srpt_recv_ioctx *ioctx)  in srpt_post_recv()  argument
781 	wr.wr_id = encode_wr_id(SRPT_RECV, ioctx->ioctx.index);  in srpt_post_recv()
783 	list.addr = ioctx->ioctx.dma;  in srpt_post_recv()
800 			  struct srpt_send_ioctx *ioctx, int len)  in srpt_post_send()  argument
815 	ib_dma_sync_single_for_device(sdev->device, ioctx->ioctx.dma, len,  in srpt_post_send()
818 	list.addr = ioctx->ioctx.dma;  in srpt_post_send()
823 	wr.wr_id = encode_wr_id(SRPT_SEND, ioctx->ioctx.index);  in srpt_post_send()
853 static int srpt_get_desc_tbl(struct srpt_send_ioctx *ioctx,  in srpt_get_desc_tbl()  argument
897 		ioctx->n_rbuf = 1;  in srpt_get_desc_tbl()
898 		ioctx->rbufs = &ioctx->single_rbuf;  in srpt_get_desc_tbl()
902 		memcpy(ioctx->rbufs, db, sizeof *db);  in srpt_get_desc_tbl()
909 		ioctx->n_rbuf = be32_to_cpu(idb->table_desc.len) / sizeof *db;  in srpt_get_desc_tbl()
911 		if (ioctx->n_rbuf >  in srpt_get_desc_tbl()
919 			ioctx->n_rbuf = 0;  in srpt_get_desc_tbl()
924 		if (ioctx->n_rbuf == 1)  in srpt_get_desc_tbl()
925 			ioctx->rbufs = &ioctx->single_rbuf;  in srpt_get_desc_tbl()
927 			ioctx->rbufs =  in srpt_get_desc_tbl()
928 				kmalloc(ioctx->n_rbuf * sizeof *db, GFP_ATOMIC);  in srpt_get_desc_tbl()
929 			if (!ioctx->rbufs) {  in srpt_get_desc_tbl()
930 				ioctx->n_rbuf = 0;  in srpt_get_desc_tbl()
937 		memcpy(ioctx->rbufs, db, ioctx->n_rbuf * sizeof *db);  in srpt_get_desc_tbl()
1048 				    struct srpt_send_ioctx *ioctx)  in srpt_unmap_sg_to_ib_sge()  argument
1054 	BUG_ON(!ioctx);  in srpt_unmap_sg_to_ib_sge()
1055 	BUG_ON(ioctx->n_rdma && !ioctx->rdma_ius);  in srpt_unmap_sg_to_ib_sge()
1057 	while (ioctx->n_rdma)  in srpt_unmap_sg_to_ib_sge()
1058 		kfree(ioctx->rdma_ius[--ioctx->n_rdma].sge);  in srpt_unmap_sg_to_ib_sge()
1060 	kfree(ioctx->rdma_ius);  in srpt_unmap_sg_to_ib_sge()
1061 	ioctx->rdma_ius = NULL;  in srpt_unmap_sg_to_ib_sge()
1063 	if (ioctx->mapped_sg_count) {  in srpt_unmap_sg_to_ib_sge()
1064 		sg = ioctx->sg;  in srpt_unmap_sg_to_ib_sge()
1066 		dir = ioctx->cmd.data_direction;  in srpt_unmap_sg_to_ib_sge()
1068 		ib_dma_unmap_sg(ch->sport->sdev->device, sg, ioctx->sg_cnt,  in srpt_unmap_sg_to_ib_sge()
1070 		ioctx->mapped_sg_count = 0;  in srpt_unmap_sg_to_ib_sge()
1078 				 struct srpt_send_ioctx *ioctx)  in srpt_map_sg_to_ib_sge()  argument
1097 	BUG_ON(!ioctx);  in srpt_map_sg_to_ib_sge()
1098 	cmd = &ioctx->cmd;  in srpt_map_sg_to_ib_sge()
1102 	ioctx->sg = sg = sg_orig = cmd->t_data_sg;  in srpt_map_sg_to_ib_sge()
1103 	ioctx->sg_cnt = sg_cnt = cmd->t_data_nents;  in srpt_map_sg_to_ib_sge()
1110 	ioctx->mapped_sg_count = count;  in srpt_map_sg_to_ib_sge()
1112 	if (ioctx->rdma_ius && ioctx->n_rdma_ius)  in srpt_map_sg_to_ib_sge()
1113 		nrdma = ioctx->n_rdma_ius;  in srpt_map_sg_to_ib_sge()
1116 			+ ioctx->n_rbuf;  in srpt_map_sg_to_ib_sge()
1118 		ioctx->rdma_ius = kzalloc(nrdma * sizeof *riu, GFP_KERNEL);  in srpt_map_sg_to_ib_sge()
1119 		if (!ioctx->rdma_ius)  in srpt_map_sg_to_ib_sge()
1122 		ioctx->n_rdma_ius = nrdma;  in srpt_map_sg_to_ib_sge()
1125 	db = ioctx->rbufs;  in srpt_map_sg_to_ib_sge()
1128 	riu = ioctx->rdma_ius;  in srpt_map_sg_to_ib_sge()
1139 	     j < count && i < ioctx->n_rbuf && tsize > 0; ++i, ++riu, ++db) {  in srpt_map_sg_to_ib_sge()
1171 				++ioctx->n_rdma;  in srpt_map_sg_to_ib_sge()
1185 		++ioctx->n_rdma;  in srpt_map_sg_to_ib_sge()
1192 	db = ioctx->rbufs;  in srpt_map_sg_to_ib_sge()
1194 	riu = ioctx->rdma_ius;  in srpt_map_sg_to_ib_sge()
1201 	     j < count && i < ioctx->n_rbuf && tsize > 0; ++i, ++riu, ++db) {  in srpt_map_sg_to_ib_sge()
1247 	srpt_unmap_sg_to_ib_sge(ch, ioctx);  in srpt_map_sg_to_ib_sge()
1257 	struct srpt_send_ioctx *ioctx;  in srpt_get_send_ioctx()  local
1262 	ioctx = NULL;  in srpt_get_send_ioctx()
1265 		ioctx = list_first_entry(&ch->free_list,  in srpt_get_send_ioctx()
1267 		list_del(&ioctx->free_list);  in srpt_get_send_ioctx()
1271 	if (!ioctx)  in srpt_get_send_ioctx()
1272 		return ioctx;  in srpt_get_send_ioctx()
1274 	BUG_ON(ioctx->ch != ch);  in srpt_get_send_ioctx()
1275 	spin_lock_init(&ioctx->spinlock);  in srpt_get_send_ioctx()
1276 	ioctx->state = SRPT_STATE_NEW;  in srpt_get_send_ioctx()
1277 	ioctx->n_rbuf = 0;  in srpt_get_send_ioctx()
1278 	ioctx->rbufs = NULL;  in srpt_get_send_ioctx()
1279 	ioctx->n_rdma = 0;  in srpt_get_send_ioctx()
1280 	ioctx->n_rdma_ius = 0;  in srpt_get_send_ioctx()
1281 	ioctx->rdma_ius = NULL;  in srpt_get_send_ioctx()
1282 	ioctx->mapped_sg_count = 0;  in srpt_get_send_ioctx()
1283 	init_completion(&ioctx->tx_done);  in srpt_get_send_ioctx()
1284 	ioctx->queue_status_only = false;  in srpt_get_send_ioctx()
1289 	memset(&ioctx->cmd, 0, sizeof(ioctx->cmd));  in srpt_get_send_ioctx()
1290 	memset(&ioctx->sense_data, 0, sizeof(ioctx->sense_data));  in srpt_get_send_ioctx()
1292 	return ioctx;  in srpt_get_send_ioctx()
1300 static int srpt_abort_cmd(struct srpt_send_ioctx *ioctx)  in srpt_abort_cmd()  argument
1305 	BUG_ON(!ioctx);  in srpt_abort_cmd()
1315 	spin_lock_irqsave(&ioctx->spinlock, flags);  in srpt_abort_cmd()
1316 	state = ioctx->state;  in srpt_abort_cmd()
1319 		ioctx->state = SRPT_STATE_DATA_IN;  in srpt_abort_cmd()
1324 		ioctx->state = SRPT_STATE_DONE;  in srpt_abort_cmd()
1329 	spin_unlock_irqrestore(&ioctx->spinlock, flags);  in srpt_abort_cmd()
1332 		struct srpt_rdma_ch *ch = ioctx->ch;  in srpt_abort_cmd()
1336 		target_put_sess_cmd(&ioctx->cmd);  in srpt_abort_cmd()
1341 		 ioctx->cmd.tag);  in srpt_abort_cmd()
1351 		WARN_ON(!transport_check_aborted_status(&ioctx->cmd, false));  in srpt_abort_cmd()
1357 		spin_lock_irqsave(&ioctx->cmd.t_state_lock, flags);  in srpt_abort_cmd()
1358 		ioctx->cmd.transport_state &= ~CMD_T_ACTIVE;  in srpt_abort_cmd()
1359 		spin_unlock_irqrestore(&ioctx->cmd.t_state_lock, flags);  in srpt_abort_cmd()
1366 		srpt_unmap_sg_to_ib_sge(ioctx->ch, ioctx);  in srpt_abort_cmd()
1367 		target_put_sess_cmd(&ioctx->cmd);  in srpt_abort_cmd()
1370 		srpt_set_cmd_state(ioctx, SRPT_STATE_DONE);  in srpt_abort_cmd()
1371 		target_put_sess_cmd(&ioctx->cmd);  in srpt_abort_cmd()
1387 	struct srpt_send_ioctx *ioctx;  in srpt_handle_send_err_comp()  local
1394 	ioctx = ch->ioctx_ring[index];  in srpt_handle_send_err_comp()
1395 	state = srpt_get_cmd_state(ioctx);  in srpt_handle_send_err_comp()
1407 	srpt_abort_cmd(ioctx);  in srpt_handle_send_err_comp()
1414 				  struct srpt_send_ioctx *ioctx)  in srpt_handle_send_comp()  argument
1420 	state = srpt_set_cmd_state(ioctx, SRPT_STATE_DONE);  in srpt_handle_send_comp()
1428 		srpt_unmap_sg_to_ib_sge(ch, ioctx);  in srpt_handle_send_comp()
1429 		transport_generic_free_cmd(&ioctx->cmd, 0);  in srpt_handle_send_comp()
1432 		       " wr_id = %u.\n", ioctx->ioctx.index);  in srpt_handle_send_comp()
1445 				  struct srpt_send_ioctx *ioctx,  in srpt_handle_rdma_comp()  argument
1448 	WARN_ON(ioctx->n_rdma <= 0);  in srpt_handle_rdma_comp()
1449 	atomic_add(ioctx->n_rdma, &ch->sq_wr_avail);  in srpt_handle_rdma_comp()
1452 		if (srpt_test_and_set_cmd_state(ioctx, SRPT_STATE_NEED_DATA,  in srpt_handle_rdma_comp()
1454 			target_execute_cmd(&ioctx->cmd);  in srpt_handle_rdma_comp()
1457 			       __LINE__, srpt_get_cmd_state(ioctx));  in srpt_handle_rdma_comp()
1459 		ioctx->rdma_aborted = true;  in srpt_handle_rdma_comp()
1469 				      struct srpt_send_ioctx *ioctx,  in srpt_handle_rdma_err_comp()  argument
1474 	state = srpt_get_cmd_state(ioctx);  in srpt_handle_rdma_err_comp()
1477 		if (ioctx->n_rdma <= 0) {  in srpt_handle_rdma_err_comp()
1480 			       ioctx->ioctx.index);  in srpt_handle_rdma_err_comp()
1483 		atomic_add(ioctx->n_rdma, &ch->sq_wr_avail);  in srpt_handle_rdma_err_comp()
1485 			srpt_abort_cmd(ioctx);  in srpt_handle_rdma_err_comp()
1514 			      struct srpt_send_ioctx *ioctx, u64 tag,  in srpt_build_cmd_rsp()  argument
1527 	srp_rsp = ioctx->ioctx.buf;  in srpt_build_cmd_rsp()
1530 	sense_data = ioctx->sense_data;  in srpt_build_cmd_rsp()
1531 	sense_data_len = ioctx->cmd.scsi_sense_length;  in srpt_build_cmd_rsp()
1532 	WARN_ON(sense_data_len > sizeof(ioctx->sense_data));  in srpt_build_cmd_rsp()
1572 				  struct srpt_send_ioctx *ioctx,  in srpt_build_tskmgmt_rsp()  argument
1582 	srp_rsp = ioctx->ioctx.buf;  in srpt_build_tskmgmt_rsp()
1674 	struct srpt_send_ioctx *ioctx = container_of(cmd,  in srpt_check_stop_free()  local
1677 	return target_put_sess_cmd(&ioctx->cmd);  in srpt_check_stop_free()
1697 	srp_cmd = recv_ioctx->ioctx.buf;  in srpt_handle_cmd()
1779 	srp_tsk = recv_ioctx->ioctx.buf;  in srpt_handle_tsk_mgmt()
1819 				   recv_ioctx->ioctx.dma, srp_max_req_size,  in srpt_handle_new_iu()
1831 	srp_cmd = recv_ioctx->ioctx.buf;  in srpt_handle_new_iu()
1877 	struct srpt_recv_ioctx *ioctx;  in srpt_process_rcv_completion()  local
1887 		ioctx = sdev->ioctx_ring[index];  in srpt_process_rcv_completion()
1888 		srpt_handle_new_iu(ch, ioctx, NULL);  in srpt_process_rcv_completion()
2634 		struct srpt_recv_ioctx *ioctx, *ioctx_tmp;  in srpt_cm_rtu_recv()  local
2638 		list_for_each_entry_safe(ioctx, ioctx_tmp, &ch->cmd_wait_list,  in srpt_cm_rtu_recv()
2640 			list_del(&ioctx->wait_list);  in srpt_cm_rtu_recv()
2641 			srpt_handle_new_iu(ch, ioctx, NULL);  in srpt_cm_rtu_recv()
2765 			      struct srpt_send_ioctx *ioctx)  in srpt_perform_rdmas()  argument
2774 	const int n_rdma = ioctx->n_rdma;  in srpt_perform_rdmas()
2776 	dir = ioctx->cmd.data_direction;  in srpt_perform_rdmas()
2788 	ioctx->rdma_aborted = false;  in srpt_perform_rdmas()
2790 	riu = ioctx->rdma_ius;  in srpt_perform_rdmas()
2799 						ioctx->ioctx.index);  in srpt_perform_rdmas()
2805 						ioctx->ioctx.index);  in srpt_perform_rdmas()
2827 		wr.wr.wr_id = encode_wr_id(SRPT_RDMA_ABORT, ioctx->ioctx.index);  in srpt_perform_rdmas()
2832 				ioctx->ioctx.index);  in srpt_perform_rdmas()
2835 		while (ch->state != CH_RELEASING && !ioctx->rdma_aborted) {  in srpt_perform_rdmas()
2837 				ioctx->ioctx.index);  in srpt_perform_rdmas()
2851 			  struct srpt_send_ioctx *ioctx)  in srpt_xfer_data()  argument
2855 	ret = srpt_map_sg_to_ib_sge(ch, ioctx);  in srpt_xfer_data()
2861 	ret = srpt_perform_rdmas(ch, ioctx);  in srpt_xfer_data()
2875 	srpt_unmap_sg_to_ib_sge(ch, ioctx);  in srpt_xfer_data()
2881 	struct srpt_send_ioctx *ioctx;  in srpt_write_pending_status()  local
2883 	ioctx = container_of(se_cmd, struct srpt_send_ioctx, cmd);  in srpt_write_pending_status()
2884 	return srpt_get_cmd_state(ioctx) == SRPT_STATE_NEED_DATA;  in srpt_write_pending_status()
2893 	struct srpt_send_ioctx *ioctx;  in srpt_write_pending()  local
2898 	ioctx = container_of(se_cmd, struct srpt_send_ioctx, cmd);  in srpt_write_pending()
2900 	new_state = srpt_set_cmd_state(ioctx, SRPT_STATE_NEED_DATA);  in srpt_write_pending()
2903 	ch = ioctx->ch;  in srpt_write_pending()
2918 			 ioctx->cmd.tag);  in srpt_write_pending()
2919 		srpt_set_cmd_state(ioctx, SRPT_STATE_DATA_IN);  in srpt_write_pending()
2923 	ret = srpt_xfer_data(ch, ioctx);  in srpt_write_pending()
2949 	struct srpt_send_ioctx *ioctx;  in srpt_queue_response()  local
2957 	ioctx = container_of(cmd, struct srpt_send_ioctx, cmd);  in srpt_queue_response()
2958 	ch = ioctx->ch;  in srpt_queue_response()
2961 	spin_lock_irqsave(&ioctx->spinlock, flags);  in srpt_queue_response()
2962 	state = ioctx->state;  in srpt_queue_response()
2966 		ioctx->state = SRPT_STATE_CMD_RSP_SENT;  in srpt_queue_response()
2969 		ioctx->state = SRPT_STATE_MGMT_RSP_SENT;  in srpt_queue_response()
2973 			ch, ioctx->ioctx.index, ioctx->state);  in srpt_queue_response()
2976 	spin_unlock_irqrestore(&ioctx->spinlock, flags);  in srpt_queue_response()
2978 	if (unlikely(transport_check_aborted_status(&ioctx->cmd, false)  in srpt_queue_response()
2981 		srpt_abort_cmd(ioctx);  in srpt_queue_response()
2985 	dir = ioctx->cmd.data_direction;  in srpt_queue_response()
2988 	if (dir == DMA_FROM_DEVICE && ioctx->cmd.data_length &&  in srpt_queue_response()
2989 	    !ioctx->queue_status_only) {  in srpt_queue_response()
2990 		ret = srpt_xfer_data(ch, ioctx);  in srpt_queue_response()
2993 			       ioctx->cmd.tag);  in srpt_queue_response()
2999 		resp_len = srpt_build_cmd_rsp(ch, ioctx, ioctx->cmd.tag,  in srpt_queue_response()
3004 		resp_len = srpt_build_tskmgmt_rsp(ch, ioctx, srp_tm_status,  in srpt_queue_response()
3005 						 ioctx->cmd.tag);  in srpt_queue_response()
3007 	ret = srpt_post_send(ch, ioctx, resp_len);  in srpt_queue_response()
3010 		       ioctx->cmd.tag);  in srpt_queue_response()
3011 		srpt_unmap_sg_to_ib_sge(ch, ioctx);  in srpt_queue_response()
3012 		srpt_set_cmd_state(ioctx, SRPT_STATE_DONE);  in srpt_queue_response()
3013 		target_put_sess_cmd(&ioctx->cmd);  in srpt_queue_response()
3030 	struct srpt_send_ioctx *ioctx = container_of(cmd,  in srpt_aborted_task()  local
3033 	srpt_unmap_sg_to_ib_sge(ioctx->ch, ioctx);  in srpt_aborted_task()
3038 	struct srpt_send_ioctx *ioctx;  in srpt_queue_status()  local
3040 	ioctx = container_of(cmd, struct srpt_send_ioctx, cmd);  in srpt_queue_status()
3041 	BUG_ON(ioctx->sense_data != cmd->sense_buffer);  in srpt_queue_status()
3045 	ioctx->queue_status_only = true;  in srpt_queue_status()
3344 	struct srpt_send_ioctx *ioctx = container_of(se_cmd,  in srpt_release_cmd()  local
3346 	struct srpt_rdma_ch *ch = ioctx->ch;  in srpt_release_cmd()
3349 	WARN_ON(ioctx->state != SRPT_STATE_DONE);  in srpt_release_cmd()
3350 	WARN_ON(ioctx->mapped_sg_count != 0);  in srpt_release_cmd()
3352 	if (ioctx->n_rbuf > 1) {  in srpt_release_cmd()
3353 		kfree(ioctx->rbufs);  in srpt_release_cmd()
3354 		ioctx->rbufs = NULL;  in srpt_release_cmd()
3355 		ioctx->n_rbuf = 0;  in srpt_release_cmd()
3359 	list_add(&ioctx->free_list, &ch->free_list);  in srpt_release_cmd()
3413 	struct srpt_send_ioctx *ioctx;  in srpt_get_tcm_cmd_state()  local
3415 	ioctx = container_of(se_cmd, struct srpt_send_ioctx, cmd);  in srpt_get_tcm_cmd_state()
3416 	return srpt_get_cmd_state(ioctx);  in srpt_get_tcm_cmd_state()