51 isert_unmap_cmd(struct isert_cmd *isert_cmd, struct isert_conn *isert_conn);
56 isert_unreg_rdma(struct isert_cmd *isert_cmd, struct isert_conn *isert_conn);
1067 		       struct isert_cmd *isert_cmd,  in isert_create_send_desc()  argument
1080 	tx_desc->isert_cmd = isert_cmd;  in isert_create_send_desc()
1116 isert_init_send_wr(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd,  in isert_init_send_wr()  argument
1119 	struct iser_tx_desc *tx_desc = &isert_cmd->tx_desc;  in isert_init_send_wr()
1121 	isert_cmd->rdma_wr.iser_ib_op = ISER_IB_SEND;  in isert_init_send_wr()
1122 	send_wr->wr_id = (uintptr_t)&isert_cmd->tx_desc;  in isert_init_send_wr()
1125 	send_wr->num_sge = isert_cmd->tx_desc.num_sge;  in isert_init_send_wr()
1284 	struct isert_cmd *isert_cmd;  in isert_allocate_cmd()  local
1292 	isert_cmd = iscsit_priv_cmd(cmd);  in isert_allocate_cmd()
1293 	isert_cmd->conn = isert_conn;  in isert_allocate_cmd()
1294 	isert_cmd->iscsi_cmd = cmd;  in isert_allocate_cmd()
1301 		      struct isert_cmd *isert_cmd, struct iscsi_cmd *cmd,  in isert_handle_scsi_cmd()  argument
1414 isert_handle_nop_out(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd,  in isert_handle_nop_out()  argument
1433 isert_handle_text_cmd(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd,  in isert_handle_text_cmd()  argument
1469 	struct isert_cmd *isert_cmd;  in isert_rx_opcode()  local
1486 		isert_cmd = iscsit_priv_cmd(cmd);  in isert_rx_opcode()
1487 		isert_cmd->read_stag = read_stag;  in isert_rx_opcode()
1488 		isert_cmd->read_va = read_va;  in isert_rx_opcode()
1489 		isert_cmd->write_stag = write_stag;  in isert_rx_opcode()
1490 		isert_cmd->write_va = write_va;  in isert_rx_opcode()
1492 		ret = isert_handle_scsi_cmd(isert_conn, isert_cmd, cmd,  in isert_rx_opcode()
1500 		isert_cmd = iscsit_priv_cmd(cmd);  in isert_rx_opcode()
1501 		ret = isert_handle_nop_out(isert_conn, isert_cmd, cmd,  in isert_rx_opcode()
1534 		isert_cmd = iscsit_priv_cmd(cmd);  in isert_rx_opcode()
1535 		ret = isert_handle_text_cmd(isert_conn, isert_cmd, cmd,  in isert_rx_opcode()
1651 isert_map_data_buf(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd,  in isert_map_data_buf()  argument
1678 		  isert_cmd, data->dma_nents, data->sg, data->nents, data->len);  in isert_map_data_buf()
1695 isert_unmap_cmd(struct isert_cmd *isert_cmd, struct isert_conn *isert_conn)  in isert_unmap_cmd()  argument
1697 	struct isert_rdma_wr *wr = &isert_cmd->rdma_wr;  in isert_unmap_cmd()
1699 	isert_dbg("Cmd %p\n", isert_cmd);  in isert_unmap_cmd()
1702 		isert_dbg("Cmd %p unmap_sg op\n", isert_cmd);  in isert_unmap_cmd()
1707 		isert_dbg("Cmd %p free send_wr\n", isert_cmd);  in isert_unmap_cmd()
1713 		isert_dbg("Cmd %p free ib_sge\n", isert_cmd);  in isert_unmap_cmd()
1720 isert_unreg_rdma(struct isert_cmd *isert_cmd, struct isert_conn *isert_conn)  in isert_unreg_rdma()  argument
1722 	struct isert_rdma_wr *wr = &isert_cmd->rdma_wr;  in isert_unreg_rdma()
1724 	isert_dbg("Cmd %p\n", isert_cmd);  in isert_unreg_rdma()
1727 		isert_dbg("Cmd %p free fr_desc %p\n", isert_cmd, wr->fr_desc);  in isert_unreg_rdma()
1739 		isert_dbg("Cmd %p unmap_sg op\n", isert_cmd);  in isert_unreg_rdma()
1748 isert_put_cmd(struct isert_cmd *isert_cmd, bool comp_err)  in isert_put_cmd()  argument
1750 	struct iscsi_cmd *cmd = isert_cmd->iscsi_cmd;  in isert_put_cmd()
1751 	struct isert_conn *isert_conn = isert_cmd->conn;  in isert_put_cmd()
1756 	isert_dbg("Cmd %p\n", isert_cmd);  in isert_put_cmd()
1781 		device->unreg_rdma_mem(isert_cmd, isert_conn);  in isert_put_cmd()
1795 		hdr = (struct iscsi_text_rsp *)&isert_cmd->tx_desc.iscsi_header;  in isert_put_cmd()
1837 isert_completion_put(struct iser_tx_desc *tx_desc, struct isert_cmd *isert_cmd,  in isert_completion_put()  argument
1840 	if (isert_cmd->pdu_buf_dma != 0) {  in isert_completion_put()
1842 		ib_dma_unmap_single(ib_dev, isert_cmd->pdu_buf_dma,  in isert_completion_put()
1843 				    isert_cmd->pdu_buf_len, DMA_TO_DEVICE);  in isert_completion_put()
1844 		isert_cmd->pdu_buf_dma = 0;  in isert_completion_put()
1848 	isert_put_cmd(isert_cmd, comp_err);  in isert_completion_put()
1897 			    struct isert_cmd *isert_cmd)  in isert_completion_rdma_write()  argument
1899 	struct isert_rdma_wr *wr = &isert_cmd->rdma_wr;  in isert_completion_rdma_write()
1900 	struct iscsi_cmd *cmd = isert_cmd->iscsi_cmd;  in isert_completion_rdma_write()
1902 	struct isert_conn *isert_conn = isert_cmd->conn;  in isert_completion_rdma_write()
1912 	device->unreg_rdma_mem(isert_cmd, isert_conn);  in isert_completion_rdma_write()
1923 			   struct isert_cmd *isert_cmd)  in isert_completion_rdma_read()  argument
1925 	struct isert_rdma_wr *wr = &isert_cmd->rdma_wr;  in isert_completion_rdma_read()
1926 	struct iscsi_cmd *cmd = isert_cmd->iscsi_cmd;  in isert_completion_rdma_read()
1928 	struct isert_conn *isert_conn = isert_cmd->conn;  in isert_completion_rdma_read()
1939 	device->unreg_rdma_mem(isert_cmd, isert_conn);  in isert_completion_rdma_read()
1943 	isert_dbg("Cmd: %p RDMA_READ comp calling execute_cmd\n", isert_cmd);  in isert_completion_rdma_read()
1961 	struct isert_cmd *isert_cmd = container_of(work,  in isert_do_control_comp()  local
1962 			struct isert_cmd, comp_work);  in isert_do_control_comp()
1963 	struct isert_conn *isert_conn = isert_cmd->conn;  in isert_do_control_comp()
1965 	struct iscsi_cmd *cmd = isert_cmd->iscsi_cmd;  in isert_do_control_comp()
1967 	isert_dbg("Cmd %p i_state %d\n", isert_cmd, cmd->i_state);  in isert_do_control_comp()
1975 		isert_completion_put(&isert_cmd->tx_desc, isert_cmd,  in isert_do_control_comp()
1990 			  struct isert_cmd *isert_cmd,  in isert_response_completion()  argument
1994 	struct iscsi_cmd *cmd = isert_cmd->iscsi_cmd;  in isert_response_completion()
2002 		INIT_WORK(&isert_cmd->comp_work, isert_do_control_comp);  in isert_response_completion()
2003 		queue_work(isert_comp_wq, &isert_cmd->comp_work);  in isert_response_completion()
2008 	isert_completion_put(tx_desc, isert_cmd, ib_dev, false);  in isert_response_completion()
2016 	struct isert_cmd *isert_cmd = tx_desc->isert_cmd;  in isert_snd_completion()  local
2019 	if (!isert_cmd) {  in isert_snd_completion()
2023 	wr = &isert_cmd->rdma_wr;  in isert_snd_completion()
2025 	isert_dbg("Cmd %p iser_ib_op %d\n", isert_cmd, wr->iser_ib_op);  in isert_snd_completion()
2029 		isert_response_completion(tx_desc, isert_cmd,  in isert_snd_completion()
2033 		isert_completion_rdma_write(tx_desc, isert_cmd);  in isert_snd_completion()
2036 		isert_completion_rdma_read(tx_desc, isert_cmd);  in isert_snd_completion()
2078 		struct isert_cmd *isert_cmd;  in isert_cq_comp_err()  local
2082 		isert_cmd = desc->isert_cmd;  in isert_cq_comp_err()
2083 		if (!isert_cmd)  in isert_cq_comp_err()
2086 			isert_completion_put(desc, isert_cmd, ib_dev, true);  in isert_cq_comp_err()
2153 isert_post_response(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd)  in isert_post_response()  argument
2158 	ret = ib_post_send(isert_conn->qp, &isert_cmd->tx_desc.send_wr,  in isert_post_response()
2170 	struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd);  in isert_put_response()  local
2172 	struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr;  in isert_put_response()
2174 				&isert_cmd->tx_desc.iscsi_header;  in isert_put_response()
2176 	isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc);  in isert_put_response()
2178 	isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc);  in isert_put_response()
2187 		struct ib_sge *tx_dsg = &isert_cmd->tx_desc.tx_sg[1];  in isert_put_response()
2198 		isert_cmd->pdu_buf_dma = ib_dma_map_single(ib_dev,  in isert_put_response()
2202 		isert_cmd->pdu_buf_len = pdu_len;  in isert_put_response()
2203 		tx_dsg->addr	= isert_cmd->pdu_buf_dma;  in isert_put_response()
2206 		isert_cmd->tx_desc.num_sge = 2;  in isert_put_response()
2209 	isert_init_send_wr(isert_conn, isert_cmd, send_wr);  in isert_put_response()
2213 	return isert_post_response(isert_conn, isert_cmd);  in isert_put_response()
2219 	struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd);  in isert_aborted_task()  local
2231 	device->unreg_rdma_mem(isert_cmd, isert_conn);  in isert_aborted_task()
2258 	struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd);  in isert_put_nopin()  local
2260 	struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr;  in isert_put_nopin()
2262 	isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc);  in isert_put_nopin()
2264 			       &isert_cmd->tx_desc.iscsi_header,  in isert_put_nopin()
2266 	isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc);  in isert_put_nopin()
2267 	isert_init_send_wr(isert_conn, isert_cmd, send_wr);  in isert_put_nopin()
2271 	return isert_post_response(isert_conn, isert_cmd);  in isert_put_nopin()
2277 	struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd);  in isert_put_logout_rsp()  local
2279 	struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr;  in isert_put_logout_rsp()
2281 	isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc);  in isert_put_logout_rsp()
2283 				&isert_cmd->tx_desc.iscsi_header);  in isert_put_logout_rsp()
2284 	isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc);  in isert_put_logout_rsp()
2285 	isert_init_send_wr(isert_conn, isert_cmd, send_wr);  in isert_put_logout_rsp()
2289 	return isert_post_response(isert_conn, isert_cmd);  in isert_put_logout_rsp()
2295 	struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd);  in isert_put_tm_rsp()  local
2297 	struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr;  in isert_put_tm_rsp()
2299 	isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc);  in isert_put_tm_rsp()
2301 				  &isert_cmd->tx_desc.iscsi_header);  in isert_put_tm_rsp()
2302 	isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc);  in isert_put_tm_rsp()
2303 	isert_init_send_wr(isert_conn, isert_cmd, send_wr);  in isert_put_tm_rsp()
2307 	return isert_post_response(isert_conn, isert_cmd);  in isert_put_tm_rsp()
2313 	struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd);  in isert_put_reject()  local
2315 	struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr;  in isert_put_reject()
2318 	struct ib_sge *tx_dsg = &isert_cmd->tx_desc.tx_sg[1];  in isert_put_reject()
2320 		(struct iscsi_reject *)&isert_cmd->tx_desc.iscsi_header;  in isert_put_reject()
2322 	isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc);  in isert_put_reject()
2324 	isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc);  in isert_put_reject()
2327 	isert_cmd->pdu_buf_dma = ib_dma_map_single(ib_dev,  in isert_put_reject()
2330 	isert_cmd->pdu_buf_len = ISCSI_HDR_LEN;  in isert_put_reject()
2331 	tx_dsg->addr	= isert_cmd->pdu_buf_dma;  in isert_put_reject()
2334 	isert_cmd->tx_desc.num_sge = 2;  in isert_put_reject()
2336 	isert_init_send_wr(isert_conn, isert_cmd, send_wr);  in isert_put_reject()
2340 	return isert_post_response(isert_conn, isert_cmd);  in isert_put_reject()
2346 	struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd);  in isert_put_text_rsp()  local
2348 	struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr;  in isert_put_text_rsp()
2350 		(struct iscsi_text_rsp *)&isert_cmd->tx_desc.iscsi_header;  in isert_put_text_rsp()
2354 	isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc);  in isert_put_text_rsp()
2360 	isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc);  in isert_put_text_rsp()
2365 		struct ib_sge *tx_dsg = &isert_cmd->tx_desc.tx_sg[1];  in isert_put_text_rsp()
2368 		isert_cmd->pdu_buf_dma = ib_dma_map_single(ib_dev,  in isert_put_text_rsp()
2371 		isert_cmd->pdu_buf_len = txt_rsp_len;  in isert_put_text_rsp()
2372 		tx_dsg->addr	= isert_cmd->pdu_buf_dma;  in isert_put_text_rsp()
2375 		isert_cmd->tx_desc.num_sge = 2;  in isert_put_text_rsp()
2377 	isert_init_send_wr(isert_conn, isert_cmd, send_wr);  in isert_put_text_rsp()
2381 	return isert_post_response(isert_conn, isert_cmd);  in isert_put_text_rsp()
2385 isert_build_rdma_wr(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd,  in isert_build_rdma_wr()  argument
2389 	struct iscsi_cmd *cmd = isert_cmd->iscsi_cmd;  in isert_build_rdma_wr()
2402 	send_wr->wr_id = (uintptr_t)&isert_cmd->tx_desc;  in isert_build_rdma_wr()
2439 	struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd);  in isert_map_rdma()  local
2447 	isert_cmd->tx_desc.isert_cmd = isert_cmd;  in isert_map_rdma()
2450 	ret = isert_map_data_buf(isert_conn, isert_cmd, se_cmd->t_data_sg,  in isert_map_rdma()
2476 	wr->isert_cmd = isert_cmd;  in isert_map_rdma()
2480 		send_wr = &isert_cmd->rdma_wr.send_wr[i];  in isert_map_rdma()
2486 			send_wr->wr.rdma.remote_addr = isert_cmd->read_va + offset;  in isert_map_rdma()
2487 			send_wr->wr.rdma.rkey = isert_cmd->read_stag;  in isert_map_rdma()
2489 				send_wr->next = &isert_cmd->tx_desc.send_wr;  in isert_map_rdma()
2494 			send_wr->wr.rdma.remote_addr = isert_cmd->write_va + va_offset;  in isert_map_rdma()
2495 			send_wr->wr.rdma.rkey = isert_cmd->write_stag;  in isert_map_rdma()
2502 		ib_sge_cnt = isert_build_rdma_wr(isert_conn, isert_cmd, ib_sge,  in isert_map_rdma()
2777 		      struct isert_cmd *isert_cmd,  in isert_handle_prot_cmd()  argument
2781 	struct se_cmd *se_cmd = &isert_cmd->iscsi_cmd->se_cmd;  in isert_handle_prot_cmd()
2796 		ret = isert_map_data_buf(isert_conn, isert_cmd,  in isert_handle_prot_cmd()
2839 	struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd);  in isert_reg_rdma()  local
2848 	isert_cmd->tx_desc.isert_cmd = isert_cmd;  in isert_reg_rdma()
2851 	ret = isert_map_data_buf(isert_conn, isert_cmd, se_cmd->t_data_sg,  in isert_reg_rdma()
2872 		ret = isert_handle_prot_cmd(isert_conn, isert_cmd, wr);  in isert_reg_rdma()
2886 	wr->isert_cmd = isert_cmd;  in isert_reg_rdma()
2888 	send_wr = &isert_cmd->rdma_wr.s_send_wr;  in isert_reg_rdma()
2891 	send_wr->wr_id = (uintptr_t)&isert_cmd->tx_desc;  in isert_reg_rdma()
2894 		send_wr->wr.rdma.remote_addr = isert_cmd->read_va;  in isert_reg_rdma()
2895 		send_wr->wr.rdma.rkey = isert_cmd->read_stag;  in isert_reg_rdma()
2900 		send_wr->wr.rdma.remote_addr = isert_cmd->write_va;  in isert_reg_rdma()
2901 		send_wr->wr.rdma.rkey = isert_cmd->write_stag;  in isert_reg_rdma()
2922 	struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd);  in isert_put_datain()  local
2923 	struct isert_rdma_wr *wr = &isert_cmd->rdma_wr;  in isert_put_datain()
2930 		 isert_cmd, se_cmd->data_length);  in isert_put_datain()
2935 		isert_err("Cmd: %p failed to prepare RDMA res\n", isert_cmd);  in isert_put_datain()
2943 		isert_create_send_desc(isert_conn, isert_cmd,  in isert_put_datain()
2944 				       &isert_cmd->tx_desc);  in isert_put_datain()
2946 				     &isert_cmd->tx_desc.iscsi_header);  in isert_put_datain()
2947 		isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc);  in isert_put_datain()
2948 		isert_init_send_wr(isert_conn, isert_cmd,  in isert_put_datain()
2949 				   &isert_cmd->tx_desc.send_wr);  in isert_put_datain()
2950 		isert_cmd->rdma_wr.s_send_wr.next = &isert_cmd->tx_desc.send_wr;  in isert_put_datain()
2960 			 "READ\n", isert_cmd);  in isert_put_datain()
2963 			 isert_cmd);  in isert_put_datain()
2972 	struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd);  in isert_get_dataout()  local
2973 	struct isert_rdma_wr *wr = &isert_cmd->rdma_wr;  in isert_get_dataout()
2980 		 isert_cmd, se_cmd->data_length, cmd->write_data_done);  in isert_get_dataout()
2984 		isert_err("Cmd: %p failed to prepare RDMA res\n", isert_cmd);  in isert_get_dataout()
2993 		 isert_cmd);  in isert_get_dataout()
3001 	struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd);  in isert_immediate_queue()  local
3009 		isert_put_cmd(isert_cmd, true);  in isert_immediate_queue()
3403 			struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd);  in isert_put_unsol_pending_cmds()  local
3406 			isert_put_cmd(isert_cmd, true);  in isert_put_unsol_pending_cmds()
3448 	.priv_size		= sizeof(struct isert_cmd),