54 	       struct isert_rdma_wr *wr);
59 	       struct isert_rdma_wr *wr);
1675 	struct isert_rdma_wr *wr = &isert_cmd->rdma_wr;  in isert_unmap_cmd()  local
1679 	if (wr->data.sg) {  in isert_unmap_cmd()
1681 		isert_unmap_data_buf(isert_conn, &wr->data);  in isert_unmap_cmd()
1684 	if (wr->rdma_wr) {  in isert_unmap_cmd()
1686 		kfree(wr->rdma_wr);  in isert_unmap_cmd()
1687 		wr->rdma_wr = NULL;  in isert_unmap_cmd()
1690 	if (wr->ib_sge) {  in isert_unmap_cmd()
1692 		kfree(wr->ib_sge);  in isert_unmap_cmd()
1693 		wr->ib_sge = NULL;  in isert_unmap_cmd()
1700 	struct isert_rdma_wr *wr = &isert_cmd->rdma_wr;  in isert_unreg_rdma()  local
1704 	if (wr->fr_desc) {  in isert_unreg_rdma()
1705 		isert_dbg("Cmd %p free fr_desc %p\n", isert_cmd, wr->fr_desc);  in isert_unreg_rdma()
1706 		if (wr->fr_desc->ind & ISERT_PROTECTED) {  in isert_unreg_rdma()
1707 			isert_unmap_data_buf(isert_conn, &wr->prot);  in isert_unreg_rdma()
1708 			wr->fr_desc->ind &= ~ISERT_PROTECTED;  in isert_unreg_rdma()
1711 		list_add_tail(&wr->fr_desc->list, &isert_conn->fr_pool);  in isert_unreg_rdma()
1713 		wr->fr_desc = NULL;  in isert_unreg_rdma()
1716 	if (wr->data.sg) {  in isert_unreg_rdma()
1718 		isert_unmap_data_buf(isert_conn, &wr->data);  in isert_unreg_rdma()
1721 	wr->ib_sge = NULL;  in isert_unreg_rdma()
1722 	wr->rdma_wr = NULL;  in isert_unreg_rdma()
1877 	struct isert_rdma_wr *wr = &isert_cmd->rdma_wr;  in isert_completion_rdma_write()  local
1884 	if (wr->fr_desc && wr->fr_desc->ind & ISERT_PROTECTED) {  in isert_completion_rdma_write()
1886 					    wr->fr_desc->pi_ctx->sig_mr);  in isert_completion_rdma_write()
1887 		wr->fr_desc->ind &= ~ISERT_PROTECTED;  in isert_completion_rdma_write()
1891 	wr->rdma_wr_num = 0;  in isert_completion_rdma_write()
1903 	struct isert_rdma_wr *wr = &isert_cmd->rdma_wr;  in isert_completion_rdma_read()  local
1910 	if (wr->fr_desc && wr->fr_desc->ind & ISERT_PROTECTED) {  in isert_completion_rdma_read()
1912 					    wr->fr_desc->pi_ctx->sig_mr);  in isert_completion_rdma_read()
1913 		wr->fr_desc->ind &= ~ISERT_PROTECTED;  in isert_completion_rdma_read()
1918 	cmd->write_data_done = wr->data.len;  in isert_completion_rdma_read()
1919 	wr->rdma_wr_num = 0;  in isert_completion_rdma_read()
1995 	struct isert_rdma_wr *wr;  in isert_snd_completion()  local
2001 	wr = &isert_cmd->rdma_wr;  in isert_snd_completion()
2003 	isert_dbg("Cmd %p iser_ib_op %d\n", isert_cmd, wr->iser_ib_op);  in isert_snd_completion()
2005 	switch (wr->iser_ib_op) {  in isert_snd_completion()
2017 		isert_err("Unknown wr->iser_ib_op: 0x%x\n", wr->iser_ib_op);  in isert_snd_completion()
2383 	rdma_wr->wr.sg_list = ib_sge;  in isert_build_rdma_wr()
2384 	rdma_wr->wr.wr_id = (uintptr_t)&isert_cmd->tx_desc;  in isert_build_rdma_wr()
2409 	rdma_wr->wr.num_sge = ++i;  in isert_build_rdma_wr()
2411 		  rdma_wr->wr.sg_list, rdma_wr->wr.num_sge);  in isert_build_rdma_wr()
2413 	return rdma_wr->wr.num_sge;  in isert_build_rdma_wr()
2418 	       struct isert_rdma_wr *wr)  in isert_map_rdma()  argument
2423 	struct isert_data_buf *data = &wr->data;  in isert_map_rdma()
2431 	offset = wr->iser_ib_op == ISER_IB_RDMA_READ ? cmd->write_data_done : 0;  in isert_map_rdma()
2434 				 offset, wr->iser_ib_op, &wr->data);  in isert_map_rdma()
2447 	wr->ib_sge = ib_sge;  in isert_map_rdma()
2449 	wr->rdma_wr_num = DIV_ROUND_UP(data->nents, isert_conn->max_sge);  in isert_map_rdma()
2450 	wr->rdma_wr = kzalloc(sizeof(struct ib_rdma_wr) * wr->rdma_wr_num,  in isert_map_rdma()
2452 	if (!wr->rdma_wr) {  in isert_map_rdma()
2458 	wr->isert_cmd = isert_cmd;  in isert_map_rdma()
2461 	for (i = 0; i < wr->rdma_wr_num; i++) {  in isert_map_rdma()
2465 		rdma_wr->wr.send_flags = 0;  in isert_map_rdma()
2466 		if (wr->iser_ib_op == ISER_IB_RDMA_WRITE) {  in isert_map_rdma()
2467 			rdma_wr->wr.opcode = IB_WR_RDMA_WRITE;  in isert_map_rdma()
2470 			if (i + 1 == wr->rdma_wr_num)  in isert_map_rdma()
2471 				rdma_wr->wr.next = &isert_cmd->tx_desc.send_wr;  in isert_map_rdma()
2473 				rdma_wr->wr.next = &wr->rdma_wr[i + 1].wr;  in isert_map_rdma()
2475 			rdma_wr->wr.opcode = IB_WR_RDMA_READ;  in isert_map_rdma()
2478 			if (i + 1 == wr->rdma_wr_num)  in isert_map_rdma()
2479 				rdma_wr->wr.send_flags = IB_SEND_SIGNALED;  in isert_map_rdma()
2481 				rdma_wr->wr.next = &wr->rdma_wr[i + 1].wr;  in isert_map_rdma()
2526 	struct ib_send_wr inv_wr, *bad_wr, *wr = NULL;  in isert_fast_reg_mr()  local
2547 		wr = &inv_wr;  in isert_fast_reg_mr()
2560 	reg_wr.wr.next = NULL;  in isert_fast_reg_mr()
2561 	reg_wr.wr.opcode = IB_WR_REG_MR;  in isert_fast_reg_mr()
2562 	reg_wr.wr.wr_id = ISER_FASTREG_LI_WRID;  in isert_fast_reg_mr()
2563 	reg_wr.wr.send_flags = 0;  in isert_fast_reg_mr()
2564 	reg_wr.wr.num_sge = 0;  in isert_fast_reg_mr()
2569 	if (!wr)  in isert_fast_reg_mr()
2570 		wr = &reg_wr.wr;  in isert_fast_reg_mr()
2572 		wr->next = &reg_wr.wr;  in isert_fast_reg_mr()
2574 	ret = ib_post_send(isert_conn->qp, wr, &bad_wr);  in isert_fast_reg_mr()
2654 	struct ib_send_wr inv_wr, *bad_wr, *wr = NULL;  in isert_reg_sig_mr()  local
2668 		wr = &inv_wr;  in isert_reg_sig_mr()
2672 	sig_wr.wr.opcode = IB_WR_REG_SIG_MR;  in isert_reg_sig_mr()
2673 	sig_wr.wr.wr_id = ISER_FASTREG_LI_WRID;  in isert_reg_sig_mr()
2674 	sig_wr.wr.sg_list = &rdma_wr->ib_sg[DATA];  in isert_reg_sig_mr()
2675 	sig_wr.wr.num_sge = 1;  in isert_reg_sig_mr()
2682 	if (!wr)  in isert_reg_sig_mr()
2683 		wr = &sig_wr.wr;  in isert_reg_sig_mr()
2685 		wr->next = &sig_wr.wr;  in isert_reg_sig_mr()
2687 	ret = ib_post_send(isert_conn->qp, wr, &bad_wr);  in isert_reg_sig_mr()
2715 		      struct isert_rdma_wr *wr)  in isert_handle_prot_cmd()  argument
2721 	if (!wr->fr_desc->pi_ctx) {  in isert_handle_prot_cmd()
2722 		ret = isert_create_pi_ctx(wr->fr_desc,  in isert_handle_prot_cmd()
2737 					 0, wr->iser_ib_op, &wr->prot);  in isert_handle_prot_cmd()
2744 		memset(&wr->ib_sg[PROT], 0, sizeof(wr->ib_sg[PROT]));  in isert_handle_prot_cmd()
2745 		ret = isert_fast_reg_mr(isert_conn, wr->fr_desc, &wr->prot,  in isert_handle_prot_cmd()
2746 					ISERT_PROT_KEY_VALID, &wr->ib_sg[PROT]);  in isert_handle_prot_cmd()
2754 	ret = isert_reg_sig_mr(isert_conn, se_cmd, wr, wr->fr_desc);  in isert_handle_prot_cmd()
2760 	wr->fr_desc->ind |= ISERT_PROTECTED;  in isert_handle_prot_cmd()
2766 		isert_unmap_data_buf(isert_conn, &wr->prot);  in isert_handle_prot_cmd()
2773 	       struct isert_rdma_wr *wr)  in isert_reg_rdma()  argument
2787 	offset = wr->iser_ib_op == ISER_IB_RDMA_READ ? cmd->write_data_done : 0;  in isert_reg_rdma()
2790 				 offset, wr->iser_ib_op, &wr->data);  in isert_reg_rdma()
2794 	if (wr->data.dma_nents != 1 || isert_prot_cmd(isert_conn, se_cmd)) {  in isert_reg_rdma()
2800 		wr->fr_desc = fr_desc;  in isert_reg_rdma()
2803 	ret = isert_fast_reg_mr(isert_conn, fr_desc, &wr->data,  in isert_reg_rdma()
2804 				ISERT_DATA_KEY_VALID, &wr->ib_sg[DATA]);  in isert_reg_rdma()
2809 		ret = isert_handle_prot_cmd(isert_conn, isert_cmd, wr);  in isert_reg_rdma()
2813 		ib_sg = &wr->ib_sg[SIG];  in isert_reg_rdma()
2815 		ib_sg = &wr->ib_sg[DATA];  in isert_reg_rdma()
2818 	memcpy(&wr->s_ib_sge, ib_sg, sizeof(*ib_sg));  in isert_reg_rdma()
2819 	wr->ib_sge = &wr->s_ib_sge;  in isert_reg_rdma()
2820 	wr->rdma_wr_num = 1;  in isert_reg_rdma()
2821 	memset(&wr->s_rdma_wr, 0, sizeof(wr->s_rdma_wr));  in isert_reg_rdma()
2822 	wr->rdma_wr = &wr->s_rdma_wr;  in isert_reg_rdma()
2823 	wr->isert_cmd = isert_cmd;  in isert_reg_rdma()
2826 	rdma_wr->wr.sg_list = &wr->s_ib_sge;  in isert_reg_rdma()
2827 	rdma_wr->wr.num_sge = 1;  in isert_reg_rdma()
2828 	rdma_wr->wr.wr_id = (uintptr_t)&isert_cmd->tx_desc;  in isert_reg_rdma()
2829 	if (wr->iser_ib_op == ISER_IB_RDMA_WRITE) {  in isert_reg_rdma()
2830 		rdma_wr->wr.opcode = IB_WR_RDMA_WRITE;  in isert_reg_rdma()
2833 		rdma_wr->wr.send_flags = !isert_prot_cmd(isert_conn, se_cmd) ?  in isert_reg_rdma()
2836 		rdma_wr->wr.opcode = IB_WR_RDMA_READ;  in isert_reg_rdma()
2839 		rdma_wr->wr.send_flags = IB_SEND_SIGNALED;  in isert_reg_rdma()
2850 	isert_unmap_data_buf(isert_conn, &wr->data);  in isert_reg_rdma()
2860 	struct isert_rdma_wr *wr = &isert_cmd->rdma_wr;  in isert_put_datain()  local
2869 	wr->iser_ib_op = ISER_IB_RDMA_WRITE;  in isert_put_datain()
2870 	rc = device->reg_rdma_mem(conn, cmd, wr);  in isert_put_datain()
2887 		isert_cmd->rdma_wr.s_rdma_wr.wr.next = &isert_cmd->tx_desc.send_wr;  in isert_put_datain()
2888 		wr->rdma_wr_num += 1;  in isert_put_datain()
2897 	rc = ib_post_send(isert_conn->qp, &wr->rdma_wr->wr, &wr_failed);  in isert_put_datain()
2916 	struct isert_rdma_wr *wr = &isert_cmd->rdma_wr;  in isert_get_dataout()  local
2924 	wr->iser_ib_op = ISER_IB_RDMA_READ;  in isert_get_dataout()
2925 	rc = device->reg_rdma_mem(conn, cmd, wr);  in isert_get_dataout()
2931 	rc = ib_post_send(isert_conn->qp, &wr->rdma_wr->wr, &wr_failed);  in isert_get_dataout()