54 	       struct isert_rdma_wr *wr);
59 	       struct isert_rdma_wr *wr);
1697 	struct isert_rdma_wr *wr = &isert_cmd->rdma_wr;  in isert_unmap_cmd()  local
1701 	if (wr->data.sg) {  in isert_unmap_cmd()
1703 		isert_unmap_data_buf(isert_conn, &wr->data);  in isert_unmap_cmd()
1706 	if (wr->send_wr) {  in isert_unmap_cmd()
1708 		kfree(wr->send_wr);  in isert_unmap_cmd()
1709 		wr->send_wr = NULL;  in isert_unmap_cmd()
1712 	if (wr->ib_sge) {  in isert_unmap_cmd()
1714 		kfree(wr->ib_sge);  in isert_unmap_cmd()
1715 		wr->ib_sge = NULL;  in isert_unmap_cmd()
1722 	struct isert_rdma_wr *wr = &isert_cmd->rdma_wr;  in isert_unreg_rdma()  local
1726 	if (wr->fr_desc) {  in isert_unreg_rdma()
1727 		isert_dbg("Cmd %p free fr_desc %p\n", isert_cmd, wr->fr_desc);  in isert_unreg_rdma()
1728 		if (wr->fr_desc->ind & ISERT_PROTECTED) {  in isert_unreg_rdma()
1729 			isert_unmap_data_buf(isert_conn, &wr->prot);  in isert_unreg_rdma()
1730 			wr->fr_desc->ind &= ~ISERT_PROTECTED;  in isert_unreg_rdma()
1733 		list_add_tail(&wr->fr_desc->list, &isert_conn->fr_pool);  in isert_unreg_rdma()
1735 		wr->fr_desc = NULL;  in isert_unreg_rdma()
1738 	if (wr->data.sg) {  in isert_unreg_rdma()
1740 		isert_unmap_data_buf(isert_conn, &wr->data);  in isert_unreg_rdma()
1743 	wr->ib_sge = NULL;  in isert_unreg_rdma()
1744 	wr->send_wr = NULL;  in isert_unreg_rdma()
1899 	struct isert_rdma_wr *wr = &isert_cmd->rdma_wr;  in isert_completion_rdma_write()  local
1906 	if (wr->fr_desc && wr->fr_desc->ind & ISERT_PROTECTED) {  in isert_completion_rdma_write()
1908 					    wr->fr_desc->pi_ctx->sig_mr);  in isert_completion_rdma_write()
1909 		wr->fr_desc->ind &= ~ISERT_PROTECTED;  in isert_completion_rdma_write()
1913 	wr->send_wr_num = 0;  in isert_completion_rdma_write()
1925 	struct isert_rdma_wr *wr = &isert_cmd->rdma_wr;  in isert_completion_rdma_read()  local
1932 	if (wr->fr_desc && wr->fr_desc->ind & ISERT_PROTECTED) {  in isert_completion_rdma_read()
1934 					    wr->fr_desc->pi_ctx->sig_mr);  in isert_completion_rdma_read()
1935 		wr->fr_desc->ind &= ~ISERT_PROTECTED;  in isert_completion_rdma_read()
1940 	cmd->write_data_done = wr->data.len;  in isert_completion_rdma_read()
1941 	wr->send_wr_num = 0;  in isert_completion_rdma_read()
2017 	struct isert_rdma_wr *wr;  in isert_snd_completion()  local
2023 	wr = &isert_cmd->rdma_wr;  in isert_snd_completion()
2025 	isert_dbg("Cmd %p iser_ib_op %d\n", isert_cmd, wr->iser_ib_op);  in isert_snd_completion()
2027 	switch (wr->iser_ib_op) {  in isert_snd_completion()
2039 		isert_err("Unknown wr->iser_ib_op: 0x%x\n", wr->iser_ib_op);  in isert_snd_completion()
2436 	       struct isert_rdma_wr *wr)  in isert_map_rdma()  argument
2441 	struct isert_data_buf *data = &wr->data;  in isert_map_rdma()
2449 	offset = wr->iser_ib_op == ISER_IB_RDMA_READ ? cmd->write_data_done : 0;  in isert_map_rdma()
2452 				 offset, wr->iser_ib_op, &wr->data);  in isert_map_rdma()
2465 	wr->ib_sge = ib_sge;  in isert_map_rdma()
2467 	wr->send_wr_num = DIV_ROUND_UP(data->nents, isert_conn->max_sge);  in isert_map_rdma()
2468 	wr->send_wr = kzalloc(sizeof(struct ib_send_wr) * wr->send_wr_num,  in isert_map_rdma()
2470 	if (!wr->send_wr) {  in isert_map_rdma()
2476 	wr->isert_cmd = isert_cmd;  in isert_map_rdma()
2479 	for (i = 0; i < wr->send_wr_num; i++) {  in isert_map_rdma()
2484 		if (wr->iser_ib_op == ISER_IB_RDMA_WRITE) {  in isert_map_rdma()
2486 			send_wr->wr.rdma.remote_addr = isert_cmd->read_va + offset;  in isert_map_rdma()
2487 			send_wr->wr.rdma.rkey = isert_cmd->read_stag;  in isert_map_rdma()
2488 			if (i + 1 == wr->send_wr_num)  in isert_map_rdma()
2491 				send_wr->next = &wr->send_wr[i + 1];  in isert_map_rdma()
2494 			send_wr->wr.rdma.remote_addr = isert_cmd->write_va + va_offset;  in isert_map_rdma()
2495 			send_wr->wr.rdma.rkey = isert_cmd->write_stag;  in isert_map_rdma()
2496 			if (i + 1 == wr->send_wr_num)  in isert_map_rdma()
2499 				send_wr->next = &wr->send_wr[i + 1];  in isert_map_rdma()
2584 	struct ib_send_wr *bad_wr, *wr = NULL;  in isert_fast_reg_mr()  local
2617 		wr = &inv_wr;  in isert_fast_reg_mr()
2624 	fr_wr.wr.fast_reg.iova_start = frpl->page_list[0] + page_off;  in isert_fast_reg_mr()
2625 	fr_wr.wr.fast_reg.page_list = frpl;  in isert_fast_reg_mr()
2626 	fr_wr.wr.fast_reg.page_list_len = pagelist_len;  in isert_fast_reg_mr()
2627 	fr_wr.wr.fast_reg.page_shift = PAGE_SHIFT;  in isert_fast_reg_mr()
2628 	fr_wr.wr.fast_reg.length = mem->len;  in isert_fast_reg_mr()
2629 	fr_wr.wr.fast_reg.rkey = mr->rkey;  in isert_fast_reg_mr()
2630 	fr_wr.wr.fast_reg.access_flags = IB_ACCESS_LOCAL_WRITE;  in isert_fast_reg_mr()
2632 	if (!wr)  in isert_fast_reg_mr()
2633 		wr = &fr_wr;  in isert_fast_reg_mr()
2635 		wr->next = &fr_wr;  in isert_fast_reg_mr()
2637 	ret = ib_post_send(isert_conn->qp, wr, &bad_wr);  in isert_fast_reg_mr()
2717 	struct ib_send_wr *bad_wr, *wr = NULL;  in isert_reg_sig_mr()  local
2731 		wr = &inv_wr;  in isert_reg_sig_mr()
2739 	sig_wr.wr.sig_handover.access_flags = IB_ACCESS_LOCAL_WRITE;  in isert_reg_sig_mr()
2740 	sig_wr.wr.sig_handover.sig_attrs = &sig_attrs;  in isert_reg_sig_mr()
2741 	sig_wr.wr.sig_handover.sig_mr = pi_ctx->sig_mr;  in isert_reg_sig_mr()
2743 		sig_wr.wr.sig_handover.prot = &rdma_wr->ib_sg[PROT];  in isert_reg_sig_mr()
2745 	if (!wr)  in isert_reg_sig_mr()
2746 		wr = &sig_wr;  in isert_reg_sig_mr()
2748 		wr->next = &sig_wr;  in isert_reg_sig_mr()
2750 	ret = ib_post_send(isert_conn->qp, wr, &bad_wr);  in isert_reg_sig_mr()
2778 		      struct isert_rdma_wr *wr)  in isert_handle_prot_cmd()  argument
2784 	if (!wr->fr_desc->pi_ctx) {  in isert_handle_prot_cmd()
2785 		ret = isert_create_pi_ctx(wr->fr_desc,  in isert_handle_prot_cmd()
2800 					 0, wr->iser_ib_op, &wr->prot);  in isert_handle_prot_cmd()
2807 		memset(&wr->ib_sg[PROT], 0, sizeof(wr->ib_sg[PROT]));  in isert_handle_prot_cmd()
2808 		ret = isert_fast_reg_mr(isert_conn, wr->fr_desc, &wr->prot,  in isert_handle_prot_cmd()
2809 					ISERT_PROT_KEY_VALID, &wr->ib_sg[PROT]);  in isert_handle_prot_cmd()
2817 	ret = isert_reg_sig_mr(isert_conn, se_cmd, wr, wr->fr_desc);  in isert_handle_prot_cmd()
2823 	wr->fr_desc->ind |= ISERT_PROTECTED;  in isert_handle_prot_cmd()
2829 		isert_unmap_data_buf(isert_conn, &wr->prot);  in isert_handle_prot_cmd()
2836 	       struct isert_rdma_wr *wr)  in isert_reg_rdma()  argument
2850 	offset = wr->iser_ib_op == ISER_IB_RDMA_READ ? cmd->write_data_done : 0;  in isert_reg_rdma()
2853 				 offset, wr->iser_ib_op, &wr->data);  in isert_reg_rdma()
2857 	if (wr->data.dma_nents != 1 || isert_prot_cmd(isert_conn, se_cmd)) {  in isert_reg_rdma()
2863 		wr->fr_desc = fr_desc;  in isert_reg_rdma()
2866 	ret = isert_fast_reg_mr(isert_conn, fr_desc, &wr->data,  in isert_reg_rdma()
2867 				ISERT_DATA_KEY_VALID, &wr->ib_sg[DATA]);  in isert_reg_rdma()
2872 		ret = isert_handle_prot_cmd(isert_conn, isert_cmd, wr);  in isert_reg_rdma()
2876 		ib_sg = &wr->ib_sg[SIG];  in isert_reg_rdma()
2878 		ib_sg = &wr->ib_sg[DATA];  in isert_reg_rdma()
2881 	memcpy(&wr->s_ib_sge, ib_sg, sizeof(*ib_sg));  in isert_reg_rdma()
2882 	wr->ib_sge = &wr->s_ib_sge;  in isert_reg_rdma()
2883 	wr->send_wr_num = 1;  in isert_reg_rdma()
2884 	memset(&wr->s_send_wr, 0, sizeof(*send_wr));  in isert_reg_rdma()
2885 	wr->send_wr = &wr->s_send_wr;  in isert_reg_rdma()
2886 	wr->isert_cmd = isert_cmd;  in isert_reg_rdma()
2889 	send_wr->sg_list = &wr->s_ib_sge;  in isert_reg_rdma()
2892 	if (wr->iser_ib_op == ISER_IB_RDMA_WRITE) {  in isert_reg_rdma()
2894 		send_wr->wr.rdma.remote_addr = isert_cmd->read_va;  in isert_reg_rdma()
2895 		send_wr->wr.rdma.rkey = isert_cmd->read_stag;  in isert_reg_rdma()
2900 		send_wr->wr.rdma.remote_addr = isert_cmd->write_va;  in isert_reg_rdma()
2901 		send_wr->wr.rdma.rkey = isert_cmd->write_stag;  in isert_reg_rdma()
2913 	isert_unmap_data_buf(isert_conn, &wr->data);  in isert_reg_rdma()
2923 	struct isert_rdma_wr *wr = &isert_cmd->rdma_wr;  in isert_put_datain()  local
2932 	wr->iser_ib_op = ISER_IB_RDMA_WRITE;  in isert_put_datain()
2933 	rc = device->reg_rdma_mem(conn, cmd, wr);  in isert_put_datain()
2951 		wr->send_wr_num += 1;  in isert_put_datain()
2954 	rc = ib_post_send(isert_conn->qp, wr->send_wr, &wr_failed);  in isert_put_datain()
2973 	struct isert_rdma_wr *wr = &isert_cmd->rdma_wr;  in isert_get_dataout()  local
2981 	wr->iser_ib_op = ISER_IB_RDMA_READ;  in isert_get_dataout()
2982 	rc = device->reg_rdma_mem(conn, cmd, wr);  in isert_get_dataout()
2988 	rc = ib_post_send(isert_conn->qp, wr->send_wr, &wr_failed);  in isert_get_dataout()