183 						struct rds_ib_send_work *send,  in rds_ib_send_unmap_op()  argument
189 	switch (send->s_wr.opcode) {  in rds_ib_send_unmap_op()
191 		if (send->s_op) {  in rds_ib_send_unmap_op()
192 			rm = container_of(send->s_op, struct rds_message, data);  in rds_ib_send_unmap_op()
193 			rds_ib_send_unmap_data(ic, send->s_op, wc_status);  in rds_ib_send_unmap_op()
198 		if (send->s_op) {  in rds_ib_send_unmap_op()
199 			rm = container_of(send->s_op, struct rds_message, rdma);  in rds_ib_send_unmap_op()
200 			rds_ib_send_unmap_rdma(ic, send->s_op, wc_status);  in rds_ib_send_unmap_op()
205 		if (send->s_op) {  in rds_ib_send_unmap_op()
206 			rm = container_of(send->s_op, struct rds_message, atomic);  in rds_ib_send_unmap_op()
207 			rds_ib_send_unmap_atomic(ic, send->s_op, wc_status);  in rds_ib_send_unmap_op()
213 			       __func__, send->s_wr.opcode);  in rds_ib_send_unmap_op()
217 	send->s_wr.opcode = 0xdead;  in rds_ib_send_unmap_op()
224 	struct rds_ib_send_work *send;  in rds_ib_send_init_ring()  local
227 	for (i = 0, send = ic->i_sends; i < ic->i_send_ring.w_nr; i++, send++) {  in rds_ib_send_init_ring()
230 		send->s_op = NULL;  in rds_ib_send_init_ring()
232 		send->s_wr.wr_id = i;  in rds_ib_send_init_ring()
233 		send->s_wr.sg_list = send->s_sge;  in rds_ib_send_init_ring()
234 		send->s_wr.ex.imm_data = 0;  in rds_ib_send_init_ring()
236 		sge = &send->s_sge[0];  in rds_ib_send_init_ring()
241 		send->s_sge[1].lkey = ic->i_mr->lkey;  in rds_ib_send_init_ring()
247 	struct rds_ib_send_work *send;  in rds_ib_send_clear_ring()  local
250 	for (i = 0, send = ic->i_sends; i < ic->i_send_ring.w_nr; i++, send++) {  in rds_ib_send_clear_ring()
251 		if (send->s_op && send->s_wr.opcode != 0xdead)  in rds_ib_send_clear_ring()
252 			rds_ib_send_unmap_op(ic, send, IB_WC_WR_FLUSH_ERR);  in rds_ib_send_clear_ring()
280 	struct rds_ib_send_work *send;  in rds_ib_send_cq_comp_handler()  local
312 			send = &ic->i_sends[oldest];  in rds_ib_send_cq_comp_handler()
313 			if (send->s_wr.send_flags & IB_SEND_SIGNALED)  in rds_ib_send_cq_comp_handler()
316 			rm = rds_ib_send_unmap_op(ic, send, wc.status);  in rds_ib_send_cq_comp_handler()
318 			if (time_after(jiffies, send->s_queued + HZ/2))  in rds_ib_send_cq_comp_handler()
321 			if (send->s_op) {  in rds_ib_send_cq_comp_handler()
322 				if (send->s_op == rm->m_final_op) {  in rds_ib_send_cq_comp_handler()
328 				send->s_op = NULL;  in rds_ib_send_cq_comp_handler()
496 					     struct rds_ib_send_work *send,  in rds_ib_set_wr_signal_state()  argument
506 		send->s_wr.send_flags |= IB_SEND_SIGNALED;  in rds_ib_set_wr_signal_state()
530 	struct rds_ib_send_work *send = NULL;  in rds_ib_xmit()  local
658 	send = &ic->i_sends[pos];  in rds_ib_xmit()
659 	first = send;  in rds_ib_xmit()
667 		send->s_wr.send_flags = send_flags;  in rds_ib_xmit()
668 		send->s_wr.opcode = IB_WR_SEND;  in rds_ib_xmit()
669 		send->s_wr.num_sge = 1;  in rds_ib_xmit()
670 		send->s_wr.next = NULL;  in rds_ib_xmit()
671 		send->s_queued = jiffies;  in rds_ib_xmit()
672 		send->s_op = NULL;  in rds_ib_xmit()
674 		send->s_sge[0].addr = ic->i_send_hdrs_dma  in rds_ib_xmit()
676 		send->s_sge[0].length = sizeof(struct rds_header);  in rds_ib_xmit()
684 			send->s_wr.num_sge = 2;  in rds_ib_xmit()
686 			send->s_sge[1].addr = ib_sg_dma_address(dev, scat) + off;  in rds_ib_xmit()
687 			send->s_sge[1].length = len;  in rds_ib_xmit()
697 		rds_ib_set_wr_signal_state(ic, send, 0);  in rds_ib_xmit()
703 			send->s_wr.send_flags |= IB_SEND_SIGNALED | IB_SEND_SOLICITED;  in rds_ib_xmit()
705 		if (send->s_wr.send_flags & IB_SEND_SIGNALED)  in rds_ib_xmit()
708 		rdsdebug("send %p wr %p num_sge %u next %p\n", send,  in rds_ib_xmit()
709 			 &send->s_wr, send->s_wr.num_sge, send->s_wr.next);  in rds_ib_xmit()
722 			prev->s_wr.next = &send->s_wr;  in rds_ib_xmit()
723 		prev = send;  in rds_ib_xmit()
726 		send = &ic->i_sends[pos];  in rds_ib_xmit()
789 	struct rds_ib_send_work *send = NULL;  in rds_ib_xmit_atomic()  local
808 	send = &ic->i_sends[pos];  in rds_ib_xmit_atomic()
809 	send->s_queued = jiffies;  in rds_ib_xmit_atomic()
812 		send->s_wr.opcode = IB_WR_MASKED_ATOMIC_CMP_AND_SWP;  in rds_ib_xmit_atomic()
813 		send->s_wr.wr.atomic.compare_add = op->op_m_cswp.compare;  in rds_ib_xmit_atomic()
814 		send->s_wr.wr.atomic.swap = op->op_m_cswp.swap;  in rds_ib_xmit_atomic()
815 		send->s_wr.wr.atomic.compare_add_mask = op->op_m_cswp.compare_mask;  in rds_ib_xmit_atomic()
816 		send->s_wr.wr.atomic.swap_mask = op->op_m_cswp.swap_mask;  in rds_ib_xmit_atomic()
818 		send->s_wr.opcode = IB_WR_MASKED_ATOMIC_FETCH_AND_ADD;  in rds_ib_xmit_atomic()
819 		send->s_wr.wr.atomic.compare_add = op->op_m_fadd.add;  in rds_ib_xmit_atomic()
820 		send->s_wr.wr.atomic.swap = 0;  in rds_ib_xmit_atomic()
821 		send->s_wr.wr.atomic.compare_add_mask = op->op_m_fadd.nocarry_mask;  in rds_ib_xmit_atomic()
822 		send->s_wr.wr.atomic.swap_mask = 0;  in rds_ib_xmit_atomic()
824 	nr_sig = rds_ib_set_wr_signal_state(ic, send, op->op_notify);  in rds_ib_xmit_atomic()
825 	send->s_wr.num_sge = 1;  in rds_ib_xmit_atomic()
826 	send->s_wr.next = NULL;  in rds_ib_xmit_atomic()
827 	send->s_wr.wr.atomic.remote_addr = op->op_remote_addr;  in rds_ib_xmit_atomic()
828 	send->s_wr.wr.atomic.rkey = op->op_rkey;  in rds_ib_xmit_atomic()
829 	send->s_op = op;  in rds_ib_xmit_atomic()
830 	rds_message_addref(container_of(send->s_op, struct rds_message, atomic));  in rds_ib_xmit_atomic()
843 	send->s_sge[0].addr = ib_sg_dma_address(ic->i_cm_id->device, op->op_sg);  in rds_ib_xmit_atomic()
844 	send->s_sge[0].length = ib_sg_dma_len(ic->i_cm_id->device, op->op_sg);  in rds_ib_xmit_atomic()
845 	send->s_sge[0].lkey = ic->i_mr->lkey;  in rds_ib_xmit_atomic()
848 		 send->s_sge[0].addr, send->s_sge[0].length);  in rds_ib_xmit_atomic()
853 	failed_wr = &send->s_wr;  in rds_ib_xmit_atomic()
854 	ret = ib_post_send(ic->i_cm_id->qp, &send->s_wr, &failed_wr);  in rds_ib_xmit_atomic()
856 		 send, &send->s_wr, ret, failed_wr);  in rds_ib_xmit_atomic()
857 	BUG_ON(failed_wr != &send->s_wr);  in rds_ib_xmit_atomic()
866 	if (unlikely(failed_wr != &send->s_wr)) {  in rds_ib_xmit_atomic()
868 		BUG_ON(failed_wr != &send->s_wr);  in rds_ib_xmit_atomic()
878 	struct rds_ib_send_work *send = NULL;  in rds_ib_xmit_rdma()  local
924 	send = &ic->i_sends[pos];  in rds_ib_xmit_rdma()
925 	first = send;  in rds_ib_xmit_rdma()
932 		send->s_wr.send_flags = 0;  in rds_ib_xmit_rdma()
933 		send->s_queued = jiffies;  in rds_ib_xmit_rdma()
934 		send->s_op = NULL;  in rds_ib_xmit_rdma()
936 		nr_sig += rds_ib_set_wr_signal_state(ic, send, op->op_notify);  in rds_ib_xmit_rdma()
938 		send->s_wr.opcode = op->op_write ? IB_WR_RDMA_WRITE : IB_WR_RDMA_READ;  in rds_ib_xmit_rdma()
939 		send->s_wr.wr.rdma.remote_addr = remote_addr;  in rds_ib_xmit_rdma()
940 		send->s_wr.wr.rdma.rkey = op->op_rkey;  in rds_ib_xmit_rdma()
943 			send->s_wr.num_sge = max_sge;  in rds_ib_xmit_rdma()
946 			send->s_wr.num_sge = num_sge;  in rds_ib_xmit_rdma()
949 		send->s_wr.next = NULL;  in rds_ib_xmit_rdma()
952 			prev->s_wr.next = &send->s_wr;  in rds_ib_xmit_rdma()
954 		for (j = 0; j < send->s_wr.num_sge && scat != &op->op_sg[op->op_count]; j++) {  in rds_ib_xmit_rdma()
956 			send->s_sge[j].addr =  in rds_ib_xmit_rdma()
958 			send->s_sge[j].length = len;  in rds_ib_xmit_rdma()
959 			send->s_sge[j].lkey = ic->i_mr->lkey;  in rds_ib_xmit_rdma()
968 		rdsdebug("send %p wr %p num_sge %u next %p\n", send,  in rds_ib_xmit_rdma()
969 			&send->s_wr, send->s_wr.num_sge, send->s_wr.next);  in rds_ib_xmit_rdma()
971 		prev = send;  in rds_ib_xmit_rdma()
972 		if (++send == &ic->i_sends[ic->i_send_ring.w_nr])  in rds_ib_xmit_rdma()
973 			send = ic->i_sends;  in rds_ib_xmit_rdma()