1926 				struct ib_send_wr *wr)  in ocrdma_build_ud_hdr()  argument
1930 	struct ocrdma_ah *ah = get_ocrdma_ah(wr->wr.ud.ah);  in ocrdma_build_ud_hdr()
1932 	ud_hdr->rsvd_dest_qpn = wr->wr.ud.remote_qpn;  in ocrdma_build_ud_hdr()
1936 		ud_hdr->qkey = wr->wr.ud.remote_qkey;  in ocrdma_build_ud_hdr()
1972 				    struct ib_send_wr *wr, u32 wqe_size)  in ocrdma_build_inline_sges()  argument
1977 	if (wr->send_flags & IB_SEND_INLINE && qp->qp_type != IB_QPT_UD) {  in ocrdma_build_inline_sges()
1978 		hdr->total_len = ocrdma_sglist_len(wr->sg_list, wr->num_sge);  in ocrdma_build_inline_sges()
1986 		for (i = 0; i < wr->num_sge; i++) {  in ocrdma_build_inline_sges()
1988 			       (void *)(unsigned long)wr->sg_list[i].addr,  in ocrdma_build_inline_sges()
1989 			       wr->sg_list[i].length);  in ocrdma_build_inline_sges()
1990 			dpp_addr += wr->sg_list[i].length;  in ocrdma_build_inline_sges()
1998 		ocrdma_build_sges(hdr, sge, wr->num_sge, wr->sg_list);  in ocrdma_build_inline_sges()
1999 		if (wr->num_sge)  in ocrdma_build_inline_sges()
2000 			wqe_size += (wr->num_sge * sizeof(struct ocrdma_sge));  in ocrdma_build_inline_sges()
2010 			     struct ib_send_wr *wr)  in ocrdma_build_send()  argument
2017 		ocrdma_build_ud_hdr(qp, hdr, wr);  in ocrdma_build_send()
2024 	status = ocrdma_build_inline_sges(qp, hdr, sge, wr, wqe_size);  in ocrdma_build_send()
2029 			      struct ib_send_wr *wr)  in ocrdma_build_write()  argument
2036 	status = ocrdma_build_inline_sges(qp, hdr, sge, wr, wqe_size);  in ocrdma_build_write()
2039 	ext_rw->addr_lo = wr->wr.rdma.remote_addr;  in ocrdma_build_write()
2040 	ext_rw->addr_hi = upper_32_bits(wr->wr.rdma.remote_addr);  in ocrdma_build_write()
2041 	ext_rw->lrkey = wr->wr.rdma.rkey;  in ocrdma_build_write()
2047 			      struct ib_send_wr *wr)  in ocrdma_build_read()  argument
2051 	u32 wqe_size = ((wr->num_sge + 1) * sizeof(struct ocrdma_sge)) +  in ocrdma_build_read()
2054 	ocrdma_build_sges(hdr, sge, wr->num_sge, wr->sg_list);  in ocrdma_build_read()
2059 	ext_rw->addr_lo = wr->wr.rdma.remote_addr;  in ocrdma_build_read()
2060 	ext_rw->addr_hi = upper_32_bits(wr->wr.rdma.remote_addr);  in ocrdma_build_read()
2061 	ext_rw->lrkey = wr->wr.rdma.rkey;  in ocrdma_build_read()
2065 static void build_frmr_pbes(struct ib_send_wr *wr, struct ocrdma_pbl *pbl_tbl,  in build_frmr_pbes()  argument
2077 	for (i = 0; i < wr->wr.fast_reg.page_list_len; i++) {  in build_frmr_pbes()
2082 		buf_addr = wr->wr.fast_reg.page_list->page_list[i];  in build_frmr_pbes()
2111 			   struct ib_send_wr *wr)  in ocrdma_build_fr()  argument
2121 	if (wr->wr.fast_reg.page_list_len > dev->attr.max_pages_per_frmr)  in ocrdma_build_fr()
2127 	if (wr->wr.fast_reg.page_list_len == 0)  in ocrdma_build_fr()
2129 	if (wr->wr.fast_reg.access_flags & IB_ACCESS_LOCAL_WRITE)  in ocrdma_build_fr()
2131 	if (wr->wr.fast_reg.access_flags & IB_ACCESS_REMOTE_WRITE)  in ocrdma_build_fr()
2133 	if (wr->wr.fast_reg.access_flags & IB_ACCESS_REMOTE_READ)  in ocrdma_build_fr()
2135 	hdr->lkey = wr->wr.fast_reg.rkey;  in ocrdma_build_fr()
2136 	hdr->total_len = wr->wr.fast_reg.length;  in ocrdma_build_fr()
2138 	fbo = wr->wr.fast_reg.iova_start -  in ocrdma_build_fr()
2139 	    (wr->wr.fast_reg.page_list->page_list[0] & PAGE_MASK);  in ocrdma_build_fr()
2141 	fast_reg->va_hi = upper_32_bits(wr->wr.fast_reg.iova_start);  in ocrdma_build_fr()
2142 	fast_reg->va_lo = (u32) (wr->wr.fast_reg.iova_start & 0xffffffff);  in ocrdma_build_fr()
2145 	fast_reg->num_sges = wr->wr.fast_reg.page_list_len;  in ocrdma_build_fr()
2147 		get_encoded_page_size(1 << wr->wr.fast_reg.page_shift);  in ocrdma_build_fr()
2150 	build_frmr_pbes(wr, mr->hwmr.pbl_table, &mr->hwmr);  in ocrdma_build_fr()
2161 int ocrdma_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr,  in ocrdma_post_send()  argument
2172 		*bad_wr = wr;  in ocrdma_post_send()
2176 	while (wr) {  in ocrdma_post_send()
2178 		    (wr->opcode != IB_WR_SEND &&  in ocrdma_post_send()
2179 		     wr->opcode != IB_WR_SEND_WITH_IMM)) {  in ocrdma_post_send()
2180 			*bad_wr = wr;  in ocrdma_post_send()
2185 		    wr->num_sge > qp->sq.max_sges) {  in ocrdma_post_send()
2186 			*bad_wr = wr;  in ocrdma_post_send()
2192 		if (wr->send_flags & IB_SEND_SIGNALED || qp->signaled)  in ocrdma_post_send()
2194 		if (wr->send_flags & IB_SEND_FENCE)  in ocrdma_post_send()
2197 		if (wr->send_flags & IB_SEND_SOLICITED)  in ocrdma_post_send()
2201 		switch (wr->opcode) {  in ocrdma_post_send()
2204 			hdr->immdt = ntohl(wr->ex.imm_data);  in ocrdma_post_send()
2207 			ocrdma_build_send(qp, hdr, wr);  in ocrdma_post_send()
2212 			hdr->lkey = wr->ex.invalidate_rkey;  in ocrdma_post_send()
2213 			status = ocrdma_build_send(qp, hdr, wr);  in ocrdma_post_send()
2217 			hdr->immdt = ntohl(wr->ex.imm_data);  in ocrdma_post_send()
2220 			status = ocrdma_build_write(qp, hdr, wr);  in ocrdma_post_send()
2223 			ocrdma_build_read(qp, hdr, wr);  in ocrdma_post_send()
2231 			hdr->lkey = wr->ex.invalidate_rkey;  in ocrdma_post_send()
2234 			status = ocrdma_build_fr(qp, hdr, wr);  in ocrdma_post_send()
2241 			*bad_wr = wr;  in ocrdma_post_send()
2244 		if (wr->send_flags & IB_SEND_SIGNALED || qp->signaled)  in ocrdma_post_send()
2248 		qp->wqe_wr_id_tbl[qp->sq.head].wrid = wr->wr_id;  in ocrdma_post_send()
2258 		wr = wr->next;  in ocrdma_post_send()
2271 static void ocrdma_build_rqe(struct ocrdma_hdr_wqe *rqe, struct ib_recv_wr *wr,  in ocrdma_build_rqe()  argument
2276 	if (wr->num_sge)  in ocrdma_build_rqe()
2277 		wqe_size = (wr->num_sge * sizeof(*sge)) + sizeof(*rqe);  in ocrdma_build_rqe()
2288 	ocrdma_build_sges(rqe, sge, wr->num_sge, wr->sg_list);  in ocrdma_build_rqe()
2292 int ocrdma_post_recv(struct ib_qp *ibqp, struct ib_recv_wr *wr,  in ocrdma_post_recv()  argument
2303 		*bad_wr = wr;  in ocrdma_post_recv()
2306 	while (wr) {  in ocrdma_post_recv()
2308 		    wr->num_sge > qp->rq.max_sges) {  in ocrdma_post_recv()
2309 			*bad_wr = wr;  in ocrdma_post_recv()
2314 		ocrdma_build_rqe(rqe, wr, 0);  in ocrdma_post_recv()
2316 		qp->rqe_wr_id_tbl[qp->rq.head] = wr->wr_id;  in ocrdma_post_recv()
2325 		wr = wr->next;  in ocrdma_post_recv()
2364 int ocrdma_post_srq_recv(struct ib_srq *ibsrq, struct ib_recv_wr *wr,  in ocrdma_post_srq_recv()  argument
2376 	while (wr) {  in ocrdma_post_srq_recv()
2378 		    wr->num_sge > srq->rq.max_sges) {  in ocrdma_post_srq_recv()
2380 			*bad_wr = wr;  in ocrdma_post_srq_recv()
2385 		ocrdma_build_rqe(rqe, wr, tag);  in ocrdma_post_srq_recv()
2387 		srq->rqe_wr_id_tbl[tag] = wr->wr_id;  in ocrdma_post_srq_recv()
2394 		wr = wr->next;  in ocrdma_post_srq_recv()