178 rpcrdma_create_chunks(struct rpc_rqst *rqst, struct xdr_buf *target,  in rpcrdma_create_chunks()  argument
181 	struct rpcrdma_req *req = rpcr_to_rdmar(rqst);  in rpcrdma_create_chunks()
182 	struct rpcrdma_xprt *r_xprt = rpcx_to_rdmax(rqst->rq_xprt);  in rpcrdma_create_chunks()
304 rpcrdma_inline_pullup(struct rpc_rqst *rqst, int pad)  in rpcrdma_inline_pullup()  argument
309 	struct rpcrdma_xprt *r_xprt = rpcx_to_rdmax(rqst->rq_xprt);  in rpcrdma_inline_pullup()
313 	destp = rqst->rq_svec[0].iov_base;  in rpcrdma_inline_pullup()
314 	curlen = rqst->rq_svec[0].iov_len;  in rpcrdma_inline_pullup()
321 	if (pad < 0 || rqst->rq_slen - curlen < RPCRDMA_INLINE_PAD_THRESH)  in rpcrdma_inline_pullup()
325 		__func__, pad, destp, rqst->rq_slen, curlen);  in rpcrdma_inline_pullup()
327 	copy_len = rqst->rq_snd_buf.page_len;  in rpcrdma_inline_pullup()
329 	if (rqst->rq_snd_buf.tail[0].iov_len) {  in rpcrdma_inline_pullup()
330 		curlen = rqst->rq_snd_buf.tail[0].iov_len;  in rpcrdma_inline_pullup()
331 		if (destp + copy_len != rqst->rq_snd_buf.tail[0].iov_base) {  in rpcrdma_inline_pullup()
333 				rqst->rq_snd_buf.tail[0].iov_base, curlen);  in rpcrdma_inline_pullup()
338 		rqst->rq_svec[0].iov_len += curlen;  in rpcrdma_inline_pullup()
342 	page_base = rqst->rq_snd_buf.page_base;  in rpcrdma_inline_pullup()
343 	ppages = rqst->rq_snd_buf.pages + (page_base >> PAGE_SHIFT);  in rpcrdma_inline_pullup()
355 		rqst->rq_svec[0].iov_len += curlen;  in rpcrdma_inline_pullup()
380 rpcrdma_marshal_req(struct rpc_rqst *rqst)  in rpcrdma_marshal_req()  argument
382 	struct rpc_xprt *xprt = rqst->rq_xprt;  in rpcrdma_marshal_req()
384 	struct rpcrdma_req *req = rpcr_to_rdmar(rqst);  in rpcrdma_marshal_req()
395 	base = rqst->rq_svec[0].iov_base;  in rpcrdma_marshal_req()
396 	rpclen = rqst->rq_svec[0].iov_len;  in rpcrdma_marshal_req()
400 	headerp->rm_xid = rqst->rq_xid;  in rpcrdma_marshal_req()
424 	if (rqst->rq_rcv_buf.buflen <= RPCRDMA_INLINE_READ_THRESHOLD(rqst))  in rpcrdma_marshal_req()
426 	else if (rqst->rq_rcv_buf.page_len == 0)  in rpcrdma_marshal_req()
428 	else if (rqst->rq_rcv_buf.flags & XDRBUF_READ)  in rpcrdma_marshal_req()
447 	if (rqst->rq_snd_buf.len <= RPCRDMA_INLINE_WRITE_THRESHOLD(rqst))  in rpcrdma_marshal_req()
449 	else if (rqst->rq_snd_buf.page_len == 0)  in rpcrdma_marshal_req()
473 		padlen = rpcrdma_inline_pullup(rqst,  in rpcrdma_marshal_req()
474 						RPCRDMA_INLINE_PAD_VALUE(rqst));  in rpcrdma_marshal_req()
479 				cpu_to_be32(RPCRDMA_INLINE_PAD_VALUE(rqst));  in rpcrdma_marshal_req()
496 			rpclen = rqst->rq_svec[0].iov_len;  in rpcrdma_marshal_req()
513 		hdrlen = rpcrdma_create_chunks(rqst, &rqst->rq_snd_buf,  in rpcrdma_marshal_req()
518 		hdrlen = rpcrdma_create_chunks(rqst, &rqst->rq_rcv_buf,  in rpcrdma_marshal_req()
554 		req->rl_send_iov[3].length = rqst->rq_slen - rpclen;  in rpcrdma_marshal_req()
611 rpcrdma_inline_fixup(struct rpc_rqst *rqst, char *srcp, int copy_len, int pad)  in rpcrdma_inline_fixup()  argument
618 	curlen = rqst->rq_rcv_buf.head[0].iov_len;  in rpcrdma_inline_fixup()
621 		rqst->rq_rcv_buf.head[0].iov_len = curlen;  in rpcrdma_inline_fixup()
628 	rqst->rq_rcv_buf.head[0].iov_base = srcp;  in rpcrdma_inline_fixup()
634 	rpcx_to_rdmax(rqst->rq_xprt)->rx_stats.fixup_copy_count += olen;  in rpcrdma_inline_fixup()
635 	page_base = rqst->rq_rcv_buf.page_base;  in rpcrdma_inline_fixup()
636 	ppages = rqst->rq_rcv_buf.pages + (page_base >> PAGE_SHIFT);  in rpcrdma_inline_fixup()
639 	if (copy_len && rqst->rq_rcv_buf.page_len) {  in rpcrdma_inline_fixup()
641 			rqst->rq_rcv_buf.page_len) >> PAGE_SHIFT;  in rpcrdma_inline_fixup()
661 	if (copy_len && rqst->rq_rcv_buf.tail[0].iov_len) {  in rpcrdma_inline_fixup()
663 		if (curlen > rqst->rq_rcv_buf.tail[0].iov_len)  in rpcrdma_inline_fixup()
664 			curlen = rqst->rq_rcv_buf.tail[0].iov_len;  in rpcrdma_inline_fixup()
665 		if (rqst->rq_rcv_buf.tail[0].iov_base != srcp)  in rpcrdma_inline_fixup()
666 			memmove(rqst->rq_rcv_buf.tail[0].iov_base, srcp, curlen);  in rpcrdma_inline_fixup()
669 		rqst->rq_rcv_buf.tail[0].iov_len = curlen;  in rpcrdma_inline_fixup()
672 		rqst->rq_rcv_buf.tail[0].iov_len = 0;  in rpcrdma_inline_fixup()
676 		unsigned char *p = rqst->rq_rcv_buf.tail[0].iov_base;  in rpcrdma_inline_fixup()
678 			p[rqst->rq_rcv_buf.tail[0].iov_len++] = 0;  in rpcrdma_inline_fixup()
687 	rqst->rq_private_buf = rqst->rq_rcv_buf;  in rpcrdma_inline_fixup()
734 	struct rpc_rqst *rqst;  in rpcrdma_reply_handler()  local
764 	rqst = xprt_lookup_rqst(xprt, headerp->rm_xid);  in rpcrdma_reply_handler()
765 	if (rqst == NULL) {  in rpcrdma_reply_handler()
781 	req = rpcr_to_rdmar(rqst);  in rpcrdma_reply_handler()
792 			__func__, rep, req, rqst,  in rpcrdma_reply_handler()
839 		rpcrdma_inline_fixup(rqst, (char *)iptr, rep->rr_len, rdmalen);  in rpcrdma_reply_handler()
883 		xprt_release_rqst_cong(rqst->rq_task);  in rpcrdma_reply_handler()
886 			__func__, xprt, rqst, status);  in rpcrdma_reply_handler()
887 	xprt_complete_rqst(rqst->rq_task, status);  in rpcrdma_reply_handler()