79 static bool rpcrdma_args_inline(struct rpc_rqst *rqst)  in rpcrdma_args_inline()  argument
81 	unsigned int callsize = RPCRDMA_HDRLEN_MIN + rqst->rq_snd_buf.len;  in rpcrdma_args_inline()
83 	return callsize <= RPCRDMA_INLINE_WRITE_THRESHOLD(rqst);  in rpcrdma_args_inline()
92 static bool rpcrdma_results_inline(struct rpc_rqst *rqst)  in rpcrdma_results_inline()  argument
94 	unsigned int repsize = RPCRDMA_HDRLEN_MIN + rqst->rq_rcv_buf.buflen;  in rpcrdma_results_inline()
96 	return repsize <= RPCRDMA_INLINE_READ_THRESHOLD(rqst);  in rpcrdma_results_inline()
243 rpcrdma_create_chunks(struct rpc_rqst *rqst, struct xdr_buf *target,  in rpcrdma_create_chunks()  argument
246 	struct rpcrdma_req *req = rpcr_to_rdmar(rqst);  in rpcrdma_create_chunks()
247 	struct rpcrdma_xprt *r_xprt = rpcx_to_rdmax(rqst->rq_xprt);  in rpcrdma_create_chunks()
365 static void rpcrdma_inline_pullup(struct rpc_rqst *rqst)  in rpcrdma_inline_pullup()  argument
370 	struct rpcrdma_xprt *r_xprt = rpcx_to_rdmax(rqst->rq_xprt);  in rpcrdma_inline_pullup()
374 	destp = rqst->rq_svec[0].iov_base;  in rpcrdma_inline_pullup()
375 	curlen = rqst->rq_svec[0].iov_len;  in rpcrdma_inline_pullup()
379 		__func__, destp, rqst->rq_slen, curlen);  in rpcrdma_inline_pullup()
381 	copy_len = rqst->rq_snd_buf.page_len;  in rpcrdma_inline_pullup()
383 	if (rqst->rq_snd_buf.tail[0].iov_len) {  in rpcrdma_inline_pullup()
384 		curlen = rqst->rq_snd_buf.tail[0].iov_len;  in rpcrdma_inline_pullup()
385 		if (destp + copy_len != rqst->rq_snd_buf.tail[0].iov_base) {  in rpcrdma_inline_pullup()
387 				rqst->rq_snd_buf.tail[0].iov_base, curlen);  in rpcrdma_inline_pullup()
392 		rqst->rq_svec[0].iov_len += curlen;  in rpcrdma_inline_pullup()
396 	page_base = rqst->rq_snd_buf.page_base;  in rpcrdma_inline_pullup()
397 	ppages = rqst->rq_snd_buf.pages + (page_base >> PAGE_SHIFT);  in rpcrdma_inline_pullup()
409 		rqst->rq_svec[0].iov_len += curlen;  in rpcrdma_inline_pullup()
433 rpcrdma_marshal_req(struct rpc_rqst *rqst)  in rpcrdma_marshal_req()  argument
435 	struct rpc_xprt *xprt = rqst->rq_xprt;  in rpcrdma_marshal_req()
437 	struct rpcrdma_req *req = rpcr_to_rdmar(rqst);  in rpcrdma_marshal_req()
445 	if (test_bit(RPC_BC_PA_IN_USE, &rqst->rq_bc_pa_state))  in rpcrdma_marshal_req()
446 		return rpcrdma_bc_marshal_reply(rqst);  in rpcrdma_marshal_req()
453 	base = rqst->rq_svec[0].iov_base;  in rpcrdma_marshal_req()
454 	rpclen = rqst->rq_svec[0].iov_len;  in rpcrdma_marshal_req()
458 	headerp->rm_xid = rqst->rq_xid;  in rpcrdma_marshal_req()
471 	if (rqst->rq_rcv_buf.flags & XDRBUF_READ)  in rpcrdma_marshal_req()
473 	else if (rpcrdma_results_inline(rqst))  in rpcrdma_marshal_req()
492 	if (rpcrdma_args_inline(rqst)) {  in rpcrdma_marshal_req()
494 	} else if (rqst->rq_snd_buf.flags & XDRBUF_WRITE) {  in rpcrdma_marshal_req()
521 		rpcrdma_inline_pullup(rqst);  in rpcrdma_marshal_req()
527 		rpclen = rqst->rq_svec[0].iov_len;  in rpcrdma_marshal_req()
529 		rpclen += rpcrdma_tail_pullup(&rqst->rq_snd_buf);  in rpcrdma_marshal_req()
531 		hdrlen = rpcrdma_create_chunks(rqst, &rqst->rq_snd_buf,  in rpcrdma_marshal_req()
536 		hdrlen = rpcrdma_create_chunks(rqst, &rqst->rq_rcv_buf,  in rpcrdma_marshal_req()
618 rpcrdma_inline_fixup(struct rpc_rqst *rqst, char *srcp, int copy_len, int pad)  in rpcrdma_inline_fixup()  argument
625 	curlen = rqst->rq_rcv_buf.head[0].iov_len;  in rpcrdma_inline_fixup()
628 		rqst->rq_rcv_buf.head[0].iov_len = curlen;  in rpcrdma_inline_fixup()
635 	rqst->rq_rcv_buf.head[0].iov_base = srcp;  in rpcrdma_inline_fixup()
641 	rpcx_to_rdmax(rqst->rq_xprt)->rx_stats.fixup_copy_count += olen;  in rpcrdma_inline_fixup()
642 	page_base = rqst->rq_rcv_buf.page_base;  in rpcrdma_inline_fixup()
643 	ppages = rqst->rq_rcv_buf.pages + (page_base >> PAGE_SHIFT);  in rpcrdma_inline_fixup()
646 	if (copy_len && rqst->rq_rcv_buf.page_len) {  in rpcrdma_inline_fixup()
648 			rqst->rq_rcv_buf.page_len) >> PAGE_SHIFT;  in rpcrdma_inline_fixup()
668 	if (copy_len && rqst->rq_rcv_buf.tail[0].iov_len) {  in rpcrdma_inline_fixup()
670 		if (curlen > rqst->rq_rcv_buf.tail[0].iov_len)  in rpcrdma_inline_fixup()
671 			curlen = rqst->rq_rcv_buf.tail[0].iov_len;  in rpcrdma_inline_fixup()
672 		if (rqst->rq_rcv_buf.tail[0].iov_base != srcp)  in rpcrdma_inline_fixup()
673 			memmove(rqst->rq_rcv_buf.tail[0].iov_base, srcp, curlen);  in rpcrdma_inline_fixup()
676 		rqst->rq_rcv_buf.tail[0].iov_len = curlen;  in rpcrdma_inline_fixup()
679 		rqst->rq_rcv_buf.tail[0].iov_len = 0;  in rpcrdma_inline_fixup()
683 		unsigned char *p = rqst->rq_rcv_buf.tail[0].iov_base;  in rpcrdma_inline_fixup()
685 			p[rqst->rq_rcv_buf.tail[0].iov_len++] = 0;  in rpcrdma_inline_fixup()
694 	rqst->rq_private_buf = rqst->rq_rcv_buf;  in rpcrdma_inline_fixup()
772 	struct rpc_rqst *rqst;  in rpcrdma_reply_handler()  local
799 	rqst = xprt_lookup_rqst(xprt, headerp->rm_xid);  in rpcrdma_reply_handler()
800 	if (!rqst)  in rpcrdma_reply_handler()
803 	req = rpcr_to_rdmar(rqst);  in rpcrdma_reply_handler()
809 			__func__, rep, req, rqst,  in rpcrdma_reply_handler()
856 		rpcrdma_inline_fixup(rqst, (char *)iptr, rep->rr_len, rdmalen);  in rpcrdma_reply_handler()
900 		xprt_release_rqst_cong(rqst->rq_task);  in rpcrdma_reply_handler()
902 	xprt_complete_rqst(rqst->rq_task, status);  in rpcrdma_reply_handler()
905 			__func__, xprt, rqst, status);  in rpcrdma_reply_handler()