58 static void rdma_build_arg_xdr(struct svc_rqst *rqstp,  in rdma_build_arg_xdr()  argument
69 	put_page(rqstp->rq_pages[0]);  in rdma_build_arg_xdr()
70 	rqstp->rq_pages[0] = page;  in rdma_build_arg_xdr()
73 	rqstp->rq_arg.head[0].iov_base = page_address(page);  in rdma_build_arg_xdr()
74 	rqstp->rq_arg.head[0].iov_len =  in rdma_build_arg_xdr()
76 	rqstp->rq_arg.len = byte_count;  in rdma_build_arg_xdr()
77 	rqstp->rq_arg.buflen = byte_count;  in rdma_build_arg_xdr()
80 	bc = byte_count - rqstp->rq_arg.head[0].iov_len;  in rdma_build_arg_xdr()
83 	rqstp->rq_arg.page_len = bc;  in rdma_build_arg_xdr()
84 	rqstp->rq_arg.page_base = 0;  in rdma_build_arg_xdr()
87 	rmsgp = (struct rpcrdma_msg *)rqstp->rq_arg.head[0].iov_base;  in rdma_build_arg_xdr()
89 		rqstp->rq_arg.pages = &rqstp->rq_pages[0];  in rdma_build_arg_xdr()
91 		rqstp->rq_arg.pages = &rqstp->rq_pages[1];  in rdma_build_arg_xdr()
96 		put_page(rqstp->rq_pages[sge_no]);  in rdma_build_arg_xdr()
97 		rqstp->rq_pages[sge_no] = page;  in rdma_build_arg_xdr()
99 		rqstp->rq_arg.buflen += ctxt->sge[sge_no].length;  in rdma_build_arg_xdr()
102 	rqstp->rq_respages = &rqstp->rq_pages[sge_no];  in rdma_build_arg_xdr()
103 	rqstp->rq_next_page = rqstp->rq_respages + 1;  in rdma_build_arg_xdr()
114 	rqstp->rq_arg.tail[0].iov_base = NULL;  in rdma_build_arg_xdr()
115 	rqstp->rq_arg.tail[0].iov_len = 0;  in rdma_build_arg_xdr()
129 			struct svc_rqst *rqstp,  in rdma_read_chunk_lcl()  argument
155 		head->arg.pages[pg_no] = rqstp->rq_arg.pages[pg_no];  in rdma_read_chunk_lcl()
160 		rqstp->rq_respages = &rqstp->rq_arg.pages[pg_no+1];  in rdma_read_chunk_lcl()
161 		rqstp->rq_next_page = rqstp->rq_respages + 1;  in rdma_read_chunk_lcl()
223 			 struct svc_rqst *rqstp,  in rdma_read_chunk_frmr()  argument
252 	frmr->kva = page_address(rqstp->rq_arg.pages[pg_no]);  in rdma_read_chunk_frmr()
261 		head->arg.pages[pg_no] = rqstp->rq_arg.pages[pg_no];  in rdma_read_chunk_frmr()
266 		rqstp->rq_respages = &rqstp->rq_arg.pages[pg_no+1];  in rdma_read_chunk_frmr()
267 		rqstp->rq_next_page = rqstp->rq_respages + 1;  in rdma_read_chunk_frmr()
374 rdma_copy_tail(struct svc_rqst *rqstp, struct svc_rdma_op_ctxt *head,  in rdma_copy_tail()  argument
390 		destp = page_address(rqstp->rq_arg.pages[page_no]);  in rdma_copy_tail()
404 	destp = page_address(rqstp->rq_arg.pages[page_no]);  in rdma_copy_tail()
408 	rqstp->rq_respages = &rqstp->rq_arg.pages[page_no+1];  in rdma_copy_tail()
409 	rqstp->rq_next_page = rqstp->rq_respages + 1;  in rdma_copy_tail()
421 			    struct svc_rqst *rqstp,  in rdma_read_chunks()  argument
443 	head->arg.head[0] = rqstp->rq_arg.head[0];  in rdma_read_chunks()
444 	head->arg.tail[0] = rqstp->rq_arg.tail[0];  in rdma_read_chunks()
448 	head->arg.len = rqstp->rq_arg.len;  in rdma_read_chunks()
449 	head->arg.buflen = rqstp->rq_arg.buflen;  in rdma_read_chunks()
476 			ret = xprt->sc_reader(xprt, rqstp, head,  in rdma_read_chunks()
500 		ret = rdma_copy_tail(rqstp, head, position,  in rdma_read_chunks()
508 	     &rqstp->rq_pages[page_no] < rqstp->rq_respages; page_no++)  in rdma_read_chunks()
509 		rqstp->rq_pages[page_no] = NULL;  in rdma_read_chunks()
514 static int rdma_read_complete(struct svc_rqst *rqstp,  in rdma_read_complete()  argument
522 		put_page(rqstp->rq_pages[page_no]);  in rdma_read_complete()
523 		rqstp->rq_pages[page_no] = head->pages[page_no];  in rdma_read_complete()
541 	rqstp->rq_arg.pages = &rqstp->rq_pages[head->hdr_count];  in rdma_read_complete()
542 	rqstp->rq_arg.page_len = head->arg.page_len;  in rdma_read_complete()
543 	rqstp->rq_arg.page_base = head->arg.page_base;  in rdma_read_complete()
546 	rqstp->rq_respages = &rqstp->rq_arg.pages[page_no];  in rdma_read_complete()
547 	rqstp->rq_next_page = rqstp->rq_respages + 1;  in rdma_read_complete()
550 	rqstp->rq_arg.head[0] = head->arg.head[0];  in rdma_read_complete()
551 	rqstp->rq_arg.tail[0] = head->arg.tail[0];  in rdma_read_complete()
552 	rqstp->rq_arg.len = head->arg.len;  in rdma_read_complete()
553 	rqstp->rq_arg.buflen = head->arg.buflen;  in rdma_read_complete()
559 	rqstp->rq_prot = IPPROTO_MAX;  in rdma_read_complete()
560 	svc_xprt_copy_addrs(rqstp, rqstp->rq_xprt);  in rdma_read_complete()
562 	ret = rqstp->rq_arg.head[0].iov_len  in rdma_read_complete()
563 		+ rqstp->rq_arg.page_len  in rdma_read_complete()
564 		+ rqstp->rq_arg.tail[0].iov_len;  in rdma_read_complete()
567 		ret, rqstp->rq_arg.len,	rqstp->rq_arg.head[0].iov_base,  in rdma_read_complete()
568 		rqstp->rq_arg.head[0].iov_len);  in rdma_read_complete()
578 int svc_rdma_recvfrom(struct svc_rqst *rqstp)  in svc_rdma_recvfrom()  argument
580 	struct svc_xprt *xprt = rqstp->rq_xprt;  in svc_rdma_recvfrom()
588 	dprintk("svcrdma: rqstp=%p\n", rqstp);  in svc_rdma_recvfrom()
597 		return rdma_read_complete(rqstp, ctxt);  in svc_rdma_recvfrom()
621 		ctxt, rdma_xprt, rqstp, ctxt->wc_status);  in svc_rdma_recvfrom()
625 	rdma_build_arg_xdr(rqstp, ctxt, ctxt->byte_len);  in svc_rdma_recvfrom()
628 	len = svc_rdma_xdr_decode_req(&rmsgp, rqstp);  in svc_rdma_recvfrom()
629 	rqstp->rq_xprt_hlen = len;  in svc_rdma_recvfrom()
639 	ret = rdma_read_chunks(rdma_xprt, rmsgp, rqstp, ctxt);  in svc_rdma_recvfrom()
649 	ret = rqstp->rq_arg.head[0].iov_len  in svc_rdma_recvfrom()
650 		+ rqstp->rq_arg.page_len  in svc_rdma_recvfrom()
651 		+ rqstp->rq_arg.tail[0].iov_len;  in svc_rdma_recvfrom()
656 		ret, rqstp->rq_arg.len,  in svc_rdma_recvfrom()
657 		rqstp->rq_arg.head[0].iov_base,  in svc_rdma_recvfrom()
658 		rqstp->rq_arg.head[0].iov_len);  in svc_rdma_recvfrom()
659 	rqstp->rq_prot = IPPROTO_MAX;  in svc_rdma_recvfrom()
660 	svc_xprt_copy_addrs(rqstp, xprt);  in svc_rdma_recvfrom()