778 static int ocrdma_alloc_lkey(struct ocrdma_dev *dev, struct ocrdma_mr *mr,  in ocrdma_alloc_lkey()  argument
783 	mr->hwmr.fr_mr = 0;  in ocrdma_alloc_lkey()
784 	mr->hwmr.local_rd = 1;  in ocrdma_alloc_lkey()
785 	mr->hwmr.remote_rd = (acc & IB_ACCESS_REMOTE_READ) ? 1 : 0;  in ocrdma_alloc_lkey()
786 	mr->hwmr.remote_wr = (acc & IB_ACCESS_REMOTE_WRITE) ? 1 : 0;  in ocrdma_alloc_lkey()
787 	mr->hwmr.local_wr = (acc & IB_ACCESS_LOCAL_WRITE) ? 1 : 0;  in ocrdma_alloc_lkey()
788 	mr->hwmr.mw_bind = (acc & IB_ACCESS_MW_BIND) ? 1 : 0;  in ocrdma_alloc_lkey()
789 	mr->hwmr.remote_atomic = (acc & IB_ACCESS_REMOTE_ATOMIC) ? 1 : 0;  in ocrdma_alloc_lkey()
790 	mr->hwmr.num_pbls = num_pbls;  in ocrdma_alloc_lkey()
792 	status = ocrdma_mbx_alloc_lkey(dev, &mr->hwmr, pdid, addr_check);  in ocrdma_alloc_lkey()
796 	mr->ibmr.lkey = mr->hwmr.lkey;  in ocrdma_alloc_lkey()
797 	if (mr->hwmr.remote_wr || mr->hwmr.remote_rd)  in ocrdma_alloc_lkey()
798 		mr->ibmr.rkey = mr->hwmr.lkey;  in ocrdma_alloc_lkey()
805 	struct ocrdma_mr *mr;  in ocrdma_get_dma_mr()  local
814 	mr = kzalloc(sizeof(*mr), GFP_KERNEL);  in ocrdma_get_dma_mr()
815 	if (!mr)  in ocrdma_get_dma_mr()
818 	status = ocrdma_alloc_lkey(dev, mr, pd->id, acc, 0,  in ocrdma_get_dma_mr()
821 		kfree(mr);  in ocrdma_get_dma_mr()
825 	return &mr->ibmr;  in ocrdma_get_dma_mr()
829 				   struct ocrdma_hw_mr *mr)  in ocrdma_free_mr_pbl_tbl()  argument
834 	if (mr->pbl_table) {  in ocrdma_free_mr_pbl_tbl()
835 		for (i = 0; i < mr->num_pbls; i++) {  in ocrdma_free_mr_pbl_tbl()
836 			if (!mr->pbl_table[i].va)  in ocrdma_free_mr_pbl_tbl()
838 			dma_free_coherent(&pdev->dev, mr->pbl_size,  in ocrdma_free_mr_pbl_tbl()
839 					  mr->pbl_table[i].va,  in ocrdma_free_mr_pbl_tbl()
840 					  mr->pbl_table[i].pa);  in ocrdma_free_mr_pbl_tbl()
842 		kfree(mr->pbl_table);  in ocrdma_free_mr_pbl_tbl()
843 		mr->pbl_table = NULL;  in ocrdma_free_mr_pbl_tbl()
847 static int ocrdma_get_pbl_info(struct ocrdma_dev *dev, struct ocrdma_mr *mr,  in ocrdma_get_pbl_info()  argument
866 	mr->hwmr.num_pbes = num_pbes;  in ocrdma_get_pbl_info()
867 	mr->hwmr.num_pbls = num_pbls;  in ocrdma_get_pbl_info()
868 	mr->hwmr.pbl_size = pbl_size;  in ocrdma_get_pbl_info()
872 static int ocrdma_build_pbl_tbl(struct ocrdma_dev *dev, struct ocrdma_hw_mr *mr)  in ocrdma_build_pbl_tbl()  argument
876 	u32 dma_len = mr->pbl_size;  in ocrdma_build_pbl_tbl()
881 	mr->pbl_table = kzalloc(sizeof(struct ocrdma_pbl) *  in ocrdma_build_pbl_tbl()
882 				mr->num_pbls, GFP_KERNEL);  in ocrdma_build_pbl_tbl()
884 	if (!mr->pbl_table)  in ocrdma_build_pbl_tbl()
887 	for (i = 0; i < mr->num_pbls; i++) {  in ocrdma_build_pbl_tbl()
890 			ocrdma_free_mr_pbl_tbl(dev, mr);  in ocrdma_build_pbl_tbl()
895 		mr->pbl_table[i].va = va;  in ocrdma_build_pbl_tbl()
896 		mr->pbl_table[i].pa = pa;  in ocrdma_build_pbl_tbl()
901 static void build_user_pbes(struct ocrdma_dev *dev, struct ocrdma_mr *mr,  in build_user_pbes()  argument
906 	struct ocrdma_pbl *pbl_tbl = mr->hwmr.pbl_table;  in build_user_pbes()
907 	struct ib_umem *umem = mr->umem;  in build_user_pbes()
910 	if (!mr->hwmr.num_pbes)  in build_user_pbes()
943 				(mr->hwmr.pbl_size / sizeof(u64))) {  in build_user_pbes()
958 	struct ocrdma_mr *mr;  in ocrdma_reg_user_mr()  local
967 	mr = kzalloc(sizeof(*mr), GFP_KERNEL);  in ocrdma_reg_user_mr()
968 	if (!mr)  in ocrdma_reg_user_mr()
970 	mr->umem = ib_umem_get(ibpd->uobject->context, start, len, acc, 0);  in ocrdma_reg_user_mr()
971 	if (IS_ERR(mr->umem)) {  in ocrdma_reg_user_mr()
975 	num_pbes = ib_umem_page_count(mr->umem);  in ocrdma_reg_user_mr()
976 	status = ocrdma_get_pbl_info(dev, mr, num_pbes);  in ocrdma_reg_user_mr()
980 	mr->hwmr.pbe_size = mr->umem->page_size;  in ocrdma_reg_user_mr()
981 	mr->hwmr.fbo = ib_umem_offset(mr->umem);  in ocrdma_reg_user_mr()
982 	mr->hwmr.va = usr_addr;  in ocrdma_reg_user_mr()
983 	mr->hwmr.len = len;  in ocrdma_reg_user_mr()
984 	mr->hwmr.remote_wr = (acc & IB_ACCESS_REMOTE_WRITE) ? 1 : 0;  in ocrdma_reg_user_mr()
985 	mr->hwmr.remote_rd = (acc & IB_ACCESS_REMOTE_READ) ? 1 : 0;  in ocrdma_reg_user_mr()
986 	mr->hwmr.local_wr = (acc & IB_ACCESS_LOCAL_WRITE) ? 1 : 0;  in ocrdma_reg_user_mr()
987 	mr->hwmr.local_rd = 1;  in ocrdma_reg_user_mr()
988 	mr->hwmr.remote_atomic = (acc & IB_ACCESS_REMOTE_ATOMIC) ? 1 : 0;  in ocrdma_reg_user_mr()
989 	status = ocrdma_build_pbl_tbl(dev, &mr->hwmr);  in ocrdma_reg_user_mr()
992 	build_user_pbes(dev, mr, num_pbes);  in ocrdma_reg_user_mr()
993 	status = ocrdma_reg_mr(dev, &mr->hwmr, pd->id, acc);  in ocrdma_reg_user_mr()
996 	mr->ibmr.lkey = mr->hwmr.lkey;  in ocrdma_reg_user_mr()
997 	if (mr->hwmr.remote_wr || mr->hwmr.remote_rd)  in ocrdma_reg_user_mr()
998 		mr->ibmr.rkey = mr->hwmr.lkey;  in ocrdma_reg_user_mr()
1000 	return &mr->ibmr;  in ocrdma_reg_user_mr()
1003 	ocrdma_free_mr_pbl_tbl(dev, &mr->hwmr);  in ocrdma_reg_user_mr()
1005 	kfree(mr);  in ocrdma_reg_user_mr()
1011 	struct ocrdma_mr *mr = get_ocrdma_mr(ib_mr);  in ocrdma_dereg_mr()  local
1014 	(void) ocrdma_mbx_dealloc_lkey(dev, mr->hwmr.fr_mr, mr->hwmr.lkey);  in ocrdma_dereg_mr()
1016 	kfree(mr->pages);  in ocrdma_dereg_mr()
1017 	ocrdma_free_mr_pbl_tbl(dev, &mr->hwmr);  in ocrdma_dereg_mr()
1020 	if (mr->umem)  in ocrdma_dereg_mr()
1021 		ib_umem_release(mr->umem);  in ocrdma_dereg_mr()
1022 	kfree(mr);  in ocrdma_dereg_mr()
2152 	struct ocrdma_mr *mr = get_ocrdma_mr(wr->mr);  in ocrdma_build_reg()  local
2153 	struct ocrdma_pbl *pbl_tbl = mr->hwmr.pbl_table;  in ocrdma_build_reg()
2170 	hdr->total_len = mr->ibmr.length;  in ocrdma_build_reg()
2172 	fbo = mr->ibmr.iova - mr->pages[0];  in ocrdma_build_reg()
2174 	fast_reg->va_hi = upper_32_bits(mr->ibmr.iova);  in ocrdma_build_reg()
2175 	fast_reg->va_lo = (u32) (mr->ibmr.iova & 0xffffffff);  in ocrdma_build_reg()
2178 	fast_reg->num_sges = mr->npages;  in ocrdma_build_reg()
2179 	fast_reg->size_sge = get_encoded_page_size(mr->ibmr.page_size);  in ocrdma_build_reg()
2182 	for (i = 0; i < mr->npages; i++) {  in ocrdma_build_reg()
2183 		u64 buf_addr = mr->pages[i];  in ocrdma_build_reg()
2193 		if (num_pbes == (mr->hwmr.pbl_size/sizeof(u64))) {  in ocrdma_build_reg()
3020 	struct ocrdma_mr *mr;  in ocrdma_alloc_mr()  local
3030 	mr = kzalloc(sizeof(*mr), GFP_KERNEL);  in ocrdma_alloc_mr()
3031 	if (!mr)  in ocrdma_alloc_mr()
3034 	mr->pages = kcalloc(max_num_sg, sizeof(u64), GFP_KERNEL);  in ocrdma_alloc_mr()
3035 	if (!mr->pages) {  in ocrdma_alloc_mr()
3040 	status = ocrdma_get_pbl_info(dev, mr, max_num_sg);  in ocrdma_alloc_mr()
3043 	mr->hwmr.fr_mr = 1;  in ocrdma_alloc_mr()
3044 	mr->hwmr.remote_rd = 0;  in ocrdma_alloc_mr()
3045 	mr->hwmr.remote_wr = 0;  in ocrdma_alloc_mr()
3046 	mr->hwmr.local_rd = 0;  in ocrdma_alloc_mr()
3047 	mr->hwmr.local_wr = 0;  in ocrdma_alloc_mr()
3048 	mr->hwmr.mw_bind = 0;  in ocrdma_alloc_mr()
3049 	status = ocrdma_build_pbl_tbl(dev, &mr->hwmr);  in ocrdma_alloc_mr()
3052 	status = ocrdma_reg_mr(dev, &mr->hwmr, pd->id, 0);  in ocrdma_alloc_mr()
3055 	mr->ibmr.rkey = mr->hwmr.lkey;  in ocrdma_alloc_mr()
3056 	mr->ibmr.lkey = mr->hwmr.lkey;  in ocrdma_alloc_mr()
3057 	dev->stag_arr[(mr->hwmr.lkey >> 8) & (OCRDMA_MAX_STAG - 1)] =  in ocrdma_alloc_mr()
3058 		(unsigned long) mr;  in ocrdma_alloc_mr()
3059 	return &mr->ibmr;  in ocrdma_alloc_mr()
3061 	ocrdma_free_mr_pbl_tbl(dev, &mr->hwmr);  in ocrdma_alloc_mr()
3063 	kfree(mr->pages);  in ocrdma_alloc_mr()
3065 	kfree(mr);  in ocrdma_alloc_mr()
3179 	struct ocrdma_mr *mr;  in ocrdma_reg_kernel_mr()  local
3188 	mr = kzalloc(sizeof(*mr), GFP_KERNEL);  in ocrdma_reg_kernel_mr()
3189 	if (!mr)  in ocrdma_reg_kernel_mr()
3197 	status = ocrdma_get_pbl_info(dev, mr, num_pbes);  in ocrdma_reg_kernel_mr()
3201 	mr->hwmr.pbe_size = pbe_size;  in ocrdma_reg_kernel_mr()
3202 	mr->hwmr.fbo = *iova_start - (buf_list[0].addr & PAGE_MASK);  in ocrdma_reg_kernel_mr()
3203 	mr->hwmr.va = *iova_start;  in ocrdma_reg_kernel_mr()
3204 	mr->hwmr.local_rd = 1;  in ocrdma_reg_kernel_mr()
3205 	mr->hwmr.remote_wr = (acc & IB_ACCESS_REMOTE_WRITE) ? 1 : 0;  in ocrdma_reg_kernel_mr()
3206 	mr->hwmr.remote_rd = (acc & IB_ACCESS_REMOTE_READ) ? 1 : 0;  in ocrdma_reg_kernel_mr()
3207 	mr->hwmr.local_wr = (acc & IB_ACCESS_LOCAL_WRITE) ? 1 : 0;  in ocrdma_reg_kernel_mr()
3208 	mr->hwmr.remote_atomic = (acc & IB_ACCESS_REMOTE_ATOMIC) ? 1 : 0;  in ocrdma_reg_kernel_mr()
3209 	mr->hwmr.mw_bind = (acc & IB_ACCESS_MW_BIND) ? 1 : 0;  in ocrdma_reg_kernel_mr()
3211 	status = ocrdma_build_pbl_tbl(dev, &mr->hwmr);  in ocrdma_reg_kernel_mr()
3214 	build_kernel_pbes(buf_list, buf_cnt, pbe_size, mr->hwmr.pbl_table,  in ocrdma_reg_kernel_mr()
3215 			  &mr->hwmr);  in ocrdma_reg_kernel_mr()
3216 	status = ocrdma_reg_mr(dev, &mr->hwmr, pd->id, acc);  in ocrdma_reg_kernel_mr()
3220 	mr->ibmr.lkey = mr->hwmr.lkey;  in ocrdma_reg_kernel_mr()
3221 	if (mr->hwmr.remote_wr || mr->hwmr.remote_rd)  in ocrdma_reg_kernel_mr()
3222 		mr->ibmr.rkey = mr->hwmr.lkey;  in ocrdma_reg_kernel_mr()
3223 	return &mr->ibmr;  in ocrdma_reg_kernel_mr()
3226 	ocrdma_free_mr_pbl_tbl(dev, &mr->hwmr);  in ocrdma_reg_kernel_mr()
3228 	kfree(mr);  in ocrdma_reg_kernel_mr()
3234 	struct ocrdma_mr *mr = get_ocrdma_mr(ibmr);  in ocrdma_set_page()  local
3236 	if (unlikely(mr->npages == mr->hwmr.num_pbes))  in ocrdma_set_page()
3239 	mr->pages[mr->npages++] = addr;  in ocrdma_set_page()
3248 	struct ocrdma_mr *mr = get_ocrdma_mr(ibmr);  in ocrdma_map_mr_sg()  local
3250 	mr->npages = 0;  in ocrdma_map_mr_sg()