713 static int ocrdma_alloc_lkey(struct ocrdma_dev *dev, struct ocrdma_mr *mr,  in ocrdma_alloc_lkey()  argument
718 	mr->hwmr.fr_mr = 0;  in ocrdma_alloc_lkey()
719 	mr->hwmr.local_rd = 1;  in ocrdma_alloc_lkey()
720 	mr->hwmr.remote_rd = (acc & IB_ACCESS_REMOTE_READ) ? 1 : 0;  in ocrdma_alloc_lkey()
721 	mr->hwmr.remote_wr = (acc & IB_ACCESS_REMOTE_WRITE) ? 1 : 0;  in ocrdma_alloc_lkey()
722 	mr->hwmr.local_wr = (acc & IB_ACCESS_LOCAL_WRITE) ? 1 : 0;  in ocrdma_alloc_lkey()
723 	mr->hwmr.mw_bind = (acc & IB_ACCESS_MW_BIND) ? 1 : 0;  in ocrdma_alloc_lkey()
724 	mr->hwmr.remote_atomic = (acc & IB_ACCESS_REMOTE_ATOMIC) ? 1 : 0;  in ocrdma_alloc_lkey()
725 	mr->hwmr.num_pbls = num_pbls;  in ocrdma_alloc_lkey()
727 	status = ocrdma_mbx_alloc_lkey(dev, &mr->hwmr, pdid, addr_check);  in ocrdma_alloc_lkey()
731 	mr->ibmr.lkey = mr->hwmr.lkey;  in ocrdma_alloc_lkey()
732 	if (mr->hwmr.remote_wr || mr->hwmr.remote_rd)  in ocrdma_alloc_lkey()
733 		mr->ibmr.rkey = mr->hwmr.lkey;  in ocrdma_alloc_lkey()
740 	struct ocrdma_mr *mr;  in ocrdma_get_dma_mr()  local
749 	mr = kzalloc(sizeof(*mr), GFP_KERNEL);  in ocrdma_get_dma_mr()
750 	if (!mr)  in ocrdma_get_dma_mr()
753 	status = ocrdma_alloc_lkey(dev, mr, pd->id, acc, 0,  in ocrdma_get_dma_mr()
756 		kfree(mr);  in ocrdma_get_dma_mr()
760 	return &mr->ibmr;  in ocrdma_get_dma_mr()
764 				   struct ocrdma_hw_mr *mr)  in ocrdma_free_mr_pbl_tbl()  argument
769 	if (mr->pbl_table) {  in ocrdma_free_mr_pbl_tbl()
770 		for (i = 0; i < mr->num_pbls; i++) {  in ocrdma_free_mr_pbl_tbl()
771 			if (!mr->pbl_table[i].va)  in ocrdma_free_mr_pbl_tbl()
773 			dma_free_coherent(&pdev->dev, mr->pbl_size,  in ocrdma_free_mr_pbl_tbl()
774 					  mr->pbl_table[i].va,  in ocrdma_free_mr_pbl_tbl()
775 					  mr->pbl_table[i].pa);  in ocrdma_free_mr_pbl_tbl()
777 		kfree(mr->pbl_table);  in ocrdma_free_mr_pbl_tbl()
778 		mr->pbl_table = NULL;  in ocrdma_free_mr_pbl_tbl()
782 static int ocrdma_get_pbl_info(struct ocrdma_dev *dev, struct ocrdma_mr *mr,  in ocrdma_get_pbl_info()  argument
801 	mr->hwmr.num_pbes = num_pbes;  in ocrdma_get_pbl_info()
802 	mr->hwmr.num_pbls = num_pbls;  in ocrdma_get_pbl_info()
803 	mr->hwmr.pbl_size = pbl_size;  in ocrdma_get_pbl_info()
807 static int ocrdma_build_pbl_tbl(struct ocrdma_dev *dev, struct ocrdma_hw_mr *mr)  in ocrdma_build_pbl_tbl()  argument
811 	u32 dma_len = mr->pbl_size;  in ocrdma_build_pbl_tbl()
816 	mr->pbl_table = kzalloc(sizeof(struct ocrdma_pbl) *  in ocrdma_build_pbl_tbl()
817 				mr->num_pbls, GFP_KERNEL);  in ocrdma_build_pbl_tbl()
819 	if (!mr->pbl_table)  in ocrdma_build_pbl_tbl()
822 	for (i = 0; i < mr->num_pbls; i++) {  in ocrdma_build_pbl_tbl()
825 			ocrdma_free_mr_pbl_tbl(dev, mr);  in ocrdma_build_pbl_tbl()
830 		mr->pbl_table[i].va = va;  in ocrdma_build_pbl_tbl()
831 		mr->pbl_table[i].pa = pa;  in ocrdma_build_pbl_tbl()
836 static void build_user_pbes(struct ocrdma_dev *dev, struct ocrdma_mr *mr,  in build_user_pbes()  argument
841 	struct ocrdma_pbl *pbl_tbl = mr->hwmr.pbl_table;  in build_user_pbes()
842 	struct ib_umem *umem = mr->umem;  in build_user_pbes()
845 	if (!mr->hwmr.num_pbes)  in build_user_pbes()
878 				(mr->hwmr.pbl_size / sizeof(u64))) {  in build_user_pbes()
893 	struct ocrdma_mr *mr;  in ocrdma_reg_user_mr()  local
902 	mr = kzalloc(sizeof(*mr), GFP_KERNEL);  in ocrdma_reg_user_mr()
903 	if (!mr)  in ocrdma_reg_user_mr()
905 	mr->umem = ib_umem_get(ibpd->uobject->context, start, len, acc, 0);  in ocrdma_reg_user_mr()
906 	if (IS_ERR(mr->umem)) {  in ocrdma_reg_user_mr()
910 	num_pbes = ib_umem_page_count(mr->umem);  in ocrdma_reg_user_mr()
911 	status = ocrdma_get_pbl_info(dev, mr, num_pbes);  in ocrdma_reg_user_mr()
915 	mr->hwmr.pbe_size = mr->umem->page_size;  in ocrdma_reg_user_mr()
916 	mr->hwmr.fbo = ib_umem_offset(mr->umem);  in ocrdma_reg_user_mr()
917 	mr->hwmr.va = usr_addr;  in ocrdma_reg_user_mr()
918 	mr->hwmr.len = len;  in ocrdma_reg_user_mr()
919 	mr->hwmr.remote_wr = (acc & IB_ACCESS_REMOTE_WRITE) ? 1 : 0;  in ocrdma_reg_user_mr()
920 	mr->hwmr.remote_rd = (acc & IB_ACCESS_REMOTE_READ) ? 1 : 0;  in ocrdma_reg_user_mr()
921 	mr->hwmr.local_wr = (acc & IB_ACCESS_LOCAL_WRITE) ? 1 : 0;  in ocrdma_reg_user_mr()
922 	mr->hwmr.local_rd = 1;  in ocrdma_reg_user_mr()
923 	mr->hwmr.remote_atomic = (acc & IB_ACCESS_REMOTE_ATOMIC) ? 1 : 0;  in ocrdma_reg_user_mr()
924 	status = ocrdma_build_pbl_tbl(dev, &mr->hwmr);  in ocrdma_reg_user_mr()
927 	build_user_pbes(dev, mr, num_pbes);  in ocrdma_reg_user_mr()
928 	status = ocrdma_reg_mr(dev, &mr->hwmr, pd->id, acc);  in ocrdma_reg_user_mr()
931 	mr->ibmr.lkey = mr->hwmr.lkey;  in ocrdma_reg_user_mr()
932 	if (mr->hwmr.remote_wr || mr->hwmr.remote_rd)  in ocrdma_reg_user_mr()
933 		mr->ibmr.rkey = mr->hwmr.lkey;  in ocrdma_reg_user_mr()
935 	return &mr->ibmr;  in ocrdma_reg_user_mr()
938 	ocrdma_free_mr_pbl_tbl(dev, &mr->hwmr);  in ocrdma_reg_user_mr()
940 	kfree(mr);  in ocrdma_reg_user_mr()
946 	struct ocrdma_mr *mr = get_ocrdma_mr(ib_mr);  in ocrdma_dereg_mr()  local
949 	(void) ocrdma_mbx_dealloc_lkey(dev, mr->hwmr.fr_mr, mr->hwmr.lkey);  in ocrdma_dereg_mr()
951 	ocrdma_free_mr_pbl_tbl(dev, &mr->hwmr);  in ocrdma_dereg_mr()
954 	if (mr->umem)  in ocrdma_dereg_mr()
955 		ib_umem_release(mr->umem);  in ocrdma_dereg_mr()
956 	kfree(mr);  in ocrdma_dereg_mr()
2115 	struct ocrdma_mr *mr;  in ocrdma_build_fr()  local
2148 	mr = (struct ocrdma_mr *) (unsigned long)  in ocrdma_build_fr()
2150 	build_frmr_pbes(wr, mr->hwmr.pbl_table, &mr->hwmr);  in ocrdma_build_fr()
2975 	struct ocrdma_mr *mr;  in ocrdma_alloc_frmr()  local
2982 	mr = kzalloc(sizeof(*mr), GFP_KERNEL);  in ocrdma_alloc_frmr()
2983 	if (!mr)  in ocrdma_alloc_frmr()
2986 	status = ocrdma_get_pbl_info(dev, mr, max_page_list_len);  in ocrdma_alloc_frmr()
2989 	mr->hwmr.fr_mr = 1;  in ocrdma_alloc_frmr()
2990 	mr->hwmr.remote_rd = 0;  in ocrdma_alloc_frmr()
2991 	mr->hwmr.remote_wr = 0;  in ocrdma_alloc_frmr()
2992 	mr->hwmr.local_rd = 0;  in ocrdma_alloc_frmr()
2993 	mr->hwmr.local_wr = 0;  in ocrdma_alloc_frmr()
2994 	mr->hwmr.mw_bind = 0;  in ocrdma_alloc_frmr()
2995 	status = ocrdma_build_pbl_tbl(dev, &mr->hwmr);  in ocrdma_alloc_frmr()
2998 	status = ocrdma_reg_mr(dev, &mr->hwmr, pd->id, 0);  in ocrdma_alloc_frmr()
3001 	mr->ibmr.rkey = mr->hwmr.lkey;  in ocrdma_alloc_frmr()
3002 	mr->ibmr.lkey = mr->hwmr.lkey;  in ocrdma_alloc_frmr()
3003 	dev->stag_arr[(mr->hwmr.lkey >> 8) & (OCRDMA_MAX_STAG - 1)] =  in ocrdma_alloc_frmr()
3004 		(unsigned long) mr;  in ocrdma_alloc_frmr()
3005 	return &mr->ibmr;  in ocrdma_alloc_frmr()
3007 	ocrdma_free_mr_pbl_tbl(dev, &mr->hwmr);  in ocrdma_alloc_frmr()
3009 	kfree(mr);  in ocrdma_alloc_frmr()
3143 	struct ocrdma_mr *mr;  in ocrdma_reg_kernel_mr()  local
3152 	mr = kzalloc(sizeof(*mr), GFP_KERNEL);  in ocrdma_reg_kernel_mr()
3153 	if (!mr)  in ocrdma_reg_kernel_mr()
3161 	status = ocrdma_get_pbl_info(dev, mr, num_pbes);  in ocrdma_reg_kernel_mr()
3165 	mr->hwmr.pbe_size = pbe_size;  in ocrdma_reg_kernel_mr()
3166 	mr->hwmr.fbo = *iova_start - (buf_list[0].addr & PAGE_MASK);  in ocrdma_reg_kernel_mr()
3167 	mr->hwmr.va = *iova_start;  in ocrdma_reg_kernel_mr()
3168 	mr->hwmr.local_rd = 1;  in ocrdma_reg_kernel_mr()
3169 	mr->hwmr.remote_wr = (acc & IB_ACCESS_REMOTE_WRITE) ? 1 : 0;  in ocrdma_reg_kernel_mr()
3170 	mr->hwmr.remote_rd = (acc & IB_ACCESS_REMOTE_READ) ? 1 : 0;  in ocrdma_reg_kernel_mr()
3171 	mr->hwmr.local_wr = (acc & IB_ACCESS_LOCAL_WRITE) ? 1 : 0;  in ocrdma_reg_kernel_mr()
3172 	mr->hwmr.remote_atomic = (acc & IB_ACCESS_REMOTE_ATOMIC) ? 1 : 0;  in ocrdma_reg_kernel_mr()
3173 	mr->hwmr.mw_bind = (acc & IB_ACCESS_MW_BIND) ? 1 : 0;  in ocrdma_reg_kernel_mr()
3175 	status = ocrdma_build_pbl_tbl(dev, &mr->hwmr);  in ocrdma_reg_kernel_mr()
3178 	build_kernel_pbes(buf_list, buf_cnt, pbe_size, mr->hwmr.pbl_table,  in ocrdma_reg_kernel_mr()
3179 			  &mr->hwmr);  in ocrdma_reg_kernel_mr()
3180 	status = ocrdma_reg_mr(dev, &mr->hwmr, pd->id, acc);  in ocrdma_reg_kernel_mr()
3184 	mr->ibmr.lkey = mr->hwmr.lkey;  in ocrdma_reg_kernel_mr()
3185 	if (mr->hwmr.remote_wr || mr->hwmr.remote_rd)  in ocrdma_reg_kernel_mr()
3186 		mr->ibmr.rkey = mr->hwmr.lkey;  in ocrdma_reg_kernel_mr()
3187 	return &mr->ibmr;  in ocrdma_reg_kernel_mr()
3190 	ocrdma_free_mr_pbl_tbl(dev, &mr->hwmr);  in ocrdma_reg_kernel_mr()
3192 	kfree(mr);  in ocrdma_reg_kernel_mr()