113 static u64 ehca_get_max_hwpage_size(struct ehca_shca *shca)  in ehca_get_max_hwpage_size()  argument
115 	return rounddown_pow_of_two(shca->hca_cap_mr_pgsize);  in ehca_get_max_hwpage_size()
162 	struct ehca_shca *shca =  in ehca_get_dma_mr()  local
165 	if (shca->maxmr) {  in ehca_get_dma_mr()
168 			ehca_err(&shca->ib_device, "out of memory");  in ehca_get_dma_mr()
173 		ret = ehca_reg_maxmr(shca, e_maxmr,  in ehca_get_dma_mr()
185 		ehca_err(&shca->ib_device, "no internal max-MR exist!");  in ehca_get_dma_mr()
192 		ehca_err(&shca->ib_device, "h_ret=%li pd=%p mr_access_flags=%x",  in ehca_get_dma_mr()
208 	struct ehca_shca *shca =  in ehca_reg_phys_mr()  local
259 		ret = ehca_reg_maxmr(shca, e_mr, iova_start, mr_access_flags,  in ehca_reg_phys_mr()
275 		hw_pgsize = ehca_get_max_hwpage_size(shca);  in ehca_reg_phys_mr()
288 		ret = ehca_reg_mr(shca, e_mr, iova_start, size, mr_access_flags,  in ehca_reg_phys_mr()
319 	struct ehca_shca *shca =  in ehca_reg_user_mr()  local
390 	while (!(hwpage_size & shca->hca_cap_mr_pgsize))  in ehca_reg_user_mr()
404 	ret = ehca_reg_mr(shca, e_mr, (u64 *)virt, length, mr_access_flags,  in ehca_reg_user_mr()
450 	struct ehca_shca *shca =  in ehca_rereg_phys_mr()  local
488 	if (e_mr == shca->maxmr) {  in ehca_rereg_phys_mr()
492 			 mr, shca->maxmr, mr->lkey);  in ehca_rereg_phys_mr()
534 		u64 hw_pgsize = ehca_get_max_hwpage_size(shca);  in ehca_rereg_phys_mr()
569 	ret = ehca_rereg_mr(shca, e_mr, new_start, new_size, new_acl,  in ehca_rereg_phys_mr()
598 	struct ehca_shca *shca =  in ehca_query_mr()  local
614 	h_ret = hipz_h_query_mr(shca->ipz_hca_handle, e_mr, &hipzout);  in ehca_query_mr()
618 			 h_ret, mr, shca->ipz_hca_handle.handle,  in ehca_query_mr()
645 	struct ehca_shca *shca =  in ehca_dereg_mr()  local
654 	} else if (e_mr == shca->maxmr) {  in ehca_dereg_mr()
658 			 mr, shca->maxmr, mr->lkey);  in ehca_dereg_mr()
664 	h_ret = hipz_h_free_resource_mr(shca->ipz_hca_handle, e_mr);  in ehca_dereg_mr()
668 			 h_ret, shca, e_mr, shca->ipz_hca_handle.handle,  in ehca_dereg_mr()
694 	struct ehca_shca *shca =  in ehca_alloc_mw()  local
707 	h_ret = hipz_h_alloc_resource_mw(shca->ipz_hca_handle, e_mw,  in ehca_alloc_mw()
712 			 h_ret, shca, shca->ipz_hca_handle.handle, e_mw);  in ehca_alloc_mw()
746 	struct ehca_shca *shca =  in ehca_dealloc_mw()  local
750 	h_ret = hipz_h_free_resource_mw(shca->ipz_hca_handle, e_mw);  in ehca_dealloc_mw()
754 			 h_ret, shca, mw, mw->rkey, shca->ipz_hca_handle.handle,  in ehca_dealloc_mw()
770 	struct ehca_shca *shca =  in ehca_alloc_fmr()  local
809 	if (!(hw_pgsize & shca->hca_cap_mr_pgsize)) {  in ehca_alloc_fmr()
830 	ret = ehca_reg_mr(shca, e_fmr, NULL,  in ehca_alloc_fmr()
861 	struct ehca_shca *shca =  in ehca_map_phys_fmr()  local
902 	ret = ehca_rereg_mr(shca, e_fmr, (u64 *)iova,  in ehca_map_phys_fmr()
927 	struct ehca_shca *shca = NULL;  in ehca_unmap_fmr()  local
935 		prev_shca = shca;  in ehca_unmap_fmr()
936 		shca = container_of(ib_fmr->device, struct ehca_shca,  in ehca_unmap_fmr()
939 		if ((shca != prev_shca) && prev_shca) {  in ehca_unmap_fmr()
940 			ehca_err(&shca->ib_device, "SHCA mismatch, shca=%p "  in ehca_unmap_fmr()
942 				 shca, prev_shca, e_fmr);  in ehca_unmap_fmr()
947 			ehca_err(&shca->ib_device, "not a FMR, e_fmr=%p "  in ehca_unmap_fmr()
959 		shca = container_of(ib_fmr->device, struct ehca_shca,  in ehca_unmap_fmr()
961 		ret = ehca_unmap_one_fmr(shca, e_fmr);  in ehca_unmap_fmr()
964 			ehca_err(&shca->ib_device, "unmap of one FMR failed, "  in ehca_unmap_fmr()
985 	struct ehca_shca *shca =  in ehca_dealloc_fmr()  local
996 	h_ret = hipz_h_free_resource_mr(shca->ipz_hca_handle, e_fmr);  in ehca_dealloc_fmr()
1000 			 h_ret, e_fmr, shca->ipz_hca_handle.handle,  in ehca_dealloc_fmr()
1011 		ehca_err(&shca->ib_device, "ret=%i fmr=%p", ret, fmr);  in ehca_dealloc_fmr()
1017 static int ehca_reg_bmap_mr_rpages(struct ehca_shca *shca,
1021 int ehca_reg_mr(struct ehca_shca *shca,  in ehca_reg_mr()  argument
1042 	h_ret = hipz_h_alloc_resource_mr(shca->ipz_hca_handle, e_mr,  in ehca_reg_mr()
1046 		ehca_err(&shca->ib_device, "hipz_alloc_mr failed, h_ret=%lli "  in ehca_reg_mr()
1047 			 "hca_hndl=%llx", h_ret, shca->ipz_hca_handle.handle);  in ehca_reg_mr()
1055 		ret = ehca_reg_bmap_mr_rpages(shca, e_mr, pginfo);  in ehca_reg_mr()
1057 		ret = ehca_reg_mr_rpages(shca, e_mr, pginfo);  in ehca_reg_mr()
1076 	h_ret = hipz_h_free_resource_mr(shca->ipz_hca_handle, e_mr);  in ehca_reg_mr()
1078 		ehca_err(&shca->ib_device, "h_ret=%lli shca=%p e_mr=%p "  in ehca_reg_mr()
1081 			 h_ret, shca, e_mr, iova_start, size, acl, e_pd,  in ehca_reg_mr()
1084 		ehca_err(&shca->ib_device, "internal error in ehca_reg_mr, "  in ehca_reg_mr()
1089 		ehca_err(&shca->ib_device, "ret=%i shca=%p e_mr=%p "  in ehca_reg_mr()
1092 			 ret, shca, e_mr, iova_start, size, acl, e_pd, pginfo,  in ehca_reg_mr()
1099 int ehca_reg_mr_rpages(struct ehca_shca *shca,  in ehca_reg_mr_rpages()  argument
1115 		ehca_err(&shca->ib_device, "kpage alloc failed");  in ehca_reg_mr_rpages()
1132 			ehca_err(&shca->ib_device, "ehca_set_pagebuf "  in ehca_reg_mr_rpages()
1141 				ehca_err(&shca->ib_device, "kpage=%p i=%x",  in ehca_reg_mr_rpages()
1150 			shca->ipz_hca_handle, e_mr,  in ehca_reg_mr_rpages()
1160 				ehca_err(&shca->ib_device, "last "  in ehca_reg_mr_rpages()
1164 					 shca->ipz_hca_handle.handle,  in ehca_reg_mr_rpages()
1172 			ehca_err(&shca->ib_device, "hipz_reg_rpage_mr failed, "  in ehca_reg_mr_rpages()
1176 				 shca->ipz_hca_handle.handle,  in ehca_reg_mr_rpages()
1189 		ehca_err(&shca->ib_device, "ret=%i shca=%p e_mr=%p pginfo=%p "  in ehca_reg_mr_rpages()
1190 			 "num_kpages=%llx num_hwpages=%llx", ret, shca, e_mr,  in ehca_reg_mr_rpages()
1197 inline int ehca_rereg_mr_rereg1(struct ehca_shca *shca,  in ehca_rereg_mr_rereg1()  argument
1220 		ehca_err(&shca->ib_device, "kpage alloc failed");  in ehca_rereg_mr_rereg1()
1228 		ehca_err(&shca->ib_device, "set pagebuf failed, e_mr=%p "  in ehca_rereg_mr_rereg1()
1236 		ehca_err(&shca->ib_device, "kpage=%p", kpage);  in ehca_rereg_mr_rereg1()
1240 	h_ret = hipz_h_reregister_pmr(shca->ipz_hca_handle, e_mr,  in ehca_rereg_mr_rereg1()
1249 		ehca_warn(&shca->ib_device, "hipz_h_reregister_pmr failed "  in ehca_rereg_mr_rereg1()
1254 		ehca_err(&shca->ib_device, "PHYP changed iova_start in "  in ehca_rereg_mr_rereg1()
1279 		ehca_err(&shca->ib_device, "ret=%i lkey=%x rkey=%x "  in ehca_rereg_mr_rereg1()
1288 int ehca_rereg_mr(struct ehca_shca *shca,  in ehca_rereg_mr()  argument
1307 		ehca_dbg(&shca->ib_device, "Rereg3 case, "  in ehca_rereg_mr()
1318 		ehca_err(&shca->ib_device, "Rereg MR for max-MR! e_mr=%p",  in ehca_rereg_mr()
1323 		ret = ehca_rereg_mr_rereg1(shca, e_mr, iova_start, size,  in ehca_rereg_mr()
1337 		h_ret = hipz_h_free_resource_mr(shca->ipz_hca_handle, e_mr);  in ehca_rereg_mr()
1339 			ehca_err(&shca->ib_device, "hipz_free_mr failed, "  in ehca_rereg_mr()
1342 				 h_ret, e_mr, shca->ipz_hca_handle.handle,  in ehca_rereg_mr()
1360 		ret = ehca_reg_mr(shca, e_mr, iova_start, size, acl,  in ehca_rereg_mr()
1372 		ehca_err(&shca->ib_device, "ret=%i shca=%p e_mr=%p "  in ehca_rereg_mr()
1375 			 "rereg_3_hcall=%x", ret, shca, e_mr, iova_start, size,  in ehca_rereg_mr()
1383 int ehca_unmap_one_fmr(struct ehca_shca *shca,  in ehca_unmap_one_fmr()  argument
1401 		h_ret = hipz_h_reregister_pmr(shca->ipz_hca_handle, e_fmr, 0,  in ehca_unmap_one_fmr()
1415 		ehca_err(&shca->ib_device, "hipz_reregister_pmr failed "  in ehca_unmap_one_fmr()
1418 			 h_ret, e_fmr, shca->ipz_hca_handle.handle,  in ehca_unmap_one_fmr()
1425 	h_ret = hipz_h_free_resource_mr(shca->ipz_hca_handle, e_fmr);  in ehca_unmap_one_fmr()
1427 		ehca_err(&shca->ib_device, "hipz_free_mr failed, "  in ehca_unmap_one_fmr()
1430 			 h_ret, e_fmr, shca->ipz_hca_handle.handle,  in ehca_unmap_one_fmr()
1451 	ret = ehca_reg_mr(shca, e_fmr, NULL,  in ehca_unmap_one_fmr()
1463 		ehca_err(&shca->ib_device, "ret=%i tmp_lkey=%x tmp_rkey=%x "  in ehca_unmap_one_fmr()
1471 int ehca_reg_smr(struct ehca_shca *shca,  in ehca_reg_smr()  argument
1488 	h_ret = hipz_h_register_smr(shca->ipz_hca_handle, e_newmr, e_origmr,  in ehca_reg_smr()
1492 		ehca_err(&shca->ib_device, "hipz_reg_smr failed, h_ret=%lli "  in ehca_reg_smr()
1495 			 h_ret, shca, e_origmr, e_newmr, iova_start, acl, e_pd,  in ehca_reg_smr()
1496 			 shca->ipz_hca_handle.handle,  in ehca_reg_smr()
1516 		ehca_err(&shca->ib_device, "ret=%i shca=%p e_origmr=%p "  in ehca_reg_smr()
1518 			 ret, shca, e_origmr, e_newmr, iova_start, acl, e_pd);  in ehca_reg_smr()
1535 			       struct ehca_shca *shca, struct ehca_mr *mr,  in ehca_reg_mr_section()  argument
1545 		ehca_err(&shca->ib_device, "reg_mr_section will probably fail:"  in ehca_reg_mr_section()
1559 		h_ret = hipz_h_register_rpage_mr(shca->ipz_hca_handle, mr,  in ehca_reg_mr_section()
1564 			ehca_err(&shca->ib_device, "register_rpage_mr failed");  in ehca_reg_mr_section()
1572 				struct ehca_shca *shca, struct ehca_mr *mr,  in ehca_reg_mr_sections()  argument
1582 		hret = ehca_reg_mr_section(top, dir, idx, kpage, shca, mr,  in ehca_reg_mr_sections()
1590 static u64 ehca_reg_mr_dir_sections(int top, u64 *kpage, struct ehca_shca *shca,  in ehca_reg_mr_dir_sections()  argument
1601 		hret = ehca_reg_mr_sections(top, dir, kpage, shca, mr, pginfo);  in ehca_reg_mr_dir_sections()
1610 	struct ehca_shca *shca,  in ehca_reg_internal_maxmr()  argument
1631 		ehca_err(&shca->ib_device, "out of memory");  in ehca_reg_internal_maxmr()
1644 	hw_pgsize = ehca_get_max_hwpage_size(shca);  in ehca_reg_internal_maxmr()
1656 	ret = ehca_reg_mr(shca, e_mr, iova_start, size_maxmr, 0, e_pd,  in ehca_reg_internal_maxmr()
1660 		ehca_err(&shca->ib_device, "reg of internal max MR failed, "  in ehca_reg_internal_maxmr()
1680 		ehca_err(&shca->ib_device, "ret=%i shca=%p e_pd=%p e_maxmr=%p",  in ehca_reg_internal_maxmr()
1681 			 ret, shca, e_pd, e_maxmr);  in ehca_reg_internal_maxmr()
1687 int ehca_reg_maxmr(struct ehca_shca *shca,  in ehca_reg_maxmr()  argument
1696 	struct ehca_mr *e_origmr = shca->maxmr;  in ehca_reg_maxmr()
1703 	h_ret = hipz_h_register_smr(shca->ipz_hca_handle, e_newmr, e_origmr,  in ehca_reg_maxmr()
1707 		ehca_err(&shca->ib_device, "hipz_reg_smr failed, h_ret=%lli "  in ehca_reg_maxmr()
1709 			 h_ret, e_origmr, shca->ipz_hca_handle.handle,  in ehca_reg_maxmr()
1729 int ehca_dereg_internal_maxmr(struct ehca_shca *shca)  in ehca_dereg_internal_maxmr()  argument
1735 	if (!shca->maxmr) {  in ehca_dereg_internal_maxmr()
1736 		ehca_err(&shca->ib_device, "bad call, shca=%p", shca);  in ehca_dereg_internal_maxmr()
1741 	e_maxmr = shca->maxmr;  in ehca_dereg_internal_maxmr()
1743 	shca->maxmr = NULL; /* remove internal max-MR indication from SHCA */  in ehca_dereg_internal_maxmr()
1747 		ehca_err(&shca->ib_device, "dereg internal max-MR failed, "  in ehca_dereg_internal_maxmr()
1749 			 ret, e_maxmr, shca, e_maxmr->ib.ib_mr.lkey);  in ehca_dereg_internal_maxmr()
1750 		shca->maxmr = e_maxmr;  in ehca_dereg_internal_maxmr()
1758 		ehca_err(&shca->ib_device, "ret=%i shca=%p shca->maxmr=%p",  in ehca_dereg_internal_maxmr()
1759 			 ret, shca, shca->maxmr);  in ehca_dereg_internal_maxmr()
2408 static int ehca_reg_bmap_mr_rpages(struct ehca_shca *shca,  in ehca_reg_bmap_mr_rpages()  argument
2417 		ehca_err(&shca->ib_device, "kpage alloc failed");  in ehca_reg_bmap_mr_rpages()
2423 		hret = ehca_reg_mr_dir_sections(top, kpage, shca, e_mr, pginfo);  in ehca_reg_bmap_mr_rpages()
2433 		ehca_err(&shca->ib_device, "ehca_reg_bmap_mr_rpages failed, "  in ehca_reg_bmap_mr_rpages()
2437 				 shca->ipz_hca_handle.handle,  in ehca_reg_bmap_mr_rpages()