457 	struct iwch_mr *mhp;  in iwch_dereg_mr()  local
465 	mhp = to_iwch_mr(ib_mr);  in iwch_dereg_mr()
466 	kfree(mhp->pages);  in iwch_dereg_mr()
467 	rhp = mhp->rhp;  in iwch_dereg_mr()
468 	mmid = mhp->attr.stag >> 8;  in iwch_dereg_mr()
469 	cxio_dereg_mem(&rhp->rdev, mhp->attr.stag, mhp->attr.pbl_size,  in iwch_dereg_mr()
470 		       mhp->attr.pbl_addr);  in iwch_dereg_mr()
471 	iwch_free_pbl(mhp);  in iwch_dereg_mr()
473 	if (mhp->kva)  in iwch_dereg_mr()
474 		kfree((void *) (unsigned long) mhp->kva);  in iwch_dereg_mr()
475 	if (mhp->umem)  in iwch_dereg_mr()
476 		ib_umem_release(mhp->umem);  in iwch_dereg_mr()
477 	PDBG("%s mmid 0x%x ptr %p\n", __func__, mmid, mhp);  in iwch_dereg_mr()
478 	kfree(mhp);  in iwch_dereg_mr()
494 	struct iwch_mr *mhp;  in iwch_register_phys_mem()  local
501 	mhp = kzalloc(sizeof(*mhp), GFP_KERNEL);  in iwch_register_phys_mem()
502 	if (!mhp)  in iwch_register_phys_mem()
505 	mhp->rhp = rhp;  in iwch_register_phys_mem()
524 	ret = iwch_alloc_pbl(mhp, npages);  in iwch_register_phys_mem()
530 	ret = iwch_write_pbl(mhp, page_list, npages, 0);  in iwch_register_phys_mem()
535 	mhp->attr.pdid = php->pdid;  in iwch_register_phys_mem()
536 	mhp->attr.zbva = 0;  in iwch_register_phys_mem()
538 	mhp->attr.perms = iwch_ib_to_tpt_access(acc);  in iwch_register_phys_mem()
539 	mhp->attr.va_fbo = *iova_start;  in iwch_register_phys_mem()
540 	mhp->attr.page_size = shift - 12;  in iwch_register_phys_mem()
542 	mhp->attr.len = (u32) total_size;  in iwch_register_phys_mem()
543 	mhp->attr.pbl_size = npages;  in iwch_register_phys_mem()
544 	ret = iwch_register_mem(rhp, php, mhp, shift);  in iwch_register_phys_mem()
548 	return &mhp->ibmr;  in iwch_register_phys_mem()
551 	iwch_free_pbl(mhp);  in iwch_register_phys_mem()
554 	kfree(mhp);  in iwch_register_phys_mem()
567 	struct iwch_mr mh, *mhp;  in iwch_reregister_phys_mem()  local
582 	mhp = to_iwch_mr(mr);  in iwch_reregister_phys_mem()
583 	rhp = mhp->rhp;  in iwch_reregister_phys_mem()
590 	memcpy(&mh, mhp, sizeof *mhp);  in iwch_reregister_phys_mem()
611 		mhp->attr.pdid = php->pdid;  in iwch_reregister_phys_mem()
613 		mhp->attr.perms = iwch_ib_to_tpt_access(acc);  in iwch_reregister_phys_mem()
615 		mhp->attr.zbva = 0;  in iwch_reregister_phys_mem()
616 		mhp->attr.va_fbo = *iova_start;  in iwch_reregister_phys_mem()
617 		mhp->attr.page_size = shift - 12;  in iwch_reregister_phys_mem()
618 		mhp->attr.len = (u32) total_size;  in iwch_reregister_phys_mem()
619 		mhp->attr.pbl_size = npages;  in iwch_reregister_phys_mem()
635 	struct iwch_mr *mhp;  in iwch_reg_user_mr()  local
642 	mhp = kzalloc(sizeof(*mhp), GFP_KERNEL);  in iwch_reg_user_mr()
643 	if (!mhp)  in iwch_reg_user_mr()
646 	mhp->rhp = rhp;  in iwch_reg_user_mr()
648 	mhp->umem = ib_umem_get(pd->uobject->context, start, length, acc, 0);  in iwch_reg_user_mr()
649 	if (IS_ERR(mhp->umem)) {  in iwch_reg_user_mr()
650 		err = PTR_ERR(mhp->umem);  in iwch_reg_user_mr()
651 		kfree(mhp);  in iwch_reg_user_mr()
655 	shift = ffs(mhp->umem->page_size) - 1;  in iwch_reg_user_mr()
657 	n = mhp->umem->nmap;  in iwch_reg_user_mr()
659 	err = iwch_alloc_pbl(mhp, n);  in iwch_reg_user_mr()
671 	for_each_sg(mhp->umem->sg_head.sgl, sg, mhp->umem->nmap, entry) {  in iwch_reg_user_mr()
675 					mhp->umem->page_size * k);  in iwch_reg_user_mr()
677 					err = iwch_write_pbl(mhp, pages, i, n);  in iwch_reg_user_mr()
687 		err = iwch_write_pbl(mhp, pages, i, n);  in iwch_reg_user_mr()
694 	mhp->attr.pdid = php->pdid;  in iwch_reg_user_mr()
695 	mhp->attr.zbva = 0;  in iwch_reg_user_mr()
696 	mhp->attr.perms = iwch_ib_to_tpt_access(acc);  in iwch_reg_user_mr()
697 	mhp->attr.va_fbo = virt;  in iwch_reg_user_mr()
698 	mhp->attr.page_size = shift - 12;  in iwch_reg_user_mr()
699 	mhp->attr.len = (u32) length;  in iwch_reg_user_mr()
701 	err = iwch_register_mem(rhp, php, mhp, shift);  in iwch_reg_user_mr()
706 		uresp.pbl_addr = (mhp->attr.pbl_addr -  in iwch_reg_user_mr()
712 			iwch_dereg_mr(&mhp->ibmr);  in iwch_reg_user_mr()
718 	return &mhp->ibmr;  in iwch_reg_user_mr()
721 	iwch_free_pbl(mhp);  in iwch_reg_user_mr()
724 	ib_umem_release(mhp->umem);  in iwch_reg_user_mr()
725 	kfree(mhp);  in iwch_reg_user_mr()
755 	struct iwch_mw *mhp;  in iwch_alloc_mw()  local
765 	mhp = kzalloc(sizeof(*mhp), GFP_KERNEL);  in iwch_alloc_mw()
766 	if (!mhp)  in iwch_alloc_mw()
770 		kfree(mhp);  in iwch_alloc_mw()
773 	mhp->rhp = rhp;  in iwch_alloc_mw()
774 	mhp->attr.pdid = php->pdid;  in iwch_alloc_mw()
775 	mhp->attr.type = TPT_MW;  in iwch_alloc_mw()
776 	mhp->attr.stag = stag;  in iwch_alloc_mw()
778 	mhp->ibmw.rkey = stag;  in iwch_alloc_mw()
779 	if (insert_handle(rhp, &rhp->mmidr, mhp, mmid)) {  in iwch_alloc_mw()
780 		cxio_deallocate_window(&rhp->rdev, mhp->attr.stag);  in iwch_alloc_mw()
781 		kfree(mhp);  in iwch_alloc_mw()
784 	PDBG("%s mmid 0x%x mhp %p stag 0x%x\n", __func__, mmid, mhp, stag);  in iwch_alloc_mw()
785 	return &(mhp->ibmw);  in iwch_alloc_mw()
791 	struct iwch_mw *mhp;  in iwch_dealloc_mw()  local
794 	mhp = to_iwch_mw(mw);  in iwch_dealloc_mw()
795 	rhp = mhp->rhp;  in iwch_dealloc_mw()
797 	cxio_deallocate_window(&rhp->rdev, mhp->attr.stag);  in iwch_dealloc_mw()
799 	PDBG("%s ib_mw %p mmid 0x%x ptr %p\n", __func__, mw, mmid, mhp);  in iwch_dealloc_mw()
800 	kfree(mhp);  in iwch_dealloc_mw()
810 	struct iwch_mr *mhp;  in iwch_alloc_mr()  local
821 	mhp = kzalloc(sizeof(*mhp), GFP_KERNEL);  in iwch_alloc_mr()
822 	if (!mhp)  in iwch_alloc_mr()
825 	mhp->pages = kcalloc(max_num_sg, sizeof(u64), GFP_KERNEL);  in iwch_alloc_mr()
826 	if (!mhp->pages) {  in iwch_alloc_mr()
831 	mhp->rhp = rhp;  in iwch_alloc_mr()
832 	ret = iwch_alloc_pbl(mhp, max_num_sg);  in iwch_alloc_mr()
835 	mhp->attr.pbl_size = max_num_sg;  in iwch_alloc_mr()
837 				 mhp->attr.pbl_size, mhp->attr.pbl_addr);  in iwch_alloc_mr()
840 	mhp->attr.pdid = php->pdid;  in iwch_alloc_mr()
841 	mhp->attr.type = TPT_NON_SHARED_MR;  in iwch_alloc_mr()
842 	mhp->attr.stag = stag;  in iwch_alloc_mr()
843 	mhp->attr.state = 1;  in iwch_alloc_mr()
845 	mhp->ibmr.rkey = mhp->ibmr.lkey = stag;  in iwch_alloc_mr()
846 	if (insert_handle(rhp, &rhp->mmidr, mhp, mmid))  in iwch_alloc_mr()
849 	PDBG("%s mmid 0x%x mhp %p stag 0x%x\n", __func__, mmid, mhp, stag);  in iwch_alloc_mr()
850 	return &(mhp->ibmr);  in iwch_alloc_mr()
852 	cxio_dereg_mem(&rhp->rdev, stag, mhp->attr.pbl_size,  in iwch_alloc_mr()
853 		       mhp->attr.pbl_addr);  in iwch_alloc_mr()
855 	iwch_free_pbl(mhp);  in iwch_alloc_mr()
857 	kfree(mhp->pages);  in iwch_alloc_mr()
859 	kfree(mhp);  in iwch_alloc_mr()
866 	struct iwch_mr *mhp = to_iwch_mr(ibmr);  in iwch_set_page()  local
868 	if (unlikely(mhp->npages == mhp->attr.pbl_size))  in iwch_set_page()
871 	mhp->pages[mhp->npages++] = addr;  in iwch_set_page()
880 	struct iwch_mr *mhp = to_iwch_mr(ibmr);  in iwch_map_mr_sg()  local
882 	mhp->npages = 0;  in iwch_map_mr_sg()