71 	struct rds_mr *mr;  in rds_mr_tree_walk()  local
75 		mr = rb_entry(parent, struct rds_mr, r_rb_node);  in rds_mr_tree_walk()
77 		if (key < mr->r_key)  in rds_mr_tree_walk()
79 		else if (key > mr->r_key)  in rds_mr_tree_walk()
82 			return mr;  in rds_mr_tree_walk()
96 static void rds_destroy_mr(struct rds_mr *mr)  in rds_destroy_mr()  argument
98 	struct rds_sock *rs = mr->r_sock;  in rds_destroy_mr()
103 			mr->r_key, atomic_read(&mr->r_refcount));  in rds_destroy_mr()
105 	if (test_and_set_bit(RDS_MR_DEAD, &mr->r_state))  in rds_destroy_mr()
109 	if (!RB_EMPTY_NODE(&mr->r_rb_node))  in rds_destroy_mr()
110 		rb_erase(&mr->r_rb_node, &rs->rs_rdma_keys);  in rds_destroy_mr()
111 	trans_private = mr->r_trans_private;  in rds_destroy_mr()
112 	mr->r_trans_private = NULL;  in rds_destroy_mr()
116 		mr->r_trans->free_mr(trans_private, mr->r_invalidate);  in rds_destroy_mr()
119 void __rds_put_mr_final(struct rds_mr *mr)  in __rds_put_mr_final()  argument
121 	rds_destroy_mr(mr);  in __rds_put_mr_final()
122 	kfree(mr);  in __rds_put_mr_final()
131 	struct rds_mr *mr;  in rds_rdma_drop_keys()  local
138 		mr = container_of(node, struct rds_mr, r_rb_node);  in rds_rdma_drop_keys()
139 		if (mr->r_trans == rs->rs_transport)  in rds_rdma_drop_keys()
140 			mr->r_invalidate = 0;  in rds_rdma_drop_keys()
141 		rb_erase(&mr->r_rb_node, &rs->rs_rdma_keys);  in rds_rdma_drop_keys()
142 		RB_CLEAR_NODE(&mr->r_rb_node);  in rds_rdma_drop_keys()
144 		rds_destroy_mr(mr);  in rds_rdma_drop_keys()
145 		rds_mr_put(mr);  in rds_rdma_drop_keys()
176 	struct rds_mr *mr = NULL, *found;  in __rds_rdma_map()  local
213 	mr = kzalloc(sizeof(struct rds_mr), GFP_KERNEL);  in __rds_rdma_map()
214 	if (!mr) {  in __rds_rdma_map()
219 	atomic_set(&mr->r_refcount, 1);  in __rds_rdma_map()
220 	RB_CLEAR_NODE(&mr->r_rb_node);  in __rds_rdma_map()
221 	mr->r_trans = rs->rs_transport;  in __rds_rdma_map()
222 	mr->r_sock = rs;  in __rds_rdma_map()
225 		mr->r_use_once = 1;  in __rds_rdma_map()
227 		mr->r_invalidate = 1;  in __rds_rdma_map()
229 		mr->r_write = 1;  in __rds_rdma_map()
265 						 &mr->r_key);  in __rds_rdma_map()
275 	mr->r_trans_private = trans_private;  in __rds_rdma_map()
278 	       mr->r_key, (void *)(unsigned long) args->cookie_addr);  in __rds_rdma_map()
284 	cookie = rds_rdma_make_cookie(mr->r_key, args->vec.addr & ~PAGE_MASK);  in __rds_rdma_map()
296 	found = rds_mr_tree_walk(&rs->rs_rdma_keys, mr->r_key, mr);  in __rds_rdma_map()
299 	BUG_ON(found && found != mr);  in __rds_rdma_map()
301 	rdsdebug("RDS: get_mr key is %x\n", mr->r_key);  in __rds_rdma_map()
303 		atomic_inc(&mr->r_refcount);  in __rds_rdma_map()
304 		*mr_ret = mr;  in __rds_rdma_map()
310 	if (mr)  in __rds_rdma_map()
311 		rds_mr_put(mr);  in __rds_rdma_map()
359 	struct rds_mr *mr;  in rds_free_mr()  local
382 	mr = rds_mr_tree_walk(&rs->rs_rdma_keys, rds_rdma_cookie_key(args.cookie), NULL);  in rds_free_mr()
383 	if (mr) {  in rds_free_mr()
384 		rb_erase(&mr->r_rb_node, &rs->rs_rdma_keys);  in rds_free_mr()
385 		RB_CLEAR_NODE(&mr->r_rb_node);  in rds_free_mr()
387 			mr->r_invalidate = 1;  in rds_free_mr()
391 	if (!mr)  in rds_free_mr()
399 	rds_destroy_mr(mr);  in rds_free_mr()
400 	rds_mr_put(mr);  in rds_free_mr()
411 	struct rds_mr *mr;  in rds_rdma_unuse()  local
416 	mr = rds_mr_tree_walk(&rs->rs_rdma_keys, r_key, NULL);  in rds_rdma_unuse()
417 	if (!mr) {  in rds_rdma_unuse()
423 	if (mr->r_use_once || force) {  in rds_rdma_unuse()
424 		rb_erase(&mr->r_rb_node, &rs->rs_rdma_keys);  in rds_rdma_unuse()
425 		RB_CLEAR_NODE(&mr->r_rb_node);  in rds_rdma_unuse()
433 	if (mr->r_trans->sync_mr)  in rds_rdma_unuse()
434 		mr->r_trans->sync_mr(mr->r_trans_private, DMA_FROM_DEVICE);  in rds_rdma_unuse()
439 		rds_destroy_mr(mr);  in rds_rdma_unuse()
440 	rds_mr_put(mr);  in rds_rdma_unuse()
716 	struct rds_mr *mr;  in rds_cmsg_rdma_dest()  local
734 	mr = rds_mr_tree_walk(&rs->rs_rdma_keys, r_key, NULL);  in rds_cmsg_rdma_dest()
735 	if (!mr)  in rds_cmsg_rdma_dest()
738 		atomic_inc(&mr->r_refcount);  in rds_cmsg_rdma_dest()
741 	if (mr) {  in rds_cmsg_rdma_dest()
742 		mr->r_trans->sync_mr(mr->r_trans_private, DMA_TO_DEVICE);  in rds_cmsg_rdma_dest()
743 		rm->rdma.op_rdma_mr = mr;  in rds_cmsg_rdma_dest()